\bordermatrix with brackets [ ] instead of parentheses ( )

An almost direct modification of \bordermatrix:

\documentclass{article}
\makeatletter
\def\bbordermatrix#1{\begingroup \m@th
  \@tempdima 4.75\p@
  \setbox\z@\vbox{%
    \def\cr{\crcr\noalign{\kern2\p@\global\let\cr\endline}}%
    \ialign{$##$\hfil\kern2\p@\kern\@tempdima&\thinspace\hfil$##$\hfil
      &&\quad\hfil$##$\hfil\crcr
      \omit\strut\hfil\crcr\noalign{\kern-\baselineskip}%
      #1\crcr\omit\strut\cr}}%
  \setbox\tw@\vbox{\unvcopy\z@\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{\unhbox\@ne\unskip\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{$\kern\wd\@ne\kern-\@tempdima\left[\kern-\wd\@ne
    \global\setbox\@ne\vbox{\box\@ne\kern2\p@}%
    \vcenter{\kern-\ht\@ne\unvbox\z@\kern-\baselineskip}\,\right]$}%
  \null\;\vbox{\kern\ht\@ne\box\tw@}\endgroup}
\makeatother

\begin{document}
$
M = \bbordermatrix{ & x & y \cr
              A & 1 & 0 \cr
              B & 0 & 1 \cr}
$
\end{document}

I acted on \left( and \right), but also changed the value given to \@tempdima, smaller because brackets are slimmer than parentheses.

If you don't want such a messy code, just do

\usepackage{etoolbox}
\let\bbordermatrix\bordermatrix
\patchcmd{\bbordermatrix}{8.75}{4.75}{}{}
\patchcmd{\bbordermatrix}{\left(}{\left[}{}{}
\patchcmd{\bbordermatrix}{\right)}{\right]}{}{}

that does exactly the same changes.

enter image description here

A slightly different version where the border entries are set in \scriptstyle.

\documentclass{article}
\makeatletter
\def\bbordermatrix#1{\begingroup \m@th
  \global\let\perhaps@scriptstyle\scriptstyle
  \@tempdima 4.75\p@
  \setbox\z@\vbox{%
    \def\cr{%
      \crcr
      \noalign{%
        \kern2\p@
        \global\let\cr\endline
        \global\let\perhaps@scriptstyle\relax
      }%
    }%
    \ialign{$\make@scriptstyle{##}$\hfil\kern2\p@\kern\@tempdima
      &\thinspace\hfil$\perhaps@scriptstyle##$\hfil
      &&\quad\hfil$\perhaps@scriptstyle##$\hfil\crcr
      \omit\strut\hfil\crcr
      \noalign{\kern-\baselineskip}%
      #1\crcr\omit\strut\cr}}%
  \setbox\tw@\vbox{\unvcopy\z@\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{\unhbox\@ne\unskip\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{$\kern\wd\@ne\kern-\@tempdima\left[\kern-\wd\@ne
    \global\setbox\@ne\vbox{\box\@ne\kern2\p@}%
    \vcenter{\kern-\ht\@ne\unvbox\z@\kern-\baselineskip}\,\right]$}%
  \null\;\vbox{\kern\ht\@ne\box\tw@}\endgroup}
\def\make@scriptstyle#1{\vcenter{\hbox{$\scriptstyle#1$}}}
\makeatother

\begin{document}
$
M = \bbordermatrix{
   & x & y \cr
 A & 1 & 0 \cr
 B & 0 & 1 \cr
}
$
\end{document}

enter image description here


Following Herbert Voss' excellent mathmode document, you can redefine the bordermatrix command to take an optional argument that determines the delimeters

enter image description here

Note that Herbert also details that the amsmath package offers the same or better macros/environments

\documentclass{article}
\makeatletter
\newif\if@borderstar
\def\bordermatrix{\@ifnextchar*{%
\@borderstartrue\@bordermatrix@i}{\@borderstarfalse\@bordermatrix@i*}%
}
\def\@bordermatrix@i*{\@ifnextchar[{\@bordermatrix@ii}{\@bordermatrix@ii[()]}}
\def\@bordermatrix@ii[#1]#2{%
\begingroup
\m@th\@tempdima8.75\p@\setbox\z@\vbox{%
\def\cr{\crcr\noalign{\kern 2\p@\global\let\cr\endline }}%
\ialign {$##$\hfil\kern 2\p@\kern\@tempdima & \thinspace %
\hfil $##$\hfil && \quad\hfil $##$\hfil\crcr\omit\strut %
\hfil\crcr\noalign{\kern -\baselineskip}#2\crcr\omit %
\strut\cr}}%
\setbox\tw@\vbox{\unvcopy\z@\global\setbox\@ne\lastbox}%
\setbox\tw@\hbox{\unhbox\@ne\unskip\global\setbox\@ne\lastbox}%
\setbox\tw@\hbox{%
$\kern\wd\@ne\kern -\@tempdima\left\@firstoftwo#1%
\if@borderstar\kern2pt\else\kern -\wd\@ne\fi%
\global\setbox\@ne\vbox{\box\@ne\if@borderstar\else\kern 2\p@\fi}%
\vcenter{\if@borderstar\else\kern -\ht\@ne\fi%
\unvbox\z@\kern-\if@borderstar2\fi\baselineskip}%
\if@borderstar\kern-2\@tempdima\kern2\p@\else\,\fi\right\@secondoftwo#1 $%
}\null \;\vbox{\kern\ht\@ne\box\tw@}%
\endgroup
}
\makeatother


\begin{document}
$
M = \bordermatrix[{[]}]{%
            & x & y \cr
          A & 1 & 0 \cr
          B & 0 & 1 }
$

$
M = \bordermatrix[\{\}]{%
            & x & y \cr
          A & 1 & 0 \cr
          B & 0 & 1 }
$
\end{document}