画像は次の通り。
LaTeXのコードは次のようになりました。
行列の外の括弧を表現するのにdecorations.pathreplacingライブラリを,
行列を上部で揃えるのにpositioningライブラリを使っています.
色付けはbackgroundsライブラリを使い,scope環境にon background layerオプションを付けて文字の背後に色を付けるようにしました.
\documentclass[a4paper,dvipdfmx,uplatex]{jsarticle}
\usepackage{amsmath}
\usepackage{tikz}
\usetikzlibrary{calc}
\usetikzlibrary{decorations.pathreplacing}
\usetikzlibrary{matrix,backgrounds,fit,positioning}
\begin{document}
\begin{equation*}
\begin{tikzpicture}[every left delimiter/.style={xshift=1ex},every right delimiter/.style={xshift=-1ex}]
%
\matrix(A)[matrix of math nodes,nodes in empty cells,
ampersand replacement=\&,
left delimiter={[},right delimiter={]},
inner sep=0.8ex,
]
{
\phantom{b_{11}}\&\hphantom{\dotsb}\&\phantom{\dotsb}\&a_{1j}\&\phantom{b_{1n}}\\
\phantom{a_{11}}\&\&\&a_{2j}\&\phantom{a_{1n}}\\
\phantom{\mathstrut}\&\&\&\phantom{\mathstrut}\&\\
\phantom{\mathstrut}\&\&\&\phantom{\mathstrut}\&\\
\phantom{a_{11}}\&\&\&a_{mj}\&\phantom{a_{1n}}\\
};
\draw[line width=1pt,line cap=round,dash pattern=on 0pt off 4\pgflinewidth](A-3-4.north)--(A-4-4.south);
\draw[decorate,decoration={brace,amplitude=5}]
($(A-1-1.north west)+(-0.5ex,1.5ex)$)--($(A-1-5.north east)+(0.5ex,1.5ex)$)
node[midway,yshift=2.5ex]{$n$};
\draw[decorate,decoration={brace,mirror,amplitude=5}]
($(A-1-1.north west)+(-1.8ex,0.2ex)$)--($(A-5-1.south west)+(-1.8ex,-0.2ex)$)
node[midway,xshift=-2.5ex]{$m$};
\begin{scope}[on background layer]
\fill[gray!40,rounded corners](A-1-4.north west) rectangle (A-5-4.south east);
\end{scope}
\matrix(B)[matrix of math nodes,nodes in empty cells,
ampersand replacement=\&,left delimiter={[},right delimiter={]},inner sep=0.8ex,
left=0.8cm of A.north west,
anchor=north east]
{
\phantom{b_{11}}\&\&\&\&\phantom{b_{11}}\\
b_{h1}\&b_{h2}\&\vphantom{b_{h1}}\phantom{\dotsb}\&\phantom{\dotsb}\vphantom{b_{h1}}\&b_{h m}\\
\phantom{b_{11}}\&\&\&\&\\
\phantom{b_{11}}\&\&\&\&\\
};
\node[inner sep=0pt,fit=(B-2-3)(B-2-4)](fit1){};
\node[baseline=(fit1.base)]at (fit1){$\dotsb\dotsb$};
\draw[decorate,decoration={brace,amplitude=5}]
($(B-1-1.north west)+(-0.5ex,1.5ex)$)--($(B-1-5.north east)+(0.5ex,1.5ex)$)
node[midway,yshift=2.5ex]{$m$};
\draw[decorate,decoration={brace,mirror,amplitude=5}]
($(B-1-1.north west)+(-1.8ex,0.2ex)$)--($(B-4-1.south west)+(-1.8ex,-0.2ex)$)
node[midway,xshift=-2.5ex]{$l$};
\begin{scope}[on background layer]
\fill[gray!40,rounded corners](B-2-1.south west) rectangle (B-2-5.north east);
\end{scope}
\matrix(BA)[matrix of math nodes,nodes in empty cells,
ampersand replacement=\&,
left delimiter={[},right delimiter={]},
inner sep=0.8ex,
right=1.4cm of A.north east,
anchor=north west
]
{
\phantom{b_{11}}\&\phantom{\dotsb}\&\phantom{\dotsb}\&\phantom{b_{hj}}\&\phantom{b_{11}}\\
\phantom{c_{11}}\&\phantom{c_{11}}\&\&\node[fill=gray!40,rounded corners]{c _{hj}};\&\phantom{c_{11}}\\
\phantom{b_{11}}\&\&\&\phantom{c_{11}}\&\\
\phantom{b_{11}}\&\&\&\phantom{b_{11}}\&\phantom{b_{11}}\\
};
\draw[decorate,decoration={brace,amplitude=5}]
($(BA-1-1.north west)+(-0.5ex,1.5ex)$)--($(BA-1-5.north east)+(0.5ex,1.5ex)$)
node[midway,yshift=2.5ex]{$n$};
\draw[decorate,decoration={brace,mirror,amplitude=5}]
($(BA-1-1.north west)+(-1.8ex,0.2ex)$)--($(BA-4-1.south west)+(-1.8ex,-0.2ex)$)
node[midway,xshift=-2.5ex]{$l$};
\begin{scope}[on background layer]
\draw[line width=1pt,line cap=round,dash pattern=on 0 pt off 4\pgflinewidth]
(BA-1-4.north)--(BA-4-4.south) (BA-2-1.west)--(BA-2-5.east);
\end{scope}
\node at($(A.east)+(10pt,15pt)$){$=$};
\node[below=0.5cm of A](A2){$A$};
\node(B2)at(B |- A2){$B$};
\node(BA2)at(BA |- A2){$BA$};
\draw[line width=1pt,line cap=round,dash pattern=on 0 pt off 4\pgflinewidth](B)--(B2) (A)--(A2) (BA)--(BA2);
\end{tikzpicture}
\end{equation*}
\end{document}