Cells with text are slighlty raised in TikZ
add anchor=north
in the definition of nodes
as below
[mat/.style={matrix of nodes,
nodes={draw, minimum size=10mm, minimum width=15mm, fill=gray!10, anchor=north, },
the output is
also changed the options slightly for the arrow
\draw [-{Latex[length=3mm]}] (array-2-5.south)--(array-5-8.north) node [near end, above, sloped] (TextNode) {\&};
the midway
replaced with near end
now no requirement of hspace
MWE
\documentclass[10pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{tikz, }
\usetikzlibrary{matrix, arrows.meta}
\begin{document}
\begin{tikzpicture}
[mat/.style={matrix of nodes,
nodes={draw, minimum size=10mm, minimum width=15mm, fill=gray!10, anchor=north, },
column sep=-\pgflinewidth,
row sep=0.5mm,
nodes in empty cells,
row 1 column 1/.style={nodes={draw=none}},
row 2 column 1/.style={nodes={draw=none, fill=none}},
row 2 column 2/.style={nodes={draw=none, fill=none}},
row 2 column 8/.style={nodes={draw=none, fill=none}},
row 2 column 9/.style={nodes={draw=none, fill=none}},
row 5 column 4/.style={nodes={draw=none, fill=none}},
row 5 column 5/.style={nodes={draw=none, fill=none}},
row 5 column 6/.style={nodes={draw=none, fill=none}},
row 1/.style={nodes={font=\ttfamily, draw=none, fill=none, minimum size=5mm}},
row 3/.style={nodes={font=\ttfamily, draw=none, fill=none, minimum size=5mm}},
row 4/.style={nodes={font=\ttfamily, draw=none, fill=none, minimum size=5mm}},
}]
\matrix[mat] (array)
{
& & & & dhaka & & & & \\
& & & & 19 & & & & \\
& & & 20021 & 20022 & 20023 & & & \\
& sylhet & & & & & & khulna & \\
& 19 & & & & & & 20022 & \\
};
\draw [-{Latex[length=3mm]}] (array-2-5.south)--(array-5-8.north) node [near end, above, sloped] (TextNode) {\&};
\draw [-{Latex[length=3mm]}] (array-2-5.south)--(array-5-2.north);
\end{tikzpicture}
\end{document}