Command with simple parsing

The idea is to make \foo open a group and set a selected character as math active inside the group, whose action is just \mathcal. If the argument is a single token, just apply \mathcal to it.

It's quite easy with expl3:

\documentclass{article}

\usepackage{xparse}

\ExplSyntaxOn

\NewDocumentCommand{\foo}{m}
 {
  \szumilo_foo:n { #1 }
 }
\cs_new_protected:Nn \szumilo_foo:n
 {
  \tl_if_single:nTF { #1 }
   { \mathcal{#1} }
   { \szumilo_foo_multiple:n { #1 } }
 }
\cs_new_protected:Nn \szumilo_foo_multiple:n
 {
  \group_begin:
  \char_set_mathcode:nn { `* } { "8000 }
  #1
  \group_end:
 }
\char_set_active_eq:NN * \mathcal

\ExplSyntaxOff

\begin{document}

$\foo{*X}+\foo{X}+\foo{X^A}+\foo{*X^A_B}$

\end{document}

If you insist on using # it's a bit more difficult.

\documentclass{article}

\usepackage{xparse}

\ExplSyntaxOn

\NewDocumentCommand{\foo}{m}
 {
  \szumilo_foo:n { #1 }
 }
\cs_new_protected:Nn \szumilo_foo:n
 {
  \tl_if_single:nTF { #1 }
   { \mathcal{#1} }
   { \szumilo_foo_multiple:n { #1 } }
 }
\cs_new_protected:Nn \szumilo_foo_multiple:n
 {
  \group_begin:
  \tl_set_rescan:Nnn \l_szumilo_foo_arg_tl { \char_set_catcode:nn { `\# } { 12 } } { #1 }
  \char_set_mathcode:nn { `\# } { "8000 }
  \tl_use:N \l_szumilo_foo_arg_tl
  \group_end:
 }
\cs_new:Nn \szumilo_foo_mathcal:NN { \mathcal{#2} }
\char_set_active_eq:nN { `\# } \szumilo_foo_mathcal:NN

\ExplSyntaxOff

\begin{document}

$\foo{#X}+\foo{X}+\foo{X^A}+\foo{#X^A_B}$

\end{document}

Besides the need of rescanning the argument for making # of category 12, we need to swallow one # (because these tokens get doubled when they have category 6).