Creating custom mathematical symbols

Picture mode!

\documentclass{article}
\usepackage{pict2e}

\makeatletter
\newcommand{\YES}{\mathord{\mathpalette\nicoud@YESNO\relax}}
\newcommand{\NO}{\mathord{\mathpalette\nicoud@YESNO{\nicoud@path{\fillpath}}}}
\newcommand{\nicoud@YESNO}[2]{%
  \begingroup
  \settoheight{\unitlength}{$#1X$}%
  \begin{picture}(0.7,1)
  \linethickness{\variable@rule{#1}}%
  \roundcap\roundjoin
  \nicoud@path{\strokepath}
  #2
  \Line(0.35,0)(0.35,0.5)
  \end{picture}%
  \endgroup
}
\newcommand{\nicoud@path}[1]{%
  \moveto(0.1,0.5)
  \lineto(0.1,1)\lineto(0.6,1)\lineto(0.6,0.5)
  \closepath
  #1
}
\newcommand{\variable@rule}[1]{%
  \fontdimen8  
  \ifx#1\displaystyle\textfont3\else
    \ifx#1\textstyle\textfont3\else
      \ifx#1\scriptstyle\scriptfont3\else
        \scriptscriptfont3\relax
  \fi\fi\fi
}
\makeatletter

\begin{document}

$\YES\NO_{\YES\NO}$

{\LARGE$\YES\NO_{\YES\NO}$}

\end{document}

Just for fun and learning, here's a version with the experimental drawing commands in expl3.

\documentclass{article}
\usepackage{xparse}

\ExplSyntaxOn
\NewDocumentCommand{\YES}{}
 {
  \mathord{\mathpalette{\nicoud_yesno:nn}{\driver_draw_stroke:}}
 }
\NewDocumentCommand{\NO}{}
 {
  \mathord{\mathpalette{\nicoud_yesno:nn}{\driver_draw_fillstroke:}}
 }

\dim_new:N \l_nicoud_yesno_ht_dim
\box_new:N \l_nicoud_yesno_box

\cs_new_protected:Nn \nicoud_yesno:nn
 {% #1 = math style, #2 = stroke or fill the path
  \group_begin:
  \hbox_set:Nn \l_tmpa_box { $#1X$ }
  \dim_set:Nn \l_nicoud_yesno_ht_dim { \box_ht:N \l_tmpa_box }
  \hbox_set:Nn \l_nicoud_yesno_box
   {
    \driver_draw_begin:
    \driver_draw_cap_round:
    \driver_draw_join_round:
    \driver_draw_linewidth:n { \nicoud_line_thickness:n { #1 } }
    \nicoud_path:n { #2 }
    \driver_draw_moveto:nn {0.35\l_nicoud_yesno_ht_dim}{0.0\l_nicoud_yesno_ht_dim}
    \driver_draw_lineto:nn {0.35\l_nicoud_yesno_ht_dim}{0.5\l_nicoud_yesno_ht_dim}
    \driver_draw_stroke:
    \driver_draw_end:
   }
  \box_set_wd:Nn \l_nicoud_yesno_box { 0.7\l_nicoud_yesno_ht_dim }
  \box_set_ht:Nn \l_nicoud_yesno_box { \l_nicoud_yesno_ht_dim }
  \box_use:N \l_nicoud_yesno_box
  \group_end:
 }
\cs_new_protected:Nn \nicoud_path:n
 {
  \driver_draw_moveto:nn {0.1\l_nicoud_yesno_ht_dim}{0.5\l_nicoud_yesno_ht_dim}
  \driver_draw_lineto:nn {0.1\l_nicoud_yesno_ht_dim}{1.0\l_nicoud_yesno_ht_dim}
  \driver_draw_lineto:nn {0.6\l_nicoud_yesno_ht_dim}{1.0\l_nicoud_yesno_ht_dim}
  \driver_draw_lineto:nn {0.6\l_nicoud_yesno_ht_dim}{0.5\l_nicoud_yesno_ht_dim}
  \driver_draw_closepath:
  #1
 }
\cs_new:Nn \nicoud_line_thickness:n
 {
  \str_case:nn { #1 }
   {
    {\displaystyle}{\fontdimen8\textfont3}
    {\textstyle}{\fontdimen8\textfont3}
    {\scriptstyle}{\fontdimen8\scriptfont3}
    {\scriptscriptstyle}{\fontdimen8\scriptscriptfont3}
   }
 }
\ExplSyntaxOff

\begin{document}

$\YES\NO_{\YES\NO}$

{\LARGE$\YES\NO_{\YES\NO}$}

\end{document}

You can buid the symbols with tikz

\documentclass{article}    
\usepackage{tikz}      

\newcommand{\myclosedsymbol}{%      
\tikz[baseline]{\fill[draw] (0,0) -- ++(0,1ex) ++(-0.25ex,0) rectangle (0.25ex,1.5ex) }%  
}   


\newcommand{\myopensymbol}{%      
\tikz[baseline]{\draw (0,0) -- ++(0,1ex) ++(-0.25ex,0) rectangle (0.25ex,1.5ex) }%  
}

\begin{document}   

{\Huge $F  \myopensymbol $}   

{\normalsize $F \myopensymbol $}   

{\tiny $ F \myopensymbol $}   

{\Huge $F  \myclosedsymbol $}   

{\normalsize $F \myclosedsymbol $}   

{\tiny $ F \myclosedsymbol $}   


\end{document}