How can I draw the circle is inscribed in triangle in 3D?
You provide some wonderful code! You only need to draw the circle in the plane of the triangle.
\documentclass[12pt, border = 1mm,tikz]{standalone}
\usepackage{tikz-3dplot}
\usetikzlibrary{3dtools}
\def\pgfmathsetmacroFPU#1#2{\begingroup%
\pgfkeys{/pgf/fpu,/pgf/fpu/output format=fixed}%
\pgfmathsetmacro{#1}{#2}%
\pgfmathsmuggle#1\endgroup}%
\begin{document}
\tdplotsetmaincoords{70}{123}
\begin{tikzpicture}[tdplot_main_coords,scale=1,tdplot_main_coords,
declare function={xA = 8; yA=5;zA=5; xB=1;yB=2;zB=0; xC=5;yC=-5;zC=0 );%
}]
\path
(xA,yA,zA) coordinate (A)
(xB,yB,zB) coordinate (B)
(xC,yC,zC) coordinate (C) ;
\pgfmathsetmacro{\AB}{{sqrt((xB- xA)*(xB- xA) +(yB- yA)*(yB- yA) + (zB- zA)*(zB- zA))} };
\pgfmathsetmacro{\AC}{{sqrt((xC- xA)*(xC- xA) +(yC- yA)*(yC- yA) + (zC- zA)*(zC- zA))} };
\pgfmathsetmacro{\BC}{{sqrt((xC- xB)*(xC- xB) +(yC- yB)*(yC- yB) + (zC- zB)*(zC- zB))} };
\pgfmathsetmacro{\P}{{(\AB + \BC + \AC)/2} };
\pgfmathsetmacro{\myarea}{{sqrt(\P*(\P - \AB) *(\P - \AC)*(\P - \BC) )} };
\pgfmathsetmacro{\r}{{\myarea/\P} };
\coordinate (I) at ({(xA*\BC + xB*\AC + xC*\AB)/(\AB + \BC + \AC)},{(yA*\BC + yB*\AC + yC*\AB)/(\AB + \BC + \AC)},{(zA*\BC + zB*\AC + zC*\AB)/(\AB + \BC + \AC)});
\foreach \p in {A,B,C,I}
\draw[fill=black] (\p) circle (1.5 pt);
\foreach \p/\g in {A/-90,B/-90,C/-90,I/0}
\path (\p)+(\g:3mm) node{$\p$};
\draw[blue] (A) -- (B) -- (C) -- cycle;
\path[overlay] [3d coordinate={(mya)=(C)-(B)},
3d coordinate={(myb)=(C)-(A)},3d coordinate={(myc)=(B)-(A)}];
\pgfmathsetmacroFPU{\tmpa}{1/sqrt(TD("(mya)o(mya)"))}
\pgfmathsetmacroFPU{\tmpb}{1/sqrt(TD("(myb)o(myb)"))}
\pgfmathsetmacroFPU{\tmpc}{1/sqrt(TD("(myc)o(myc)"))}
\path[overlay] [3d coordinate={(mya)=\tmpa*(mya)},
3d coordinate={(myb)=\tmpb*(myb)},3d coordinate={(myc)=\tmpc*(myc)},
3d coordinate={(myn)=(myc)x(mya)},
3d coordinate={(mym)=(myn)x(B)-(A)}];
\pgfmathsetmacroFPU{\tmpm}{1/sqrt(TD("(mym)o(mym)"))}
\path[overlay] [3d coordinate={(mym)=\tmpm*(mym)}];
\begin{scope}[x={(myc)},y={(mym)}]
\draw[red,dashed] (I) circle[radius= \r];
\end{scope}
\end{tikzpicture}
\end{document}
You can choose the coordinates (more or less) as you wish
\documentclass[12pt, border = 1mm,tikz]{standalone}
\usepackage{tikz-3dplot}
\usetikzlibrary{3dtools}
\def\pgfmathsetmacroFPU#1#2{\begingroup%
\pgfkeys{/pgf/fpu,/pgf/fpu/output format=fixed}%
\pgfmathsetmacro{#1}{#2}%
\pgfmathsmuggle#1\endgroup}%
\begin{document}
\tdplotsetmaincoords{70}{123}
\foreach \X in {-6,-5.8,...,6}
{\begin{tikzpicture}[tdplot_main_coords,scale=1,tdplot_main_coords,
declare function={xA = 8; yA=5;zA=\X; xB=1;yB=2;zB=0; xC=5;yC=-5;zC=0 );%
}]
\draw[tdplot_screen_coords] (-8,-11) rectangle (3,3);
\path
(xA,yA,zA) coordinate (A)
(xB,yB,zB) coordinate (B)
(xC,yC,zC) coordinate (C) ;
\pgfmathsetmacro{\AB}{{sqrt((xB- xA)*(xB- xA) +(yB- yA)*(yB- yA) + (zB- zA)*(zB- zA))} };
\pgfmathsetmacro{\AC}{{sqrt((xC- xA)*(xC- xA) +(yC- yA)*(yC- yA) + (zC- zA)*(zC- zA))} };
\pgfmathsetmacro{\BC}{{sqrt((xC- xB)*(xC- xB) +(yC- yB)*(yC- yB) + (zC- zB)*(zC- zB))} };
\pgfmathsetmacro{\P}{{(\AB + \BC + \AC)/2} };
\pgfmathsetmacro{\myarea}{{sqrt(\P*(\P - \AB) *(\P - \AC)*(\P - \BC) )} };
\pgfmathsetmacro{\r}{{\myarea/\P} };
\coordinate (I) at ({(xA*\BC + xB*\AC + xC*\AB)/(\AB + \BC + \AC)},{(yA*\BC + yB*\AC + yC*\AB)/(\AB + \BC + \AC)},{(zA*\BC + zB*\AC + zC*\AB)/(\AB + \BC + \AC)});
\foreach \p in {A,B,C,I}
\draw[fill=black] (\p) circle (1.5 pt);
\foreach \p/\g in {A/-90,B/-90,C/-90,I/0}
\path (\p)+(\g:3mm) node{$\p$};
\draw[blue] (A) -- (B) -- (C) -- cycle;
\path[overlay] [3d coordinate={(mya)=(C)-(B)},
3d coordinate={(myb)=(C)-(A)},3d coordinate={(myc)=(B)-(A)}];
\pgfmathsetmacroFPU{\tmpa}{1/sqrt(TD("(mya)o(mya)"))}
\pgfmathsetmacroFPU{\tmpb}{1/sqrt(TD("(myb)o(myb)"))}
\pgfmathsetmacroFPU{\tmpc}{1/sqrt(TD("(myc)o(myc)"))}
\path[overlay] [3d coordinate={(mya)=\tmpa*(mya)},
3d coordinate={(myb)=\tmpb*(myb)},3d coordinate={(myc)=\tmpc*(myc)},
3d coordinate={(myn)=(myc)x(mya)},
3d coordinate={(mym)=(myn)x(B)-(A)}];
\pgfmathsetmacroFPU{\tmpm}{1/sqrt(TD("(mym)o(mym)"))}
\path[overlay] [3d coordinate={(mym)=\tmpm*(mym)}];
\begin{scope}[x={(myc)},y={(mym)}]
\draw[red,dashed] (I) circle[radius= \r];
\end{scope}
\end{tikzpicture}}
\end{document}
Note that I normalized various vectors to avoid dimension too large
errors.