"Erroneous nesting of equation structures" message- I can't see what's wrong?

You should not nest align in equation. The align environment is enough:

\documentclass[]{article}

\usepackage{amsmath}

\begin{document}

\begin{align}
\begin{split}
  a(t_{n+1}) \cos \varphi_{n+1} = a(t_{n}) \cos \varphi_{n} + \sqrt{2E_{n}}(t_{n+1} - t_{n}) \cos( \alpha_{n}+ \theta{_n}),
\\
b(t_{n+1})\sin \varphi_{n+1} \left(1+ \delta \sin^{2} \varphi_{n+1} \right) = b(t_{n})\sin \varphi_{n} \left(1+ \delta \sin^{2} \varphi_{n} \right) + \sqrt{2E_{n}}(t_{n+1}-t_{n}) \sin (\alpha_{n} + \theta_{n}),
 \\
 \theta_{n} + \alpha_{n} + \theta^{*}_{n+1} - \alpha_{n+1} = 0,
 \\
 \sqrt{2E_{n+1}}\cos \theta_{n+1} = \sqrt{2E_{n}}\cos\theta^{*}_{n+1},
 \\
 \sqrt{2E_{n+1}}\sin\theta_{n+1} = \sqrt{2E_{n}}\sin\theta^{*}_{n+1}-2u(\varphi_{n+1}, t_{n+1}).
\end{split}
\label{system}
\end{align}
\end{document}

This compiles (but looks terrible, imho).

With horizontal alignment it looks better I think:

\documentclass[]{article}

\usepackage{amsmath}
\usepackage[margin=1cm]{geometry}

\begin{document}
Without alignment:
\begin{align}
\begin{split}
  a(t_{n+1}) \cos \varphi_{n+1} = a(t_{n}) \cos \varphi_{n} + \sqrt{2E_{n}}(t_{n+1} - t_{n}) \cos( \alpha_{n}+ \theta{_n}),
\\
b(t_{n+1})\sin \varphi_{n+1} \left(1+ \delta \sin^{2} \varphi_{n+1} \right) = b(t_{n})\sin \varphi_{n} \left(1+ \delta \sin^{2} \varphi_{n} \right) + \sqrt{2E_{n}}(t_{n+1}-t_{n}) \sin (\alpha_{n} + \theta_{n}),
 \\
 \theta_{n} + \alpha_{n} + \theta^{*}_{n+1} - \alpha_{n+1} = 0,
 \\
 \sqrt{2E_{n+1}}\cos \theta_{n+1} = \sqrt{2E_{n}}\cos\theta^{*}_{n+1},
 \\
 \sqrt{2E_{n+1}}\sin\theta_{n+1} = \sqrt{2E_{n}}\sin\theta^{*}_{n+1}-2u(\varphi_{n+1}, t_{n+1}).
\end{split}
\label{system}
\end{align}

With alignment:
\begin{align}
\begin{split}
  a(t_{n+1}) \cos \varphi_{n+1} &= a(t_{n}) \cos \varphi_{n} + \sqrt{2E_{n}}(t_{n+1} - t_{n}) \cos( \alpha_{n}+ \theta{_n}),
\\
b(t_{n+1})\sin \varphi_{n+1} \left(1+ \delta \sin^{2} \varphi_{n+1} \right) &= b(t_{n})\sin \varphi_{n} \left(1+ \delta \sin^{2} \varphi_{n} \right) + \sqrt{2E_{n}}(t_{n+1}-t_{n}) \sin (\alpha_{n} + \theta_{n}),
 \\
 \theta_{n} + \alpha_{n} + \theta^{*}_{n+1} - \alpha_{n+1} &= 0,
 \\
 \sqrt{2E_{n+1}}\cos \theta_{n+1} &= \sqrt{2E_{n}}\cos\theta^{*}_{n+1},
 \\
 \sqrt{2E_{n+1}}\sin\theta_{n+1} &= \sqrt{2E_{n}}\sin\theta^{*}_{n+1}-2u(\varphi_{n+1}, t_{n+1}).
\end{split}
\label{system2}
\end{align}
\end{document}

align already puts you in math mode, so get rid of the redundant equation environment. I also moved the \label inside the align.

\documentclass{article}
\usepackage[margin=1cm]{geometry}
\usepackage{amsmath}
\begin{document}
%\begin{equation}
\begin{align}
\begin{split}
  a(t_{n+1}) \cos \varphi_{n+1} = a(t_{n}) \cos \varphi_{n} + \sqrt{2E_{n}}(t_{n+1} - t_{n}) \cos( \alpha_{n}+ \theta{_n}),
\\
b(t_{n+1})\sin \varphi_{n+1} \left(1+ \delta \sin^{2} \varphi_{n+1} \right) = b(t_{n})\sin \varphi_{n} \left(1+ \delta \sin^{2} \varphi_{n} \right) + \sqrt{2E_{n}}(t_{n+1}-t_{n}) \sin (\alpha_{n} + \theta_{n}),
 \\
 \theta_{n} + \alpha_{n} + \theta^{*}_{n+1} - \alpha_{n+1} = 0,
 \\
 \sqrt{2E_{n+1}}\cos \theta_{n+1} = \sqrt{2E_{n}}\cos\theta^{*}_{n+1},
 \\
 \sqrt{2E_{n+1}}\sin\theta_{n+1} = \sqrt{2E_{n}}\sin\theta^{*}_{n+1}-2u(\varphi_{n+1}, t_{n+1}).
\end{split}
\label{system}
\end{align}
%\end{equation}
\end{document}

Alternately, you can keep the equation environment, and change align to aligned:

\documentclass{article}
\usepackage[margin=1cm]{geometry}
\usepackage{amsmath}
\begin{document}
\begin{equation}
\begin{aligned}
\begin{split}
  a(t_{n+1}) \cos \varphi_{n+1} = a(t_{n}) \cos \varphi_{n} + \sqrt{2E_{n}}(t_{n+1} - t_{n}) \cos( \alpha_{n}+ \theta{_n}),
\\
b(t_{n+1})\sin \varphi_{n+1} \left(1+ \delta \sin^{2} \varphi_{n+1} \right) = b(t_{n})\sin \varphi_{n} \left(1+ \delta \sin^{2} \varphi_{n} \right) + \sqrt{2E_{n}}(t_{n+1}-t_{n}) \sin (\alpha_{n} + \theta_{n}),
 \\
 \theta_{n} + \alpha_{n} + \theta^{*}_{n+1} - \alpha_{n+1} = 0,
 \\
 \sqrt{2E_{n+1}}\cos \theta_{n+1} = \sqrt{2E_{n}}\cos\theta^{*}_{n+1},
 \\
 \sqrt{2E_{n+1}}\sin\theta_{n+1} = \sqrt{2E_{n}}\sin\theta^{*}_{n+1}-2u(\varphi_{n+1}, t_{n+1}).
\end{split}
\label{system}
\end{aligned}
\end{equation}
\end{document}