Fixing overlapping values

1. Scale the x-ticks

scaled x ticks = base 10:2,

2. Change the number format

xticklabel style={
/pgf/number format/fixed,
/pgf/number format/precision=2},

3. Reduce the number of xticks.

List only the needed xtick labels

xtick={0.00,0.02,...,0.12},

OR, increase xtick distance (as commented by Stefan Pinnow)

xtick distance=0.02,

Code

\documentclass[border=3mm]{standalone}
\usepackage{subcaption}
\usepackage{caption}
\usepackage{pgfplots, pgfplotstable}
\usetikzlibrary{positioning}
\usepgfplotslibrary{units}
\pgfplotsset{compat=1.16,width=0.98\textwidth}
%\pgfplotsset{compat=1.16,width=0.98\textwidth}

\usepackage{tikz}
\usepackage[english,spanish]{babel}         % multilenguaje

% grid style

\pgfdeclareplotmark{mystar}{
    \node[star,star point ratio=2.25,minimum size=6pt,
    inner sep=0pt,draw=black,solid,fill=red] {};
}

\decimalpoint
\definecolor{color0}{rgb}{1,1,0}
\definecolor{color1}{rgb}{1,0.647058823529412,0}
\begin{document}

    \begin{tikzpicture}
    \begin{axis}[
    axis line style={black},
%    scaled x ticks = base 10:2,
%   xticklabel style={
%   /pgf/number format/fixed,
%   /pgf/number format/precision=2},
    xtick={0.00,0.02,...,0.12},
    legend cell align={left},
    legend style={at={(0.03,0.97)}, anchor=north west, draw=black},
    tick align=outside,
    x grid style={dashed,black!60},
    xlabel={$\lambda$},
    xmajorticks=true,
    xmin=0.0055, xmax=0.1045,
    xtick style={color=black},
    y grid style={dashed,black!60},
    ylabel={RMSE},
    ymajorticks=true,
    ymin=0.912938021884183, ymax=1.01339429908853,
    ytick style={black},
    xtick align=inside,
    ytick align=inside,
    grid = none,
    y tick label style={
        /pgf/number format/.cd,
        fixed,
        fixed zerofill,
        precision=4
    },
    ]
\addplot [mark = *,line width=0.8pt,blue, mark size=1pt]
table {%
    0.01 0.97820029742643
    0.02 0.962938632883304
    0.03 0.944377590107486
    0.04 0.934490513382265
    0.05 0.926770707258887
    0.09 0.921652885011797
    0.1 0.922493653631392
};
\addlegendentry{20 f}
\addplot [mark = *,line width=0.8pt,black, mark size=1pt]
table {%
    0.01 0.992629285376604
    0.02 0.970555305088483
    0.03 0.950769057081716
    0.04 0.937408149963347
    0.05 0.92888998569171
    0.09 0.919757726792486
    0.1 0.922265347358195
};
\addlegendentry{30 f}
\addplot [mark = *,line width=0.8pt,red, mark size=1pt]
table {%
    0.01 1.00429685084556
    0.02 0.976819790163343
    0.03 0.955650609854943
    0.04 0.939869451503815
    0.05 0.927933873176105
    0.09 0.917634061468206
    0.1 0.920597626159782
};
\addlegendentry{50 f}
\addplot [mark = *,line width=0.8pt,yellow, mark size=1pt]
table {%
    0.01 1.00882810467015
    0.02 0.972771038904076
    0.03 0.955085239938932
    0.04 0.941800125612627
    0.05 0.930788565993547
    0.09 0.917504216302563
    0.1 0.920804593468368
};
\addlegendentry{100 f}
\addplot [mark = *,line width=0.8pt,orange, mark size=1pt]
table {%
    0.01 1.00130716487781
    0.02 0.97125309142477
    0.03 0.952896925523292
    0.04 0.940895422722576
    0.05 0.931374232049292
    0.09 0.920535667968586
    0.1 0.920701204441381
};
\addlegendentry{150 f}
\end{axis}
\end{tikzpicture}
\end{document}

Edit: Now I notice to late that other answer has added the same solution for xtick labels as I try proposed :-(. So only some off-topic suggestions ...

I would design your diagram with use of the filecontents package and reduced number of decimal digits for y values to four (to my opinion they still provide satisfactory accuracy):

\documentclass[margin=3mm]{standalone}
\usepackage[english,spanish]{babel}         % multilenguaje
\usepackage{pgfplots}
\usetikzlibrary{babel,
                positioning}
\pgfplotsset{compat=1.16,width=0.98\linewidth}

\usepackage{filecontents}
\begin{filecontents}{performance.data}
x    A       B         C        D        E 
0.01 0.9782  0.99262   1.0042   1.0088   1.0013
0.02 0.9629  0.97055   0.9768   0.9727   0.9712
0.04 0.9344  0.93740   0.9398   0.9418   0.9408
0.05 0.9267  0.92888   0.9279   0.9307   0.9313
0.09 0.9216  0.91975   0.9176   0.9175   0.9205
0.1  0.9224  0.92226   0.9205   0.9208   0.9207
\end{filecontents}

\begin{document}
    \begin{tikzpicture}
\begin{axis}[
    grid = none,
    legend style={font=\small},
    legend cell align=left,         
    legend pos=north east,
    xmin=0.005, xmax=0.105,
    ylabel={RMSE},
    xlabel={$\lambda$},
%
xticklabel style={font=\small,
    /pgf/number format/fixed},  
yticklabel style={font=\small},
%
every axis plot post/.append style={line width=0.8pt},
%
mark size=1pt,
cycle list={
    {blue,mark=*},
    {black,mark=*},
    {red,mark=*},
    {yellow,mark=*},
    {orange,mark=otimes*}% <-- don't add a comma here
            },
    ]
\addplot    table [y=A] {performance.data};
\addlegendentry{20 f}
\addplot    table [y=B] {performance.data};
\addlegendentry{30 f}
\addplot    table [y=C] {performance.data};
\addlegendentry{50 f}
\addplot    table [y=D] {performance.data};
\addlegendentry{100 f}
\addplot    table [y=E] {performance.data};
\addlegendentry{150 f}
\end{axis}
    \end{tikzpicture}
\end{document}