\documentclass[preview,border=1mm]{standalone}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{mathtools}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{bm}

\renewcommand{\qedsymbol}{$\blacksquare$}
\def\d{\mathrm{d}}
\def\vx{\ensuremath{\bm{x}}}
\def\vw{\ensuremath{\bm{w}}}
\def\vu{\ensuremath{\bm{u}}}
\def\vv{\ensuremath{\bm{v}}}
\def\mPhi{\ensuremath{\bm{\Phi}}}
\def\vlambda{\ensuremath{\bm{\lambda}}}
\def\vphi{\ensuremath{\bm{\phi}}}
\def\boldt{\ensuremath{\mathsf{t}}}
\begin{document}
Recall that the design matrix $\mPhi$ for $M$ base functions and $N$ input entries is defined by:
\[
\mPhi_{nj} = \phi_j(\vx_n)
\]
\[
\mPhi = 
\begin{bmatrix}
\vphi_1 & \vphi_2 & \cdots & \vphi_M
\end{bmatrix}
=
\begin{bmatrix}
\vphi(\vx_1)^\top \\
\vphi(\vx_2)^\top \\
\vdots\\
\vphi(\vx_N)^\top \\
\end{bmatrix}
\]
which is basically the vertical stack of transformed inputs $\vphi(\vx_n)^\top$.
\\[8pt]
We consider an arbitrary vector $\vv\in \mathbb{R}^N$. Given that $\operatorname{Span}(\{\vphi_j\}_{j\in\{1,\dots, M\}}) = F$ is a closed subspace of Hilbert space (true for any finite-dimensional subspace), so that $\mathbb{R}^N = F \oplus F^\bot$, where $F^\bot$ is the orthogonal complement of $F$ and $\oplus$ is inner direct sum.
\begin{itemize}
	\item If $\vv\in F$, which  means that $\sum_{j=1}^M \lambda_j\vphi_j = \mPhi\vlambda$, then we have
	\[\mPhi(\mPhi^\top\mPhi)^{-1}\mPhi^\top\vv = \mPhi(\mPhi^\top\mPhi)^{-1}\mPhi^\top\mPhi\vlambda = \mPhi\vlambda = \vv\]
	\item If $\vv\in F^\bot$, so that $\vphi_j^\top\vv = 0$, then
	\[\mPhi(\mPhi^\top\mPhi)^{-1}\mPhi^\top\vv = \mPhi(\mPhi^\top\mPhi)^{-1}\bm{0} = \bm{0}\]
	\item If $\vv$ in the subspace other than former cases, it can be decomposed into two vectors $\vv = \vv_F + \vv_{F^\bot}$, such that $\vv_F\in F, \vv_{F^\bot}\in F^\bot$.
	\[\mPhi(\mPhi^\top\mPhi)^{-1}\mPhi^\top\vv = \mPhi(\mPhi^\top\mPhi)^{-1}\mPhi^\top(\vv_F + \vv_{F^\bot}) = \vv_F + \bm{0} = \vv_F \]
\end{itemize}
For the least-square solution
\[
\vw_\text{ML} = (\mPhi^\top\mPhi)^{-1}\mPhi^\top\boldt \tag{3.15},
\]
we know, by using the results above, that there exist a vector $\vlambda_F\in \mathbb{R}^M$ in the manifold $\mathcal{S}$ such that after the orthogonal projection by $(\mPhi^\top\mPhi)^{-1}\mPhi^\top$, $\boldt$ becomes
\[
\begin{split}
	& (\mPhi^\top\mPhi)^{-1}\mPhi^\top\boldt \\
	= & (\mPhi^\top\mPhi)^{-1}\mPhi^\top(\boldt_F + \boldt_{F^\bot}) \\
	= & (\mPhi^\top\mPhi)^{-1}\mPhi^\top\boldt_F \\
	= & (\mPhi^\top\mPhi)^{-1}\mPhi^\top\mPhi\vlambda_F = \vlambda_F \\
\end{split}
\]

\qed

\end{document}
