\documentclass[12pt,a4paper]{article}
\usepackage{graphicx}
\graphicspath{{eps/}{pdf/}}
\pagestyle{empty}
\oddsidemargin 2.1mm
\textwidth     155mm
\topmargin     -15mm
\textheight    242mm

\def\IR{\mbox{I\kern-0.2em R}}
\def\IQ{\mbox{$\;$\raise0.1ex\hbox{\rule{0.5pt}{1.4ex}}\kern-0.26em Q}}

\begin{document}

%-----------------------------------------------------------------------

\noindent
{\bf Artificial Neural Networks and Deep Learning}
\hfill Summer 2018 \\
Christian Borgelt and Christoph Doell \hfill from 2018.06.05

%-----------------------------------------------------------------------

\vskip3ex
\centerline{\bf Exercise Sheet 5}

%-----------------------------------------------------------------------

\subsubsection*{Exercise 19 \quad\rm Function Approximation}

\begin{enumerate}\itemsep0pt
\item[a)] Construct a multi-layer perceptron with about 10~neurons
          that approximates the function $y = x^2$ in the
          interval~$[0.5,4.5]$ by a step function! Consider that there
          are (at least) two possibilities for doing this: one that
          uses two hidden layers, and one that uses only one hidden
          layer.
\item[b)] How can the approximation be improved?
          (State at least two possibilities.)
\end{enumerate}

%-----------------------------------------------------------------------

\subsubsection*{Exercise 20 \quad\rm Function Approximation}

\begin{enumerate}\itemsep0pt
\item[a)] Consider the indicator function of the rational numbers
          over the set of the real numbers (also known as the
          Dirichlet function), that is, the function
          \[ f : \IR \to \{ 0, 1 \}, \qquad
                 x \mapsto \left\{\begin{array}{ll}
                           1, & \mbox{if $x \in \IQ$,} \\
                           0, & \mbox{otherwise.}
                           \end{array}\right. \]
          Is it possible to approximate this function with a
          neural network (multi-layer perceptron) with arbitrary
          precision (for simplicity: in the interval $[-1,1]$)?
\item[b)] Consider the simple function
          \[ g: \IR \to \IR, \qquad x \mapsto x^{-1}. \]
          Is it possible to approximate this function with a
          neural network (multi-layer perceptron) with arbitrary
          precision (for simplicity: in the interval $[-1,1]$)?
\item[c)] What can be concluded from the result of parts~a) and b)
          about the computational capabilities of neural networks
          (multi-layer perceptrons)?
\end{enumerate}

%-----------------------------------------------------------------------

\subsubsection*{Exercise 21 \quad\rm Gradient Descent}

Approximate the minimum of the function
$f(x_1,x_2) = 2x_1^2 -2x_1 +x_2^2 -x_2$
with the help of gradient descent!
Start from the initial approximation $(x_1,x_2) = (0,0)$ and use
$\eta = 0.2$ as the step width parameter (learning rate)!

%-----------------------------------------------------------------------

\subsubsection*{Exercise 22 \quad\rm Gradient Descent}

In Exercise~15a we considered fitting a straight line $y = a+bx$
to the data points $(-2,0), (0,1), (1,3), (2,5)$. In that exercise
we used the analytical solution via setting up and solving the system
of normal equations. In this exercise we want to solve the same problem
with the help of gradient descent.
\begin{enumerate}\itemsep0pt
\item[a)] What is the error function for this task?
\item[b)] What is the gradient descent update rule for this task?
\item[c)] Compute a few update steps starting from $a = b = 0$!
          Try different step width parameters,
          e.g.\ from the sequence 0.001, 0.003, 0.01, 0.03, 0.1,
          0.3, 1, 3 etc.!
\end{enumerate}

%-----------------------------------------------------------------------

\end{document}
