\documentclass[12pt,a4paper]{article}
\usepackage{graphicx}
\graphicspath{{eps/}{pdf/}}
\pagestyle{empty}
\oddsidemargin 2.1mm
\textwidth     155mm
\topmargin     -15mm
\textheight    240mm

\def\net{\mathop{\rm net}\nolimits}

\begin{document}

%-----------------------------------------------------------------------

\noindent
{\bf Artificial Neural Networks and Deep Learning}
\hfill Summer 2018 \\
Christian Borgelt and Christoph Doell \hfill from 2018.06.12

%-----------------------------------------------------------------------

\vskip3ex
\centerline{\bf Exercise Sheet 6}

%-----------------------------------------------------------------------

\subsubsection*{Exercise 23 \quad\rm Gradient Descent}

\hangindent-60mm\hangafter-4\leavevmode
\hbox to0pt{\vbox to10pt{\hbox to\textwidth{\hss
\includegraphics{nnex-04}}\vss}\hss}%
Consider a 2-layer perceptron with $n$~inputs and one output, in which
the output neuron has the weighted sum of its inputs as the network
input function, a semi-linear function \\
\parbox{95mm}{
\[ f_{\rm act}(\net,\theta) = \left\{\begin{array}{ll}
   1, & \mbox{if $\net > \theta +\frac{1}{2},$} \\[.5ex]
   0, & \mbox{if $\net < \theta -\frac{1}{2},$} \\
   (\net -\theta) +\frac{1}{2}, & \mbox{otherwise,}
   \end{array}\right. \]}\\
(see diagram) as its activation function and the identity as its
output function. Derive the rule for changing the weights that results
from an approach based on gradient descent if the network error is
computed as the sum (over all patterns) of the squared differences
between desired and actual output!

%-----------------------------------------------------------------------

\subsubsection*{Exercise 24 \quad\rm Gradient Descent}

\hangindent-60mm\hangafter-4\leavevmode
\hbox to0pt{\vbox to10pt{\hbox to\textwidth{\hss
\includegraphics{nnex-05}}\vss}\hss}%
Consider a 2-layer perceptron with $n$~inputs and one output, in which
the output neuron has the weighted sum of its inputs as the network
input function, a logistic function \\
\parbox{90mm}{%
\[ f_{\rm act}(\net,\theta) = \frac{1}{1 +e^{-(\net -\theta)}} \]} \\
(see diagram) as its activation function and the identity as its
output function. Derive the rule for changing the weights after a
single training pattern was presented that results from an approach
based on gradient descent if the network error is computed as the
absolute value of the difference between desired and actual output!
What changes to the backpropagation procedure would be necessary
(for a multi-layer perceptron)?

%-----------------------------------------------------------------------

\subsubsection*{Exercise 25 \quad\rm {\tt xmlp/wmlp}:
                                     Biimplication}

Start one of the programs {\tt xmlp} (for GNU/Linux)
or {\tt wmlp} (for Windows) (these programs are available at
{\tt http://www.borgelt.net/mlpd.html}), which show the training
process of a 3-layer perceptron. Choose {\tt Actions > Biimplication}
or {\tt Actions > Exclusive~Or}. Following the explanations in the
lecture concerning the biimplication problem and its solution, the
representation should be comprehensible. You can change the parameters
of the training procedure via {\tt Settings > Parameters...} Execute
the training multiple times and experiment with the values of the
parameters (momentum term, learning rate and range of the initial
weights).

%-----------------------------------------------------------------------

\subsubsection*{Exercise 26 \quad\rm {\tt xmlp/wmlp}:
                                     Function Approximation}

With the two programs {\tt xmlp} or {\tt wmlp} that were used in
Exercise 25 it is also possible to visualize the approximation of
two simple real-valued functions. Choose {\tt Actions > Function 1}
or {\tt Actions > Function 2}. For both functions, execute the training
multiple times. How are the functions approximated by the neural
network? Explain the meaning of the weights and the threshold/bias
values. Also, experiment again with the values of the parameters
(momentum term, learning rate and range of the initial
weights). What effects can be achieved?

%-----------------------------------------------------------------------

\end{document}
