\documentclass[utf8x,xcolor=pdftex,dvipsnames,table]{beamer}
\usetheme{Malmoe}  % Now it's a beamer presentation with the lisa theme!
\setbeamertemplate{footline}[page number]
\usecolortheme{beaver}
\usepackage[T1]{fontenc}
\usepackage{amsmath}
\usepackage[utf8x]{inputenc}
%\logo{\includegraphics[width=.8in]{UdeM_NoirBleu_logo_Marie_crop}}
\usepackage{listings}

\newcommand{\superscript}[1]{\ensuremath{^{\textrm{#1}}}}

\mode<presentation>

\title{Theano, Pylearn2, libgpuarray Presentation}

\author{%
\footnotesize
Frédéric Bastien, Bart van Merriënboer \newline
Département d'Informatique et de Recherche Opérationnelle \newline
Université de Montréal \newline
Montréal, Canada \newline
\texttt{\{bastienf, vanmerb\}@iro.umontreal.ca} \newline \newline
}

\date{OML Workshop 2014}

\setbeamertemplate{navigation symbols}{}

\begin{document}

\begin{frame}[plain]
 \titlepage
 \vspace{-5em}
 \includegraphics[width=1in]{../hpcs2011_tutorial/pics/lisabook_logo_text_3.png}
 \hfill
 \includegraphics[width=.8in]{../hpcs2011_tutorial/pics/UdeM_NoirBleu_logo_Marie_crop}
\end{frame}

\section{Introduction}
\begin{frame}{High level}\setcounter{page}{1}
  Python <- \{NumPy/SciPy/libgpuarray\} <- Theano <- Pylearn2
  \begin{itemize}
  \item Python: OO coding language
  \item Numpy: n-dimensional array object and scientific computing toolbox
  \item SciPy: sparse matrix objects and more scientific computing functionality
  \item libgpuarray: GPU n-dimentional array object in C for CUDA and OpenCL
  \item Theano: compiler/symbolic graph manipulation
  \item Pylearn2: machine learning framework
  \end{itemize}
\end{frame}

\begin{frame}{Python}
  \begin{itemize}
  \item General-purpose high-level OO interpreted language
  \item Emphasizes code readability
  \item Comprehensive standard library
  \item Dynamic type and memory management
  \item Slow execution
  \item Easily extensible with C
  \item Popular in *web development* and *scientific communities*
  \end{itemize}
\end{frame}

\begin{frame}{NumPy/SciPy}
  \begin{itemize}
  \item Python floats are full-fledged objects on the heap
      \begin{itemize}
      \item Not suitable for high-performance computing!
      \end{itemize}

  \item NumPy provides an $n$-dimensional numeric array in Python
      \begin{itemize}
      \item Perfect for high-performance computing
      \item Slices of arrays are views (no copy)
      \end{itemize}

  \item NumPy provides
      \begin{itemize}
      \item Elementwise computations
      \item Linear algebra, Fourier transforms
      \item Pseudorandom number generators, supporting many distributions
      \end{itemize}

  \item SciPy provides lots more, including
      \begin{itemize}
      \item Sparse matrices
      \item More linear algebra
      \item Solvers and optimization algorithms
      \item Matlab-compatible I/O
      \item I/O and signal processing for images and audio
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{What's missing?}
  \begin{itemize}
    \item Non-lazy evaluation (required by Python) hurts performance
    \item Bound to the CPU
    \item Lacks symbolic or automatic differentiation
    \item No automatic speed and stability optimization
  \end{itemize}

\end{frame}

\begin{frame}{Why scripting for GPUs?}
  \begin{bf}They complement each other\end{bf}

  GPUs are everything that high level languages are not

  \begin{itemize}
    \item Highly parallel
    \item Very architecture-sensitive
    \item Built for maximum FP/memory throughput
    \item So hard to program that meta-programming is easier.
  \end{itemize}

  \begin{bf}Best of both:\end{bf} easily scripted code which invokes high-performance GPU kernels.

  \begin{bf}Theano C code generation removes overhead\end{bf} from
  function call between Python and C, by launching many C functions at once.

\end{frame}

\begin{frame}{Theano}

  High-level domain-specific language tailored to numeric computation.

  \begin{itemize}
    \item Syntax as close to NumPy as possible
    \item Compiles most common expressions to C for CPU and/or GPU
    \item Limited expressivity means more opportunities optimizations
    \begin{itemize}
      \item No subroutines -> global optimization
      \item Strongly typed -> compiles to machine instructions
      \item Array oriented -> easy parallelism
      \item Support for looping and branching in expressions
    \end{itemize}
    \item Automatic speed and stability optimizations
    \item Can reuse other technologies for best performance.
    \begin{itemize}
      \item BLAS, SciPy, Cython, Numba, PyCUDA, CUDA
    \end{itemize}
    \item Automatic differentiation and R op
    \item Sparse matrices
  \end{itemize}
\end{frame}


\begin{frame}{Pylearn2}

  Machine Learning library aimed at researchers

  \begin{itemize}
    \item Built on top of Theano, for fast execution and use of GPU
    \item Easy to try variants of implemented algorithms, and to extend them (using Theano)
    \item Very modular, each component of the library can be used in isolation
    \item Experiments can be specified through a YAML config file, or by a Python script
    \item Scripts for visualizing weights, plot monitored values
  \end{itemize}
\end{frame}


\begin{frame}{Goal of the stack}
You can't have your cake and eat it too.\newline
We want our cake and eat it too.???\newline
Target the holy grail: \begin{bf}fast to develop\end{bf} and \begin{bf}fast to run\end{bf}.
\end{frame}


\begin{frame}{libgpuarray}
  GOAL: A common GPU n dimensions array that can be reused by all projects. It support CUDA and OpenCL.
  \newline \newline
  Motivation:
  \begin{itemize}
  \item Currently there are at least 6 different gpu arrays in python
    \begin{itemize}
    \item CudaNdarray(Theano), GPUArray(pycuda), CUDAMatrix(cudamat), GPUArray(pyopencl), Clyther, Copperhead, ...
    \item There are even more if we include other languages.
    \end{itemize}
  \item They are incompatible
    \begin{itemize}
    \item None have the same properties and interface.
    \end{itemize}
  \item All of them are a subset of numpy.ndarray on the gpu!
  \end{itemize}
\end{frame}


\section{Theano}
\begin{frame}{Description}
  \begin{itemize}
    \item Mathematical symbolic expression compiler
    \item Expressions mimic NumPy's syntax and semantics (easier to use)
    \item Dynamic C/CUDA code generation
    \begin{itemize}
      \item Is used with other technologie to generate fast code: C/C++, CUDA, OpenCL, PyCUDA, Cython, Numba, \ldots
    \end{itemize}
    \item Efficient symbolic differeniation
    \begin{itemize}
      \item Theano computes derivatives of functions with one or many inputs.
      \item Also support computation of the Jacobian, Hessian, R and L op.
    \end{itemize}
    \item Speed and stability optimizations
    \begin{itemize}
      \item Gives the right answer for ``log(1+x)'' even if x is really tiny.
    \end{itemize}
    \item Works on Linux, Mac and Windows
    \item Transparent use of a GPU
    \begin{itemize}
      \item {\tt float32} only for now (working on other data types)
      \item Still in experimental state on Windows
    \end{itemize}

    \item Extensive unit-testing and self-verification
    \begin{itemize}
      \item Detects and diagnoses many types of errors
    \end{itemize}

%    \item Statically typed and purely functional
    \item Sparse operations (CPU only)
  \end{itemize}
\end{frame}

% The following does not work with lstset, for some reason
%\begin{frame}{Simple example}
\begin{frame}[fragile]
  \frametitle{Simple example}

\lstset{language=Python,
        commentstyle=\itshape\color{blue}
        }
\begin{lstlisting}
import theano
# declare symbolic variable
a = theano.tensor.vector("a")
# build symbolic expression
b = a + a ** 10
# compile function
f = theano.function([a], b)
print f([0, 1, 2])
# prints `array([0, 2, 1026])`
\end{lstlisting}
\end{frame}

\begin{frame}{Simple example: graph optimization}
\center
\includegraphics[width=0.35\textwidth]{../hpcs2011_tutorial/pics/f_unoptimized.png}
\hspace{0.1\textwidth}
\includegraphics[width=0.35\textwidth]{../hpcs2011_tutorial/pics/f_optimized.png}
%Symbolic programming = *Paradigm shift*: people need to use it to understand it.

\end{frame}


\begin{frame}{Project status?}
  \begin{itemize}
    \item Mature: Theano has been developed and used since January 2008 (6.5 yrs old)
    \item Driven over 100 research papers
    \item Good user documentation
    \item Active mailing list with participants from outside our lab
    \item Core technology for a few Silicon-Valley start-ups
    \item Many contributors (some from outside our lab)
    \item Used to teach many university classes
    \item Has been used for research at Google and Yahoo.
  \end{itemize}
  Theano: \url{deeplearning.net/software/theano/}

  Deep Learning Tutorials: \url{deeplearning.net/tutorial/}
\end{frame}


\section{Pylearn2}
\begin{frame}{Pylearn2 details}
  \begin{itemize}
    \item The core library contains a collection of:
    \begin{itemize}
      \item Training algorithms (i.e., Stochastic and Batch GD, model-specific rules)
      \begin{itemize}
        \item Costs, supervised/unsupervised and exact/estimated (NLL, Score matching, NCE)
        \item Monitor, history of (functions of) parameters and hyperparameters on different data sets (training, validation, test)
        \item TerminationCriterion, determines when to stop training
      \end{itemize}
      \item Training extensions, perform actions throughout the training process based on the model's state (e.g. early stopping)
      \item Models (NNets, ConvNets, RBMs, k-means, PCA, SVMs)
      \item Datasets (MNIST, CIFAR-10) and preprocessors (LCN, ZCA)
    \end{itemize}
  \item Data specifications which give semantics to data
  \begin{itemize}
    \item IndexSpace, labels in the form of a 1D array of integers
    \item VectorSpace, numerica data as 1D float32 array
    \item Conv2DSpace, color images as 3D float32 arrays
  \end{itemize}
  \item Allows for automatic conversion e.g. labels to one-hot vectors, images to flattened vectors
\end{itemize}
\end{frame}
\begin{frame}{Project status}
  \begin{itemize}
    \item Has been used for scientific publications, Kaggle competitions, used by many researchers at LISA
    \item Still under rapid development, however the API shouldn't break without warning
    \item Documentation is incomplete, but improving
    \item Features currently in development:
    \begin{itemize}
      \item Recurrent neural networks (RNNs), based on the GroundHog framework developed at LISA
      \item Better hyperparameter search support for YAML files
    \end{itemize}
  \end{itemize}
\end{frame}

\section{libgpuarray}
\begin{frame}{libgpuarray: Design Goals}
  \begin{itemize}
  \item Have the base object in C to allow collaboration with more projects.
    \begin{itemize}
    \item We want people from C, C++, ruby, R, ... all use the same base GPU ndarray.
    \end{itemize}
  \item Be compatible with CUDA and OpenCL.
  \item Not too simple, (don’t support just matrix).
  \item Support all dtype.
  \item Allow strided views.
  \item But still easy to develop new code that support only a few memory layout.
    \begin{itemize}
    \item This ease the development of new code.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Project status?}
  \begin{itemize}
  \item Usable directly, but not all implementation available.
  \item Multiple GPU work.
  \item Is the next GPU array container for Theano and is working.
    \begin{itemize}
    \item Not all Theano implementation available now.
    \item OpenCL miss more implementation.
    \item Multiple GPU on the way.
    \end{itemize}
  \item Web site: \url{http://deeplearning.net/software/libgpuarray/}
  \end{itemize}
\end{frame}

\section{sharing}
\begin{frame}{Simplifing code sharing}
\begin{enumerate}
  \item<1-> License: \begin{bf}Suggest BSD\end{bf} as it is used by many software in our field.
    \begin{itemize}
    \item Common license help share code.
    \item When reusing code, don't forget to keep the license and the copyright notice
    \end{itemize}
  \item<2-> Common base object! \begin{bf}libgpuarray\end{bf}
  \item<3-> Otherwise: put important implementation(e.g convolution) in \begin{bf}separate file\end{bf} and \begin{bf}use raw ptr/shape/strides\end{bf} as inputs. Document that interface.
  \item<4-> Acknowledge reuse \begin{bf}in section on web site\end{bf} AND \begin{bf}in papers\end{bf} about the software we reuse! (and use too)
\end{enumerate}

\end{frame}

\begin{frame}{Theano/PyLearn2 Future}
\end{frame}

\section{Features}

\begin{frame}{OLD STUFF Strides}
\only<1>{Strides is a way to specify how much memory to skip between each element of a dimension.}
\only<2>{We can use strides to take submatrix ${\color{cyan!50}B}$ without copying any memory.}
\begin{center}
\onslide<1->{Matrix ${\color{red!50}A}$}\hspace{5em}\onslide<2->{Matrix ${\color{cyan!50}B}$}
\end{center}
\begin{center}
%%\only<1>{\includegraphics{strides-1}}
%%\only<2>{\includegraphics{strides-2}}
\end{center}
\end{frame}

\begin{frame}{Features desired}
\begin{itemize}
\item {\color{gray!80} Support for varying datatypes}
\item {\color{gray!80} Support for an arbitrary number of dimensions}
\item {\color{gray!80} Support for strides}
\item Support for broadcasting
\item {\color{gray!80} Compatibility with CUDA and OpenCL}
\end{itemize}
\end{frame}

\begin{frame}{Comparison of existing implementations}
\begin{table}
\rowcolors{2}{RoyalBlue!5}{RoyalBlue!23}
\begin{tabular}{|l|c|c|c|c|c|}
\hline
Package & strides & bcast & dims & types & backends \\
\hline
\hline
Theano & yes\footnote{as number of elements} & yes & any & float32 & CUDA \\
PyCUDA& no & no & any & all & CUDA \\
PyOpenCL & no & no & any & all & OpenCL \\
CUDAMat & no & yes\footnote{via a function} & 2 & float32 & CUDA \\
Gnumpy & no & yes & any & float32\footnote{and a hackish form of boolean} & CUDA \\
Thrust & no & no & 1 & all & CUDA \\
\hline
\hiderowcolors
Desired & yes & yes & any & all & both \\
\hline
\end{tabular}
\end{table}
\end{frame}


%%\vspace{-1em}

\section{Conclusion}

\begin{frame}{Acknowledgment}
\begin{itemize}
\item James Bergstra
\item Compute Canada, RQCHP, NSERC, and Canada Research Chairs for providing funds or access to compute resources.
\end{itemize}
\end{frame}

\begin{frame}
\begin{center}
\Huge
Questions?
\end{center}
\end{frame}


\end{document}
