Merged with all other changes

.hgignore

@@ -2,6 +2,7 @@ syntax: glob
 *.pyo
 *~
 \#*\#
+doc/oplist.txt
 compiled/*.cpp
 cutils_ext.cpp
 html

__init__.py

@@ -19,6 +19,8 @@ To learn more, check out:
 
 - Index of Howto documents (:wiki:`IndexHowto`)
 
+- Op List (:doc:`oplist`)
+
 """
 
 __docformat__ = "restructuredtext en"

_test_compile.py

@@ -77,7 +77,7 @@ class T_Function(unittest.TestCase):
     def test_closure(self):
         x, y, z = tensor.scalars('xyz')
         v = tensor.value(numpy.zeros(()))
-        e = x + tensor.add_inplace(v, 1)
+        e = x + tensor._add_inplace(v, 1)
         f = function([x], [e])
         assert f(1.) == 2.
         assert f(1.) == 3.
@@ -109,7 +109,7 @@ class T_Function(unittest.TestCase):
     def test_borrow_false_through_inplace(self):
         x, y, z = tensor.scalars('xyz')
         # if borrow_outputs is False, we must not reuse the temporary created for x+y
-        e = tensor.add_inplace(x + y, z)
+        e = tensor._add_inplace(x + y, z)
         for linker in 'py c c|py c&py'.split():
             f = function([x, y, z], [e], borrow_outputs = False, linker = linker)
             res1 = f(1.0, 2.0, 3.0)

_test_tensor_opt.py

@@ -3,12 +3,12 @@
 
 import unittest
 
-from theano import gof
-from theano.tensor_opt import *
-from theano import tensor
-from theano.tensor import Tensor
-from theano.gof import Env
-from theano.elemwise import DimShuffle
+import gof
+from tensor_opt import *
+import tensor
+from tensor import Tensor
+from gof import Env
+from elemwise import DimShuffle
 import numpy
 #import scalar_opt
 
@@ -43,7 +43,7 @@ def inputs(xbc = (0, 0), ybc = (0, 0), zbc = (0, 0)):
 #     def test_user_inplace(self):
 #         x, y, z = inputs()
 #         e0 = x + y
-#         e1 = tensor.mul_inplace(x, y)
+#         e1 = tensor._mul_inplace(x, y)
 #         g = Env([x, y], [e0, e1])
 #         self.failUnless(str(g) == "[Broadcast{Add}(x, y), Broadcast{Mul}{0: 0}(x, y)]")
 #         inplace_optimizer.optimize(g)
@@ -52,7 +52,7 @@ def inputs(xbc = (0, 0), ybc = (0, 0), zbc = (0, 0)):
 #     def test_inplace_on_second_argument(self):
 #         x, y, z = inputs()
 #         e0 = x + y
-#         e1 = tensor.mul_inplace(x, z)
+#         e1 = tensor._mul_inplace(x, z)
 #         g = Env([x, y], [e0, e1])
 #         self.failUnless(str(g) == "[Broadcast{Add}(x, y), Broadcast{Mul}{0: 0}(x, z)]")
 #         inplace_optimizer.optimize(g)
@@ -98,9 +98,9 @@ class _test_dimshuffle_lift(unittest.TestCase):
 
 
 
-from theano.tensor import *
+from tensor import *
 
-from theano.sandbox  import pprint
+from sandbox  import pprint
 
 class _test_greedy_distribute(unittest.TestCase):
     def test_main(self):
@@ -279,8 +279,8 @@ class _test_canonize(unittest.TestCase):
 
 # #     def test_inplace(self):
 # #         x, y, z = inputs()
-# #         #e = tensor.add_inplace(x, y + z)
-# #         e = x + tensor.add_inplace(y, z)
+# #         #e = tensor._add_inplace(x, y + z)
+# #         e = x + tensor._add_inplace(y, z)
 # #         g = Env([x, y, z], [e])
 # #         opt = CliqueOptimizer(through_broadcast = False,
 # #                               scalar_optimizer = None,

doc/apirst2html.py

+#!/usr/bin/python
+# -*- coding: iso-8859-1 -*-
+
+"""An HTML writer supporting link to external documentation.
+
+This module is a frontend for the Docutils_ HTML writer. It allows a document
+to reference objects documented in the API documentation generated by
+extraction tools such as Doxygen_ or Epydoc_.
+
+.. _Docutils:   http://docutils.sourceforge.net/
+.. _Doxygen:    http://www.doxygen.org/
+.. _Epydoc:     http://epydoc.sourceforge.net/
+"""
+
+# $Id: apirst2html.py 1531 2007-02-18 23:07:25Z dvarrazzo $
+__version__ = "$Revision: 1531 $"[11:-2]
+__author__ = "Daniele Varrazzo"
+__copyright__ = "Copyright (C) 2007 by Daniele Varrazzo"
+__docformat__ = 'reStructuredText en'
+
+try:
+    import locale
+    locale.setlocale(locale.LC_ALL, '')
+except:
+    pass
+
+# We have to do some path magic to prevent Python from getting
+# confused about the difference between the ``epydoc.py`` script, and the
+# real ``epydoc`` package.  So remove ``sys.path[0]``, which contains the
+# directory of the script.
+import sys, os.path
+script_path = os.path.abspath(sys.path[0])
+sys.path = [p for p in sys.path if os.path.abspath(p) != script_path]
+
+import epydoc.docwriter.xlink as xlink
+
+from docutils.core import publish_cmdline, default_description
+description = ('Generates (X)HTML documents with API documentation links.  '
+                + default_description)
+publish_cmdline(reader=xlink.ApiLinkReader(), writer_name='html',
+                description=description)

doc/doc.idx

@@ -2,3 +2,4 @@ graph	graph.html
 tensor	graph.html
 result	graph.html
 howto	graph.html
+oplist	oplist.html

epydoc

-# TODO:
-#   Get all graphs to work!
+#!/usr/bin/python
+#
+# Call the command line interface for Epydoc.
+#
+
+# Make sure that we don't get confused between an epydoc.py script and
+# the real epydoc package.
+import sys, os.path, inspect
+if os.path.exists(os.path.join(sys.path[0], 'epydoc.py')):
+    del sys.path[0]
+
+from epydoc import docintrospecter 
+from epydoc.apidoc import RoutineDoc
+
+def Op_to_RoutineDoc(op, routine_doc, module_name=None):
+    routine_doc.specialize_to(RoutineDoc)
+
+    #NB: this code is lifted from
+    # /u/bergstrj/pub/prefix/x86_64-unknown-linux-gnu-Fedora_release_7__Moonshine_/lib/python2.5/site-packages/epydoc
+    # /u/bergstrj/pub/prefix/x86_64-unknown-linux-gnu-Fedora_release_7__Moonshine_/lib/python2.5/site-packages/epydoc/docintrospecter.py
+
+
+    # op should be an op instance
+    assert hasattr(op, 'perform')
+
+    # Record the function's docstring.
+    routine_doc.docstring = getattr(op, '__doc__', '')
+
+    # Record the function's signature.
+    func = op.__epydoc_asRoutine
+    if isinstance(func, type(Op_to_RoutineDoc)):
+        (args, vararg, kwarg, defaults) = inspect.getargspec(func)
+
+        # Add the arguments.
+        routine_doc.posargs = args
+        routine_doc.vararg = vararg
+        routine_doc.kwarg = kwarg
+
+        # Set default values for positional arguments.
+        routine_doc.posarg_defaults = [None]*len(args)
+
+        # Set the routine's line number.
+        if hasattr(func, 'func_code'):
+            routine_doc.lineno = func.func_code.co_firstlineno
+    else:
+        # [XX] I should probably use UNKNOWN here??
+        # dvarrazzo: if '...' is to be changed, also check that
+        # `docstringparser.process_arg_field()` works correctly.
+        # See SF bug #1556024.
+        routine_doc.posargs = ['...']
+        routine_doc.posarg_defaults = [None]
+        routine_doc.kwarg = None
+        routine_doc.vararg = None
+
+    return routine_doc
+
+docintrospecter.register_introspecter(
+        lambda value: getattr(value, '__epydoc_asRoutine', False), 
+        Op_to_RoutineDoc, 
+        priority=-1)
+
+from epydoc.cli import cli
+cli()
 
-
-[epydoc] # Epydoc section marker (required by ConfigParser)
-
-# The list of objects to document.  Objects can be named using
-# dotted names, module filenames, or package directory names.
-# Alases for this option include "objects" and "values".
-modules: *.py, gof/*.py, theano/*.py, theano/gof/*.py, joseph/*.py, pylearn/*.py, scipy, numpy
-
-# The type of output that should be generated.  Should be one
-# of: html, text, latex, dvi, ps, pdf.
-output: html
-
-# The path to the output directory.  May be relative or absolute.
-target: html/
-
-# An integer indicating how verbose epydoc should be.  The default
-# value is 0; negative values will supress warnings and errors;
-# positive values will give more verbose output.
-verbosity: 1
-
-# A boolean value indicating that Epydoc should show a tracaback
-# in case of unexpected error. By default don't show tracebacks
-debug: 0
-
-# If True, don't try to use colors or cursor control when doing
-# textual output. The default False assumes a rich text prompt
-simple-term: 0
-
-
-### Generation options
-
-# The default markup language for docstrings, for modules that do
-# not define __docformat__.  Defaults to epytext.
-docformat: epytext
-
-# Whether or not parsing should be used to examine objects.
-parse: yes
-
-# Whether or not introspection should be used to examine objects.
-introspect: yes
-
-# Don't examine in any way the modules whose dotted name match this
-# regular expression pattern.
-#exclude
-
-# Don't perform introspection on the modules whose dotted name match this
-# regular expression pattern.
-#exclude-introspect
-
-# Don't perform parsing on the modules whose dotted name match this
-# regular expression pattern.
-#exclude-parse
-
-# The format for showing inheritance objects.
-# It should be one of: 'grouped', 'listed', 'included'.
-inheritance: grouped
-
-# Whether or not to inclue private variables.  (Even if included,
-# private variables will be hidden by default.)
-private: yes
-
-# Whether or not to list each module's imports.
-imports: yes
-
-# Whether or not to include syntax highlighted source code in
-# the output (HTML only).
-sourcecode: yes
-
-# Whether or not to includea a page with Epydoc log, containing
-# effective option at the time of generation and the reported logs.
-include-log: yes
-
-
-### Output options
-
-# The documented project's name.
-name: Theano
-
-# The CSS stylesheet for HTML output.  Can be the name of a builtin
-# stylesheet, or the name of a file.
-css: white
-
-# The documented project's URL.
-url: http://lgcm.iro.umontreal.ca/theano/
-
-# HTML code for the project link in the navigation bar.  If left
-# unspecified, the project link will be generated based on the
-# project's name and URL.
-#link: <a href="somewhere">My Cool Project</a>
-
-# The "top" page for the documentation.  Can be a URL, the name
-# of a module or class, or one of the special names "trees.html",
-# "indices.html", or "help.html"
-#top: os.path
-
-# An alternative help file.  The named file should contain the
-# body of an HTML file; navigation bars will be added to it.
-#help: my_helpfile.html
-
-# Whether or not to include a frames-based table of contents.
-#frames: yes
-frames: no
-
-# Whether each class should be listed in its own section when
-# generating LaTeX or PDF output.
-separate-classes: no
-
-
-### API linking options
-
-# Define a new API document.  A new interpreted text role
-# will be created
-#external-api: epydoc
-
-# Use the records in this file to resolve objects in the API named NAME.
-#external-api-file: epydoc:api-objects.txt
-
-# Use this URL prefix to configure the string returned for external API.
-#external-api-root: epydoc:http://epydoc.sourceforge.net/api
-
-
-### Graph options
-
-# The list of graph types that should be automatically included
-# in the output.  Graphs are generated using the Graphviz "dot"
-# executable.  Graph types include: "classtree", "callgraph",
-# "umlclass".  Use "all" to include all graph types
-graph: all
-
-# The path to the Graphviz "dot" executable, used to generate
-# graphs.
-dotpath: /usr/bin/dot
-
-# The name of one or more pstat files (generated by the profile
-# or hotshot module).  These are used to generate call graphs.
-pstat: autotest.pstat
-
-# Specify the font used to generate Graphviz graphs.
-# (e.g., helvetica or times).
-graph-font: Helvetica
-
-# Specify the font size used to generate Graphviz graphs.
-graph-font-size: 10
-
-
-### Return value options
-
-# The condition upon which Epydoc should exit with a non-zero
-# exit status. Possible values are error, warning, docstring_warning
-#fail-on: error

gen_oplist.py

+"""script to generate doc/oplist.txt, which compiles to :doc:`oplist`. """
+__docformat__ = "restructuredtext en"
+import sys
+import gof
+
+def isOpClass(thing):
+    return hasattr(thing, 'perform') and not isinstance(thing, gof.Op)
+
+def isOpConstructor(thing, module):
+    return hasattr(thing, 'perform') and isinstance(thing, gof.Op)\
+            or thing in getattr(module, '_constructor_list', [])
+
+def print_title(title_string, under_char):
+    print title_string
+    print under_char * len(title_string)
+    print ""
+
+def chomp(s):
+    """interpret and left-align a docstring"""
+
+    if 'subtensor' in s: 
+        debug = 0
+    else:
+        debug = 0
+
+    r = []
+    leadspace = True
+    for c in s:
+        if leadspace and c in ' \n\t':
+            continue
+        else:
+            leadspace = False
+
+        if c == '\n':
+            if debug:
+                print >> sys.stderr, 'breaking'
+            break
+        if c == '\t': 
+            c = ' ';
+        r.append(c)
+
+    if debug: 
+        print >> sys.stderr, r
+
+    return "".join(r)
+
+
+def generate():
+    """Generate the op list"""
+    import scalar, sparse, tensor
+
+    print_title("Theano Op List", "~")
+    print ""
+    print ".. contents:: "
+    print ""
+
+    for module in [scalar, sparse, tensor]:
+	print_title('module: `%s`' % module.__name__, '=')
+
+	print_title('Op Classes', '-')
+
+        symbol_name_list = [s for s in dir(module) if not s[0] == '_']
+
+	for symbol_name in symbol_name_list:
+
+	    symbol = getattr(module, symbol_name)
+
+	    if isOpClass(symbol):
+		print ""
+		print "- :api:`%s.%s`" % (symbol.__module__, symbol_name)
+		docstring = getattr(symbol, '__doc__', "")
+
+		if not docstring: 
+		    print " ", '(no doc)'
+		elif len(docstring) < 50:
+		    print " ", chomp(docstring)
+		else:
+		    print " ", chomp(docstring[:40]), "..."
+	# a little trailing whitespace
+	print ""
+
+	print_title('Op Constructors', '-')
+	for symbol_name in symbol_name_list:
+
+	    symbol = getattr(module, symbol_name)
+
+	    if isOpConstructor(symbol, module):
+		print ""
+		print "- :api:`%s.%s`" % (symbol.__module__, symbol_name)
+		docstring = getattr(symbol, '__doc__', "")
+
+		if not docstring: 
+		    print " ", 'No documentation'
+		elif len(docstring) < 50:
+		    print " ", chomp(docstring)
+		else:
+		    print " ", chomp(docstring[:40]), "..."
+	# a little trailing whitespace
+	print ""
+
+if __name__ == "__main__":
+    generate()

gof/op.py

@@ -29,18 +29,18 @@ class CLinkerOp(object):
         given names for the inputs and outputs.
         
         :Parameters:
-         `node`: Apply instance
+         `node` : Apply instance
            WRITEME
-         `name`: WRITEME
+         `name` : WRITEME
            WRITEME
-         `inputs`: list of strings
+         `inputs` : list of strings
            There is a string for each input of the function, and the string is the name of a C
            `PyObject` variable pointing to that input.
-         `outputs`: list of strings
+         `outputs` : list of strings
            Each string is the name of a `PyObject` pointer where the Op should store its
            results.  The `CLinker` guarantees that on entry to this code block, each pointer
            is either NULL or is unchanged from the end of the previous execution.
-         `sub`: dict of strings
+         `sub` : dict of strings
            extra symbols defined in `CLinker` sub symbols (such as 'fail').
            WRITEME
 
@@ -59,18 +59,18 @@ class CLinkerOp(object):
         This is a convenient place to clean up things allocated by c_code().  
         
         :Parameters:
-         `node`: Apply instance
+         `node` : Apply instance
            WRITEME
-         `name`: WRITEME
+         `name` : WRITEME
            WRITEME
-         `inputs`: list of strings
+         `inputs` : list of strings
            There is a string for each input of the function, and the string is the name of a C
            `PyObject` variable pointing to that input.
-         `outputs`: list of strings
+         `outputs` : list of strings
            Each string is the name of a `PyObject` pointer where the Op should store its
            results.  The `CLinker` guarantees that on entry to this code block, each pointer
            is either NULL or is unchanged from the end of the previous execution.
-         `sub`: dict of strings
+         `sub` : dict of strings
            extra symbols defined in `CLinker` sub symbols (such as 'fail').
            WRITEME
         
@@ -225,11 +225,11 @@ class PureOp(object):
         output storage.  Return None.
 
         :Parameters:
-         `node`: Apply instance 
+         `node` : Apply instance 
             contains the symbolic inputs and outputs
-         `inputs`: list
+         `inputs` : list
             sequence of inputs (immutable)
-         `output_storage`: list
+         `output_storage` : list
              list of mutable 1-element lists (do not change the length of these lists)
 
         The `output_storage` list might contain data. If an element of

local.build_html.sh

 #!/bin/bash
 
-mkdir -p html/api
-epydoc --config local.epydoc
-
-cd doc
-sh build_html.sh
-cd ../
-rm -Rf html/doc
-mv doc/html html/doc
+APIRST2HTML=doc/apirst2html.py
+EPYDOC_ARGS='--external-api=api --external-api-file=api:html/api/api-objects.txt --external-api-root=api:../api/'
+
+
+mkdir -p html/api && mkdir -p html/doc
+
+# this builds some stuff or something... basically makes the rest work properly
+# for a reason I don't understand.  -JB 20080924
+python __init__.py
+
+if [ " $1" != " rst" ]; then
+./epydoc --config local.epydoc
+fi
+
+if [ " $1" != " epydoc" ]; then
+python gen_oplist.py > doc/oplist.txt
+for RST in graph oplist ;  do
+
+    $APIRST2HTML $EPYDOC_ARGS doc/$RST.txt html/doc/$RST.html
+done
+fi
 

local.epydoc

@@ -7,17 +7,12 @@
 # The list of objects to document.  Objects can be named using
 # dotted names, module filenames, or package directory names.
 # Alases for this option include "objects" and "values".
-modules: *.py, gof/__init__.py,
-    gof/a*,
-    gof/c*,
-    gof/d*,
-    gof/e*,
-    gof/g*,
-    gof/l*,
-    gof/o*,
-    gof/p*,
-    gof/t*,
-    gof/u*
+modules: __init__.py,
+    [a-z]*.py,
+    [A-Z]*.py,
+    gof/__init__.py,
+    gof/[a-z]*.py,
+    gof/[A-Z]*.py
 
 # The type of output that should be generated.  Should be one
 # of: html, text, latex, dvi, ps, pdf.

scalar.py

@@ -586,7 +586,7 @@ class Abs(UnaryScalarOp):
             return "%(z)s = fabs(%(x)s);" % locals()
         #complex, other?
         raise NotImplementedError('type not supported', type)
-abs = Abs(same_out)
+_abs = Abs(same_out)
 
 class Sgn(UnaryScalarOp):
     def impl(self, x):

tensor_opt.py

@@ -20,7 +20,7 @@ def in2out(*local_opts):
 
 # gemm: (d,a,b,c,s) -> d = d*s + a*dot(b,c)
 # Transforms d -= a * dot(b, c) into gemm(d, -a, b, c, 1.0)
-gemm_pattern_1 = gof.PatternSub((T.sub_inplace,
+gemm_pattern_1 = gof.PatternSub((T._sub_inplace,
                                  'd',
                                  (T.mul,
                                   dict(pattern = (T.DimShuffle((), ['x', 'x'], inplace = True), 'a'),