docgen progress

__transit/scripts/gen_oplist.py

 """script to generate doc/oplist.txt, which compiles to :doc:`oplist`. """
 __docformat__ = "restructuredtext en"
 import sys
-import gof
+from theano import gof
 
 def print_title(title_string, under_char, over_char=''):
     l = len(title_string)
@@ -94,15 +94,25 @@ class EntryConstructor(Entry):
         Entry.__init__(self, symbol, name, module)
 
 
-def search_entries(module_list):
-    ops = []
-    constructors = []
+def search_entries(module_list, ops = None, constructors = None, seen = None):
+    if ops is None: ops = []
+    if constructors is None: constructors = []
+    if seen is None: seen = set()
+    modules = []
 
     for module in module_list:
         symbol_name_list = [s for s in dir(module) if not s[0] == '_']
-
+        
 	for symbol_name in symbol_name_list:
 	    symbol = getattr(module, symbol_name)
+            try:
+                if symbol in seen:
+                    continue
+                seen.add(symbol)
+            except TypeError:
+                pass
+            if type(symbol) == type(module): # module
+                modules.append(symbol)
             try:
                 ops.append(EntryOp(symbol, symbol_name, module))
             except TypeError:
@@ -111,6 +121,9 @@ def search_entries(module_list):
                 except TypeError:
                     pass
 
+    for symbol in modules:
+        search_entries([symbol], ops, constructors, seen)
+
     return ops, constructors
 
 def print_entries(ops, constructors):
@@ -141,7 +154,7 @@ def print_entries(ops, constructors):
 
 if __name__ == "__main__":
     """Generate the op list"""
-    import scalar, sparse, tensor
+    import theano
 
     print_title("Op List", "~", "~")
     print """
@@ -156,11 +169,8 @@ In the future, this list may distinguish `constructors` that are Op instances fr
     print ".. contents:: "
     print ""
 
-    ops, constructors = search_entries([scalar, sparse, tensor])
+    ops, constructors = search_entries([theano])
 
     print_entries(ops, constructors)
 
     print ""
-
-    for line in open("doc/header.txt"):
-        print line[:-1]

doc/ext.py

 
+import os
 from docutils import nodes
 import epydoc.docwriter.xlink as xlink
 
@@ -7,6 +8,15 @@ def role_fn(name, rawtext, text, lineno, inliner,
     node = nodes.reference(rawtext, text, refuri = "http://pylearn.org/theano/wiki/%s" % text)
     return [node], []
 
+
 def setup(app):
-#    print dir(xlink)
+    help(xlink)
+    #role = xlink.create_api_role('api', True)
+    #print role
+
+    xlink.register_api('api', xlink.DocUrlGenerator())
+    xlink.set_api_file('api', os.path.join(app.outdir, 'api', 'api-objects.txt'))
+    xlink.set_api_root('api', os.path.join(app.outdir, 'api', ''))
+    xlink.create_api_role('api', True)
+    
     app.add_role("wiki", role_fn)

scripts/docgen.py

 
 import sys
 import os
+import inspect
 
-throot = "/".join(sys.path[0].split("/")[:-1])
+from epydoc import docintrospecter 
+from epydoc.apidoc import RoutineDoc
 
-os.chdir(throot)
+def Op_to_RoutineDoc(op, routine_doc, module_name=None):
+    routine_doc.specialize_to(RoutineDoc)
 
-def mkdir(path):
-    try:
-        os.mkdir(path)
-    except OSError:
-        pass
+    #NB: this code is lifted from
+    # /u/bergstrj/pub/prefix/x86_64-unknown-linux-gnu-Fedora_release_7__Moonshine_/lib/python2.5/site-packages/epydoc
+    # /u/bergstrj/pub/prefix/x86_64-unknown-linux-gnu-Fedora_release_7__Moonshine_/lib/python2.5/site-packages/epydoc/docintrospecter.py
 
-mkdir("html")
-mkdir("html/doc")
-mkdir("html/api")
 
-os.system("epydoc --config doc/api/epydoc.conf -o html/api")
+    # op should be an op instance
+    assert hasattr(op, 'perform')
 
-import sphinx
-sys.path[0:0] = [os.path.realpath('doc')]
-sphinx.main([sys.argv[0], 'doc', 'html'])
+    # Record the function's docstring.
+    routine_doc.docstring = getattr(op, '__doc__', '')
+
+    # Record the function's signature.
+    func = op.__epydoc_asRoutine
+    if isinstance(func, type(Op_to_RoutineDoc)):
+        (args, vararg, kwarg, defaults) = inspect.getargspec(func)
+
+        # Add the arguments.
+        routine_doc.posargs = args
+        routine_doc.vararg = vararg
+        routine_doc.kwarg = kwarg
+
+        # Set default values for positional arguments.
+        routine_doc.posarg_defaults = [None]*len(args)
+
+        # Set the routine's line number.
+        if hasattr(func, 'func_code'):
+            routine_doc.lineno = func.func_code.co_firstlineno
+    else:
+        # [XX] I should probably use UNKNOWN here??
+        # dvarrazzo: if '...' is to be changed, also check that
+        # `docstringparser.process_arg_field()` works correctly.
+        # See SF bug #1556024.
+        routine_doc.posargs = ['...']
+        routine_doc.posarg_defaults = [None]
+        routine_doc.kwarg = None
+        routine_doc.vararg = None
+
+    return routine_doc
+
+docintrospecter.register_introspecter(
+    lambda value: getattr(value, '__epydoc_asRoutine', False), 
+    Op_to_RoutineDoc, 
+    priority=-1)
+
+
+
+if __name__ == '__main__':
+
+    throot = "/".join(sys.path[0].split("/")[:-1])
+
+    os.chdir(throot)
+
+    def mkdir(path):
+        try:
+            os.mkdir(path)
+        except OSError:
+            pass
+
+    mkdir("html")
+    mkdir("html/doc")
+    mkdir("html/api")
+
+    if len(sys.argv) == 1 or sys.argv[1] != 'rst':
+        from epydoc.cli import cli
+        sys.path[0:0] = os.path.realpath('.')
+        sys.argv[:] = ['', '--config', 'doc/api/epydoc.conf', '-o', 'html/api']
+        cli()
+#        os.system("epydoc --config doc/api/epydoc.conf -o html/api")
+
+    if len(sys.argv) == 1 or sys.argv[1] != 'epydoc':
+        import sphinx
+        sys.path[0:0] = [os.path.realpath('doc')]
+        sphinx.main(['', 'doc', 'html'])
 
 

theano/tensor/basic.py

@@ -648,7 +648,7 @@ def cast(t, dtype):
 #to be removed as we get the epydoc routine-documenting thing going -JB 20080924
 def _conversion(real_value):
     __oplist_tag(real_value, 'casting')
-    real_value.__module__='tensor'
+    real_value.__module__='tensor.basic'
     return real_value
 
 convert_to_int8  = _conversion(elemwise.Elemwise(scal.Identity(scal.specific_out(scal.int8))))

theano/tensor/raw_random.py

@@ -64,6 +64,20 @@ class RandomFunction(gof.Op):
         return hash(self.fn) ^ hash(self.outtype) ^ hash(self.args) ^ hash(self.inplace)
 
 
+__oplist_constructor_list = []
+"""List of functions to be listed as op constructors in the oplist (`gen_oplist`, doc/oplist.txt)."""
+def constructor(f):
+    """Add `f` to :doc:`oplist`.
+    
+    Make `f` appear as a constructor in the oplist (`gen_oplist`, doc/oplist.txt).
+    """
+    __oplist_constructor_list.append(f)
+    return f
+def __oplist_tag(thing, tag):
+    tags = getattr(thing, '__oplist_tags', [])
+    tags.append(tag)
+    thing.__oplist_tags = tags
+
 def random_function(fn, dtype, *rfargs, **rfkwargs):
     """
     Returns a wrapper around RandomFunction which automatically infers the number 
@@ -76,6 +90,7 @@ def random_function(fn, dtype, *rfargs, **rfkwargs):
     - make_lvector(x, y, z, ...)
     - constants
     """
+    @constructor
     def f(ndim, *args, **kwargs):
         if isinstance(ndim, int):
             r, shape, args = args[0], args[1], args[2:]
@@ -94,10 +109,48 @@ def random_function(fn, dtype, *rfargs, **rfkwargs):
 RS = numpy.random.RandomState
 
 # we need to provide defaults for all the functions in order to infer the argument types...
+
 uniform = random_function(RS.uniform, 'float64', 0.0, 1.0)
+uniform.__doc__ = """
+Usage: uniform(random_state, size, low=0.0, high=1.0)
+Sample from a uniform distribution between low and high.
+
+If the size argument is ambiguous on the number of
+dimensions, the first argument may be a plain integer
+to supplement the missing information.
+"""
+
 binomial = random_function(RS.binomial, 'int64', 1, 0.5)
+binomial.__doc__ = """
+Usage: binomial(random_state, size, n=1, prob=0.5)
+Sample n times with probability of success prob for each trial,
+return the number of successes.
+
+If the size argument is ambiguous on the number of
+dimensions, the first argument may be a plain integer
+to supplement the missing information.
+"""
+
 normal = random_function(RS.normal, 'float64', 0.0, 1.0)
+normal.__doc__ = """
+Usage: normal(random_state, size, avg=0.0, std=1.0)
+Sample from a normal distribution centered on avg with
+the specified standard deviation (std)
+
+If the size argument is ambiguous on the number of
+dimensions, the first argument may be a plain integer
+to supplement the missing information.
+"""
+
 random_integers = random_function(RS.random_integers, 'int64', 0, 1)
+random_integers.__doc__ = """
+Usage: random_integers(random_state, size, low=0, high=1)
+Sample a random integer between low and high, both inclusive.
+
+If the size argument is ambiguous on the number of
+dimensions, the first argument may be a plain integer
+to supplement the missing information.
+"""
 
 
 @gof.local_optimizer