tags, docs, syntax highlighting in code-block

doc/apirst2html.py

@@ -29,12 +29,21 @@ except:
 # real ``epydoc`` package.  So remove ``sys.path[0]``, which contains the
 # directory of the script.
 import sys, os.path
-script_path = os.path.abspath(sys.path[0])
-sys.path = [p for p in sys.path if os.path.abspath(p) != script_path]
+
+# I leave this in place actually, so that I can import pygments_code_block_directive
+
+#script_path = os.path.abspath(sys.path[0])
+#sys.path = [p for p in sys.path if os.path.abspath(p) != script_path]
 
 import epydoc.docwriter.xlink as xlink
 
 from docutils.core import publish_cmdline, default_description
+try:
+    # .. code-block:: python should look nice with this
+    import pygments_code_block_directive
+except:
+    print >> sys.stderr, "failed to get pygments"
+
 description = ('Generates (X)HTML documents with API documentation links.  '
                 + default_description)
 publish_cmdline(reader=xlink.ApiLinkReader(), writer_name='html',

doc/graph.txt

@@ -12,18 +12,12 @@ Subtitle
 
 Here is some stuff.
 
- .. code-block::
-  def fib(n):
-     if n == 0:
-       return 1
-     if n == 1:
-       return 1
-     return fib(n-1) + fib(n-1)
-
- .. python::
-  def fib(n):
-     if n == 0:
-       return 1
-     if n == 1:
-       return 1
-     return fib(n-1) + fib(n-1)
+.. code-block:: python
+
+    def fib(n):
+        if n == 0:
+            return 1
+        if n == 1:
+            return 1
+        return fib(n-1) + fib(n-1)
+

doc/pygments_code_block_directive.py

+#!/usr/bin/python
+
+# :Author: a Pygments author|contributor; Felix Wiemann; Guenter Milde
+# :Date: $Date: 2007-06-13 12:20:42 +0200 (Wed, 13 Jun 2007) $
+# :Copyright: This module has been placed in the public domain.
+# 
+# This is a merge of `Using Pygments in ReST documents`_ from the pygments_
+# documentation, and a `proof of concept`_ by Felix Wiemann.
+# 
+# ========== ===========================================================
+# 2007-06-01 Removed redundancy from class values.
+# 2007-06-04 Merge of successive tokens of same type
+#            (code taken from pygments.formatters.others).
+# 2007-06-05 Separate docutils formatter script
+#            Use pygments' CSS class names (like the html formatter)
+#            allowing the use of pygments-produced style sheets.
+# 2007-06-07 Merge in the formatting of the parsed tokens
+#            (misnamed as docutils_formatter) as class DocutilsInterface
+# 2007-06-08 Failsave implementation (fallback to a standard literal block 
+#            if pygments not found)
+# ========== ===========================================================
+# 
+# ::
+
+"""Define and register a code-block directive using pygments
+"""
+
+# Requirements
+# ------------
+# ::
+
+from docutils import nodes
+from docutils.parsers.rst import directives
+try:
+    import pygments
+    from pygments.lexers import get_lexer_by_name
+    from pygments.formatters.html import _get_ttype_class
+except ImportError:
+    pass
+
+
+
+# Customisation
+# -------------
+# 
+# Do not insert inline nodes for the following tokens.
+# (You could add e.g. Token.Punctuation like ``['', 'p']``.) ::
+
+unstyled_tokens = ['']
+
+# DocutilsInterface
+# -----------------
+# 
+# This interface class combines code from
+# pygments.formatters.html and pygments.formatters.others.
+# 
+# It does not require anything of docutils and could also become a part of
+# pygments::
+
+class DocutilsInterface(object):
+    """Parse `code` string and yield "classified" tokens.
+    
+    Arguments
+    
+      code     -- string of source code to parse
+      language -- formal language the code is written in.
+    
+    Merge subsequent tokens of the same token-type. 
+    
+    Yields the tokens as ``(ttype_class, value)`` tuples, 
+    where ttype_class is taken from pygments.token.STANDARD_TYPES and 
+    corresponds to the class argument used in pygments html output.
+
+    """
+
+    def __init__(self, code, language):
+        self.code = code
+        self.language = language
+        
+    def lex(self):
+        # Get lexer for language (use text as fallback)
+        try:
+            lexer = get_lexer_by_name(self.language)
+        except ValueError:
+            # info: "no pygments lexer for %s, using 'text'"%self.language
+            lexer = get_lexer_by_name('text')
+        return pygments.lex(self.code, lexer)
+        
+            
+    def join(self, tokens):
+        """join subsequent tokens of same token-type
+        """
+        tokens = iter(tokens)
+        (lasttype, lastval) = tokens.next()
+        for ttype, value in tokens:
+            if ttype is lasttype:
+                lastval += value
+            else:
+                yield(lasttype, lastval)
+                (lasttype, lastval) = (ttype, value)
+        yield(lasttype, lastval)
+
+    def __iter__(self):
+        """parse code string and yield "clasified" tokens
+        """
+        try:
+            tokens = self.lex()
+        except IOError:
+            print "INFO: Pygments lexer not found, using fallback"
+            # TODO: write message to INFO 
+            yield ('', self.code)
+            return
+
+        for ttype, value in self.join(tokens):
+            yield (_get_ttype_class(ttype), value)
+
+
+
+# code_block_directive
+# --------------------
+# ::
+
+def code_block_directive(name, arguments, options, content, lineno,
+                       content_offset, block_text, state, state_machine):
+    """parse and classify content of a code_block
+    """
+    language = arguments[0]
+    # create a literal block element and set class argument
+    code_block = nodes.literal_block(classes=["code-block", language])
+    
+    # parse content with pygments and add to code_block element
+    for cls, value in DocutilsInterface(u'\n'.join(content), language):
+        if cls in unstyled_tokens:
+            # insert as Text to decrease the verbosity of the output.
+            code_block += nodes.Text(value, value)
+        else:
+            code_block += nodes.inline(value, value, classes=[cls])
+
+    return [code_block]
+
+
+# Register Directive
+# ------------------
+# ::
+
+code_block_directive.arguments = (1, 0, 1)
+code_block_directive.content = 1
+directives.register_directive('code-block', code_block_directive)
+
+# .. _doctutils: http://docutils.sf.net/
+# .. _pygments: http://pygments.org/
+# .. _Using Pygments in ReST documents: http://pygments.org/docs/rstdirective/
+# .. _proof of concept:
+#      http://article.gmane.org/gmane.text.docutils.user/3689
+# 
+# Test output
+# -----------
+# 
+# If called from the command line, call the docutils publisher to render the
+# input::
+
+if __name__ == '__main__':
+    from docutils.core import publish_cmdline, default_description
+    description = "code-block directive test output" + default_description
+    try:
+        import locale
+        locale.setlocale(locale.LC_ALL, '')
+    except:
+        pass
+    # Uncomment the desired output format:
+    publish_cmdline(writer_name='pseudoxml', description=description)
+    # publish_cmdline(writer_name='xml', description=description)
+    # publish_cmdline(writer_name='html', description=description)
+    # publish_cmdline(writer_name='latex', description=description)
+    # publish_cmdline(writer_name='newlatex2e', description=description)
+    
+
+

gen_oplist.py

@@ -3,100 +3,141 @@ __docformat__ = "restructuredtext en"
 import sys
 import gof
 
-def isOpClass(thing):
-    return hasattr(thing, 'perform') and not isinstance(thing, gof.Op)
-
-def isOpConstructor(thing, module):
-    return hasattr(thing, 'perform') and isinstance(thing, gof.Op)\
-            or thing in getattr(module, '_constructor_list', [])
-
 def print_title(title_string, under_char):
     print title_string
     print under_char * len(title_string)
     print ""
 
-def chomp(s):
-    """interpret and left-align a docstring"""
-
-    if 'subtensor' in s: 
-        debug = 0
-    else:
-        debug = 0
 
-    r = []
-    leadspace = True
-    for c in s:
-        if leadspace and c in ' \n\t':
-            continue
+class Entry:
+    """Structure for generating the oplist file"""
+    symbol = None
+    name = None
+    module = None
+    docstring = None
+    tags = []
+
+    def __init__(self, symbol, name, current_module):
+        self.symbol = symbol
+        self.name = name
+        self.module = symbol.__module__ #current_module.__name__ # symbol.__module__
+        self.docstring = symbol.__doc__
+        self.tags = ['module:%s' % current_module.__name__] + getattr(symbol, '__oplist_tags', [])
+
+    def mini_desc(self, maxlen=50):
+        """Return a short description of the op"""
+        def chomp(s):
+            """interpret and left-align a docstring"""
+
+            if 'subtensor' in s: 
+                debug = 0
+            else:
+                debug = 0
+
+            r = []
+            leadspace = True
+            for c in s:
+                if leadspace and c in ' \n\t':
+                    continue
+                else:
+                    leadspace = False
+
+                if c == '\n':
+                    if debug:
+                        print >> sys.stderr, 'breaking'
+                    break
+                if c in '\t*`': 
+                    c = ' ';
+                r.append(c)
+
+            if debug: 
+                print >> sys.stderr, r
+
+            return "".join(r)
+
+        minmax = 5
+        assert maxlen >= minmax
+        if not self.docstring: 
+            return "" #+ '(no doc)'
+        elif len(self.docstring) < maxlen:
+            return chomp(self.docstring)
         else:
-            leadspace = False
-
-        if c == '\n':
-            if debug:
-                print >> sys.stderr, 'breaking'
-            break
-        if c == '\t': 
-            c = ' ';
-        r.append(c)
-
-    if debug: 
-        print >> sys.stderr, r
-
-    return "".join(r)
+            return "%s ..."% chomp(self.docstring[:maxlen-minmax])
+
+    apilink = property(lambda self: ":api:`%s.%s`"% (self.module, self.name))
+    """Return the ReST link into the epydoc of this symbol"""
+
+class EntryOp(Entry):
+    def __init__(self, symbol, *args):
+        has_perform = hasattr(symbol, 'perform')
+        if symbol is gof.Op:
+            raise TypeError('not an Op subclass')
+        if not issubclass(symbol, gof.Op):
+            raise TypeError('not an Op subclass')
+        Entry.__init__(self, symbol, *args)
+
+class EntryConstructor(Entry):
+    def __init__(self, symbol, name, module):
+        is_op = isinstance(symbol, gof.Op)
+        is_ctor = symbol in getattr(module, '__oplist_constructor_list', [])
+        if not (is_op or is_ctor):
+            raise TypeError('not a constructor', symbol)
+        Entry.__init__(self, symbol, name, module)
+
+
+def search_entries(module_list):
+    ops = []
+    constructors = []
+
+    for module in module_list:
+        symbol_name_list = [s for s in dir(module) if not s[0] == '_']
 
+	for symbol_name in symbol_name_list:
+	    symbol = getattr(module, symbol_name)
+            try:
+                ops.append(EntryOp(symbol, symbol_name, module))
+            except TypeError:
+                try:
+                    constructors.append(EntryConstructor(symbol, symbol_name, module))
+                except TypeError:
+                    pass
 
-def generate():
-    """Generate the op list"""
-    import scalar, sparse, tensor
+    return ops, constructors
 
+def print_entries(ops, constructors):
     print_title("Theano Op List", "~")
     print ""
     print ".. contents:: "
     print ""
 
-    for module in [scalar, sparse, tensor]:
-	print_title('module: `%s`' % module.__name__, '=')
+    tags = {}
+    for o in ops + constructors:
+        for t in o.tags:
+            tags.setdefault(t, []).append(o)
 
-	print_title('Op Classes', '-')
+    for t in tags:
+        print_title(t, '=')
 
-        symbol_name_list = [s for s in dir(module) if not s[0] == '_']
+        tagged_ops = [op for op in tags[t] if isinstance(op, EntryOp)]
+        if len(tagged_ops):
+            print_title('Op Classes', '-')
+            for op in tagged_ops:
+                print "- %s" % op.apilink
+                print "  %s" % op.mini_desc()
+                print ""
 
-	for symbol_name in symbol_name_list:
-
-	    symbol = getattr(module, symbol_name)
+        tagged_ops = [op for op in tags[t] if isinstance(op, EntryConstructor)]
+        if len(tagged_ops):
+            print_title('Op Constructors', '-')
+            for op in tagged_ops:
+                print "- %s" % op.apilink
+                print "  %s" % op.mini_desc()
+                print ""
 
-	    if isOpClass(symbol):
-		print ""
-		print "- :api:`%s.%s`" % (symbol.__module__, symbol_name)
-		docstring = getattr(symbol, '__doc__', "")
-
-		if not docstring: 
-		    print " ", '(no doc)'
-		elif len(docstring) < 50:
-		    print " ", chomp(docstring)
-		else:
-		    print " ", chomp(docstring[:40]), "..."
-	# a little trailing whitespace
-	print ""
-
-	print_title('Op Constructors', '-')
-	for symbol_name in symbol_name_list:
-
-	    symbol = getattr(module, symbol_name)
-
-	    if isOpConstructor(symbol, module):
-		print ""
-		print "- :api:`%s.%s`" % (symbol.__module__, symbol_name)
-		docstring = getattr(symbol, '__doc__', "")
-
-		if not docstring: 
-		    print " ", 'No documentation'
-		elif len(docstring) < 50:
-		    print " ", chomp(docstring)
-		else:
-		    print " ", chomp(docstring[:40]), "..."
-	# a little trailing whitespace
-	print ""
 
 if __name__ == "__main__":
-    generate()
+    """Generate the op list"""
+    import scalar, sparse, tensor
+
+    ops, constructors = search_entries([scalar, sparse, tensor])
+    print_entries(ops, constructors)

tensor.py

@@ -28,12 +28,16 @@ import compile
 from elemwise import Elemwise, DimShuffle, CAReduce, Sum
 
 
-_constructor_list = []
+__oplist_constructor_list = []
 """List of functions to be listed as op constructors in the oplist (`gen_oplist`, doc/oplist.txt)."""
 def constructor(f):
     """Make `f` appear as a constructor in the oplist (`gen_oplist`, doc/oplist.txt)."""
-    _constructor_list.append(f)
+    __oplist_constructor_list.append(f)
     return f
+def __oplist_tag(thing, tag):
+    tags = getattr(thing, '__oplist_tags', [])
+    tags.append(tag)
+    thing.__oplist_tags = tags
 
 
 def as_tensor(x, name = None):
@@ -539,11 +543,12 @@ def _elemwise(scalar_op, name, doc_prefix=''):
 
     return straight, inplace
 
-def _redefine(real_symbol_value):
+def _redefine(real_symbol_value, module='tensor'):
     """Replace the value associated with a function symbol.
     
     This is useful to trick epydoc into doing what we want.  It's a hack.
     """
+    real_symbol_value.__module__ = 'tensor'
     def decorator(f):
         return real_symbol_value
     return decorator
@@ -573,6 +578,7 @@ def _scal_elemwise(symbol):
     #for the meaning of this see the ./epydoc script
     # it makes epydoc display rval as if it were a function, not an object
     rval.__epydoc_asRoutine = symbol
+    rval.__module__ = 'tensor'
 
     return rval
 
@@ -622,6 +628,8 @@ def cast(t, dtype):
 
 #to be removed as we get the epydoc routine-documenting thing going -JB 20080924
 def _conversion(real_value):
+    __oplist_tag(real_value, 'casting')
+    real_value.__module__='tensor'
     return real_value
 
 convert_to_int8  = _conversion(elemwise.Elemwise(scal.Identity(scal.specific_out(scal.int8))))