Skip to content

P
pytensor
提交 dcbe40c8 authored 作者: Pierre Luc Carrier's avatar Pierre Luc Carrier 提交者： Arnaud Bergeron
浏览文件
操作

Remove compile.module as well as references to it and tests for it

上级 238b38d1
隐藏空白字符变更
内嵌 并排
正在显示 包含 0 行增加 和 3052 行删除
theano/__init__.py
浏览文件 @ dcbe40c8
......@@ -58,15 +58,11 @@ from theano.compile import \
Mode, \
predefined_modes, predefined_linkers, predefined_optimizers, \
FunctionMaker, function, function_dump, OpFromGraph, \
Component, External, Member, Method, \
Composite, ComponentList, ComponentDict, Module, \
ProfileMode, ProfileStats, \
Param, shared, as_op
from theano.misc.safe_asarray import _asarray
FancyModule = Module
from theano.printing import pprint, pp
from theano.scan_module import scan, map, reduce, foldl, foldr, clone
......
theano/compile/__init__.py
浏览文件 @ dcbe40c8
......@@ -12,8 +12,6 @@ from theano.compile.mode import *
from theano.compile.io import *
from theano.compile.module import *
from theano.compile.debugmode import DebugMode
from theano.compile.monitormode import MonitorMode
......
theano/compile/module.py deleted 100644 → 0
浏览文件 @ 238b38d1
"""Classes implementing Theano's Module system.
For design notes, see doc/advanced/module.txt
"""
__docformat__ = "restructuredtext en"
import sys, warnings
from itertools import chain
from theano import gof
from theano.printing import pprint
from theano.compile import io
from theano.compat.python2x import all
import theano.compile.function_module
import theano.compile.mode
#This module is imported by other parts of theano, and worse still, other
#parts of theano do "from module import *"
#So rather than printing out a warning if you import from this module, we
#must stick a warning inside all of the components
def deprecation_warning():
# Make sure the warning is displayed only once.
if deprecation_warning.already_displayed:
return
warnings.warn(
"theano modules are deprecated and will be removed in release 0.7",
stacklevel=3)
deprecation_warning.already_displayed = True
deprecation_warning.already_displayed = False
def name_join(*args):
deprecation_warning()
"""
Creates a string representation for the given names:
join('a', 'b', 'c') => 'a.b.c'
"""
return ".".join(arg for arg in args if arg)
def name_split(sym, n=-1):
deprecation_warning()
"""
Gets the names from their joined representation
split('a.b.c') => ['a', 'b', 'c']
Returns the n first names, if n==-1 returns all of them.
split('a.b.c',1) => ['a', 'b.c']
"""
return sym.split('.', n)
class AllocationError(Exception):
"""
Exception raised when a Variable has no associated storage.
"""
pass
class Component(object):
"""
Base class for the various kinds of components which are not
structural but may be meaningfully used in structures (Member,
Method, etc.)
"""
def __init__(self, no_warn = False):
if not no_warn:
deprecation_warning()
self.__dict__['_name'] = ''
self.__dict__['parent'] = None
def allocate(self, memo):
"""
Populates the memo dictionary with gof.Variable -> io.In
pairings. The value field of the In instance should contain a
gof.Container instance. The memo dictionary is meant to tell
the build method of Components where the values associated to
certain variables are stored and how they should behave if they
are implicit inputs to a Method (needed to compute its
output(s) but not in the inputs or updates lists).
"""
raise NotImplementedError
def build(self, mode, memo):
"""
Makes an instance of this Component using the mode provided
and taking the containers in the memo dictionary.
A Component which builds nothing, such as External, may return
None.
The return value of this function will show up in the Module graph produced by make().
"""
raise NotImplementedError()
def make_no_init(self, mode=None):
"""
Allocates the necessary containers using allocate() and uses
build() with the provided mode to make an instance which will
be returned. The initialize() method of the instance will not
be called.
"""
if mode is None:
mode = theano.compile.mode.get_default_mode()
memo = {}
self.allocate(memo)
rval = self.build(mode, memo)
return rval
def make(self, *args, **kwargs):
"""
Allocates the necessary containers using allocate() and uses
build() to make an instance which will be returned. The
initialize() method of the instance will be called with the
arguments and the keyword arguments. If 'mode' is in the
keyword arguments it will be passed to build().
"""
mode = kwargs.pop('mode', theano.compile.mode.get_default_mode())
rval = self.make_no_init(mode)
if hasattr(rval, 'initialize'):
rval.initialize(*args, **kwargs)
return rval
def __repr__(self):
return str(self)
def __str__(self):
return self.__class__.__name__
def pretty(self, **kwargs):
"""
Returns a pretty representation of this Component, suitable
for reading.
"""
raise NotImplementedError
def __get_name__(self):
"""
Getter for self.name
"""
return self._name
def __set_name__(self, name):
"""
Setter for self.name
"""
self._name = name
name = property(lambda self: self.__get_name__(),
lambda self, value: self.__set_name__(value),
"Contains the name of this Component")
class _RComponent(Component):
"""
Base class for a Component wrapping a Variable. For internal use.
"""
def __init__(self, r):
deprecation_warning()
super(_RComponent, self).__init__()
self.r = r
# If self.owns_name is True, then the name of the variable
# may be adjusted when the name of the Component is. Else,
# the variable will always keep its original name. The component
# will only be allowed to own a variable's name if it has no
# original name to begin with. This allows the user to opt out
# of the automatic naming scheme if he or she wants to. It is
# also usually the case that a Variable used in more than one
# Component should only retain the first name it gets.
self.owns_name = r.name is None
def __set_name__(self, name):
super(_RComponent, self).__set_name__(name)
if self.owns_name:
self.r.name = name
def __str__(self):
return "%s(%s)" % (self.__class__.__name__, self.r)
def pretty(self, **kwargs):
rval = '%s :: %s' % (self.__class__.__name__, self.r.type)
return rval
class External(_RComponent):
"""
External represents a Variable which comes from somewhere else
(another module) or is a temporary calculation.
"""
def allocate(self, memo):
deprecation_warning()
# nothing to allocate
return None
def build(self, mode, memo):
"""
Builds nothing.
"""
return None
def pretty(self, **kwargs):
rval = super(External, self).pretty()
if self.r.owner:
rval += '\n= %s' % (pprint(self.r, dict(target = self.r)))
return rval
class Member(_RComponent):
"""
Member represents a Variable which is a state of a Composite. That
Variable will be accessible from a built Composite and it is
possible to do updates on Members.
Member builds a gof.Container.
"""
def allocate(self, memo):
"""
If the memo does not have a Container associated to this
Member's Variable, instantiates one and sets it in the memo.
"""
deprecation_warning()
r = self.r
if memo and r in memo:
return memo[r]
assert isinstance(r, gof.Variable)
rval = gof.Container(r, storage = [getattr(r, 'data', None)],
readonly=isinstance(r, gof.Constant))
memo[r] = io.In(variable=r,
value=rval,
mutable=False)
return memo[r]
def build(self, mode, memo):
"""
Returns the Container associated to this Member's Variable.
"""
return memo[self.r].value
class Method(Component):
"""
Method is a declaration of a function. It contains inputs,
outputs and updates. If the Method is part of a Composite
which holds references to Members, the Method may use them
without declaring them in the inputs, outputs or updates list.
inputs, outputs or updates may be strings. In that case, they
will be resolved in the Composite which is the parent of this
Method.
Method builds a Function (same structure as a call to
theano.function)
"""
inputs = []
"""function inputs (see `compile.function`)
If Module members are named explicitly in this list, then they will not use shared storage.
Storage must be provided either via an `io.In` value argument, or at the point of the
function call.
"""
outputs = None
"""function outputs (see `compile.function`)"""
updates = {}
"""update expressions for module members
If this method should update the shared storage value for a Module member, then the
update expression must be given in this dictionary.
Keys in this dictionary must be members of the module graph--variables for which this Method
will use the shared storage.
The value associated with each key should be a Variable (or a string that can be resolved to
a Variable) representing the computation of a new value for this shared storage after
each function call.
"""
mode = None
"""This will override the Module compilation mode for this Method"""
def __init__(self, inputs, outputs, updates=None, mode=None):
"""Initialize attributes
:param inputs: value for `Method.inputs`
:param outputs: value for `Method.outputs`
:param updates: value for `Method.updates`
:param mode: value for `Method.mode`
:type inputs: list of (str or `Variable` or `io.In`)
:type outputs: None or str or `Variable` or `io.Out` or list of (str or `Variable` or
`io.Out`)
:type updates: dict of `Variable` or str -> `Variable` or str
:type mode: None or any mode accepted by `compile.function`
"""
deprecation_warning()
if updates is None:
updates = {}
super(Method, self).__init__()
self.inputs = inputs
self.outputs = outputs
self.updates = dict(updates)
self.mode = mode
def resolve_all(self):
"""Convert all inputs, outputs, and updates specified as strings to Variables.
This works by searching the attribute list of the Module to which this Method is bound.
"""
def resolve_variable(x, passthrough=(gof.Variable,)):
if isinstance(x, passthrough):
return x
elif isinstance(x, _RComponent):
return x.r
else:
raise Exception('The following thing is not of the following types', x,
passthrough + (_RComponent,))
# return self.resolve(x).r
def resolve_inputs():
if isinstance(self.inputs, (io.In, gof.Variable, basestring)):
inputs = [self.inputs]
else:
inputs = list(self.inputs)
self.inputs = [resolve_variable(input,
passthrough=(gof.Variable, io.In)) for input in inputs]
def resolve_outputs():
if isinstance(self.outputs, (io.Out, gof.Variable, basestring, type(None))):
output = self.outputs
self.outputs = resolve_variable(output,
passthrough=(gof.Variable, io.Out, type(None)))
else:
outputs = list(self.outputs)
self.outputs = [resolve_variable(output,
passthrough=(gof.Variable, io.Out)) for output in outputs]
def resolve_updates():
updates = self.updates
self.updates = {}
for k, v in updates.iteritems():
k, v = resolve_variable(k), resolve_variable(v)
self.updates[k] = v
resolve_inputs()
resolve_outputs()
resolve_updates()
def allocate(self, memo):
"""
Method allocates nothing.
"""
return None
def build(self, mode, memo, allocate_all=False):
"""Compile a function for this Method.
:param allocate_all: if True, storage will be
allocated for all needed Variables even if there is no
associated storage for them in the memo. If allocate_all is
False, storage will only be allocated for Variables that are
reachable from the inputs list.
:returns: a function that implements this method
:rtype: `Function` instance
"""
if self in memo:
return memo[self]
self.resolve_all() # resolve all so we don't have to mess with strings
def get_storage(r, require=False):
# If require is True, we can only get storage from the memo.
try:
return memo[r]
except KeyError:
if require:
raise AllocationError('There is no storage associated to %s used by %s = %s.'
' Verify that it is indeed a Member of the'
' enclosing module or of one of its submodules.' % (r, self.name, self))
else:
return io.In(variable=r,
value=gof.Container(r,
storage=[getattr(r, 'data', None)],
readonly=(isinstance(r, gof.Constant))),
mutable=False)
inputs = self.inputs
# Deal with explicit inputs
inputs = []
for input in self.inputs:
if type(input) is io.In:
inputs.append(input)
elif isinstance(input, gof.Variable):
input_in = io.In(
variable=input,
mutable=False)
inputs.append(input_in)
else:
raise TypeError(input, type(input))
# Deal with updates to shared storage
for k, v in self.updates.iteritems():
assert isinstance(k, gof.Variable)
if isinstance(k, gof.Constant):
raise TypeError('Module Constants cannot be updated', k)
assert isinstance(v, gof.Variable)
#identify an input for variable k
input_k = None
for input in inputs:
if input.variable == k:
input_k = input
#print 'METHOD UPDATE', k, v, input_k
if input_k is None:
# this is an implicit input,
# use shared storage
input_k = io.In(
variable=k,
update=v,
value=get_storage(k, not allocate_all).value,
mutable=True,
implicit = True)
inputs.append(input_k)
else:
raise ValueError(('Variable listed in both inputs and updates.'
' Use inputs to use your own storage, use updates to '
'work on module-shared storage'), k)
# Deal with module inputs that are not updated
outputs = self.outputs
_inputs = [x.variable for x in inputs]
# Grab the variables that are not accessible from either the inputs or the updates.
if outputs is None:
outputs_list = []
else:
if isinstance(outputs, (list, tuple)):
outputs_list = list(outputs)
else:
outputs_list = [outputs]
#backport
#outputs_list = [] if outputs is None else (list(outputs) if isinstance(outputs, (list, tuple)) else [outputs])
outputs_variable_list = []
for o in outputs_list:
if isinstance(o, io.Out):
outputs_variable_list += [o.variable]
else:
outputs_variable_list += [o]
#backport
#outputs_variable_list = [o.variable if isinstance(o, io.Out) else o for o in outputs_list]
for input in gof.graph.inputs(outputs_variable_list
+ [x.update for x in inputs if getattr(x, 'update', False)],
blockers = _inputs):
if input not in _inputs:
# Add this input to the inputs; we require that storage already exists for them,
# but otherwise they are immutable.
if isinstance(input, gof.Constant):
#input might be Constant
storage = get_storage(input)
assert type(storage) is io.In
container = storage.value
#the user is allowed to change this value between function calls if it isn't a constant
assert container.readonly == (isinstance(input, gof.Constant))
#the function is not allowed to change this value
assert storage.mutable == False
else:
storage = get_storage(input, not allocate_all)
# Declare as an implicit input.
# TODO Note from OD: is this dangerous? (in case this storage
# is shared, and would sometimes need to be implicit, sometimes
# not).
storage.implicit = True
assert type(storage) is io.In
inputs.append(storage)
if self.mode is None:
effective_mode = mode
else:
effective_mode = self.mode
# We ignore unused inputs, since all the inputs are passed
rval = theano.compile.function_module.orig_function(inputs, outputs, effective_mode,
on_unused_input='ignore')
memo[self] = rval
return rval
def pretty(self, **kwargs):
self.resolve_all()
if self.inputs:
rval = 'inputs: %s\n' % ", ".join(map(str, self.inputs))
else:
rval = ''
if isinstance(self.outputs, (list, tuple)):
inputs, outputs, updates = self.inputs, self.outputs
else:
inputs, outputs, updates = [self.outputs], self.updates
#backport
#inputs, outputs, updates = self.inputs, self.outputs if isinstance(self.outputs, (list, tuple)) else [self.outputs], self.updates
# If mode is in kwargs, prints the optimized version of the method
mode = kwargs.pop('mode', None)
if mode:
f = self.build(mode, {}, True)
einputs, eoutputs = f.maker.fgraph.inputs, f.maker.fgraph.outputs
updates = dict(((k, v) for k, v in zip(einputs[len(inputs):], eoutputs[len(outputs):])))
inputs, outputs = einputs[:len(inputs)], eoutputs[:len(outputs)]
rval += pprint(inputs, outputs, updates, False)
return rval
def __str__(self):
if self.updates:
sep = "; "
else:
sep = ""
return "Method(%s -> %s%s%s)" % \
(self.inputs,
self.outputs,
sep,
#backport
#"; " if self.updates else "",
", ".join("%s <= %s" % (old, new) for old, new in self.updates.iteritems()))
def __call__(self, *args, **kwargs):
raise TypeError("'Method' object is not callable"
" (Hint: compile your module first. See Component.make())")
class CompositeInstance(object):
"""
Generic type which various Composite subclasses are intended to
build.
"""
def __init__(self, component, __items__):
deprecation_warning()
# The Component that built this CompositeInstance
self.__dict__['component'] = component
# Some data structure indexable using []
self.__dict__['__items__'] = __items__
def __getitem__(self, item):
x = self.__items__[item]
# For practical reasons, if the item is a Container, we
# return its contents.
if isinstance(x, gof.Container):
return x.value
return x
def __setitem__(self, item, value):
x = self.__items__[item]
if isinstance(x, gof.Container):
# If the item is a Container, we set its value
x.value = value
elif hasattr(x, 'initialize'):
# If the item has an initialize() method, we use
# it with the value as argument
x.initialize(value)
else:
##self.__items__[item] = value
raise KeyError('Cannot set item %s' % item)
class Composite(Component):
"""
Composite represents a structure that contains Components.
"""
def resolve(self, name):
raise NotImplementedError
def components(self):
"""
Returns all components.
"""
raise NotImplementedError
def components_map(self):
"""
Returns (key, value) pairs corresponding to each component.
"""
raise NotImplementedError
def flat_components(self, include_self=False):
"""
Generator that yields each component in a flattened hierarchy
of composites and components. If include_self is True, the
list will include the Composite instances, else it will only
yield the list of leaves.
"""
if include_self:
yield self
for component in self.components():
if isinstance(component, Composite):
for x in component.flat_components(include_self):
yield x
else:
yield component
def flat_components_map(self, include_self=False, path=None):
"""
Generator that yields (path, component) pairs in a flattened
hierarchy of composites and components, where path is a
sequence of keys such that::
component is self[path[0]][path[1]]...
If include_self is True, the list will include the Composite
instances, else it will only yield the list of leaves.
"""
if path is None:
path = []
if include_self:
yield path, self
for name, component in self.components_map():
path2 = path + [name]
if isinstance(component, Composite):
for fullpath, x in component.flat_components_map(include_self, path2):
yield fullpath, x
else:
yield path2, component
def allocate(self, memo):
"""
Does allocation for each component in the composite.
"""
deprecation_warning()
for member in self.components():
member.allocate(memo)
def get(self, item):
"""
Get the Component associated to the key.
"""
raise NotImplementedError
def set(self, item, value):
"""
Set the Component associated to the key.
"""
raise NotImplementedError
def __getitem__(self, item):
# Uses get() internally
print 'COMPOSITE GETITEM', item
x = self.get(item)
if isinstance(x, (External, Member)):
return x.r
return x
def __setitem__(self, item, value):
# Uses set() internally
self.set(item, value)
def __iter__(self):
retval = []
for c in self.components():
if isinstance(c, (External, Member)):
retval += [c.r]
else:
retval += [c]
return retval
#backport
#return (c.r if isinstance(c, (External, Member)) else c for c in self.components())
class ComponentListInstance(CompositeInstance):
def __str__(self):
return '[%s]' % ', '.join(map(str, self.__items__))
def __len__(self):
return len(self.__items__)
def initialize(self, init):
deprecation_warning()
for i, initv in enumerate(init):
self[i] = initv
class ComponentList(Composite):
"""
ComponentList represents a sequence of Component. It builds a
ComponentListInstance.
"""
def __init__(self, *_components):
deprecation_warning()
super(ComponentList, self).__init__()
if len(_components) == 1 and isinstance(_components[0], (list, tuple)):
_components = _components[0]
self._components = []
for c in _components:
if not isinstance(c, Component):
raise TypeError(c, type(c))
self.append(c)
def components(self):
return self._components
def components_map(self):
return enumerate(self._components)
def build(self, mode, memo):
if self in memo:
return memo[self]
builds = [c.build(mode, memo) for c in self._components]
rval = ComponentListInstance(self, builds)
memo[self] = rval
return rval
def get(self, item):
return self._components[item]
def set(self, item, value):
if isinstance(value, gof.Variable):
value = Member(value)
elif not isinstance(value, Component):
raise TypeError('ComponentList may only contain Components.', value, type(value))
#value = value.bind(self, str(item))
value.name = name_join(self.name, str(item))
self._components[item] = value
def append(self, c):
if isinstance(c, gof.Variable):
c = Member(c)
elif not isinstance(c, Component):
raise TypeError('ComponentList may only contain Components.', c, type(c))
self._components.append(c)
def extend(self, other):
for o in other:
self.append(o)
def __add__(self, other):
if isinstance(other, (list, tuple)):
return ComponentList(self._components + map(wrap,other))
elif isinstance(other, ComponentList):
return ComponentList(self._components + other._components)
else:
return NotImplemented
def __radd__(self, other):
if isinstance(other, (list, tuple)):
return ComponentList(map(wrap,other) + self._components)
elif isinstance(other, ComponentList):
return ComponentList(other._components + self._components)
else:
return NotImplemented
def __str__(self):
return str(self._components)
def __len__(self):
return len(self._components)
def pretty(self, **kwargs):
cr = '\n ' #if header else '\n'
strings = []
#if header:
# rval += "ComponentList:"
for i, c in self.components_map():
strings.append('%i:%s%s' % (i, cr, c.pretty(**kwargs).replace('\n', cr)))
#rval += cr + '%i -> %s' % (i, c.pretty(header = True, **kwargs).replace('\n', cr))
return '\n'.join(strings)
def __set_name__(self, name):
super(ComponentList, self).__set_name__(name)
for i, member in enumerate(self._components):
member.name = '%s.%i' % (name, i)
def default_initialize(self, init=None, **kwinit):
deprecation_warning()
if init is None:
init = {}
for k, initv in dict(init, **kwinit).iteritems():
self[k] = initv
class ComponentDictInstanceNoInit(CompositeInstance):
"""Component Instance that allows new items to be added"""
def __setitem__(self, item, value):
if item not in self.__items__:
# Set it if it's not there
# TODO: is this needed here? move to ModuleInstance?
self.__items__[item] = value
else:
super(ComponentDictInstanceNoInit, self).__setitem__(item, value)
def __str__(self):
strings = []
for k, v in sorted(self.__items__.iteritems()):
if isinstance(v, gof.Container):
v = v.value
if not k.startswith('_') and not callable(v) and not k in getattr(self, '__hide__', []):
pre = '%s: ' % k
strings.append('%s%s' % (pre, str(v).replace('\n', '\n' + ' '*len(pre))))
return '{%s}' % '\n'.join(strings).replace('\n', '\n ')
class ComponentDictInstance(ComponentDictInstanceNoInit):
"""
ComponentDictInstance is meant to be instantiated by ComponentDict.
"""
def initialize(self, init=None, **kwinit):
deprecation_warning()
if init is None:
init = {}
for k, initv in dict(init, **kwinit).iteritems():
self[k] = initv
class ComponentDict(Composite):
InstanceType = ComponentDictInstance # Type used by build() to make the instance
def __init__(self, components=None, **kwcomponents):
deprecation_warning()
if components is None:
components = {}
super(ComponentDict, self).__init__()
components = dict(components, **kwcomponents)
for val in components.itervalues():
if not isinstance(val, Component):
raise TypeError(val, type(val))
self.__dict__['_components'] = components
def components(self):
return self._components.itervalues()
def components_map(self):
return self._components.iteritems()
def build(self, mode, memo):
if self in memo:
return self[memo]
inst = self.InstanceType(self, {})
for name, c in self._components.iteritems():
x = c.build(mode, memo)
if x is not None:
inst[name] = x
memo[self] = inst
return inst
def get(self, item):
return self._components[item]
def set(self, item, value):
if not isinstance(value, Component):
msg = """
ComponentDict may only contain Components.
(Hint: maybe value here needs to be wrapped, see theano.compile.module.register_wrapper.)"""
raise TypeError(msg, value, type(value))
#value = value.bind(self, item)
value.name = name_join(self.name, str(item))
self._components[item] = value
def pretty(self, **kwargs):
cr = '\n ' #if header else '\n'
strings = []
# if header:
# rval += "ComponentDict:"
for name, component in self.components_map():
if name.startswith('_'):
continue
strings.append('%s:%s%s' % (name, cr, component.pretty(**kwargs).replace('\n', cr)))
strings.sort()
return '\n'.join(strings)
def __str__(self):
return self.__class__.__name__+"(%s)" % ', '.join(x for x in sorted(map(str, self._components)) if x[0] != '_')
def __set_name__(self, name):
super(ComponentDict, self).__set_name__(name)
for mname, member in self._components.iteritems():
member.name = '%s.%s' % (name, mname)
__autowrappers = []
def register_wrapper(condition, wrapper, no_warn = False):
"""
:type condition: function x -> bool
:param condition: this function should return True iff `wrapper` can
sensibly turn x into a Component.
:type wrapper: function x -> Component
:param wrapper: this function should convert `x` into an instance of
a Component subclass.
"""
if not no_warn:
deprecation_warning()
__autowrappers.append((condition, wrapper))
def wrapper(x):
"""Returns a wrapper function appropriate for `x`
Returns None if not appropriate wrapper is found
"""
deprecation_warning()
for condition, wrap_fn in __autowrappers:
if condition(x):
return wrap_fn
return None
def wrap(x):
"""
Wraps `x` in a `Component`. Wrappers can be registered using
`register_wrapper` to allow wrapping more types.
It is necessary for Module attributes to be wrappable.
A Module with an attribute that is not wrappable as a Component, will cause
`Component.make` to fail.
"""
deprecation_warning()
w = wrapper(x)
if w is not None:
return w(x)
else:
return x
def dict_wrap(d):
deprecation_warning()
d_copy = {}
for k,v in d.iteritems():
d_copy[k]=wrap(v)
return d_copy
# Component -> itself
register_wrapper(lambda x: isinstance(x, Component),
lambda x: x, no_warn = True)
# Variable -> Member
register_wrapper(lambda x: isinstance(x, gof.Variable) and not x.owner,
lambda x: Member(x), no_warn = True)
# Variable -> External
register_wrapper(lambda x: isinstance(x, gof.Variable) and x.owner,
lambda x: External(x), no_warn = True)
# [[Variable1], {Variable2}, Variable3...] -> ComponentList(Member(Variable1), Member(Variable2), ...)
register_wrapper(lambda x: isinstance(x, (list, tuple)) \
and all(wrapper(r) is not None for r in x),
lambda x: ComponentList(*map(wrap, x)), no_warn = True)
#{ "name1":{Component,Variable,list,tuple,dict},...} -> ComponentDict({Component,Variable,list,tuple,dict},...)
register_wrapper(lambda x: isinstance(x, dict) \
and all(wrapper(r) is not None for r in x.itervalues()),
lambda x: ComponentDict(dict_wrap(x)),no_warn = True)
class Curry:
def __init__(self, obj, name, arg):
deprecation_warning()
self.obj = obj
self.name = name
self.meth = getattr(self.obj, self.name)
self.arg = arg
def __call__(self, *args, **kwargs):
return self.meth(self.arg, *args, **kwargs)
def __getstate__(self):
return [self.obj, self.name, self.arg]
def __setstate__(self, state):
self.obj, self.name, self.arg = state
self.meth = getattr(self.obj, self.name)
class ModuleInstance(ComponentDictInstanceNoInit):
"""
WRITEME
:note: ModuleInstance is meant to be instantiated by Module. This differs
from ComponentDictInstance on a key point, which is that getattr
does a similar thing to getitem.
:note: ModuleInstance is compatible for use as ComponentDict.InstanceType.
"""
def __getattr__(self, attr):
if attr == '__items__' and '__items__' not in self.__dict__:
self.__dict__['__items__'] = {}
try:
return self[attr]
except KeyError:
raise AttributeError('%s has no %s attribute.' % (self.__class__, attr))
def __setattr__(self, attr, value):
try:
self[attr] = value
except KeyError:
self.__dict__[attr] = value
class Module(ComponentDict):
"""WRITEME
You should inherit from Module with the members will be other Modules or Components. To
make more specialized elements of a Module graph, consider inheriting from Component
directly.
"""
InstanceType = ModuleInstance # By default, we use build ModuleInstance
def __init__(self, *args, **kw):
deprecation_warning()
super(Module, self).__init__(*args, **kw)
self.__dict__["local_attr"]={}
self.__dict__["_components"]={}
def __wrapper__(self, x):
"""
This function is called whenever x is set as an attribute of
the Module.
"""
return wrap(x)
def __setattr__(self, attr, value):
# a is a new attribute
# we will use the local_attr dict to store it
v = self.unpack_member_and_external(value)
# this __setattr__ function overrides property.__set__, so we don't worry about a
# setter here
self.__dict__[attr] = v
self.__dict__["local_attr"][attr] = v
@staticmethod
def unpack_member_and_external(v):
if isinstance(v, Member):
print >> sys.stderr, ("WARNING: assignment of Member "
"object %s (either directly or indirectly) to Module "
"is deprecated. Just use Variable." % v)
return v.r
elif isinstance(v, External):
print >> sys.stderr, ("WARNING: assignment of External "
"object %s (either directly or indirectly) to Module "
"is deprecated. Just use Variable." % v)
return v.r
elif isinstance(v, (gof.Variable,Method,Module)):
return v
elif isinstance(v,(int,bool)):
return v
elif isinstance(v, (list)):
return map(Module.unpack_member_and_external,v)
elif isinstance(v, (tuple)):
return tuple(map(Module.unpack_member_and_external,v))
elif isinstance(v,dict):
v_copy = dict()
for k,vv in v.iteritems():
v_copy[k]=Module.unpack_member_and_external(vv)
return v
else:
# raise NotImplementedError
# print "WARNING: unknow:",v
return v
def old__getattr__(self, attr):
if attr == '_components' and '_components' not in self.__dict__:
self.__dict__['_components'] = {}
try:
rval = self.__dict__["local_attr"][attr]
except KeyError:
raise AttributeError('%s has no %s attribute.' % (self.__class__, attr))
return rval
def old__setattr__(self, attr, value):
"""
"""
if attr in ('parent', '_components'):
self.__dict__[attr] = value
return
self.__dict__["local_attr"][attr] = self.unpack_member_and_external(value)
def build(self, mode, memo):
if self in memo:
return memo[self]
for k,v in self.local_attr.iteritems():
self.__setattr__(k,v)
inst = super(Module, self).build(mode, memo)
if not isinstance(inst, ModuleInstance):
raise TypeError('The InstanceType of a Module should inherit from ModuleInstance',
(self, type(inst)))
for methodname in dir(self):
# Any method with a name like '_instance_XXX' is added to
# the object built under the name obj.XXX
if methodname.startswith('_instance_'):
new_methodname = methodname[len('_instance_'):]
if hasattr(inst, new_methodname):
print >> sys.stderr, "WARNING: not overriding already-defined method",
print >> sys.stderr, getattr(inst, new_methodname),
print >> sys.stderr, "with",
print >> sys.stderr, getattr(self, methodname)
else:
curried = Curry(self, methodname, inst)
# setattr doesn't work here because we overrode __setattr__
# setattr(inst, new_methodname, curried)
inst.__dict__[new_methodname] = curried
assert getattr(inst, new_methodname) == curried
#print 'ADDING METHOD', method, 'to', id(inst), new_methodname, getattr(inst, new_methodname)
memo[self] = inst
return inst
def _instance_initialize(self, inst, init=None, **kwinit):
"""
Default initialization method.
"""
if init is None:
init = {}
for name, value in chain(init.iteritems(), kwinit.iteritems()):
inst[name] = value
def make_module_instance(self, *args, **kwargs):
"""
Module's __setattr__ method hides all members under local_attr. This
method iterates over those elements and wraps them so they can be used
in a computation graph. The "wrapped" members are then set as object
attributes accessible through the dotted notation syntax (<module_name>
<dot> <member_name>). Submodules are handled recursively.
"""
# Function to go through member lists and dictionaries recursively,
# to look for submodules on which make_module_instance needs to be called
def recurse(v):
if isinstance(v,list):
iterv = enumerate(v)
else:
iterv = v.iteritems()
#backport
#iter = enumerate(v) if isinstance(v,list) else v.iteritems()
for sk,sv in iterv:
if isinstance(sv,(list,dict)):
sv = recurse(sv)
elif isinstance(sv,Module):
sv = sv.make_module_instance(args,kwargs)
v[sk] = sv
return v
for k,v in self.local_attr.iteritems():
if isinstance(v,Module):
v = v.make_module_instance(args,kwargs)
self[k] = self.__wrapper__(v)
elif isinstance(v,Method):
self.__setitem__(k,v)
else:
# iterate through lists and dictionaries to wrap submodules
if isinstance(v,(list,dict)):
self[k] = self.__wrapper__(recurse(v))
try:
self[k] = self.__wrapper__(v)
except Exception:
if isinstance(v, Component):
raise
else:
self.__dict__[k] = v
return self
def make(self, *args, **kwargs):
"""
Allocates the necessary containers using allocate() and uses
build() to make an instance which will be returned. The
initialize() method of the instance will be called with the
arguments and the keyword arguments. If 'mode' is in the
keyword arguments it will be passed to build().
"""
self.make_module_instance(args,kwargs)
mode = kwargs.pop('mode', theano.compile.mode.get_default_mode())
rval = self.make_no_init(mode)
if hasattr(rval, 'initialize'):
rval.initialize(*args, **kwargs)
return rval
def __str__(self):
return self.__class__.__name__+"(%s)" % ', '.join(x for x in sorted(map(str, self.local_attr)) if x[0] != '_')
def __get_name__(self):
"""
Getter for self.name
"""
return self._name
def __set_name__(self, name):
"""
Setter for self.name
"""
self._name = name
name = property(lambda self: self.__get_name__(),
lambda self, value: self.__set_name__(value),
"Contains the name of this Component")
FancyModule = Module
FancyModuleInstance = ModuleInstance
def func_to_mod(f):
"""
Creates a dummy module, with external member variables for the input
parameters required by the function f, and a member output defined as::
output <= f(**kwinit)
"""
deprecation_warning()
def make(**kwinit):
m = Module()
outputs = f(**kwinit)
if isinstance(outputs, list):
for i,o in enumerate(outputs):
setattr(m, 'output%(i)i', o)
else:
m.output = outputs
return m
return make
theano/compile/tests/test_inplace_opt_for_value.py deleted 100644 → 0
浏览文件 @ 238b38d1
#!/usr/bin/env python
import numpy as N
import theano
from theano import Op, Apply, tensor as T, Module, Method, Mode, compile
from theano.gof import OpSub, TopoOptimizer
from theano.printing import Print
from theano.tests import unittest_tools
####################
# Library-type stuff
####################
from theano.compile import module
from theano import tensor as T
class StochasticGradientDescent(module.FancyModule):
"""Fixed stepsize gradient descent"""
def __init__(self, args, cost, params, gradients=None, stepsize=None, WEIRD_STUFF=True):
"""
:param stepsize: the step to take in (negative) gradient direction
:type stepsize: None, scalar value, or scalar TensorVariable
"""
super(StochasticGradientDescent, self).__init__()
self.WEIRD_STUFF = WEIRD_STUFF
self.stepsize_init = None
if stepsize is None:
self.stepsize = (T.dscalar())
elif isinstance(stepsize, T.TensorVariable):
self.stepsize = stepsize
else:
if self.WEIRD_STUFF:
#TODO: why is this necessary? why does the else clause not work?
# self.stepsize = module.Member(T.dscalar(), init = stepsize)
self.stepsize = (T.dscalar())
self.stepsize_init = stepsize
else:
# self.stepsize = module.Member(T.value(stepsize))
self.stepsize = (T.constant(stepsize))#work!
if self.stepsize.ndim != 0:
raise ValueError('stepsize must be a scalar', stepsize)
self.params = params
if gradients is None:
self.gparams = T.grad(cost, self.params)
else:
self.gparams = gradients
self.updates = dict((p, p - self.stepsize * g) for p, g in zip(self.params, self.gparams))
self.step = module.Method(
args, [],
updates=self.updates)
self.step_cost = module.Method(
args, cost,
updates=self.updates)
def _instance_initialize(self, obj):
if self.WEIRD_STUFF:
obj.stepsize = self.stepsize_init
else:
pass
def sgd_minimizer(stepsize=None, **args):
def m(i,c,p,g=None):
return StochasticGradientDescent(i, c, p, stepsize=stepsize, **args)
return m
class TanhRnn(Op):
"""
This class implements the recurrent part of a recurrent neural network.
There is not a neat way to include this in a more fine-grained way in Theano at the moment,
so to get something working, I'm implementing a relatively complicated Op that could be
broken down later into constituents.
Anyway, this Op implements recursive computation of the form:
.. latex-eqn:
z_t &= \tanh( z_{t-1} A + x_{t-1})
For z0 a vector, and x a TxM matrix, it returns a matrix z of shape (T+1, M),
in which z[0] = z0.
"""
def __eq__(self, other):
return (type(self) == type(other))
def __hash__(self):
return hash(type(self))
def make_node(self, x, z0, A):
"""
:type x: matrix (each row is an x_t) (shape: (T, M))
:type z0: vector (the first row of output) (shape: M)
:type A: matrix (M by M)
"""
x = T.as_tensor_variable(x)
z0 = T.as_tensor_variable(z0)
A = T.as_tensor_variable(A)
z = x.type() #make a new symbolic variable with the same type as x
return Apply(self, [x, z0, A], [z])
def perform(self, node, inp, out):
x, z0, A = inp
assert x is not None
assert z0 is not None
assert A is not None
T,M = x.shape
z = N.zeros((T+1, M))
z[0] = z0
for i in xrange(T):
z[i+1] = N.tanh(N.dot(z[i], A) + x[i])
out[0][0] = z
def grad(self, inp, grads):
x, z0, A = inp
gz, = grads
z = tanh_rnn(x, z0, A)
gz_incl_rnn, gx = tanh_rnn_grad(A, z, gz)
return [gx, gz_incl_rnn[0], (T.dot(z[:-1].T, gx))]
tanh_rnn = TanhRnn()
class TanhRnnGrad(Op):
"""Gradient calculation for TanhRnn"""
view_map = {0: [2]}
def __init__(self):
pass
def __eq__(self, other):
return (type(self) == type(other))
def __hash__(self):
return hash(type(self))
def make_node(self, A, z, gz):
return Apply(self, [A,z,gz], (z.type(), gz.type()))
def perform(self, node, inp, out):
A, z, gz = inp
Tp1,M = z.shape
T = Tp1 - 1
gx = N.zeros((T, M))
for i in xrange(T-1, -1, -1):
#back through the tanh
gx[i] = gz[i+1] * (1.0 - z[i+1] * z[i+1])
out[0][0] = gz
out[1][0] = gx
def __str__(self):
return super(TanhRnnGrad, self).__str__()
tanh_rnn_grad = TanhRnnGrad()
#######################
# Experiment-type stuff
#######################
class ExampleRNN(Module):
def __init__(self, n_vis, minimizer):
super(ExampleRNN, self).__init__()
self.n_vis = n_vis
#recurrent weight matrix in latent space
self.z0 = (T.dvector())
self.w = (T.dmatrix())
self.params = [self.z0, self.w]
#input and target
x, y = T.dmatrix(), T.dmatrix()
z = tanh_rnn(x, self.z0, self.w)
self.cost = T.sum(z[1:])
# using the make_minimizer protocol
self.minimizer = minimizer([x, y], self.cost, self.params)
def _instance_initialize(self, obj):
#print 'INITIALIZE EXAMPLE RNN'
n_vis = self.n_vis
rng = N.random.RandomState(unittest_tools.fetch_seed(2342))
obj.z0 = N.zeros(n_vis)
obj.w = rng.randn(n_vis, n_vis) * 0.01
obj.minimizer.initialize()
def test_example_rnn():
minimizer_fn = sgd_minimizer(stepsize = 0.001)
n_vis = 5
n_out = 3
n_hid = 4
rnn_module = ExampleRNN(n_vis, minimizer_fn)
rnn = rnn_module.make()
rng = N.random.RandomState(unittest_tools.fetch_seed(7722342))
x = rng.randn(10,n_vis)
y = rng.randn(10,n_out)
#set y to be like x with a lag of LAG
LAG = 4
y[LAG:] = x[:-LAG, 0:n_out]
if 0:
for i, node in enumerate(rnn.minimizer.step_cost.maker.fgraph.toposort()):
print i, node
niter=1500
if theano.config.mode=='DEBUG_MODE':
niter=30
for i in xrange(niter):
rnn.minimizer.step_cost(x, y)
if theano.config.mode=='DEBUG_MODE':
assert rnn.minimizer.step_cost(x,y) < -.9 #it starts around -.28
else:
assert rnn.minimizer.step_cost(x,y) < -20 #it starts around -.28
def test_WEIRD_STUFF():
n_vis = 3
rng = N.random.RandomState(unittest_tools.fetch_seed(7722342))
x = rng.randn(10,n_vis)
y = rng.randn(10,n_vis)
#set y to be like x with a lag of LAG
LAG = 4
y[LAG:] = x[:-LAG, 0:n_vis]
minimizer_fn1 = sgd_minimizer(stepsize = 0.001, WEIRD_STUFF = False)
minimizer_fn2 = sgd_minimizer(stepsize = 0.001, WEIRD_STUFF = True)
rnn_module1 = ExampleRNN(n_vis, minimizer_fn1)
rnn_module2 = ExampleRNN(n_vis, minimizer_fn2)
rnn1 = rnn_module1.make(mode='FAST_RUN')
# rnn2 = rnn_module1.make(mode='FAST_COMPILE')#work
# rnn2 = rnn_module1.make(mode='FAST_RUN')#fail
rnn2 = rnn_module2.make(mode=Mode('c|py', 'fast_run'))#fail
# rnn2 = rnn_module1.make(mode=Mode('c|py', 'fast_run').excluding("inplace"))#work
# rnn2 = rnn_module1.make(mode=Mode('c|py', 'fast_compile'))#work
# rnn2 = rnn_module1.make(mode=Mode('py', 'fast_run_stable'))#work
# rnn2 = rnn_module1.make(mode=Mode('py', 'merge'))#work
# rnn2 = rnn_module1.make(mode=Mode('c|py', 'fast_run').excluding("inplace_opt"))#work
# rnn2 = rnn_module1.make(mode=Mode('py', 'fast_run'))#fail
m = Mode('py', 'fast_run')
# for n in m.optimizer: print n.name
if 0:
topo1=rnn1.minimizer.step_cost.maker.fgraph.toposort()
topo2=rnn2.minimizer.step_cost.maker.fgraph.toposort()
for i in range(len(topo1)):
print '1',i, topo1[i]
print '2',i, topo2[i]
if 0:
topo1=rnn1.minimizer.step.maker.fgraph.toposort()
topo2=rnn2.minimizer.step.maker.fgraph.toposort()
for i in range(len(topo1)):
print '1',i, topo1[i]
print '2',i, topo2[i]
import theano.printing
#print len(rnn1.minimizer.step.maker.inputs)
#print len(rnn2.minimizer.step.maker.inputs)
#print rnn1.minimizer.step.maker.inputs
#print rnn2.minimizer.step.maker.inputs
# for i in range(1,len(rnn1.minimizer.step.maker.inputs)):
# print "valid update:",theano.printing.pp(rnn1.minimizer.step.maker.inputs[i].update),
# print rnn1.minimizer.step.maker.inputs[i].update.name
# print "other update",theano.printing.pp(rnn2.minimizer.step.maker.inputs[i].update),
# print rnn2.minimizer.step.maker.inputs[i].update.name
# print dir(rnn1.minimizer.step.maker.inputs[5].update)
# print dir(rnn2.minimizer.step.maker.inputs[5].update)
niter=3
for i in xrange(niter):
#print rnn1.minimizer.step_cost(x, y)
#print rnn2.minimizer.step_cost(x, y)
# assert rnn1.n_vis != rnn2.n_vis or slef.n_hid != rnn2.n_hid or rnn1.n_out != rnn2.n_out
assert (N.abs(rnn1.z0-rnn2.z0)<1e-8).all()
#print (N.abs(rnn1.w-rnn2.w)<1e-8).all()
#print (N.abs(rnn1.w-rnn2.w))
#print rnn1.w
#print rnn2.w
assert (N.abs(rnn1.w-rnn2.w)<1e-8).all()
# assert b
if __name__ == '__main__':
# from theano.tests import main
# main(__file__)
# test_example_rnn()
test_WEIRD_STUFF()
theano/compile/tests/test_module.py deleted 100644 → 0
浏览文件 @ 238b38d1
#!/usr/bin/env python
"""Test compile.module"""
__docformat__ = "restructuredtext en"
import cPickle, numpy, unittest
from theano import config
from theano.compat import exc_message
from theano.compile.module import *
from theano.compile.function_module import AliasedMemoryError
import theano.tensor as T
import sys, copy
import theano
#TODO: add test for module.make(member=init_value)
class T_module(unittest.TestCase):
def test_empty_module(self):
m = Module()
m.make()
def test_whats_up_with_submembers(self):
class Blah(Module):
def __init__(self, stepsize):
super(Blah, self).__init__()
self.stepsize = T.constant(stepsize)
x = T.dscalar()
self.step = Method([x], x - self.stepsize)
B = Blah(0.0)
b = B.make(mode='FAST_RUN')
assert b.stepsize == 0.0
b.step(1.0)
assert b.stepsize == 0.0
def test_members_in_list_tuple_or_dict(self):
"""Test that a Member which is only included via a list, tuple or dictionary is still treated as if it
were a toplevel attribute and not shared
"""
def local_test(x,y):
m1=Module()
m1.x=x()
m1.y=y()
m1.emtpylist = []
m1.lx=[x()]#cast Variable]
m1.ly=[y()]
m1.llx=[[x()]]#cast Variable]
m1.lly=[[y()]]
m1.ltx=[(x(),)]
m1.lty=[(y(),)]
m1.ldx=[{"x":x()}]
m1.ldy=[{"y":y()}]
m1.tx=(x(),)
m1.ty=(y(),)
m1.tlx=[(x(),)]
m1.tly=[(y(),)]
m1.ttx=((x(),),)
m1.tty=((y(),),)
m1.tdx=({"x":x()},)
m1.tdy=({"y":y()},)
m1.dx={"x":x()}
m1.dy={"y":y()}
m1.dlx={"x":[x()]}
m1.dly={"y":[y()]}
m1.dtx={"x":(x(),)}
m1.dty={"y":(y(),)}
m1.ddx={"x":{"x":x()}}
m1.ddy={"y":{"y":y()}}
assert isinstance(m1.x,(gof.Variable))
assert isinstance(m1.y,(gof.Variable))
for i, obj in enumerate([
m1.lx[0], #0
m1.llx[0][0],
m1.ltx[0][0],
m1.ldx[0]['x'],
m1.lty[0][0],#5
m1.ldy[0]['y'],
m1.ly[0],
m1.lly[0][0],
m1.tx[0], #8
m1.ty[0], m1.tlx[0][0],
m1.tly[0][0], m1.ttx[0][0], m1.tty[0][0], m1.tdx[0]['x'],
m1.tdy[0]['y'], m1.dx['x'],
m1.dy['y'], m1.dlx['x'][0], m1.dly['y'][0],
m1.dtx['x'][0], m1.dty['y'][0], m1.ddx['x']['x'],
m1.ddy['y']['y']]):
assert isinstance(obj,(gof.Variable))
inst=m1.make()
def get_l():
return [inst.lx, inst.ly, inst.tx, inst.ty, inst.dx, inst.dy, inst.llx, inst.lly, inst.ltx, inst.lty, inst.ldx, inst.ldy, inst.tlx, inst.tly, inst.ttx, inst.tty, inst.tdx, inst.tdy, inst.dly, inst.dlx, inst.dty, inst.dtx, inst.ddy, inst.ddx]
def get_l2():
# return [inst.lx[0], inst.ly[0], inst.tx[0], inst.ty[0], inst.dx['x'], inst.dy['y'], inst.llx[0][0], inst.lly[0][0], inst.ltx[0][0], inst.lty[0][0], inst.ldx[0]['x'], inst.ldy[0]['y'], inst.tlx[0][0], inst.tly[0][0], inst.ttx[0][0], inst.tty[0][0], inst.tdx, inst.tdy, inst.dly, inst.dlx, inst.dty, inst.dtx, inst.ddy, inst.ddx]
return [inst.lx, inst.ly, inst.tx, inst.ty, inst.llx[0], inst.lly[0], inst.ltx[0], inst.lty[0], inst.ldx[0], inst.ldy[0], inst.tlx[0], inst.tly[0], inst.ttx[0], inst.tty[0], inst.tdx[0], inst.tdy[0], inst.dly['y'], inst.dlx['x'], inst.dty['y'], inst.dtx['x']]#, inst.ddy['y'], inst.ddx['x']]
#test that we can access the data
inst.x
inst.y
for i in get_l():
assert i
#test that we can set a value to the data the get this value
if not isinstance(m1.x, gof.Constant):
inst.x=-1
inst.y=-2
inst.ldx[0]['x']=-3
inst.ldy[0]['y']=-4
inst.tdx[0]['x']=-5
inst.tdy[0]['y']=-6
inst.ddx['x']['x']=-7
inst.ddy['y']['y']=-8
for i,j in zip(get_l2(),range(len(get_l2()))):
i[0]=j
assert inst.x==-1
assert inst.y==-2
assert inst.ldx[0]['x']==-3
assert inst.ldy[0]['y']==-4
assert inst.tdx[0]['x']==-5
assert inst.tdy[0]['y']==-6
assert inst.ddx['x']['x']==-7
assert inst.ddy['y']['y']==-8
for i,j in zip(get_l2(),range(len(get_l2()))):
assert i[0]==j
local_test(lambda:T.dscalar(),lambda:T.dscalar())
local_test(lambda:T.constant(1),lambda:T.constant(2))
local_test(lambda:T.constant(1),lambda:T.constant(2))
def test_list_assign(self):
"""Test that list members can be assigned list-wise"""
def local_test(x,y):
m1=Module()
#create a list with some variables in it
m1.l=[x(), y()]
# create a Method that makes the second list element a shared Member
m1.f=Method([], m1.l[1])
m1.g=Method([], m1.l[0])
m = m1.make()
#assign 4 and 5 to the two variables' containers in m
m.l = [4, 5]
m.f()
assert numpy.all(5 == m.f())
assert numpy.all(4 == m.g())
local_test(lambda:T.dscalar(),lambda:T.dscalar())
def test_tuple_assign(self):
"""Test that list members can be assigned tuple-wise"""
def local_test(x,y):
m1=Module()
m1.l=(x(), y())
# create a Method that makes the second list element a shared Member
m1.g=Method([], m1.l[0])
m1.f=Method([], m1.l[1])
m = m1.make()
#assign 4 and 5 to the two variables' containers in m
m.l = (4, 5)
assert 5 == m.f()
assert 4 == m.g()
local_test(lambda:T.dscalar(),lambda:T.dscalar())
def test_dict_assign(self):
"""Test that list members can be assigned dict-wise"""
def local_test(x,y):
m1=Module()
##DICT
m1.l={'x':x(), 'y':y()}
# create a Method that makes the second list element a shared Member
m1.f=Method([], m1.l['y'])
m1.g=Method([], m1.l['x'])
m = m1.make()
#assign 4 and 5 to the two variables' containers in m
m.l = dict(x=4, y=5)
assert 5 == m.f()
assert 4 == m.g()
#print 'dscalar test'
local_test(lambda:T.dscalar(),lambda:T.dscalar())
def test_method_in_list_or_dict(self):
"""Test that a Method which is only included via a list or dictionary is still treated as if it
were a toplevel attribute
Fred: why we don't do this of direct fct of variables?
"""
m1=Module()
x=T.dscalar()
m1.x=T.dscalar()
m1.y=Method(x,x*2)
m1.z=Method([],m1.x*2)
m1.ly=[Method(x,x*2)]
m1.lz=[Method([],m1.x*2)]
m1.ty=(Method(x,x*2),)
m1.tz=(Method([],m1.x*2),)
m1.dy={'y':Method(x,x*2)}
m1.dz={'z':Method([],m1.x*2)}
m1.lly=[[Method(x,x*2)]]
m1.llz=[[Method([],m1.x*2)]]
m1.lty=[(Method(x,x*2),)]
m1.ltz=[(Method([],m1.x*2),)]
m1.ldy=[{'y':Method(x,x*2)}]
m1.ldz=[{'z':Method([],m1.x*2)}]
m1.tly=([Method(x,x*2)],)
m1.tlz=([Method([],m1.x*2)],)
m1.tty=((Method(x,x*2),),)
m1.ttz=((Method([],m1.x*2),),)
m1.tdy=({'y':Method(x,x*2)},)
m1.tdz=({'z':Method([],m1.x*2)},)
m1.dly={'y':[Method(x,x*2)]}
m1.dlz={'z':[Method([],m1.x*2)]}
m1.dty={'y':(Method(x,x*2),)}
m1.dtz={'z':(Method([],m1.x*2),)}
m1.ddy={'y':{'y':Method(x,x*2)}}
m1.ddz={'z':{'z':Method([],m1.x*2)}}
inst=m1.make()
inst.x=1
assert inst.y(2)==4
assert inst.z()==2
assert inst.ly[0](2)==4
assert inst.lz[0]()==2
assert inst.ty[0](2)==4
assert inst.tz[0]()==2
assert inst.dy['y'](2)==4
assert inst.dz['z']()==2
for f in inst.lly[0][0], inst.lty[0][0], inst.ldy[0]['y'], inst.tly[0][0], inst.tty[0][0], inst.tdy[0]['y'], inst.dly['y'][0], inst.dty['y'][0], inst.ddy['y']['y']:
assert f(2)==4
for f in inst.llz[0][0], inst.ltz[0][0], inst.ldz[0]['z'], inst.tlz[0][0], inst.ttz[0][0], inst.tdz[0]['z'], inst.dlz['z'][0], inst.dtz['z'][0], inst.ddz['z']['z']:
assert f()==2
assert isinstance(inst.z,theano.compile.function_module.Function)
assert isinstance(inst.y,theano.compile.function_module.Function)
for f in inst.ly,inst.lz,inst.ty,inst.tz:
assert isinstance(f[0],theano.compile.function_module.Function)
for f in inst.lly,inst.llz,inst.lty,inst.ltz,inst.tly,inst.tlz,inst.tty,inst.ttz:
assert isinstance(f[0][0],theano.compile.function_module.Function)
for f in inst.dly['y'][0],inst.dty['y'][0], inst.dlz['z'][0],inst.dtz['z'][0], inst.ddy['y']['y'], inst.ddz['z']['z']:
assert isinstance(f,theano.compile.function_module.Function)
def test_shared_members(self):
"""Test that under a variety of tricky conditions, the shared-ness of Variables and Members
is respected."""
def populate_module(m,x):
m.x=x
m.lx=[x]
m.llx=[[x],[x]]
m.ltx=[(x,)]
m.ldx=[{'x':x}]
m.tx=(x,)
m.tlx=([x],)
m.ttx=((x,),)
m.tdx=({'x':x},)
m.dx={'x':x}
m.dlx={'x':[x]}
m.dtx={'x':(x,)}
m.ddx={'x':{'x':x}}
def get_element(i):
return [i.x,i.lx[0],i.tx[0],i.dx['x'],i.llx[0][0], i.llx[1][0], i.ltx[0][0], i.ldx[0]['x'], i.tlx[0][0], i.tlx[0][0], i.tdx[0]['x'], i.dlx['x'][0], i.dtx['x'][0], i.ddx['x']['x']]
m1=Module()
m2=Module()
x=T.dscalar()
populate_module(m1,x)
populate_module(m2,x)
#m1.x and m2.x should not be shared as their is no hierarchi link between them.
inst1=m1.make()
inst2=m2.make()
m1.m2=m2
#m1.x and m2.x should be shared as their is a hierarchi link between them.
inst3=m1.make()
inst1.x=1
inst2.x=2
inst3.x=3
for f in get_element(inst1):
assert f==1
for f in get_element(inst2):
assert f==2
for f in get_element(inst3)+get_element(inst3.m2):
assert f==3
inst3.m2.x=4
for f in get_element(inst3)+get_element(inst3.m2):
assert f==4
def test_shared_members_N(self):
"""Test that Members can be shared an arbitrary number of times between
many submodules and internal data structures."""
def populate_module(m,x):
m.x=x
m.lx=[x]
m.llx=[[x],[x]]
m.ltx=[(x,)]
m.ldx=[{'x':x}]
m.tx=(x,)
m.tlx=([x],)
m.ttx=((x,),)
m.tdx=({'x':x},)
m.dx={'x':x}
m.dlx={'x':[x]}
m.dtx={'x':(x,)}
m.ddx={'x':{'x':x}}
def get_element(i):
return [i.x,i.lx[0],i.tx[0],i.dx['x'],i.llx[0][0], i.llx[1][0], i.ltx[0][0], i.ldx[0]['x'], i.tlx[0][0], i.tlx[0][0], i.tdx[0]['x'], i.dlx['x'][0], i.dtx['x'][0], i.ddx['x']['x']]
m1=Module()
m2=Module()
m3=Module()
m4=Module()
x=T.dscalar()
populate_module(m1,x)
populate_module(m2,(x))
populate_module(m4,(x))
#m1.x and m2.x should not be shared as their is no hierarchi link between them.
inst1=m1.make()
inst2=m2.make()
m1.m2=m2
m2.m3=m3
m3.m4=m4
#m1.x and m2.x should be shared as their is a hierarchi link between them.
inst3=m1.make()
inst1.x=1
inst2.x=2
inst3.x=3
for f in get_element(inst1):
assert f==1
for f in get_element(inst2):
assert f==2
for f in get_element(inst3)+get_element(inst3.m2)+get_element(inst3.m2.m3.m4):
assert f==3
inst3.m2.x=4
for f in get_element(inst3)+get_element(inst3.m2)+get_element(inst3.m2.m3.m4):
assert f==4
def test_shared_method(self):
"""Test that under a variety of tricky conditions, the shared-ness of Variables and Methods
is respected.
Fred: the test create different method event if they are shared. What do we want?
"""
m1=Module()
m1.x=T.dscalar()
x=T.dscalar()
fy=Method(x,x*2)
fz=Method([],m1.x*2)
m1.y=fy
m1.z=fz
m1.ly=[fy]
m1.lz=[fz]
m1.lly=[[fy]]
m1.llz=[[fz]]
m1.ty=(fy,)
m1.tz=(fz,)
m1.tty=((fy,),)
m1.ttz=((fz,),)
m1.dy={'y':fy}
m1.dz={'z':fz}
inst=m1.make()
inst.x=1
assert inst.y(2)==4
assert inst.z()==2
assert inst.ly[0](2)==4
assert inst.lz[0]()==2
assert inst.ty[0](2)==4
assert inst.tz[0]()==2
assert inst.dy['y'](2)==4
assert inst.dz['z']()==2
assert inst.lly[0][0](2)==4
assert inst.llz[0][0]()==2
assert inst.tty[0][0](2)==4
assert inst.ttz[0][0]()==2
assert isinstance(inst.z,theano.compile.function_module.Function)
assert isinstance(inst.lz[0],theano.compile.function_module.Function)
assert isinstance(inst.llz[0][0],theano.compile.function_module.Function)
assert isinstance(inst.tz[0],theano.compile.function_module.Function)
assert isinstance(inst.dz['z'],theano.compile.function_module.Function)
assert isinstance(inst.ttz[0][0],theano.compile.function_module.Function)
assert isinstance(inst.y,theano.compile.function_module.Function)
assert isinstance(inst.ly[0],theano.compile.function_module.Function)
assert isinstance(inst.lly[0][0],theano.compile.function_module.Function)
assert isinstance(inst.ty[0],theano.compile.function_module.Function)
assert isinstance(inst.dy['y'],theano.compile.function_module.Function)
assert isinstance(inst.tty[0][0],theano.compile.function_module.Function)
assert m1.y is m1.ly[0]
assert inst.y is inst.ly[0]
assert inst.y is inst.lly[0][0]
assert inst.y is inst.ty[0]
assert inst.y is inst.tty[0][0]
assert inst.y is inst.dy['y']
def test_member_method_inputs(self):
"""Test that module Members can be named as Method inputs, in which case the function will
*not* use the storage allocated for the Module's version of that Member.
"""
# test that explicit Method inputs don't use shared storage
M = Module()
M.x = T.dscalar()
M.y = T.dscalar()
M.f = Method([M.x], M.x + M.y)
M.g = Method([M.y], M.x - M.y)
m = M.make()
m.y = 77
assert m.f(23) == 100
assert m.x is None
m.x = 1000
assert m.g(23) == 977
assert m.y == 77
assert m.x == 1000
def test_member_input_flags(self):
"""Test that we can manipulate the mutable, strict, etc. flags (see SymbolicInput) of
Method inputs"""
if config.mode == 'FAST_COMPILE':
return
M = Module()
M.x = T.dvector()
M.y = T.dvector()
xval= numpy.asarray([0, 0.5])
M.f = Method([io.In(M.x,
mutable=True,
update=(M.x - M.y),
value=xval)], M.x + M.y)
m = M.make()
m.y = numpy.asarray([1, 2])
assert numpy.all(m.f(xval) == [1, 2.5])
assert numpy.all(xval == [-1, -1.5])
def test_member_constant(self):
"""Test that module Members of Constant work correctly.
As Variable with more optimization?"""
M = Module()
x = T.dscalar()
M.y = T.constant(40)
M.f = Method([x], x + 2 * M.y)
m = M.make()
try:
m.y = 77 #fail?
except Exception:
pass
assert m.y == 40
assert m.f(20) == 100
def test_raise_NotImplemented(self):
c=Component()
self.assertRaises(NotImplementedError, c.allocate,"")
self.assertRaises(NotImplementedError, c.build,"","")
self.assertRaises(NotImplementedError, c.pretty)
c=Composite()
self.assertRaises(NotImplementedError, c.components)
self.assertRaises(NotImplementedError, c.components_map)
self.assertRaises(NotImplementedError, c.get,"n")
self.assertRaises(NotImplementedError, c.set,"n",1)
def test_wrappable_as_tensor(self):
M = Module()
M.a = [1,2,3]
M.make()
m = M.make()
#print m.a
#print m.a[0], type(m.a[0]), m.a[0] == 1
#print list(m.a)
assert list(m.a) == [1,2,3]
assert m.a is not M.a
try:
m.a = [4, 5, 6]
assert False
except Exception, e:
if exc_message(e).startswith("Cannot set readonly"):
pass
else:
raise
try:
m.a[0] = 4
assert False
except Exception, e:
if exc_message(e).startswith("Cannot set readonly"):
pass
else:
raise
def test_mixed_list(self):
M = Module()
M.a = [1,2,T.lscalar()]
m = M.make()
assert list(m.a) == [1,2,None]
assert m.a is not M.a
try:
m.a[0] = 4
assert False
except Exception, e:
if exc_message(e).startswith("Cannot set readonly"):
pass
else:
raise
m.a[2] = 3
assert list(m.a) == [1,2,3]
def test_multiple_references():
class A(theano.Module):
def __init__(self, sub_module):
super(A, self).__init__()
self.sub_module = sub_module
def _instance_initialize(self, obj):
pass
#print 'Initializing A'
class B(theano.Module):
def __init__(self, sub_module):
super(B, self).__init__()
self.sub_module = sub_module
def _instance_initialize(self, obj):
pass
#print 'Initializing B'
class C(theano.Module):
def __init__(self):
super(C, self).__init__()
self.value = theano.tensor.scalar()
def _instance_initialize(self, obj):
#print 'Initializing C'
obj.value = 0
def _instance_set(self, obj, value):
#print 'Setting C'
obj.value = value
class D(theano.Module):
def __init__(self):
super(D, self).__init__()
self.c = C()
self.a = A(self.c)
self.b = B(self.c)
# Workaround for bug exhibited in a previous email.
self.bug = theano.tensor.scalar()
def _instance_initialize(self, obj):
#print 'Initializing D'
obj.c.set(1)
d = D()
d_instance = d.make(mode = 'FAST_COMPILE')
assert d_instance.c.value == 1
assert d_instance.a.sub_module.value == 1
assert d_instance.b.sub_module.value == 1
def test_tuple_members():
M = Module()
M.a = (1,1)
assert isinstance(M.a, tuple)
class Temp(Module):
def __init__(self):
super(Temp, self).__init__()
self.a = (1,1)
M = Temp()
assert isinstance(M.a, tuple)
def test_method_updates():
# updates work
M = Module()
M.x = T.dvector()
x = T.dvector()
xval= numpy.asarray([0, 0.5])
M.f = Method([x], M.x*4, updates={M.x:M.x * 2}, mode='FAST_COMPILE')
m = M.make(mode='FAST_RUN')
m.x = xval
m.f([9,9])
assert numpy.all(m.x == [0, 1])
assert numpy.all(xval == [0, 0.5])
# In(update) works
M = Module()
M.x = T.dvector()
x = T.dvector()
M.f = Method([x, io.In(M.x, value=xval, update=M.x*2)], M.x*4)
m = M.make()
m.f([9,9])
assert m.x is None
assert numpy.all(m.f[M.x] == [0, 1])
# when a variable is listed explicitly and in an update, then there's a problem.
M = Module()
M.x = T.dvector()
x = T.dvector()
M.f = Method([x, io.In(M.x, value=xval, update=M.x*2)], M.x*4,
updates={M.x:M.x * 7})
try:
m = M.make()
assert False
except ValueError, e:
if str(exc_message(e)).startswith('Variable listed in both inputs and up'):
pass
else:
raise
def test_method_mode():
"""Test that Methods can override the module build mode"""
M = Module()
M.x = T.dvector()
M.f = Method([M.x], M.x*4, mode='FAST_COMPILE')
M.g = Method([M.x], M.x*4)
M.h = Method([M.x], M.x*4)
m = M.make(mode='FAST_RUN')
assert m.f.maker.mode != m.g.maker.mode
assert m.h.maker.mode == m.g.maker.mode
assert numpy.all(m.f([1,2]) == m.g([1,2]))
def test_method_implicit_ticket_384():
"""
Ensure it is not possible to accidentally overwrite module variables
added as implicit inputs.
"""
M = Module()
M.x = T.scalar()
M.f = Method([M.x], M.x * 3)
m = M.make()
m.f(0)
try:
m.f(0, 0)
assert False
except TypeError, e:
if not str(e).startswith('Tried to provide value for implicit input'):
raise
def get_mode():
if config.mode not in ['DEBUG_MODE', 'DebugMode']:
mode = config.mode
else: mode = 'FAST_RUN'
return mode
def test_pickle():
"""Test that a module can be pickled"""
M = Module()
M.x = (T.dmatrix())
M.y = (T.dmatrix())
a = T.dmatrix()
M.f = Method([a], a + M.x + M.y)
M.g = Method([a], a * M.x * M.y)
mode = get_mode()
m = M.make(x=numpy.zeros((4,5)), y=numpy.ones((2,3)), mode=mode)
m_dup = cPickle.loads(cPickle.dumps(m, protocol=-1))
assert numpy.all(m.x == m_dup.x) and numpy.all(m.y == m_dup.y)
m_dup.x[0,0] = 3.142
assert m_dup.f.input_storage[1].data[0,0] == 3.142
assert m.x[0,0] == 0.0 #ensure that m is not aliased to m_dup
#check that the unpickled version has the same argument/property aliasing
assert m_dup.x is m_dup.f.input_storage[1].data
assert m_dup.y is m_dup.f.input_storage[2].data
assert m_dup.x is m_dup.g.input_storage[1].data
assert m_dup.y is m_dup.g.input_storage[2].data
def test_pickle_aliased_memory():
M = Module()
M.x = (T.dmatrix())
M.y = (T.dmatrix())
a = T.dmatrix()
M.f = Method([a], a + M.x + M.y)
M.g = Method([a], a * M.x * M.y)
mode = get_mode()
m = M.make(x=numpy.zeros((4,5)), y=numpy.ones((2,3)), mode=mode)
m.y = m.x[:]
#m's x and y memory is aliased....
m.x[0,0] = 3.14
assert m.y[0,0] == 3.14
import logging
from theano.compat.six import StringIO
sio = StringIO()
handler = logging.StreamHandler(sio)
logging.getLogger('theano.compile.function_module').addHandler(handler)
# Silence original handler when intentionnally generating warning messages
logging.getLogger('theano').removeHandler(theano.logging_default_handler)
try:
m.f.pickle_aliased_memory_strategy = 'warn'
m.g.pickle_aliased_memory_strategy = 'warn'
m_dup = cPickle.loads(cPickle.dumps(m, protocol=-1))
assert sio.getvalue().startswith('aliased relat')
finally:
logging.getLogger('theano.compile.function_module').removeHandler(handler)
logging.getLogger('theano').addHandler(theano.logging_default_handler)
try:
m.f.pickle_aliased_memory_strategy = 'raise'
m.g.pickle_aliased_memory_strategy = 'raise'
m_dup = cPickle.loads(cPickle.dumps(m, protocol=-1))
except AliasedMemoryError, e:
return
assert 0 #should have failed to pickle
#is m_dup's memory aliased?
m_dup.x[0,0] = 3.14
assert m_dup.y[0,0] == 3.14
#m's memory is aliased differently....
m.y = m.x[1:2]
m_dup = cPickle.loads(cPickle.dumps(m, protocol=-1))
if 0:
#is m_dup's memory aliased the same way?
m.x[1,0] = 3.142
assert m.y[0,0] == 3.142
m_dup.x[1,0] = 3.142
assert m_dup.y[0,0] == 3.142
def test_default_instance_initialize():
"""
Testing the default _instance_initialize provided by module.
"""
class M1(Module):
def __init__(self):
super(M1, self).__init__()
self.a = T.dscalar()
self.b = T.lscalar()
self.c = T.lvector()
class M2(Module):
def __init__(self):
super(M2, self).__init__()
self.a = T.lscalar()
self.x = M1()
self.y = self.x
self.z = M1()
m = M2().make(a = 13,
x = dict(a = 1, b = 2, c = [3, 4]),
z = dict(a = 5, b = 6, c = [7, 8]))
assert m.a == 13
assert m.x.a == 1
assert m.x.b == 2
assert all(m.x.c == [3, 4])
assert m.y.a == 1
assert m.y.b == 2
assert all(m.y.c == [3, 4])
assert m.z.a == 5
assert m.z.b == 6
assert all(m.z.c == [7, 8])
class MyModule(Module):
def __init__(self):
Module.__init__(self)
self.a = T.dscalar()
self.b = T.dscalar()
def _instance_initialize(self, obj):
obj.a = 4.5
obj.b = 3.5
def _instance_something(self, obj, a):
return obj.a + a
def test_pickle_module():
M = MyModule()
m = M.make()
mm = copy.deepcopy(m)
assert m.a == mm.a
assert m.b == mm.b
if __name__ == '__main__':
from theano.tests import main
# main(__file__[:-3])
main("test_module")
# t=T_test_module()
# t.test_shared_members()
# tests = unittest.TestLoader().loadTestsFromModule("T_test_module")
# tests.debug()
theano/tensor/basic.py
浏览文件 @ dcbe40c8
......@@ -443,12 +443,6 @@ def _obj_is_wrappable_as_tensor(x):
return False
def _wrap_tensor_into_member(x):
return compile.module.Member(constant(x))
compile.module.register_wrapper(_obj_is_wrappable_as_tensor,
_wrap_tensor_into_member, no_warn=True)
if int(config.tensor.cmp_sloppy) > 1:
# This config variable is a quick-and-dirty way to get low-precision
# comparisons. For a more precise setting of these tolerances set
......
theano/tensor/tests/test_naacl09.py deleted 100644 → 0
浏览文件 @ 238b38d1
#CUT-and-PASTE from pylearn.algorithms.daa
import theano
from theano import tensor as T
from theano.tensor import nnet as NN
from theano.compile import module
from theano.compile.mode import get_default_mode
from theano import config
from theano import tensor as T, sparse as S
import numpy as N
import sys
from theano.tests import unittest_tools
from numpy.testing.noseclasses import KnownFailureTest
from nose.plugins.attrib import attr
def cross_entropy(target, output, axis=1):
"""
@todo: This is essentially duplicated as tensor.nnet.binary_crossentropy
@warning: OUTPUT and TARGET are reversed in tensor.nnet.binary_crossentropy
"""
return -T.mean(target * T.log(output) + (1 - target) * T.log(1 - output), axis=axis)
def quadratic(target, output, axis=1):
return T.mean(T.sqr(target - output), axis=axis)
class QuadraticDenoisingAA(module.Module):
"""Quadratic de-noising Auto-encoder
WRITEME
Abstract base class. Requires subclass with functions:
- build_corrupted_input()
Introductory article about this model WRITEME.
"""
def __init__(self,
input=None,
# regularize = False,
tie_weights=False,
n_quadratic_filters=1,
_w1=None,
_w2=None,
_b1=None,
_b2=None,
_qfilters=None,
activation_function=NN.sigmoid,
reconstruction_cost_function=cross_entropy):
"""
:param input: WRITEME
:param regularize: WRITEME
:param tie_weights: WRITEME
:param activation_function: WRITEME
:param reconstruction_cost: Should return one cost per example (row)
:todo: Default noise level for all daa levels
"""
super(QuadraticDenoisingAA, self).__init__()
self.random = T.randomstreams.RandomStreams()
# MODEL CONFIGURATION
# self.regularize = regularize
self.tie_weights = tie_weights
self.activation_function = activation_function
self.reconstruction_cost_function = reconstruction_cost_function
# ACQUIRE/MAKE INPUT
if not input:
input = T.matrix('input')
#self.input = theano.External(input)
self.input = (input)
# HYPER-PARAMETERS
#self.lr = theano.Member(T.scalar())
self.lr = (T.scalar())
# PARAMETERS
if _qfilters is None:
#self.qfilters = [theano.Member(T.dmatrix('q%i'%i)) for i in xrange(n_quadratic_filters)]
self.qfilters = [(T.dmatrix('q%i' % i))
for i in xrange(n_quadratic_filters)]
else:
#self.qfilters = [theano.Member(q) for q in _qfilters]
self.qfilters = [(q) for q in _qfilters]
#self.w1 = theano.Member(T.matrix('w1')) if _w1 is None else theano.Member(_w1)
if _w1 is None:
self.w1 = (T.matrix('w1'))
else:
self.w1 = (_w1)
if _w2 is None:
if not tie_weights:
#self.w2 = theano.Member(T.matrix())
self.w2 = (T.matrix())
else:
self.w2 = self.w1.T
else:
#self.w2 = theano.Member(_w2)
self.w2 = (_w2)
#self.b1 = theano.Member(T.vector('b1')) if _b1 is None else theano.Member(_b1)
if _b1 is None:
self.b1 = (T.vector('b1'))
else:
self.b1 = (_b1)
#self.b2 = theano.Member(T.vector('b2')) if _b2 is None else theano.Member(_b2)
if _b2 is None:
self.b2 = (T.vector('b2'))
else:
self.b2 = (_b2)
# # REGULARIZATION COST
# self.regularization = self.build_regularization()
### NOISELESS ###
# HIDDEN LAYER
def _act(x):
if len(self.qfilters) > 0:
qsum = 10e-10 # helps to control the gradient in the square-root below
for qf in self.qfilters:
qsum = qsum + T.dot(x, qf) ** 2
return T.dot(x, self.w1) + self.b1 + T.sqrt(qsum)
else:
return T.dot(x, self.w1) + self.b1
self.hidden_activation = _act(self.input) # noise-free hidden
self.hidden = self.hid_activation_function(self.hidden_activation)
# RECONSTRUCTION LAYER
self.output_activation = T.dot(self.hidden, self.w2) + self.b2
self.output = self.out_activation_function(self.output_activation)
# RECONSTRUCTION COST
self.reconstruction_costs = self.build_reconstruction_costs(self.output)
self.reconstruction_cost = T.mean(self.reconstruction_costs)
# TOTAL COST
self.cost = self.reconstruction_cost
# if self.regularize:
# self.cost = self.cost + self.regularization
### WITH NOISE ###
self.corrupted_input = self.build_corrupted_input()
# HIDDEN LAYER
self.nhidden_activation = _act(self.corrupted_input)
self.nhidden = self.hid_activation_function(self.nhidden_activation)
# RECONSTRUCTION LAYER
self.noutput_activation = T.dot(self.nhidden, self.w2) + self.b2
self.noutput = self.out_activation_function(self.noutput_activation)
# RECONSTRUCTION COST
self.nreconstruction_costs = self.build_reconstruction_costs(self.noutput)
self.nreconstruction_cost = T.mean(self.nreconstruction_costs)
# TOTAL COST
self.ncost = self.nreconstruction_cost
# if self.regularize:
# self.ncost = self.ncost + self.regularization
# GRADIENTS AND UPDATES
if self.tie_weights:
self.params = [self.w1, self.b1, self.b2] + self.qfilters
else:
self.params = [self.w1, self.w2, self.b1, self.b2] + self.qfilters
gradients = T.grad(self.ncost, self.params)
updates = dict((p, p - self.lr * g) for p, g in zip(self.
params, gradients))
# INTERFACE METHODS
#self.update = theano.Method(self.input, self.ncost, updates)
#self.compute_cost = theano.Method(self.input, self.cost)
#self.noisify = theano.Method(self.input, self.corrupted_input)
#self.reconstruction = theano.Method(self.input, self.output)
#self.representation = theano.Method(self.input, self.hidden)
#self.reconstruction_through_noise = theano.Method(self.input, [self.corrupted_input, self.noutput])
#self.validate = theano.Method(self.input, [self.cost, self.output])
def _instance_initialize(self, obj, input_size, hidden_size, seed, lr, qfilter_relscale):
#print 'QDAA init'
"""
qfilter_relscale is the initial range for any quadratic filters (relative to the linear
filter's initial range)
"""
if (input_size is None) ^ (hidden_size is None):
raise ValueError(
"Must specify input_size and hidden_size or neither.")
super(QuadraticDenoisingAA, self)._instance_initialize(obj, {})
obj.random.initialize()
R = N.random.RandomState(unittest_tools.fetch_seed(seed))
if input_size is not None:
sz = (input_size, hidden_size)
inf = 1 / N.sqrt(input_size)
hif = 1 / N.sqrt(hidden_size)
obj.w1 = N.asarray(R.uniform(size=sz, low=-inf, high=inf),
dtype=config.floatX)
if not self.tie_weights:
obj.w2 = N.asarray(
R.uniform(size=list(reversed(sz)), low=-hif, high=hif),
dtype=config.floatX)
obj.b1 = N.zeros(hidden_size, dtype=config.floatX)
obj.b2 = N.zeros(input_size, dtype=config.floatX)
obj.qfilters = [R.uniform(size = sz, low = -inf, high = inf) * qfilter_relscale \
for qf in self.qfilters]
if seed is not None:
obj.random.seed(seed)
obj.lr = N.asarray(lr, dtype=config.floatX)
obj.__hide__ = ['params']
# def build_regularization(self):
# """
# @todo: Why do we need this function?
# """
# return T.zero() # no regularization!
class SigmoidXEQuadraticDenoisingAA(QuadraticDenoisingAA):
"""
@todo: Merge this into the above.
@todo: Default noise level for all daa levels
"""
def setUp(self):
unittest_tools.seed_rng()
def build_corrupted_input(self):
#self.noise_level = theano.Member(T.scalar())
self.noise_level = (T.scalar())
return self.random.binomial(T.shape(self.input), 1, 1 - self.noise_level) * self.input
def hid_activation_function(self, activation):
return self.activation_function(activation)
def out_activation_function(self, activation):
return self.activation_function(activation)
def build_reconstruction_costs(self, output):
return self.reconstruction_cost_function(self.input, output)
# def build_regularization(self):
# self.l2_coef = theano.Member(T.scalar())
# if self.tie_weights:
# return self.l2_coef * T.sum(self.w1 * self.w1)
# else:
# return self.l2_coef * (T.sum(self.w1 * self.w1) + T.sum(self.w2 * self.w2))
def _instance_initialize(self, obj, input_size, hidden_size, noise_level, seed, lr, qfilter_relscale):
# obj.l2_coef = 0.0
obj.noise_level = N.asarray(noise_level, dtype=config.floatX)
super(SigmoidXEQuadraticDenoisingAA, self)._instance_initialize(
obj, input_size, hidden_size, seed, lr, qfilter_relscale)
QDAA = SigmoidXEQuadraticDenoisingAA
class Loss01(object):
def loss_01(self, x, targ):
return N.mean(self.classify(x) != targ)
class Module_Nclass(module.FancyModule):
def _instance_initialize(mod_self, self, n_in, n_out, lr, seed):
#self.component is the LogisticRegressionTemplate instance that built this guy.
"""
@todo: Remove seed. Used only to keep Stacker happy.
"""
self.w = N.zeros((n_in, n_out))
self.b = N.zeros(n_out)
self.lr = lr
self.__hide__ = ['params']
self.input_dimension = n_in
self.output_dimension = n_out
def __init__(self, x=None, targ=None, w=None, b=None, lr=None, regularize=False):
super(Module_Nclass, self).__init__() # boilerplate
#self.x = module.Member(x) if x is not None else T.matrix('input')
if x is not None:
self.x = (x)
else:
self.x = T.matrix('input')
#self.targ = module.Member(targ) if targ is not None else T.lvector()
if targ is not None:
self.targ = (targ)
else:
self.targ = T.lvector()
#self.w = module.Member(w) if w is not None else module.Member(T.dmatrix())
if w is not None:
self.w = (w)
else:
self.w = (T.dmatrix())
#self.b = module.Member(b) if b is not None else module.Member(T.dvector())
if b is not None:
self.b = (b)
else:
self.b = (T.dvector())
#self.lr = module.Member(lr) if lr is not None else module.Member(T.dscalar())
if lr is not None:
self.lr = (lr)
else:
self.lr = (T.dscalar())
self.params = [p for p in [self.w, self.b] if p.owner is None]
linear_output = T.dot(self.x, self.w) + self.b
(xent, softmax, max_pr, argmax) = NN.crossentropy_softmax_max_and_argmax_1hot(
linear_output, self.targ)
sum_xent = T.sum(xent)
self.softmax = softmax
self.argmax = argmax
self.max_pr = max_pr
self.sum_xent = sum_xent
# Softmax being computed directly.
softmax_unsupervised = NN.softmax(linear_output)
self.softmax_unsupervised = softmax_unsupervised
#compatibility with current implementation of stacker/daa or something
#TODO: remove this, make a wrapper
self.cost = self.sum_xent
self.input = self.x
# TODO: I want to make output = linear_output.
self.output = self.softmax_unsupervised
#define the apply method
self.pred = T.argmax(linear_output, axis=1)
#self.apply = module.Method([self.input], self.pred)
#self.validate = module.Method([self.input, self.targ], [self.cost, self.argmax, self.max_pr])
#self.softmax_output = module.Method([self.input], self.softmax_unsupervised)
if self.params:
gparams = T.grad(sum_xent, self.params)
#self.update = module.Method([self.input, self.targ], sum_xent,
#updates = dict((p, p - self.lr * g) for p, g in zip(self.params, gparams)))
class ConvolutionalMLP(module.FancyModule):
def __init__(self,
window_size,
n_quadratic_filters,
activation_function,
reconstruction_cost_function,
tie_weights=False,
# _input,
# _targ
):
super(ConvolutionalMLP, self).__init__()
#self.lr = module.Member(T.scalar())
self.lr = (T.scalar())
self.inputs = [T.dmatrix() for i in range(window_size)]
self.targ = T.lvector()
self.input_representations = []
self.input_representations.append(QDAA(
input=self.inputs[0],
tie_weights=tie_weights,
n_quadratic_filters=n_quadratic_filters,
activation_function=activation_function,
reconstruction_cost_function = reconstruction_cost_function
)
)
for i in self.inputs[1:]:
self.input_representations.append(
QDAA(
input=i,
tie_weights=tie_weights,
n_quadratic_filters=n_quadratic_filters,
activation_function=activation_function,
reconstruction_cost_function = reconstruction_cost_function,
_w1 = self.input_representations[0].w1,
_w2 = self.input_representations[0].w2,
_b1 = self.input_representations[0].b1,
_b2 = self.input_representations[0].b2,
_qfilters = self.input_representations[0].qfilters
)
)
assert self.input_representations[-1].w1 is \
self.input_representations[0].w1
self.input_representation = T.concatenate([i.
hidden for i in self.input_representations], axis=1)
self.hidden = QDAA(
input=self.input_representation,
tie_weights=tie_weights,
n_quadratic_filters=n_quadratic_filters,
activation_function=activation_function,
reconstruction_cost_function = reconstruction_cost_function
)
self.output = Module_Nclass(x=self.hidden.hidden, targ=self.targ)
input_pretraining_params = [
self.input_representations[0].w1,
self.input_representations[0].w2,
self.input_representations[0].b1,
self.input_representations[0].b2
] + self.input_representations[0].qfilters
hidden_pretraining_params = [
self.hidden.w1,
self.hidden.w2,
self.hidden.b1,
self.hidden.b2
] + self.hidden.qfilters
input_pretraining_cost = sum(i.ncost for i in self.
input_representations)
hidden_pretraining_cost = self.hidden.ncost
input_pretraining_gradients = T.grad(input_pretraining_cost,
input_pretraining_params)
hidden_pretraining_gradients = T.grad(
hidden_pretraining_cost, hidden_pretraining_params)
pretraining_updates = \
dict((p, p - self.lr * g) for p, g in \
zip(input_pretraining_params, input_pretraining_gradients) \
+ zip(hidden_pretraining_params, hidden_pretraining_gradients))
self.pretraining_update = module.Method(self.inputs,
[input_pretraining_cost, hidden_pretraining_cost],
pretraining_updates)
finetuning_params = \
[self.input_representations[0].w1, self.input_representations[0].b1] + self.input_representations[0].qfilters + \
[self.hidden.w1, self.hidden.b1] + self.hidden.qfilters + \
[self.output.w, self.output.b]
finetuning_cost = self.output.cost
finetuning_gradients = T.grad(finetuning_cost, finetuning_params)
finetuning_updates = dict((p, p - self.lr * g) for p,
g in zip(finetuning_params, finetuning_gradients))
self.finetuning_update = module.Method(self.inputs + [self.
targ], self.output.cost, finetuning_updates)
#self.validate = module.Method(self.inputs + [self.targ], [self.output.cost, self.output.argmax, self.output.max_pr])
#self.softmax_output = module.Method(self.inputs, self.output.softmax_unsupervised)
def _instance_initialize(mod_self, self, input_size, input_representation_size, hidden_representation_size, output_size, lr, seed, noise_level, qfilter_relscale):
R = N.random.RandomState(unittest_tools.fetch_seed(seed))
self.input_size = input_size
self.input_representation_size = input_representation_size
self.hidden_representation_size = hidden_representation_size
self.output_size = output_size
self.lr = N.asarray(lr, dtype=config.floatX)
# for layer in obj.layers:
# if layer.lr is None:
# layer.lr = lr
assert self.input_representations[-1] \
is not self.input_representations[0]
assert self.input_representations[-1].w1 is\
self.input_representations[0].w1
for i in self.input_representations:
# i.initialize(input_size=self.input_size, hidden_size=self.input_representation_size, seed=R.random_integers(2**30), noise_level=noise_level, qfilter_relscale=qfilter_relscale)
i.initialize(input_size=self.input_size,
hidden_size=self.input_representation_size, noise_level=noise_level,
seed=int(R.random_integers(2**30)), lr=lr, qfilter_relscale=qfilter_relscale)
#print type(i.w1)
assert isinstance(i.w1, N.ndarray)
for i in self.input_representations[1:]:
#print type(i.w1)
assert isinstance(i.w1, N.ndarray)
assert (i.w1 == self.input_representations[0].w1).all()
assert (i.w2 == self.input_representations[0].w2).all()
assert (i.b1 == self.input_representations[0].b1).all()
assert (i.b2 == self.input_representations[0].b2).all()
assert N.all((a == b).all() for a, b in zip(i.
qfilters, self.input_representations[0].qfilters))
self.hidden.initialize(input_size=(len(self.inputs) * self.input_representation_size),
hidden_size=self.hidden_representation_size, noise_level=noise_level,
seed=int(R.random_integers(2**30)), lr=lr, qfilter_relscale=qfilter_relscale)
self.output.initialize(n_in=self.
hidden_representation_size, n_out=self.output_size, lr=lr, seed=R.random_integers(2**30))
def create(window_size=3,
input_dimension=9,
output_vocabsize=8,
n_quadratic_filters=2,
token_representation_size=5,
concatenated_representation_size=7,
lr=0.01,
seed=123,
noise_level=0.2,
qfilter_relscale=0.1,
compile_mode=None):
""" Create a convolutional model. """
activation_function = T.tanh
architecture = ConvolutionalMLP( \
window_size=window_size,
n_quadratic_filters=n_quadratic_filters,
activation_function=activation_function,
reconstruction_cost_function=quadratic,
tie_weights=False
)
backup = config.warn.sum_div_dimshuffle_bug
config.warn.sum_div_dimshuffle_bug = False
try:
model = architecture.make(input_size=input_dimension,
input_representation_size=token_representation_size, hidden_representation_size=concatenated_representation_size, output_size=output_vocabsize, lr=lr, seed=seed, noise_level=noise_level, qfilter_relscale=qfilter_relscale, mode=compile_mode)
finally:
config.warn.sum_div_dimshuffle_bug = backup
return model
def create_realistic(window_size=3, # 7,
input_dimension=200,
output_vocabsize=23,
n_quadratic_filters=2,
token_representation_size=150,
concatenated_representation_size=400,
lr=0.001,
seed=123,
noise_level=0.2,
qfilter_relscale=0.1,
compile_mode=None):
""" Create a convolutional model. """
activation_function = T.tanh
architecture = ConvolutionalMLP( \
window_size=window_size,
n_quadratic_filters=n_quadratic_filters,
activation_function=activation_function,
reconstruction_cost_function=quadratic,
tie_weights=False
)
model = architecture.make(input_size=input_dimension,
input_representation_size=token_representation_size, hidden_representation_size=concatenated_representation_size, output_size=output_vocabsize, lr=lr, seed=seed, noise_level=noise_level, qfilter_relscale=qfilter_relscale, mode=compile_mode)
return model
@attr('slow')
def test_naacl_model(iters_per_unsup=3, iters_per_sup=3,
optimizer=None, realistic=False):
#print "BUILDING MODEL"
import time
t = time.time()
if optimizer:
mode = theano.Mode(linker='c|py', optimizer=optimizer)
else:
mode = get_default_mode()
if mode.__class__.__name__ == 'DebugMode':
iters_per_unsup = 1
iters_per_sup = 1
if realistic:
m = create_realistic(compile_mode=mode)
else:
m = create(compile_mode=mode)
#print 'BUILD took %.3fs'%(time.time() - t)
prog_str = []
idx_of_node = {}
for i, node in enumerate(m.pretraining_update.maker.fgraph.toposort()):
idx_of_node[node] = i
if False and i > -1:
print ' ', i, node, [(ii, idx_of_node.get(ii.
owner, 'IN')) for ii in node.inputs]
prog_str.append(str(node))
#print input_pretraining_gradients[4].owner.inputs
#print input_pretraining_gradients[4].owner.inputs[1].owner.inputs
#sys.exit()
#print "PROGRAM LEN %i HASH %i"% (len(m.pretraining_update.maker.fgraph.apply_nodes), reduce(lambda a, b: hash(a) ^ hash(b),prog_str))
rng = N.random.RandomState(unittest_tools.fetch_seed(23904))
inputs = [rng.rand(10, m.input_size) for i in 1, 2, 3]
targets = N.asarray([0, 3, 4, 2, 3, 4, 4, 2, 1, 0])
#print inputs
#print 'UNSUPERVISED PHASE'
t = time.time()
for i in xrange(3):
for j in xrange(iters_per_unsup):
try:
known_fail = False
m.pretraining_update(*inputs)
except ValueError:
known_fail = True
except TypeError:
known_fail = True
if known_fail:
raise KnownFailureTest("Deprecated compile.module fails to "
"give a sensible warning when updates to a variable "
"have the wrong type")
s0, s1 = [str(j) for j in m.pretraining_update(*inputs)]
#print 'huh?', i, iters_per_unsup, iters_per_unsup * (i+1), s0, s1
if iters_per_unsup == 3:
assert s0.startswith('0.927793') # '0.403044')
assert s1.startswith('0.068035') # '0.074898')
#print 'UNSUPERVISED took %.3fs'%(time.time() - t)
#print 'FINETUNING GRAPH'
#print 'SUPERVISED PHASE COSTS (%s)'%optimizer
t = time.time()
for i in xrange(3):
for j in xrange(iters_per_unsup):
m.finetuning_update(*(inputs + [targets]))
s0 = str(m.finetuning_update(*(inputs + [targets])))
#print iters_per_sup * (i+1), s0
if iters_per_sup == 10:
s0f = float(s0)
assert 19.7042 < s0f and s0f < 19.7043
#print 'SUPERVISED took %.3fs'%( time.time() - t)
def jtest_main():
from theano import gof
JTEST = theano.compile.mode.optdb.query(*sys.argv[2:])
#print 'JTEST', JTEST
theano.compile.register_optimizer('JTEST', JTEST)
optimizer = eval(sys.argv[1])
test_naacl_model(optimizer, 10, 10, realistic=False)
def profile_main():
import cProfile
import pstats
from theano.compat.six import StringIO
prof = cProfile.Profile()
prof = prof.runctx("real_main()", globals(), locals())
stream = StringIO()
stats = pstats.Stats(prof)
stats.sort_stats("time") # Or cumulative
stats.print_stats(80) # 80 = how many to print
# The rest is optional.
# stats.print_callees()
# stats.print_callers()
if __name__ == '__main__':
profile_main()
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论