Skip to content

P
pytensor
提交 11477dfe authored 作者: Frederic Bastien's avatar Frederic Bastien
浏览文件
操作

backport merge

上级 6ee3d69f a4a54be6
隐藏空白字符变更
内嵌 并排
正在显示 包含 848 行增加 和 652 行删除
theano/compile/builders.py
浏览文件 @ 11477dfe
......@@ -56,7 +56,7 @@ class OpFromGraph(gof.Op):
def make_node(self, *inputs):
for input, type in zip(inputs, self.input_types):
if not type == input.type:
raise TypeError("Wrong type, expected %s but got %s" % type, input.type)
raise TypeError("Wrong type, expected %s but got %s" % (type, input.type))
return gof.Apply(self,
inputs,
[type() for type in self.output_types])
......
theano/compile/debugmode.py
浏览文件 @ 11477dfe
"""Provides `DebugMode`, an evaluation mode for debugging theano internals."""
__docformat__ = "restructuredtext en"
import time, copy, sys, copy_reg
import time, copy, sys, copy_reg, gc
from StringIO import StringIO
import numpy
......@@ -20,6 +20,22 @@ from theano.compile.function_module import (FunctionMaker,
Supervisor)
from theano.compile.mode import Mode, register_mode
import logging
_logger=logging.getLogger("theano.compile.debugmode")
_logger.setLevel(logging.WARNING)
def error(*args):
#sys.stderr.write('ERROR:'+ ' '.join(str(a) for a in args)+'\n')
_logger.error("ERROR: "+' '.join(str(a) for a in args))
def warning(*args):
#sys.stderr.write('WARNING:'+ ' '.join(str(a) for a in args)+'\n')
_logger.warning("WARNING: "+' '.join(str(a) for a in args))
def info(*args):
#sys.stderr.write('INFO:'+ ' '.join(str(a) for a in args)+'\n')
_logger.info("INFO: "+' '.join(str(a) for a in args))
def debug(*args):
#sys.stderr.write('DEBUG:'+ ' '.join(str(a) for a in args)+'\n')
_logger.debug("DEBUG: "+' '.join(str(a) for a in args))
########################
#
......@@ -116,8 +132,16 @@ class BadOptimization(DebugModeError):
print >> sio, " Variable: id", id(self.new_r), self.new_r
print >> sio, " Op", self.new_r.owner
print >> sio, " Value Type:", type(self.new_r_val)
print >> sio, " Old Value: ", self.old_r_val
print >> sio, " New Value: ", self.new_r_val
str_old_r_val = str(self.old_r_val)
if len(str_old_r_val) > 80:
print >> sio, " Old Value: ", str(self.old_r_val)[:80], '...'
else:
print >> sio, " Old Value: ", str(self.old_r_val)
str_new_r_val = str(self.new_r_val)
if len(str_new_r_val) > 80:
print >> sio, " New Value: ", str(self.new_r_val)[:80], '...'
else:
print >> sio, " New Value: ", str(self.new_r_val)
print >> sio, " Reason: ", str(self.reason)
print >> sio, " Old Graph:"
print >> sio, self.old_graph
......@@ -135,23 +159,28 @@ class BadDestroyMap(DebugModeError):
self.new_val = new_val
def __str__(self):
sio = StringIO()
print >> sio, " node:", self.node
print >> sio, " node.inputs:", [(str(i), id(i)) for i in self.node.inputs]
print >> sio, " destroy_map:", getattr(self.node.op, 'destroy_map', {})
print >> sio, " changed input idx:", self.idx
print >> sio, " changed input type:", self.node.inputs[self.idx].type
print >> sio, " repr (old val):", repr(self.old_val)
print >> sio, " repr (new val):", repr(self.new_val)
print >> sio, " value dtype (new <space> old):", self.new_val.dtype, self.old_val.dtype
print >> sio, " value shape (new <space> old):", self.new_val.shape, self.old_val.shape
print >> sio, " value min (new <space> old):", self.new_val.min(), self.old_val.min()
print >> sio, " value max (new <space> old):", self.new_val.max(), self.old_val.max()
print >> sio, " value min (new-old):", (self.new_val-self.old_val).min()
print >> sio, " value max (new-old):", (self.new_val-self.old_val).max()
print >> sio, ""
print >> sio, " Hint: this can also be caused by a deficient values_eq_approx() or __eq__() implementation that compares node input values"
return sio.getvalue()
npy_old_val = numpy.asarray(self.old_val)
npy_new_val = numpy.asarray(self.new_val)
try:
sio = StringIO()
print >> sio, " node:", self.node
print >> sio, " node.inputs:", [(str(i), id(i)) for i in self.node.inputs]
print >> sio, " destroy_map:", getattr(self.node.op, 'destroy_map', {})
print >> sio, " changed input idx:", self.idx
print >> sio, " changed input type:", self.node.inputs[self.idx].type
print >> sio, " repr (old val):", repr(self.old_val)
print >> sio, " repr (new val):", repr(self.new_val)
print >> sio, " value dtype (new <space> old):", npy_new_val.dtype, npy_old_val.dtype
print >> sio, " value shape (new <space> old):", npy_new_val.shape, npy_old_val.shape
print >> sio, " value min (new <space> old):", npy_new_val.min(), npy_old_val.min()
print >> sio, " value max (new <space> old):", npy_new_val.max(), npy_old_val.max()
print >> sio, " value min (new-old):", (npy_new_val-npy_old_val).min()
print >> sio, " value max (new-old):", (npy_new_val-npy_old_val).max()
print >> sio, ""
print >> sio, " Hint: this can also be caused by a deficient values_eq_approx() or __eq__() implementation that compares node input values"
return sio.getvalue()
except Exception, e:
return str(e)
class BadViewMap(DebugModeError):
"""Exception: Some perform() or c_code() created a memory alias that wasn't in the view_map"""
......@@ -191,15 +220,25 @@ class StochasticOrder(DebugModeError):
class InvalidValueError(DebugModeError):
"""Exception: some Op an output value that is inconsistent with the Type of that output"""
def __init__(self, r, v):
def __init__(self, r, v, client_node=None):
super(InvalidValueError, self).__init__()
self.r = r
self.v = v
self.client_node = client_node
def __str__(self):
r, v = self.r, self.v
return "InvalidValueError: Variable %s, Type %s, type(Value) %s, Value %s"\
% (str(r), str(r.type), str(type(v)), str(v)[0:100])
type_r = type(r)
type_v = type(v)
v_val = str(v)[0:100]
client_node = self.client_node
return """InvalidValueError
type(variable) = %(type_r)s
variable = %(r)s
type(value) = %(type_v)s
value = %(v_val)s
client_node = %(client_node)s
""" % locals()
########################
#
......@@ -393,7 +432,7 @@ def _lessbroken_deepcopy(a):
Returns a copy of `a` that shares no internal storage with the original. A deep copy.
This function handles numpy arrays specially to avoid some bug I had one time... (possibly
about copying 1-d arrays?)
about copying 0-d arrays?)
"""
# this exists because numpy copies are broken
if type(a) is numpy.ndarray:
......@@ -822,6 +861,7 @@ class _Linker(gof.link.LocalLinker):
# This is the function that runs when you evaluate the graph
#####
def f():
debug("starting f")
for x in no_recycling:
x[0] = None
......@@ -844,6 +884,7 @@ class _Linker(gof.link.LocalLinker):
assert len(thunks_py) == len(order)
# transfer the initial values from the storage_map to the r_vals
debug("DEBUGMODE: transfer initial values")
for r in storage_map:
if (r.owner is None):
if (storage_map[r][0] is None):
......@@ -865,18 +906,27 @@ class _Linker(gof.link.LocalLinker):
for i, (thunk_py, thunk_c, node) in enumerate(zip(thunks_py, thunks_c, order)):
this_node_destroyed_variables = set()
debug(i, "DEBUGMODE: starting node", i, node)
debug(i, "DEBUGMODE: inputs broadcastable", [i.type.broadcastable for i in node.inputs])
debug(i, "DEBUGMODE: outputs broadcastable", [i.type.broadcastable for i in node.outputs])
# put a copy of each input into the storage_map
# also, check that inputs have valid values
for r in node.inputs:
assert isinstance(r, gof.Variable)
assert r in r_vals
# print >> sys.stderr,i, "DEBUGMODE: deepcopy input ", r
storage_map[r][0] = _lessbroken_deepcopy(r_vals[r])
if not r.type.is_valid_value(storage_map[r][0]):
raise InvalidValueError(r, storage_map[r][0])
raise InvalidValueError(r, storage_map[r][0], client_node=node)
debug(i, "DEBUGMODE running thunk_py")
if thunk_py:
try:
thunk_py()
except utils.MethodNotDefined:
thunk_py = None #shouldn't have put it into the list in the first place
if thunk_py:
thunk_py()
# check output values for type-correctness
for r in node.outputs:
if not r.type.is_valid_value(storage_map[r][0]):
......@@ -888,9 +938,15 @@ class _Linker(gof.link.LocalLinker):
_check_viewmap(node, storage_map)
# print >> sys.stderr, i, "DEBUGMODE thunk_py %100s %50s %30s" % (node,
#[(id(o), numpy.asarray(storage_map[o][0])[0,0]) for o in node.inputs],
#[(id(o), numpy.asarray(storage_map[o][0])[0,0]) for o in node.outputs])
sys.stdout.flush()
#retrieve each output from the storage_map
for r in node.outputs:
assert r not in r_vals
# print >> sys.stderr, i, "DEBUGMODE storing reference output %x" % id(storage_map[r][0])
r_vals[r] = storage_map[r][0]
storage_map[r][0] = None #clear the storage_map of outputs for the thunk_c
......@@ -898,8 +954,10 @@ class _Linker(gof.link.LocalLinker):
for r in node.inputs:
# TODO: we only need to overwrite the non-destroyed inputs
# print >> sys.stderr, i, "DEBUGMODE replacing input", r
storage_map[r][0] = _lessbroken_deepcopy(r_vals[r])
debug(i, "DEBUGMODE running thunk_c")
thunk_c()
for r in node.outputs:
......@@ -918,21 +976,35 @@ class _Linker(gof.link.LocalLinker):
_check_viewmap(node, storage_map)
# print >> sys.stderr, i, "DEBUGMODE thunk_c %100s %50s %30s" % (node,
#[(id(o), numpy.asarray(storage_map[o][0])[0,0]) for o in node.inputs],
#[(id(o), numpy.asarray(storage_map[o][0])[0,0]) for o in node.outputs])
sys.stdout.flush()
for r in node.outputs:
if r in r_vals:
#print >> sys.stderr, i, "DEBUGMODE clearing output", r
# compares the version from thunk_py (in r_vals)
# to the version produced by thunk_c (in storage_map)
if not r.type.values_eq_approx(r_vals[r], storage_map[r][0]):
raise BadCLinkerOutput(r, val_py=r_vals[r], val_c=storage_map[r][0])
else:
#print >> sys.stderr, i, "DEBUGMODE storing reference output %x" % id(storage_map[r][0])
#retrieve each output from the storage_map
r_vals[r] = storage_map[r][0]
storage_map[r][0] = None #clear the storage_map for the thunk_c
# we're done with this thunk
# clear everything out of the storage_map
for r in node.inputs:
#print >> sys.stderr, i, "DEBUGMODE clearing input", r
storage_map[r][0] = None
debug("done with node")
if True:
gc.collect()
#except:
# raise_with_op(node)
......
theano/compile/function_module.py
浏览文件 @ 11477dfe
......@@ -15,7 +15,7 @@ import numpy
import theano.gof
#from theano import gof
import copy
import time
import mode as mode_module
from io import *
......@@ -840,6 +840,7 @@ def function(inputs, outputs, mode=None, accept_inplace = False):
f[<kitname>] = seed #re-seed the elements of a RandomKit
"""
t1 = time.time()
if mode is None:
mode = mode_module.default_mode
#backport
......@@ -885,6 +886,10 @@ def function(inputs, outputs, mode=None, accept_inplace = False):
else:
Maker = getattr(mode, 'function_maker', FunctionMaker)
fn = Maker(inputs, outputs, mode, accept_inplace = accept_inplace).create(defaults)
t2 = time.time()
if hasattr(mode, 'compile_time'):
mode.compile_time+=t2-t1
return fn
......
theano/compile/profilemode.py
浏览文件 @ 11477dfe
......@@ -9,8 +9,10 @@ class ProfileMode(Mode):
def __init__(self, linker=OpWiseCLinker(), optimizer=None):
local_time = [0.0]
apply_time = {}
apply_call = {}
op_time = {}
op_cimpl = {}
op_call = {}
def blah(i, node, th):
if hasattr(th, 'cthunk'):
......@@ -24,13 +26,18 @@ class ProfileMode(Mode):
local_time[0] += dt
apply_time[(i,node.op)] = apply_time.get((i,node.op), 0.0) + dt
apply_call[(i,node.op)] = apply_call.get((i,node.op), 0) + 1
op_time[node.op] = op_time.get(node.op, 0.0) + dt
op_cimpl[node.op] = hasattr(th, 'cthunk')
op_call[node.op] = op_call.get(node.op,0) + 1
self.local_time = local_time
self.apply_time = apply_time
self.apply_call = apply_call
self.op_time = op_time
self.op_cimpl = op_cimpl
self.op_call = op_call
self.compile_time = 0 #time passed in function()
if isinstance(linker, str):
linker = predefined_linkers[linker]
......@@ -48,6 +55,8 @@ class ProfileMode(Mode):
The Op-wise summary print the execution time of all Apply nodes executing the same Op are grouped together and the total execution time per Op is shown (so if you use dot twice, you will see only one entry there corresponding to the sum of the time spent in each of them). If two Op have different hash value, they will be separate.
The type-Op-wise summary group the result by type of op. So event if two Op have different hash value, they will be merged.
Their is an hack with the Op-wise summary. Go see it if you want to know more.
param: n_apply_to_print the number of apply to print. Default 15.
param: n_ops_to_print the number of ops to print. Default 20.
......@@ -68,14 +77,23 @@ class ProfileMode(Mode):
tot=0
for f,t,a in atimes[:n_apply_to_print]:
tot+=t
print ' %.2f%% %.3fs %.3fs %i %s' % (f*100, tot, t, a[0], a[1])
print ' %4.1f%% %.3fs %.3fs %i %s' % (f*100, tot, t, a[0], a[1])
print ' ... (remaining %i Apply instances account for %.2f%%(%.2fs) of the runtime)'\
%(max(0, len(atimes)-n_apply_to_print),
sum(f for f, t, a in atimes[n_apply_to_print:])*100,
sum(t for f, t, a in atimes[n_apply_to_print:]))
flops=False
flops_msg=''
for a,t in op_time.items():
if hasattr(a,'flops'):
flops=True
flops_msg=' <MFlops/s>'
print '\nHACK WARNING: we print the flops for some OP, but the logic don\' always work. You need to know the internal of Theano to make it work correctly. Otherwise don\'t use!'
break
print '\nOp-wise summary: < of local_time spent on this kind of Op> <cumulative seconds> <self seconds>%s <Op name>'%(flops_msg)
print '\nOp-wise summary: <% of local_time spent on this kind of Op> <cumulative seconds> <self seconds> <Op name>'
otimes = [(t/local_time, t, a, self.op_cimpl[a]) for a, t in op_time.items()]
otimes.sort()
otimes.reverse()
......@@ -86,10 +104,14 @@ class ProfileMode(Mode):
msg = '*'
else:
msg = ' '
print ' %.2f%% %.3fs %.3fs %s %s' % (f*100, tot, t, msg, a)
#backport
#print ' %.2f%% %.3fs %.3fs %s %s' % (f*100, tot, t, '*' if ci else ' ', a)
print ' ... (remaining %i Ops account for %.2f%%(%.2fs) of the runtime)'\
m=-1
if hasattr(a,'flops'):
m=a.flops*self.op_call[a]/t/1e6
if flops:
print ' %4.1f%% %.3fs %.3fs %s %7.1f %s' % (f*100, tot, t, msg, m,a)
else:
print ' %4.1f%% %.3fs %.3fs %s %s' % (f*100, tot, t, msg, a)
print ' ... (remaining %i Ops account for %6.2f%%(%.2fs) of the runtime)'\
%(max(0, len(otimes)-n_ops_to_print),
sum(f for f, t, a, ci in otimes[n_ops_to_print:])*100,
sum(t for f, t, a, ci in otimes[n_ops_to_print:]))
......@@ -114,15 +136,13 @@ class ProfileMode(Mode):
msg = '*'
else:
msg = ' '
print ' %.2f%% %.3fs %.3fs %s %s' % (f*100, tot, t, msg, a)
#backport
#print ' %.2f%% %.3fs %.3fs %s %s' % (f*100, tot, t, '*' if ci else ' ', a)
print ' %4.1f%% %.3fs %.3fs %s %s' % (f*100, tot, t, msg, a)
print ' ... (remaining %i Ops account for %.2f%%(%.2fs) of the runtime)'\
%(max(0, len(sotimes)-n_ops_to_print),
sum(f for f, t, a in sotimes[n_ops_to_print:])*100,
sum(t for f, t, a in sotimes[n_ops_to_print:]))
print '(*) Op is running a c implementation'
print 'compile time: %.3fs'%self.compile_time
register_mode('PROFILE_MODE',ProfileMode())
......@@ -138,3 +158,4 @@ def atexit_print_default_profile_mode():
#Register atexit_print_default_profile_mode to have the summary of the
#predefined mode PROFILE_MODE if it is used printed when the program terminate.
atexit.register(atexit_print_default_profile_mode)
theano/gof/cc.py
浏览文件 @ 11477dfe
......@@ -357,7 +357,6 @@ class CLinker(link.Linker):
self.env = env
self.fetch_variables()
self.no_recycling = no_recycling
self.module_compile_str = cmodule.gcc_module_compile_str
return self
def fetch_variables(self):
......@@ -397,6 +396,8 @@ class CLinker(link.Linker):
self.consts = []
c_support_code_apply = []
symbol = {}
# (init_)tasks contains a list of pairs (Op/Variable, task_name)
......@@ -477,7 +478,7 @@ class CLinker(link.Linker):
id += 2
for node in self.node_order:
for node_num, node in enumerate(self.node_order):
# We populate sub with a mapping from the variable names specified by the op's c_var_names
# method to the actual variable names that we will use.
......@@ -486,7 +487,7 @@ class CLinker(link.Linker):
## for variable, vname in zip(op.inputs + op.outputs, ivnames + ovnames):
## sub[vname] = symbol[variable]
name = "<invalid_c_thing>"
name = "node_%i" % node_num
isyms, osyms = [symbol[r] for r in node.inputs], [symbol[r] for r in node.outputs]
# c_validate_update is deprecated
......@@ -498,6 +499,11 @@ class CLinker(link.Linker):
sub['fail'] = failure_code(sub)
op = node.op
# type-specific support code
try: c_support_code_apply.append(op.c_support_code_apply(node, name))
except utils.MethodNotDefined: pass
# emit c_code
try: behavior = op.c_code(node, name, isyms, osyms, sub)
except utils.MethodNotDefined:
raise NotImplementedError("%s cannot produce C code" % op)
......@@ -539,6 +545,7 @@ class CLinker(link.Linker):
self.blocks = blocks
self.tasks = tasks
all = self.inputs + self.outputs + self.orphans
self.c_support_code_apply = c_support_code_apply
if (self.init_tasks, self.tasks) != self.get_init_tasks():
print >> sys.stderr, "init_tasks\n", self.init_tasks
......@@ -561,6 +568,7 @@ class CLinker(link.Linker):
This might contain duplicates.
"""
ret = []
# generic support code
for x in [y.type for y in self.variables] + [y.op for y in self.node_order]:
try: ret.append(x.c_support_code())
except utils.MethodNotDefined: pass
......@@ -599,28 +607,70 @@ class CLinker(link.Linker):
def headers(self):
"""WRITEME
Returns a list of headers that are needed by one
or more Variables or Ops.
or more Types or Ops.
This might contain duplicates.
The return value will not contain duplicates.
"""
ret = []
for x in [y.type for y in self.variables] + [y.op for y in self.node_order]:
try: ret += x.c_headers()
except utils.MethodNotDefined: pass
return ret
return list(set(ret))
def c_compiler(self):
c_compiler = None
for x in [y.type for y in self.variables] + [y.op for y in self.node_order]:
if hasattr(x, 'c_compiler'):
x_compiler = x.c_compiler()
else:
continue
if c_compiler is None:
c_compiler = x_compiler
else:
if x_compiler and (x_compiler != c_compiler):
raise Exception('Nodes have requested specific different compilers',
(c_compiler, x_compiler))
return cmodule.gcc_module_compile_str if (c_compiler is None) else c_compiler
def header_dirs(self):
"""WRITEME
Returns a list of lib directories that are needed by one
or more Types or Ops.
The return value will not contain duplicates.
"""
ret = []
for x in [y.type for y in self.variables] + [y.op for y in self.node_order]:
try: ret += x.c_header_dirs()
except utils.MethodNotDefined: pass
return list(set(ret))
def libraries(self):
"""WRITEME
Returns a list of libraries that are needed by one
or more Variables or Ops.
or more Types or Ops.
This might contain duplicates.
The return value will not contain duplicates.
"""
ret = []
for x in [y.type for y in self.variables] + [y.op for y in self.node_order]:
try: ret += x.c_libraries()
except utils.MethodNotDefined: pass
return ret
return list(set(ret))
def lib_dirs(self):
"""WRITEME
Returns a list of lib directories that are needed by one
or more Types or Ops.
The return value will not contain duplicates.
"""
ret = []
for x in [y.type for y in self.variables] + [y.op for y in self.node_order]:
try: ret += x.c_lib_dirs()
except utils.MethodNotDefined: pass
return list(set(ret))
def __compile__(self, input_storage = None, output_storage = None):
"""WRITEME
......@@ -787,11 +837,13 @@ class CLinker(link.Linker):
get_lock()
try:
debug("LOCATION", location)
module = self.module_compile_str(
c_compiler = self.c_compiler()
module = c_compiler(
module_name=mod.name,
src_code = mod.code(),
location=location,
include_dirs=[],
include_dirs=self.header_dirs(),
lib_dirs=self.lib_dirs(),
libs=self.libraries(),
preargs=self.compile_args())
finally:
......@@ -834,7 +886,7 @@ class CLinker(link.Linker):
""" % dict(struct_name = self.struct_name)
# We add all the support code, compile args, headers and libs we need.
for support_code in self.support_code():
for support_code in self.support_code() + self.c_support_code_apply:
mod.add_support_code(support_code)
mod.add_support_code(self.struct_code)
mod.add_support_code(static)
......
theano/gof/cmodule.py
浏览文件 @ 11477dfe
......@@ -93,7 +93,12 @@ class DynamicModule(object):
def code(self):
sio = StringIO.StringIO()
for inc in self.includes:
print >> sio, "#include", inc
if not inc:
continue
if inc[0] == '<' or inc[0] == '"':
print >> sio, "#include", inc
else:
print >> sio, '#include "%s"'%inc
print >> sio, "//////////////////////"
print >> sio, "//// Support Code"
......@@ -236,12 +241,8 @@ class ModuleCache(object):
self.module_from_name = dict(self.module_from_name)
self.entry_from_key = dict(self.entry_from_key)
self.stats = [0, 0, 0]
if force_fresh is None:
self.force_fresh = self.force_fresh
else:
if force_fresh is not None:
self.force_fresh = force_fresh
#backport
#self.force_fresh = self.force_fresh if force_fresh is None else force_fresh
self.loaded_key_pkl = set()
self.refresh()
......@@ -426,8 +427,6 @@ class ModuleCache(object):
if age_thresh is None:
age_thresh = self.age_thresh
#backport
#age_thresh = self.age_thresh if age_thresh is None else age_thresh
compilelock.get_lock()
try:
# update the age of modules that have been accessed by other processes
......@@ -516,30 +515,58 @@ def get_gcc_shared_library_arg():
else:
return '-shared'
def std_include_dirs():
return [distutils.sysconfig.get_python_inc()] + numpy.distutils.misc_util.get_numpy_include_dirs()
def std_lib_dirs_and_libs():
python_inc = distutils.sysconfig.get_python_inc()
if sys.platform == 'win32':
# Typical include directory: C:\Python26\include
libname = os.path.basename(os.path.dirname(python_inc)).lower()
# Also add directory containing the Python library to the library
# directories.
python_lib_dir = os.path.join(os.path.dirname(python_inc), 'libs')
lib_dirs = [python_lib_dir]
return [libname], [python_lib_dir]
else:
# Typical include directory: /usr/include/python2.6
libname = os.path.basename(python_inc)
return [libname], []
def std_libs():
return std_lib_dirs_and_libs()[0]
def std_lib_dirs():
return std_lib_dirs_and_libs()[1]
def gcc_module_compile_str(module_name, src_code, location=None, include_dirs=[], lib_dirs=[], libs=[],
preargs=[], tmpdir=None):
preargs=[]):
"""
:param module_name: string (this has been embedded in the src_code
:param src_code: a complete c or c++ source listing for the module
:param location: a pre-existing filesystem directory where the cpp file and .so will be written
:param include_dirs: a list of include directory names (each gets prefixed with -I)
:param lib_dirs: a list of library search path directory names (each gets prefixed with -L)
:param libs: a list of libraries to link with (each gets prefixed with -l)
:param preargs: a list of extra compiler arguments
:returns: dynamically-imported python module of the compiled code.
"""
#TODO: don't to the dlimport in this function
if preargs is None:
preargs = []
else:
preargs = list(preargs)
#backport
#preargs= [] if preargs is None else list(preargs)
preargs.append('-fPIC')
no_opt = False
include_dirs = [distutils.sysconfig.get_python_inc()] + \
numpy.distutils.misc_util.get_numpy_include_dirs()\
+ include_dirs
python_inc = distutils.sysconfig.get_python_inc()
include_dirs = std_include_dirs() + include_dirs
libs = std_libs() + libs
lib_dirs = std_lib_dirs() + lib_dirs
if sys.platform == 'win32':
python_inc = distutils.sysconfig.get_python_inc()
# Typical include directory: C:\Python26\include
libname = os.path.basename(os.path.dirname(python_inc)).lower()
# Also add directory containing the Python library to the library
......@@ -548,6 +575,7 @@ def gcc_module_compile_str(module_name, src_code, location=None, include_dirs=[]
lib_dirs = [python_lib_dir] + lib_dirs
else:
# Typical include directory: /usr/include/python2.6
python_inc = distutils.sysconfig.get_python_inc()
libname = os.path.basename(python_inc)
......@@ -561,7 +589,8 @@ def gcc_module_compile_str(module_name, src_code, location=None, include_dirs=[]
workdir = location
cppfilename = os.path.join(workdir, 'mod.cpp')
cppfilename = os.path.join(location, 'mod.cpp')
cppfile = file(cppfilename, 'w')
debug('Writing module C++ code to', cppfilename)
......@@ -571,7 +600,7 @@ def gcc_module_compile_str(module_name, src_code, location=None, include_dirs=[]
cppfile.write(src_code)
cppfile.close()
lib_filename = os.path.join(workdir, '%s.%s' %
lib_filename = os.path.join(location, '%s.%s' %
(module_name, get_lib_extension()))
debug('Generating shared lib', lib_filename)
......@@ -586,93 +615,21 @@ def gcc_module_compile_str(module_name, src_code, location=None, include_dirs=[]
cmd.extend(['-L%s'%ldir for ldir in lib_dirs])
cmd.extend(['-l%s'%l for l in libs])
debug('Running cmd', ' '.join(cmd))
p = subprocess.Popen(cmd)
status = p.wait()
if status:
error('g++ return status', status)
else:
#touch the __init__ file
file(os.path.join(workdir, "__init__.py"),'w').close()
rval = dlimport(lib_filename)
return rval
def nvcc_module_compile_str(module_name, src_code, location=None, include_dirs=[], lib_dirs=[], libs=[],
preargs=[], tmpdir=None):
if preargs is None:
preargs = []
else:
preargs = list(preargs)
#backport
#preargs= [] if preargs is None else list(preargs)
preargs.append('-fPIC')
no_opt = False
raise NotImplementedError()
#TODO: -O preargs should be passed globally, not to -Xcompiler
#TODO: where to find these strings? sys? distutils?
include_dirs = ['/usr/include/python2.6'] + include_dirs
libs = ['python2.6', 'cudart'] + libs
lib_dirs = ['/usr/local/cuda/lib']+lib_dirs
workdir = tempfile.mkdtemp(dir=location)
cppfilename = os.path.join(workdir, 'mod.cpp') #.cpp to use g++
cppfilename = os.path.join(workdir, 'mod.cu') #.cu to use nvopencc
cppfile = file(cppfilename, 'w')
debug('Writing module C++ code to', cppfilename)
ofiles = []
rval = None
try:
cppfile.write(src_code)
cppfile.close()
lib_filename = os.path.join(workdir, '%s.%s' %
(module_name, get_lib_extension()))
debug('Generating shared lib', lib_filename)
cmd = ['nvcc', '-shared', '-g']
cmd.extend(['-Xcompiler', ','.join(preargs)])
cmd.extend('-I%s'%idir for idir in include_dirs)
cmd.extend(['-o',lib_filename])
cmd.append(cppfilename)
cmd.extend(['-L%s'%ldir for ldir in lib_dirs])
cmd.extend(['-l%s'%l for l in libs])
debug('Running cmd', ' '.join(cmd))
p = subprocess.Popen(cmd)
status = p.wait()
if status:
warning('nvcc return status', status)
else:
#touch the __init__ file
file(os.path.join(workdir, "__init__.py"),'w').close()
#load the module
sys.path.insert(0, workdir)
try:
rval = __import__(module_name, {}, {}, [module_name])
if not rval:
debug('__import__ failed')
finally:
del sys.path[0]
assert pathcopy == sys.path
finally:
warning("TODO: cleanup")
#os.remove(cppfilename)
for ofile in ofiles:
#os.remove(ofiles[0])
pass
return rval
print '==============================='
for i, l in enumerate(src_code.split('\n')):
#gcc put its messages to stderr, so we add ours now
print >> sys.stderr, '%05i\t%s'%(i+1, l)
print '==============================='
raise Exception('g++ return status', status)
#touch the __init__ file
file(os.path.join(location, "__init__.py"),'w').close()
return dlimport(lib_filename)
def icc_module_compile_str(*args):
raise NotImplementedError()
......
theano/gof/op.py
浏览文件 @ 11477dfe
......@@ -11,8 +11,138 @@ __docformat__ = "restructuredtext en"
import utils
import traceback
class CLinkerObject(object):
"""Standard elements of an Op or Type used with the CLinker
"""
def c_headers(self):
"""Optional: Return a list of header files required by code returned by
this class.
For example: return ['<iostream>', '<math.h>', '/full/path/to/header.h']
These strings will be prefixed with "#include " and inserted at the beginning of the c
source code.
Strings in this list that start neither with '<' nor '"' will be enclosed in
double-quotes.
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise utils.MethodNotDefined("c_headers", type(self), self.__class__.__name__)
def c_header_dirs(self):
"""Optional: Return a list of header search paths required by code returned by
this class.
For example: return ['/usr/local/include', '/opt/weirdpath/src/include'].
Provide search paths for headers, in addition to those in any relevant environment
variables.
Hint: for unix compilers, these are the things that get '-I' prefixed in the compiler
cmdline.
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise utils.MethodNotDefined("c_lib_dirs", type(self), self.__class__.__name__)
def c_libraries(self):
"""Optional: Return a list of libraries required by code returned by
this class.
For example: return ['gsl', 'gslcblas', 'm', 'fftw3', 'g2c'].
The compiler will search the directories specified by the environment
variable LD_LIBRARY_PATH in addition to any returned by `c_lib_dirs`.
Hint: for unix compilers, these are the things that get '-l' prefixed in the compiler
cmdline.
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise utils.MethodNotDefined("c_libraries", type(self), self.__class__.__name__)
def c_lib_dirs(self):
"""Optional: Return a list of library search paths required by code returned by
this class.
For example: return ['/usr/local/lib', '/opt/weirdpath/build/libs'].
Provide search paths for libraries, in addition to those in any relevant environment
variables (e.g. LD_LIBRARY_PATH).
Hint: for unix compilers, these are the things that get '-L' prefixed in the compiler
cmdline.
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise utils.MethodNotDefined("c_lib_dirs", type(self), self.__class__.__name__)
def c_support_code(self):
"""Optional: Return utility code for use by a `Variable` or `Op` to be
included at global scope prior to the rest of the code for this class.
class CLinkerOp(object):
QUESTION: How many times will this support code be emitted for a graph
with many instances of the same type?
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise utils.MethodNotDefined("c_support_code", type(self), self.__class__.__name__)
def c_code_cache_version(self):
"""Return a tuple of integers indicating the version of this Op.
An empty tuple indicates an 'unversioned' Op that will not be cached between processes.
The cache mechanism may erase cached modules that have been superceded by newer
versions. See `ModuleCache` for details.
"""
return (1,)
def c_compile_args(self):
"""Optional: Return a list of compile args recommended to compile the
code returned by other methods in this class.
Example: return ['-ffast-math']
Compiler arguments related to headers, libraries and search paths should be provided
via the functions `c_headers`, `c_libraries`, `c_header_dirs`, and `c_lib_dirs`.
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise utils.MethodNotDefined("c_compile_args", type(self), self.__class__.__name__)
def c_no_compile_args(self):
"""Optional: Return a list of incompatible gcc compiler arguments.
We will remove those arguments from the command line of gcc. So if
another Op adds a compile arg in the graph that is incompatible
with this Op, the incompatible arg will not be used.
Useful for instance to remove -ffast-math.
EXAMPLE
WRITEME
:Exceptions:
- `MethodNotDefined`: the subclass does not override this method
"""
raise utils.MethodNotDefined("c_no_compile_args", type(self), self.__class__.__name__)
class CLinkerOp(CLinkerObject):
"""
Interface definition for `Op` subclasses compiled by `CLinker`.
......@@ -83,94 +213,22 @@ class CLinkerOp(object):
raise utils.MethodNotDefined('%s.c_code_cleanup' \
% self.__class__.__name__)
def c_compile_args(self):
"""Optional: Return a list of recommended gcc compiler arguments.
QUESTION: is this function optional?
This is only a hint.
EXAMPLE
WRITEME
"""
raise utils.MethodNotDefined('%s.c_compile_args' \
% self.__class__.__name__)
def c_no_compile_args(self):
"""Optional: Return a list of incompatible gcc compiler arguments.
We will remove those arguments from the command line of gcc. So if
another Op adds a compile arg in the graph that is incompatible
with this Op, the incompatible arg will not be used.
Useful for instance to remove -ffast-math.
def c_support_code_apply(self, node, name):
"""Optional: Return utility code for use by an `Op` that will be inserted at global
scope, that can be specialized for the support of a particular `Apply` node.
EXAMPLE
:param node: an Apply instance in the graph being compiled
WRITEME
:param node_id: a string or number that serves to uniquely identify this node.
Symbol names defined by this support code should include the node_id, so that they can
be called from the c_code, and so that they do not cause name collisions.
:Exceptions:
- `MethodNotDefined`: the subclass does not override this method
- `MethodNotDefined`: Subclass does not implement this method
"""
raise utils.MethodNotDefined('%s.c_no_compile_args' \
% self.__class__.__name__)
def c_headers(self):
"""Optional: Return a list of header files that must be included to compile the C code.
A subclass should override this method.
EXAMPLE
WRITEME
:Exceptions:
- `MethodNotDefined`: the subclass does not override this method
raise utils.MethodNotDefined("c_support_code_apply", type(self), self.__class__.__name__)
"""
raise utils.MethodNotDefined('%s.c_headers' \
% self.__class__.__name__)
def c_libraries(self):
"""Optional: Return a list of libraries to link against to manipulate this `Op`.
A subclass should override this method.
WRITEME
:Exceptions:
- `MethodNotDefined`: the subclass does not override this method
"""
raise utils.MethodNotDefined('%s.c_libraries' \
% self.__class__.__name__)
def c_support_code(self):
"""Optional: Return support code for use by the code that is returned by `c_code`.
Support code is inserted into the C code at global scope.
A subclass should override this method.
WRITEME
:Exceptions:
- `MethodNotDefined`: the subclass does not override this method
"""
raise utils.MethodNotDefined('%s.c_support_code' \
% self.__class__.__name__)
def c_code_cache_version(self):
"""Return a tuple of integers indicating the version of this Op.
An empty tuple indicates an 'unversioned' Op that will not be cached between processes.
The cache mechanism may erase cached modules that have been superceded by newer
versions. See `ModuleCache` for details.
"""
return (1,)
class PureOp(object):
"""
......
theano/gof/opt.py
浏览文件 @ 11477dfe
......@@ -750,6 +750,7 @@ class NavigatorOptimizer(Optimizer):
raise
if replacements is False or replacements is None:
return False
assert len(node.outputs) == len(replacements)
repl_pairs = zip(node.outputs, replacements)
try:
env.replace_all_validate(repl_pairs, reason=lopt)
......
theano/gof/optdb.py
浏览文件 @ 11477dfe
import sys
import sys, StringIO
if sys.version_info[:2] >= (2,5):
from collections import defaultdict
......@@ -145,4 +145,14 @@ class SequenceDB(DB):
opts.sort(key = lambda obj: self.__priority__[obj.name])
return opt.SeqOptimizer(opts, failure_callback = self.failure_callback)
def print_summary(self, stream=sys.stdout):
print >> stream, "SequenceDB (id %i)"%id(self)
print >> stream, " priority", self.__priority__
print >> stream, " names", self._names
print >> stream, " db", self.__db__
def __str__(self):
sio = StringIO.StringIO()
self.print_summary(sio)
return sio.getvalue()
theano/gof/type.py
浏览文件 @ 11477dfe
......@@ -12,8 +12,9 @@ import traceback
########
# Type #
########
from .op import CLinkerObject
class CLinkerType(object):
class CLinkerType(CLinkerObject):
"""Interface specification for Types that can be arguments to a `CLinkerOp`.
A CLinkerType instance is mainly reponsible for providing the C code that
......@@ -176,89 +177,8 @@ class CLinkerType(object):
"""
raise MethodNotDefined("c_sync", type(self), self.__class__.__name__)
def c_compile_args(self):
"""Optional: Return a list of compile args recommended to compile the
code returned by other methods in this class.
WRITEME: example of formatting for -I, -L, -f args.
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise MethodNotDefined("c_compile_args", type(self), self.__class__.__name__)
def c_no_compile_args(self):
"""Optional: Return a list of incompatible gcc compiler arguments.
We will remove those arguments from the command line of gcc. So if
another Op adds a compile arg in the graph that is incompatible
with this Op, the incompatible arg will not be used.
Useful for instance to remove -ffast-math.
EXAMPLE
WRITEME
:Exceptions:
- `MethodNotDefined`: the subclass does not override this method
"""
raise MethodNotDefined("c_no_compile_args", type(self), self.__class__.__name__)
def c_headers(self):
"""Optional: Return a list of header files required by code returned by
this class.
WRITEME: example of local file, standard file.
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise MethodNotDefined("c_headers", type(self), self.__class__.__name__)
def c_libraries(self):
"""Optional: Return a list of libraries required by code returned by
this class.
For example: return ['gsl', 'gslcblas', 'm', 'fftw3', 'g2c'].
The compiler will search the directories specified by the environment
variable LD_LIBRARY_PATH. No option is provided for an Op to provide an
extra library directory because this would change the linking path for
other Ops in a potentially disasterous way.
QUESTION: What about via the c_compile_args? a -L option is allowed no?
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise MethodNotDefined("c_libraries", type(self), self.__class__.__name__)
def c_support_code(self):
"""Optional: Return utility code for use by a `Variable` or `Op` to be
included at global scope prior to the rest of the code for this class.
QUESTION: How many times will this support code be emitted for a graph
with many instances of the same type?
:Exceptions:
- `MethodNotDefined`: Subclass does not implement this method
"""
raise MethodNotDefined("c_support_code", type(self), self.__class__.__name__)
def c_code_cache_version(self):
"""Return a tuple of integers indicating the version of this Op.
An empty tuple indicates an 'unversioned' Op that will not be cached between processes.
The cache mechanism may erase cached modules that have been superceded by newer
versions. See `ModuleCache` for details.
"""
return (1,)
class PureType(object):
"""Interface specification for variable type instances.
......
theano/gradient.py
浏览文件 @ 11477dfe
......@@ -10,26 +10,16 @@ from gof.python25 import all
import gof.utils
import logging
_logger=logging.getLogger("theano.gradient")
_logger.setLevel(logging.WARN)
def error(*args):
#sys.stderr.write('ERROR:'+ ' '.join(str(a) for a in args)+'\n')
_logger.error("ERROR: "+' '.join(str(a) for a in args))
def warning(*args):
#sys.stderr.write('WARNING:'+ ' '.join(str(a) for a in args)+'\n')
_logger.warning("WARNING: "+' '.join(str(a) for a in args))
def info(*args):
#sys.stderr.write('INFO:'+ ' '.join(str(a) for a in args)+'\n')
_logger.info("INFO: "+' '.join(str(a) for a in args))
def debug(*args):
#sys.stderr.write('DEBUG:'+ ' '.join(str(a) for a in args)+'\n')
_logger.debug("DEBUG: "+' '.join(str(a) for a in args))
_logger = logging.getLogger('theano.gradient')
def warning(*msg):
_logger.warning('WARNING theano.gradient: '+' '.join(msg))
def info(*msg):
_logger.info('INFO theano.gradient: '+' '.join(msg))
_msg_retType = 'op.grad(...) returned a non-list'
_msg_badlen = 'op.grad(...) returned wrong number of gradients'
def grad_sources_inputs(sources, graph_inputs):
def grad_sources_inputs(sources, graph_inputs, warn_type=True):
"""
A gradient source is a pair (``r``, ``g_r``), in which ``r`` is a `Variable`, and ``g_r`` is a
`Variable` that is a gradient wrt ``r``.
......@@ -114,6 +104,11 @@ def grad_sources_inputs(sources, graph_inputs):
len(g_inputs),
len(node.inputs))
for ii, (r, g_r) in enumerate(zip(node.inputs, g_inputs)):
if warn_type:
if g_r and (getattr(r,'type',0) != getattr(g_r,'type', 1)):
r_type = getattr(r,'type', None)
g_r_type = getattr(g_r,'type', None)
info('%s.grad returned a different type for input %i: %s vs. %s'%(node.op, ii, r_type, g_r_type))
if g_r and len(sources) == 1 and sources[0][0].name and r.name:
g_r.name = "(d%s/d%s)" % (sources[0][0].name, r.name)
if g_r is not None:
......
theano/sandbox/conv.py
浏览文件 @ 11477dfe
......@@ -47,7 +47,6 @@ class ConvOp(Op):
unroll_batch - c code generation option
unroll_kern - c code generation option
The reason that this op does the summation over convolutions within the 'stack' is that
it allows us to be memory-efficient about how gradients are calculated. If, for
example, we had a convolution op that took a list of images, a list of kernels, and
......@@ -92,14 +91,25 @@ class ConvOp(Op):
if self.bsize<=self.unroll_batch:
self.unroll_batch = self.bsize
else:
print "OPTIMISATION WARNING: in ConvOp.__init__() unroll_batch(%s) must be 0 or a divisor of bsize(%s). We revert it to 1. This won't change the result, but may make it slower."%(str(self.unroll_batch),str(self.bsize))
self.unroll_batch=1
#find the maximum value under unroll_batch that would work
new=self.unroll_batch
assert(new>=1)
while self.bsize % new!=0:
new-=1
print "OPTIMISATION WARNING: in ConvOp.__init__() unroll_batch(%s) must be 0 or a divisor of bsize(%s). We revert it to %d. This won't change the result, but may make it slower."%(str(self.unroll_batch),str(self.bsize),new)
self.unroll_batch=new
if self.unroll_kern>0 and self.nkern % unroll_kern!=0:
if self.nkern<=self.unroll_kern:
self.unroll_kern = self.nkern
else:
print "OPTIMISATION WARNING: in ConvOp.__init__() unroll_kern(%s) should be 0 or a divisor of nkern(%s)We revert it to 1. This won't change the result, but may make it slower."%(str(self.unroll_kern),str(self.nkern))
self.unroll_kern=1
#find the maximum value under unroll_kern that would work
new=self.unroll_kern
assert(new>=1)
while self.nkern % new!=0:
new-=1
print "OPTIMISATION WARNING: in ConvOp.__init__() unroll_kern(%s) should be 0 or a divisor of nkern(%s)We revert it to %d. This won't change the result, but may make it slower."%(str(self.unroll_kern),str(self.nkern),new)
self.unroll_kern=new
self.outshp = getFilterOutShp(self.imshp_logical, self.kshp_logical, (dx,dy), output_mode)
self.fulloutshp = getFilterOutShp(self.imshp_logical, self.kshp_logical, (1,1), output_mode)
self.out_mode = output_mode
......@@ -137,6 +147,33 @@ class ConvOp(Op):
def __str__(self):
return "ConvOp{" +",".join(str((a, getattr(self, a))) for a in self.__attrnames) + "}"
def set_flops(self):
""" Usefull with the hack in profilemode to print the MFlops"""
if self.out_mode=="valid":
self.flops=self.kshp[0]*self.kshp[1]*2#nb mul and add by output pixed
self.flops*=self.outshp[0]*self.outshp[1]#nb flops by output image
self.flops*=self.imshp[0]*self.nkern*self.bsize#for all outputs images#n_stack==self.imshp[0]
else: #full mode not implemented
self.flops=0
for out_row in range(self.outshp[0]):#loop over output row
for out_col in range(self.outshp[0]):#loop over output col
for row in range(self.kshp[0]):#loop over kern row
if row+out_row-self.kshp[0]+1<0 or row+out_row-self.kshp[0]+1>=self.imshp[1]: continue
col=0
max_col=self.kshp[1]
img_col=out_col-self.kshp[1]+1
max_col=min(max_col,self.imshp[2]-img_col)
if img_col<0:
col=-img_col
img_col+=col
while col < max_col: #loop over kern col
self.flops+=1
col+=1
self.flops*=self.imshp[0]*self.nkern*self.bsize#for all outputs images#n_stack==self.imshp[0]
def make_node(self, inputs, kerns):
# TODO: find a way to make ConvOp work for N-D (after NIPS09)
"""
......@@ -188,7 +225,6 @@ class ConvOp(Op):
buf = N.zeros((batchsize,)+ self.imshp_logical, dtype=img2d.dtype)
buf[:,:,::rstride, ::cstride] = img2d
img2d = buf
print 'A'
del buf, rstride, cstride
if self.kshp != self.kshp_logical:
......@@ -204,7 +240,6 @@ class ConvOp(Op):
assert coffset >= 0
buf[:,:,roffset::rstride, coffset::cstride] = filtersflipped
filtersflipped = buf
print 'B'
del buf, rstride, cstride
for b in range(batchsize):
......@@ -293,7 +328,10 @@ class ConvOp(Op):
unroll_batch=un_b, unroll_kern=un_k,
imshp_logical=imshp_logical,
kshp_logical=kshp_logical,
kshp_logical_top_aligned=kshp_logical_top_aligned)(img,filters)
kshp_logical_top_aligned=kshp_logical_top_aligned)
if hasattr(self,'flops'):
dw.set_flops()
dw = dw(img,filters)
assert (dw.owner.op.outshp==self.kshp).all()
if self.out_mode == 'valid':
# before DimShuffle, dw is of shape visdim x nkern x kshp[0] x kshp[1]
......@@ -311,7 +349,10 @@ class ConvOp(Op):
1,1, output_mode=mode,
unroll_batch=un_b, unroll_kern=un_k,
imshp_logical=(self.nkern, self.fulloutshp[0], self.fulloutshp[1]),
kshp_logical=None)(gz,filters)
kshp_logical=None)
if hasattr(self,'flops'):
din.set_flops()
din = din(gz,filters)
assert (din.owner.op.outshp==self.imshp[1:]).all()
return [din, dw]
......
theano/scalar/basic.py
浏览文件 @ 11477dfe
......@@ -31,10 +31,25 @@ def as_scalar(x, name = None):
def constant(x):
if isinstance(x, float):
return ScalarConstant(float64, x)
for dtype in ['float32', 'float64']:
x_ = numpy.asarray(x, dtype=dtype)
if numpy.all(x == x_):
break
x_ = None
assert x_ is not None
return ScalarConstant(Scalar(str(x_.dtype)), x)
if isinstance(x, int):
return ScalarConstant(int64, x)
return ScalarConstant(float64, float(x))
for dtype in ['int8', 'int16', 'int32', 'int64']:
x_ = numpy.asarray(x, dtype=dtype)
if numpy.all(x == x_):
break
x_ = None
assert x_ is not None
return ScalarConstant(Scalar(str(x_.dtype)), x)
if isinstance(x, complex):
raise NotImplementedError()
raise TypeError(x)
#return ScalarConstant(float64, float(x))
class Scalar(Type):
......@@ -206,9 +221,9 @@ class _scalar_py_operators:
def __neg__(self): return neg(self)
#CASTS
def __int__(self): return AsInt(self).out
def __float__(self): return AsInt(self).out
def __complex__(self): return AsComplex(self).out
#def __int__(self): return AsInt(self).out
#def __float__(self): return AsDouble(self).out
#def __complex__(self): return AsComplex(self).out
#BITWISE
def __invert__(self): return invert(self)
......
theano/tensor/basic.py
浏览文件 @ 11477dfe
......@@ -133,7 +133,7 @@ _as_tensor_variable = as_tensor_variable
as_tensor = as_tensor_variable
def constant_or_value(x, rtype, name=None, ndim=None):
def constant_or_value(x, rtype, name=None, ndim=None, dtype=None):
"""Return a symbolic `Constant` with value `x`
:Exceptions:
......@@ -141,10 +141,28 @@ def constant_or_value(x, rtype, name=None, ndim=None):
- `ValueError`: `x` could not be expanded to have ndim dimensions
"""
if isinstance(x, numpy.ndarray):
x_ = x
if dtype is not None:
x_ = numpy.asarray(x, dtype=dtype)
else:
x_ = numpy.asarray(x)
x_ = None
if rtype is TensorConstant and isinstance(x, int):
for dtype in ['int8', 'int16', 'int32', 'int64']:
x_ = numpy.asarray(x, dtype=dtype)
if numpy.all(x == x_):
break
x_ = None
elif rtype is TensorConstant and isinstance(x, float):
for dtype in ['float32', 'float64']:
x_ = numpy.asarray(x, dtype=dtype)
if numpy.all(x == x_):
break
x_ = None
elif isinstance(x, numpy.ndarray):
x_ = x
else:
x_ = numpy.asarray(x)
assert type(x_) == numpy.ndarray
bcastable = [d == 1 for d in x_.shape]
if ndim is not None:
......@@ -160,11 +178,11 @@ def constant_or_value(x, rtype, name=None, ndim=None):
except:
raise TypeError("Could not convert %s to TensorType" % x, type(x))
def constant(x, name=None, ndim=None):
return constant_or_value(x, rtype=TensorConstant, name=name, ndim=ndim)
def constant(x, name=None, ndim=None, dtype=None):
return constant_or_value(x, rtype=TensorConstant, name=name, ndim=ndim, dtype=dtype)
def value(x, name=None, ndim=None):
return constant_or_value(x, rtype=TensorValue, name=name, ndim=ndim)
def value(x, name=None, ndim=None, dtype=None):
return constant_or_value(x, rtype=TensorValue, name=name, ndim=ndim, dtype=dtype)
def _obj_is_wrappable_as_tensor(x):
try:
......@@ -262,7 +280,8 @@ class TensorType(Type):
"""Compare True iff other is the same kind of TensorType"""
return type(self) == type(other) and other.dtype == self.dtype and other.broadcastable == self.broadcastable
def values_eq_approx(self, a, b):
@staticmethod
def values_eq_approx(a, b):
if type(a) is numpy.ndarray and type(b) is numpy.ndarray:
if a.shape != b.shape:
return False
......@@ -337,19 +356,18 @@ class TensorType(Type):
return self.name
else:
b = self.broadcastable
#bcast = str(self.broadcastable)
# TODO: verify this backport
if not any(b):
retval = len(b)
else:
retval = str(b)
bcast = {(): 'scalar',
named_broadcastable = {(): 'scalar',
(False,): 'vector',
(False, True): 'col',
(True, False): 'row',
#(False, False): 'matrix'}.get(b, "%iD" % retval) #TODO backport
(False, False): 'matrix'}.get(b, "%iD" % len(b) if not any(b) else str(b))
(False, False): 'matrix'}
if b in named_broadcastable:
bcast = named_broadcastable[b]
else:
if any(b):
bcast = str(b)
else:
bcast = '%iD' % len(b)
return "TensorType(%s, %s)" % (str(self.dtype), bcast)
def __repr__(self):
......@@ -626,18 +644,22 @@ class _tensor_py_operators:
"""
return reshape(self, shape, ndim=ndim)
def dimshuffle(self, pattern):
def dimshuffle(self, *pattern):
"""Reorder the dimensions of this variable, optionally inserting broadcasted dimensions.
:param pattern: list of int mixed with 'x' for broadcastable dimensions
:param pattern: list/tuple of int mixed with 'x' for broadcastable dimensions
For example, to create a 3D view of a [2D] matrix, call ``dimshuffle([0,'x',1])``. This
will create a 3D view such that the middle dimension is an implicit broadcasted
dimension. To do the same thing on the transpose of that matrix, call ``dimshuffle([1,
'x', 0])``.
This function supports the pattern passed as a tuple, or as a variable-length argument (e.g. ``a.dimshuffle(pattern)`` is equivalent to ``a.dimshuffle(*pattern)`` where ``pattern`` is a list/tuple of ints mixed with 'x' characters).
For more information, see `DimShuffle`.
"""
if (len(pattern) == 1) and (isinstance(pattern[0], (list, tuple))):
pattern = pattern[0]
op = DimShuffle(list(self.type.broadcastable), pattern)
return op(self)
......@@ -656,7 +678,8 @@ class _tensor_py_operators:
return Subtensor(args)(self, *Subtensor.collapse(args, lambda entry: isinstance(entry, Variable)))
#COPYING
def copy(self): return tensor_copy(self)
def copy(self):
return tensor_copy(self)
def __iter__(self):
try:
......@@ -790,8 +813,6 @@ def _scal_elemwise(symbol):
else:
msg = "no_inplace"
n="Elemwise{%s,%s}"%(symbolname,msg)
#backport
#n="Elemwise{%s,%s}"%(symbolname,"inplace" if inplace else "no_inplace")
if inplace:
scalar_op = getattr(scal, symbolname[:-len('_inplace')])
......@@ -928,11 +949,11 @@ class MaxAndArgmax(Op):
inputs = [x, axis]
broadcastable = [False] * (x.type.ndim - 1) #TODO: be less conservative
outputs = [tensor(x.type.dtype, broadcastable),
tensor(axis.type.dtype, broadcastable)]
tensor('int32', broadcastable)]
return Apply(self, inputs, outputs)
def perform(self, node, (x, axis), (max, max_idx)):
max[0] = numpy.asarray(numpy.max(x, axis))
max_idx[0] = numpy.asarray(numpy.argmax(x, axis))
max_idx[0] = numpy.asarray(numpy.argmax(x, axis), dtype='int32')
def grad(self, (x, axis), (g_max, g_max_idx)):
# @warning: This only works if axis is 0, else the max is
# broadcasted wrong in the call to eq.
......@@ -949,17 +970,11 @@ class MaxAndArgmax(Op):
g_max_pad = shape_padleft(g_max)
else:
g_max_pad = shape_padright(g_max)
#backport
#g_max_pad = shape_padleft(g_max) if axis.data==0 else \
# shape_padright(g_max)
xmax = max(x, axis)
if axis.data==0:
xmax_pad = shape_padleft(xmax)
else:
xmax_pad = shape_padright(xmax)
#backport
#xmax_pad = shape_padleft(xmax) if axis.data==0 else \
# shape_padright(xmax)
g_x = eq(xmax_pad, x) * g_max_pad
return g_x, None
......@@ -1187,13 +1202,14 @@ pprint.assign(fill, printing.FunctionPrinter('fill'))
def ones_like(model):
"""WRITEME"""
#return Ones(model.type.ndim)(shape(model))
return fill(model, 1.0)
ret= fill(model, constant(1.0, dtype=model.type.dtype))
return ret
@constructor
def zeros_like(model):
"""WRITEME"""
#return Zeros(model.type.ndim)(shape(model))
return fill(model, 0.0)
return fill(model, constant(0.0, dtype=model.type.dtype))
class Filler(gof.Op):
"""WRITEME"""
......@@ -1560,6 +1576,7 @@ class Subtensor(Op):
return type(self) == type(other) and self.idx_list == other.idx_list
def __hash__(self):
#TODO: optimize by cache this hash value
msg = []
for entry in self.idx_list:
if isinstance(entry, slice):
......@@ -2226,9 +2243,11 @@ class Reshape(Op):
self.name = name
def __eq__(self, other):
return (type(other) is Reshape) and (other.ndim == self.ndim) and self.name == other.name
# .name does not participate because it doesn't affect computations
return (type(other) is Reshape) and (other.ndim == self.ndim)
def __hash__(self):
return hash(Reshape) ^ hash(self.ndim) ^ hash(self.name)
# .name does not participate because it doesn't affect computations
return hash(Reshape) ^ hash(self.ndim)
def make_node(self, x, shp):
x = as_tensor_variable(x)
shp = as_tensor_variable(shp)
......
theano/tensor/blas.py
浏览文件 @ 11477dfe
......@@ -4,8 +4,8 @@ import os, sys, traceback
import numpy
from theano.gof import (utils, Op, Apply, view_roots, PatternSub, DestroyHandler,
SeqOptimizer, local_optimizer, LocalOptimizer, OpKeyOptimizer,
InconsistencyError)
SeqOptimizer, local_optimizer, Optimizer, LocalOptimizer, OpKeyOptimizer,
InconsistencyError, toolbox)
from theano.printing import pprint, FunctionPrinter
from theano.tensor.opt import register_specialize, out2in, insert_inplace_optimizer
# opt.py
......@@ -363,7 +363,6 @@ class Gemm(GemmRelated):
gemm = Gemm()
pprint.assign(gemm, FunctionPrinter('gemm'))
def res_is_a(node, op, maxclients=None):
if maxclients is not None:
retval = (len(node.clients) <= maxclients)
......@@ -373,226 +372,220 @@ def res_is_a(node, op, maxclients=None):
return node.owner \
and node.owner.op == op \
and retval
#backport
# and (len(node.clients) <= maxclients if maxclients is not None else True)
class GemmLocalOptimizer(LocalOptimizer):
"""This is a massive beast for recognizing all the ways that a subtraction or addition
could be replaced by a GEMM
It depends on `local_transposed_dot` to canonicalize the graph a bit by swapping
dot(a,b).T -> dot(b.T, a.T)
"""
def __init__(self):
super(LocalOptimizer, self).__init__()
def op_key(self):
return [T.add, T.sub]
def _as_scalar(res):
"""Return None or a TensorVariable whose type is in T.float_scalar_types"""
if res.owner and isinstance(res.owner.op, T.DimShuffle):
return _as_scalar(res.owner.inputs[0])
elif res.type in T.float_scalar_types:
return res
elif isinstance(res, T.Constant) and res.data.size == 1:
return res.data.flatten()[0]
else:
return None
def _is_real_matrix(res):
return res.type.dtype in ('float32', 'float64') \
and res.type.ndim == 2 \
and res.type.broadcastable[0] == False \
and res.type.broadcastable[1] == False #cope with tuple vs. list
def _as_isolated_scalar_times_matrix(res):
if res_is_a(res, T.mul, 1):
if len(res.owner.inputs) == 2:
L, R = res.owner.inputs
sL = _as_scalar(L)
sR = _as_scalar(R)
if (sL is not None) and _is_real_matrix(R):
return (sL, R)
if (sR is not None) and _is_real_matrix(L):
return (sR, L)
else:
scalars = []
matrices = []
for input in res.owner.inputs:
scalar_input = _as_scalar(input)
if scalar_input is not None:
scalars.append(scalar_input)
elif _is_real_matrix(input):
matrices.append(input)
else:
return None
if len(matrices) == 1:
rval = (T.mul(*scalars), matrices[0])
return rval
def add_requirements(self, env):
super(GemmLocalOptimizer,self).add_requirements(env)
env.extend(DestroyHandler())
def _beta_L_plus_alpha_M(beta, L, alpha, M, recurse_flip = True):
#print 'BETA L + ALPHA M', beta, L, alpha, M, recurse_flip
#EXPRESSION: (beta * L) + (alpha * M)
if res_is_a(M, _dot22, 1):
Ml, Mr = M.owner.inputs
rval = [gemm(L, alpha, Ml, Mr, beta)]
#print 'GEMM 0', rval, beta, L, alpha, M
return rval
# this is False'd out because of inadequate testing.
# TODO see ticket #237
if False and res_is_a(M, gemm, 1):
#EXPRESSION: (beta * L) + (alpha * (gemm(G, a, u, v, b)))
#EXPRESSION: (beta * L) + alpha * (b * G) + alpha * a * dot(u, v)
G, a, u, v, b = M.owner.inputs
#print 'GEMM', G, L
if res_is_a(G, _dot22, 1):
#EXPRESSION: (beta * L) + (alpha * (gemm(dot(x,y), a, u, v, b)))
x, y = G.owner.inputs
#EXPRESSION: (beta * L) + (alpha * ((b*dot(x,y) + (a * dot(u, v)))))
#EXPRESSION: (beta * L) + (alpha*b*dot(x,y)) + (alpha * a * dot(u, v))
rval = [gemm(gemm(L, alpha * b, x, y, beta), alpha * a, u, v, 1.0)]
print 'GEMM 1', rval
return rval
if (G is L):
#EXPRESSION: (beta * L) + (alpha*b*L) + (alpha * a * dot(u, v))
rval = [gemm(L, alpha*a, u, v, alpha * b + beta)]
print 'GEMM 2', rval
return rval
if (1.0 != alpha):
#at the very least, move the alpha inside the gemm
rval = [beta * L + gemm(G, alpha * a, u, v, alpha * b)]
print 'GEMM 3', rval
return rval
def transform(self, node):
_as_scalar, _is_real_matrix, _as_isolated_scalar_times_matrix, beta_L_plus_alpha_M\
= (GemmLocalOptimizer._as_scalar,
GemmLocalOptimizer._is_real_matrix,
GemmLocalOptimizer._as_isolated_scalar_times_matrix,
GemmLocalOptimizer.beta_L_plus_alpha_M)
if node.op == T.sub:
L, R = node.inputs
if not _is_real_matrix(L):
return False
if not _is_real_matrix(R):
return False
tmp = _as_isolated_scalar_times_matrix(L)
try:
sL, mL = tmp
except:
sL, mL = 1.0, L
if recurse_flip:
return _beta_L_plus_alpha_M(alpha, M, beta, L, recurse_flip = False)
else:
return False
tmp = _as_isolated_scalar_times_matrix(R)
try:
sR, mR = tmp
except:
sR, mR = 1.0, R
rval = beta_L_plus_alpha_M(sL, mL, -sR, mR)
return rval
if node.op == T.add:
# arguments of the form scalar * matrix
sM_list = []
def _gemm_from_node(node):
"""
:todo: In many expressions, there are many ways to turn it into a gemm. For example
dot(a,b) + c + d. This function should return all of them, so that if one version of gemm
causes a cycle in the graph, then another application of gemm can be tried.
# arguments that can be interpreted as scalar * matrix
sM_orig = []
"""
if node.op == T.sub:
L, R = node.inputs
if not _is_real_matrix(L):
return False
if not _is_real_matrix(R):
return False
# arguments not of the form scalar * matrix (i.e., vectors, scalars)
other_inputs = []
tmp = _as_isolated_scalar_times_matrix(L)
try:
sL, mL = tmp
except:
sL, mL = 1.0, L
for input in node.inputs:
tmp = _as_isolated_scalar_times_matrix(input)
if tmp:
sM_list.append(tmp)
sM_orig.append(input)
elif _is_real_matrix(input):
sM_list.append((1.0, input))
sM_orig.append(input)
else:
other_inputs.append(input)
assert len(sM_list) == len(sM_orig)
assert len(sM_list) + len(other_inputs) == len(node.inputs)
if len(sM_list) == 2:
(sL, mL), (sR, mR) = sM_list
gemm_of_sM_list = beta_L_plus_alpha_M(sL, mL, sR, mR)
if gemm_of_sM_list:
#we turned the two candidates into a gemm
# now we have to add the other_inputs and return the replacement graph
if other_inputs:
return [T.add(*(other_inputs + gemm_of_sM_list))]
else:
return gemm_of_sM_list
tmp = _as_isolated_scalar_times_matrix(R)
try:
sR, mR = tmp
except:
sR, mR = 1.0, R
rval = _beta_L_plus_alpha_M(sL, mL, -sR, mR)
return rval
if node.op == T.add:
# arguments of the form scalar * matrix
sM_list = []
# arguments that can be interpreted as scalar * matrix
sM_orig = []
# arguments not of the form scalar * matrix (i.e., vectors, scalars)
other_inputs = []
for input in node.inputs:
tmp = _as_isolated_scalar_times_matrix(input)
if tmp:
sM_list.append(tmp)
sM_orig.append(input)
elif _is_real_matrix(input):
sM_list.append((1.0, input))
sM_orig.append(input)
else:
# Try every pair in the sM_list, trying to turn it into a gemm operation
for i in xrange(len(sM_list) - 1):
for j in xrange(i+1, len(sM_list)):
assert i != j
sL, mL = sM_list[i]
sR, mR = sM_list[j]
gemm_of_sM_list = beta_L_plus_alpha_M(sL, mL, sR, mR)
if gemm_of_sM_list:
assert len(gemm_of_sM_list) == 1
inputs_without_ij = [input for k, input in enumerate(sM_orig) if k not in (i,j)]
new_add_inputs = (inputs_without_ij + gemm_of_sM_list + other_inputs)
if False: #SUPER DEBUG MODE :(
if len(new_add_inputs) + 1 != len(node.inputs):
print 'inputs', node.inputs
print 'sM, other', sM_list, other_inputs
print 'i,j', i, j
print 'gemm', gemm_of_sM_list
print 'without ij', inputs_without_ij
print 'new inputs', new_add_inputs
sys.exit(1)
# this should be True because we've combined a pair of arguments
# into a single GEMM
assert len(new_add_inputs) + 1 == len(node.inputs)
return [T.add(*new_add_inputs)]
return False
@staticmethod
def _as_scalar(res):
"""Return None or a TensorVariable whose type is in T.float_scalar_types"""
if res.owner and isinstance(res.owner.op, T.DimShuffle):
return GemmLocalOptimizer._as_scalar(res.owner.inputs[0])
elif res.type in T.float_scalar_types:
return res
elif isinstance(res, T.Constant) and res.data.size == 1:
return res.data.flatten()[0]
other_inputs.append(input)
assert len(sM_list) == len(sM_orig)
assert len(sM_list) + len(other_inputs) == len(node.inputs)
if len(sM_list) == 2:
(sL, mL), (sR, mR) = sM_list
gemm_of_sM_list = _beta_L_plus_alpha_M(sL, mL, sR, mR)
if gemm_of_sM_list:
#we turned the two candidates into a gemm
# now we have to add the other_inputs and return the replacement graph
if other_inputs:
return [T.add(*(other_inputs + gemm_of_sM_list))]
else:
return gemm_of_sM_list
else:
return None
@staticmethod
def _is_real_matrix(res):
return res.type in T.float_matrix_types \
and res.type.broadcastable[0] == False \
and res.type.broadcastable[1] == False #cope with tuple vs. list
@staticmethod
def _as_isolated_scalar_times_matrix(res):
_as_scalar, _is_real_matrix, _as_isolated_scalar_times_matrix, beta_L_plus_alpha_M\
= (GemmLocalOptimizer._as_scalar,
GemmLocalOptimizer._is_real_matrix,
GemmLocalOptimizer._as_isolated_scalar_times_matrix,
GemmLocalOptimizer.beta_L_plus_alpha_M)
if res_is_a(res, T.mul, 1):
if len(res.owner.inputs) == 2:
L, R = res.owner.inputs
sL = _as_scalar(L)
sR = _as_scalar(R)
if (sL is not None) and _is_real_matrix(R):
return (sL, R)
if (sR is not None) and _is_real_matrix(L):
return (sR, L)
else:
scalars = []
matrices = []
for input in res.owner.inputs:
scalar_input = _as_scalar(input)
if scalar_input is not None:
scalars.append(scalar_input)
elif _is_real_matrix(input):
matrices.append(input)
else:
return None
if len(matrices) == 1:
rval = (T.mul(*scalars), matrices[0])
return rval
@staticmethod
def beta_L_plus_alpha_M(beta, L, alpha, M, recurse_flip = True):
#print 'BETA L + ALPHA M', beta, L, alpha, M, recurse_flip
#EXPRESSION: (beta * L) + (alpha * M)
if res_is_a(M, _dot22, 1):
Ml, Mr = M.owner.inputs
rval = [gemm(L, alpha, Ml, Mr, beta)]
#print 'GEMM 0', rval, beta, L, alpha, M
return rval
# Try every pair in the sM_list, trying to turn it into a gemm operation
for i in xrange(len(sM_list) - 1):
for j in xrange(i+1, len(sM_list)):
assert i != j
sL, mL = sM_list[i]
sR, mR = sM_list[j]
gemm_of_sM_list = _beta_L_plus_alpha_M(sL, mL, sR, mR)
if gemm_of_sM_list:
assert len(gemm_of_sM_list) == 1
inputs_without_ij = [input for k, input in enumerate(sM_orig) if k not in (i,j)]
new_add_inputs = (inputs_without_ij + gemm_of_sM_list + other_inputs)
if False: #SUPER DEBUG MODE :(
if len(new_add_inputs) + 1 != len(node.inputs):
print 'inputs', node.inputs
print 'sM, other', sM_list, other_inputs
print 'i,j', i, j
print 'gemm', gemm_of_sM_list
print 'without ij', inputs_without_ij
print 'new inputs', new_add_inputs
sys.exit(1)
# this should be True because we've combined a pair of arguments
# into a single GEMM
assert len(new_add_inputs) + 1 == len(node.inputs)
return [T.add(*new_add_inputs)]
return False
class GemmOptimizer(Optimizer):
"""Graph optimizer for inserting Gemm operations"""
def __init__(self):
Optimizer.__init__(self)
# this is False'd out because of inadequate testing.
# TODO see ticket #237
if False and res_is_a(M, gemm, 1):
#EXPRESSION: (beta * L) + (alpha * (gemm(G, a, u, v, b)))
#EXPRESSION: (beta * L) + alpha * (b * G) + alpha * a * dot(u, v)
G, a, u, v, b = M.owner.inputs
#print 'GEMM', G, L
if res_is_a(G, _dot22, 1):
#EXPRESSION: (beta * L) + (alpha * (gemm(dot(x,y), a, u, v, b)))
x, y = G.owner.inputs
#EXPRESSION: (beta * L) + (alpha * ((b*dot(x,y) + (a * dot(u, v)))))
#EXPRESSION: (beta * L) + (alpha*b*dot(x,y)) + (alpha * a * dot(u, v))
rval = [gemm(gemm(L, alpha * b, x, y, beta), alpha * a, u, v, 1.0)]
print 'GEMM 1', rval
return rval
if (G is L):
#EXPRESSION: (beta * L) + (alpha*b*L) + (alpha * a * dot(u, v))
rval = [gemm(L, alpha*a, u, v, alpha * b + beta)]
print 'GEMM 2', rval
return rval
if (1.0 != alpha):
#at the very least, move the alpha inside the gemm
rval = [beta * L + gemm(G, alpha * a, u, v, alpha * b)]
print 'GEMM 3', rval
return rval
def add_requirements(self, env):
env.extend(toolbox.ReplaceValidate())
env.extend(DestroyHandler())
if recurse_flip:
return GemmLocalOptimizer.beta_L_plus_alpha_M(alpha, M, beta, L, recurse_flip = False)
else:
return False
def apply(self, env):
did_something = True
while did_something:
nodelist = list(env.toposort())
did_something = False
for node in nodelist:
new_outputs = _gemm_from_node(node)
if new_outputs:
assert len(new_outputs) == len(node.outputs)
try:
env.replace_all_validate(
zip(node.outputs, new_outputs),
reason = 'GemmOptimizer')
did_something = True
break
except InconsistencyError, e:
#TODO: retry other applications of gemm (see comment in _gemm_from_node
pass
compile.optdb.register('inplace_gemm', GemmOptimizer(), 70.00, 'fast_run', 'inplace', 'gemm')
#I think that three passes should suffice to catch all the GEMMs.
# TODO: This could be an equilibriumOptmizer, but I don't know how to combine an OpKeyOptimizer and
# an EquilibriumOptimizer.
compile.optdb.register('inplace_gemm_0', OpKeyOptimizer(GemmLocalOptimizer(),
failure_callback=OpKeyOptimizer.warn_inplace), 70.00, 'fast_run', 'inplace', 'gemm')
compile.optdb.register('inplace_gemm_1', OpKeyOptimizer(GemmLocalOptimizer(),
failure_callback=OpKeyOptimizer.warn_inplace), 70.01, 'fast_run', 'inplace', 'gemm')
compile.optdb.register('inplace_gemm_2', OpKeyOptimizer(GemmLocalOptimizer(),
failure_callback=OpKeyOptimizer.warn_inplace), 70.02, 'fast_run', 'inplace', 'gemm')
class Dot22(GemmRelated):
"""Compute a matrix-matrix product.
This is a specialization of the more general Dot()
"""
def make_node(self, x, y):
assert GemmLocalOptimizer._is_real_matrix(x)
assert _is_real_matrix(x)
assert y.type == x.type #makes sure y is a matrix
bz = [False, False]
outputs = [T.tensor(x.type.dtype, bz)]
......@@ -645,7 +638,7 @@ _dot22 = Dot22()
def local_dot_to_dot22(node):
if node.op == T.dot:
x,y = node.inputs
if GemmLocalOptimizer._is_real_matrix(x) and y.type == x.type:
if _is_real_matrix(x) and y.type == x.type:
return [_dot22(*node.inputs)]
else:
return False
......
theano/tensor/elemwise.py
浏览文件 @ 11477dfe
......@@ -136,7 +136,8 @@ class DimShuffle(Op):
self.__dict__.update(d)
self._rehash()
def make_node(self, input):
def make_node(self, _input):
input = as_tensor_variable(_input)
ib = tuple(input.type.broadcastable)
if not ib == self.input_broadcastable:
raise TypeError("The number of dimensions and/or broadcastable pattern of the input is incorrect for this op. Expected %s, got %s." % (self.input_broadcastable, ib))
......@@ -659,7 +660,7 @@ class Elemwise(Op):
task_code = self.scalar_op.c_code(Apply(self.scalar_op,
[Scalar(dtype = input.type.dtype)() for input in node.inputs],
[Scalar(dtype = output.type.dtype)() for input in node.outputs]),
[Scalar(dtype = output.type.dtype)() for output in node.outputs]),
name + '_scalar_',
["%s_i" % s for s in _inames],
["%s_i" % s for s in onames],
......@@ -886,11 +887,12 @@ class Sum(CAReduce):
CAReduce.__init__(self, scalar.add, axis)
def _output_dtype(self, idtype):
if idtype.startswith('int'):
return 'int64' #we want to protect against overflow
else:
return idtype
# we want to protect against overflow
return dict(
int8='int32',
int16='int32',
int32='int64',
).get(idtype, idtype)
def grad(self, (x, ), (gz, )):
gz = as_tensor_variable(gz)
......
theano/tensor/nnet.py
浏览文件 @ 11477dfe
......@@ -77,7 +77,6 @@ pprint.assign(softplus, printing.FunctionPrinter('softplus'))
# TENSOR OPS
#
class SoftmaxWithBias(gof.Op):
"""
An L{Op} for the output of neural-net multiclass classifiers.
......@@ -133,6 +132,8 @@ class SoftmaxWithBias(gof.Op):
def c_headers(self):
return ['<iostream>','<cmath>']
def c_code_cache_version(self):
return ()
@staticmethod
def c_code_template():
# this implementation was lifted from
......@@ -157,14 +158,14 @@ class SoftmaxWithBias(gof.Op):
PyErr_SetString(PyExc_ValueError, "b not 1d tensor");
%(fail)s;
}
if (%(x)s->descr->type_num != PyArray_DOUBLE)
if ((%(x)s->descr->type_num != PyArray_DOUBLE)&&(%(x)s->descr->type_num != PyArray_FLOAT))
{
PyErr_SetString(PyExc_TypeError, "a not float64");
PyErr_SetString(PyExc_TypeError, "a not float");
%(fail)s;
}
if (%(b)s->descr->type_num != PyArray_DOUBLE)
if ((%(b)s->descr->type_num != PyArray_DOUBLE) && (%(b)s->descr->type_num != PyArray_FLOAT))
{
PyErr_SetString(PyExc_TypeError, "b not float64");
PyErr_SetString(PyExc_TypeError, "b not float");
%(fail)s;
}
if ((%(x)s->dimensions[1] != %(b)s->dimensions[0]))
......@@ -193,22 +194,22 @@ class SoftmaxWithBias(gof.Op):
double sum = 0.0;
bool discount_max = false;
const double* __restrict__ x_i = (double*)(%(x)s->data + %(x)s->strides[0] * i);
const double* __restrict__ b_i = (double*)(%(b)s->data);
double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i);
const dtype_%(x)s* __restrict__ x_i = (dtype_%(x)s*)(%(x)s->data + %(x)s->strides[0] * i);
const dtype_%(b)s* __restrict__ b_i = (dtype_%(b)s*)(%(b)s->data);
dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*)(%(sm)s->data + %(sm)s->strides[0] * i);
"""
inside_row_loop = """
npy_intp Sx = %(x)s->strides[1]/sizeof(double);
npy_intp Sb = %(b)s->strides[0]/sizeof(double);
npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
npy_intp Sx = %(x)s->strides[1]/sizeof(dtype_%(x)s);
npy_intp Sb = %(b)s->strides[0]/sizeof(dtype_%(b)s);
npy_intp Ssm = %(sm)s->strides[1]/sizeof(dtype_%(sm)s);
size_t row_max_j=0;
double row_max = x_i[0] + b_i[0];
dtype_%(sm)s row_max = x_i[0] + b_i[0];
// Get the maximum value of the row
for (j = 0; j < Nx[1]; ++j)
{
double row_ij = x_i[j * Sx] + b_i[j * Sb];
dtype_%(sm)s row_ij = x_i[j * Sx] + b_i[j * Sb];
// std::cout << "1" << row_ij << "\\n";
row_max_j = (row_ij > row_max) ? j : row_max_j;
row_max = (row_ij > row_max) ? row_ij : row_max;
......@@ -216,9 +217,9 @@ class SoftmaxWithBias(gof.Op):
for (j = 0; j < Nx[1]; ++j)
{
double row_ij = x_i[j * Sx] + b_i[j * Sb];
dtype_%(sm)s row_ij = x_i[j * Sx] + b_i[j * Sb];
// std::cout << "2" << row_ij << "\\n";
double sm_ij = exp(row_ij - row_max);
dtype_%(sm)s sm_ij = exp(row_ij - row_max);
// std::cout << "3" << sm_ij << "\\n";
sum += sm_ij;
sm_i[j * Ssm] = sm_ij;
......@@ -292,12 +293,18 @@ class SoftmaxGrad(gof.Op):
def grad(self, *args):
raise NotImplementedError()
def c_code_cache_version(self):
return ()
def c_code(self, node, name, (dy, sm), (dx,), sub):
return '''
if ((%(dy)s->descr->type_num != PyArray_DOUBLE)
|| (%(sm)s->descr->type_num != PyArray_DOUBLE))
if ((%(dy)s->descr->type_num != PyArray_DOUBLE) && (%(dy)s->descr->type_num != PyArray_FLOAT))
{
PyErr_SetString(PyExc_TypeError, "types should be float or float64");
%(fail)s;
}
if ((%(sm)s->descr->type_num != PyArray_DOUBLE) && (%(sm)s->descr->type_num != PyArray_FLOAT))
{
PyErr_SetString(PyExc_TypeError, "types should be float64, float64");
PyErr_SetString(PyExc_TypeError, "types should be float or float64");
%(fail)s;
}
if ((%(dy)s->nd != 2)
......@@ -327,12 +334,12 @@ class SoftmaxGrad(gof.Op):
for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i)
{
const double* __restrict__ dy_i = (double*) (%(dy)s->data + %(dy)s->strides[0] * i);
npy_intp Sdy = %(dy)s->strides[1]/sizeof(double);
const double* __restrict__ sm_i = (double*) (%(sm)s->data + %(sm)s->strides[0] * i);
npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
double* __restrict__ dx_i = (double*) (%(dx)s->data + %(dx)s->strides[0] * i);
npy_intp Sdx = %(dx)s->strides[1]/sizeof(double);
const dtype_%(dy)s* __restrict__ dy_i = (dtype_%(dy)s*) (%(dy)s->data + %(dy)s->strides[0] * i);
npy_intp Sdy = %(dy)s->strides[1]/sizeof(dtype_%(dy)s);
const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*) (%(sm)s->data + %(sm)s->strides[0] * i);
npy_intp Ssm = %(sm)s->strides[1]/sizeof(dtype_%(sm)s);
dtype_%(dx)s* __restrict__ dx_i = (dtype_%(dx)s*) (%(dx)s->data + %(dx)s->strides[0] * i);
npy_intp Sdx = %(dx)s->strides[1]/sizeof(dtype_%(dx)s);
double sum_dy_times_sm = 0.;
for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
......@@ -503,7 +510,7 @@ class CrossentropySoftmaxArgmax1HotWithBias(gof.Op):
raise ValueError('y_idx must have same number of rows as x')
sm = numpy.zeros_like(x) # softmax
nll = numpy.zeros(x.shape[0]) #nll(y | softmax(x))
nll = numpy.zeros(x.shape[0], dtype=node.outputs[0].type.dtype) #nll(y | softmax(x))
am = numpy.zeros_like(y_idx)
for i in xrange(sm.shape[0]):
#add the bias vector to the i'th row of x
......@@ -598,7 +605,7 @@ class CrossentropySoftmaxArgmax1HotWithBias(gof.Op):
begin_row_loop,
"""
const %(y_idx_type)s y_i = ((%(y_idx_type)s*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0];
double* __restrict__ nll_i = (double*)(%(nll)s->data + %(nll)s->strides[0] * i);
dtype_%(nll)s* __restrict__ nll_i = (dtype_%(nll)s*)(%(nll)s->data + %(nll)s->strides[0] * i);
%(am_type)s* __restrict__ am_i = (%(am_type)s*) (%(am)s->data + %(am)s->strides[0] * i);
""",
inside_row_loop,
......@@ -617,6 +624,8 @@ class CrossentropySoftmaxArgmax1HotWithBias(gof.Op):
end_row_loop)
def c_code_cache_version(self):
return ()
def c_code(self, node, name, (x, b, y_idx), (nll, sm, am), sub):
y_idx_type = node.inputs[2].type.dtype_specs()[1]
am_type = y_idx_type
......@@ -647,15 +656,20 @@ class CrossentropySoftmax1HotWithBiasDx (gof.Op):
output_storage[0][0] = dx
def grad(self, *args):
raise NotImplementedError()
def c_code_cache_version(self):
return ()
def c_code(self, node, name, (dnll, sm, y_idx), (dx,), sub):
y_idx_type = node.inputs[2].type.dtype_specs()[1]
return """
if ((%(dnll)s->descr->type_num != PyArray_DOUBLE)
|| (%(sm)s->descr->type_num != PyArray_DOUBLE)
)
if ((%(dnll)s->descr->type_num != PyArray_DOUBLE) && (%(dnll)s->descr->type_num != PyArray_FLOAT))
{
PyErr_SetString(PyExc_TypeError, "dnll type should be float32 or float64");
%(fail)s;
}
if ((%(sm)s->descr->type_num != PyArray_DOUBLE) && (%(sm)s->descr->type_num != PyArray_FLOAT))
{
PyErr_SetString(PyExc_TypeError, "types should be float64, float64, int64");
PyErr_SetString(PyExc_TypeError, "sm type should be float32 or float64");
%(fail)s;
}
if ((%(y_idx)s->descr->type_num != PyArray_INT64)
......@@ -697,15 +711,15 @@ class CrossentropySoftmax1HotWithBiasDx (gof.Op):
for (size_t i = 0; i < %(dx)s->dimensions[0]; ++i)
{
const double dnll_i = ((double*)(%(dnll)s->data + %(dnll)s->strides[0] * i))[0];
const dtype_%(dnll)s dnll_i = ((dtype_%(dnll)s*)(%(dnll)s->data + %(dnll)s->strides[0] * i))[0];
const %(y_idx_type)s y_i = ((%(y_idx_type)s*)(%(y_idx)s->data + %(y_idx)s->strides[0] * i))[0];
const double* __restrict__ sm_i = (double*)(%(sm)s->data + %(sm)s->strides[0] * i);
npy_intp Ssm = %(sm)s->strides[1]/sizeof(double);
const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*)(%(sm)s->data + %(sm)s->strides[0] * i);
npy_intp Ssm = %(sm)s->strides[1]/sizeof(dtype_%(sm)s);
double* __restrict__ dx_i = (double*)(%(dx)s->data + %(dx)s->strides[0] * i);
npy_intp Sdx = %(dx)s->strides[1]/sizeof(double);
dtype_%(dx)s* __restrict__ dx_i = (dtype_%(dx)s*)(%(dx)s->data + %(dx)s->strides[0] * i);
npy_intp Sdx = %(dx)s->strides[1]/sizeof(dtype_%(dx)s);
for (size_t j = 0; j < %(dx)s->dimensions[1]; ++j)
{
......@@ -801,7 +815,8 @@ class CrossentropyCategorical1Hot(gof.Op):
'(got type: %s instead of: %s)' % (_true_one_of_n.type,
tensor.lvector))
return gof.Apply(self, [_coding_dist, _true_one_of_n], [tensor.dvector()])
return gof.Apply(self, [_coding_dist, _true_one_of_n],
[tensor.Tensor(dtype=_coding_dist.dtype, broadcastable=[False])()])
def perform(self, node, (coding, one_of_n), (y_out,)):
y = numpy.zeros_like(coding[:,0])
......
theano/tensor/opt.py
浏览文件 @ 11477dfe
......@@ -114,7 +114,7 @@ def local_dimshuffle_lift(node):
input = node.inputs[0]
inode = input.owner
if inode and isinstance(inode.op, Elemwise):
if inode and isinstance(inode.op, Elemwise) and (len(input.clients)==1):
return inode.op.make_node(*[DimShuffle(input.type.broadcastable,
op.new_order,
op.inplace)(input) for input in inode.inputs]).outputs
......
theano/tensor/raw_random.py
浏览文件 @ 11477dfe
......@@ -143,7 +143,11 @@ class RandomFunction(gof.Op):
# build the inputs to this Apply by overlaying args on self.args
inputs = []
for arg, default in zip(args, self.args):
assert arg is None or default.type.dtype == arg.type.dtype
# The NAACL test is failing because of this assert.
# I am commenting out the requirement that the dtypes match because it doesn't seem
# to me to be necessary (although I agree it is typically true).
# -JB 20090819
#assert arg is None or default.type.dtype == arg.type.dtype
if arg is None:
input = default
else:
......
theano/tensor/tests/test_blas.py
浏览文件 @ 11477dfe
......@@ -4,14 +4,11 @@ from theano.gof import Env
from theano.printing import pp
import numpy
from theano.tensor.blas import *
from theano.tensor.blas import _dot22, res_is_a
from theano.tensor.blas import _dot22, res_is_a, _as_scalar, _is_real_matrix
from unittest import TestCase
from theano.tests import unittest_tools
from copy import copy
_as_scalar = GemmLocalOptimizer._as_scalar
_is_real_matrix = GemmLocalOptimizer._is_real_matrix
from theano import In, Out
from test_basic import (_approx_eq, as_tensor_variable, inplace_func,
compile, value, constant, inplace, eval_outputs)
......
theano/tensor/tests/test_nnet.py
浏览文件 @ 11477dfe
......@@ -206,18 +206,35 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase):
print 'BEFORE'
for node in env.toposort():
print node.op
print node.op, node.inputs
print '----'
theano.compile.mode.optdb.query(
theano.compile.mode.OPT_FAST_RUN).optimize(env)
print 'AFTER'
for node in env.toposort():
print node.op
assert env.toposort()[3].op == crossentropy_softmax_argmax_1hot_with_bias
assert env.toposort()[5].op == crossentropy_softmax_1hot_with_bias_dx
assert len(env.toposort()) == 6 #shorthand for actually checking what I really
print node.op, node.inputs
# the function has 9 ops because the dimshuffle and elemwise{second} aren't getting
# cleaned up as well as we'd like.
has_cx1hot = False
has_cx1hotdx = False
has_softmax = False
has_softmaxdx = False
for node in env.toposort():
if node.op == crossentropy_softmax_argmax_1hot_with_bias:
has_cx1hot = True
if node.op == crossentropy_softmax_1hot_with_bias_dx :
has_cx1hotdx = True
if node.op == softmax:
has_softmax = True
if node.op == softmax_grad:
has_softmaxdx = True
assert has_cx1hot
assert has_cx1hotdx
assert not has_softmax
assert not has_softmaxdx
def test_argmax_pushdown():
x = tensor.dmatrix()
......
theano/tests/test_gradient.py
浏览文件 @ 11477dfe
......@@ -10,7 +10,9 @@ from theano.gradient import *
from theano import gradient
_grad_sources_inputs = grad_sources_inputs
def _grad_sources_inputs(*args):
# warn_type was introduced after this code, it complains throughout for nothing.
return grad_sources_inputs(warn_type=False, *args)
class test_grad_sources_inputs(unittest.TestCase):
def test_retNone1(self):
......@@ -148,7 +150,7 @@ class test_grad_sources_inputs(unittest.TestCase):
return [1]
i = gof.generic()
a1 = O(self).make_node(i)
g = grad_sources_inputs([(a1.outputs[0], 1)], None)
g = grad_sources_inputs([(a1.outputs[0], 1)], None, warn_type=False)
self.failUnless(g[i] is 1)
def test_some_None_igrads(self):
......@@ -170,7 +172,7 @@ class test_grad_sources_inputs(unittest.TestCase):
k = gof.generic()
a1 = O(self, True).make_node(i,j)
a2 = O(self, True).make_node(a1.outputs[1], k)
g = grad_sources_inputs([(a2.outputs[0], 1)], None)
g = grad_sources_inputs([(a2.outputs[0], 1)], None, warn_type=False)
self.failUnless(g[i] is 1 and j not in g and k not in g)
a1 = O(self, True).make_node(i,j)
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论