提交 357c85bf authored 作者: Frederic's avatar Frederic

Move hash_from_dict and make it better support OrderedDict

上级 eb37aded
import theano import theano
from theano.gof.utils import give_variables_names, unique, remove from theano.compat import OrderedDict
from theano.gof.utils import (
give_variables_names, hash_from_dict, remove, unique)
def test_give_variables_names(): def test_give_variables_names():
...@@ -44,3 +46,20 @@ def test_remove(): ...@@ -44,3 +46,20 @@ def test_remove():
# The list are needed as with python 3, remove and filter return generators # The list are needed as with python 3, remove and filter return generators
# and we can't compare generators. # and we can't compare generators.
assert list(remove(even, range(5))) == list(filter(odd, range(5))) assert list(remove(even, range(5))) == list(filter(odd, range(5)))
def test_hash_from_dict():
dicts = [{}, {0: 0}, {0: 1}, {1: 0}, {1: 1},
{0: (0,)}, {0: [1]},
{0: (0, 1)}, {0: [1, 0]},
]
for elem in dicts[:]:
dicts.append(OrderedDict(elem))
hashs = []
for idx, d in enumerate(dicts):
h = hash_from_dict(d)
assert h not in hashs
hashs.append(h)
# List are not hashable. So they are transformed into tuple.
assert hash_from_dict({0: (0,)}) == hash_from_dict({0: [0]})
...@@ -7,7 +7,7 @@ import numpy ...@@ -7,7 +7,7 @@ import numpy
from six import iteritems from six import iteritems
from theano import config from theano import config
from theano.compat import PY3 from theano.compat import OrderedDict, PY3
def simple_extract_stack(f=None, limit=None): def simple_extract_stack(f=None, limit=None):
...@@ -465,3 +465,34 @@ else: ...@@ -465,3 +465,34 @@ else:
def hash_from_file(file_path): def hash_from_file(file_path):
"""Return the MD5 hash of a file.""" """Return the MD5 hash of a file."""
return hash_from_code(open(file_path, 'rb').read()) return hash_from_code(open(file_path, 'rb').read())
def hash_from_dict(d):
"""Work around the fact that dict are not hashable in python
This request that all object have a sorted order that depend only
on the value of the object. This is true for integer/float/string
We do not verify that the objects in the dict have this property.
Also, we transform values that are list into tuple as list are not
hashable.
:note: special case for OrderedDict, it use the order of the dict,
so the key don't need to be sortable.
"""
if isinstance(d, OrderedDict):
items = list(d.iteritems())
else:
items = list(d.items())
items.sort()
first_part = [k for k, v in items]
second_part = []
for k, v in items:
if isinstance(v, (tuple, list)):
second_part += [tuple(v)]
else:
second_part += [v]
tuple_items = tuple(first_part + second_part + [d.__class__])
return hash(tuple_items)
...@@ -27,11 +27,11 @@ import theano ...@@ -27,11 +27,11 @@ import theano
from six.moves import xrange from six.moves import xrange
from theano.compat import izip from theano.compat import izip
from theano.gof import Op, Apply, local_optimizer, EquilibriumDB from theano.gof import Op, Apply, local_optimizer, EquilibriumDB
from theano.gof.utils import hash_from_dict
from theano.sandbox.cuda import GpuElemwise, CudaNdarrayType, GpuOp from theano.sandbox.cuda import GpuElemwise, CudaNdarrayType, GpuOp
from theano.sandbox.cuda.basic_ops import (as_cuda_ndarray_variable, from theano.sandbox.cuda.basic_ops import (as_cuda_ndarray_variable,
gpu_contiguous) gpu_contiguous)
from theano.sandbox.cuda.opt import gpu_seqopt from theano.sandbox.cuda.opt import gpu_seqopt
from theano.tensor.utils import hash_from_dict
from . import pycuda_init from . import pycuda_init
if not pycuda_init.pycuda_available: if not pycuda_init.pycuda_available:
......
...@@ -13,9 +13,9 @@ from theano.gof import Apply, Op, OpenMPOp ...@@ -13,9 +13,9 @@ from theano.gof import Apply, Op, OpenMPOp
from theano import scalar from theano import scalar
from theano.scalar import get_scalar_type from theano.scalar import get_scalar_type
from theano.printing import pprint from theano.printing import pprint
from theano.tensor.utils import hash_from_dict
from theano.gradient import DisconnectedType from theano.gradient import DisconnectedType
from theano.gof.null_type import NullType from theano.gof.null_type import NullType
from theano.gof.utils import hash_from_dict
from theano.tensor import elemwise_cgen as cgen from theano.tensor import elemwise_cgen as cgen
config = theano.config config = theano.config
......
...@@ -3,8 +3,7 @@ import unittest ...@@ -3,8 +3,7 @@ import unittest
import numpy import numpy
import theano import theano
from theano.tensor.utils import (hash_from_ndarray, hash_from_dict, from theano.tensor.utils import (hash_from_ndarray, shape_of_variables)
shape_of_variables)
def test_hash_from_ndarray(): def test_hash_from_ndarray():
...@@ -37,21 +36,6 @@ def test_hash_from_ndarray(): ...@@ -37,21 +36,6 @@ def test_hash_from_ndarray():
assert hash_from_ndarray(rng[::-1]) == hash_from_ndarray(rng[::-1].copy()) assert hash_from_ndarray(rng[::-1]) == hash_from_ndarray(rng[::-1].copy())
def test_hash_from_dict():
dicts = [{}, {0: 0}, {0: 1}, {1: 0}, {1: 1},
{0: (0,)}, {0: [1]},
{0: (0, 1)}, {0: [1, 0]},
]
hashs = []
for idx, d in enumerate(dicts):
h = hash_from_dict(d)
assert h not in hashs
hashs.append(h)
# List are not hashable. So they are transformed into tuple.
assert hash_from_dict({0: (0,)}) == hash_from_dict({0: [0]})
class Tshape_of_variables(unittest.TestCase): class Tshape_of_variables(unittest.TestCase):
def test_simple(self): def test_simple(self):
x = theano.tensor.matrix('x') x = theano.tensor.matrix('x')
......
...@@ -30,31 +30,6 @@ def hash_from_ndarray(data): ...@@ -30,31 +30,6 @@ def hash_from_ndarray(data):
hash_from_code(str(data.dtype))) hash_from_code(str(data.dtype)))
def hash_from_dict(d):
"""Work around the fact that dict are not hashable in python
This request that all object have a sorted order that depend only
on the value of the object. This is true for integer/float/string
We do not verify that the objects in the dict have this property.
Also, we transform values that are list into tuple as list are not
hashable.
"""
items = list(d.items())
items.sort()
first_part = [k for k, v in items]
second_part = []
for k, v in items:
if isinstance(v, (tuple, list)):
second_part += [tuple(v)]
else:
second_part += [v]
tuple_items = tuple(first_part + second_part)
return hash(tuple_items)
def shape_of_variables(fgraph, input_shapes): def shape_of_variables(fgraph, input_shapes):
""" """
Compute the numeric shape of all intermediate variables given input shapes Compute the numeric shape of all intermediate variables given input shapes
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论