Skip to content

P
pytensor
提交 57615e21 authored 作者: Brandon T. Willard's avatar Brandon T. Willard 提交者： Thomas Wiecki
浏览文件
操作

Remove unused theano.misc files

上级 67015ca9
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 0 行增加 和 491 行删除
theano/misc/buildbot_filter.py deleted 100644 → 0
浏览文件 @ 67015ca9
#!/usr/bin/env python
import sys
def filter_output(fd_in):
s = ""
for line in fd_in:
toks = line.split()
if len(toks):
if toks[0] == "File" and toks[-1].startswith("test"):
s += line
elif toks[0].startswith("ImportError"):
s += line
elif toks[0] in [
"KnownFailureTest:",
"Exception:",
"Failure:",
"AssertionError",
"AssertionError:",
"GradientError:",
]:
s += line
elif toks[0] == "Executing" and toks[1] in ["tests", "nosetests"]:
s += line
return s
if __name__ == "__main__":
import pdb
pdb.set_trace()
if len(sys.argv) > 1:
with open(sys.argv[1]) as f:
print(filter_output(f))
else:
print(filter_output(sys.stdin))
theano/misc/do_nightly_build deleted 100755 → 0
浏览文件 @ 67015ca9
#!/bin/bash
date
START=`date +%s`
ARGS="$@"
PROFILING=""
RELEASE=""
# If not jenkins, set workspace to local Tmp
if [ -v $WORKSPACE ]; then
if [ -v $TMPDIR ]; then
TMPDIR=/tmp
fi
WORKSPACE=$TMPDIR
fi
echo "Hostname:"
hostname
if [ "$1" == "--release" ]; then
RELEASE="True"
shift
ARGS="$@"
fi
if [ "$1" == "--buildbot" ]; then
shift
ARGS="$@"
BASE_COMPILEDIR=$WORKSPACE/compile/theano_compile_dir_theano
ROOT_CWD=$WORKSPACE/nightly_build
FLAGS=base_compiledir=$BASE_COMPILEDIR
COMPILEDIR=`THEANO_FLAGS=$FLAGS python -c "from __future__ import print_function; import theano; print(theano.config.compiledir)"`
cd ${ROOT_CWD}/Theano
git rev-parse HEAD
#Run tests from inside the Theano directory to prevent import problem.
# PROFILING="--with-coverage --cover-package=theano"
export PYTHONPATH=${ROOT_CWD}:$PYTHONPATH
else
ROOT_CWD=.
COMPILEDIR=`python -c "from __future__ import print_function; import theano; print(theano.config.compiledir)"|tail -1`
fi
# tests xunit for test profiling
XUNIT="--with-xunit --xunit-file="
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
# We don't want warnings in the buildbot for errors already fixed.
FLAGS=${THEANO_FLAGS},warn__argmax_pushdown_bug=False,warn__gpusum_01_011_0111_bug=False,warn__sum_sum_bug=False,warn__sum_div_dimshuffle_bug=False,warn__subtensor_merge_bug=False,$FLAGS
# We want to see correctly optimization/shape errors, so make make them raise an
# error.
FLAGS=on_opt_error=raise,$FLAGS
FLAGS=on_shape_error=raise,$FLAGS
# Ignore user device and floatX config, because:
# 1. Tests are intended to be run with device=cpu.
# 2. We explicitly add 'floatX=float32' in one run of the test suite below,
# while we want all other runs to run with 'floatX=float64'.
FLAGS=${FLAGS},device=cpu,floatX=float64
if [ "$RELEASE" ]; then
echo "Executing tests with default mode and compute_test_value"
date
THEANO_FLAGS=${FLAGS},compute_test_value=ignore pytest ${ARGS}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
echo "Executing tests with linker=vm,floatX=float32"
echo "THEANO_FLAGS=${FLAGS},linker=vm,floatX=float32 pytest ${ARGS}"
date
THEANO_FLAGS=${FLAGS},linker=vm,floatX=float32 pytest ${ARGS}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
echo "Executing tests with cxx="
echo "THEANO_FLAGS=${FLAGS},cxx= pytest ${ARGS}"
date
THEANO_FLAGS=${FLAGS},cxx= pytest ${ARGS}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
fi
echo "Executing tests with mode=FAST_RUN"
FILE=${ROOT_CWD}/theano_fastrun_tests.xml
echo "THEANO_FLAGS=cmodule__warn_no_version=True,${FLAGS},mode=FAST_RUN pytest ${PROFILING} ${ARGS} ${XUNIT}${FILE}"
date
THEANO_FLAGS=cmodule__warn_no_version=True,${FLAGS},mode=FAST_RUN pytest ${PROFILING} ${ARGS} ${XUNIT}${FILE}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
echo "Executing tests with mode=FAST_RUN,floatX=float32"
FILE=${ROOT_CWD}/theano_fastrun_float32_tests.xml
echo "THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 pytest ${ARGS} ${XUNIT}${FILE}"
date
THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 pytest ${ARGS} ${XUNIT}${FILE}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
echo "Executing tests with linker=vm,vm__lazy=True,floatX=float32"
FILE=${ROOT_CWD}/theano_fastrun_float32_lazyvm_tests.xml
echo "THEANO_FLAGS=${FLAGS},linker=vm,vm__lazy=True,floatX=float32 pytest ${ARGS} ${XUNIT}${FILE}"
date
THEANO_FLAGS=${FLAGS},linker=vm,vm__lazy=True,floatX=float32 pytest ${ARGS} ${XUNIT}${FILE}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
#we change the seed and record it everyday to test different combination. We record it to be able to reproduce bug caused by different seed. We don't want multiple test in DEBUG_MODE each day as this take too long.
seed=$RANDOM
echo "Executing tests with mode=DEBUG_MODE with seed of the day $seed"
FILE=${ROOT_CWD}/theano_debug_tests.xml
echo "THEANO_FLAGS=${FLAGS},unittests__rseed=$seed,mode=DEBUG_MODE,DebugMode__check_strides=0,DebugMode__patience=3,DebugMode__check_preallocated_output= pytest ${ARGS} ${XUNIT}${FILE}"
date
THEANO_FLAGS=${FLAGS},unittests__rseed=$seed,mode=DEBUG_MODE,DebugMode__check_strides=0,DebugMode__patience=3,DebugMode__check_preallocated_output= pytest ${ARGS} ${XUNIT}${FILE}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
#We put this at the end as it have a tendency to loop infinitly.
#Until we fix the root of the problem we let the rest run, then we can kill this one in the morning.
# with --batch=1000" # The buildbot freeze sometimes when collecting the tests to run
echo "Executing tests with mode=FAST_COMPILE"
FILE=${ROOT_CWD}/theano_fastcompile_tests.xml
echo "THEANO_FLAGS=${FLAGS},mode=FAST_COMPILE pytest ${ARGS} ${XUNIT}${FILE}"
date
THEANO_FLAGS=${FLAGS},mode=FAST_COMPILE pytest ${ARGS} ${XUNIT}${FILE}
echo "Number of elements in the compiledir:"
ls ${COMPILEDIR}|wc -l
echo
echo
END=`date +%s`
python -c "from __future__ import print_function; print('Total test time: %dm %ds'%((${END} - ${START})/60, (${END} - ${START})%60))"
date
theano/misc/do_nightly_build_send deleted 100755 → 0
浏览文件 @ 67015ca9
#!/bin/env python
# Import smtplib for the actual sending function
import os.path
import smtplib
import sys
from theano.misc.buildbot_filter import filter_output
# me == the sender's email address
# family = the list of all recipients' email addresses
family=['theano-buildbot@googlegroups.com']
me='lisa@iro.umontreal.ca'
#Those file contain the output of the do_nightly_build script.
files=["do_nightly_build_theano.log",
"do_nightly_build_deeplearning.log",
"do_nightly_build_theano_python3.3.0.log",
]
msgs=['Theano buildbot',
'Deep Learning Tutorial buildbot',
'Theano Python3.3.0 buildbot']
print('files', files)
print("msgs", msgs)
print("args", sys.argv)
if len(sys.argv) == 3:
#We send just a file with a message
files = [sys.argv[1]]
msgs = [sys.argv[2]]
elif len(sys.argv) == 2:
#This is a prefix where the output files are
files=[os.path.join(sys.argv[1], x) for x in files]
else:
files=[os.path.join('/tmp', x) for x in files]
print('path', files)
from email.mime.multipart import MIMEMultipart
# Here are the email package modules we'll need
from email.mime.text import MIMEText
COMMASPACE = ', '
def mysend(subject, file):
# Create the container (outer) email message.
if not os.path.isfile(file):
print("Error: no file", file)
return
msg = MIMEMultipart()
msg['From'] = me
msg['To'] = COMMASPACE.join(family)
msg.preamble = 'The output of the buildbot'
# Open the files in binary mode. Let the MIMEImage class automatically
# guess the specific image type.
with open(file, 'rb') as fp:
s=fp.read()
failures=0
errors=0
ran=False
nb_ran=0
skip=0
speed_failure=0
show_speed_failure=False
knownfail=0
gpu_time = None
float32_time = None
float64_time = None
for token in s.split():
token=token.strip('(,)')
if token.startswith("failures="):
failures+=int(token[9:])
elif token.startswith("errors="):
errors+=int(token[+7:])
elif token == "Ran":
ran=True
elif token.startswith("SKIP="):
skip+=int(token[5:])
elif token == "KnownFailureTest:":
# This means that KnownFailure plugin is off,
# so knownfails are also counted as errors
knownfail+=1
errors-=1
elif token.startswith("KNOWNFAIL="):
knownfail += int(token.split('=')[1])
elif token.startswith("speed_failure_"):
speed_failure+=int(token.split('=')[1])
show_speed_failure=True
elif ran:
ran=False
try:
nb_ran+=int(token)
except Exception as e:
print(e)
start = ""
for line in s.splitlines():
if gpu_time is None and line.startswith("gpu % expected/get"):
start=line
elif float32_time is None and line.startswith("float32 % expected/get"):
start=line
elif float64_time is None and line.startswith("float64 % expected/get"):
start=line
elif start:
start+=line
if start[-1]=="]":
if start.startswith("gpu % expected/get"):
gpu_time = start
start = ""
elif start.startswith("float32 % expected/get"):
float32_time = start
start = ""
elif start.startswith("float64 % expected/get"):
float64_time = start
start = ""
s = ("Summary of the output:\n\n" + filter_output(open(file)) +
"\n\nFull output:\n\n" + s)
img = MIMEText(s)
msg.attach(img)
# Send the email via our own SMTP server.
if show_speed_failure:
msg['Subject'] = subject+" Fail="+str(failures)+" Err="+str(errors)+" Ran="+str(nb_ran)+" Skip="+str(skip)+" KnownFail="+str(knownfail)+ " SpeedFailure="+str(speed_failure)
else:
msg['Subject'] = subject+" Fail="+str(failures)+" Err="+str(errors)+" Ran="+str(nb_ran)+" Skip="+str(skip)+" KnownFail="+str(knownfail)
print(msg['Subject'])
s = smtplib.SMTP()
s.connect()
s.sendmail(me, family, msg.as_string())
s.close()
print("Finished sending email for", subject)
for msg, file in zip(msgs, files):
mysend(msg, file)
theano/misc/gh_api.py deleted 100644 → 0
浏览文件 @ 67015ca9
"""Functions for Github authorisation."""
try:
input = raw_input
except NameError:
pass
import getpass
import json
import requests
# Keyring stores passwords by a 'username', but we're not storing a username and
# password
fake_username = "ipython_tools"
token = None
def get_auth_token():
global token
if token is not None:
return token
import keyring
token = keyring.get_password("github", fake_username)
if token is not None:
return token
print(
"Please enter your github username and password. These are not "
"stored, only used to get an oAuth token. You can revoke this at "
"any time on Github."
)
user = input("Username: ")
pw = getpass.getpass("Password: ")
auth_request = {
"scopes": ["public_repo", "gist"],
"note": "IPython tools",
"note_url": "https://github.com/ipython/ipython/tree/master/tools",
}
response = requests.post(
"https://api.github.com/authorizations",
auth=(user, pw),
data=json.dumps(auth_request),
)
response.raise_for_status()
token = json.loads(response.text)["token"]
keyring.set_password("github", fake_username, token)
return token
def make_auth_header():
return {"Authorization": "token " + get_auth_token()}
def post_issue_comment(project, num, body):
url = f"https://api.github.com/repos/{project}/issues/{num}/comments"
payload = json.dumps({"body": body})
requests.post(url, data=payload, headers=make_auth_header())
def post_gist(content, description="", filename="file", auth=False):
"""Post some text to a Gist, and return the URL."""
post_data = json.dumps(
{
"description": description,
"public": True,
"files": {filename: {"content": content}},
}
).encode("utf-8")
headers = make_auth_header() if auth else {}
response = requests.post(
"https://api.github.com/gists", data=post_data, headers=headers
)
response.raise_for_status()
response_data = json.loads(response.text)
return response_data["html_url"]
def get_pull_request(project, num, github_api=3):
"""get pull request info by number
github_api : version of github api to use
"""
if github_api == 2:
url = f"http://github.com/api/v2/json/pulls/{project}/{num}"
elif github_api == 3:
url = f"https://api.github.com/repos/{project}/pulls/{num}"
response = requests.get(url)
response.raise_for_status()
if github_api == 2:
return json.loads(response.text)["pull"]
return json.loads(response.text)
def get_pulls_list(project, github_api=3):
"""get pull request list
github_api : version of github api to use
"""
if github_api == 3:
url = f"https://api.github.com/repos/{project}/pulls"
else:
url = f"http://github.com/api/v2/json/pulls/{project}"
response = requests.get(url)
response.raise_for_status()
if github_api == 2:
return json.loads(response.text)["pulls"]
return json.loads(response.text)
theano/misc/hooks/check_whitespace.py deleted 100755 → 0
浏览文件 @ 67015ca9
差异被折叠。
theano/misc/hooks/reindent.py deleted 100644 → 0
浏览文件 @ 67015ca9
差异被折叠。
theano/misc/strutil.py deleted 100755 → 0
浏览文件 @ 67015ca9
def render_string(string, sub):
"""
string: a string, containing formatting instructions
sub: a dictionary containing keys and values to substitute for
them.
returns: string % sub
The only difference between this function and the % operator
is that it raises an exception with a more informative error
message than the % operator does.
"""
try:
finalCode = string % sub
except Exception as E:
# If unable to render the string, render longer and longer
# initial substrings until we find the minimal initial substring
# that causes an error
i = 0
while i <= len(string):
try:
finalCode = string[0:i] % sub
except Exception as F:
if str(F) == str(E):
raise Exception(string[0:i] + "<<<< caused exception " + str(F))
i += 1
raise AssertionError()
return finalCode
def pretty_format(string):
lines = string.split("\n")
lines = [strip_leading_white_space(line) for line in lines]
indent = 0
for i in range(len(lines)):
indent -= lines[i].count("}")
if indent < 0:
indent = 0
#
lines[i] = (" " * indent) + lines[i]
indent += lines[i].count("{")
#
rval = "\n".join(lines)
return rval
def strip_leading_white_space(line):
while len(line) > 0 and (line[0] == " " or line[0] == "\t"):
line = line[1:]
return line
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论