mirror of
https://github.com/c64scene-ar/llvm-6502.git
synced 2025-07-30 02:25:19 +00:00
lit: Sink code into a 'lit' package.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@92168 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Daniel Dunbar
575
utils/lit/lit.py
575
utils/lit/lit.py
@@ -1,576 +1,5 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
"""
|
|
||||||
lit - LLVM Integrated Tester.
|
|
||||||
|
|
||||||
See lit.pod for more information.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import math, os, platform, random, re, sys, time, threading, traceback
|
|
||||||
|
|
||||||
import ProgressBar
|
|
||||||
import TestRunner
|
|
||||||
import Util
|
|
||||||
|
|
||||||
from TestingConfig import TestingConfig
|
|
||||||
import LitConfig
|
|
||||||
import Test
|
|
||||||
|
|
||||||
# Configuration files to look for when discovering test suites. These can be
|
|
||||||
# overridden with --config-prefix.
|
|
||||||
#
|
|
||||||
# FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
|
|
||||||
gConfigName = 'lit.cfg'
|
|
||||||
gSiteConfigName = 'lit.site.cfg'
|
|
||||||
|
|
||||||
kLocalConfigName = 'lit.local.cfg'
|
|
||||||
|
|
||||||
class TestingProgressDisplay:
|
|
||||||
def __init__(self, opts, numTests, progressBar=None):
|
|
||||||
self.opts = opts
|
|
||||||
self.numTests = numTests
|
|
||||||
self.current = None
|
|
||||||
self.lock = threading.Lock()
|
|
||||||
self.progressBar = progressBar
|
|
||||||
self.completed = 0
|
|
||||||
|
|
||||||
def update(self, test):
|
|
||||||
# Avoid locking overhead in quiet mode
|
|
||||||
if self.opts.quiet and not test.result.isFailure:
|
|
||||||
self.completed += 1
|
|
||||||
return
|
|
||||||
|
|
||||||
# Output lock.
|
|
||||||
self.lock.acquire()
|
|
||||||
try:
|
|
||||||
self.handleUpdate(test)
|
|
||||||
finally:
|
|
||||||
self.lock.release()
|
|
||||||
|
|
||||||
def finish(self):
|
|
||||||
if self.progressBar:
|
|
||||||
self.progressBar.clear()
|
|
||||||
elif self.opts.quiet:
|
|
||||||
pass
|
|
||||||
elif self.opts.succinct:
|
|
||||||
sys.stdout.write('\n')
|
|
||||||
|
|
||||||
def handleUpdate(self, test):
|
|
||||||
self.completed += 1
|
|
||||||
if self.progressBar:
|
|
||||||
self.progressBar.update(float(self.completed)/self.numTests,
|
|
||||||
test.getFullName())
|
|
||||||
|
|
||||||
if self.opts.succinct and not test.result.isFailure:
|
|
||||||
return
|
|
||||||
|
|
||||||
if self.progressBar:
|
|
||||||
self.progressBar.clear()
|
|
||||||
|
|
||||||
print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
|
|
||||||
self.completed, self.numTests)
|
|
||||||
|
|
||||||
if test.result.isFailure and self.opts.showOutput:
|
|
||||||
print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
|
|
||||||
'*'*20)
|
|
||||||
print test.output
|
|
||||||
print "*" * 20
|
|
||||||
|
|
||||||
sys.stdout.flush()
|
|
||||||
|
|
||||||
class TestProvider:
|
|
||||||
def __init__(self, tests, maxTime):
|
|
||||||
self.maxTime = maxTime
|
|
||||||
self.iter = iter(tests)
|
|
||||||
self.lock = threading.Lock()
|
|
||||||
self.startTime = time.time()
|
|
||||||
|
|
||||||
def get(self):
|
|
||||||
# Check if we have run out of time.
|
|
||||||
if self.maxTime is not None:
|
|
||||||
if time.time() - self.startTime > self.maxTime:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Otherwise take the next test.
|
|
||||||
self.lock.acquire()
|
|
||||||
try:
|
|
||||||
item = self.iter.next()
|
|
||||||
except StopIteration:
|
|
||||||
item = None
|
|
||||||
self.lock.release()
|
|
||||||
return item
|
|
||||||
|
|
||||||
class Tester(threading.Thread):
|
|
||||||
def __init__(self, litConfig, provider, display):
|
|
||||||
threading.Thread.__init__(self)
|
|
||||||
self.litConfig = litConfig
|
|
||||||
self.provider = provider
|
|
||||||
self.display = display
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
while 1:
|
|
||||||
item = self.provider.get()
|
|
||||||
if item is None:
|
|
||||||
break
|
|
||||||
self.runTest(item)
|
|
||||||
|
|
||||||
def runTest(self, test):
|
|
||||||
result = None
|
|
||||||
startTime = time.time()
|
|
||||||
try:
|
|
||||||
result, output = test.config.test_format.execute(test,
|
|
||||||
self.litConfig)
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
# This is a sad hack. Unfortunately subprocess goes
|
|
||||||
# bonkers with ctrl-c and we start forking merrily.
|
|
||||||
print '\nCtrl-C detected, goodbye.'
|
|
||||||
os.kill(0,9)
|
|
||||||
except:
|
|
||||||
if self.litConfig.debug:
|
|
||||||
raise
|
|
||||||
result = Test.UNRESOLVED
|
|
||||||
output = 'Exception during script execution:\n'
|
|
||||||
output += traceback.format_exc()
|
|
||||||
output += '\n'
|
|
||||||
elapsed = time.time() - startTime
|
|
||||||
|
|
||||||
test.setResult(result, output, elapsed)
|
|
||||||
self.display.update(test)
|
|
||||||
|
|
||||||
def dirContainsTestSuite(path):
|
|
||||||
cfgpath = os.path.join(path, gSiteConfigName)
|
|
||||||
if os.path.exists(cfgpath):
|
|
||||||
return cfgpath
|
|
||||||
cfgpath = os.path.join(path, gConfigName)
|
|
||||||
if os.path.exists(cfgpath):
|
|
||||||
return cfgpath
|
|
||||||
|
|
||||||
def getTestSuite(item, litConfig, cache):
|
|
||||||
"""getTestSuite(item, litConfig, cache) -> (suite, relative_path)
|
|
||||||
|
|
||||||
Find the test suite containing @arg item.
|
|
||||||
|
|
||||||
@retval (None, ...) - Indicates no test suite contains @arg item.
|
|
||||||
@retval (suite, relative_path) - The suite that @arg item is in, and its
|
|
||||||
relative path inside that suite.
|
|
||||||
"""
|
|
||||||
def search1(path):
|
|
||||||
# Check for a site config or a lit config.
|
|
||||||
cfgpath = dirContainsTestSuite(path)
|
|
||||||
|
|
||||||
# If we didn't find a config file, keep looking.
|
|
||||||
if not cfgpath:
|
|
||||||
parent,base = os.path.split(path)
|
|
||||||
if parent == path:
|
|
||||||
return (None, ())
|
|
||||||
|
|
||||||
ts, relative = search(parent)
|
|
||||||
return (ts, relative + (base,))
|
|
||||||
|
|
||||||
# We found a config file, load it.
|
|
||||||
if litConfig.debug:
|
|
||||||
litConfig.note('loading suite config %r' % cfgpath)
|
|
||||||
|
|
||||||
cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
|
|
||||||
source_root = os.path.realpath(cfg.test_source_root or path)
|
|
||||||
exec_root = os.path.realpath(cfg.test_exec_root or path)
|
|
||||||
return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
|
|
||||||
|
|
||||||
def search(path):
|
|
||||||
# Check for an already instantiated test suite.
|
|
||||||
res = cache.get(path)
|
|
||||||
if res is None:
|
|
||||||
cache[path] = res = search1(path)
|
|
||||||
return res
|
|
||||||
|
|
||||||
# Canonicalize the path.
|
|
||||||
item = os.path.realpath(item)
|
|
||||||
|
|
||||||
# Skip files and virtual components.
|
|
||||||
components = []
|
|
||||||
while not os.path.isdir(item):
|
|
||||||
parent,base = os.path.split(item)
|
|
||||||
if parent == item:
|
|
||||||
return (None, ())
|
|
||||||
components.append(base)
|
|
||||||
item = parent
|
|
||||||
components.reverse()
|
|
||||||
|
|
||||||
ts, relative = search(item)
|
|
||||||
return ts, tuple(relative + tuple(components))
|
|
||||||
|
|
||||||
def getLocalConfig(ts, path_in_suite, litConfig, cache):
|
|
||||||
def search1(path_in_suite):
|
|
||||||
# Get the parent config.
|
|
||||||
if not path_in_suite:
|
|
||||||
parent = ts.config
|
|
||||||
else:
|
|
||||||
parent = search(path_in_suite[:-1])
|
|
||||||
|
|
||||||
# Load the local configuration.
|
|
||||||
source_path = ts.getSourcePath(path_in_suite)
|
|
||||||
cfgpath = os.path.join(source_path, kLocalConfigName)
|
|
||||||
if litConfig.debug:
|
|
||||||
litConfig.note('loading local config %r' % cfgpath)
|
|
||||||
return TestingConfig.frompath(cfgpath, parent, litConfig,
|
|
||||||
mustExist = False,
|
|
||||||
config = parent.clone(cfgpath))
|
|
||||||
|
|
||||||
def search(path_in_suite):
|
|
||||||
key = (ts, path_in_suite)
|
|
||||||
res = cache.get(key)
|
|
||||||
if res is None:
|
|
||||||
cache[key] = res = search1(path_in_suite)
|
|
||||||
return res
|
|
||||||
|
|
||||||
return search(path_in_suite)
|
|
||||||
|
|
||||||
def getTests(path, litConfig, testSuiteCache, localConfigCache):
|
|
||||||
# Find the test suite for this input and its relative path.
|
|
||||||
ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
|
|
||||||
if ts is None:
|
|
||||||
litConfig.warning('unable to find test suite for %r' % path)
|
|
||||||
return (),()
|
|
||||||
|
|
||||||
if litConfig.debug:
|
|
||||||
litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
|
|
||||||
path_in_suite))
|
|
||||||
|
|
||||||
return ts, getTestsInSuite(ts, path_in_suite, litConfig,
|
|
||||||
testSuiteCache, localConfigCache)
|
|
||||||
|
|
||||||
def getTestsInSuite(ts, path_in_suite, litConfig,
|
|
||||||
testSuiteCache, localConfigCache):
|
|
||||||
# Check that the source path exists (errors here are reported by the
|
|
||||||
# caller).
|
|
||||||
source_path = ts.getSourcePath(path_in_suite)
|
|
||||||
if not os.path.exists(source_path):
|
|
||||||
return
|
|
||||||
|
|
||||||
# Check if the user named a test directly.
|
|
||||||
if not os.path.isdir(source_path):
|
|
||||||
lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
|
|
||||||
yield Test.Test(ts, path_in_suite, lc)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Otherwise we have a directory to search for tests, start by getting the
|
|
||||||
# local configuration.
|
|
||||||
lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
|
|
||||||
|
|
||||||
# Search for tests.
|
|
||||||
for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
|
|
||||||
litConfig, lc):
|
|
||||||
yield res
|
|
||||||
|
|
||||||
# Search subdirectories.
|
|
||||||
for filename in os.listdir(source_path):
|
|
||||||
# FIXME: This doesn't belong here?
|
|
||||||
if filename in ('Output', '.svn') or filename in lc.excludes:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Ignore non-directories.
|
|
||||||
file_sourcepath = os.path.join(source_path, filename)
|
|
||||||
if not os.path.isdir(file_sourcepath):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Check for nested test suites, first in the execpath in case there is a
|
|
||||||
# site configuration and then in the source path.
|
|
||||||
file_execpath = ts.getExecPath(path_in_suite + (filename,))
|
|
||||||
if dirContainsTestSuite(file_execpath):
|
|
||||||
sub_ts, subiter = getTests(file_execpath, litConfig,
|
|
||||||
testSuiteCache, localConfigCache)
|
|
||||||
elif dirContainsTestSuite(file_sourcepath):
|
|
||||||
sub_ts, subiter = getTests(file_sourcepath, litConfig,
|
|
||||||
testSuiteCache, localConfigCache)
|
|
||||||
else:
|
|
||||||
# Otherwise, continue loading from inside this test suite.
|
|
||||||
subiter = getTestsInSuite(ts, path_in_suite + (filename,),
|
|
||||||
litConfig, testSuiteCache,
|
|
||||||
localConfigCache)
|
|
||||||
sub_ts = None
|
|
||||||
|
|
||||||
N = 0
|
|
||||||
for res in subiter:
|
|
||||||
N += 1
|
|
||||||
yield res
|
|
||||||
if sub_ts and not N:
|
|
||||||
litConfig.warning('test suite %r contained no tests' % sub_ts.name)
|
|
||||||
|
|
||||||
def runTests(numThreads, litConfig, provider, display):
|
|
||||||
# If only using one testing thread, don't use threads at all; this lets us
|
|
||||||
# profile, among other things.
|
|
||||||
if numThreads == 1:
|
|
||||||
t = Tester(litConfig, provider, display)
|
|
||||||
t.run()
|
|
||||||
return
|
|
||||||
|
|
||||||
# Otherwise spin up the testing threads and wait for them to finish.
|
|
||||||
testers = [Tester(litConfig, provider, display)
|
|
||||||
for i in range(numThreads)]
|
|
||||||
for t in testers:
|
|
||||||
t.start()
|
|
||||||
try:
|
|
||||||
for t in testers:
|
|
||||||
t.join()
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
sys.exit(2)
|
|
||||||
|
|
||||||
def main():
|
|
||||||
global options
|
|
||||||
from optparse import OptionParser, OptionGroup
|
|
||||||
parser = OptionParser("usage: %prog [options] {file-or-path}")
|
|
||||||
|
|
||||||
parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
|
|
||||||
help="Number of testing threads",
|
|
||||||
type=int, action="store", default=None)
|
|
||||||
parser.add_option("", "--config-prefix", dest="configPrefix",
|
|
||||||
metavar="NAME", help="Prefix for 'lit' config files",
|
|
||||||
action="store", default=None)
|
|
||||||
parser.add_option("", "--param", dest="userParameters",
|
|
||||||
metavar="NAME=VAL",
|
|
||||||
help="Add 'NAME' = 'VAL' to the user defined parameters",
|
|
||||||
type=str, action="append", default=[])
|
|
||||||
|
|
||||||
group = OptionGroup(parser, "Output Format")
|
|
||||||
# FIXME: I find these names very confusing, although I like the
|
|
||||||
# functionality.
|
|
||||||
group.add_option("-q", "--quiet", dest="quiet",
|
|
||||||
help="Suppress no error output",
|
|
||||||
action="store_true", default=False)
|
|
||||||
group.add_option("-s", "--succinct", dest="succinct",
|
|
||||||
help="Reduce amount of output",
|
|
||||||
action="store_true", default=False)
|
|
||||||
group.add_option("-v", "--verbose", dest="showOutput",
|
|
||||||
help="Show all test output",
|
|
||||||
action="store_true", default=False)
|
|
||||||
group.add_option("", "--no-progress-bar", dest="useProgressBar",
|
|
||||||
help="Do not use curses based progress bar",
|
|
||||||
action="store_false", default=True)
|
|
||||||
parser.add_option_group(group)
|
|
||||||
|
|
||||||
group = OptionGroup(parser, "Test Execution")
|
|
||||||
group.add_option("", "--path", dest="path",
|
|
||||||
help="Additional paths to add to testing environment",
|
|
||||||
action="append", type=str, default=[])
|
|
||||||
group.add_option("", "--vg", dest="useValgrind",
|
|
||||||
help="Run tests under valgrind",
|
|
||||||
action="store_true", default=False)
|
|
||||||
group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
|
|
||||||
help="Specify an extra argument for valgrind",
|
|
||||||
type=str, action="append", default=[])
|
|
||||||
group.add_option("", "--time-tests", dest="timeTests",
|
|
||||||
help="Track elapsed wall time for each test",
|
|
||||||
action="store_true", default=False)
|
|
||||||
group.add_option("", "--no-execute", dest="noExecute",
|
|
||||||
help="Don't execute any tests (assume PASS)",
|
|
||||||
action="store_true", default=False)
|
|
||||||
parser.add_option_group(group)
|
|
||||||
|
|
||||||
group = OptionGroup(parser, "Test Selection")
|
|
||||||
group.add_option("", "--max-tests", dest="maxTests", metavar="N",
|
|
||||||
help="Maximum number of tests to run",
|
|
||||||
action="store", type=int, default=None)
|
|
||||||
group.add_option("", "--max-time", dest="maxTime", metavar="N",
|
|
||||||
help="Maximum time to spend testing (in seconds)",
|
|
||||||
action="store", type=float, default=None)
|
|
||||||
group.add_option("", "--shuffle", dest="shuffle",
|
|
||||||
help="Run tests in random order",
|
|
||||||
action="store_true", default=False)
|
|
||||||
parser.add_option_group(group)
|
|
||||||
|
|
||||||
group = OptionGroup(parser, "Debug and Experimental Options")
|
|
||||||
group.add_option("", "--debug", dest="debug",
|
|
||||||
help="Enable debugging (for 'lit' development)",
|
|
||||||
action="store_true", default=False)
|
|
||||||
group.add_option("", "--show-suites", dest="showSuites",
|
|
||||||
help="Show discovered test suites",
|
|
||||||
action="store_true", default=False)
|
|
||||||
group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
|
|
||||||
help="Don't run Tcl scripts using 'sh'",
|
|
||||||
action="store_false", default=True)
|
|
||||||
group.add_option("", "--repeat", dest="repeatTests", metavar="N",
|
|
||||||
help="Repeat tests N times (for timing)",
|
|
||||||
action="store", default=None, type=int)
|
|
||||||
parser.add_option_group(group)
|
|
||||||
|
|
||||||
(opts, args) = parser.parse_args()
|
|
||||||
|
|
||||||
if not args:
|
|
||||||
parser.error('No inputs specified')
|
|
||||||
|
|
||||||
if opts.configPrefix is not None:
|
|
||||||
global gConfigName, gSiteConfigName
|
|
||||||
gConfigName = '%s.cfg' % opts.configPrefix
|
|
||||||
gSiteConfigName = '%s.site.cfg' % opts.configPrefix
|
|
||||||
|
|
||||||
if opts.numThreads is None:
|
|
||||||
opts.numThreads = Util.detectCPUs()
|
|
||||||
|
|
||||||
inputs = args
|
|
||||||
|
|
||||||
# Create the user defined parameters.
|
|
||||||
userParams = {}
|
|
||||||
for entry in opts.userParameters:
|
|
||||||
if '=' not in entry:
|
|
||||||
name,val = entry,''
|
|
||||||
else:
|
|
||||||
name,val = entry.split('=', 1)
|
|
||||||
userParams[name] = val
|
|
||||||
|
|
||||||
# Create the global config object.
|
|
||||||
litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
|
|
||||||
path = opts.path,
|
|
||||||
quiet = opts.quiet,
|
|
||||||
useValgrind = opts.useValgrind,
|
|
||||||
valgrindArgs = opts.valgrindArgs,
|
|
||||||
useTclAsSh = opts.useTclAsSh,
|
|
||||||
noExecute = opts.noExecute,
|
|
||||||
debug = opts.debug,
|
|
||||||
isWindows = (platform.system()=='Windows'),
|
|
||||||
params = userParams)
|
|
||||||
|
|
||||||
# Load the tests from the inputs.
|
|
||||||
tests = []
|
|
||||||
testSuiteCache = {}
|
|
||||||
localConfigCache = {}
|
|
||||||
for input in inputs:
|
|
||||||
prev = len(tests)
|
|
||||||
tests.extend(getTests(input, litConfig,
|
|
||||||
testSuiteCache, localConfigCache)[1])
|
|
||||||
if prev == len(tests):
|
|
||||||
litConfig.warning('input %r contained no tests' % input)
|
|
||||||
|
|
||||||
# If there were any errors during test discovery, exit now.
|
|
||||||
if litConfig.numErrors:
|
|
||||||
print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
|
|
||||||
sys.exit(2)
|
|
||||||
|
|
||||||
if opts.showSuites:
|
|
||||||
suitesAndTests = dict([(ts,[])
|
|
||||||
for ts,_ in testSuiteCache.values()
|
|
||||||
if ts])
|
|
||||||
for t in tests:
|
|
||||||
suitesAndTests[t.suite].append(t)
|
|
||||||
|
|
||||||
print '-- Test Suites --'
|
|
||||||
suitesAndTests = suitesAndTests.items()
|
|
||||||
suitesAndTests.sort(key = lambda (ts,_): ts.name)
|
|
||||||
for ts,ts_tests in suitesAndTests:
|
|
||||||
print ' %s - %d tests' %(ts.name, len(ts_tests))
|
|
||||||
print ' Source Root: %s' % ts.source_root
|
|
||||||
print ' Exec Root : %s' % ts.exec_root
|
|
||||||
|
|
||||||
# Select and order the tests.
|
|
||||||
numTotalTests = len(tests)
|
|
||||||
if opts.shuffle:
|
|
||||||
random.shuffle(tests)
|
|
||||||
else:
|
|
||||||
tests.sort(key = lambda t: t.getFullName())
|
|
||||||
if opts.maxTests is not None:
|
|
||||||
tests = tests[:opts.maxTests]
|
|
||||||
|
|
||||||
extra = ''
|
|
||||||
if len(tests) != numTotalTests:
|
|
||||||
extra = ' of %d' % numTotalTests
|
|
||||||
header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
|
|
||||||
opts.numThreads)
|
|
||||||
|
|
||||||
if opts.repeatTests:
|
|
||||||
tests = [t.copyWithIndex(i)
|
|
||||||
for t in tests
|
|
||||||
for i in range(opts.repeatTests)]
|
|
||||||
|
|
||||||
progressBar = None
|
|
||||||
if not opts.quiet:
|
|
||||||
if opts.succinct and opts.useProgressBar:
|
|
||||||
try:
|
|
||||||
tc = ProgressBar.TerminalController()
|
|
||||||
progressBar = ProgressBar.ProgressBar(tc, header)
|
|
||||||
except ValueError:
|
|
||||||
print header
|
|
||||||
progressBar = ProgressBar.SimpleProgressBar('Testing: ')
|
|
||||||
else:
|
|
||||||
print header
|
|
||||||
|
|
||||||
# Don't create more threads than tests.
|
|
||||||
opts.numThreads = min(len(tests), opts.numThreads)
|
|
||||||
|
|
||||||
startTime = time.time()
|
|
||||||
display = TestingProgressDisplay(opts, len(tests), progressBar)
|
|
||||||
provider = TestProvider(tests, opts.maxTime)
|
|
||||||
runTests(opts.numThreads, litConfig, provider, display)
|
|
||||||
display.finish()
|
|
||||||
|
|
||||||
if not opts.quiet:
|
|
||||||
print 'Testing Time: %.2fs'%(time.time() - startTime)
|
|
||||||
|
|
||||||
# Update results for any tests which weren't run.
|
|
||||||
for t in tests:
|
|
||||||
if t.result is None:
|
|
||||||
t.setResult(Test.UNRESOLVED, '', 0.0)
|
|
||||||
|
|
||||||
# List test results organized by kind.
|
|
||||||
hasFailures = False
|
|
||||||
byCode = {}
|
|
||||||
for t in tests:
|
|
||||||
if t.result not in byCode:
|
|
||||||
byCode[t.result] = []
|
|
||||||
byCode[t.result].append(t)
|
|
||||||
if t.result.isFailure:
|
|
||||||
hasFailures = True
|
|
||||||
|
|
||||||
# FIXME: Show unresolved and (optionally) unsupported tests.
|
|
||||||
for title,code in (('Unexpected Passing Tests', Test.XPASS),
|
|
||||||
('Failing Tests', Test.FAIL)):
|
|
||||||
elts = byCode.get(code)
|
|
||||||
if not elts:
|
|
||||||
continue
|
|
||||||
print '*'*20
|
|
||||||
print '%s (%d):' % (title, len(elts))
|
|
||||||
for t in elts:
|
|
||||||
print ' %s' % t.getFullName()
|
|
||||||
print
|
|
||||||
|
|
||||||
if opts.timeTests:
|
|
||||||
# Collate, in case we repeated tests.
|
|
||||||
times = {}
|
|
||||||
for t in tests:
|
|
||||||
key = t.getFullName()
|
|
||||||
times[key] = times.get(key, 0.) + t.elapsed
|
|
||||||
|
|
||||||
byTime = list(times.items())
|
|
||||||
byTime.sort(key = lambda (name,elapsed): elapsed)
|
|
||||||
if byTime:
|
|
||||||
Util.printHistogram(byTime, title='Tests')
|
|
||||||
|
|
||||||
for name,code in (('Expected Passes ', Test.PASS),
|
|
||||||
('Expected Failures ', Test.XFAIL),
|
|
||||||
('Unsupported Tests ', Test.UNSUPPORTED),
|
|
||||||
('Unresolved Tests ', Test.UNRESOLVED),
|
|
||||||
('Unexpected Passes ', Test.XPASS),
|
|
||||||
('Unexpected Failures', Test.FAIL),):
|
|
||||||
if opts.quiet and not code.isFailure:
|
|
||||||
continue
|
|
||||||
N = len(byCode.get(code,[]))
|
|
||||||
if N:
|
|
||||||
print ' %s: %d' % (name,N)
|
|
||||||
|
|
||||||
# If we encountered any additional errors, exit abnormally.
|
|
||||||
if litConfig.numErrors:
|
|
||||||
print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
|
|
||||||
sys.exit(2)
|
|
||||||
|
|
||||||
# Warn about warnings.
|
|
||||||
if litConfig.numWarnings:
|
|
||||||
print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
|
|
||||||
|
|
||||||
if hasFailures:
|
|
||||||
sys.exit(1)
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
if __name__=='__main__':
|
if __name__=='__main__':
|
||||||
# Bump the GIL check interval, its more important to get any one thread to a
|
import lit
|
||||||
# blocking operation (hopefully exec) than to try and unblock other threads.
|
lit.main()
|
||||||
import sys
|
|
||||||
sys.setcheckinterval(1000)
|
|
||||||
main()
|
|
||||||
|
|||||||
10
utils/lit/lit/__init__.py
Normal file
10
utils/lit/lit/__init__.py
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
"""'lit' Testing Tool"""
|
||||||
|
|
||||||
|
from lit import main
|
||||||
|
|
||||||
|
__author__ = 'Daniel Dunbar'
|
||||||
|
__email__ = 'daniel@zuster.org'
|
||||||
|
__versioninfo__ = (0, 1, 0)
|
||||||
|
__version__ = '.'.join(map(str, __versioninfo__))
|
||||||
|
|
||||||
|
__all__ = []
|
||||||
579
utils/lit/lit/lit.py
Executable file
579
utils/lit/lit/lit.py
Executable file
@@ -0,0 +1,579 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
"""
|
||||||
|
lit - LLVM Integrated Tester.
|
||||||
|
|
||||||
|
See lit.pod for more information.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import math, os, platform, random, re, sys, time, threading, traceback
|
||||||
|
|
||||||
|
import ProgressBar
|
||||||
|
import TestRunner
|
||||||
|
import Util
|
||||||
|
|
||||||
|
from TestingConfig import TestingConfig
|
||||||
|
import LitConfig
|
||||||
|
import Test
|
||||||
|
|
||||||
|
# Configuration files to look for when discovering test suites. These can be
|
||||||
|
# overridden with --config-prefix.
|
||||||
|
#
|
||||||
|
# FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
|
||||||
|
gConfigName = 'lit.cfg'
|
||||||
|
gSiteConfigName = 'lit.site.cfg'
|
||||||
|
|
||||||
|
kLocalConfigName = 'lit.local.cfg'
|
||||||
|
|
||||||
|
class TestingProgressDisplay:
|
||||||
|
def __init__(self, opts, numTests, progressBar=None):
|
||||||
|
self.opts = opts
|
||||||
|
self.numTests = numTests
|
||||||
|
self.current = None
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
self.progressBar = progressBar
|
||||||
|
self.completed = 0
|
||||||
|
|
||||||
|
def update(self, test):
|
||||||
|
# Avoid locking overhead in quiet mode
|
||||||
|
if self.opts.quiet and not test.result.isFailure:
|
||||||
|
self.completed += 1
|
||||||
|
return
|
||||||
|
|
||||||
|
# Output lock.
|
||||||
|
self.lock.acquire()
|
||||||
|
try:
|
||||||
|
self.handleUpdate(test)
|
||||||
|
finally:
|
||||||
|
self.lock.release()
|
||||||
|
|
||||||
|
def finish(self):
|
||||||
|
if self.progressBar:
|
||||||
|
self.progressBar.clear()
|
||||||
|
elif self.opts.quiet:
|
||||||
|
pass
|
||||||
|
elif self.opts.succinct:
|
||||||
|
sys.stdout.write('\n')
|
||||||
|
|
||||||
|
def handleUpdate(self, test):
|
||||||
|
self.completed += 1
|
||||||
|
if self.progressBar:
|
||||||
|
self.progressBar.update(float(self.completed)/self.numTests,
|
||||||
|
test.getFullName())
|
||||||
|
|
||||||
|
if self.opts.succinct and not test.result.isFailure:
|
||||||
|
return
|
||||||
|
|
||||||
|
if self.progressBar:
|
||||||
|
self.progressBar.clear()
|
||||||
|
|
||||||
|
print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
|
||||||
|
self.completed, self.numTests)
|
||||||
|
|
||||||
|
if test.result.isFailure and self.opts.showOutput:
|
||||||
|
print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
|
||||||
|
'*'*20)
|
||||||
|
print test.output
|
||||||
|
print "*" * 20
|
||||||
|
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
class TestProvider:
|
||||||
|
def __init__(self, tests, maxTime):
|
||||||
|
self.maxTime = maxTime
|
||||||
|
self.iter = iter(tests)
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
self.startTime = time.time()
|
||||||
|
|
||||||
|
def get(self):
|
||||||
|
# Check if we have run out of time.
|
||||||
|
if self.maxTime is not None:
|
||||||
|
if time.time() - self.startTime > self.maxTime:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Otherwise take the next test.
|
||||||
|
self.lock.acquire()
|
||||||
|
try:
|
||||||
|
item = self.iter.next()
|
||||||
|
except StopIteration:
|
||||||
|
item = None
|
||||||
|
self.lock.release()
|
||||||
|
return item
|
||||||
|
|
||||||
|
class Tester(threading.Thread):
|
||||||
|
def __init__(self, litConfig, provider, display):
|
||||||
|
threading.Thread.__init__(self)
|
||||||
|
self.litConfig = litConfig
|
||||||
|
self.provider = provider
|
||||||
|
self.display = display
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
while 1:
|
||||||
|
item = self.provider.get()
|
||||||
|
if item is None:
|
||||||
|
break
|
||||||
|
self.runTest(item)
|
||||||
|
|
||||||
|
def runTest(self, test):
|
||||||
|
result = None
|
||||||
|
startTime = time.time()
|
||||||
|
try:
|
||||||
|
result, output = test.config.test_format.execute(test,
|
||||||
|
self.litConfig)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
# This is a sad hack. Unfortunately subprocess goes
|
||||||
|
# bonkers with ctrl-c and we start forking merrily.
|
||||||
|
print '\nCtrl-C detected, goodbye.'
|
||||||
|
os.kill(0,9)
|
||||||
|
except:
|
||||||
|
if self.litConfig.debug:
|
||||||
|
raise
|
||||||
|
result = Test.UNRESOLVED
|
||||||
|
output = 'Exception during script execution:\n'
|
||||||
|
output += traceback.format_exc()
|
||||||
|
output += '\n'
|
||||||
|
elapsed = time.time() - startTime
|
||||||
|
|
||||||
|
test.setResult(result, output, elapsed)
|
||||||
|
self.display.update(test)
|
||||||
|
|
||||||
|
def dirContainsTestSuite(path):
|
||||||
|
cfgpath = os.path.join(path, gSiteConfigName)
|
||||||
|
if os.path.exists(cfgpath):
|
||||||
|
return cfgpath
|
||||||
|
cfgpath = os.path.join(path, gConfigName)
|
||||||
|
if os.path.exists(cfgpath):
|
||||||
|
return cfgpath
|
||||||
|
|
||||||
|
def getTestSuite(item, litConfig, cache):
|
||||||
|
"""getTestSuite(item, litConfig, cache) -> (suite, relative_path)
|
||||||
|
|
||||||
|
Find the test suite containing @arg item.
|
||||||
|
|
||||||
|
@retval (None, ...) - Indicates no test suite contains @arg item.
|
||||||
|
@retval (suite, relative_path) - The suite that @arg item is in, and its
|
||||||
|
relative path inside that suite.
|
||||||
|
"""
|
||||||
|
def search1(path):
|
||||||
|
# Check for a site config or a lit config.
|
||||||
|
cfgpath = dirContainsTestSuite(path)
|
||||||
|
|
||||||
|
# If we didn't find a config file, keep looking.
|
||||||
|
if not cfgpath:
|
||||||
|
parent,base = os.path.split(path)
|
||||||
|
if parent == path:
|
||||||
|
return (None, ())
|
||||||
|
|
||||||
|
ts, relative = search(parent)
|
||||||
|
return (ts, relative + (base,))
|
||||||
|
|
||||||
|
# We found a config file, load it.
|
||||||
|
if litConfig.debug:
|
||||||
|
litConfig.note('loading suite config %r' % cfgpath)
|
||||||
|
|
||||||
|
cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
|
||||||
|
source_root = os.path.realpath(cfg.test_source_root or path)
|
||||||
|
exec_root = os.path.realpath(cfg.test_exec_root or path)
|
||||||
|
return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
|
||||||
|
|
||||||
|
def search(path):
|
||||||
|
# Check for an already instantiated test suite.
|
||||||
|
res = cache.get(path)
|
||||||
|
if res is None:
|
||||||
|
cache[path] = res = search1(path)
|
||||||
|
return res
|
||||||
|
|
||||||
|
# Canonicalize the path.
|
||||||
|
item = os.path.realpath(item)
|
||||||
|
|
||||||
|
# Skip files and virtual components.
|
||||||
|
components = []
|
||||||
|
while not os.path.isdir(item):
|
||||||
|
parent,base = os.path.split(item)
|
||||||
|
if parent == item:
|
||||||
|
return (None, ())
|
||||||
|
components.append(base)
|
||||||
|
item = parent
|
||||||
|
components.reverse()
|
||||||
|
|
||||||
|
ts, relative = search(item)
|
||||||
|
return ts, tuple(relative + tuple(components))
|
||||||
|
|
||||||
|
def getLocalConfig(ts, path_in_suite, litConfig, cache):
|
||||||
|
def search1(path_in_suite):
|
||||||
|
# Get the parent config.
|
||||||
|
if not path_in_suite:
|
||||||
|
parent = ts.config
|
||||||
|
else:
|
||||||
|
parent = search(path_in_suite[:-1])
|
||||||
|
|
||||||
|
# Load the local configuration.
|
||||||
|
source_path = ts.getSourcePath(path_in_suite)
|
||||||
|
cfgpath = os.path.join(source_path, kLocalConfigName)
|
||||||
|
if litConfig.debug:
|
||||||
|
litConfig.note('loading local config %r' % cfgpath)
|
||||||
|
return TestingConfig.frompath(cfgpath, parent, litConfig,
|
||||||
|
mustExist = False,
|
||||||
|
config = parent.clone(cfgpath))
|
||||||
|
|
||||||
|
def search(path_in_suite):
|
||||||
|
key = (ts, path_in_suite)
|
||||||
|
res = cache.get(key)
|
||||||
|
if res is None:
|
||||||
|
cache[key] = res = search1(path_in_suite)
|
||||||
|
return res
|
||||||
|
|
||||||
|
return search(path_in_suite)
|
||||||
|
|
||||||
|
def getTests(path, litConfig, testSuiteCache, localConfigCache):
|
||||||
|
# Find the test suite for this input and its relative path.
|
||||||
|
ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
|
||||||
|
if ts is None:
|
||||||
|
litConfig.warning('unable to find test suite for %r' % path)
|
||||||
|
return (),()
|
||||||
|
|
||||||
|
if litConfig.debug:
|
||||||
|
litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
|
||||||
|
path_in_suite))
|
||||||
|
|
||||||
|
return ts, getTestsInSuite(ts, path_in_suite, litConfig,
|
||||||
|
testSuiteCache, localConfigCache)
|
||||||
|
|
||||||
|
def getTestsInSuite(ts, path_in_suite, litConfig,
|
||||||
|
testSuiteCache, localConfigCache):
|
||||||
|
# Check that the source path exists (errors here are reported by the
|
||||||
|
# caller).
|
||||||
|
source_path = ts.getSourcePath(path_in_suite)
|
||||||
|
if not os.path.exists(source_path):
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check if the user named a test directly.
|
||||||
|
if not os.path.isdir(source_path):
|
||||||
|
lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
|
||||||
|
yield Test.Test(ts, path_in_suite, lc)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Otherwise we have a directory to search for tests, start by getting the
|
||||||
|
# local configuration.
|
||||||
|
lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
|
||||||
|
|
||||||
|
# Search for tests.
|
||||||
|
for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
|
||||||
|
litConfig, lc):
|
||||||
|
yield res
|
||||||
|
|
||||||
|
# Search subdirectories.
|
||||||
|
for filename in os.listdir(source_path):
|
||||||
|
# FIXME: This doesn't belong here?
|
||||||
|
if filename in ('Output', '.svn') or filename in lc.excludes:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Ignore non-directories.
|
||||||
|
file_sourcepath = os.path.join(source_path, filename)
|
||||||
|
if not os.path.isdir(file_sourcepath):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check for nested test suites, first in the execpath in case there is a
|
||||||
|
# site configuration and then in the source path.
|
||||||
|
file_execpath = ts.getExecPath(path_in_suite + (filename,))
|
||||||
|
if dirContainsTestSuite(file_execpath):
|
||||||
|
sub_ts, subiter = getTests(file_execpath, litConfig,
|
||||||
|
testSuiteCache, localConfigCache)
|
||||||
|
elif dirContainsTestSuite(file_sourcepath):
|
||||||
|
sub_ts, subiter = getTests(file_sourcepath, litConfig,
|
||||||
|
testSuiteCache, localConfigCache)
|
||||||
|
else:
|
||||||
|
# Otherwise, continue loading from inside this test suite.
|
||||||
|
subiter = getTestsInSuite(ts, path_in_suite + (filename,),
|
||||||
|
litConfig, testSuiteCache,
|
||||||
|
localConfigCache)
|
||||||
|
sub_ts = None
|
||||||
|
|
||||||
|
N = 0
|
||||||
|
for res in subiter:
|
||||||
|
N += 1
|
||||||
|
yield res
|
||||||
|
if sub_ts and not N:
|
||||||
|
litConfig.warning('test suite %r contained no tests' % sub_ts.name)
|
||||||
|
|
||||||
|
def runTests(numThreads, litConfig, provider, display):
|
||||||
|
# If only using one testing thread, don't use threads at all; this lets us
|
||||||
|
# profile, among other things.
|
||||||
|
if numThreads == 1:
|
||||||
|
t = Tester(litConfig, provider, display)
|
||||||
|
t.run()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Otherwise spin up the testing threads and wait for them to finish.
|
||||||
|
testers = [Tester(litConfig, provider, display)
|
||||||
|
for i in range(numThreads)]
|
||||||
|
for t in testers:
|
||||||
|
t.start()
|
||||||
|
try:
|
||||||
|
for t in testers:
|
||||||
|
t.join()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# Bump the GIL check interval, its more important to get any one thread to a
|
||||||
|
# blocking operation (hopefully exec) than to try and unblock other threads.
|
||||||
|
#
|
||||||
|
# FIXME: This is a hack.
|
||||||
|
import sys
|
||||||
|
sys.setcheckinterval(1000)
|
||||||
|
|
||||||
|
global options
|
||||||
|
from optparse import OptionParser, OptionGroup
|
||||||
|
parser = OptionParser("usage: %prog [options] {file-or-path}")
|
||||||
|
|
||||||
|
parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
|
||||||
|
help="Number of testing threads",
|
||||||
|
type=int, action="store", default=None)
|
||||||
|
parser.add_option("", "--config-prefix", dest="configPrefix",
|
||||||
|
metavar="NAME", help="Prefix for 'lit' config files",
|
||||||
|
action="store", default=None)
|
||||||
|
parser.add_option("", "--param", dest="userParameters",
|
||||||
|
metavar="NAME=VAL",
|
||||||
|
help="Add 'NAME' = 'VAL' to the user defined parameters",
|
||||||
|
type=str, action="append", default=[])
|
||||||
|
|
||||||
|
group = OptionGroup(parser, "Output Format")
|
||||||
|
# FIXME: I find these names very confusing, although I like the
|
||||||
|
# functionality.
|
||||||
|
group.add_option("-q", "--quiet", dest="quiet",
|
||||||
|
help="Suppress no error output",
|
||||||
|
action="store_true", default=False)
|
||||||
|
group.add_option("-s", "--succinct", dest="succinct",
|
||||||
|
help="Reduce amount of output",
|
||||||
|
action="store_true", default=False)
|
||||||
|
group.add_option("-v", "--verbose", dest="showOutput",
|
||||||
|
help="Show all test output",
|
||||||
|
action="store_true", default=False)
|
||||||
|
group.add_option("", "--no-progress-bar", dest="useProgressBar",
|
||||||
|
help="Do not use curses based progress bar",
|
||||||
|
action="store_false", default=True)
|
||||||
|
parser.add_option_group(group)
|
||||||
|
|
||||||
|
group = OptionGroup(parser, "Test Execution")
|
||||||
|
group.add_option("", "--path", dest="path",
|
||||||
|
help="Additional paths to add to testing environment",
|
||||||
|
action="append", type=str, default=[])
|
||||||
|
group.add_option("", "--vg", dest="useValgrind",
|
||||||
|
help="Run tests under valgrind",
|
||||||
|
action="store_true", default=False)
|
||||||
|
group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
|
||||||
|
help="Specify an extra argument for valgrind",
|
||||||
|
type=str, action="append", default=[])
|
||||||
|
group.add_option("", "--time-tests", dest="timeTests",
|
||||||
|
help="Track elapsed wall time for each test",
|
||||||
|
action="store_true", default=False)
|
||||||
|
group.add_option("", "--no-execute", dest="noExecute",
|
||||||
|
help="Don't execute any tests (assume PASS)",
|
||||||
|
action="store_true", default=False)
|
||||||
|
parser.add_option_group(group)
|
||||||
|
|
||||||
|
group = OptionGroup(parser, "Test Selection")
|
||||||
|
group.add_option("", "--max-tests", dest="maxTests", metavar="N",
|
||||||
|
help="Maximum number of tests to run",
|
||||||
|
action="store", type=int, default=None)
|
||||||
|
group.add_option("", "--max-time", dest="maxTime", metavar="N",
|
||||||
|
help="Maximum time to spend testing (in seconds)",
|
||||||
|
action="store", type=float, default=None)
|
||||||
|
group.add_option("", "--shuffle", dest="shuffle",
|
||||||
|
help="Run tests in random order",
|
||||||
|
action="store_true", default=False)
|
||||||
|
parser.add_option_group(group)
|
||||||
|
|
||||||
|
group = OptionGroup(parser, "Debug and Experimental Options")
|
||||||
|
group.add_option("", "--debug", dest="debug",
|
||||||
|
help="Enable debugging (for 'lit' development)",
|
||||||
|
action="store_true", default=False)
|
||||||
|
group.add_option("", "--show-suites", dest="showSuites",
|
||||||
|
help="Show discovered test suites",
|
||||||
|
action="store_true", default=False)
|
||||||
|
group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
|
||||||
|
help="Don't run Tcl scripts using 'sh'",
|
||||||
|
action="store_false", default=True)
|
||||||
|
group.add_option("", "--repeat", dest="repeatTests", metavar="N",
|
||||||
|
help="Repeat tests N times (for timing)",
|
||||||
|
action="store", default=None, type=int)
|
||||||
|
parser.add_option_group(group)
|
||||||
|
|
||||||
|
(opts, args) = parser.parse_args()
|
||||||
|
|
||||||
|
if not args:
|
||||||
|
parser.error('No inputs specified')
|
||||||
|
|
||||||
|
if opts.configPrefix is not None:
|
||||||
|
global gConfigName, gSiteConfigName
|
||||||
|
gConfigName = '%s.cfg' % opts.configPrefix
|
||||||
|
gSiteConfigName = '%s.site.cfg' % opts.configPrefix
|
||||||
|
|
||||||
|
if opts.numThreads is None:
|
||||||
|
opts.numThreads = Util.detectCPUs()
|
||||||
|
|
||||||
|
inputs = args
|
||||||
|
|
||||||
|
# Create the user defined parameters.
|
||||||
|
userParams = {}
|
||||||
|
for entry in opts.userParameters:
|
||||||
|
if '=' not in entry:
|
||||||
|
name,val = entry,''
|
||||||
|
else:
|
||||||
|
name,val = entry.split('=', 1)
|
||||||
|
userParams[name] = val
|
||||||
|
|
||||||
|
# Create the global config object.
|
||||||
|
litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
|
||||||
|
path = opts.path,
|
||||||
|
quiet = opts.quiet,
|
||||||
|
useValgrind = opts.useValgrind,
|
||||||
|
valgrindArgs = opts.valgrindArgs,
|
||||||
|
useTclAsSh = opts.useTclAsSh,
|
||||||
|
noExecute = opts.noExecute,
|
||||||
|
debug = opts.debug,
|
||||||
|
isWindows = (platform.system()=='Windows'),
|
||||||
|
params = userParams)
|
||||||
|
|
||||||
|
# Load the tests from the inputs.
|
||||||
|
tests = []
|
||||||
|
testSuiteCache = {}
|
||||||
|
localConfigCache = {}
|
||||||
|
for input in inputs:
|
||||||
|
prev = len(tests)
|
||||||
|
tests.extend(getTests(input, litConfig,
|
||||||
|
testSuiteCache, localConfigCache)[1])
|
||||||
|
if prev == len(tests):
|
||||||
|
litConfig.warning('input %r contained no tests' % input)
|
||||||
|
|
||||||
|
# If there were any errors during test discovery, exit now.
|
||||||
|
if litConfig.numErrors:
|
||||||
|
print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
if opts.showSuites:
|
||||||
|
suitesAndTests = dict([(ts,[])
|
||||||
|
for ts,_ in testSuiteCache.values()
|
||||||
|
if ts])
|
||||||
|
for t in tests:
|
||||||
|
suitesAndTests[t.suite].append(t)
|
||||||
|
|
||||||
|
print '-- Test Suites --'
|
||||||
|
suitesAndTests = suitesAndTests.items()
|
||||||
|
suitesAndTests.sort(key = lambda (ts,_): ts.name)
|
||||||
|
for ts,ts_tests in suitesAndTests:
|
||||||
|
print ' %s - %d tests' %(ts.name, len(ts_tests))
|
||||||
|
print ' Source Root: %s' % ts.source_root
|
||||||
|
print ' Exec Root : %s' % ts.exec_root
|
||||||
|
|
||||||
|
# Select and order the tests.
|
||||||
|
numTotalTests = len(tests)
|
||||||
|
if opts.shuffle:
|
||||||
|
random.shuffle(tests)
|
||||||
|
else:
|
||||||
|
tests.sort(key = lambda t: t.getFullName())
|
||||||
|
if opts.maxTests is not None:
|
||||||
|
tests = tests[:opts.maxTests]
|
||||||
|
|
||||||
|
extra = ''
|
||||||
|
if len(tests) != numTotalTests:
|
||||||
|
extra = ' of %d' % numTotalTests
|
||||||
|
header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
|
||||||
|
opts.numThreads)
|
||||||
|
|
||||||
|
if opts.repeatTests:
|
||||||
|
tests = [t.copyWithIndex(i)
|
||||||
|
for t in tests
|
||||||
|
for i in range(opts.repeatTests)]
|
||||||
|
|
||||||
|
progressBar = None
|
||||||
|
if not opts.quiet:
|
||||||
|
if opts.succinct and opts.useProgressBar:
|
||||||
|
try:
|
||||||
|
tc = ProgressBar.TerminalController()
|
||||||
|
progressBar = ProgressBar.ProgressBar(tc, header)
|
||||||
|
except ValueError:
|
||||||
|
print header
|
||||||
|
progressBar = ProgressBar.SimpleProgressBar('Testing: ')
|
||||||
|
else:
|
||||||
|
print header
|
||||||
|
|
||||||
|
# Don't create more threads than tests.
|
||||||
|
opts.numThreads = min(len(tests), opts.numThreads)
|
||||||
|
|
||||||
|
startTime = time.time()
|
||||||
|
display = TestingProgressDisplay(opts, len(tests), progressBar)
|
||||||
|
provider = TestProvider(tests, opts.maxTime)
|
||||||
|
runTests(opts.numThreads, litConfig, provider, display)
|
||||||
|
display.finish()
|
||||||
|
|
||||||
|
if not opts.quiet:
|
||||||
|
print 'Testing Time: %.2fs'%(time.time() - startTime)
|
||||||
|
|
||||||
|
# Update results for any tests which weren't run.
|
||||||
|
for t in tests:
|
||||||
|
if t.result is None:
|
||||||
|
t.setResult(Test.UNRESOLVED, '', 0.0)
|
||||||
|
|
||||||
|
# List test results organized by kind.
|
||||||
|
hasFailures = False
|
||||||
|
byCode = {}
|
||||||
|
for t in tests:
|
||||||
|
if t.result not in byCode:
|
||||||
|
byCode[t.result] = []
|
||||||
|
byCode[t.result].append(t)
|
||||||
|
if t.result.isFailure:
|
||||||
|
hasFailures = True
|
||||||
|
|
||||||
|
# FIXME: Show unresolved and (optionally) unsupported tests.
|
||||||
|
for title,code in (('Unexpected Passing Tests', Test.XPASS),
|
||||||
|
('Failing Tests', Test.FAIL)):
|
||||||
|
elts = byCode.get(code)
|
||||||
|
if not elts:
|
||||||
|
continue
|
||||||
|
print '*'*20
|
||||||
|
print '%s (%d):' % (title, len(elts))
|
||||||
|
for t in elts:
|
||||||
|
print ' %s' % t.getFullName()
|
||||||
|
print
|
||||||
|
|
||||||
|
if opts.timeTests:
|
||||||
|
# Collate, in case we repeated tests.
|
||||||
|
times = {}
|
||||||
|
for t in tests:
|
||||||
|
key = t.getFullName()
|
||||||
|
times[key] = times.get(key, 0.) + t.elapsed
|
||||||
|
|
||||||
|
byTime = list(times.items())
|
||||||
|
byTime.sort(key = lambda (name,elapsed): elapsed)
|
||||||
|
if byTime:
|
||||||
|
Util.printHistogram(byTime, title='Tests')
|
||||||
|
|
||||||
|
for name,code in (('Expected Passes ', Test.PASS),
|
||||||
|
('Expected Failures ', Test.XFAIL),
|
||||||
|
('Unsupported Tests ', Test.UNSUPPORTED),
|
||||||
|
('Unresolved Tests ', Test.UNRESOLVED),
|
||||||
|
('Unexpected Passes ', Test.XPASS),
|
||||||
|
('Unexpected Failures', Test.FAIL),):
|
||||||
|
if opts.quiet and not code.isFailure:
|
||||||
|
continue
|
||||||
|
N = len(byCode.get(code,[]))
|
||||||
|
if N:
|
||||||
|
print ' %s: %d' % (name,N)
|
||||||
|
|
||||||
|
# If we encountered any additional errors, exit abnormally.
|
||||||
|
if litConfig.numErrors:
|
||||||
|
print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
# Warn about warnings.
|
||||||
|
if litConfig.numWarnings:
|
||||||
|
print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
|
||||||
|
|
||||||
|
if hasFailures:
|
||||||
|
sys.exit(1)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
if __name__=='__main__':
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user