mirror of
https://github.com/classilla/tenfourfox.git
synced 2024-10-03 22:55:12 +00:00
198 lines
5.9 KiB
Python
198 lines
5.9 KiB
Python
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||
|
|
||
|
import sys
|
||
|
import tempfile
|
||
|
from cStringIO import StringIO
|
||
|
from collections import defaultdict
|
||
|
|
||
|
import wptrunner
|
||
|
import wpttest
|
||
|
|
||
|
from mozlog import commandline, reader
|
||
|
|
||
|
logger = None
|
||
|
|
||
|
|
||
|
def setup_logging(args, defaults):
|
||
|
global logger
|
||
|
logger = commandline.setup_logging("web-platform-tests-unstable", args, defaults)
|
||
|
wptrunner.setup_stdlib_logger()
|
||
|
|
||
|
for name in args.keys():
|
||
|
if name.startswith("log_"):
|
||
|
args.pop(name)
|
||
|
|
||
|
return logger
|
||
|
|
||
|
|
||
|
def group(items, size):
|
||
|
rv = []
|
||
|
i = 0
|
||
|
while i < len(items):
|
||
|
rv.append(items[i:i + size])
|
||
|
i += size
|
||
|
|
||
|
return rv
|
||
|
|
||
|
|
||
|
def next_power_of_two(num):
|
||
|
rv = 1
|
||
|
while rv < num:
|
||
|
rv = rv << 1
|
||
|
return rv
|
||
|
|
||
|
|
||
|
class Reducer(object):
|
||
|
def __init__(self, target, **kwargs):
|
||
|
self.target = target
|
||
|
|
||
|
self.test_type = kwargs["test_types"][0]
|
||
|
run_info = wpttest.get_run_info(kwargs["metadata_root"],
|
||
|
kwargs["product"],
|
||
|
debug=False)
|
||
|
test_filter = wptrunner.TestFilter(include=kwargs["include"])
|
||
|
self.test_loader = wptrunner.TestLoader(kwargs["tests_root"],
|
||
|
kwargs["metadata_root"],
|
||
|
[self.test_type],
|
||
|
run_info,
|
||
|
manifest_filer=test_filter)
|
||
|
if kwargs["repeat"] == 1:
|
||
|
logger.critical("Need to specify --repeat with more than one repetition")
|
||
|
sys.exit(1)
|
||
|
self.kwargs = kwargs
|
||
|
|
||
|
def run(self):
|
||
|
all_tests = self.get_initial_tests()
|
||
|
|
||
|
tests = all_tests[:-1]
|
||
|
target_test = [all_tests[-1]]
|
||
|
|
||
|
if self.unstable(target_test):
|
||
|
return target_test
|
||
|
|
||
|
if not self.unstable(all_tests):
|
||
|
return []
|
||
|
|
||
|
chunk_size = next_power_of_two(int(len(tests) / 2))
|
||
|
logger.debug("Using chunk size %i" % chunk_size)
|
||
|
|
||
|
while chunk_size >= 1:
|
||
|
logger.debug("%i tests remain" % len(tests))
|
||
|
chunks = group(tests, chunk_size)
|
||
|
chunk_results = [None] * len(chunks)
|
||
|
|
||
|
for i, chunk in enumerate(chunks):
|
||
|
logger.debug("Running chunk %i/%i of size %i" % (i + 1, len(chunks), chunk_size))
|
||
|
trial_tests = []
|
||
|
chunk_str = ""
|
||
|
for j, inc_chunk in enumerate(chunks):
|
||
|
if i != j and chunk_results[j] in (None, False):
|
||
|
chunk_str += "+"
|
||
|
trial_tests.extend(inc_chunk)
|
||
|
else:
|
||
|
chunk_str += "-"
|
||
|
logger.debug("Using chunks %s" % chunk_str)
|
||
|
trial_tests.extend(target_test)
|
||
|
|
||
|
chunk_results[i] = self.unstable(trial_tests)
|
||
|
|
||
|
# if i == len(chunks) - 2 and all(item is False for item in chunk_results[:-1]):
|
||
|
# Dangerous? optimisation that if you got stability for 0..N-1 chunks
|
||
|
# it must be unstable with the Nth chunk
|
||
|
# chunk_results[i+1] = True
|
||
|
# continue
|
||
|
|
||
|
new_tests = []
|
||
|
keep_str = ""
|
||
|
for result, chunk in zip(chunk_results, chunks):
|
||
|
if not result:
|
||
|
keep_str += "+"
|
||
|
new_tests.extend(chunk)
|
||
|
else:
|
||
|
keep_str += "-"
|
||
|
|
||
|
logger.debug("Keeping chunks %s" % keep_str)
|
||
|
|
||
|
tests = new_tests
|
||
|
|
||
|
chunk_size = int(chunk_size / 2)
|
||
|
|
||
|
return tests + target_test
|
||
|
|
||
|
def unstable(self, tests):
|
||
|
logger.debug("Running with %i tests" % len(tests))
|
||
|
|
||
|
self.test_loader.tests = {self.test_type: tests}
|
||
|
|
||
|
stdout, stderr = sys.stdout, sys.stderr
|
||
|
sys.stdout = StringIO()
|
||
|
sys.stderr = StringIO()
|
||
|
|
||
|
with tempfile.NamedTemporaryFile() as f:
|
||
|
args = self.kwargs.copy()
|
||
|
args["log_raw"] = [f]
|
||
|
args["capture_stdio"] = False
|
||
|
wptrunner.setup_logging(args, {})
|
||
|
wptrunner.run_tests(test_loader=self.test_loader, **args)
|
||
|
wptrunner.logger.remove_handler(wptrunner.logger.handlers[0])
|
||
|
is_unstable = self.log_is_unstable(f)
|
||
|
|
||
|
sys.stdout, sys.stderr = stdout, stderr
|
||
|
|
||
|
logger.debug("Result was unstable with chunk removed"
|
||
|
if is_unstable else "stable")
|
||
|
|
||
|
return is_unstable
|
||
|
|
||
|
def log_is_unstable(self, log_f):
|
||
|
log_f.seek(0)
|
||
|
|
||
|
statuses = defaultdict(set)
|
||
|
|
||
|
def handle_status(item):
|
||
|
if item["test"] == self.target:
|
||
|
statuses[item["subtest"]].add(item["status"])
|
||
|
|
||
|
def handle_end(item):
|
||
|
if item["test"] == self.target:
|
||
|
statuses[None].add(item["status"])
|
||
|
|
||
|
reader.each_log(reader.read(log_f),
|
||
|
{"test_status": handle_status,
|
||
|
"test_end": handle_end})
|
||
|
|
||
|
logger.debug(str(statuses))
|
||
|
|
||
|
if not statuses:
|
||
|
logger.error("Didn't get any useful output from wptrunner")
|
||
|
log_f.seek(0)
|
||
|
for item in reader.read(log_f):
|
||
|
logger.debug(item)
|
||
|
return None
|
||
|
|
||
|
return any(len(item) > 1 for item in statuses.itervalues())
|
||
|
|
||
|
def get_initial_tests(self):
|
||
|
# Need to pass in arguments
|
||
|
|
||
|
all_tests = self.test_loader.tests[self.test_type]
|
||
|
tests = []
|
||
|
for item in all_tests:
|
||
|
tests.append(item)
|
||
|
if item.url == self.target:
|
||
|
break
|
||
|
|
||
|
logger.debug("Starting with tests: %s" % ("\n".join(item.id for item in tests)))
|
||
|
|
||
|
return tests
|
||
|
|
||
|
|
||
|
def do_reduce(**kwargs):
|
||
|
target = kwargs.pop("target")
|
||
|
reducer = Reducer(target, **kwargs)
|
||
|
|
||
|
unstable_set = reducer.run()
|
||
|
return unstable_set
|