mirror of
https://github.com/classilla/tenfourfox.git
synced 2024-06-28 09:29:40 +00:00
411 lines
17 KiB
Python
Executable File
411 lines
17 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# ***** BEGIN LICENSE BLOCK *****
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
# ***** END LICENSE BLOCK *****
|
|
|
|
import copy
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
# load modules from parent dir
|
|
sys.path.insert(1, os.path.dirname(sys.path[0]))
|
|
|
|
from mozharness.base.errors import BaseErrorList, TarErrorList
|
|
from mozharness.base.log import ERROR, WARNING
|
|
from mozharness.base.script import (
|
|
BaseScript,
|
|
PreScriptAction,
|
|
)
|
|
from mozharness.base.vcs.vcsbase import VCSMixin
|
|
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
|
|
from mozharness.mozilla.testing.errors import LogcatErrorList
|
|
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
|
|
from mozharness.mozilla.buildbot import TBPL_SUCCESS
|
|
|
|
|
|
class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
|
|
test_suites = ('jsreftest', 'reftest', 'mochitest', 'mochitest-chrome', 'xpcshell', 'crashtest', 'cppunittest')
|
|
config_options = [[
|
|
["--type"],
|
|
{"action": "store",
|
|
"dest": "test_type",
|
|
"default": "browser",
|
|
"help": "The type of tests to run",
|
|
}
|
|
], [
|
|
["--busybox-url"],
|
|
{"action": "store",
|
|
"dest": "busybox_url",
|
|
"default": None,
|
|
"help": "URL to the busybox binary",
|
|
}
|
|
], [
|
|
["--emulator-url"],
|
|
{"action": "store",
|
|
"dest": "emulator_url",
|
|
"default": None,
|
|
"help": "URL to the emulator zip",
|
|
}
|
|
], [
|
|
["--xre-url"],
|
|
{"action": "store",
|
|
"dest": "xre_url",
|
|
"default": None,
|
|
"help": "URL to the desktop xre zip",
|
|
}
|
|
], [
|
|
["--gecko-url"],
|
|
{"action": "store",
|
|
"dest": "gecko_url",
|
|
"default": None,
|
|
"help": "URL to the gecko build injected into the emulator",
|
|
}
|
|
], [
|
|
["--test-manifest"],
|
|
{"action": "store",
|
|
"dest": "test_manifest",
|
|
"default": None,
|
|
"help": "Path to test manifest to run",
|
|
}
|
|
], [
|
|
["--test-suite"],
|
|
{"action": "store",
|
|
"dest": "test_suite",
|
|
"type": "choice",
|
|
"choices": test_suites,
|
|
"help": "Which test suite to run",
|
|
}
|
|
], [
|
|
["--adb-path"],
|
|
{"action": "store",
|
|
"dest": "adb_path",
|
|
"default": None,
|
|
"help": "Path to adb",
|
|
}
|
|
], [
|
|
["--total-chunks"],
|
|
{"action": "store",
|
|
"dest": "total_chunks",
|
|
"help": "Number of total chunks",
|
|
}
|
|
], [
|
|
["--this-chunk"],
|
|
{"action": "store",
|
|
"dest": "this_chunk",
|
|
"help": "Number of this chunk",
|
|
}
|
|
], [
|
|
["--test-path"],
|
|
{"action": "store",
|
|
"dest": "test_path",
|
|
"help": "Path of tests to run",
|
|
}
|
|
]] + copy.deepcopy(testing_config_options) \
|
|
+ copy.deepcopy(blobupload_config_options)
|
|
|
|
error_list = [
|
|
{'substr': 'FAILED (errors=', 'level': ERROR},
|
|
{'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
|
|
{'substr': r'''Could not communicate with Marionette server. Is the Gecko process still running''', 'level': ERROR},
|
|
{'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
|
|
{'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
|
|
{'regex': re.compile(r'''(Timeout|NoSuchAttribute|Javascript|NoSuchElement|XPathLookup|NoSuchWindow|StaleElement|ScriptTimeout|ElementNotVisible|NoSuchFrame|InvalidElementState|NoAlertPresent|InvalidCookieDomain|UnableToSetCookie|InvalidSelector|MoveTargetOutOfBounds)Exception'''), 'level': ERROR},
|
|
]
|
|
|
|
def __init__(self, require_config_file=False):
|
|
super(B2GEmulatorTest, self).__init__(
|
|
config_options=self.config_options,
|
|
all_actions=['clobber',
|
|
'read-buildbot-config',
|
|
'download-and-extract',
|
|
'create-virtualenv',
|
|
'install',
|
|
'run-tests'],
|
|
default_actions=['clobber',
|
|
'download-and-extract',
|
|
'create-virtualenv',
|
|
'install',
|
|
'run-tests'],
|
|
require_config_file=require_config_file,
|
|
config={
|
|
'require_test_zip': True,
|
|
# This is a special IP that has meaning to the emulator
|
|
'remote_webserver': '10.0.2.2',
|
|
}
|
|
)
|
|
|
|
# these are necessary since self.config is read only
|
|
c = self.config
|
|
self.adb_path = c.get('adb_path', self._query_adb())
|
|
self.installer_url = c.get('installer_url')
|
|
self.installer_path = c.get('installer_path')
|
|
self.test_url = c.get('test_url')
|
|
self.test_packages_url = c.get('test_packages_url')
|
|
self.test_manifest = c.get('test_manifest')
|
|
self.busybox_path = None
|
|
|
|
# TODO detect required config items and fail if not set
|
|
|
|
def query_abs_dirs(self):
|
|
if self.abs_dirs:
|
|
return self.abs_dirs
|
|
abs_dirs = super(B2GEmulatorTest, self).query_abs_dirs()
|
|
dirs = {}
|
|
dirs['abs_test_install_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'tests')
|
|
dirs['abs_xre_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'xre')
|
|
dirs['abs_emulator_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'emulator')
|
|
dirs['abs_blob_upload_dir'] = os.path.join(
|
|
abs_dirs['abs_work_dir'], 'blobber_upload_dir')
|
|
dirs['abs_b2g-distro_dir'] = os.path.join(
|
|
dirs['abs_emulator_dir'], 'b2g-distro')
|
|
dirs['abs_mochitest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'mochitest')
|
|
dirs['abs_mochitest-chrome_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'mochitest')
|
|
dirs['abs_certs_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'certs')
|
|
dirs['abs_modules_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'modules')
|
|
dirs['abs_reftest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'reftest')
|
|
dirs['abs_crashtest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'reftest')
|
|
dirs['abs_jsreftest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'reftest')
|
|
dirs['abs_xpcshell_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'xpcshell')
|
|
dirs['abs_cppunittest_dir'] = os.path.join(
|
|
dirs['abs_test_install_dir'], 'cppunittest')
|
|
for key in dirs.keys():
|
|
if key not in abs_dirs:
|
|
abs_dirs[key] = dirs[key]
|
|
self.abs_dirs = abs_dirs
|
|
return self.abs_dirs
|
|
|
|
def download_and_extract(self):
|
|
target_suite = self.config['test_suite']
|
|
super(B2GEmulatorTest, self).download_and_extract(suite_categories=[target_suite])
|
|
dirs = self.query_abs_dirs()
|
|
|
|
self.mkdir_p(dirs['abs_emulator_dir'])
|
|
tar = self.query_exe('tar', return_type='list')
|
|
self.run_command(tar + ['zxf', self.installer_path],
|
|
cwd=dirs['abs_emulator_dir'],
|
|
error_list=TarErrorList,
|
|
halt_on_failure=True, fatal_exit_code=3)
|
|
|
|
self.download_unzip(self.config['xre_url'],
|
|
dirs['abs_xre_dir'])
|
|
|
|
if self.config.get('busybox_url'):
|
|
self.download_file(self.config['busybox_url'],
|
|
file_name='busybox',
|
|
parent_dir=dirs['abs_work_dir'])
|
|
self.busybox_path = os.path.join(dirs['abs_work_dir'], 'busybox')
|
|
|
|
@PreScriptAction('create-virtualenv')
|
|
def _pre_create_virtualenv(self, action):
|
|
dirs = self.query_abs_dirs()
|
|
requirements = os.path.join(dirs['abs_test_install_dir'],
|
|
'config',
|
|
'marionette_requirements.txt')
|
|
if os.path.isfile(requirements):
|
|
self.register_virtualenv_module(requirements=[requirements],
|
|
two_pass=True)
|
|
return
|
|
|
|
# XXX Bug 879765: Dependent modules need to be listed before parent
|
|
# modules, otherwise they will get installed from the pypi server.
|
|
# XXX Bug 908356: This block can be removed as soon as the
|
|
# in-tree requirements files propagate to all active trees.
|
|
mozbase_dir = os.path.join('tests', 'mozbase')
|
|
self.register_virtualenv_module(
|
|
'manifestparser',
|
|
url=os.path.join(mozbase_dir, 'manifestdestiny')
|
|
)
|
|
|
|
for m in ('mozfile', 'mozlog', 'mozinfo', 'moznetwork', 'mozhttpd',
|
|
'mozcrash', 'mozinstall', 'mozdevice', 'mozprofile', 'mozprocess',
|
|
'mozrunner'):
|
|
self.register_virtualenv_module(
|
|
m, url=os.path.join(mozbase_dir, m))
|
|
|
|
self.register_virtualenv_module(
|
|
'marionette', url=os.path.join('tests', 'marionette'))
|
|
|
|
def _query_abs_base_cmd(self, suite):
|
|
dirs = self.query_abs_dirs()
|
|
cmd = [self.query_python_path('python')]
|
|
cmd.append(self.config['run_file_names'][suite])
|
|
|
|
raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
|
|
'%s_raw.log' % suite)
|
|
error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
|
|
'%s_errorsummary.log' % suite)
|
|
emulator_type = 'x86' if os.path.isdir(os.path.join(dirs['abs_b2g-distro_dir'],
|
|
'out', 'target', 'product', 'generic_x86')) else 'arm'
|
|
self.info("The emulator type: %s" % emulator_type)
|
|
|
|
str_format_values = {
|
|
'adbpath': self.adb_path,
|
|
'b2gpath': dirs['abs_b2g-distro_dir'],
|
|
'emulator': emulator_type,
|
|
'logcat_dir': dirs['abs_work_dir'],
|
|
'modules_dir': dirs['abs_modules_dir'],
|
|
'remote_webserver': self.config['remote_webserver'],
|
|
'xre_path': os.path.join(dirs['abs_xre_dir'], 'bin'),
|
|
'utility_path': os.path.join(dirs['abs_test_install_dir'], 'bin'),
|
|
'symbols_path': self.symbols_path,
|
|
'busybox': self.busybox_path,
|
|
'total_chunks': self.config.get('total_chunks'),
|
|
'this_chunk': self.config.get('this_chunk'),
|
|
'test_path': self.config.get('test_path'),
|
|
'certificate_path': dirs['abs_certs_dir'],
|
|
'raw_log_file': raw_log_file,
|
|
'error_summary_file': error_summary_file,
|
|
}
|
|
|
|
if suite not in self.config["suite_definitions"]:
|
|
self.fatal("'%s' not defined in the config!" % suite)
|
|
|
|
try_options, try_tests = self.try_args(suite)
|
|
|
|
options = self.query_options(self.config["suite_definitions"][suite]["options"],
|
|
try_options,
|
|
str_format_values=str_format_values)
|
|
cmd.extend(opt for opt in options if not opt.endswith('None'))
|
|
|
|
tests = self.query_tests_args(self.config["suite_definitions"][suite].get("tests"),
|
|
try_tests,
|
|
str_format_values=str_format_values)
|
|
cmd.extend(opt for opt in tests if not opt.endswith('None'))
|
|
|
|
return cmd
|
|
|
|
def _query_adb(self):
|
|
return self.which('adb') or \
|
|
os.getenv('ADB_PATH') or \
|
|
os.path.join(self.query_abs_dirs()['abs_b2g-distro_dir'],
|
|
'out', 'host', 'linux-x86', 'bin', 'adb')
|
|
|
|
def preflight_run_tests(self):
|
|
super(B2GEmulatorTest, self).preflight_run_tests()
|
|
suite = self.config['test_suite']
|
|
# set default test manifest by suite if none specified
|
|
if not self.test_manifest:
|
|
if suite == 'reftest':
|
|
self.test_manifest = os.path.join('tests', 'layout',
|
|
'reftests', 'reftest.list')
|
|
elif suite == 'xpcshell':
|
|
self.test_manifest = os.path.join('tests', 'xpcshell_b2g.ini')
|
|
elif suite == 'crashtest':
|
|
self.test_manifest = os.path.join('tests', 'testing',
|
|
'crashtest', 'crashtests.list')
|
|
elif suite == 'jsreftest':
|
|
self.test_manifest = os.path.join('jsreftest', 'tests',
|
|
'jstests.list')
|
|
|
|
if not os.path.isfile(self.adb_path):
|
|
self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path)
|
|
|
|
def install(self):
|
|
# The emulator was extracted during the download_and_extract step;
|
|
# there's no separate binary to install. We call pass to prevent
|
|
# the base implementation from running here.
|
|
pass
|
|
|
|
def _get_success_codes(self, suite_name):
|
|
success_codes = None
|
|
if suite_name == 'xpcshell':
|
|
# bug 773703
|
|
success_codes = [0, 1]
|
|
return success_codes
|
|
|
|
def run_tests(self):
|
|
"""
|
|
Run the tests
|
|
"""
|
|
dirs = self.query_abs_dirs()
|
|
|
|
error_list = self.error_list
|
|
error_list.extend(BaseErrorList)
|
|
|
|
suite = self.config['test_suite']
|
|
if suite not in self.test_suites:
|
|
self.fatal("Don't know how to run --test-suite '%s'!" % suite)
|
|
|
|
cmd = self._query_abs_base_cmd(suite)
|
|
cwd = dirs['abs_%s_dir' % suite]
|
|
|
|
# TODO we probably have to move some of the code in
|
|
# scripts/desktop_unittest.py and scripts/marionette.py to
|
|
# mozharness.mozilla.testing.unittest so we can share it.
|
|
# In the short term, I'm ok with some duplication of code if it
|
|
# expedites things; please file bugs to merge if that happens.
|
|
|
|
suite_name = [x for x in self.test_suites if x in self.config['test_suite']][0]
|
|
if self.config.get('this_chunk'):
|
|
suite = '%s-%s' % (suite_name, self.config['this_chunk'])
|
|
else:
|
|
suite = suite_name
|
|
|
|
env = {}
|
|
if self.query_minidump_stackwalk():
|
|
env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
|
|
env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
|
|
if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
|
|
self.mkdir_p(env['MOZ_UPLOAD_DIR'])
|
|
env = self.query_env(partial_env=env)
|
|
|
|
success_codes = self._get_success_codes(suite_name)
|
|
parser = self.get_test_output_parser(suite_name,
|
|
config=self.config,
|
|
log_obj=self.log_obj,
|
|
error_list=error_list)
|
|
return_code = self.run_command(cmd, cwd=cwd, env=env,
|
|
output_timeout=1000,
|
|
output_parser=parser,
|
|
success_codes=success_codes)
|
|
|
|
logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
|
|
|
|
qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
|
|
if os.path.isfile(qemu):
|
|
self.copyfile(qemu, os.path.join(env['MOZ_UPLOAD_DIR'],
|
|
os.path.basename(qemu)))
|
|
|
|
tbpl_status, log_level = parser.evaluate_parser(return_code,
|
|
success_codes=success_codes)
|
|
|
|
if os.path.isfile(logcat):
|
|
if tbpl_status != TBPL_SUCCESS:
|
|
# On failure, dump logcat, check if the emulator is still
|
|
# running, and if it is still accessible via adb.
|
|
self.info('dumping logcat')
|
|
self.run_command(['cat', logcat], error_list=LogcatErrorList)
|
|
|
|
self.run_command(['ps', '-C', 'emulator'])
|
|
self.run_command([self.adb_path, 'devices'])
|
|
|
|
# upload logcat to blobber
|
|
self.copyfile(logcat, os.path.join(env['MOZ_UPLOAD_DIR'],
|
|
os.path.basename(logcat)))
|
|
else:
|
|
self.info('no logcat file found')
|
|
|
|
parser.append_tinderboxprint_line(suite_name)
|
|
|
|
self.buildbot_status(tbpl_status, level=log_level)
|
|
self.log("The %s suite: %s ran with return status: %s" %
|
|
(suite_name, suite, tbpl_status), level=log_level)
|
|
|
|
if __name__ == '__main__':
|
|
emulatorTest = B2GEmulatorTest()
|
|
emulatorTest.run_and_exit()
|