277 lines
9.0 KiB
Python
Executable File
277 lines
9.0 KiB
Python
Executable File
#!/usr/bin/python2.4
|
|
|
|
"""Run reliability tests using Android instrumentation.
|
|
|
|
A test file consists of list web sites to test is needed as a parameter
|
|
|
|
Usage:
|
|
run_reliability_tests.py path/to/url/list
|
|
"""
|
|
|
|
import logging
|
|
import optparse
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from Numeric import *
|
|
|
|
TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
|
|
TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
|
|
TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
|
|
TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt"
|
|
HTTP_URL_FILE = "urllist_http"
|
|
HTTPS_URL_FILE = "urllist_https"
|
|
NUM_URLS = 25
|
|
|
|
|
|
def DumpRenderTreeFinished(adb_cmd):
|
|
"""Check if DumpRenderTree finished running.
|
|
|
|
Args:
|
|
adb_cmd: adb command string
|
|
|
|
Returns:
|
|
True if DumpRenderTree has finished, False otherwise
|
|
"""
|
|
|
|
# pull test status file and look for "#DONE"
|
|
shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
|
|
adb_output = subprocess.Popen(shell_cmd_str,
|
|
shell=True, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE).communicate()[0]
|
|
return adb_output.strip() == "#DONE"
|
|
|
|
|
|
def RemoveDeviceFile(adb_cmd, file_name):
|
|
shell_cmd_str = adb_cmd + " shell rm " + file_name
|
|
subprocess.Popen(shell_cmd_str,
|
|
shell=True, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE).communicate()
|
|
|
|
|
|
def Bugreport(url, bugreport_dir, adb_cmd):
|
|
"""Pull a bugreport from the device."""
|
|
bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
|
|
int(time.time()))
|
|
|
|
# prepend the report with url
|
|
handle = open(bugreport_filename, "w")
|
|
handle.writelines("Bugreport for crash in url - %s\n\n" % url)
|
|
handle.close()
|
|
|
|
cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
|
|
os.system(cmd)
|
|
|
|
|
|
def ProcessPageLoadTime(raw_log):
|
|
"""Processes the raw page load time logged by test app."""
|
|
log_handle = open(raw_log, "r")
|
|
load_times = {}
|
|
|
|
for line in log_handle:
|
|
line = line.strip()
|
|
pair = line.split("|")
|
|
if len(pair) != 2:
|
|
logging.info("Line has more than one '|': " + line)
|
|
continue
|
|
if pair[0] not in load_times:
|
|
load_times[pair[0]] = []
|
|
try:
|
|
pair[1] = int(pair[1])
|
|
except ValueError:
|
|
logging.info("Lins has non-numeric load time: " + line)
|
|
continue
|
|
load_times[pair[0]].append(pair[1])
|
|
|
|
log_handle.close()
|
|
|
|
# rewrite the average time to file
|
|
log_handle = open(raw_log, "w")
|
|
for url, times in load_times.iteritems():
|
|
# calculate std
|
|
arr = array(times)
|
|
avg = average(arr)
|
|
d = arr - avg
|
|
std = sqrt(sum(d * d) / len(arr))
|
|
output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" %
|
|
(url, min(arr), max(arr), avg, std,
|
|
array2string(arr)))
|
|
log_handle.write(output)
|
|
log_handle.close()
|
|
|
|
|
|
def main(options, args):
|
|
"""Send the url list to device and start testing, restart if crashed."""
|
|
|
|
# Set up logging format.
|
|
log_level = logging.INFO
|
|
if options.verbose:
|
|
log_level = logging.DEBUG
|
|
logging.basicConfig(level=log_level,
|
|
format="%(message)s")
|
|
|
|
# Include all tests if none are specified.
|
|
if not args:
|
|
print "Missing URL list file"
|
|
sys.exit(1)
|
|
else:
|
|
path = args[0]
|
|
|
|
if not options.crash_file:
|
|
print "Missing crash file name, use --crash-file to specify"
|
|
sys.exit(1)
|
|
else:
|
|
crashed_file = options.crash_file
|
|
|
|
if not options.timeout_file:
|
|
print "Missing timeout file, use --timeout-file to specify"
|
|
sys.exit(1)
|
|
else:
|
|
timedout_file = options.timeout_file
|
|
|
|
if not options.delay:
|
|
manual_delay = 0
|
|
else:
|
|
manual_delay = options.delay
|
|
|
|
if not options.bugreport:
|
|
bugreport_dir = "."
|
|
else:
|
|
bugreport_dir = options.bugreport
|
|
if not os.path.exists(bugreport_dir):
|
|
os.makedirs(bugreport_dir)
|
|
if not os.path.isdir(bugreport_dir):
|
|
logging.error("Cannot create results dir: " + bugreport_dir)
|
|
sys.exit(1)
|
|
|
|
adb_cmd = "adb "
|
|
if options.adb_options:
|
|
adb_cmd += options.adb_options + " "
|
|
|
|
# push url list to device
|
|
test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
|
|
proc = subprocess.Popen(test_cmd, shell=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
(adb_output, adb_error) = proc.communicate()
|
|
if proc.returncode != 0:
|
|
logging.error("failed to push url list to device.")
|
|
logging.error(adb_output)
|
|
logging.error(adb_error)
|
|
sys.exit(1)
|
|
|
|
# clean up previous results
|
|
RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
|
|
RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
|
|
RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE)
|
|
|
|
logging.info("Running the test ...")
|
|
|
|
# Count crashed tests.
|
|
crashed_tests = []
|
|
|
|
if options.time_out_ms:
|
|
timeout_ms = options.time_out_ms
|
|
|
|
# Run test until it's done
|
|
test_cmd_prefix = adb_cmd + " shell am instrument"
|
|
test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
|
|
|
|
# Call ReliabilityTestsAutoTest#startReliabilityTests
|
|
test_cmd = (test_cmd_prefix + " -e class "
|
|
"com.android.dumprendertree.ReliabilityTest#"
|
|
"runReliabilityTest -e timeout %s -e delay %s" %
|
|
(str(timeout_ms), str(manual_delay)))
|
|
|
|
if options.logtime:
|
|
test_cmd += " -e logtime true"
|
|
|
|
test_cmd += test_cmd_postfix
|
|
|
|
adb_output = subprocess.Popen(test_cmd, shell=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE).communicate()[0]
|
|
while not DumpRenderTreeFinished(adb_cmd):
|
|
logging.error("DumpRenderTree exited before all URLs are visited.")
|
|
shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
|
|
crashed_test = ""
|
|
while not crashed_test:
|
|
(crashed_test, err) = subprocess.Popen(
|
|
shell_cmd_str, shell=True, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE).communicate()
|
|
crashed_test = crashed_test.strip()
|
|
if not crashed_test:
|
|
logging.error('Cannot get crashed test name, device offline?')
|
|
logging.error('stderr: ' + err)
|
|
logging.error('retrying in 10s...')
|
|
time.sleep(10)
|
|
|
|
logging.info(crashed_test + " CRASHED")
|
|
crashed_tests.append(crashed_test)
|
|
Bugreport(crashed_test, bugreport_dir, adb_cmd)
|
|
logging.info("Resuming reliability test runner...")
|
|
|
|
adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE).communicate()[0]
|
|
|
|
if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
|
|
adb_output.find("Process crashed.") != -1):
|
|
logging.error("Error happened : " + adb_output)
|
|
sys.exit(1)
|
|
|
|
logging.info(adb_output)
|
|
logging.info("Done\n")
|
|
|
|
if crashed_tests:
|
|
file_handle = open(crashed_file, "w")
|
|
file_handle.writelines("\n".join(crashed_tests))
|
|
logging.info("Crashed URL list stored in: " + crashed_file)
|
|
file_handle.close()
|
|
else:
|
|
logging.info("No crash found.")
|
|
|
|
# get timeout file from sdcard
|
|
test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
|
|
+ timedout_file + "\"")
|
|
subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE).communicate()
|
|
|
|
if options.logtime:
|
|
# get logged page load times from sdcard
|
|
test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \""
|
|
+ options.logtime + "\"")
|
|
subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE).communicate()
|
|
ProcessPageLoadTime(options.logtime)
|
|
|
|
|
|
if "__main__" == __name__:
|
|
option_parser = optparse.OptionParser()
|
|
option_parser.add_option("-t", "--time-out-ms",
|
|
default=60000,
|
|
help="set the timeout for each test")
|
|
option_parser.add_option("-v", "--verbose", action="store_true",
|
|
default=False,
|
|
help="include debug-level logging")
|
|
option_parser.add_option("-a", "--adb-options",
|
|
default=None,
|
|
help="pass options to adb, such as -d -e, etc")
|
|
option_parser.add_option("-c", "--crash-file",
|
|
default="reliability_crashed_sites.txt",
|
|
help="the list of sites that cause browser to crash")
|
|
option_parser.add_option("-f", "--timeout-file",
|
|
default="reliability_timedout_sites.txt",
|
|
help="the list of sites that timedout during test")
|
|
option_parser.add_option("-d", "--delay",
|
|
default=0,
|
|
help="add a manual delay between pages (in ms)")
|
|
option_parser.add_option("-b", "--bugreport",
|
|
default=".",
|
|
help="the directory to store bugreport for crashes")
|
|
option_parser.add_option("-l", "--logtime",
|
|
default=None,
|
|
help="Logs page load time for each url to the file")
|
|
opts, arguments = option_parser.parse_args()
|
|
main(opts, arguments)
|