M7350v1_en_gpl

base/tests/DumpRenderTree/assets/results/layout_tests_crashed.txt

@@ -0,0 +1,4 @@
+/sdcard/android/layout_tests/http/tests/xmlhttprequest/basic-auth.html
+/sdcard/android/layout_tests/http/tests/xmlhttprequest/failed-auth.html
+/sdcard/android/layout_tests/http/tests/xmlhttprequest/cross-origin-authorization.html
+/sdcard/android/layout_tests/http/tests/xmlhttprequest/cross-origin-no-authorization.html

base/tests/DumpRenderTree/assets/run_layout_tests.py

@@ -0,0 +1,320 @@
+#!/usr/bin/python
+
+"""Run layout tests using Android emulator and instrumentation.
+
+  First, you need to get an SD card or sdcard image that has layout tests on it.
+  Layout tests are in following directory:
+    /sdcard/android/layout_tests
+  For example, /sdcard/android/layout_tests/fast
+
+  Usage:
+    Run all tests under fast/ directory:
+      run_layout_tests.py, or
+      run_layout_tests.py fast
+
+    Run all tests under a sub directory:
+      run_layout_tests.py fast/dom
+
+    Run a single test:
+      run_layout_tests.py fast/dom/
+
+  After a merge, if there are changes of layout tests in SD card, you need to
+  use --refresh-test-list option *once* to re-generate test list on the card.
+
+  Some other options are:
+    --rebaseline generates expected layout tests results under /sdcard/android/expected_result/
+    --time-out-ms (default is 8000 millis) for each test
+    --adb-options="-e" passes option string to adb
+    --results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
+    --js-engine the JavaScript engine currently in use, determines which set of Android-specific expected results we should use, should be 'jsc' or 'v8'
+"""
+
+import logging
+import optparse
+import os
+import subprocess
+import sys
+import time
+
+def CountLineNumber(filename):
+  """Compute the number of lines in a given file.
+
+  Args:
+    filename: a file name related to the current directory.
+  """
+
+  fp = open(os.path.abspath(filename), "r");
+  lines = 0
+  for line in fp.readlines():
+    lines = lines + 1
+  fp.close()
+  return lines
+
+def DumpRenderTreeFinished(adb_cmd):
+  """ Check if DumpRenderTree finished running tests
+
+  Args:
+    output: adb_cmd string
+  """
+
+  # pull /sdcard/android/running_test.txt, if the content is "#DONE", it's done
+  shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
+  adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
+  return adb_output.strip() == "#DONE"
+
+def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
+                new_count_first=True):
+   """ Given two result files, generate diff and
+       write to diff_results file. All arguments are absolute paths
+       to files.
+   """
+   old_file = open(old_results, "r")
+   new_file = open(new_results, "r")
+   diff_file = open(diff_results, "a")
+
+   # Read lines from each file
+   ndict = new_file.readlines()
+   cdict = old_file.readlines()
+
+   # Write marker to diff file
+   diff_file.writelines(marker + "\n")
+   diff_file.writelines("###############\n")
+
+   # Strip reason from result lines
+   if strip_reason is True:
+     for i in range(0, len(ndict)):
+       ndict[i] = ndict[i].split(' ')[0] + "\n"
+     for i in range(0, len(cdict)):
+       cdict[i] = cdict[i].split(' ')[0] + "\n"
+
+   params = {
+       "new": [0, ndict, cdict, "+"],
+       "miss": [0, cdict, ndict, "-"]
+       }
+   if new_count_first:
+     order = ["new", "miss"]
+   else:
+     order = ["miss", "new"]
+
+   for key in order:
+     for line in params[key][1]:
+       if line not in params[key][2]:
+         if line[-1] != "\n":
+           line += "\n";
+         diff_file.writelines(params[key][3] + line)
+         params[key][0] += 1
+
+   logging.info(marker + "  >>> " + str(params["new"][0]) + " new, " +
+                str(params["miss"][0]) + " misses")
+
+   diff_file.writelines("\n\n")
+
+   old_file.close()
+   new_file.close()
+   diff_file.close()
+   return
+
+def CompareResults(ref_dir, results_dir):
+  """Compare results in two directories
+
+  Args:
+    ref_dir: the reference directory having layout results as references
+    results_dir: the results directory
+  """
+  logging.info("Comparing results to " + ref_dir)
+
+  diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
+  if os.path.exists(diff_result):
+    os.remove(diff_result)
+
+  files=["crashed", "failed", "passed", "nontext"]
+  for f in files:
+    result_file_name = "layout_tests_" + f + ".txt"
+    DiffResults(f, os.path.join(results_dir, result_file_name),
+                os.path.join(ref_dir, result_file_name), diff_result,
+                False, f != "passed")
+  logging.info("Detailed diffs are in " + diff_result)
+
+def main(options, args):
+  """Run the tests. Will call sys.exit when complete.
+
+  Args:
+    options: a dictionary of command line options
+    args: a list of sub directories or files to test
+  """
+
+  # Set up logging format.
+  log_level = logging.INFO
+  if options.verbose:
+    log_level = logging.DEBUG
+  logging.basicConfig(level=log_level,
+                      format='%(message)s')
+
+  # Include all tests if none are specified.
+  if not args:
+    path = '/';
+  else:
+    path = ' '.join(args);
+
+  adb_cmd = "adb ";
+  if options.adb_options:
+    adb_cmd += options.adb_options
+
+  # Re-generate the test list if --refresh-test-list is on
+  if options.refresh_test_list:
+    logging.info("Generating test list.");
+    generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
+    adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
+
+    if adb_output.find('Process crashed') != -1:
+       logging.info("Aborting because cannot generate test list.\n" + adb_output)
+       sys.exit(1)
+
+
+  logging.info("Running tests")
+
+  # Count crashed tests.
+  crashed_tests = []
+
+  timeout_ms = '30000'
+  if options.time_out_ms:
+    timeout_ms = options.time_out_ms
+
+  # Run test until it's done
+
+  run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
+
+  run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
+  if options.rebaseline:
+    run_layout_test_cmd_postfix += " -e rebaseline true"
+
+  # If the JS engine is not specified on the command line, try reading the
+  # JS_ENGINE environment  variable, which is used by the build system in
+  # external/webkit/Android.mk.
+  js_engine = options.js_engine
+  if not js_engine and os.environ.has_key('JS_ENGINE'):
+    js_engine = os.environ['JS_ENGINE']
+  if js_engine:
+    run_layout_test_cmd_postfix += " -e jsengine " + js_engine
+
+  run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
+
+  # Call LayoutTestsAutoTest::startLayoutTests.
+  run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
+
+  adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
+  while not DumpRenderTreeFinished(adb_cmd):
+    # Get the running_test.txt
+    logging.error("DumpRenderTree crashed, output:\n" + adb_output)
+
+    shell_cmd_str = adb_cmd + " shell cat /sdcard/android/running_test.txt"
+    crashed_test = ""
+    while not crashed_test:
+      (crashed_test, err) = subprocess.Popen(
+          shell_cmd_str, shell=True, stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE).communicate()
+      crashed_test = crashed_test.strip()
+      if not crashed_test:
+        logging.error('Cannot get crashed test name, device offline?')
+        logging.error('stderr: ' + err)
+        logging.error('retrying in 10s...')
+        time.sleep(10)
+
+    logging.info(crashed_test + " CRASHED");
+    crashed_tests.append(crashed_test);
+
+    logging.info("Resuming layout test runner...");
+    # Call LayoutTestsAutoTest::resumeLayoutTests
+    run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
+
+    adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
+
+  if adb_output.find('INSTRUMENTATION_FAILED') != -1:
+    logging.error("Error happened : " + adb_output)
+    sys.exit(1)
+
+  logging.debug(adb_output);
+  logging.info("Done\n");
+
+  # Pull results from /sdcard
+  results_dir = options.results_directory
+  if not os.path.exists(results_dir):
+    os.makedirs(results_dir)
+  if not os.path.isdir(results_dir):
+    logging.error("Cannot create results dir: " + results_dir);
+    sys.exit(1);
+
+  result_files = ["/sdcard/layout_tests_passed.txt",
+                  "/sdcard/layout_tests_failed.txt",
+                  "/sdcard/layout_tests_ignored.txt",
+                  "/sdcard/layout_tests_nontext.txt"]
+  for file in result_files:
+    shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
+    adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
+    logging.debug(adb_output)
+
+  # Create the crash list.
+  fp = open(results_dir + "/layout_tests_crashed.txt", "w");
+  for crashed_test in crashed_tests:
+    fp.writelines(crashed_test + '\n')
+  fp.close()
+
+  # Count the number of tests in each category.
+  passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
+  logging.info(str(passed_tests) + " passed")
+  failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
+  logging.info(str(failed_tests) + " failed")
+  ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
+  logging.info(str(ignored_tests) + " ignored results")
+  crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
+  logging.info(str(crashed_tests) + " crashed")
+  nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
+  logging.info(str(nontext_tests) + " no dumpAsText")
+  logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")
+
+  logging.info("Results are stored under: " + results_dir + "\n")
+
+  # Comparing results to references to find new fixes and regressions.
+  results_dir = os.path.abspath(options.results_directory)
+  ref_dir = options.ref_directory
+
+  # if ref_dir is null, cannonify ref_dir to the script dir.
+  if not ref_dir:
+    script_self = sys.argv[0]
+    script_dir = os.path.dirname(script_self)
+    ref_dir = os.path.join(script_dir, "results")
+
+  ref_dir = os.path.abspath(ref_dir)
+
+  CompareResults(ref_dir, results_dir)
+
+if '__main__' == __name__:
+  option_parser = optparse.OptionParser()
+  option_parser.add_option("", "--rebaseline", action="store_true",
+                           default=False,
+                           help="generate expected results for those tests not having one")
+  option_parser.add_option("", "--time-out-ms",
+                           default=None,
+                           help="set the timeout for each test")
+  option_parser.add_option("", "--verbose", action="store_true",
+                           default=False,
+                           help="include debug-level logging")
+  option_parser.add_option("", "--refresh-test-list", action="store_true",
+                           default=False,
+                           help="re-generate test list, it may take some time.")
+  option_parser.add_option("", "--adb-options",
+                           default=None,
+                           help="pass options to adb, such as -d -e, etc");
+  option_parser.add_option("", "--results-directory",
+                           default="layout-test-results",
+                           help="directory which results are stored.")
+  option_parser.add_option("", "--ref-directory",
+                           default=None,
+                           dest="ref_directory",
+                           help="directory where reference results are stored.")
+  option_parser.add_option("", "--js-engine",
+                           default=None,
+                           help="The JavaScript engine currently in use, which determines which set of Android-specific expected results we should use. Should be 'jsc' or 'v8'.");
+
+  options, args = option_parser.parse_args();
+  main(options, args)

base/tests/DumpRenderTree/assets/run_page_cycler.py

@@ -0,0 +1,134 @@
+#!/usr/bin/python
+
+"""Run page cycler tests using Android instrumentation.
+
+  First, you need to get an SD card or sdcard image that has page cycler tests.
+
+  Usage:
+    Run a single page cycler test:
+      run_page_cycler.py "file:///sdcard/android/page_cycler/moz/start.html?auto=1\&iterations=10"
+"""
+
+import logging
+import optparse
+import os
+import subprocess
+import sys
+import time
+
+
+
+def main(options, args):
+  """Run the tests. Will call sys.exit when complete.
+
+  """
+
+  # Set up logging format.
+  log_level = logging.INFO
+  if options.verbose:
+    log_level = logging.DEBUG
+  logging.basicConfig(level=log_level,
+                      format='%(message)s')
+
+  # Include all tests if none are specified.
+  if not args:
+    print "need a URL, e.g. file:///sdcard/android/page_cycler/moz/start.html"
+    sys.exit(1)
+  else:
+    path = ' '.join(args);
+
+  adb_cmd = "adb ";
+  if options.adb_options:
+    adb_cmd += options.adb_options
+
+  logging.info("Running the test ...")
+
+  # Count crashed tests.
+  crashed_tests = []
+
+  timeout_ms = '0'
+  if options.time_out_ms:
+    timeout_ms = options.time_out_ms
+
+  # Run test until it's done
+
+  run_load_test_cmd_prefix = adb_cmd + " shell am instrument"
+  run_load_test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
+
+  # Call LoadTestsAutoTest::runTest.
+  run_load_test_cmd = run_load_test_cmd_prefix + " -e class com.android.dumprendertree.LoadTestsAutoTest#runPageCyclerTest -e path \"" + path + "\" -e timeout " + timeout_ms
+
+  if options.drawtime:
+    run_load_test_cmd += " -e drawtime true "
+
+  if options.save_image:
+    run_load_test_cmd += " -e saveimage \"%s\"" % options.save_image
+
+  run_load_test_cmd += run_load_test_cmd_postfix
+
+  (adb_output, adb_error) = subprocess.Popen(run_load_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+  fail_flag = False
+  for line in adb_output.splitlines():
+    line = line.strip()
+    if line.find('INSTRUMENTATION_CODE') == 0:
+      if not line[22:] == '-1':
+        fail_flag = True
+        break
+    if (line.find('INSTRUMENTATION_FAILED') != -1 or
+        line.find('Process crashed.') != -1):
+      fail_flag = True
+      break
+  if fail_flag:
+    logging.error("Error happened : " + adb_output)
+    sys.exit(1)
+
+  logging.info(adb_output);
+  logging.info(adb_error);
+  logging.info("Done\n");
+
+  # Pull results from /sdcard/load_test_result.txt
+  results_dir = options.results_directory
+  if not os.path.exists(results_dir):
+    os.makedirs(results_dir)
+  if not os.path.isdir(results_dir):
+    logging.error("Cannot create results dir: " + results_dir)
+    sys.exit(1)
+
+  result_file = "/sdcard/load_test_result.txt"
+  shell_cmd_str = adb_cmd + " pull " + result_file + " " + results_dir
+  (adb_output, err) = subprocess.Popen(
+      shell_cmd_str, shell=True,
+      stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+  if not os.path.isfile(os.path.join(results_dir, "load_test_result.txt")):
+    logging.error("Failed to pull result file.")
+    logging.error("adb stdout:")
+    logging.error(adb_output)
+    logging.error("adb stderr:")
+    logging.error(err)
+  logging.info("Results are stored under: " + results_dir + "/load_test_result.txt\n")
+
+if '__main__' == __name__:
+  option_parser = optparse.OptionParser()
+  option_parser.add_option("-t", "--time-out-ms",
+                           default=None,
+                           help="set the timeout for each test")
+  option_parser.add_option("-v", "--verbose", action="store_true",
+                           default=False,
+                           help="include debug-level logging")
+  option_parser.add_option("-a", "--adb-options",
+                           default=None,
+                           help="pass options to adb, such as -d -e, etc");
+  option_parser.add_option("-r", "--results-directory",
+                           default="layout-test-results",
+                           help="directory which results are stored.")
+
+  option_parser.add_option("-d", "--drawtime", action="store_true",
+                           default=False,
+                           help="log draw time for each page rendered.")
+
+  option_parser.add_option("-s", "--save-image",
+                           default=None,
+                           help="stores rendered page to a location on device.")
+
+  options, args = option_parser.parse_args();
+  main(options, args)

base/tests/DumpRenderTree/assets/run_reliability_tests.py

@@ -0,0 +1,276 @@
+#!/usr/bin/python2.4
+
+"""Run reliability tests using Android instrumentation.
+
+  A test file consists of list web sites to test is needed as a parameter
+
+  Usage:
+    run_reliability_tests.py path/to/url/list
+"""
+
+import logging
+import optparse
+import os
+import subprocess
+import sys
+import time
+from Numeric import *
+
+TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
+TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
+TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
+TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt"
+HTTP_URL_FILE = "urllist_http"
+HTTPS_URL_FILE = "urllist_https"
+NUM_URLS = 25
+
+
+def DumpRenderTreeFinished(adb_cmd):
+  """Check if DumpRenderTree finished running.
+
+  Args:
+    adb_cmd: adb command string
+
+  Returns:
+    True if DumpRenderTree has finished, False otherwise
+  """
+
+  # pull test status file and look for "#DONE"
+  shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
+  adb_output = subprocess.Popen(shell_cmd_str,
+                                shell=True, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE).communicate()[0]
+  return adb_output.strip() == "#DONE"
+
+
+def RemoveDeviceFile(adb_cmd, file_name):
+  shell_cmd_str = adb_cmd + " shell rm " + file_name
+  subprocess.Popen(shell_cmd_str,
+                   shell=True, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE).communicate()
+
+
+def Bugreport(url, bugreport_dir, adb_cmd):
+  """Pull a bugreport from the device."""
+  bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
+                                                            int(time.time()))
+
+  # prepend the report with url
+  handle = open(bugreport_filename, "w")
+  handle.writelines("Bugreport for crash in url - %s\n\n" % url)
+  handle.close()
+
+  cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
+  os.system(cmd)
+
+
+def ProcessPageLoadTime(raw_log):
+  """Processes the raw page load time logged by test app."""
+  log_handle = open(raw_log, "r")
+  load_times = {}
+
+  for line in log_handle:
+    line = line.strip()
+    pair = line.split("|")
+    if len(pair) != 2:
+      logging.info("Line has more than one '|': " + line)
+      continue
+    if pair[0] not in load_times:
+      load_times[pair[0]] = []
+    try:
+      pair[1] = int(pair[1])
+    except ValueError:
+      logging.info("Lins has non-numeric load time: " + line)
+      continue
+    load_times[pair[0]].append(pair[1])
+
+  log_handle.close()
+
+  # rewrite the average time to file
+  log_handle = open(raw_log, "w")
+  for url, times in load_times.iteritems():
+    # calculate std
+    arr = array(times)
+    avg = average(arr)
+    d = arr - avg
+    std = sqrt(sum(d * d) / len(arr))
+    output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" %
+              (url, min(arr), max(arr), avg, std,
+               array2string(arr)))
+    log_handle.write(output)
+  log_handle.close()
+
+
+def main(options, args):
+  """Send the url list to device and start testing, restart if crashed."""
+
+  # Set up logging format.
+  log_level = logging.INFO
+  if options.verbose:
+    log_level = logging.DEBUG
+  logging.basicConfig(level=log_level,
+                      format="%(message)s")
+
+  # Include all tests if none are specified.
+  if not args:
+    print "Missing URL list file"
+    sys.exit(1)
+  else:
+    path = args[0]
+
+  if not options.crash_file:
+    print "Missing crash file name, use --crash-file to specify"
+    sys.exit(1)
+  else:
+    crashed_file = options.crash_file
+
+  if not options.timeout_file:
+    print "Missing timeout file, use --timeout-file to specify"
+    sys.exit(1)
+  else:
+    timedout_file = options.timeout_file
+
+  if not options.delay:
+    manual_delay = 0
+  else:
+    manual_delay = options.delay
+
+  if not options.bugreport:
+    bugreport_dir = "."
+  else:
+    bugreport_dir = options.bugreport
+  if not os.path.exists(bugreport_dir):
+    os.makedirs(bugreport_dir)
+  if not os.path.isdir(bugreport_dir):
+    logging.error("Cannot create results dir: " + bugreport_dir)
+    sys.exit(1)
+
+  adb_cmd = "adb "
+  if options.adb_options:
+    adb_cmd += options.adb_options + " "
+
+  # push url list to device
+  test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
+  proc = subprocess.Popen(test_cmd, shell=True,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.PIPE)
+  (adb_output, adb_error) = proc.communicate()
+  if proc.returncode != 0:
+    logging.error("failed to push url list to device.")
+    logging.error(adb_output)
+    logging.error(adb_error)
+    sys.exit(1)
+
+  # clean up previous results
+  RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
+  RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
+  RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE)
+
+  logging.info("Running the test ...")
+
+  # Count crashed tests.
+  crashed_tests = []
+
+  if options.time_out_ms:
+    timeout_ms = options.time_out_ms
+
+  # Run test until it's done
+  test_cmd_prefix = adb_cmd + " shell am instrument"
+  test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
+
+  # Call ReliabilityTestsAutoTest#startReliabilityTests
+  test_cmd = (test_cmd_prefix + " -e class "
+              "com.android.dumprendertree.ReliabilityTest#"
+              "runReliabilityTest -e timeout %s -e delay %s" %
+              (str(timeout_ms), str(manual_delay)))
+
+  if options.logtime:
+    test_cmd += " -e logtime true"
+
+  test_cmd += test_cmd_postfix
+
+  adb_output = subprocess.Popen(test_cmd, shell=True,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE).communicate()[0]
+  while not DumpRenderTreeFinished(adb_cmd):
+    logging.error("DumpRenderTree exited before all URLs are visited.")
+    shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
+    crashed_test = ""
+    while not crashed_test:
+      (crashed_test, err) = subprocess.Popen(
+          shell_cmd_str, shell=True, stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE).communicate()
+      crashed_test = crashed_test.strip()
+      if not crashed_test:
+        logging.error('Cannot get crashed test name, device offline?')
+        logging.error('stderr: ' + err)
+        logging.error('retrying in 10s...')
+        time.sleep(10)
+
+    logging.info(crashed_test + " CRASHED")
+    crashed_tests.append(crashed_test)
+    Bugreport(crashed_test, bugreport_dir, adb_cmd)
+    logging.info("Resuming reliability test runner...")
+
+    adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE).communicate()[0]
+
+  if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
+      adb_output.find("Process crashed.") != -1):
+    logging.error("Error happened : " + adb_output)
+    sys.exit(1)
+
+  logging.info(adb_output)
+  logging.info("Done\n")
+
+  if crashed_tests:
+    file_handle = open(crashed_file, "w")
+    file_handle.writelines("\n".join(crashed_tests))
+    logging.info("Crashed URL list stored in: " + crashed_file)
+    file_handle.close()
+  else:
+    logging.info("No crash found.")
+
+  # get timeout file from sdcard
+  test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
+              + timedout_file +  "\"")
+  subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE).communicate()
+
+  if options.logtime:
+    # get logged page load times from sdcard
+    test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \""
+                + options.logtime +  "\"")
+    subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
+                     stderr=subprocess.PIPE).communicate()
+    ProcessPageLoadTime(options.logtime)
+
+
+if "__main__" == __name__:
+  option_parser = optparse.OptionParser()
+  option_parser.add_option("-t", "--time-out-ms",
+                           default=60000,
+                           help="set the timeout for each test")
+  option_parser.add_option("-v", "--verbose", action="store_true",
+                           default=False,
+                           help="include debug-level logging")
+  option_parser.add_option("-a", "--adb-options",
+                           default=None,
+                           help="pass options to adb, such as -d -e, etc")
+  option_parser.add_option("-c", "--crash-file",
+                           default="reliability_crashed_sites.txt",
+                           help="the list of sites that cause browser to crash")
+  option_parser.add_option("-f", "--timeout-file",
+                           default="reliability_timedout_sites.txt",
+                           help="the list of sites that timedout during test")
+  option_parser.add_option("-d", "--delay",
+                           default=0,
+                           help="add a manual delay between pages (in ms)")
+  option_parser.add_option("-b", "--bugreport",
+                           default=".",
+                           help="the directory to store bugreport for crashes")
+  option_parser.add_option("-l", "--logtime",
+                           default=None,
+                           help="Logs page load time for each url to the file")
+  opts, arguments = option_parser.parse_args()
+  main(opts, arguments)