OLD | NEW |
(Empty) | |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. |
| 4 |
| 5 """Test runners for iOS.""" |
| 6 |
| 7 import argparse |
| 8 import collections |
| 9 import errno |
| 10 import os |
| 11 import shutil |
| 12 import subprocess |
| 13 import sys |
| 14 import tempfile |
| 15 import time |
| 16 |
| 17 import find_xcode |
| 18 import gtest_utils |
| 19 import xctest_utils |
| 20 |
| 21 |
| 22 XCTEST_PROJECT = os.path.abspath(os.path.join( |
| 23 os.path.dirname(__file__), |
| 24 'TestProject', |
| 25 'TestProject.xcodeproj', |
| 26 )) |
| 27 |
| 28 XCTEST_SCHEME = 'TestProject' |
| 29 |
| 30 |
| 31 class Error(Exception): |
| 32 """Base class for errors.""" |
| 33 pass |
| 34 |
| 35 |
| 36 class TestRunnerError(Error): |
| 37 """Base class for TestRunner-related errors.""" |
| 38 pass |
| 39 |
| 40 |
| 41 class AppLaunchError(TestRunnerError): |
| 42 """The app failed to launch.""" |
| 43 pass |
| 44 |
| 45 |
| 46 class AppNotFoundError(TestRunnerError): |
| 47 """The requested app was not found.""" |
| 48 def __init__(self, app_path): |
| 49 super(AppNotFoundError, self).__init__( |
| 50 'App does not exist: %s' % app_path) |
| 51 |
| 52 |
| 53 class DeviceDetectionError(TestRunnerError): |
| 54 """Unexpected number of devices detected.""" |
| 55 def __init__(self, udids): |
| 56 super(DeviceDetectionError, self).__init__( |
| 57 'Expected one device, found %s:\n%s' % (len(udids), '\n'.join(udids))) |
| 58 |
| 59 |
| 60 class PlugInsNotFoundError(TestRunnerError): |
| 61 """The PlugIns directory was not found.""" |
| 62 def __init__(self, plugins_dir): |
| 63 super(PlugInsNotFoundError, self).__init__( |
| 64 'PlugIns directory does not exist: %s' % plugins_dir) |
| 65 |
| 66 |
| 67 class SimulatorNotFoundError(TestRunnerError): |
| 68 """The given simulator binary was not found.""" |
| 69 def __init__(self, iossim_path): |
| 70 super(SimulatorNotFoundError, self).__init__( |
| 71 'Simulator does not exist: %s' % iossim_path) |
| 72 |
| 73 |
| 74 class XcodeVersionNotFoundError(TestRunnerError): |
| 75 """The requested version of Xcode was not found.""" |
| 76 def __init__(self, xcode_version): |
| 77 super(XcodeVersionNotFoundError, self).__init__( |
| 78 'Xcode version not found: %s', xcode_version) |
| 79 |
| 80 |
| 81 class XCTestPlugInNotFoundError(TestRunnerError): |
| 82 """The .xctest PlugIn was not found.""" |
| 83 def __init__(self, xctest_path): |
| 84 super(XCTestPlugInNotFoundError, self).__init__( |
| 85 'XCTest not found: %s', xctest_path) |
| 86 |
| 87 |
| 88 def get_kif_test_filter(tests, invert=False): |
| 89 """Returns the KIF test filter to filter the given test cases. |
| 90 |
| 91 Args: |
| 92 tests: List of test cases to filter. |
| 93 invert: Whether to invert the filter or not. Inverted, the filter will match |
| 94 everything except the given test cases. |
| 95 |
| 96 Returns: |
| 97 A string which can be supplied to GKIF_SCENARIO_FILTER. |
| 98 """ |
| 99 # A pipe-separated list of test cases with the "KIF." prefix omitted. |
| 100 # e.g. NAME:a|b|c matches KIF.a, KIF.b, KIF.c. |
| 101 # e.g. -NAME:a|b|c matches everything except KIF.a, KIF.b, KIF.c. |
| 102 test_filter = '|'.join(test.split('KIF.', 1)[-1] for test in tests) |
| 103 if invert: |
| 104 return '-NAME:%s' % test_filter |
| 105 return 'NAME:%s' % test_filter |
| 106 |
| 107 |
| 108 def get_gtest_filter(tests, invert=False): |
| 109 """Returns the GTest filter to filter the given test cases. |
| 110 |
| 111 Args: |
| 112 tests: List of test cases to filter. |
| 113 invert: Whether to invert the filter or not. Inverted, the filter will match |
| 114 everything except the given test cases. |
| 115 |
| 116 Returns: |
| 117 A string which can be supplied to --gtest_filter. |
| 118 """ |
| 119 # A colon-separated list of tests cases. |
| 120 # e.g. a:b:c matches a, b, c. |
| 121 # e.g. -a:b:c matches everything except a, b, c. |
| 122 test_filter = ':'.join(test for test in tests) |
| 123 if invert: |
| 124 return '-%s' % test_filter |
| 125 return test_filter |
| 126 |
| 127 |
| 128 class TestRunner(object): |
| 129 """Base class containing common functionality.""" |
| 130 |
| 131 def __init__( |
| 132 self, |
| 133 app_path, |
| 134 xcode_version, |
| 135 out_dir, |
| 136 env_vars=None, |
| 137 test_args=None, |
| 138 xctest=False, |
| 139 ): |
| 140 """Initializes a new instance of this class. |
| 141 |
| 142 Args: |
| 143 app_path: Path to the compiled .app to run. |
| 144 xcode_version: Version of Xcode to use when running the test. |
| 145 out_dir: Directory to emit test data into. |
| 146 env_vars: List of environment variables to pass to the test itself. |
| 147 test_args: List of strings to pass as arguments to the test when |
| 148 launching. |
| 149 xctest: Whether or not this is an XCTest. |
| 150 |
| 151 Raises: |
| 152 AppNotFoundError: If the given app does not exist. |
| 153 PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests. |
| 154 XcodeVersionNotFoundError: If the given Xcode version does not exist. |
| 155 XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist. |
| 156 """ |
| 157 app_path = os.path.abspath(app_path) |
| 158 if not os.path.exists(app_path): |
| 159 raise AppNotFoundError(app_path) |
| 160 |
| 161 if not find_xcode.find_xcode(xcode_version)['found']: |
| 162 raise XcodeVersionNotFoundError(xcode_version) |
| 163 |
| 164 if not os.path.exists(out_dir): |
| 165 os.makedirs(out_dir) |
| 166 |
| 167 self.app_name = os.path.splitext(os.path.split(app_path)[-1])[0] |
| 168 self.app_path = app_path |
| 169 self.cfbundleid = subprocess.check_output([ |
| 170 '/usr/libexec/PlistBuddy', |
| 171 '-c', 'Print:CFBundleIdentifier', |
| 172 os.path.join(app_path, 'Info.plist'), |
| 173 ]).rstrip() |
| 174 self.env_vars = env_vars or [] |
| 175 self.logs = collections.OrderedDict() |
| 176 self.out_dir = out_dir |
| 177 self.test_args = test_args or [] |
| 178 self.xcode_version = xcode_version |
| 179 self.xctest_path = '' |
| 180 |
| 181 if xctest: |
| 182 plugins_dir = os.path.join(self.app_path, 'PlugIns') |
| 183 if not os.path.exists(plugins_dir): |
| 184 raise PlugInsNotFoundError(plugins_dir) |
| 185 for plugin in os.listdir(plugins_dir): |
| 186 if plugin.endswith('.xctest'): |
| 187 self.xctest_path = os.path.join(plugins_dir, plugin) |
| 188 if not os.path.exists(self.xctest_path): |
| 189 raise XCTestPlugInNotFoundError(self.xctest_path) |
| 190 |
| 191 def get_launch_command(self, test_filter=None, invert=False): |
| 192 """Returns the command that can be used to launch the test app. |
| 193 |
| 194 Args: |
| 195 test_filter: List of test cases to filter. |
| 196 invert: Whether to invert the filter or not. Inverted, the filter will |
| 197 match everything except the given test cases. |
| 198 |
| 199 Returns: |
| 200 A list of strings forming the command to launch the test. |
| 201 """ |
| 202 raise NotImplementedError |
| 203 |
| 204 def get_launch_env(self): |
| 205 """Returns a dict of environment variables to use to launch the test app. |
| 206 |
| 207 Returns: |
| 208 A dict of environment variables. |
| 209 """ |
| 210 return os.environ.copy() |
| 211 |
| 212 def set_up(self): |
| 213 """Performs setup actions which must occur prior to every test launch.""" |
| 214 raise NotImplementedError |
| 215 |
| 216 def tear_down(self): |
| 217 """Performs cleanup actions which must occur after every test launch.""" |
| 218 raise NotImplementedError |
| 219 |
| 220 def screenshot_desktop(self): |
| 221 """Saves a screenshot of the desktop in the output directory.""" |
| 222 subprocess.check_call([ |
| 223 'screencapture', |
| 224 os.path.join(self.out_dir, 'desktop_%s.png' % time.time()), |
| 225 ]) |
| 226 |
| 227 def _run(self, cmd): |
| 228 """Runs the specified command, parsing GTest output. |
| 229 |
| 230 Args: |
| 231 cmd: List of strings forming the command to run. |
| 232 |
| 233 Returns: |
| 234 GTestResult instance. |
| 235 """ |
| 236 print ' '.join(cmd) |
| 237 print |
| 238 |
| 239 result = gtest_utils.GTestResult(cmd) |
| 240 if self.xctest_path: |
| 241 parser = xctest_utils.XCTestLogParser() |
| 242 else: |
| 243 parser = gtest_utils.GTestLogParser() |
| 244 |
| 245 proc = subprocess.Popen( |
| 246 cmd, |
| 247 env=self.get_launch_env(), |
| 248 stdout=subprocess.PIPE, |
| 249 stderr=subprocess.STDOUT, |
| 250 ) |
| 251 |
| 252 while True: |
| 253 line = proc.stdout.readline() |
| 254 if not line: |
| 255 break |
| 256 line = line.rstrip() |
| 257 parser.ProcessLine(line) |
| 258 print line |
| 259 sys.stdout.flush() |
| 260 |
| 261 proc.wait() |
| 262 sys.stdout.flush() |
| 263 |
| 264 for test in parser.FailedTests(include_flaky=True): |
| 265 # Test cases are named as <test group>.<test case>. If the test case |
| 266 # is prefixed with "FLAKY_", it should be reported as flaked not failed. |
| 267 if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): |
| 268 result.flaked_tests[test] = parser.FailureDescription(test) |
| 269 else: |
| 270 result.failed_tests[test] = parser.FailureDescription(test) |
| 271 |
| 272 result.passed_tests.extend(parser.PassedTests(include_flaky=True)) |
| 273 |
| 274 print '%s returned %s' % (cmd[0], proc.returncode) |
| 275 print |
| 276 |
| 277 # iossim can return 5 if it exits noncleanly even if all tests passed. |
| 278 # Therefore we cannot rely on process exit code to determine success. |
| 279 result.finalize(proc.returncode, parser.CompletedWithoutFailure()) |
| 280 return result |
| 281 |
| 282 def launch(self): |
| 283 """Launches the test app.""" |
| 284 self.set_up() |
| 285 cmd = self.get_launch_command() |
| 286 try: |
| 287 result = self._run(cmd) |
| 288 if result.crashed and not result.crashed_test: |
| 289 # If the app crashed but not during any particular test case, assume |
| 290 # it crashed on startup. Try one more time. |
| 291 print 'Crashed on startup, retrying...' |
| 292 print |
| 293 result = self._run(cmd) |
| 294 |
| 295 if result.crashed and not result.crashed_test: |
| 296 raise AppLaunchError |
| 297 |
| 298 passed = result.passed_tests |
| 299 failed = result.failed_tests |
| 300 flaked = result.flaked_tests |
| 301 |
| 302 try: |
| 303 # XCTests cannot currently be resumed at the next test case. |
| 304 while not self.xctest_path and result.crashed and result.crashed_test: |
| 305 # If the app crashes during a specific test case, then resume at the |
| 306 # next test case. This is achieved by filtering out every test case |
| 307 # which has already run. |
| 308 print 'Crashed during %s, resuming...' % result.crashed_test |
| 309 print |
| 310 result = self._run(self.get_launch_command( |
| 311 test_filter=passed + failed.keys() + flaked.keys(), invert=True, |
| 312 )) |
| 313 passed.extend(result.passed_tests) |
| 314 failed.update(result.failed_tests) |
| 315 flaked.update(result.flaked_tests) |
| 316 except OSError as e: |
| 317 if e.errno == errno.E2BIG: |
| 318 print 'Too many test cases to resume.' |
| 319 print |
| 320 else: |
| 321 raise |
| 322 |
| 323 self.logs['passed tests'] = passed |
| 324 for test, log_lines in failed.iteritems(): |
| 325 self.logs[test] = log_lines |
| 326 for test, log_lines in flaked.iteritems(): |
| 327 self.logs[test] = log_lines |
| 328 |
| 329 return not failed |
| 330 finally: |
| 331 self.tear_down() |
| 332 |
| 333 |
| 334 class SimulatorTestRunner(TestRunner): |
| 335 """Class for running tests on iossim.""" |
| 336 |
| 337 def __init__( |
| 338 self, |
| 339 app_path, |
| 340 iossim_path, |
| 341 platform, |
| 342 version, |
| 343 xcode_version, |
| 344 out_dir, |
| 345 env_vars=None, |
| 346 test_args=None, |
| 347 xctest=False, |
| 348 ): |
| 349 """Initializes a new instance of this class. |
| 350 |
| 351 Args: |
| 352 app_path: Path to the compiled .app or .ipa to run. |
| 353 iossim_path: Path to the compiled iossim binary to use. |
| 354 platform: Name of the platform to simulate. Supported values can be found |
| 355 by running "iossim -l". e.g. "iPhone 5s", "iPad Retina". |
| 356 version: Version of iOS the platform should be running. Supported values |
| 357 can be found by running "iossim -l". e.g. "9.3", "8.2", "7.1". |
| 358 xcode_version: Version of Xcode to use when running the test. |
| 359 out_dir: Directory to emit test data into. |
| 360 env_vars: List of environment variables to pass to the test itself. |
| 361 test_args: List of strings to pass as arguments to the test when |
| 362 launching. |
| 363 xctest: Whether or not this is an XCTest. |
| 364 |
| 365 Raises: |
| 366 AppNotFoundError: If the given app does not exist. |
| 367 PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests. |
| 368 XcodeVersionNotFoundError: If the given Xcode version does not exist. |
| 369 XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist. |
| 370 """ |
| 371 super(SimulatorTestRunner, self).__init__( |
| 372 app_path, |
| 373 xcode_version, |
| 374 out_dir, |
| 375 env_vars=env_vars, |
| 376 test_args=test_args, |
| 377 xctest=xctest, |
| 378 ) |
| 379 |
| 380 iossim_path = os.path.abspath(iossim_path) |
| 381 if not os.path.exists(iossim_path): |
| 382 raise SimulatorNotFoundError(iossim_path) |
| 383 |
| 384 self.homedir = '' |
| 385 self.iossim_path = iossim_path |
| 386 self.platform = platform |
| 387 self.start_time = None |
| 388 self.version = version |
| 389 |
| 390 @staticmethod |
| 391 def kill_simulators(): |
| 392 """Kills all running simulators.""" |
| 393 try: |
| 394 subprocess.check_call([ |
| 395 'pkill', |
| 396 '-9', |
| 397 '-x', |
| 398 # The simulator's name varies by Xcode version. |
| 399 'iPhone Simulator', # Xcode 5 |
| 400 'iOS Simulator', # Xcode 6 |
| 401 'Simulator', # Xcode 7+ |
| 402 'simctl', # https://crbug.com/637429 |
| 403 ]) |
| 404 # If a signal was sent, wait for the simulators to actually be killed. |
| 405 time.sleep(5) |
| 406 except subprocess.CalledProcessError as e: |
| 407 if e.returncode != 1: |
| 408 # Ignore a 1 exit code (which means there were no simulators to kill). |
| 409 raise |
| 410 |
| 411 def wipe_simulator(self): |
| 412 """Wipes the simulator.""" |
| 413 subprocess.check_call([ |
| 414 self.iossim_path, |
| 415 '-d', self.platform, |
| 416 '-s', self.version, |
| 417 '-w', |
| 418 ]) |
| 419 |
| 420 def get_home_directory(self): |
| 421 """Returns the simulator's home directory.""" |
| 422 return subprocess.check_output([ |
| 423 self.iossim_path, |
| 424 '-d', self.platform, |
| 425 '-p', |
| 426 '-s', self.version, |
| 427 ]).rstrip() |
| 428 |
| 429 def set_up(self): |
| 430 """Performs setup actions which must occur prior to every test launch.""" |
| 431 self.kill_simulators() |
| 432 self.wipe_simulator() |
| 433 self.homedir = self.get_home_directory() |
| 434 # Crash reports have a timestamp in their file name, formatted as |
| 435 # YYYY-MM-DD-HHMMSS. Save the current time in the same format so |
| 436 # we can compare and fetch crash reports from this run later on. |
| 437 self.start_time = time.strftime('%Y-%m-%d-%H%M%S', time.localtime()) |
| 438 |
| 439 def extract_test_data(self): |
| 440 """Extracts data emitted by the test.""" |
| 441 # Find the Documents directory of the test app. The app directory names |
| 442 # don't correspond with any known information, so we have to examine them |
| 443 # all until we find one with a matching CFBundleIdentifier. |
| 444 apps_dir = os.path.join( |
| 445 self.homedir, 'Containers', 'Data', 'Application') |
| 446 if os.path.exists(apps_dir): |
| 447 for appid_dir in os.listdir(apps_dir): |
| 448 docs_dir = os.path.join(apps_dir, appid_dir, 'Documents') |
| 449 metadata_plist = os.path.join( |
| 450 apps_dir, |
| 451 appid_dir, |
| 452 '.com.apple.mobile_container_manager.metadata.plist', |
| 453 ) |
| 454 if os.path.exists(docs_dir) and os.path.exists(metadata_plist): |
| 455 cfbundleid = subprocess.check_output([ |
| 456 '/usr/libexec/PlistBuddy', |
| 457 '-c', 'Print:MCMMetadataIdentifier', |
| 458 metadata_plist, |
| 459 ]).rstrip() |
| 460 if cfbundleid == self.cfbundleid: |
| 461 shutil.copytree(docs_dir, os.path.join(self.out_dir, 'Documents')) |
| 462 return |
| 463 |
| 464 def retrieve_crash_reports(self): |
| 465 """Retrieves crash reports produced by the test.""" |
| 466 # A crash report's naming scheme is [app]_[timestamp]_[hostname].crash. |
| 467 # e.g. net_unittests_2014-05-13-15-0900_vm1-a1.crash. |
| 468 crash_reports_dir = os.path.expanduser(os.path.join( |
| 469 '~', 'Library', 'Logs', 'DiagnosticReports')) |
| 470 |
| 471 if not os.path.exists(crash_reports_dir): |
| 472 return |
| 473 |
| 474 for crash_report in os.listdir(crash_reports_dir): |
| 475 report_name, ext = os.path.splitext(crash_report) |
| 476 if report_name.startswith(self.app_name) and ext == '.crash': |
| 477 report_time = report_name[len(self.app_name) + 1:].split('_')[0] |
| 478 |
| 479 # The timestamp format in a crash report is big-endian and therefore |
| 480 # a staight string comparison works. |
| 481 if report_time > self.start_time: |
| 482 with open(os.path.join(crash_reports_dir, crash_report)) as f: |
| 483 self.logs['crash report (%s)' % report_time] = ( |
| 484 f.read().splitlines()) |
| 485 |
| 486 def tear_down(self): |
| 487 """Performs cleanup actions which must occur after every test launch.""" |
| 488 self.extract_test_data() |
| 489 self.retrieve_crash_reports() |
| 490 self.screenshot_desktop() |
| 491 self.kill_simulators() |
| 492 self.wipe_simulator() |
| 493 if os.path.exists(self.homedir): |
| 494 shutil.rmtree(self.homedir, ignore_errors=True) |
| 495 self.homedir = '' |
| 496 |
| 497 def get_launch_command(self, test_filter=None, invert=False): |
| 498 """Returns the command that can be used to launch the test app. |
| 499 |
| 500 Args: |
| 501 test_filter: List of test cases to filter. |
| 502 invert: Whether to invert the filter or not. Inverted, the filter will |
| 503 match everything except the given test cases. |
| 504 |
| 505 Returns: |
| 506 A list of strings forming the command to launch the test. |
| 507 """ |
| 508 cmd = [ |
| 509 self.iossim_path, |
| 510 '-d', self.platform, |
| 511 '-s', self.version, |
| 512 ] |
| 513 |
| 514 if test_filter: |
| 515 kif_filter = get_kif_test_filter(test_filter, invert=invert) |
| 516 gtest_filter = get_gtest_filter(test_filter, invert=invert) |
| 517 cmd.extend(['-e', 'GKIF_SCENARIO_FILTER=%s' % kif_filter]) |
| 518 cmd.extend(['-c', '--gtest_filter=%s' % gtest_filter]) |
| 519 |
| 520 for env_var in self.env_vars: |
| 521 cmd.extend(['-e', env_var]) |
| 522 |
| 523 for test_arg in self.test_args: |
| 524 cmd.extend(['-c', test_arg]) |
| 525 |
| 526 cmd.append(self.app_path) |
| 527 if self.xctest_path: |
| 528 cmd.append(self.xctest_path) |
| 529 return cmd |
| 530 |
| 531 def get_launch_env(self): |
| 532 """Returns a dict of environment variables to use to launch the test app. |
| 533 |
| 534 Returns: |
| 535 A dict of environment variables. |
| 536 """ |
| 537 env = super(SimulatorTestRunner, self).get_launch_env() |
| 538 if self.xctest_path: |
| 539 env['NSUnbufferedIO'] = 'YES' |
| 540 return env |
| 541 |
| 542 |
| 543 class DeviceTestRunner(TestRunner): |
| 544 """Class for running tests on devices.""" |
| 545 |
| 546 def __init__( |
| 547 self, |
| 548 app_path, |
| 549 xcode_version, |
| 550 out_dir, |
| 551 env_vars=None, |
| 552 test_args=None, |
| 553 xctest=False, |
| 554 ): |
| 555 """Initializes a new instance of this class. |
| 556 |
| 557 Args: |
| 558 app_path: Path to the compiled .app to run. |
| 559 xcode_version: Version of Xcode to use when running the test. |
| 560 out_dir: Directory to emit test data into. |
| 561 env_vars: List of environment variables to pass to the test itself. |
| 562 test_args: List of strings to pass as arguments to the test when |
| 563 launching. |
| 564 xctest: Whether or not this is an XCTest. |
| 565 |
| 566 Raises: |
| 567 AppNotFoundError: If the given app does not exist. |
| 568 PlugInsNotFoundError: If the PlugIns directory does not exist for XCTests. |
| 569 XcodeVersionNotFoundError: If the given Xcode version does not exist. |
| 570 XCTestPlugInNotFoundError: If the .xctest PlugIn does not exist. |
| 571 """ |
| 572 super(DeviceTestRunner, self).__init__( |
| 573 app_path, |
| 574 xcode_version, |
| 575 out_dir, |
| 576 env_vars=env_vars, |
| 577 test_args=test_args, |
| 578 xctest=xctest, |
| 579 ) |
| 580 |
| 581 self.udid = subprocess.check_output(['idevice_id', '--list']).rstrip() |
| 582 if len(self.udid.splitlines()) != 1: |
| 583 raise DeviceDetectionError(self.udid) |
| 584 |
| 585 def uninstall_apps(self): |
| 586 """Uninstalls all apps found on the device.""" |
| 587 for app in subprocess.check_output( |
| 588 ['idevicefs', '--udid', self.udid, 'ls', '@']).splitlines(): |
| 589 subprocess.check_call( |
| 590 ['ideviceinstaller', '--udid', self.udid, '--uninstall', app]) |
| 591 |
| 592 def install_app(self): |
| 593 """Installs the app.""" |
| 594 subprocess.check_call( |
| 595 ['ideviceinstaller', '--udid', self.udid, '--install', self.app_path]) |
| 596 |
| 597 def set_up(self): |
| 598 """Performs setup actions which must occur prior to every test launch.""" |
| 599 self.uninstall_apps() |
| 600 self.install_app() |
| 601 |
| 602 def extract_test_data(self): |
| 603 """Extracts data emitted by the test.""" |
| 604 subprocess.check_call([ |
| 605 'idevicefs', |
| 606 '--udid', self.udid, |
| 607 'pull', |
| 608 '@%s/Documents' % self.cfbundleid, |
| 609 os.path.join(self.out_dir, 'Documents'), |
| 610 ]) |
| 611 |
| 612 def retrieve_crash_reports(self): |
| 613 """Retrieves crash reports produced by the test.""" |
| 614 logs_dir = os.path.join(self.out_dir, 'Logs') |
| 615 os.mkdir(logs_dir) |
| 616 subprocess.check_call([ |
| 617 'idevicecrashreport', |
| 618 '--extract', |
| 619 '--udid', self.udid, |
| 620 logs_dir, |
| 621 ]) |
| 622 |
| 623 def tear_down(self): |
| 624 """Performs cleanup actions which must occur after every test launch.""" |
| 625 self.extract_test_data() |
| 626 self.retrieve_crash_reports() |
| 627 self.screenshot_desktop() |
| 628 self.uninstall_apps() |
| 629 |
| 630 def get_launch_command(self, test_filter=None, invert=False): |
| 631 """Returns the command that can be used to launch the test app. |
| 632 |
| 633 Args: |
| 634 test_filter: List of test cases to filter. |
| 635 invert: Whether to invert the filter or not. Inverted, the filter will |
| 636 match everything except the given test cases. |
| 637 |
| 638 Returns: |
| 639 A list of strings forming the command to launch the test. |
| 640 """ |
| 641 if self.xctest_path: |
| 642 return [ |
| 643 'xcodebuild', |
| 644 'test-without-building', |
| 645 'BUILT_PRODUCTS_DIR=%s' % os.path.dirname(self.app_path), |
| 646 '-destination', 'id=%s' % self.udid, |
| 647 '-project', XCTEST_PROJECT, |
| 648 '-scheme', XCTEST_SCHEME, |
| 649 ] |
| 650 |
| 651 cmd = [ |
| 652 'idevice-app-runner', |
| 653 '--udid', self.udid, |
| 654 '--start', self.cfbundleid, |
| 655 ] |
| 656 args = [] |
| 657 |
| 658 if test_filter: |
| 659 kif_filter = get_kif_test_filter(test_filter, invert=invert) |
| 660 gtest_filter = get_gtest_filter(test_filter, invert=invert) |
| 661 cmd.extend(['-D', 'GKIF_SCENARIO_FILTER=%s' % kif_filter]) |
| 662 args.append('--gtest-filter=%s' % gtest_filter) |
| 663 |
| 664 for env_var in self.env_vars: |
| 665 cmd.extend(['-D', env_var]) |
| 666 |
| 667 if args or self.test_args: |
| 668 cmd.append('--args') |
| 669 cmd.extend(self.test_args) |
| 670 cmd.extend(args) |
| 671 |
| 672 return cmd |
| 673 |
| 674 def get_launch_env(self): |
| 675 """Returns a dict of environment variables to use to launch the test app. |
| 676 |
| 677 Returns: |
| 678 A dict of environment variables. |
| 679 """ |
| 680 env = super(DeviceTestRunner, self).get_launch_env() |
| 681 if self.xctest_path: |
| 682 env['NSUnbufferedIO'] = 'YES' |
| 683 # e.g. ios_web_shell_egtests |
| 684 env['APP_TARGET_NAME'] = os.path.splitext( |
| 685 os.path.basename(self.app_path))[0] |
| 686 # e.g. ios_web_shell_egtests_module |
| 687 env['TEST_TARGET_NAME'] = env['APP_TARGET_NAME'] + '_module' |
| 688 return env |
OLD | NEW |