Index: experimental/soundwave/alert_analyzer.py |
diff --git a/experimental/soundwave/alert_analyzer.py b/experimental/soundwave/alert_analyzer.py |
new file mode 100755 |
index 0000000000000000000000000000000000000000..569dab3969365bfef11545aeca01a1b3b1e09a09 |
--- /dev/null |
+++ b/experimental/soundwave/alert_analyzer.py |
@@ -0,0 +1,210 @@ |
+#!/usr/bin/env python |
+# Copyright 2017 The Chromium Authors. All rights reserved. |
+# Use of this source code is governed by a BSD-style license that can be |
+# found in the LICENSE file. |
+ |
+import argparse |
+import httplib2 |
+import json |
+import numpy |
+from oauth2client import client |
+from oauth2client import service_account # pylint: disable=no-name-in-module |
+import time |
+ |
+ |
+# TODO(rnephew): Integrate into catapult/experimental/benchmark_health_report. |
+REQUEST_URL = 'https://chromeperf.appspot.com/api/' |
+# pylint: disable=line-too-long |
+HELP_SITE = 'https://developers.google.com/api-client-library/python/auth/service-accounts#creatinganaccount' |
+ |
+OAUTH_CLIENT_ID = ( |
+ '62121018386-h08uiaftreu4dr3c4alh3l7mogskvb7i.apps.googleusercontent.com') |
+OAUTH_CLIENT_SECRET = 'vc1fZfV1cZC6mgDSHV-KSPOz' |
+SCOPES = 'https://www.googleapis.com/auth/userinfo.email' |
+ |
+ |
+def AuthorizeAccount(args): |
+ """A factory for authorized account credentials.""" |
+ if args.credentials: |
+ try: |
+ return AuthorizeAccountServiceAccount(args.credentials) |
+ except Exception: # pylint: disable=broad-except |
+ print ('Failure authenticating with service account. Falling back to user' |
+ ' authentication.') |
+ return AuthorizeAccountUserAccount() |
+ |
+ |
+def AuthorizeAccountServiceAccount(json_key): |
+ """Used to create a service account connection with the performance dashboard. |
+ |
+ args: |
+ json_key: Path to json file that contains credentials. |
+ returns: |
+ An object that can be used to communicate with the dashboard. |
+ """ |
+ creds = service_account.ServiceAccountCredentials.from_json_keyfile_name( |
+ json_key, [SCOPES]) |
+ return creds.authorize(httplib2.Http()) |
+ |
+ |
+def AuthorizeAccountUserAccount(): |
+ """Used to create an user account connection with the performance dashboard. |
+ |
+ returns: |
+ An object that can be used to communicate with the dashboard. |
+ """ |
+ flow = client.OAuth2WebServerFlow( |
+ OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET, [SCOPES], approval_prompt='force') |
+ flow.redirect_uri = client.OOB_CALLBACK_URN |
+ print('Go to the followinhg link in your browser:\n' |
+ ' %s\n' % flow.step1_get_authorize_url()) |
+ code = raw_input('Enter verification code: ').strip() |
+ try: |
+ creds = flow.step2_exchange(code) |
+ return creds.authorize(httplib2.Http()) |
+ except client.FlowExchangeError: |
+ print 'User authentication has failed.' |
+ raise |
+ |
+ |
+ |
+def MakeApiRequest(credentials, request, retry=True): |
+ """Used to communicate with perf dashboard. |
+ |
+ args: |
+ credentials: Set of credentials generated by |
+ request: String that contains POST request to dashboard. |
+ returns: |
+ Contents of the response from the dashboard. |
+ """ |
+ print 'Making API request: %s' % request |
+ resp, content = credentials.request( |
+ REQUEST_URL + request, |
+ method="POST", |
+ headers={'Content-length': 0}) |
+ if resp['status'] != '200': |
+ print ('Error detected while making api request. Returned: %s' |
+ % (resp['status'])) |
+ if retry: |
+ print 'Retrying command after 3 seconds...' |
+ time.sleep(3) |
+ return MakeApiRequest(credentials, request, retry=False) |
+ return (resp, content) |
+ |
+ |
+def _ProcessTimeseriesData(ts): |
+ """Does noise processing of timeseries data. |
+ args: |
+ ts: Timeseries from dashboard. |
+ returns: |
+ Dict of noise metrics. |
+ """ |
+ ts_values = [t[1] for t in ts] |
+ mean = numpy.mean(ts_values) |
+ std = numpy.std(ts_values) |
+ return { |
+ 'count': len(ts_values), |
+ 'sum': sum(ts_values), |
+ 'mean': mean, |
+ 'variance': numpy.var(ts_values), |
+ 'stdev': std, |
+ 'cv': std / mean * 100 if mean else None, |
+ } |
+ |
+ |
+def GetBugData(creds, bug, cache): |
+ """Returns data for given bug.""" |
+ try: |
+ if not bug: |
+ return {'bug': {'state': None, 'status': None, 'summary': None}} |
+ if int(bug) == -1: |
+ return {'bug': {'state': None, 'status': None, 'summary': 'Invalid'}} |
+ if int(bug) == -2: |
+ return {'bug': {'state': None, 'status': None, 'summary': 'Ignored'}} |
+ r = 'bugs/%s' % bug |
+ _, output = MakeApiRequest(creds, r) |
+ |
+ if cache.get(bug): |
+ print 'Returning cached data for bug %s' % bug |
+ return cache[bug] |
+ data = json.loads(output) |
+ # Only care about date of comments, not connent. |
+ data['bug']['comments'] = [a['published'] for a in data['bug']['comments']] |
+ cache[bug] = data |
+ return data |
+ except Exception: # pylint: disable=broad-except |
+ print 'Problem when collecting bug data for bug %s: %s' % (bug, output) |
+ raise |
+ |
+ |
+def GetAlertData(credentials, benchmark, days): |
+ """Returns alerts for given benchmark.""" |
+ r = 'alerts/history/%s/?benchmark=%s' %(str(days), benchmark) |
+ _, output = MakeApiRequest(credentials, r) |
+ try: |
+ data = json.loads(output)['anomalies'] |
+ return data |
+ except: |
+ print 'Problem getting alerts for benchmark %s: %s' % (benchmark, output) |
+ raise |
+ |
+ |
+def GetNoiseData(credentials, metric, days): |
+ """Returns noise data for given metric.""" |
+ r = 'timeseries/%s?num_days=%s' % (metric, str(days)) |
+ if not metric: |
+ return None |
+ _, output = MakeApiRequest(credentials, r) |
+ try: |
+ data = json.loads(output) |
+ if not data: |
+ print 'No data found for metric %s in the last %s days.' % (metric, days) |
+ return None |
+ ts = data['timeseries'][1:] # First entry is book keeping. |
+ return _ProcessTimeseriesData(ts) |
+ except Exception: |
+ print 'Problem getting timeseries for %s: %s' % (metric, output) |
+ raise |
+ |
+ |
+def Main(): |
+ parser = argparse.ArgumentParser() |
+ parser.add_argument('-b', '--benchmark', required=True, |
+ help='Benchmark to pull data for.') |
+ parser.add_argument('-d', '--days', required=False, default=30, |
+ help='Number of days to collect data for. Default 30') |
+ parser.add_argument('--credentials', |
+ help=('Path to json credentials file. See %s for ' |
+ 'information about generating this.' % HELP_SITE)) |
+ parser.add_argument('--output-path', default='alert_analyzer.json', |
+ help='Path to save file to. Default: alert_analyzer.json') |
+ args = parser.parse_args() |
+ |
+ credentials = AuthorizeAccount(args) |
+ data = [] |
+ |
+ alerts = GetAlertData(credentials, args.benchmark, args.days) |
+ bug_cache = {} |
+ print '%s alerts found! Collecting data related to them...' % len(alerts) |
+ for alert in alerts: |
+ entry = {'alert': alert} |
+ bug_id = alert.get('bug_id') |
+ metric = '%s/%s/%s/%s' % (alert['master'], alert['bot'], alert['testsuite'], |
+ alert['test']) |
+ |
+ entry['noise'] = { |
+ 'reg': GetNoiseData(credentials, metric, args.days), |
+ 'ref': GetNoiseData(credentials, alert['ref_test'], args.days) |
+ } |
+ entry['bug'] = GetBugData(credentials, bug_id, bug_cache)['bug'] |
+ |
+ data.append(entry) |
+ |
+ # Save at end. |
+ with open(args.output_path, 'w') as fp: |
+ print 'Saving data to %s.' % args.output_path |
+ json.dump(data, fp, sort_keys=True, indent=2) |
+ |
+ |
+if __name__ == '__main__': |
+ Main() |