[Soundwave] Add tool that displays alerts and related noise and bug data.

experimental/soundwave/alert_analyzer.py

+#!/usr/bin/env python
+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import httplib2
+import json
+import numpy
+from oauth2client import client
+from oauth2client import service_account # pylint: disable=no-name-in-module
+import time
+
+
+# TODO(rnephew): Integrate into catapult/experimental/benchmark_health_report.
+REQUEST_URL = 'https://chromeperf.appspot.com/api/'
+# pylint: disable=line-too-long
+HELP_SITE = 'https://developers.google.com/api-client-library/python/auth/service-accounts#creatinganaccount'
+
+OAUTH_CLIENT_ID = (
+    '62121018386-h08uiaftreu4dr3c4alh3l7mogskvb7i.apps.googleusercontent.com')
+OAUTH_CLIENT_SECRET = 'vc1fZfV1cZC6mgDSHV-KSPOz'
+SCOPES = 'https://www.googleapis.com/auth/userinfo.email'
+
+
+def AuthorizeAccount(args):
+  """A factory for authorized account credentials."""
+  if args.credentials:
+    try:
+      return AuthorizeAccountServiceAccount(args.credentials)
+    except Exception:  # pylint: disable=broad-except
+      print ('Failure authenticating with service account. Falling back to user'
+             ' authentication.')
+  return AuthorizeAccountUserAccount()
+
+
+def AuthorizeAccountServiceAccount(json_key):
+  """Used to create a service account connection with the performance dashboard.
+
+  args:
+    json_key: Path to json file that contains credentials.
+  returns:
+    An object that can be used to communicate with the dashboard.
+  """
+  creds = service_account.ServiceAccountCredentials.from_json_keyfile_name(
+      json_key, [SCOPES])
+  return creds.authorize(httplib2.Http())
+
+
+def AuthorizeAccountUserAccount():
+  """Used to create an user account connection with the performance dashboard.
+
+  returns:
+    An object that can be used to communicate with the dashboard.
+  """
+  flow = client.OAuth2WebServerFlow(
+      OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET, [SCOPES], approval_prompt='force')
+  flow.redirect_uri = client.OOB_CALLBACK_URN
+  print('Go to the followinhg link in your browser:\n'
+        '    %s\n' % flow.step1_get_authorize_url())
+  code = raw_input('Enter verification code: ').strip()
+  try:
+    creds = flow.step2_exchange(code)
+    return creds.authorize(httplib2.Http())
+  except client.FlowExchangeError:
+    print 'User authentication has failed.'
+    raise
+
+
+
+def MakeApiRequest(credentials, request, retry=True):
+  """Used to communicate with perf dashboard.
+
+  args:
+    credentials: Set of credentials generated by
+    request: String that contains POST request to dashboard.
+  returns:
+    Contents of the response from the dashboard.
+  """
+  print 'Making API request: %s' % request
+  resp, content = credentials.request(
+      REQUEST_URL + request,
+      method="POST",
+      headers={'Content-length': 0})
+  if resp['status'] != '200':
+    print ('Error detected while making api request. Returned: %s'
+           % (resp['status']))
+    if retry:
+      print 'Retrying command after 3 seconds...'
+      time.sleep(3)
+      return MakeApiRequest(credentials, request, retry=False)
+  return (resp, content)
+
+
+def _ProcessTimeseriesData(ts):
+  """Does noise processing of timeseries data.
+  args:
+    ts: Timeseries from dashboard.
+  returns:
+    Dict of noise metrics.
+  """
+  ts_values = [t[1] for t in ts]
+  mean = numpy.mean(ts_values)
+  std = numpy.std(ts_values)
+  return {
+      'count': len(ts_values),
+      'sum': sum(ts_values),
+      'mean': mean,
+      'variance': numpy.var(ts_values),
+      'stdev': std,
+      'cv': std / mean * 100 if mean else None,
+  }
+
+
+def GetBugData(creds, bug, cache):
+  """Returns data for given bug."""
+  try:
+    if not bug:
+      return {'bug': {'state': None, 'status': None, 'summary': None}}
+    if int(bug) == -1:
+      return {'bug': {'state': None, 'status': None, 'summary': 'Invalid'}}
+    if int(bug) == -2:
+      return {'bug': {'state': None, 'status': None, 'summary': 'Ignored'}}
+    r = 'bugs/%s' % bug
+    _, output = MakeApiRequest(creds, r)
+
+    if cache.get(bug):
+      print 'Returning cached data for bug %s' % bug
+      return cache[bug]
+    data = json.loads(output)
+    # Only care about date of comments, not connent.
+    data['bug']['comments'] = [a['published'] for a in data['bug']['comments']]
+    cache[bug] = data
+    return data
+  except Exception:  # pylint: disable=broad-except
+    print 'Problem when collecting bug data for bug %s: %s' % (bug, output)
+    raise
+
+
+def GetAlertData(credentials, benchmark, days):
+  """Returns alerts for given benchmark."""
+  r = 'alerts/history/%s/?benchmark=%s' %(str(days), benchmark)
+  _, output = MakeApiRequest(credentials, r)
+  try:
+    data = json.loads(output)['anomalies']
+    return data
+  except:
+    print 'Problem getting alerts for benchmark %s: %s' % (benchmark, output)
+    raise
+
+
+def GetNoiseData(credentials, metric, days):
+  """Returns noise data for given metric."""
+  r = 'timeseries/%s?num_days=%s' % (metric, str(days))
+  if not metric:
+    return None
+  _, output = MakeApiRequest(credentials, r)
+  try:
+    data = json.loads(output)
+    if not data:
+      print 'No data found for metric %s in the last %s days.' % (metric, days)
+      return None
+    ts = data['timeseries'][1:]  # First entry is book keeping.
+    return _ProcessTimeseriesData(ts)
+  except Exception:
+    print 'Problem getting timeseries for %s: %s' % (metric, output)
+    raise
+
+
+def Main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-b', '--benchmark', required=True,
+                      help='Benchmark to pull data for.')
+  parser.add_argument('-d', '--days', required=False, default=30,
+                      help='Number of days to collect data for. Default 30')
+  parser.add_argument('--credentials',
+                      help=('Path to json credentials file. See %s for '
+                            'information about generating this.' % HELP_SITE))
+  parser.add_argument('--output-path', default='alert_analyzer.json',
+                      help='Path to save file to. Default: alert_analyzer.json')
+  args = parser.parse_args()
+
+  credentials = AuthorizeAccount(args)
+  data = []
+
+  alerts = GetAlertData(credentials, args.benchmark, args.days)
+  bug_cache = {}
+  print '%s alerts found! Collecting data related to them...' % len(alerts)
+  for alert in alerts:
+    entry = {'alert': alert}
+    bug_id = alert.get('bug_id')
+    metric = '%s/%s/%s/%s' % (alert['master'], alert['bot'], alert['testsuite'],
+                              alert['test'])
+
+    entry['noise'] = {
+        'reg': GetNoiseData(credentials, metric, args.days),
+        'ref': GetNoiseData(credentials, alert['ref_test'], args.days)
+    }
+    entry['bug'] = GetBugData(credentials, bug_id, bug_cache)['bug']
+
+    data.append(entry)
+
+  # Save at end.
+  with open(args.output_path, 'w') as fp:
+    print 'Saving data to %s.' % args.output_path
+    json.dump(data, fp, sort_keys=True, indent=2)
+
+
+if __name__ == '__main__':
+  Main()