Autoroller: Support for rolling individual DEPS entries.

tools/autoroller/roll_chromium_revision.py

 #!/usr/bin/env python
 # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
 # that can be found in the LICENSE file in the root of the source
 # tree. An additional intellectual property rights grant can be found
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
 """Script to roll chromium_revision in the WebRTC DEPS file."""
 
 import argparse
 import base64
 import collections
 import logging
 import os
 import re
 import subprocess
 import sys
 import urllib
 
 
+# Skip these dependencies (list without solution name prefix).

[phoglund, 2016/12/13 09:57:02]

It's not entirely clear what "skipping" means in this context. Maybe
DONT_AUTOROLL_THESE is a better name? :)

[kjellander_webrtc, 2016/12/13 10:25:00]

Done.

+SKIPPED_AUTOROLL_DEPS = [
+  'src/third_party/gflags/src',
+  'src/third_party/winsdk_samples/src',
+]
+
+WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc'
 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
 
 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
 ROLL_BRANCH_NAME = 'roll_chromium_revision'
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
-                                                  os.pardir))
-sys.path.append(CHECKOUT_ROOT_DIR)
+CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
+                                                 os.pardir))
+CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
+CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src')
+
+sys.path.append(CHECKOUT_SRC_DIR)
 import setup_links
 
-sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build'))
+sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
 import find_depot_tools
 find_depot_tools.add_depot_tools_to_path()
 from gclient import GClientKeywords
 
 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
 CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts',
                                               'update.py')
 
 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
 ChangedDep = collections.namedtuple('ChangedDep',
@@ skipping 38 matching lines @@
 
 def _RunCommand(command, working_dir=None, ignore_exit_code=False,
                 extra_env=None):
   """Runs a command and returns the output from that command.
 
   If the command fails (exit code != 0), the function will exit the process.
 
   Returns:
     A tuple containing the stdout and stderr outputs as strings.
   """
-  working_dir = working_dir or CHECKOUT_ROOT_DIR
+  working_dir = working_dir or CHECKOUT_SRC_DIR
   logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
   env = os.environ.copy()
   if extra_env:
     assert all(type(value) == str for value in extra_env.values())
     logging.debug('extra env: %s', extra_env)
     env.update(extra_env)
   p = subprocess.Popen(command, stdout=subprocess.PIPE,
                        stderr=subprocess.PIPE, env=env,
                        cwd=working_dir, universal_newlines=True)
   std_output = p.stdout.read()
@@ skipping 78 matching lines @@
       result.append(depsentry)
     else:
       parts = path.split('/')
       if all(part == parts[i]
              for i, part in enumerate(dir_path.split('/'))):
         result.append(depsentry)
   return result
 
 
 def BuildDepsentryDict(deps_dict):
-  """Builds a dict of DepsEntry object from a raw parsed deps dict."""
+  """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
   result = {}
   def AddDepsEntries(deps_subdict):
     for path, deps_url in deps_subdict.iteritems():
       if not result.has_key(path):
         url, revision = deps_url.split('@') if deps_url else (None, None)
         result[path] = DepsEntry(path, url, revision)
 
   AddDepsEntries(deps_dict['deps'])
   for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
-    AddDepsEntries(deps_dict['deps_os'].get(deps_os, {}))
+    AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
   return result
 
 
-def CalculateChangedDeps(current_deps, new_deps):
+def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps):
+  """
+  Calculate changed deps entries based on:
+  1. Entries defined in the WebRTC DEPS file - rolls them to the same revision
+     as the new_cr_deps dict, if available. Otherwise HEAD unless the deps is

[phoglund, 2016/12/13 09:57:02]

I find this confusing. When exactly is something rolled to HEAD? If it isn't
specified in the new_cr_deps dict? What does it mean for it not to be specified
in the new_cr_deps dict? Does that mean new_cr_deps specifies a subset of all
possible deps, and the ones not in the subset are rolled to head? In that case,
what is the list of all possible deps?

[kjellander_webrtc, 2016/12/13 10:25:00]

I clarified it now.
The new_cr_deps dict contains a parsed Chromium DEPS file for the revision the
autoroller is trying to roll to (i.e. the --revision flag passed to the script).

+     configured to be skipped.
+  2. Entried present in the setup_links.py file. If the dir has changed between

[phoglund, 2016/12/13 09:57:02]

Nit: entries

[kjellander_webrtc, 2016/12/13 10:25:00]

Done.

+     old_cr_deps and new_cr_deps, it is considered changed.
+
+  Returns:
+    A list of ChangedDep objects representing the changed deps.
+  """
+  return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) +
+                CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps))
+
+
+def CalculateChangedDepsProper(webrtc_deps, new_cr_deps):
   result = []
-  current_entries = BuildDepsentryDict(current_deps)
-  new_entries = BuildDepsentryDict(new_deps)
+  webrtc_entries = BuildDepsentryDict(webrtc_deps)
+  new_cr_entries = BuildDepsentryDict(new_cr_deps)
+  for path, webrtc_deps_entry in webrtc_entries.iteritems():
+    if path in SKIPPED_AUTOROLL_DEPS:
+      continue
+    cr_deps_entry = new_cr_entries.get(path)
+    if cr_deps_entry:
+      # Use the revision from Chromium's DEPS file.
+      new_rev = cr_deps_entry.revision
+      assert webrtc_deps_entry.url == cr_deps_entry.url, (
+        'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
+        (path, webrtc_deps_entry.url, cr_deps_entry.url))
+    else:
+      # Use the HEAD of the deps repo.
+      stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url,
+                               'HEAD'])
+      new_rev = stdout.strip().split('\t')[0]
 
+    # Check if an update is necessary.
+    if webrtc_deps_entry.revision != new_rev:
+      logging.debug('Roll dependency %s to %s', path, new_rev)
+      result.append(ChangedDep(path, webrtc_deps_entry.url,
+                               webrtc_deps_entry.revision, new_rev))
+  return result
+
+
+def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps):
+  result = []
+  new_cr_entries = BuildDepsentryDict(new_cr_deps)
+  old_cr_entries = BuildDepsentryDict(old_cr_deps)
   all_deps_dirs = setup_links.DIRECTORIES
   for deps_dir in all_deps_dirs:
     # All deps have 'src' prepended to the path in the Chromium DEPS file.
     dir_path = 'src/%s' % deps_dir
 
-    for entry in GetMatchingDepsEntries(current_entries, dir_path):
-      new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path)
+    for entry in GetMatchingDepsEntries(old_cr_entries, dir_path):
+      new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path)
       assert len(new_matching_entries) <= 1, (
           'Should never find more than one entry matching %s in %s, found %d' %
-          (entry.path, new_entries, len(new_matching_entries)))
+          (entry.path, new_cr_entries, len(new_matching_entries)))
       if not new_matching_entries:
         result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
       elif entry != new_matching_entries[0]:
         result.append(ChangedDep(entry.path, entry.url, entry.revision,
                                  new_matching_entries[0].revision))
   return result
 
 
 def CalculateChangedClang(new_cr_rev):
   def GetClangRev(lines):
     for line in lines:
       match = CLANG_REVISION_RE.match(line)
       if match:
         return match.group(1)
     raise RollError('Could not parse Clang revision!')
 
-  chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src',
+  chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR,
                                    CLANG_UPDATE_SCRIPT_LOCAL_PATH)
   with open(chromium_src_path, 'rb') as f:
     current_lines = f.readlines()
   current_rev = GetClangRev(current_lines)
 
   new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
                                              new_cr_rev).splitlines()
   new_rev = GetClangRev(new_clang_update_py)
   return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
 
@@ skipping 35 matching lines @@
                                            CLANG_UPDATE_SCRIPT_URL_PATH)
     commit_msg.append('Details: %s\n' % change_url)
   else:
     commit_msg.append('No update to Clang.\n')
 
   commit_msg.append('TBR=%s' % tbr_authors)
   commit_msg.append('BUG=None')
   return '\n'.join(commit_msg)
 
 
-def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision):
+def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
+                   changed_deps):
   """Update the DEPS file with the new revision."""
+
+  # Update the chromium_revision variable.
   with open(deps_filename, 'rb') as deps_file:
     deps_content = deps_file.read()
   deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
   with open(deps_filename, 'wb') as deps_file:
     deps_file.write(deps_content)
 
+  # Update each individual DEPS entry.
+  for dep in changed_deps:
+    _, stderr = _RunCommand(
+      ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
+      working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
+    if stderr:
+      logging.warning('roll-dep-svn: %s', stderr)
+
+
 def _IsTreeClean():
   stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
   if len(stdout) == 0:
     return True
 
   logging.error('Dirty/unversioned files:\n%s', stdout)
   return False
 
 
 def _EnsureUpdatedMasterBranch(dry_run):
@@ skipping 56 matching lines @@
                  help=('Chromium Git revision to roll to. Defaults to the '
                        'Chromium HEAD revision if omitted.'))
   p.add_argument('-u', '--rietveld-email',
                  help=('E-mail address to use for creating the CL at Rietveld'
                        'If omitted a previously cached one will be used or an '
                        'error will be thrown during upload.'))
   p.add_argument('--dry-run', action='store_true', default=False,
                  help=('Calculate changes and modify DEPS, but don\'t create '
                        'any local branch, commit, upload CL or send any '
                        'tryjobs.'))
-  p.add_argument('--allow-reverse', action='store_true', default=False,
-                 help=('Allow rolling back in time (disabled by default but '
-                       'may be useful to be able do to manually).'))
   p.add_argument('--skip-cq', action='store_true', default=False,
                  help='Skip sending the CL to the CQ (default: %(default)s)')
   p.add_argument('-v', '--verbose', action='store_true', default=False,
                  help='Be extra verbose in printing of log messages.')
   opts = p.parse_args()
 
   if opts.verbose:
     logging.basicConfig(level=logging.DEBUG)
   else:
     logging.basicConfig(level=logging.INFO)
 
   if not _IsTreeClean():
     logging.error('Please clean your local checkout first.')
     return 1
 
   if opts.clean:
     _RemovePreviousRollBranch(opts.dry_run)
 
   _EnsureUpdatedMasterBranch(opts.dry_run)
 
   new_cr_rev = opts.revision
   if not new_cr_rev:
     stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
     head_rev = stdout.strip().split('\t')[0]
     logging.info('No revision specified. Using HEAD: %s', head_rev)
     new_cr_rev = head_rev
 
-  deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
-  local_deps = ParseLocalDepsFile(deps_filename)
-  current_cr_rev = local_deps['vars']['chromium_revision']
+  deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
+  webrtc_deps = ParseLocalDepsFile(deps_filename)
+  current_cr_rev = webrtc_deps['vars']['chromium_revision']
 
   current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
   new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
 
   current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
   new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
 
-  if new_commit_pos > current_commit_pos or opts.allow_reverse:
-    changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps))
-    clang_change = CalculateChangedClang(new_cr_rev)
-    commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
-                                       current_commit_pos, new_commit_pos,
-                                       changed_deps, clang_change)
-    logging.debug('Commit message:\n%s', commit_msg)
-  else:
-    logging.info('Currently pinned chromium_revision: %s (#%s) is newer than '
-                 '%s (#%s). To roll to older revisions, you must pass the '
-                 '--allow-reverse flag.\n'
-                 'Aborting without action.', current_cr_rev, current_commit_pos,
-                 new_cr_rev, new_commit_pos)
-    return 0
+  changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps)
+  clang_change = CalculateChangedClang(new_cr_rev)
+  commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
+                                     current_commit_pos, new_commit_pos,
+                                     changed_deps, clang_change)
+  logging.debug('Commit message:\n%s', commit_msg)
 
   _CreateRollBranch(opts.dry_run)
-  UpdateDeps(deps_filename, current_cr_rev, new_cr_rev)
+  UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
   _LocalCommit(commit_msg, opts.dry_run)
   _UploadCL(opts.dry_run, opts.rietveld_email)
   _SendToCQ(opts.dry_run, opts.skip_cq)
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())