DEPS: Sync Git subtree mirrors instead of symlinking into chromium/src

tools-webrtc/autoroller/roll_deps.py

 #!/usr/bin/env python
 # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
 # that can be found in the LICENSE file in the root of the source
 # tree. An additional intellectual property rights grant can be found
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
 """Script to automatically roll dependencies in the WebRTC DEPS file."""
 
 import argparse
 import base64
 import collections
 import logging
 import os
 import re
 import subprocess
 import sys
 import urllib
 
 
 # Skip these dependencies (list without solution name prefix).
 DONT_AUTOROLL_THESE = [
   'src/third_party/gflags/src',
-  'src/third_party/winsdk_samples/src',
+  'src/third_party/winsdk_samples',
 ]
 
 WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc'
 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
 
 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
 ROLL_BRANCH_NAME = 'roll_chromium_revision'
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
                                                  os.pardir))
 CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
-CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src')
-
-sys.path.append(CHECKOUT_SRC_DIR)
-import setup_links
 
 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
 import find_depot_tools
 find_depot_tools.add_depot_tools_to_path()
 from gclient import GClientKeywords
 
 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
-CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts',
-                                              'update.py')
+CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools',
+                                              'clang', 'scripts', 'update.py')
 
 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
 ChangedDep = collections.namedtuple('ChangedDep',
                                     'path url current_rev new_rev')
 
 class RollError(Exception):
   pass
 
 
 def ParseDepsDict(deps_content):
@@ skipping 144 matching lines @@
       if not result.has_key(path):
         url, revision = deps_url.split('@') if deps_url else (None, None)
         result[path] = DepsEntry(path, url, revision)
 
   AddDepsEntries(deps_dict['deps'])
   for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
     AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
   return result
 
 
-def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps):
+def CalculateChangedDeps(webrtc_deps, new_cr_deps):
   """
-  Calculate changed deps entries based on:
-  1. Entries defined in the WebRTC DEPS file:
+  Calculate changed deps entries based on entries defined in the WebRTC DEPS
+  file:
      - If a shared dependency with the Chromium DEPS file: roll it to the same
        revision as Chromium (i.e. entry in the new_cr_deps dict)
      - If it's a Chromium sub-directory, roll it to the HEAD revision (notice
        this means it may be ahead of the chromium_revision, but generally these
        should be close).
      - If it's another DEPS entry (not shared with Chromium), roll it to HEAD
        unless it's configured to be skipped.
-  2. Entries present in the setup_links.py file. If the dir has changed between
-     old_cr_deps and new_cr_deps, it is considered changed and updated to the
-     revision for the entry in the new_cr_deps dict.
 
   Returns:
     A list of ChangedDep objects representing the changed deps.
   """
-  return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) +
-                CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps))
-
-
-def CalculateChangedDepsProper(webrtc_deps, new_cr_deps):
   result = []
   webrtc_entries = BuildDepsentryDict(webrtc_deps)
   new_cr_entries = BuildDepsentryDict(new_cr_deps)
   for path, webrtc_deps_entry in webrtc_entries.iteritems():
     if path in DONT_AUTOROLL_THESE:
       continue
     cr_deps_entry = new_cr_entries.get(path)
     if cr_deps_entry:
       # Use the revision from Chromium's DEPS file.
       new_rev = cr_deps_entry.revision
       assert webrtc_deps_entry.url == cr_deps_entry.url, (
         'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
         (path, webrtc_deps_entry.url, cr_deps_entry.url))
     else:
       # Use the HEAD of the deps repo.
       stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url,
                                'HEAD'])
       new_rev = stdout.strip().split('\t')[0]
 
     # Check if an update is necessary.
     if webrtc_deps_entry.revision != new_rev:
       logging.debug('Roll dependency %s to %s', path, new_rev)
       result.append(ChangedDep(path, webrtc_deps_entry.url,
                                webrtc_deps_entry.revision, new_rev))
-  return result
-
-
-def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps):
-  result = []
-  new_cr_entries = BuildDepsentryDict(new_cr_deps)
-  old_cr_entries = BuildDepsentryDict(old_cr_deps)
-  all_deps_dirs = setup_links.DIRECTORIES
-  for deps_dir in all_deps_dirs:
-    # All deps have 'src' prepended to the path in the Chromium DEPS file.
-    dir_path = 'src/%s' % deps_dir
-
-    for entry in GetMatchingDepsEntries(old_cr_entries, dir_path):
-      new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path)
-      assert len(new_matching_entries) <= 1, (
-          'Should never find more than one entry matching %s in %s, found %d' %
-          (entry.path, new_cr_entries, len(new_matching_entries)))
-      if not new_matching_entries:
-        result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
-      elif entry != new_matching_entries[0]:
-        result.append(ChangedDep(entry.path, entry.url, entry.revision,
-                                 new_matching_entries[0].revision))
-  return result
+  return sorted(result)
 
 
 def CalculateChangedClang(new_cr_rev):
   def GetClangRev(lines):
     for line in lines:
       match = CLANG_REVISION_RE.match(line)
       if match:
         return match.group(1)
     raise RollError('Could not parse Clang revision!')
 
-  chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR,
-                                   CLANG_UPDATE_SCRIPT_LOCAL_PATH)
-  with open(chromium_src_path, 'rb') as f:
+  with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f:
     current_lines = f.readlines()
   current_rev = GetClangRev(current_lines)
 
   new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
                                              new_cr_rev).splitlines()
   new_rev = GetClangRev(new_clang_update_py)
   return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
 
 
 def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
@@ skipping 46 matching lines @@
 
   # Update the chromium_revision variable.
   with open(deps_filename, 'rb') as deps_file:
     deps_content = deps_file.read()
   deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
   with open(deps_filename, 'wb') as deps_file:
     deps_file.write(deps_content)
 
   # Update each individual DEPS entry.
   for dep in changed_deps:
+    local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path)
+    if not os.path.isdir(local_dep_dir):
+      raise RollError(
+          'Cannot find local directory %s. Either run\n'
+          'gclient sync --deps=all\n'
+          'or make sure the .gclient file for your solution contains all '
+          'platforms in the target_os list, i.e.\n'
+          'target_os = ["android", "unix", "mac", "ios", "win"];\n'
+          'Then run "gclient sync" again.' % local_dep_dir)
     _, stderr = _RunCommand(
       ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
       working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
     if stderr:
       logging.warning('roll-dep-svn: %s', stderr)
 
 
 def _IsTreeClean():
   stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
   if len(stdout) == 0:
@@ skipping 102 matching lines @@
     logging.info('No revision specified. Using HEAD: %s', head_rev)
     new_cr_rev = head_rev
 
   deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
   webrtc_deps = ParseLocalDepsFile(deps_filename)
   current_cr_rev = webrtc_deps['vars']['chromium_revision']
 
   current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
   new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
 
-  current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
   new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
-
-  changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps)
+  changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
   clang_change = CalculateChangedClang(new_cr_rev)
   commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
                                      current_commit_pos, new_commit_pos,
                                      changed_deps, clang_change)
   logging.debug('Commit message:\n%s', commit_msg)
 
   _CreateRollBranch(opts.dry_run)
   UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
   _LocalCommit(commit_msg, opts.dry_run)
   _UploadCL(opts.dry_run, opts.rietveld_email)
   _SendToCQ(opts.dry_run, opts.skip_cq)
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())