Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(6)

Side by Side Diff: tools-webrtc/autoroller/roll_deps.py

Issue 2864213004: Rename tools-webrtc -> tools_webrtc (Closed)
Patch Set: REmove symlink Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 #
4 # Use of this source code is governed by a BSD-style license
5 # that can be found in the LICENSE file in the root of the source
6 # tree. An additional intellectual property rights grant can be found
7 # in the file PATENTS. All contributing project authors may
8 # be found in the AUTHORS file in the root of the source tree.
9
10 """Script to automatically roll dependencies in the WebRTC DEPS file."""
11
12 import argparse
13 import base64
14 import collections
15 import logging
16 import os
17 import re
18 import subprocess
19 import sys
20 import urllib
21
22
23 # Skip these dependencies (list without solution name prefix).
24 DONT_AUTOROLL_THESE = [
25 'src/third_party/gflags/src',
26 'src/third_party/winsdk_samples',
27 ]
28
29 WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc'
30 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
31 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
32 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
33 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
34
35 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
36 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
37 ROLL_BRANCH_NAME = 'roll_chromium_revision'
38
39 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
40 CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
41 os.pardir))
42 CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
43
44 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
45 import find_depot_tools
46 find_depot_tools.add_depot_tools_to_path()
47 from gclient import GClientKeywords
48
49 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
50 CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools',
51 'clang', 'scripts', 'update.py')
52
53 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
54 ChangedDep = collections.namedtuple('ChangedDep',
55 'path url current_rev new_rev')
56
57 class RollError(Exception):
58 pass
59
60
61 def ParseDepsDict(deps_content):
62 local_scope = {}
63 var = GClientKeywords.VarImpl({}, local_scope)
64 global_scope = {
65 'From': GClientKeywords.FromImpl,
66 'Var': var.Lookup,
67 'deps_os': {},
68 }
69 exec(deps_content, global_scope, local_scope)
70 return local_scope
71
72
73 def ParseLocalDepsFile(filename):
74 with open(filename, 'rb') as f:
75 deps_content = f.read()
76 return ParseDepsDict(deps_content)
77
78
79 def ParseRemoteCrDepsFile(revision):
80 deps_content = ReadRemoteCrFile('DEPS', revision)
81 return ParseDepsDict(deps_content)
82
83
84 def ParseCommitPosition(commit_message):
85 for line in reversed(commit_message.splitlines()):
86 m = COMMIT_POSITION_RE.match(line.strip())
87 if m:
88 return m.group(1)
89 logging.error('Failed to parse commit position id from:\n%s\n',
90 commit_message)
91 sys.exit(-1)
92
93
94 def _RunCommand(command, working_dir=None, ignore_exit_code=False,
95 extra_env=None):
96 """Runs a command and returns the output from that command.
97
98 If the command fails (exit code != 0), the function will exit the process.
99
100 Returns:
101 A tuple containing the stdout and stderr outputs as strings.
102 """
103 working_dir = working_dir or CHECKOUT_SRC_DIR
104 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
105 env = os.environ.copy()
106 if extra_env:
107 assert all(type(value) == str for value in extra_env.values())
108 logging.debug('extra env: %s', extra_env)
109 env.update(extra_env)
110 p = subprocess.Popen(command, stdout=subprocess.PIPE,
111 stderr=subprocess.PIPE, env=env,
112 cwd=working_dir, universal_newlines=True)
113 std_output = p.stdout.read()
114 err_output = p.stderr.read()
115 p.wait()
116 p.stdout.close()
117 p.stderr.close()
118 if not ignore_exit_code and p.returncode != 0:
119 logging.error('Command failed: %s\n'
120 'stdout:\n%s\n'
121 'stderr:\n%s\n', ' '.join(command), std_output, err_output)
122 sys.exit(p.returncode)
123 return std_output, err_output
124
125
126 def _GetBranches():
127 """Returns a tuple of active,branches.
128
129 The 'active' is the name of the currently active branch and 'branches' is a
130 list of all branches.
131 """
132 lines = _RunCommand(['git', 'branch'])[0].split('\n')
133 branches = []
134 active = ''
135 for line in lines:
136 if '*' in line:
137 # The assumption is that the first char will always be the '*'.
138 active = line[1:].strip()
139 branches.append(active)
140 else:
141 branch = line.strip()
142 if branch:
143 branches.append(branch)
144 return active, branches
145
146
147 def _ReadGitilesContent(url):
148 # Download and decode BASE64 content until
149 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
150 base64_content = ReadUrlContent(url + '?format=TEXT')
151 return base64.b64decode(base64_content[0])
152
153
154 def ReadRemoteCrFile(path_below_src, revision):
155 """Reads a remote Chromium file of a specific revision. Returns a string."""
156 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision,
157 path_below_src))
158
159
160 def ReadRemoteCrCommit(revision):
161 """Reads a remote Chromium commit message. Returns a string."""
162 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
163
164
165 def ReadUrlContent(url):
166 """Connect to a remote host and read the contents. Returns a list of lines."""
167 conn = urllib.urlopen(url)
168 try:
169 return conn.readlines()
170 except IOError as e:
171 logging.exception('Error connecting to %s. Error: %s', url, e)
172 raise
173 finally:
174 conn.close()
175
176
177 def GetMatchingDepsEntries(depsentry_dict, dir_path):
178 """Gets all deps entries matching the provided path.
179
180 This list may contain more than one DepsEntry object.
181 Example: dir_path='src/testing' would give results containing both
182 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS.
183 Example 2: dir_path='src/build' should return 'src/build' but not
184 'src/buildtools'.
185
186 Returns:
187 A list of DepsEntry objects.
188 """
189 result = []
190 for path, depsentry in depsentry_dict.iteritems():
191 if path == dir_path:
192 result.append(depsentry)
193 else:
194 parts = path.split('/')
195 if all(part == parts[i]
196 for i, part in enumerate(dir_path.split('/'))):
197 result.append(depsentry)
198 return result
199
200
201 def BuildDepsentryDict(deps_dict):
202 """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
203 result = {}
204 def AddDepsEntries(deps_subdict):
205 for path, deps_url in deps_subdict.iteritems():
206 if not result.has_key(path):
207 url, revision = deps_url.split('@') if deps_url else (None, None)
208 result[path] = DepsEntry(path, url, revision)
209
210 AddDepsEntries(deps_dict['deps'])
211 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
212 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
213 return result
214
215
216 def CalculateChangedDeps(webrtc_deps, new_cr_deps):
217 """
218 Calculate changed deps entries based on entries defined in the WebRTC DEPS
219 file:
220 - If a shared dependency with the Chromium DEPS file: roll it to the same
221 revision as Chromium (i.e. entry in the new_cr_deps dict)
222 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice
223 this means it may be ahead of the chromium_revision, but generally these
224 should be close).
225 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD
226 unless it's configured to be skipped.
227
228 Returns:
229 A list of ChangedDep objects representing the changed deps.
230 """
231 result = []
232 webrtc_entries = BuildDepsentryDict(webrtc_deps)
233 new_cr_entries = BuildDepsentryDict(new_cr_deps)
234 for path, webrtc_deps_entry in webrtc_entries.iteritems():
235 if path in DONT_AUTOROLL_THESE:
236 continue
237 cr_deps_entry = new_cr_entries.get(path)
238 if cr_deps_entry:
239 # Use the revision from Chromium's DEPS file.
240 new_rev = cr_deps_entry.revision
241 assert webrtc_deps_entry.url == cr_deps_entry.url, (
242 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
243 (path, webrtc_deps_entry.url, cr_deps_entry.url))
244 else:
245 # Use the HEAD of the deps repo.
246 stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url,
247 'HEAD'])
248 new_rev = stdout.strip().split('\t')[0]
249
250 # Check if an update is necessary.
251 if webrtc_deps_entry.revision != new_rev:
252 logging.debug('Roll dependency %s to %s', path, new_rev)
253 result.append(ChangedDep(path, webrtc_deps_entry.url,
254 webrtc_deps_entry.revision, new_rev))
255 return sorted(result)
256
257
258 def CalculateChangedClang(new_cr_rev):
259 def GetClangRev(lines):
260 for line in lines:
261 match = CLANG_REVISION_RE.match(line)
262 if match:
263 return match.group(1)
264 raise RollError('Could not parse Clang revision!')
265
266 with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f:
267 current_lines = f.readlines()
268 current_rev = GetClangRev(current_lines)
269
270 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
271 new_cr_rev).splitlines()
272 new_rev = GetClangRev(new_clang_update_py)
273 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
274
275
276 def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
277 new_commit_pos, changed_deps_list, clang_change):
278 current_cr_rev = current_cr_rev[0:10]
279 new_cr_rev = new_cr_rev[0:10]
280 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
281 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
282
283 commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
284 git_number_interval)]
285 commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval))
286 commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
287 rev_interval))
288 # TBR field will be empty unless in some custom cases, where some engineers
289 # are added.
290 tbr_authors = ''
291 if changed_deps_list:
292 commit_msg.append('Changed dependencies:')
293
294 for c in changed_deps_list:
295 commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
296 c.current_rev[0:10],
297 c.new_rev[0:10]))
298 if 'libvpx' in c.path:
299 tbr_authors += 'marpan@webrtc.org, '
300
301 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
302 commit_msg.append('DEPS diff: %s\n' % change_url)
303 else:
304 commit_msg.append('No dependencies changed.')
305
306 if clang_change.current_rev != clang_change.new_rev:
307 commit_msg.append('Clang version changed %s:%s' %
308 (clang_change.current_rev, clang_change.new_rev))
309 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
310 CLANG_UPDATE_SCRIPT_URL_PATH)
311 commit_msg.append('Details: %s\n' % change_url)
312 else:
313 commit_msg.append('No update to Clang.\n')
314
315 commit_msg.append('TBR=%s' % tbr_authors)
316 commit_msg.append('BUG=None')
317 return '\n'.join(commit_msg)
318
319
320 def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
321 changed_deps):
322 """Update the DEPS file with the new revision."""
323
324 # Update the chromium_revision variable.
325 with open(deps_filename, 'rb') as deps_file:
326 deps_content = deps_file.read()
327 deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
328 with open(deps_filename, 'wb') as deps_file:
329 deps_file.write(deps_content)
330
331 # Update each individual DEPS entry.
332 for dep in changed_deps:
333 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path)
334 if not os.path.isdir(local_dep_dir):
335 raise RollError(
336 'Cannot find local directory %s. Either run\n'
337 'gclient sync --deps=all\n'
338 'or make sure the .gclient file for your solution contains all '
339 'platforms in the target_os list, i.e.\n'
340 'target_os = ["android", "unix", "mac", "ios", "win"];\n'
341 'Then run "gclient sync" again.' % local_dep_dir)
342 _, stderr = _RunCommand(
343 ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
344 working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
345 if stderr:
346 logging.warning('roll-dep-svn: %s', stderr)
347
348
349 def _IsTreeClean():
350 stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
351 if len(stdout) == 0:
352 return True
353
354 logging.error('Dirty/unversioned files:\n%s', stdout)
355 return False
356
357
358 def _EnsureUpdatedMasterBranch(dry_run):
359 current_branch = _RunCommand(
360 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0]
361 if current_branch != 'master':
362 logging.error('Please checkout the master branch and re-run this script.')
363 if not dry_run:
364 sys.exit(-1)
365
366 logging.info('Updating master branch...')
367 _RunCommand(['git', 'pull'])
368
369
370 def _CreateRollBranch(dry_run):
371 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
372 if not dry_run:
373 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
374
375
376 def _RemovePreviousRollBranch(dry_run):
377 active_branch, branches = _GetBranches()
378 if active_branch == ROLL_BRANCH_NAME:
379 active_branch = 'master'
380 if ROLL_BRANCH_NAME in branches:
381 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
382 if not dry_run:
383 _RunCommand(['git', 'checkout', active_branch])
384 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
385
386
387 def _LocalCommit(commit_msg, dry_run):
388 logging.info('Committing changes locally.')
389 if not dry_run:
390 _RunCommand(['git', 'add', '--update', '.'])
391 _RunCommand(['git', 'commit', '-m', commit_msg])
392
393
394 def _UploadCL(dry_run, rietveld_email=None):
395 logging.info('Uploading CL...')
396 if not dry_run:
397 cmd = ['git', 'cl', 'upload', '-f']
398 if rietveld_email:
399 cmd.append('--email=%s' % rietveld_email)
400 _RunCommand(cmd, extra_env={'EDITOR': 'true'})
401
402
403 def _SendToCQ(dry_run, skip_cq):
404 logging.info('Sending the CL to the CQ...')
405 if not dry_run and not skip_cq:
406 _RunCommand(['git', 'cl', 'set_commit'])
407 logging.info('Sent the CL to the CQ.')
408
409
410 def main():
411 p = argparse.ArgumentParser()
412 p.add_argument('--clean', action='store_true', default=False,
413 help='Removes any previous local roll branch.')
414 p.add_argument('-r', '--revision',
415 help=('Chromium Git revision to roll to. Defaults to the '
416 'Chromium HEAD revision if omitted.'))
417 p.add_argument('-u', '--rietveld-email',
418 help=('E-mail address to use for creating the CL at Rietveld'
419 'If omitted a previously cached one will be used or an '
420 'error will be thrown during upload.'))
421 p.add_argument('--dry-run', action='store_true', default=False,
422 help=('Calculate changes and modify DEPS, but don\'t create '
423 'any local branch, commit, upload CL or send any '
424 'tryjobs.'))
425 p.add_argument('-i', '--ignore-unclean-workdir', action='store_true',
426 default=False,
427 help=('Ignore if the current branch is not master or if there '
428 'are uncommitted changes (default: %(default)s).'))
429 p.add_argument('--skip-cq', action='store_true', default=False,
430 help='Skip sending the CL to the CQ (default: %(default)s)')
431 p.add_argument('-v', '--verbose', action='store_true', default=False,
432 help='Be extra verbose in printing of log messages.')
433 opts = p.parse_args()
434
435 if opts.verbose:
436 logging.basicConfig(level=logging.DEBUG)
437 else:
438 logging.basicConfig(level=logging.INFO)
439
440 if not opts.ignore_unclean_workdir and not _IsTreeClean():
441 logging.error('Please clean your local checkout first.')
442 return 1
443
444 if opts.clean:
445 _RemovePreviousRollBranch(opts.dry_run)
446
447 if not opts.ignore_unclean_workdir:
448 _EnsureUpdatedMasterBranch(opts.dry_run)
449
450 new_cr_rev = opts.revision
451 if not new_cr_rev:
452 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
453 head_rev = stdout.strip().split('\t')[0]
454 logging.info('No revision specified. Using HEAD: %s', head_rev)
455 new_cr_rev = head_rev
456
457 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
458 webrtc_deps = ParseLocalDepsFile(deps_filename)
459 current_cr_rev = webrtc_deps['vars']['chromium_revision']
460
461 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
462 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
463
464 new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
465 changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps)
466 clang_change = CalculateChangedClang(new_cr_rev)
467 commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
468 current_commit_pos, new_commit_pos,
469 changed_deps, clang_change)
470 logging.debug('Commit message:\n%s', commit_msg)
471
472 _CreateRollBranch(opts.dry_run)
473 UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
474 _LocalCommit(commit_msg, opts.dry_run)
475 _UploadCL(opts.dry_run, opts.rietveld_email)
476 _SendToCQ(opts.dry_run, opts.skip_cq)
477 return 0
478
479
480 if __name__ == '__main__':
481 sys.exit(main())
OLDNEW
« no previous file with comments | « tools-webrtc/audio_quality/win/vcomp120.dll.sha1 ('k') | tools-webrtc/autoroller/unittests/roll_deps_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698