Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(17)

Side by Side Diff: tools/autoroller/roll_chromium_revision.py

Issue 2581493003: Move tools/autoroller to tools-webrtc/ + rename script (Closed)
Patch Set: Renamed the script Created 4 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 #
4 # Use of this source code is governed by a BSD-style license
5 # that can be found in the LICENSE file in the root of the source
6 # tree. An additional intellectual property rights grant can be found
7 # in the file PATENTS. All contributing project authors may
8 # be found in the AUTHORS file in the root of the source tree.
9
10 """Script to roll chromium_revision in the WebRTC DEPS file."""
11
12 import argparse
13 import base64
14 import collections
15 import logging
16 import os
17 import re
18 import subprocess
19 import sys
20 import urllib
21
22
23 # Skip these dependencies (list without solution name prefix).
24 DONT_AUTOROLL_THESE = [
25 'src/third_party/gflags/src',
26 'src/third_party/winsdk_samples/src',
27 ]
28
29 WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc'
30 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
31 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
32 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
33 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
34
35 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
36 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
37 ROLL_BRANCH_NAME = 'roll_chromium_revision'
38
39 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
40 CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
41 os.pardir))
42 CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
43 CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src')
44
45 sys.path.append(CHECKOUT_SRC_DIR)
46 import setup_links
47
48 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
49 import find_depot_tools
50 find_depot_tools.add_depot_tools_to_path()
51 from gclient import GClientKeywords
52
53 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
54 CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts',
55 'update.py')
56
57 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
58 ChangedDep = collections.namedtuple('ChangedDep',
59 'path url current_rev new_rev')
60
61 class RollError(Exception):
62 pass
63
64
65 def ParseDepsDict(deps_content):
66 local_scope = {}
67 var = GClientKeywords.VarImpl({}, local_scope)
68 global_scope = {
69 'From': GClientKeywords.FromImpl,
70 'Var': var.Lookup,
71 'deps_os': {},
72 }
73 exec(deps_content, global_scope, local_scope)
74 return local_scope
75
76
77 def ParseLocalDepsFile(filename):
78 with open(filename, 'rb') as f:
79 deps_content = f.read()
80 return ParseDepsDict(deps_content)
81
82
83 def ParseRemoteCrDepsFile(revision):
84 deps_content = ReadRemoteCrFile('DEPS', revision)
85 return ParseDepsDict(deps_content)
86
87
88 def ParseCommitPosition(commit_message):
89 for line in reversed(commit_message.splitlines()):
90 m = COMMIT_POSITION_RE.match(line.strip())
91 if m:
92 return m.group(1)
93 logging.error('Failed to parse commit position id from:\n%s\n',
94 commit_message)
95 sys.exit(-1)
96
97
98 def _RunCommand(command, working_dir=None, ignore_exit_code=False,
99 extra_env=None):
100 """Runs a command and returns the output from that command.
101
102 If the command fails (exit code != 0), the function will exit the process.
103
104 Returns:
105 A tuple containing the stdout and stderr outputs as strings.
106 """
107 working_dir = working_dir or CHECKOUT_SRC_DIR
108 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
109 env = os.environ.copy()
110 if extra_env:
111 assert all(type(value) == str for value in extra_env.values())
112 logging.debug('extra env: %s', extra_env)
113 env.update(extra_env)
114 p = subprocess.Popen(command, stdout=subprocess.PIPE,
115 stderr=subprocess.PIPE, env=env,
116 cwd=working_dir, universal_newlines=True)
117 std_output = p.stdout.read()
118 err_output = p.stderr.read()
119 p.wait()
120 p.stdout.close()
121 p.stderr.close()
122 if not ignore_exit_code and p.returncode != 0:
123 logging.error('Command failed: %s\n'
124 'stdout:\n%s\n'
125 'stderr:\n%s\n', ' '.join(command), std_output, err_output)
126 sys.exit(p.returncode)
127 return std_output, err_output
128
129
130 def _GetBranches():
131 """Returns a tuple of active,branches.
132
133 The 'active' is the name of the currently active branch and 'branches' is a
134 list of all branches.
135 """
136 lines = _RunCommand(['git', 'branch'])[0].split('\n')
137 branches = []
138 active = ''
139 for line in lines:
140 if '*' in line:
141 # The assumption is that the first char will always be the '*'.
142 active = line[1:].strip()
143 branches.append(active)
144 else:
145 branch = line.strip()
146 if branch:
147 branches.append(branch)
148 return active, branches
149
150
151 def _ReadGitilesContent(url):
152 # Download and decode BASE64 content until
153 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
154 base64_content = ReadUrlContent(url + '?format=TEXT')
155 return base64.b64decode(base64_content[0])
156
157
158 def ReadRemoteCrFile(path_below_src, revision):
159 """Reads a remote Chromium file of a specific revision. Returns a string."""
160 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision,
161 path_below_src))
162
163
164 def ReadRemoteCrCommit(revision):
165 """Reads a remote Chromium commit message. Returns a string."""
166 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
167
168
169 def ReadUrlContent(url):
170 """Connect to a remote host and read the contents. Returns a list of lines."""
171 conn = urllib.urlopen(url)
172 try:
173 return conn.readlines()
174 except IOError as e:
175 logging.exception('Error connecting to %s. Error: %s', url, e)
176 raise
177 finally:
178 conn.close()
179
180
181 def GetMatchingDepsEntries(depsentry_dict, dir_path):
182 """Gets all deps entries matching the provided path.
183
184 This list may contain more than one DepsEntry object.
185 Example: dir_path='src/testing' would give results containing both
186 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS.
187 Example 2: dir_path='src/build' should return 'src/build' but not
188 'src/buildtools'.
189
190 Returns:
191 A list of DepsEntry objects.
192 """
193 result = []
194 for path, depsentry in depsentry_dict.iteritems():
195 if path == dir_path:
196 result.append(depsentry)
197 else:
198 parts = path.split('/')
199 if all(part == parts[i]
200 for i, part in enumerate(dir_path.split('/'))):
201 result.append(depsentry)
202 return result
203
204
205 def BuildDepsentryDict(deps_dict):
206 """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
207 result = {}
208 def AddDepsEntries(deps_subdict):
209 for path, deps_url in deps_subdict.iteritems():
210 if not result.has_key(path):
211 url, revision = deps_url.split('@') if deps_url else (None, None)
212 result[path] = DepsEntry(path, url, revision)
213
214 AddDepsEntries(deps_dict['deps'])
215 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
216 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
217 return result
218
219
220 def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps):
221 """
222 Calculate changed deps entries based on:
223 1. Entries defined in the WebRTC DEPS file:
224 - If a shared dependency with the Chromium DEPS file: roll it to the same
225 revision as Chromium (i.e. entry in the new_cr_deps dict)
226 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice
227 this means it may be ahead of the chromium_revision, but generally these
228 should be close).
229 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD
230 unless it's configured to be skipped.
231 2. Entries present in the setup_links.py file. If the dir has changed between
232 old_cr_deps and new_cr_deps, it is considered changed and updated to the
233 revision for the entry in the new_cr_deps dict.
234
235 Returns:
236 A list of ChangedDep objects representing the changed deps.
237 """
238 return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) +
239 CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps))
240
241
242 def CalculateChangedDepsProper(webrtc_deps, new_cr_deps):
243 result = []
244 webrtc_entries = BuildDepsentryDict(webrtc_deps)
245 new_cr_entries = BuildDepsentryDict(new_cr_deps)
246 for path, webrtc_deps_entry in webrtc_entries.iteritems():
247 if path in DONT_AUTOROLL_THESE:
248 continue
249 cr_deps_entry = new_cr_entries.get(path)
250 if cr_deps_entry:
251 # Use the revision from Chromium's DEPS file.
252 new_rev = cr_deps_entry.revision
253 assert webrtc_deps_entry.url == cr_deps_entry.url, (
254 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' %
255 (path, webrtc_deps_entry.url, cr_deps_entry.url))
256 else:
257 # Use the HEAD of the deps repo.
258 stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url,
259 'HEAD'])
260 new_rev = stdout.strip().split('\t')[0]
261
262 # Check if an update is necessary.
263 if webrtc_deps_entry.revision != new_rev:
264 logging.debug('Roll dependency %s to %s', path, new_rev)
265 result.append(ChangedDep(path, webrtc_deps_entry.url,
266 webrtc_deps_entry.revision, new_rev))
267 return result
268
269
270 def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps):
271 result = []
272 new_cr_entries = BuildDepsentryDict(new_cr_deps)
273 old_cr_entries = BuildDepsentryDict(old_cr_deps)
274 all_deps_dirs = setup_links.DIRECTORIES
275 for deps_dir in all_deps_dirs:
276 # All deps have 'src' prepended to the path in the Chromium DEPS file.
277 dir_path = 'src/%s' % deps_dir
278
279 for entry in GetMatchingDepsEntries(old_cr_entries, dir_path):
280 new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path)
281 assert len(new_matching_entries) <= 1, (
282 'Should never find more than one entry matching %s in %s, found %d' %
283 (entry.path, new_cr_entries, len(new_matching_entries)))
284 if not new_matching_entries:
285 result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
286 elif entry != new_matching_entries[0]:
287 result.append(ChangedDep(entry.path, entry.url, entry.revision,
288 new_matching_entries[0].revision))
289 return result
290
291
292 def CalculateChangedClang(new_cr_rev):
293 def GetClangRev(lines):
294 for line in lines:
295 match = CLANG_REVISION_RE.match(line)
296 if match:
297 return match.group(1)
298 raise RollError('Could not parse Clang revision!')
299
300 chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR,
301 CLANG_UPDATE_SCRIPT_LOCAL_PATH)
302 with open(chromium_src_path, 'rb') as f:
303 current_lines = f.readlines()
304 current_rev = GetClangRev(current_lines)
305
306 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
307 new_cr_rev).splitlines()
308 new_rev = GetClangRev(new_clang_update_py)
309 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
310
311
312 def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
313 new_commit_pos, changed_deps_list, clang_change):
314 current_cr_rev = current_cr_rev[0:10]
315 new_cr_rev = new_cr_rev[0:10]
316 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
317 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
318
319 commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
320 git_number_interval)]
321 commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval))
322 commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
323 rev_interval))
324 # TBR field will be empty unless in some custom cases, where some engineers
325 # are added.
326 tbr_authors = ''
327 if changed_deps_list:
328 commit_msg.append('Changed dependencies:')
329
330 for c in changed_deps_list:
331 commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
332 c.current_rev[0:10],
333 c.new_rev[0:10]))
334 if 'libvpx' in c.path:
335 tbr_authors += 'marpan@webrtc.org, '
336
337 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
338 commit_msg.append('DEPS diff: %s\n' % change_url)
339 else:
340 commit_msg.append('No dependencies changed.')
341
342 if clang_change.current_rev != clang_change.new_rev:
343 commit_msg.append('Clang version changed %s:%s' %
344 (clang_change.current_rev, clang_change.new_rev))
345 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
346 CLANG_UPDATE_SCRIPT_URL_PATH)
347 commit_msg.append('Details: %s\n' % change_url)
348 else:
349 commit_msg.append('No update to Clang.\n')
350
351 commit_msg.append('TBR=%s' % tbr_authors)
352 commit_msg.append('BUG=None')
353 return '\n'.join(commit_msg)
354
355
356 def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
357 changed_deps):
358 """Update the DEPS file with the new revision."""
359
360 # Update the chromium_revision variable.
361 with open(deps_filename, 'rb') as deps_file:
362 deps_content = deps_file.read()
363 deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
364 with open(deps_filename, 'wb') as deps_file:
365 deps_file.write(deps_content)
366
367 # Update each individual DEPS entry.
368 for dep in changed_deps:
369 _, stderr = _RunCommand(
370 ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
371 working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
372 if stderr:
373 logging.warning('roll-dep-svn: %s', stderr)
374
375
376 def _IsTreeClean():
377 stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
378 if len(stdout) == 0:
379 return True
380
381 logging.error('Dirty/unversioned files:\n%s', stdout)
382 return False
383
384
385 def _EnsureUpdatedMasterBranch(dry_run):
386 current_branch = _RunCommand(
387 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0]
388 if current_branch != 'master':
389 logging.error('Please checkout the master branch and re-run this script.')
390 if not dry_run:
391 sys.exit(-1)
392
393 logging.info('Updating master branch...')
394 _RunCommand(['git', 'pull'])
395
396
397 def _CreateRollBranch(dry_run):
398 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
399 if not dry_run:
400 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
401
402
403 def _RemovePreviousRollBranch(dry_run):
404 active_branch, branches = _GetBranches()
405 if active_branch == ROLL_BRANCH_NAME:
406 active_branch = 'master'
407 if ROLL_BRANCH_NAME in branches:
408 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
409 if not dry_run:
410 _RunCommand(['git', 'checkout', active_branch])
411 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
412
413
414 def _LocalCommit(commit_msg, dry_run):
415 logging.info('Committing changes locally.')
416 if not dry_run:
417 _RunCommand(['git', 'add', '--update', '.'])
418 _RunCommand(['git', 'commit', '-m', commit_msg])
419
420
421 def _UploadCL(dry_run, rietveld_email=None):
422 logging.info('Uploading CL...')
423 if not dry_run:
424 cmd = ['git', 'cl', 'upload', '-f']
425 if rietveld_email:
426 cmd.append('--email=%s' % rietveld_email)
427 _RunCommand(cmd, extra_env={'EDITOR': 'true'})
428
429
430 def _SendToCQ(dry_run, skip_cq):
431 logging.info('Sending the CL to the CQ...')
432 if not dry_run and not skip_cq:
433 _RunCommand(['git', 'cl', 'set_commit'])
434 logging.info('Sent the CL to the CQ.')
435
436
437 def main():
438 p = argparse.ArgumentParser()
439 p.add_argument('--clean', action='store_true', default=False,
440 help='Removes any previous local roll branch.')
441 p.add_argument('-r', '--revision',
442 help=('Chromium Git revision to roll to. Defaults to the '
443 'Chromium HEAD revision if omitted.'))
444 p.add_argument('-u', '--rietveld-email',
445 help=('E-mail address to use for creating the CL at Rietveld'
446 'If omitted a previously cached one will be used or an '
447 'error will be thrown during upload.'))
448 p.add_argument('--dry-run', action='store_true', default=False,
449 help=('Calculate changes and modify DEPS, but don\'t create '
450 'any local branch, commit, upload CL or send any '
451 'tryjobs.'))
452 p.add_argument('-i', '--ignore-unclean-workdir', action='store_true',
453 default=False,
454 help=('Ignore if the current branch is not master or if there '
455 'are uncommitted changes (default: %(default)s).'))
456 p.add_argument('--skip-cq', action='store_true', default=False,
457 help='Skip sending the CL to the CQ (default: %(default)s)')
458 p.add_argument('-v', '--verbose', action='store_true', default=False,
459 help='Be extra verbose in printing of log messages.')
460 opts = p.parse_args()
461
462 if opts.verbose:
463 logging.basicConfig(level=logging.DEBUG)
464 else:
465 logging.basicConfig(level=logging.INFO)
466
467 if not opts.ignore_unclean_workdir and not _IsTreeClean():
468 logging.error('Please clean your local checkout first.')
469 return 1
470
471 if opts.clean:
472 _RemovePreviousRollBranch(opts.dry_run)
473
474 if not opts.ignore_unclean_workdir:
475 _EnsureUpdatedMasterBranch(opts.dry_run)
476
477 new_cr_rev = opts.revision
478 if not new_cr_rev:
479 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
480 head_rev = stdout.strip().split('\t')[0]
481 logging.info('No revision specified. Using HEAD: %s', head_rev)
482 new_cr_rev = head_rev
483
484 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
485 webrtc_deps = ParseLocalDepsFile(deps_filename)
486 current_cr_rev = webrtc_deps['vars']['chromium_revision']
487
488 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
489 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
490
491 current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
492 new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
493
494 changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps)
495 clang_change = CalculateChangedClang(new_cr_rev)
496 commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
497 current_commit_pos, new_commit_pos,
498 changed_deps, clang_change)
499 logging.debug('Commit message:\n%s', commit_msg)
500
501 _CreateRollBranch(opts.dry_run)
502 UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
503 _LocalCommit(commit_msg, opts.dry_run)
504 _UploadCL(opts.dry_run, opts.rietveld_email)
505 _SendToCQ(opts.dry_run, opts.skip_cq)
506 return 0
507
508
509 if __name__ == '__main__':
510 sys.exit(main())
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698