OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
3 # | |
4 # Use of this source code is governed by a BSD-style license | |
5 # that can be found in the LICENSE file in the root of the source | |
6 # tree. An additional intellectual property rights grant can be found | |
7 # in the file PATENTS. All contributing project authors may | |
8 # be found in the AUTHORS file in the root of the source tree. | |
9 | |
10 """Script to roll chromium_revision in the WebRTC DEPS file.""" | |
11 | |
12 import argparse | |
13 import base64 | |
14 import collections | |
15 import logging | |
16 import os | |
17 import re | |
18 import subprocess | |
19 import sys | |
20 import urllib | |
21 | |
22 | |
23 # Skip these dependencies (list without solution name prefix). | |
24 DONT_AUTOROLL_THESE = [ | |
25 'src/third_party/gflags/src', | |
26 'src/third_party/winsdk_samples/src', | |
27 ] | |
28 | |
29 WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc' | |
30 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' | |
31 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' | |
32 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' | |
33 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' | |
34 | |
35 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') | |
36 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$') | |
37 ROLL_BRANCH_NAME = 'roll_chromium_revision' | |
38 | |
39 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
40 CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, | |
41 os.pardir)) | |
42 CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) | |
43 CHROMIUM_CHECKOUT_SRC_DIR = os.path.join(CHECKOUT_SRC_DIR, 'chromium', 'src') | |
44 | |
45 sys.path.append(CHECKOUT_SRC_DIR) | |
46 import setup_links | |
47 | |
48 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) | |
49 import find_depot_tools | |
50 find_depot_tools.add_depot_tools_to_path() | |
51 from gclient import GClientKeywords | |
52 | |
53 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' | |
54 CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts', | |
55 'update.py') | |
56 | |
57 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') | |
58 ChangedDep = collections.namedtuple('ChangedDep', | |
59 'path url current_rev new_rev') | |
60 | |
61 class RollError(Exception): | |
62 pass | |
63 | |
64 | |
65 def ParseDepsDict(deps_content): | |
66 local_scope = {} | |
67 var = GClientKeywords.VarImpl({}, local_scope) | |
68 global_scope = { | |
69 'From': GClientKeywords.FromImpl, | |
70 'Var': var.Lookup, | |
71 'deps_os': {}, | |
72 } | |
73 exec(deps_content, global_scope, local_scope) | |
74 return local_scope | |
75 | |
76 | |
77 def ParseLocalDepsFile(filename): | |
78 with open(filename, 'rb') as f: | |
79 deps_content = f.read() | |
80 return ParseDepsDict(deps_content) | |
81 | |
82 | |
83 def ParseRemoteCrDepsFile(revision): | |
84 deps_content = ReadRemoteCrFile('DEPS', revision) | |
85 return ParseDepsDict(deps_content) | |
86 | |
87 | |
88 def ParseCommitPosition(commit_message): | |
89 for line in reversed(commit_message.splitlines()): | |
90 m = COMMIT_POSITION_RE.match(line.strip()) | |
91 if m: | |
92 return m.group(1) | |
93 logging.error('Failed to parse commit position id from:\n%s\n', | |
94 commit_message) | |
95 sys.exit(-1) | |
96 | |
97 | |
98 def _RunCommand(command, working_dir=None, ignore_exit_code=False, | |
99 extra_env=None): | |
100 """Runs a command and returns the output from that command. | |
101 | |
102 If the command fails (exit code != 0), the function will exit the process. | |
103 | |
104 Returns: | |
105 A tuple containing the stdout and stderr outputs as strings. | |
106 """ | |
107 working_dir = working_dir or CHECKOUT_SRC_DIR | |
108 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) | |
109 env = os.environ.copy() | |
110 if extra_env: | |
111 assert all(type(value) == str for value in extra_env.values()) | |
112 logging.debug('extra env: %s', extra_env) | |
113 env.update(extra_env) | |
114 p = subprocess.Popen(command, stdout=subprocess.PIPE, | |
115 stderr=subprocess.PIPE, env=env, | |
116 cwd=working_dir, universal_newlines=True) | |
117 std_output = p.stdout.read() | |
118 err_output = p.stderr.read() | |
119 p.wait() | |
120 p.stdout.close() | |
121 p.stderr.close() | |
122 if not ignore_exit_code and p.returncode != 0: | |
123 logging.error('Command failed: %s\n' | |
124 'stdout:\n%s\n' | |
125 'stderr:\n%s\n', ' '.join(command), std_output, err_output) | |
126 sys.exit(p.returncode) | |
127 return std_output, err_output | |
128 | |
129 | |
130 def _GetBranches(): | |
131 """Returns a tuple of active,branches. | |
132 | |
133 The 'active' is the name of the currently active branch and 'branches' is a | |
134 list of all branches. | |
135 """ | |
136 lines = _RunCommand(['git', 'branch'])[0].split('\n') | |
137 branches = [] | |
138 active = '' | |
139 for line in lines: | |
140 if '*' in line: | |
141 # The assumption is that the first char will always be the '*'. | |
142 active = line[1:].strip() | |
143 branches.append(active) | |
144 else: | |
145 branch = line.strip() | |
146 if branch: | |
147 branches.append(branch) | |
148 return active, branches | |
149 | |
150 | |
151 def _ReadGitilesContent(url): | |
152 # Download and decode BASE64 content until | |
153 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. | |
154 base64_content = ReadUrlContent(url + '?format=TEXT') | |
155 return base64.b64decode(base64_content[0]) | |
156 | |
157 | |
158 def ReadRemoteCrFile(path_below_src, revision): | |
159 """Reads a remote Chromium file of a specific revision. Returns a string.""" | |
160 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, | |
161 path_below_src)) | |
162 | |
163 | |
164 def ReadRemoteCrCommit(revision): | |
165 """Reads a remote Chromium commit message. Returns a string.""" | |
166 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) | |
167 | |
168 | |
169 def ReadUrlContent(url): | |
170 """Connect to a remote host and read the contents. Returns a list of lines.""" | |
171 conn = urllib.urlopen(url) | |
172 try: | |
173 return conn.readlines() | |
174 except IOError as e: | |
175 logging.exception('Error connecting to %s. Error: %s', url, e) | |
176 raise | |
177 finally: | |
178 conn.close() | |
179 | |
180 | |
181 def GetMatchingDepsEntries(depsentry_dict, dir_path): | |
182 """Gets all deps entries matching the provided path. | |
183 | |
184 This list may contain more than one DepsEntry object. | |
185 Example: dir_path='src/testing' would give results containing both | |
186 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. | |
187 Example 2: dir_path='src/build' should return 'src/build' but not | |
188 'src/buildtools'. | |
189 | |
190 Returns: | |
191 A list of DepsEntry objects. | |
192 """ | |
193 result = [] | |
194 for path, depsentry in depsentry_dict.iteritems(): | |
195 if path == dir_path: | |
196 result.append(depsentry) | |
197 else: | |
198 parts = path.split('/') | |
199 if all(part == parts[i] | |
200 for i, part in enumerate(dir_path.split('/'))): | |
201 result.append(depsentry) | |
202 return result | |
203 | |
204 | |
205 def BuildDepsentryDict(deps_dict): | |
206 """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" | |
207 result = {} | |
208 def AddDepsEntries(deps_subdict): | |
209 for path, deps_url in deps_subdict.iteritems(): | |
210 if not result.has_key(path): | |
211 url, revision = deps_url.split('@') if deps_url else (None, None) | |
212 result[path] = DepsEntry(path, url, revision) | |
213 | |
214 AddDepsEntries(deps_dict['deps']) | |
215 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: | |
216 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) | |
217 return result | |
218 | |
219 | |
220 def CalculateChangedDeps(webrtc_deps, old_cr_deps, new_cr_deps): | |
221 """ | |
222 Calculate changed deps entries based on: | |
223 1. Entries defined in the WebRTC DEPS file: | |
224 - If a shared dependency with the Chromium DEPS file: roll it to the same | |
225 revision as Chromium (i.e. entry in the new_cr_deps dict) | |
226 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice | |
227 this means it may be ahead of the chromium_revision, but generally these | |
228 should be close). | |
229 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD | |
230 unless it's configured to be skipped. | |
231 2. Entries present in the setup_links.py file. If the dir has changed between | |
232 old_cr_deps and new_cr_deps, it is considered changed and updated to the | |
233 revision for the entry in the new_cr_deps dict. | |
234 | |
235 Returns: | |
236 A list of ChangedDep objects representing the changed deps. | |
237 """ | |
238 return sorted(CalculateChangedDepsProper(webrtc_deps, new_cr_deps) + | |
239 CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps)) | |
240 | |
241 | |
242 def CalculateChangedDepsProper(webrtc_deps, new_cr_deps): | |
243 result = [] | |
244 webrtc_entries = BuildDepsentryDict(webrtc_deps) | |
245 new_cr_entries = BuildDepsentryDict(new_cr_deps) | |
246 for path, webrtc_deps_entry in webrtc_entries.iteritems(): | |
247 if path in DONT_AUTOROLL_THESE: | |
248 continue | |
249 cr_deps_entry = new_cr_entries.get(path) | |
250 if cr_deps_entry: | |
251 # Use the revision from Chromium's DEPS file. | |
252 new_rev = cr_deps_entry.revision | |
253 assert webrtc_deps_entry.url == cr_deps_entry.url, ( | |
254 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' % | |
255 (path, webrtc_deps_entry.url, cr_deps_entry.url)) | |
256 else: | |
257 # Use the HEAD of the deps repo. | |
258 stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url, | |
259 'HEAD']) | |
260 new_rev = stdout.strip().split('\t')[0] | |
261 | |
262 # Check if an update is necessary. | |
263 if webrtc_deps_entry.revision != new_rev: | |
264 logging.debug('Roll dependency %s to %s', path, new_rev) | |
265 result.append(ChangedDep(path, webrtc_deps_entry.url, | |
266 webrtc_deps_entry.revision, new_rev)) | |
267 return result | |
268 | |
269 | |
270 def CalculateChangedDepsLegacy(old_cr_deps, new_cr_deps): | |
271 result = [] | |
272 new_cr_entries = BuildDepsentryDict(new_cr_deps) | |
273 old_cr_entries = BuildDepsentryDict(old_cr_deps) | |
274 all_deps_dirs = setup_links.DIRECTORIES | |
275 for deps_dir in all_deps_dirs: | |
276 # All deps have 'src' prepended to the path in the Chromium DEPS file. | |
277 dir_path = 'src/%s' % deps_dir | |
278 | |
279 for entry in GetMatchingDepsEntries(old_cr_entries, dir_path): | |
280 new_matching_entries = GetMatchingDepsEntries(new_cr_entries, entry.path) | |
281 assert len(new_matching_entries) <= 1, ( | |
282 'Should never find more than one entry matching %s in %s, found %d' % | |
283 (entry.path, new_cr_entries, len(new_matching_entries))) | |
284 if not new_matching_entries: | |
285 result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None')) | |
286 elif entry != new_matching_entries[0]: | |
287 result.append(ChangedDep(entry.path, entry.url, entry.revision, | |
288 new_matching_entries[0].revision)) | |
289 return result | |
290 | |
291 | |
292 def CalculateChangedClang(new_cr_rev): | |
293 def GetClangRev(lines): | |
294 for line in lines: | |
295 match = CLANG_REVISION_RE.match(line) | |
296 if match: | |
297 return match.group(1) | |
298 raise RollError('Could not parse Clang revision!') | |
299 | |
300 chromium_src_path = os.path.join(CHROMIUM_CHECKOUT_SRC_DIR, | |
301 CLANG_UPDATE_SCRIPT_LOCAL_PATH) | |
302 with open(chromium_src_path, 'rb') as f: | |
303 current_lines = f.readlines() | |
304 current_rev = GetClangRev(current_lines) | |
305 | |
306 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, | |
307 new_cr_rev).splitlines() | |
308 new_rev = GetClangRev(new_clang_update_py) | |
309 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) | |
310 | |
311 | |
312 def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, | |
313 new_commit_pos, changed_deps_list, clang_change): | |
314 current_cr_rev = current_cr_rev[0:10] | |
315 new_cr_rev = new_cr_rev[0:10] | |
316 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) | |
317 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) | |
318 | |
319 commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, | |
320 git_number_interval)] | |
321 commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) | |
322 commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % | |
323 rev_interval)) | |
324 # TBR field will be empty unless in some custom cases, where some engineers | |
325 # are added. | |
326 tbr_authors = '' | |
327 if changed_deps_list: | |
328 commit_msg.append('Changed dependencies:') | |
329 | |
330 for c in changed_deps_list: | |
331 commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url, | |
332 c.current_rev[0:10], | |
333 c.new_rev[0:10])) | |
334 if 'libvpx' in c.path: | |
335 tbr_authors += 'marpan@webrtc.org, ' | |
336 | |
337 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') | |
338 commit_msg.append('DEPS diff: %s\n' % change_url) | |
339 else: | |
340 commit_msg.append('No dependencies changed.') | |
341 | |
342 if clang_change.current_rev != clang_change.new_rev: | |
343 commit_msg.append('Clang version changed %s:%s' % | |
344 (clang_change.current_rev, clang_change.new_rev)) | |
345 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, | |
346 CLANG_UPDATE_SCRIPT_URL_PATH) | |
347 commit_msg.append('Details: %s\n' % change_url) | |
348 else: | |
349 commit_msg.append('No update to Clang.\n') | |
350 | |
351 commit_msg.append('TBR=%s' % tbr_authors) | |
352 commit_msg.append('BUG=None') | |
353 return '\n'.join(commit_msg) | |
354 | |
355 | |
356 def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, | |
357 changed_deps): | |
358 """Update the DEPS file with the new revision.""" | |
359 | |
360 # Update the chromium_revision variable. | |
361 with open(deps_filename, 'rb') as deps_file: | |
362 deps_content = deps_file.read() | |
363 deps_content = deps_content.replace(old_cr_revision, new_cr_revision) | |
364 with open(deps_filename, 'wb') as deps_file: | |
365 deps_file.write(deps_content) | |
366 | |
367 # Update each individual DEPS entry. | |
368 for dep in changed_deps: | |
369 _, stderr = _RunCommand( | |
370 ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev], | |
371 working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True) | |
372 if stderr: | |
373 logging.warning('roll-dep-svn: %s', stderr) | |
374 | |
375 | |
376 def _IsTreeClean(): | |
377 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) | |
378 if len(stdout) == 0: | |
379 return True | |
380 | |
381 logging.error('Dirty/unversioned files:\n%s', stdout) | |
382 return False | |
383 | |
384 | |
385 def _EnsureUpdatedMasterBranch(dry_run): | |
386 current_branch = _RunCommand( | |
387 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] | |
388 if current_branch != 'master': | |
389 logging.error('Please checkout the master branch and re-run this script.') | |
390 if not dry_run: | |
391 sys.exit(-1) | |
392 | |
393 logging.info('Updating master branch...') | |
394 _RunCommand(['git', 'pull']) | |
395 | |
396 | |
397 def _CreateRollBranch(dry_run): | |
398 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) | |
399 if not dry_run: | |
400 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) | |
401 | |
402 | |
403 def _RemovePreviousRollBranch(dry_run): | |
404 active_branch, branches = _GetBranches() | |
405 if active_branch == ROLL_BRANCH_NAME: | |
406 active_branch = 'master' | |
407 if ROLL_BRANCH_NAME in branches: | |
408 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) | |
409 if not dry_run: | |
410 _RunCommand(['git', 'checkout', active_branch]) | |
411 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) | |
412 | |
413 | |
414 def _LocalCommit(commit_msg, dry_run): | |
415 logging.info('Committing changes locally.') | |
416 if not dry_run: | |
417 _RunCommand(['git', 'add', '--update', '.']) | |
418 _RunCommand(['git', 'commit', '-m', commit_msg]) | |
419 | |
420 | |
421 def _UploadCL(dry_run, rietveld_email=None): | |
422 logging.info('Uploading CL...') | |
423 if not dry_run: | |
424 cmd = ['git', 'cl', 'upload', '-f'] | |
425 if rietveld_email: | |
426 cmd.append('--email=%s' % rietveld_email) | |
427 _RunCommand(cmd, extra_env={'EDITOR': 'true'}) | |
428 | |
429 | |
430 def _SendToCQ(dry_run, skip_cq): | |
431 logging.info('Sending the CL to the CQ...') | |
432 if not dry_run and not skip_cq: | |
433 _RunCommand(['git', 'cl', 'set_commit']) | |
434 logging.info('Sent the CL to the CQ.') | |
435 | |
436 | |
437 def main(): | |
438 p = argparse.ArgumentParser() | |
439 p.add_argument('--clean', action='store_true', default=False, | |
440 help='Removes any previous local roll branch.') | |
441 p.add_argument('-r', '--revision', | |
442 help=('Chromium Git revision to roll to. Defaults to the ' | |
443 'Chromium HEAD revision if omitted.')) | |
444 p.add_argument('-u', '--rietveld-email', | |
445 help=('E-mail address to use for creating the CL at Rietveld' | |
446 'If omitted a previously cached one will be used or an ' | |
447 'error will be thrown during upload.')) | |
448 p.add_argument('--dry-run', action='store_true', default=False, | |
449 help=('Calculate changes and modify DEPS, but don\'t create ' | |
450 'any local branch, commit, upload CL or send any ' | |
451 'tryjobs.')) | |
452 p.add_argument('-i', '--ignore-unclean-workdir', action='store_true', | |
453 default=False, | |
454 help=('Ignore if the current branch is not master or if there ' | |
455 'are uncommitted changes (default: %(default)s).')) | |
456 p.add_argument('--skip-cq', action='store_true', default=False, | |
457 help='Skip sending the CL to the CQ (default: %(default)s)') | |
458 p.add_argument('-v', '--verbose', action='store_true', default=False, | |
459 help='Be extra verbose in printing of log messages.') | |
460 opts = p.parse_args() | |
461 | |
462 if opts.verbose: | |
463 logging.basicConfig(level=logging.DEBUG) | |
464 else: | |
465 logging.basicConfig(level=logging.INFO) | |
466 | |
467 if not opts.ignore_unclean_workdir and not _IsTreeClean(): | |
468 logging.error('Please clean your local checkout first.') | |
469 return 1 | |
470 | |
471 if opts.clean: | |
472 _RemovePreviousRollBranch(opts.dry_run) | |
473 | |
474 if not opts.ignore_unclean_workdir: | |
475 _EnsureUpdatedMasterBranch(opts.dry_run) | |
476 | |
477 new_cr_rev = opts.revision | |
478 if not new_cr_rev: | |
479 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) | |
480 head_rev = stdout.strip().split('\t')[0] | |
481 logging.info('No revision specified. Using HEAD: %s', head_rev) | |
482 new_cr_rev = head_rev | |
483 | |
484 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') | |
485 webrtc_deps = ParseLocalDepsFile(deps_filename) | |
486 current_cr_rev = webrtc_deps['vars']['chromium_revision'] | |
487 | |
488 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) | |
489 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) | |
490 | |
491 current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev) | |
492 new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) | |
493 | |
494 changed_deps = CalculateChangedDeps(webrtc_deps, current_cr_deps, new_cr_deps) | |
495 clang_change = CalculateChangedClang(new_cr_rev) | |
496 commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, | |
497 current_commit_pos, new_commit_pos, | |
498 changed_deps, clang_change) | |
499 logging.debug('Commit message:\n%s', commit_msg) | |
500 | |
501 _CreateRollBranch(opts.dry_run) | |
502 UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) | |
503 _LocalCommit(commit_msg, opts.dry_run) | |
504 _UploadCL(opts.dry_run, opts.rietveld_email) | |
505 _SendToCQ(opts.dry_run, opts.skip_cq) | |
506 return 0 | |
507 | |
508 | |
509 if __name__ == '__main__': | |
510 sys.exit(main()) | |
OLD | NEW |