OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
3 # | |
4 # Use of this source code is governed by a BSD-style license | |
5 # that can be found in the LICENSE file in the root of the source | |
6 # tree. An additional intellectual property rights grant can be found | |
7 # in the file PATENTS. All contributing project authors may | |
8 # be found in the AUTHORS file in the root of the source tree. | |
9 | |
10 """Script to automatically roll dependencies in the WebRTC DEPS file.""" | |
11 | |
12 import argparse | |
13 import base64 | |
14 import collections | |
15 import logging | |
16 import os | |
17 import re | |
18 import subprocess | |
19 import sys | |
20 import urllib | |
21 | |
22 | |
23 # Skip these dependencies (list without solution name prefix). | |
24 DONT_AUTOROLL_THESE = [ | |
25 'src/third_party/gflags/src', | |
26 'src/third_party/winsdk_samples', | |
27 ] | |
28 | |
29 WEBRTC_URL = 'https://chromium.googlesource.com/external/webrtc' | |
30 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' | |
31 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' | |
32 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' | |
33 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' | |
34 | |
35 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') | |
36 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$') | |
37 ROLL_BRANCH_NAME = 'roll_chromium_revision' | |
38 | |
39 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) | |
40 CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, | |
41 os.pardir)) | |
42 CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) | |
43 | |
44 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) | |
45 import find_depot_tools | |
46 find_depot_tools.add_depot_tools_to_path() | |
47 from gclient import GClientKeywords | |
48 | |
49 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' | |
50 CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', | |
51 'clang', 'scripts', 'update.py') | |
52 | |
53 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') | |
54 ChangedDep = collections.namedtuple('ChangedDep', | |
55 'path url current_rev new_rev') | |
56 | |
57 class RollError(Exception): | |
58 pass | |
59 | |
60 | |
61 def ParseDepsDict(deps_content): | |
62 local_scope = {} | |
63 var = GClientKeywords.VarImpl({}, local_scope) | |
64 global_scope = { | |
65 'From': GClientKeywords.FromImpl, | |
66 'Var': var.Lookup, | |
67 'deps_os': {}, | |
68 } | |
69 exec(deps_content, global_scope, local_scope) | |
70 return local_scope | |
71 | |
72 | |
73 def ParseLocalDepsFile(filename): | |
74 with open(filename, 'rb') as f: | |
75 deps_content = f.read() | |
76 return ParseDepsDict(deps_content) | |
77 | |
78 | |
79 def ParseRemoteCrDepsFile(revision): | |
80 deps_content = ReadRemoteCrFile('DEPS', revision) | |
81 return ParseDepsDict(deps_content) | |
82 | |
83 | |
84 def ParseCommitPosition(commit_message): | |
85 for line in reversed(commit_message.splitlines()): | |
86 m = COMMIT_POSITION_RE.match(line.strip()) | |
87 if m: | |
88 return m.group(1) | |
89 logging.error('Failed to parse commit position id from:\n%s\n', | |
90 commit_message) | |
91 sys.exit(-1) | |
92 | |
93 | |
94 def _RunCommand(command, working_dir=None, ignore_exit_code=False, | |
95 extra_env=None): | |
96 """Runs a command and returns the output from that command. | |
97 | |
98 If the command fails (exit code != 0), the function will exit the process. | |
99 | |
100 Returns: | |
101 A tuple containing the stdout and stderr outputs as strings. | |
102 """ | |
103 working_dir = working_dir or CHECKOUT_SRC_DIR | |
104 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) | |
105 env = os.environ.copy() | |
106 if extra_env: | |
107 assert all(type(value) == str for value in extra_env.values()) | |
108 logging.debug('extra env: %s', extra_env) | |
109 env.update(extra_env) | |
110 p = subprocess.Popen(command, stdout=subprocess.PIPE, | |
111 stderr=subprocess.PIPE, env=env, | |
112 cwd=working_dir, universal_newlines=True) | |
113 std_output = p.stdout.read() | |
114 err_output = p.stderr.read() | |
115 p.wait() | |
116 p.stdout.close() | |
117 p.stderr.close() | |
118 if not ignore_exit_code and p.returncode != 0: | |
119 logging.error('Command failed: %s\n' | |
120 'stdout:\n%s\n' | |
121 'stderr:\n%s\n', ' '.join(command), std_output, err_output) | |
122 sys.exit(p.returncode) | |
123 return std_output, err_output | |
124 | |
125 | |
126 def _GetBranches(): | |
127 """Returns a tuple of active,branches. | |
128 | |
129 The 'active' is the name of the currently active branch and 'branches' is a | |
130 list of all branches. | |
131 """ | |
132 lines = _RunCommand(['git', 'branch'])[0].split('\n') | |
133 branches = [] | |
134 active = '' | |
135 for line in lines: | |
136 if '*' in line: | |
137 # The assumption is that the first char will always be the '*'. | |
138 active = line[1:].strip() | |
139 branches.append(active) | |
140 else: | |
141 branch = line.strip() | |
142 if branch: | |
143 branches.append(branch) | |
144 return active, branches | |
145 | |
146 | |
147 def _ReadGitilesContent(url): | |
148 # Download and decode BASE64 content until | |
149 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. | |
150 base64_content = ReadUrlContent(url + '?format=TEXT') | |
151 return base64.b64decode(base64_content[0]) | |
152 | |
153 | |
154 def ReadRemoteCrFile(path_below_src, revision): | |
155 """Reads a remote Chromium file of a specific revision. Returns a string.""" | |
156 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, | |
157 path_below_src)) | |
158 | |
159 | |
160 def ReadRemoteCrCommit(revision): | |
161 """Reads a remote Chromium commit message. Returns a string.""" | |
162 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) | |
163 | |
164 | |
165 def ReadUrlContent(url): | |
166 """Connect to a remote host and read the contents. Returns a list of lines.""" | |
167 conn = urllib.urlopen(url) | |
168 try: | |
169 return conn.readlines() | |
170 except IOError as e: | |
171 logging.exception('Error connecting to %s. Error: %s', url, e) | |
172 raise | |
173 finally: | |
174 conn.close() | |
175 | |
176 | |
177 def GetMatchingDepsEntries(depsentry_dict, dir_path): | |
178 """Gets all deps entries matching the provided path. | |
179 | |
180 This list may contain more than one DepsEntry object. | |
181 Example: dir_path='src/testing' would give results containing both | |
182 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. | |
183 Example 2: dir_path='src/build' should return 'src/build' but not | |
184 'src/buildtools'. | |
185 | |
186 Returns: | |
187 A list of DepsEntry objects. | |
188 """ | |
189 result = [] | |
190 for path, depsentry in depsentry_dict.iteritems(): | |
191 if path == dir_path: | |
192 result.append(depsentry) | |
193 else: | |
194 parts = path.split('/') | |
195 if all(part == parts[i] | |
196 for i, part in enumerate(dir_path.split('/'))): | |
197 result.append(depsentry) | |
198 return result | |
199 | |
200 | |
201 def BuildDepsentryDict(deps_dict): | |
202 """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" | |
203 result = {} | |
204 def AddDepsEntries(deps_subdict): | |
205 for path, deps_url in deps_subdict.iteritems(): | |
206 if not result.has_key(path): | |
207 url, revision = deps_url.split('@') if deps_url else (None, None) | |
208 result[path] = DepsEntry(path, url, revision) | |
209 | |
210 AddDepsEntries(deps_dict['deps']) | |
211 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: | |
212 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) | |
213 return result | |
214 | |
215 | |
216 def CalculateChangedDeps(webrtc_deps, new_cr_deps): | |
217 """ | |
218 Calculate changed deps entries based on entries defined in the WebRTC DEPS | |
219 file: | |
220 - If a shared dependency with the Chromium DEPS file: roll it to the same | |
221 revision as Chromium (i.e. entry in the new_cr_deps dict) | |
222 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice | |
223 this means it may be ahead of the chromium_revision, but generally these | |
224 should be close). | |
225 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD | |
226 unless it's configured to be skipped. | |
227 | |
228 Returns: | |
229 A list of ChangedDep objects representing the changed deps. | |
230 """ | |
231 result = [] | |
232 webrtc_entries = BuildDepsentryDict(webrtc_deps) | |
233 new_cr_entries = BuildDepsentryDict(new_cr_deps) | |
234 for path, webrtc_deps_entry in webrtc_entries.iteritems(): | |
235 if path in DONT_AUTOROLL_THESE: | |
236 continue | |
237 cr_deps_entry = new_cr_entries.get(path) | |
238 if cr_deps_entry: | |
239 # Use the revision from Chromium's DEPS file. | |
240 new_rev = cr_deps_entry.revision | |
241 assert webrtc_deps_entry.url == cr_deps_entry.url, ( | |
242 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' % | |
243 (path, webrtc_deps_entry.url, cr_deps_entry.url)) | |
244 else: | |
245 # Use the HEAD of the deps repo. | |
246 stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url, | |
247 'HEAD']) | |
248 new_rev = stdout.strip().split('\t')[0] | |
249 | |
250 # Check if an update is necessary. | |
251 if webrtc_deps_entry.revision != new_rev: | |
252 logging.debug('Roll dependency %s to %s', path, new_rev) | |
253 result.append(ChangedDep(path, webrtc_deps_entry.url, | |
254 webrtc_deps_entry.revision, new_rev)) | |
255 return sorted(result) | |
256 | |
257 | |
258 def CalculateChangedClang(new_cr_rev): | |
259 def GetClangRev(lines): | |
260 for line in lines: | |
261 match = CLANG_REVISION_RE.match(line) | |
262 if match: | |
263 return match.group(1) | |
264 raise RollError('Could not parse Clang revision!') | |
265 | |
266 with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f: | |
267 current_lines = f.readlines() | |
268 current_rev = GetClangRev(current_lines) | |
269 | |
270 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, | |
271 new_cr_rev).splitlines() | |
272 new_rev = GetClangRev(new_clang_update_py) | |
273 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) | |
274 | |
275 | |
276 def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, | |
277 new_commit_pos, changed_deps_list, clang_change): | |
278 current_cr_rev = current_cr_rev[0:10] | |
279 new_cr_rev = new_cr_rev[0:10] | |
280 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) | |
281 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) | |
282 | |
283 commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, | |
284 git_number_interval)] | |
285 commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) | |
286 commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % | |
287 rev_interval)) | |
288 # TBR field will be empty unless in some custom cases, where some engineers | |
289 # are added. | |
290 tbr_authors = '' | |
291 if changed_deps_list: | |
292 commit_msg.append('Changed dependencies:') | |
293 | |
294 for c in changed_deps_list: | |
295 commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url, | |
296 c.current_rev[0:10], | |
297 c.new_rev[0:10])) | |
298 if 'libvpx' in c.path: | |
299 tbr_authors += 'marpan@webrtc.org, ' | |
300 | |
301 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') | |
302 commit_msg.append('DEPS diff: %s\n' % change_url) | |
303 else: | |
304 commit_msg.append('No dependencies changed.') | |
305 | |
306 if clang_change.current_rev != clang_change.new_rev: | |
307 commit_msg.append('Clang version changed %s:%s' % | |
308 (clang_change.current_rev, clang_change.new_rev)) | |
309 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, | |
310 CLANG_UPDATE_SCRIPT_URL_PATH) | |
311 commit_msg.append('Details: %s\n' % change_url) | |
312 else: | |
313 commit_msg.append('No update to Clang.\n') | |
314 | |
315 commit_msg.append('TBR=%s' % tbr_authors) | |
316 commit_msg.append('BUG=None') | |
317 return '\n'.join(commit_msg) | |
318 | |
319 | |
320 def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, | |
321 changed_deps): | |
322 """Update the DEPS file with the new revision.""" | |
323 | |
324 # Update the chromium_revision variable. | |
325 with open(deps_filename, 'rb') as deps_file: | |
326 deps_content = deps_file.read() | |
327 deps_content = deps_content.replace(old_cr_revision, new_cr_revision) | |
328 with open(deps_filename, 'wb') as deps_file: | |
329 deps_file.write(deps_content) | |
330 | |
331 # Update each individual DEPS entry. | |
332 for dep in changed_deps: | |
333 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) | |
334 if not os.path.isdir(local_dep_dir): | |
335 raise RollError( | |
336 'Cannot find local directory %s. Either run\n' | |
337 'gclient sync --deps=all\n' | |
338 'or make sure the .gclient file for your solution contains all ' | |
339 'platforms in the target_os list, i.e.\n' | |
340 'target_os = ["android", "unix", "mac", "ios", "win"];\n' | |
341 'Then run "gclient sync" again.' % local_dep_dir) | |
342 _, stderr = _RunCommand( | |
343 ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev], | |
344 working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True) | |
345 if stderr: | |
346 logging.warning('roll-dep-svn: %s', stderr) | |
347 | |
348 | |
349 def _IsTreeClean(): | |
350 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) | |
351 if len(stdout) == 0: | |
352 return True | |
353 | |
354 logging.error('Dirty/unversioned files:\n%s', stdout) | |
355 return False | |
356 | |
357 | |
358 def _EnsureUpdatedMasterBranch(dry_run): | |
359 current_branch = _RunCommand( | |
360 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] | |
361 if current_branch != 'master': | |
362 logging.error('Please checkout the master branch and re-run this script.') | |
363 if not dry_run: | |
364 sys.exit(-1) | |
365 | |
366 logging.info('Updating master branch...') | |
367 _RunCommand(['git', 'pull']) | |
368 | |
369 | |
370 def _CreateRollBranch(dry_run): | |
371 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) | |
372 if not dry_run: | |
373 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) | |
374 | |
375 | |
376 def _RemovePreviousRollBranch(dry_run): | |
377 active_branch, branches = _GetBranches() | |
378 if active_branch == ROLL_BRANCH_NAME: | |
379 active_branch = 'master' | |
380 if ROLL_BRANCH_NAME in branches: | |
381 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) | |
382 if not dry_run: | |
383 _RunCommand(['git', 'checkout', active_branch]) | |
384 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) | |
385 | |
386 | |
387 def _LocalCommit(commit_msg, dry_run): | |
388 logging.info('Committing changes locally.') | |
389 if not dry_run: | |
390 _RunCommand(['git', 'add', '--update', '.']) | |
391 _RunCommand(['git', 'commit', '-m', commit_msg]) | |
392 | |
393 | |
394 def _UploadCL(dry_run, rietveld_email=None): | |
395 logging.info('Uploading CL...') | |
396 if not dry_run: | |
397 cmd = ['git', 'cl', 'upload', '-f'] | |
398 if rietveld_email: | |
399 cmd.append('--email=%s' % rietveld_email) | |
400 _RunCommand(cmd, extra_env={'EDITOR': 'true'}) | |
401 | |
402 | |
403 def _SendToCQ(dry_run, skip_cq): | |
404 logging.info('Sending the CL to the CQ...') | |
405 if not dry_run and not skip_cq: | |
406 _RunCommand(['git', 'cl', 'set_commit']) | |
407 logging.info('Sent the CL to the CQ.') | |
408 | |
409 | |
410 def main(): | |
411 p = argparse.ArgumentParser() | |
412 p.add_argument('--clean', action='store_true', default=False, | |
413 help='Removes any previous local roll branch.') | |
414 p.add_argument('-r', '--revision', | |
415 help=('Chromium Git revision to roll to. Defaults to the ' | |
416 'Chromium HEAD revision if omitted.')) | |
417 p.add_argument('-u', '--rietveld-email', | |
418 help=('E-mail address to use for creating the CL at Rietveld' | |
419 'If omitted a previously cached one will be used or an ' | |
420 'error will be thrown during upload.')) | |
421 p.add_argument('--dry-run', action='store_true', default=False, | |
422 help=('Calculate changes and modify DEPS, but don\'t create ' | |
423 'any local branch, commit, upload CL or send any ' | |
424 'tryjobs.')) | |
425 p.add_argument('-i', '--ignore-unclean-workdir', action='store_true', | |
426 default=False, | |
427 help=('Ignore if the current branch is not master or if there ' | |
428 'are uncommitted changes (default: %(default)s).')) | |
429 p.add_argument('--skip-cq', action='store_true', default=False, | |
430 help='Skip sending the CL to the CQ (default: %(default)s)') | |
431 p.add_argument('-v', '--verbose', action='store_true', default=False, | |
432 help='Be extra verbose in printing of log messages.') | |
433 opts = p.parse_args() | |
434 | |
435 if opts.verbose: | |
436 logging.basicConfig(level=logging.DEBUG) | |
437 else: | |
438 logging.basicConfig(level=logging.INFO) | |
439 | |
440 if not opts.ignore_unclean_workdir and not _IsTreeClean(): | |
441 logging.error('Please clean your local checkout first.') | |
442 return 1 | |
443 | |
444 if opts.clean: | |
445 _RemovePreviousRollBranch(opts.dry_run) | |
446 | |
447 if not opts.ignore_unclean_workdir: | |
448 _EnsureUpdatedMasterBranch(opts.dry_run) | |
449 | |
450 new_cr_rev = opts.revision | |
451 if not new_cr_rev: | |
452 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) | |
453 head_rev = stdout.strip().split('\t')[0] | |
454 logging.info('No revision specified. Using HEAD: %s', head_rev) | |
455 new_cr_rev = head_rev | |
456 | |
457 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') | |
458 webrtc_deps = ParseLocalDepsFile(deps_filename) | |
459 current_cr_rev = webrtc_deps['vars']['chromium_revision'] | |
460 | |
461 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) | |
462 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) | |
463 | |
464 new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) | |
465 changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) | |
466 clang_change = CalculateChangedClang(new_cr_rev) | |
467 commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, | |
468 current_commit_pos, new_commit_pos, | |
469 changed_deps, clang_change) | |
470 logging.debug('Commit message:\n%s', commit_msg) | |
471 | |
472 _CreateRollBranch(opts.dry_run) | |
473 UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) | |
474 _LocalCommit(commit_msg, opts.dry_run) | |
475 _UploadCL(opts.dry_run, opts.rietveld_email) | |
476 _SendToCQ(opts.dry_run, opts.skip_cq) | |
477 return 0 | |
478 | |
479 | |
480 if __name__ == '__main__': | |
481 sys.exit(main()) | |
OLD | NEW |