OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 | |
3 # Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. | |
4 # | |
5 # Use of this source code is governed by a BSD-style license | |
6 # that can be found in the LICENSE file in the root of the source | |
7 # tree. An additional intellectual property rights grant can be found | |
8 # in the file PATENTS. All contributing project authors may | |
9 # be found in the AUTHORS file in the root of the source tree. | |
10 | |
11 """Given the output of -t commands from a ninja build for a gyp and GN generated | |
12 build, report on differences between the command lines. | |
13 | |
14 | |
15 When invoked from the command line, this script assumes that the GN and GYP | |
16 targets have been generated in the specified folders. It is meant to be used as | |
17 follows: | |
18 $ python tools/gyp_flag_compare.py gyp_dir gn_dir target | |
19 | |
20 When the GN and GYP target names differ, it should be called invoked as follows: | |
21 $ python tools/gyp_flag_compare.py gyp_dir gn_dir gyp_target gn_target | |
22 | |
23 When all targets want to be compared, it should be called without a target name, | |
24 i.e.: | |
25 $ python tools/gyp_flag_compare.py gyp_dir gn_dir | |
26 | |
27 This script can also be used interactively. Then ConfigureBuild can optionally | |
28 be used to generate ninja files with GYP and GN. | |
29 Here's an example setup. Note that the current working directory must be the | |
30 project root: | |
31 $ PYTHONPATH=tools python | |
32 >>> import sys | |
33 >>> import pprint | |
34 >>> sys.displayhook = pprint.pprint | |
35 >>> import gyp_flag_compare as fc | |
36 >>> fc.ConfigureBuild(['gyp_define=1', 'define=2'], ['gn_arg=1', 'arg=2']) | |
37 >>> modules_unittests = fc.Comparison('modules_unittests') | |
38 | |
39 The above starts interactive Python, sets up the output to be pretty-printed | |
40 (useful for making lists, dicts, and sets readable), configures the build with | |
41 GN arguments and GYP defines, and then generates a comparison for that build | |
42 configuration for the "modules_unittests" target. | |
43 | |
44 After that, the |modules_unittests| object can be used to investigate | |
45 differences in the build. | |
46 | |
47 To configure an official build, use this configuration. Disabling NaCl produces | |
48 a more meaningful comparison, as certain files need to get compiled twice | |
49 for the IRT build, which uses different flags: | |
50 >>> fc.ConfigureBuild( | |
51 ['disable_nacl=1', 'buildtype=Official', 'branding=Chrome'], | |
52 ['enable_nacl=false', 'is_official_build=true', | |
53 'is_chrome_branded=true']) | |
54 """ | |
55 | |
56 | |
57 import os | |
58 import shlex | |
59 import subprocess | |
60 import sys | |
61 | |
62 # Must be in src/. | |
63 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
64 os.chdir(BASE_DIR) | |
65 | |
66 _DEFAULT_GN_DIR = 'out/gn' | |
67 _DEFAULT_GYP_DIR = 'out/Release' | |
68 | |
69 | |
70 def FilterChromium(filename): | |
71 """Replaces 'chromium/src/' by '' in the filename.""" | |
72 return filename.replace('chromium/src/', '') | |
73 | |
74 | |
75 def ConfigureBuild(gyp_args=None, gn_args=None, gn_dir=_DEFAULT_GN_DIR): | |
76 """Generates gn and gyp targets with the given arguments.""" | |
77 gyp_args = gyp_args or [] | |
78 gn_args = gn_args or [] | |
79 | |
80 print >> sys.stderr, 'Regenerating GN in %s...' % gn_dir | |
81 # Currently only Release, non-component. | |
82 Run('gn gen %s --args="is_debug=false is_component_build=false %s"' % \ | |
83 (gn_dir, ' '.join(gn_args))) | |
84 | |
85 os.environ.pop('GYP_DEFINES', None) | |
86 # Remove environment variables required by gn but conflicting with GYP. | |
87 # Relevant if Windows toolchain isn't provided by depot_tools. | |
88 os.environ.pop('GYP_MSVS_OVERRIDE_PATH', None) | |
89 os.environ.pop('WINDOWSSDKDIR', None) | |
90 | |
91 gyp_defines = '' | |
92 if len(gyp_args) > 0: | |
93 gyp_defines = '-D' + ' -D'.join(gyp_args) | |
94 | |
95 print >> sys.stderr, 'Regenerating GYP in %s...' % _DEFAULT_GYP_DIR | |
96 Run('python webrtc/build/gyp_webrtc.py -Gconfig=Release %s' % gyp_defines) | |
97 | |
98 | |
99 def Counts(dict_of_list): | |
100 """Given a dictionary whose value are lists, returns a dictionary whose values | |
101 are the length of the list. This can be used to summarize a dictionary. | |
102 """ | |
103 return {k: len(v) for k, v in dict_of_list.iteritems()} | |
104 | |
105 | |
106 def CountsByDirname(dict_of_list): | |
107 """Given a list of files, returns a dict of dirname to counts in that dir.""" | |
108 r = {} | |
109 for path in dict_of_list: | |
110 dirname = os.path.dirname(path) | |
111 r.setdefault(dirname, 0) | |
112 r[dirname] += 1 | |
113 return r | |
114 | |
115 | |
116 class Comparison(object): | |
117 """A comparison of the currently-configured build for a target.""" | |
118 | |
119 def __init__(self, gyp_target="", gn_target=None, gyp_dir=_DEFAULT_GYP_DIR, | |
120 gn_dir=_DEFAULT_GN_DIR): | |
121 """Creates a comparison of a GN and GYP target. If the target names differ | |
122 between the two build systems, then two names may be passed. | |
123 """ | |
124 if gn_target is None: | |
125 gn_target = gyp_target | |
126 | |
127 self._gyp_target = gyp_target | |
128 self._gn_target = gn_target | |
129 | |
130 self._gyp_dir = gyp_dir | |
131 self._gn_dir = gn_dir | |
132 | |
133 self._skipped = [] | |
134 | |
135 self._total_diffs = 0 | |
136 | |
137 self._missing_gyp_flags = {} | |
138 self._missing_gn_flags = {} | |
139 | |
140 self._missing_gyp_files = {} | |
141 self._missing_gn_files = {} | |
142 | |
143 self._CompareFiles() | |
144 | |
145 @property | |
146 def gyp_files(self): | |
147 """Returns the set of files that are in the GYP target.""" | |
148 return set(self._gyp_flags.keys()) | |
149 | |
150 @property | |
151 def gn_files(self): | |
152 """Returns the set of files that are in the GN target.""" | |
153 return set(self._gn_flags.keys()) | |
154 | |
155 @property | |
156 def skipped(self): | |
157 """Returns the list of compiler commands that were not processed during the | |
158 comparison. | |
159 """ | |
160 return self._skipped | |
161 | |
162 @property | |
163 def total_differences(self): | |
164 """Returns the total number of differences detected.""" | |
165 return self._total_diffs | |
166 | |
167 @property | |
168 def missing_in_gyp(self): | |
169 """Differences that are only in GYP build but not in GN, indexed by the | |
170 difference.""" | |
171 return self._missing_gyp_flags | |
172 | |
173 @property | |
174 def missing_in_gn(self): | |
175 """Differences that are only in the GN build but not in GYP, indexed by | |
176 the difference.""" | |
177 return self._missing_gn_flags | |
178 | |
179 @property | |
180 def missing_in_gyp_by_file(self): | |
181 """Differences that are only in the GYP build but not in GN, indexed by | |
182 file. | |
183 """ | |
184 return self._missing_gyp_files | |
185 | |
186 @property | |
187 def missing_in_gn_by_file(self): | |
188 """Differences that are only in the GYP build but not in GN, indexed by | |
189 file. | |
190 """ | |
191 return self._missing_gn_files | |
192 | |
193 def _CompareFiles(self): | |
194 """Performs the actual target comparison.""" | |
195 if sys.platform == 'win32': | |
196 # On Windows flags are stored in .rsp files which are created by building. | |
197 print >> sys.stderr, 'Building in %s...' % self._gn_dir | |
198 Run('ninja -C %s -d keeprsp %s' % (self._gn_dir, self._gn_target)) | |
199 print >> sys.stderr, 'Building in %s...' % self._gyp_dir | |
200 Run('ninja -C %s -d keeprsp %s' % (self._gyp_dir, self._gn_target)) | |
201 | |
202 gn = Run('ninja -C %s -t commands %s' % (self._gn_dir, self._gn_target)) | |
203 gyp = Run('ninja -C %s -t commands %s' % (self._gyp_dir, self._gyp_target)) | |
204 | |
205 self._gn_flags = self._GetFlags(gn.splitlines(), | |
206 os.path.join(os.getcwd(), self._gn_dir)) | |
207 self._gyp_flags = self._GetFlags(gyp.splitlines(), | |
208 os.path.join(os.getcwd(), self._gyp_dir)) | |
209 | |
210 self._gn_flags = dict((FilterChromium(filename), value) | |
211 for filename, value in self._gn_flags.iteritems()) | |
212 self._gyp_flags = dict((FilterChromium(filename), value) | |
213 for filename, value in self._gyp_flags.iteritems()) | |
214 | |
215 all_files = sorted(self.gn_files & self.gyp_files) | |
216 for filename in all_files: | |
217 gyp_flags = self._gyp_flags[filename] | |
218 gn_flags = self._gn_flags[filename] | |
219 self._CompareLists(filename, gyp_flags, gn_flags, 'dash_f') | |
220 self._CompareLists(filename, gyp_flags, gn_flags, 'defines', | |
221 # These defines are not used by WebRTC | |
222 dont_care_gyp=[ | |
223 '-DENABLE_WEBVR', | |
224 '-DUSE_EXTERNAL_POPUP_MENU', | |
225 '-DUSE_LIBJPEG_TURBO=1', | |
226 '-DUSE_MINIKIN_HYPHENATION=1', | |
227 '-DV8_USE_EXTERNAL_STARTUP_DATA', | |
228 '-DCR_CLANG_REVISION=280106-1', | |
229 '-DUSE_LIBPCI=1' | |
230 ], | |
231 dont_care_gn=[ | |
232 '-DUSE_EXTERNAL_POPUP_MENU=1' | |
233 ]) | |
234 self._CompareLists(filename, gyp_flags, gn_flags, 'include_dirs') | |
235 self._CompareLists(filename, gyp_flags, gn_flags, 'warnings', | |
236 # More conservative warnings in GN we consider to be OK. | |
237 dont_care_gyp=[ | |
238 '/wd4091', # 'keyword' : ignored on left of 'type' when no variable | |
239 # is declared. | |
240 '/wd4456', # Declaration hides previous local declaration. | |
241 '/wd4457', # Declaration hides function parameter. | |
242 '/wd4458', # Declaration hides class member. | |
243 '/wd4459', # Declaration hides global declaration. | |
244 '/wd4702', # Unreachable code. | |
245 '/wd4800', # Forcing value to bool 'true' or 'false'. | |
246 '/wd4838', # Conversion from 'type' to 'type' requires a narrowing | |
247 # conversion. | |
248 ] if sys.platform == 'win32' else None, | |
249 dont_care_gn=[ | |
250 '-Wendif-labels', | |
251 '-Wextra', | |
252 '-Wsign-compare', | |
253 ] if not sys.platform == 'win32' else None) | |
254 self._CompareLists(filename, gyp_flags, gn_flags, 'other', | |
255 dont_care_gyp=['-g'], dont_care_gn=['-g2']) | |
256 | |
257 def _CompareLists(self, filename, gyp, gn, name, | |
258 dont_care_gyp=None, dont_care_gn=None): | |
259 """Return a report of any differences between gyp and gn lists, ignoring | |
260 anything in |dont_care_{gyp|gn}| respectively.""" | |
261 if gyp[name] == gn[name]: | |
262 return | |
263 if not dont_care_gyp: | |
264 dont_care_gyp = [] | |
265 if not dont_care_gn: | |
266 dont_care_gn = [] | |
267 gyp_set = set(gyp[name]) | |
268 gn_set = set(gn[name]) | |
269 missing_in_gyp = gyp_set - gn_set | |
270 missing_in_gn = gn_set - gyp_set | |
271 missing_in_gyp -= set(dont_care_gyp) | |
272 missing_in_gn -= set(dont_care_gn) | |
273 | |
274 for m in missing_in_gyp: | |
275 self._missing_gyp_flags.setdefault(name, {}) \ | |
276 .setdefault(m, []).append(filename) | |
277 self._total_diffs += 1 | |
278 self._missing_gyp_files.setdefault(filename, {}) \ | |
279 .setdefault(name, set()).update(missing_in_gyp) | |
280 | |
281 for m in missing_in_gn: | |
282 self._missing_gn_flags.setdefault(name, {}) \ | |
283 .setdefault(m, []).append(filename) | |
284 self._total_diffs += 1 | |
285 self._missing_gn_files.setdefault(filename, {}) \ | |
286 .setdefault(name, set()).update(missing_in_gn) | |
287 | |
288 def _GetFlags(self, lines, build_dir): | |
289 """Turn a list of command lines into a semi-structured dict.""" | |
290 is_win = sys.platform == 'win32' | |
291 flags_by_output = {} | |
292 for line in lines: | |
293 line = FilterChromium(line) | |
294 line = line.replace(os.getcwd(), '../../') | |
295 line = line.replace('//', '/') | |
296 command_line = shlex.split(line.strip(), posix=not is_win)[1:] | |
297 | |
298 output_name = _FindAndRemoveArgWithValue(command_line, '-o') | |
299 dep_name = _FindAndRemoveArgWithValue(command_line, '-MF') | |
300 | |
301 command_line = _MergeSpacedArgs(command_line, '-Xclang') | |
302 | |
303 cc_file = [x for x in command_line if x.endswith('.cc') or | |
304 x.endswith('.c') or | |
305 x.endswith('.cpp') or | |
306 x.endswith('.mm') or | |
307 x.endswith('.m')] | |
308 if len(cc_file) != 1: | |
309 self._skipped.append(command_line) | |
310 continue | |
311 assert len(cc_file) == 1 | |
312 | |
313 if is_win: | |
314 rsp_file = [x for x in command_line if x.endswith('.rsp')] | |
315 assert len(rsp_file) <= 1 | |
316 if rsp_file: | |
317 rsp_file = os.path.join(build_dir, rsp_file[0][1:]) | |
318 with open(rsp_file, "r") as open_rsp_file: | |
319 command_line = shlex.split(open_rsp_file, posix=False) | |
320 | |
321 defines = [x for x in command_line if x.startswith('-D')] | |
322 include_dirs = [x for x in command_line if x.startswith('-I')] | |
323 dash_f = [x for x in command_line if x.startswith('-f')] | |
324 warnings = \ | |
325 [x for x in command_line if x.startswith('/wd' if is_win else '-W')] | |
326 others = [x for x in command_line if x not in defines and \ | |
327 x not in include_dirs and \ | |
328 x not in dash_f and \ | |
329 x not in warnings and \ | |
330 x not in cc_file] | |
331 | |
332 for index, value in enumerate(include_dirs): | |
333 if value == '-Igen': | |
334 continue | |
335 path = value[2:] | |
336 if not os.path.isabs(path): | |
337 path = os.path.join(build_dir, path) | |
338 include_dirs[index] = '-I' + os.path.normpath(path) | |
339 | |
340 # GYP supports paths above the source root like <(DEPTH)/../foo while such | |
341 # paths are unsupported by gn. But gn allows to use system-absolute paths | |
342 # instead (paths that start with single '/'). Normalize all paths. | |
343 cc_file = [os.path.normpath(os.path.join(build_dir, cc_file[0]))] | |
344 | |
345 # Filter for libFindBadConstructs.so having a relative path in one and | |
346 # absolute path in the other. | |
347 others_filtered = [] | |
348 for x in others: | |
349 if x.startswith('-Xclang ') and \ | |
350 (x.endswith('libFindBadConstructs.so') or \ | |
351 x.endswith('libFindBadConstructs.dylib')): | |
352 others_filtered.append( | |
353 '-Xclang ' + | |
354 os.path.join(os.getcwd(), os.path.normpath( | |
355 os.path.join('out/gn_flags', x.split(' ', 1)[1])))) | |
356 elif x.startswith('-B'): | |
357 others_filtered.append( | |
358 '-B' + | |
359 os.path.join(os.getcwd(), os.path.normpath( | |
360 os.path.join('out/gn_flags', x[2:])))) | |
361 else: | |
362 others_filtered.append(x) | |
363 others = others_filtered | |
364 | |
365 flags_by_output[cc_file[0]] = { | |
366 'output': output_name, | |
367 'depname': dep_name, | |
368 'defines': sorted(defines), | |
369 'include_dirs': sorted(include_dirs), # TODO(scottmg): This is wrong. | |
370 'dash_f': sorted(dash_f), | |
371 'warnings': sorted(warnings), | |
372 'other': sorted(others), | |
373 } | |
374 return flags_by_output | |
375 | |
376 | |
377 def _FindAndRemoveArgWithValue(command_line, argname): | |
378 """Given a command line as a list, remove and return the value of an option | |
379 that takes a value as a separate entry. | |
380 | |
381 Modifies |command_line| in place. | |
382 """ | |
383 if argname not in command_line: | |
384 return '' | |
385 location = command_line.index(argname) | |
386 value = command_line[location + 1] | |
387 command_line[location:location + 2] = [] | |
388 return value | |
389 | |
390 | |
391 def _MergeSpacedArgs(command_line, argname): | |
392 """Combine all arguments |argname| with their values, separated by a space.""" | |
393 i = 0 | |
394 result = [] | |
395 while i < len(command_line): | |
396 arg = command_line[i] | |
397 if arg == argname: | |
398 result.append(arg + ' ' + command_line[i + 1]) | |
399 i += 1 | |
400 else: | |
401 result.append(arg) | |
402 i += 1 | |
403 return result | |
404 | |
405 | |
406 def Run(command_line): | |
407 """Run |command_line| as a subprocess and return stdout. Raises on error.""" | |
408 print >> sys.stderr, command_line | |
409 return subprocess.check_output(command_line, shell=True) | |
410 | |
411 | |
412 def main(): | |
413 if len(sys.argv) < 3: | |
414 print 'usage: %s gyp_dir gn_dir' % __file__ | |
415 print ' or: %s gyp_dir gn_dir target' % __file__ | |
416 print ' or: %s gyp_dir gn_dir gyp_target gn_target' % __file__ | |
417 return 1 | |
418 | |
419 gyp_dir = sys.argv[1] | |
420 gn_dir = sys.argv[2] | |
421 | |
422 gyp_target = gn_target = "" | |
423 | |
424 if len(sys.argv) > 3: | |
425 gyp_target = sys.argv[3] | |
426 if len(sys.argv) > 4: | |
427 gn_target = sys.argv[4] | |
428 | |
429 print 'GYP output directory is %s' % gyp_dir | |
430 print 'GN output directory is %s' % gn_dir | |
431 | |
432 comparison = Comparison(gyp_target, gn_target, gyp_dir, gn_dir) | |
433 | |
434 differing_files = set(comparison.missing_in_gn_by_file.keys()) & \ | |
435 set(comparison.missing_in_gyp_by_file.keys()) | |
436 files_with_given_differences = {} | |
437 for filename in differing_files: | |
438 output = '' | |
439 missing_in_gyp = comparison.missing_in_gyp_by_file.get(filename, {}) | |
440 missing_in_gn = comparison.missing_in_gn_by_file.get(filename, {}) | |
441 difference_types = sorted(set(missing_in_gyp.keys() + missing_in_gn.keys())) | |
442 for difference_type in difference_types: | |
443 if (len(missing_in_gyp[difference_type]) == 0 and | |
444 len(missing_in_gn[difference_type]) == 0): | |
445 continue | |
446 output += ' %s differ:\n' % difference_type | |
447 if (difference_type in missing_in_gyp and | |
448 len(missing_in_gyp[difference_type])): | |
449 output += ' In gyp, but not in GN:\n %s' % '\n '.join( | |
450 sorted(missing_in_gyp[difference_type])) + '\n' | |
451 if (difference_type in missing_in_gn and | |
452 len(missing_in_gn[difference_type])): | |
453 output += ' In GN, but not in gyp:\n %s' % '\n '.join( | |
454 sorted(missing_in_gn[difference_type])) + '\n' | |
455 if output: | |
456 files_with_given_differences.setdefault(output, []).append(filename) | |
457 | |
458 for diff, files in files_with_given_differences.iteritems(): | |
459 print '\n'.join(sorted(files)) | |
460 print diff | |
461 | |
462 print 'Total differences:', comparison.total_differences | |
463 # TODO(scottmg): Return failure on difference once we're closer to identical. | |
464 return 0 | |
465 | |
466 | |
467 if __name__ == '__main__': | |
468 sys.exit(main()) | |
OLD | NEW |