OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2017 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2017 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """This script is used to extract network traffic annotations from Chrome. | 6 """This script is used to extract network traffic annotations from Chrome. |
7 Please refer to README.md for running steps.""" | 7 Please refer to README.md for running steps.""" |
8 | 8 |
9 import argparse | 9 import argparse |
10 import os | 10 import os |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
68 raw_annotations += stdout_text | 68 raw_annotations += stdout_text |
69 if stderr_text: | 69 if stderr_text: |
70 print stderr_text | 70 print stderr_text |
71 return raw_annotations | 71 return raw_annotations |
72 | 72 |
73 | 73 |
74 def _ParsRawAnnotations(raw_annotations): | 74 def _ParsRawAnnotations(raw_annotations): |
75 """Parses raw annotations texts which are received from the clang tool. | 75 """Parses raw annotations texts which are received from the clang tool. |
76 Args: | 76 Args: |
77 raw_annotations: str Serialization of annotations and metadata. Each | 77 raw_annotations: str Serialization of annotations and metadata. Each |
78 annotation should have the following lines: | 78 annotation should have either of the following lines: |
79 1- "==== NEW ANNOTATION ====" | 79 1- "==== NEW ANNOTATION ====" |
80 2- File path. | 80 2- File path. |
81 3- Name of the function including this position. | 81 3- Name of the function including this position. |
82 4- Line number. | 82 4- Line number. |
83 5- Function Type. | 83 5- Function Type. |
84 6- Unique id of annotation. | 84 6- Unique id of annotation. |
85 7- Completing id or group id, when applicable, empty otherwise. | 85 7- Completing id or group id, when applicable, empty otherwise. |
86 8- Serialization of annotation text (several lines) | 86 8- Serialization of annotation text (several lines) |
87 n- "==== ANNOTATION ENDS ====" | 87 n- "==== ANNOTATION ENDS ====" |
| 88 or: |
| 89 1: "==== NEW CALL ====" |
| 90 2: File path. |
| 91 3: Name of the function in which the call is made. |
| 92 4: Name of the called function. |
| 93 5: Does the call have an annotation? |
| 94 6: "==== CALL ENDS ====" |
88 | 95 |
89 Returns: | 96 Returns: |
90 annotations: ExtractedNetworkTrafficAnnotation A protobuf including all | 97 annotations: ExtractedNetworkTrafficAnnotation A protobuf including all |
91 extracted annotations. | 98 extracted annotations. |
92 metadata: list of dict List of metadata for each annotation. Each item | 99 metadata: list of dict List of metadata for each annotation. Each item |
93 includes the following fields: | 100 includes the following fields: |
94 function_type: str Type of the function that defines the annotation. | 101 function_type: str Type of the function that defines the annotation. |
95 extra_id: str Possible prefix for annotation completion. | 102 extra_id: str Possible prefix for annotation completion. |
96 errors: list of str List of errors. | 103 errors: list of str List of errors. |
97 """ | 104 """ |
98 annotations = traffic_annotation_pb2.ExtractedNetworkTrafficAnnotation() | 105 annotations = traffic_annotation_pb2.ExtractedNetworkTrafficAnnotation() |
99 errors = [] | 106 errors = [] |
100 metadata = [] | 107 metadata = [] |
101 | 108 |
102 lines = [line.strip("\r\n") for line in raw_annotations.split("\n")] | 109 lines = [line.strip("\r\n") for line in raw_annotations.split("\n")] |
103 current = 0 | 110 current = 0 |
104 | 111 |
105 try: | 112 try: |
106 while current < len(lines) - 1: | 113 while current < len(lines) - 1: |
107 if lines[current] != "==== NEW ANNOTATION ====": | 114 if lines[current] == "==== NEW ANNOTATION ====": |
| 115 if current + 6 >= len(lines): |
| 116 raise Exception( |
| 117 "Not enough header lines at line %i." % current) |
| 118 |
| 119 # Extract header lines. |
| 120 source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource() |
| 121 source.file = lines[current + 1] |
| 122 source.function = lines[current + 2] |
| 123 source.line = int(lines[current + 3]) |
| 124 unique_id = lines[current + 5] |
| 125 |
| 126 new_metadata = {'function_type': lines[current + 4], |
| 127 'extra_id': lines[current + 6], |
| 128 'unique_id_hash': _ComputeStringHash(unique_id)} |
| 129 # Extract serialized proto. |
| 130 current += 7 |
| 131 annotation_text = "" |
| 132 |
| 133 while current < len(lines): |
| 134 if lines[current] == "==== ANNOTATION ENDS ====": |
| 135 break |
| 136 else: |
| 137 annotation_text += lines[current] |
| 138 current += 1 |
| 139 else: |
| 140 raise Exception( |
| 141 "Error at line %i, expected annotation end tag." % current) |
| 142 current += 1 |
| 143 |
| 144 # Process unittests and undefined tags. |
| 145 if unique_id in ("test", "test_partial"): |
| 146 continue |
| 147 if unique_id in ("undefined", "missing"): |
| 148 errors.append("Annotation is not defined for file '%s', line %i." % |
| 149 (source.file, source.line)) |
| 150 continue |
| 151 |
| 152 # Decode serialized proto. |
| 153 annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation() |
| 154 try: |
| 155 text_format.Parse(annotation_text, annotation_proto) |
| 156 except Exception as error: |
| 157 errors.append("Annotation in file '%s', line %i, has an error: %s" % |
| 158 (source.file, source.line, error)) |
| 159 |
| 160 # Add new proto. |
| 161 annotation_proto.unique_id = unique_id |
| 162 annotation_proto.source.CopyFrom(source) |
| 163 annotations.network_traffic_annotation.add().CopyFrom(annotation_proto) |
| 164 metadata.append(new_metadata) |
| 165 elif lines[current] == "==== NEW CALL ====": |
| 166 # Ignore calls for now. |
| 167 while current < len(lines): |
| 168 if lines[current] == "==== CALL ENDS ====": |
| 169 break |
| 170 current += 1 |
| 171 else: |
| 172 raise Exception( |
| 173 "Error at line %i, expected call end tag." % current) |
| 174 current += 1 |
| 175 else: # The line is neither new annotation nor new call. |
108 raise Exception( | 176 raise Exception( |
109 "Error at line %i, expected starting new annotaion." % current) | 177 "Error at line %i, expected starting new annotation or call." % |
110 if current + 5 >= len(lines): | 178 current) |
111 raise Exception( | |
112 "Not enough header lines at line %i." % current) | |
113 | |
114 # Extract header lines. | |
115 source = traffic_annotation_pb2.NetworkTrafficAnnotation.TrafficSource() | |
116 source.file = lines[current + 1] | |
117 source.function = lines[current + 2] | |
118 source.line = int(lines[current + 3]) | |
119 unique_id = lines[current + 5] | |
120 | |
121 new_metadata = {'function_type': lines[current + 4], | |
122 'extra_id': lines[current + 6], | |
123 'unique_id_hash': _ComputeStringHash(unique_id)} | |
124 # Extract serialized proto. | |
125 current += 7 | |
126 annotation_text = "" | |
127 | |
128 while current < len(lines): | |
129 current += 1 | |
130 if lines[current - 1] == "==== ANNOTATION ENDS ====": | |
131 break | |
132 else: | |
133 annotation_text += lines[current - 1] | |
134 else: | |
135 raise Exception( | |
136 "Error at line %i, expected annotation end tag." % current) | |
137 | |
138 # Process unittests and undefined tags. | |
139 if unique_id in ("test", "test_partial"): | |
140 continue | |
141 if unique_id in ("undefined", "missing"): | |
142 errors.append("Annotation is not defined for file '%s', line %i." % | |
143 (source.file, source.line)) | |
144 continue | |
145 | |
146 # Decode serialized proto. | |
147 annotation_proto = traffic_annotation_pb2.NetworkTrafficAnnotation() | |
148 try: | |
149 text_format.Parse(annotation_text, annotation_proto) | |
150 except Exception as error: | |
151 errors.append("Annotation in file '%s', line %i, has error: %s" % | |
152 (source.file, source.line, error)) | |
153 | |
154 # Add new proto. | |
155 annotation_proto.unique_id = unique_id | |
156 annotation_proto.source.CopyFrom(source) | |
157 annotations.network_traffic_annotation.add().CopyFrom(annotation_proto) | |
158 metadata.append(new_metadata) | |
159 | 179 |
160 except Exception as error: | 180 except Exception as error: |
161 errors.append(str(error)) | 181 errors.append(str(error)) |
162 | 182 |
163 print "Extracted %i annotations with %i errors." % \ | 183 print "Extracted %i annotations with %i errors." % \ |
164 (len(annotations.network_traffic_annotation), len(errors)) | 184 (len(annotations.network_traffic_annotation), len(errors)) |
165 return annotations, metadata, errors | 185 return annotations, metadata, errors |
166 | 186 |
167 | 187 |
168 def _WriteSummaryFile(annotations, metadata, errors, file_path): | 188 def _WriteSummaryFile(annotations, metadata, errors, file_path): |
(...skipping 96 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
265 _WriteSummaryFile(annotations, metadata, errors, args.summary_file) | 285 _WriteSummaryFile(annotations, metadata, errors, args.summary_file) |
266 | 286 |
267 if args.hash_codes_file: | 287 if args.hash_codes_file: |
268 _WriteHashCodesFile(annotations, metadata, args.hash_codes_file) | 288 _WriteHashCodesFile(annotations, metadata, args.hash_codes_file) |
269 | 289 |
270 return 0 | 290 return 0 |
271 | 291 |
272 | 292 |
273 if __name__ == '__main__': | 293 if __name__ == '__main__': |
274 sys.exit(main()) | 294 sys.exit(main()) |
OLD | NEW |