code-annotations 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_annotations/__init__.py +6 -0
- code_annotations/annotation_errors.py +64 -0
- code_annotations/base.py +647 -0
- code_annotations/cli.py +271 -0
- code_annotations/contrib/README.rst +8 -0
- code_annotations/contrib/config/__init__.py +14 -0
- code_annotations/contrib/config/feature_toggle_annotations.yaml +29 -0
- code_annotations/contrib/config/openedx_events_annotations.yaml +20 -0
- code_annotations/contrib/config/setting_annotations.yaml +13 -0
- code_annotations/contrib/sphinx/extensions/__init__.py +3 -0
- code_annotations/contrib/sphinx/extensions/base.py +53 -0
- code_annotations/contrib/sphinx/extensions/featuretoggles.py +137 -0
- code_annotations/contrib/sphinx/extensions/openedx_events.py +182 -0
- code_annotations/contrib/sphinx/extensions/settings.py +139 -0
- code_annotations/exceptions.py +9 -0
- code_annotations/extensions/__init__.py +3 -0
- code_annotations/extensions/base.py +166 -0
- code_annotations/extensions/javascript.py +20 -0
- code_annotations/extensions/python.py +20 -0
- code_annotations/find_django.py +460 -0
- code_annotations/find_static.py +96 -0
- code_annotations/generate_docs.py +165 -0
- code_annotations/helpers.py +182 -0
- code_annotations/report_templates/html/annotation.tpl +14 -0
- code_annotations/report_templates/html/annotation_data.tpl +8 -0
- code_annotations/report_templates/html/annotation_list.tpl +27 -0
- code_annotations/report_templates/html/base.tpl +92 -0
- code_annotations/report_templates/rst/annotation.tpl +13 -0
- code_annotations/report_templates/rst/annotation_data.tpl +8 -0
- code_annotations/report_templates/rst/annotation_group.tpl +2 -0
- code_annotations/report_templates/rst/annotation_list.tpl +34 -0
- code_annotations/report_templates/rst/base.tpl +41 -0
- code_annotations-3.1.0.dist-info/METADATA +82 -0
- code_annotations-3.1.0.dist-info/RECORD +39 -0
- code_annotations-3.1.0.dist-info/WHEEL +5 -0
- code_annotations-3.1.0.dist-info/entry_points.txt +6 -0
- code_annotations-3.1.0.dist-info/licenses/LICENSE.txt +202 -0
- code_annotations-3.1.0.dist-info/licenses/NOTICE.txt +13 -0
- code_annotations-3.1.0.dist-info/top_level.txt +1 -0
code_annotations/base.py
ADDED
|
@@ -0,0 +1,647 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Click command to do static annotation searching via Stevedore plugins.
|
|
3
|
+
"""
|
|
4
|
+
import datetime
|
|
5
|
+
import errno
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
from abc import ABCMeta, abstractmethod
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
from stevedore import named
|
|
12
|
+
|
|
13
|
+
from code_annotations import annotation_errors
|
|
14
|
+
from code_annotations.exceptions import ConfigurationException
|
|
15
|
+
from code_annotations.helpers import VerboseEcho
|
|
16
|
+
|
|
17
|
+
PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
18
|
+
DEFAULT_TEMPLATE_DIR = os.path.join(PACKAGE_DIR, "report_templates")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AnnotationConfig:
|
|
22
|
+
"""
|
|
23
|
+
Configuration shared among all Code Annotations commands.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, config_file_path, report_path_override=None, verbosity=1, source_path_override=None):
|
|
27
|
+
"""
|
|
28
|
+
Initialize AnnotationConfig.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
config_file_path: Path to the configuration file
|
|
32
|
+
report_path_override: Path to write reports to, if overridden on the command line
|
|
33
|
+
verbosity: Verbosity level from the command line
|
|
34
|
+
source_path_override: Path to search if we're static code searching, if overridden on the command line
|
|
35
|
+
"""
|
|
36
|
+
self.groups = {}
|
|
37
|
+
self.choices = {}
|
|
38
|
+
self.optional_groups = []
|
|
39
|
+
self.annotation_tokens = []
|
|
40
|
+
self.annotation_regexes = []
|
|
41
|
+
self.mgr = None
|
|
42
|
+
|
|
43
|
+
# Global logger, other objects can hold handles to this
|
|
44
|
+
self.echo = VerboseEcho()
|
|
45
|
+
|
|
46
|
+
with open(config_file_path) as config_file:
|
|
47
|
+
raw_config = yaml.safe_load(config_file)
|
|
48
|
+
|
|
49
|
+
self._check_raw_config_keys(raw_config)
|
|
50
|
+
|
|
51
|
+
self.safelist_path = raw_config['safelist_path']
|
|
52
|
+
self.extensions = raw_config['extensions']
|
|
53
|
+
|
|
54
|
+
self.verbosity = verbosity
|
|
55
|
+
self.echo.set_verbosity(verbosity)
|
|
56
|
+
|
|
57
|
+
self.report_path = report_path_override if report_path_override else raw_config['report_path']
|
|
58
|
+
self.echo(f"Configured for report path: {self.report_path}")
|
|
59
|
+
|
|
60
|
+
self.source_path = source_path_override if source_path_override else raw_config['source_path']
|
|
61
|
+
self.echo(f"Configured for source path: {self.source_path}")
|
|
62
|
+
|
|
63
|
+
self._configure_coverage(raw_config.get('coverage_target', None))
|
|
64
|
+
|
|
65
|
+
self.rendered_report_format = raw_config.get('rendered_report_format', 'rst')
|
|
66
|
+
self.report_template_dir = raw_config.get(
|
|
67
|
+
'report_template_dir',
|
|
68
|
+
os.path.join(DEFAULT_TEMPLATE_DIR, self.rendered_report_format)
|
|
69
|
+
)
|
|
70
|
+
self.rendered_report_dir = raw_config.get('rendered_report_dir', 'annotation_reports')
|
|
71
|
+
self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix', None)
|
|
72
|
+
self.trim_filename_prefixes = raw_config.get('trim_filename_prefixes', [])
|
|
73
|
+
self.third_party_package_location = raw_config.get('third_party_package_location', "site-packages")
|
|
74
|
+
self.rendered_report_file_extension = f".{self.rendered_report_format}"
|
|
75
|
+
|
|
76
|
+
self._configure_annotations(raw_config)
|
|
77
|
+
self._configure_extensions()
|
|
78
|
+
|
|
79
|
+
def _check_raw_config_keys(self, raw_config):
|
|
80
|
+
"""
|
|
81
|
+
Validate that all required keys exist in the configuration file.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
raw_config: Python representation of the YAML config file
|
|
85
|
+
|
|
86
|
+
Raises:
|
|
87
|
+
ConfigurationException on any missing keys
|
|
88
|
+
"""
|
|
89
|
+
errors = []
|
|
90
|
+
for k in ('report_path', 'source_path', 'safelist_path', 'annotations', 'extensions'):
|
|
91
|
+
if k not in raw_config:
|
|
92
|
+
errors.append(k)
|
|
93
|
+
|
|
94
|
+
if errors:
|
|
95
|
+
raise ConfigurationException(
|
|
96
|
+
'The following required keys are missing from the configuration file: \n{}'.format(
|
|
97
|
+
'\n'.join(errors)
|
|
98
|
+
)
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
def _is_annotation_group(self, token_or_group):
|
|
102
|
+
"""
|
|
103
|
+
Determine if an annotation is a group or not.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
token_or_group: The annotation being checked
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
True if the type of the annotation is correct for a group, otherwise False
|
|
110
|
+
"""
|
|
111
|
+
return isinstance(token_or_group, list)
|
|
112
|
+
|
|
113
|
+
def _is_choice_group(self, token_or_group):
|
|
114
|
+
"""
|
|
115
|
+
Determine if an annotation is a choice group.
|
|
116
|
+
|
|
117
|
+
Args:
|
|
118
|
+
token_or_group: The annotation being checked
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
True if the type of the annotation is correct for a choice group, otherwise False
|
|
122
|
+
"""
|
|
123
|
+
return isinstance(token_or_group, dict) and "choices" in token_or_group
|
|
124
|
+
|
|
125
|
+
def _is_optional_group(self, token_or_group):
|
|
126
|
+
"""
|
|
127
|
+
Determine if an annotation is an optional group.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
token_or_group: The annotation being checked
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
True if the annotation is optional, otherwise False.
|
|
134
|
+
"""
|
|
135
|
+
return isinstance(token_or_group, dict) and bool(token_or_group.get("optional"))
|
|
136
|
+
|
|
137
|
+
def _is_annotation_token(self, token_or_group):
|
|
138
|
+
"""
|
|
139
|
+
Determine if an annotation has the right format.
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
token_or_group: The annotation being checked
|
|
143
|
+
|
|
144
|
+
Returns:
|
|
145
|
+
True if the type of the annotation is correct for a text type, otherwise False
|
|
146
|
+
"""
|
|
147
|
+
if token_or_group is None:
|
|
148
|
+
return True
|
|
149
|
+
if isinstance(token_or_group, dict):
|
|
150
|
+
# If annotation is a dict, only a few keys are tolerated
|
|
151
|
+
return set(token_or_group.keys()).issubset({"choices", "optional"})
|
|
152
|
+
return False
|
|
153
|
+
|
|
154
|
+
def _add_annotation_token(self, token):
|
|
155
|
+
if token in self.annotation_tokens:
|
|
156
|
+
raise ConfigurationException(f'{token} is configured more than once, tokens must be unique.')
|
|
157
|
+
self.annotation_tokens.append(token)
|
|
158
|
+
|
|
159
|
+
def _configure_coverage(self, coverage_target):
|
|
160
|
+
"""
|
|
161
|
+
Set coverage_target to the specified value.
|
|
162
|
+
|
|
163
|
+
Args:
|
|
164
|
+
coverage_target:
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
|
|
168
|
+
"""
|
|
169
|
+
if coverage_target:
|
|
170
|
+
try:
|
|
171
|
+
self.coverage_target = float(coverage_target)
|
|
172
|
+
except (TypeError, ValueError) as error:
|
|
173
|
+
raise ConfigurationException(
|
|
174
|
+
f'Coverage target must be a number between 0 and 100 not "{coverage_target}".'
|
|
175
|
+
) from error
|
|
176
|
+
|
|
177
|
+
if self.coverage_target < 0.0 or self.coverage_target > 100.0:
|
|
178
|
+
raise ConfigurationException(
|
|
179
|
+
f'Invalid coverage target. {self.coverage_target} is not between 0 and 100.'
|
|
180
|
+
)
|
|
181
|
+
else:
|
|
182
|
+
self.coverage_target = None
|
|
183
|
+
|
|
184
|
+
def _configure_group(self, group_name, group):
|
|
185
|
+
"""
|
|
186
|
+
Perform group configuration and add annotations from the group to global configuration.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
group_name: The name of the group (the key in the configuration dictionary)
|
|
190
|
+
group: The list of annotations that comprise the group
|
|
191
|
+
|
|
192
|
+
Raises:
|
|
193
|
+
TypeError if the group is misconfigured
|
|
194
|
+
"""
|
|
195
|
+
self.groups[group_name] = []
|
|
196
|
+
|
|
197
|
+
if not group or len(group) == 1:
|
|
198
|
+
raise ConfigurationException(f'Group "{group_name}" must have more than one annotation.')
|
|
199
|
+
|
|
200
|
+
for annotation in group:
|
|
201
|
+
for annotation_token in annotation:
|
|
202
|
+
annotation_value = annotation[annotation_token]
|
|
203
|
+
|
|
204
|
+
# Otherwise it should be a text type, if not then error out
|
|
205
|
+
if not self._is_annotation_token(annotation_value):
|
|
206
|
+
raise ConfigurationException(f'{annotation} is an unknown annotation type.')
|
|
207
|
+
# The annotation comment is a choice group
|
|
208
|
+
if self._is_choice_group(annotation_value):
|
|
209
|
+
self._configure_choices(annotation_token, annotation_value)
|
|
210
|
+
# The annotation comment is not mandatory
|
|
211
|
+
if self._is_optional_group(annotation_value):
|
|
212
|
+
self.optional_groups.append(annotation_token)
|
|
213
|
+
|
|
214
|
+
self.groups[group_name].append(annotation_token)
|
|
215
|
+
self._add_annotation_token(annotation_token)
|
|
216
|
+
self.annotation_regexes.append(re.escape(annotation_token))
|
|
217
|
+
|
|
218
|
+
def _configure_choices(self, annotation_token, annotation):
|
|
219
|
+
"""
|
|
220
|
+
Configure the choices list for an annotation.
|
|
221
|
+
|
|
222
|
+
Args:
|
|
223
|
+
annotation_token: The annotation token we are setting choices for
|
|
224
|
+
annotation: The annotation body (list of choices)
|
|
225
|
+
"""
|
|
226
|
+
self.choices[annotation_token] = annotation['choices']
|
|
227
|
+
|
|
228
|
+
def _configure_annotations(self, raw_config):
|
|
229
|
+
"""
|
|
230
|
+
Transform the configured annotations into more usable pieces and validate.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
raw_config: The dictionary form of our configuration file
|
|
234
|
+
Raises:
|
|
235
|
+
TypeError if annotations are misconfigured
|
|
236
|
+
"""
|
|
237
|
+
annotation_tokens = raw_config['annotations']
|
|
238
|
+
|
|
239
|
+
for annotation_token_or_group_name in annotation_tokens:
|
|
240
|
+
annotation = annotation_tokens[annotation_token_or_group_name]
|
|
241
|
+
|
|
242
|
+
if self._is_annotation_group(annotation):
|
|
243
|
+
self._configure_group(annotation_token_or_group_name, annotation)
|
|
244
|
+
|
|
245
|
+
elif self._is_choice_group(annotation):
|
|
246
|
+
self._configure_choices(annotation_token_or_group_name, annotation)
|
|
247
|
+
self._add_annotation_token(annotation_token_or_group_name)
|
|
248
|
+
self.annotation_regexes.append(re.escape(annotation_token_or_group_name))
|
|
249
|
+
|
|
250
|
+
elif not self._is_annotation_token(annotation): # pragma: no cover
|
|
251
|
+
raise TypeError(
|
|
252
|
+
f'{annotation_token_or_group_name} is an unknown type, must be strings or lists.'
|
|
253
|
+
)
|
|
254
|
+
else:
|
|
255
|
+
self._add_annotation_token(annotation_token_or_group_name)
|
|
256
|
+
self.annotation_regexes.append(re.escape(annotation_token_or_group_name))
|
|
257
|
+
|
|
258
|
+
self.echo.echo_v(f"Groups configured: {self.groups}")
|
|
259
|
+
self.echo.echo_v(f"Choices configured: {self.choices}")
|
|
260
|
+
self.echo.echo_v(f"Annotation tokens configured: {self.annotation_tokens}")
|
|
261
|
+
|
|
262
|
+
def _plugin_load_failed_handler(self, *args, **kwargs):
|
|
263
|
+
"""
|
|
264
|
+
Handle failures to load an extension.
|
|
265
|
+
|
|
266
|
+
Dumps the error and raises an exception. By default these
|
|
267
|
+
errors just fail silently.
|
|
268
|
+
|
|
269
|
+
Args:
|
|
270
|
+
*args:
|
|
271
|
+
**kwargs:
|
|
272
|
+
|
|
273
|
+
Raises:
|
|
274
|
+
ConfigurationException
|
|
275
|
+
"""
|
|
276
|
+
self.echo(str(args), fg='red')
|
|
277
|
+
self.echo(str(kwargs), fg='red')
|
|
278
|
+
raise ConfigurationException('Failed to load a plugin, aborting.')
|
|
279
|
+
|
|
280
|
+
def _configure_extensions(self):
|
|
281
|
+
"""
|
|
282
|
+
Configure the Stevedore NamedExtensionManager.
|
|
283
|
+
|
|
284
|
+
Raises:
|
|
285
|
+
ConfigurationException
|
|
286
|
+
"""
|
|
287
|
+
# These are the names of all of our configured extensions
|
|
288
|
+
configured_extension_names = self.extensions.keys()
|
|
289
|
+
|
|
290
|
+
# Load Stevedore extensions that we are configured for (and only those)
|
|
291
|
+
self.mgr = named.NamedExtensionManager(
|
|
292
|
+
names=configured_extension_names,
|
|
293
|
+
namespace='annotation_finder.searchers',
|
|
294
|
+
invoke_on_load=True,
|
|
295
|
+
on_load_failure_callback=self._plugin_load_failed_handler,
|
|
296
|
+
invoke_args=(self, self.echo),
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
# Output extension names listed in configuration
|
|
300
|
+
self.echo.echo_vv("Configured extension names: {}".format(" ".join(configured_extension_names)))
|
|
301
|
+
|
|
302
|
+
# Output found extension entry points from setup.py|cfg (whether or not they were loaded)
|
|
303
|
+
self.echo.echo_vv("Stevedore entry points found: {}".format(str(self.mgr.list_entry_points())))
|
|
304
|
+
|
|
305
|
+
# Output extensions that were actually able to load
|
|
306
|
+
self.echo.echo_v("Loaded extensions: {}".format(" ".join([x.name for x in self.mgr.extensions])))
|
|
307
|
+
|
|
308
|
+
if len(self.mgr.extensions) != len(configured_extension_names):
|
|
309
|
+
raise ConfigurationException('Not all configured extensions could be loaded! Asked for {} got {}.'.format(
|
|
310
|
+
configured_extension_names, self.mgr.extensions
|
|
311
|
+
))
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
class BaseSearch(metaclass=ABCMeta):
|
|
315
|
+
"""
|
|
316
|
+
Base class for searchers.
|
|
317
|
+
"""
|
|
318
|
+
|
|
319
|
+
def __init__(self, config):
|
|
320
|
+
"""
|
|
321
|
+
Initialize for StaticSearch.
|
|
322
|
+
|
|
323
|
+
Args:
|
|
324
|
+
config: Configuration object
|
|
325
|
+
"""
|
|
326
|
+
self.config = config
|
|
327
|
+
self.echo = self.config.echo
|
|
328
|
+
# errors contains formatted error messages
|
|
329
|
+
self.errors = []
|
|
330
|
+
# annotation_errors contains (annotation, AnnotationErrorType, args) tuples
|
|
331
|
+
# This attribute may be parsed by 3rd-parties, such as edx-lint.
|
|
332
|
+
self.annotation_errors = []
|
|
333
|
+
|
|
334
|
+
def format_file_results(self, all_results, results):
|
|
335
|
+
"""
|
|
336
|
+
Add all extensions' search results for a file to the overall results.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
all_results: Aggregated results to add the results to
|
|
340
|
+
results: Results of search() on a single file
|
|
341
|
+
|
|
342
|
+
Returns:
|
|
343
|
+
None, modifies all_results
|
|
344
|
+
"""
|
|
345
|
+
for annotations in results:
|
|
346
|
+
if not annotations:
|
|
347
|
+
continue
|
|
348
|
+
|
|
349
|
+
# TODO: The file_path should be the same for all of these results
|
|
350
|
+
# so we should be able to optimize getting file_path and making
|
|
351
|
+
# sure it exists in the dict to do this less often.
|
|
352
|
+
file_path = annotations[0]['filename']
|
|
353
|
+
|
|
354
|
+
if file_path not in all_results: # pragma: no cover
|
|
355
|
+
all_results[file_path] = []
|
|
356
|
+
|
|
357
|
+
for annotation in annotations:
|
|
358
|
+
# If this is a "choices" type of annotation, split the comment into a list.
|
|
359
|
+
# Actually checking the choice validity happens later in _check_results_choices.
|
|
360
|
+
if annotation['annotation_token'] in self.config.choices:
|
|
361
|
+
annotation['annotation_data'] = re.split(r',\s?|\s', annotation['annotation_data'])
|
|
362
|
+
|
|
363
|
+
# TODO: De-dupe results? Should only be necessary if more than one
|
|
364
|
+
# Stevedore extension is working on the same file type
|
|
365
|
+
all_results[file_path].extend(annotations)
|
|
366
|
+
|
|
367
|
+
def _check_results_choices(self, annotation):
|
|
368
|
+
"""
|
|
369
|
+
Check that a search result has appropriate choices.
|
|
370
|
+
|
|
371
|
+
If the following errors are found:
|
|
372
|
+
- no choices
|
|
373
|
+
- multiple of the same choice
|
|
374
|
+
- a choice which is not configured
|
|
375
|
+
|
|
376
|
+
This function will add the error to self.errors.
|
|
377
|
+
|
|
378
|
+
Args:
|
|
379
|
+
annotation: A single search result dict.
|
|
380
|
+
"""
|
|
381
|
+
# Not a choice type of annotation, nothing to do
|
|
382
|
+
if annotation['annotation_token'] not in self.config.choices:
|
|
383
|
+
return None
|
|
384
|
+
|
|
385
|
+
token = annotation['annotation_token']
|
|
386
|
+
found_valid_choices = []
|
|
387
|
+
|
|
388
|
+
# If the line begins with an annotation token that should have choices, but has no text after the token,
|
|
389
|
+
# the first split will be empty.
|
|
390
|
+
if annotation['annotation_data'][0] != "":
|
|
391
|
+
for choice in annotation['annotation_data']:
|
|
392
|
+
if choice not in self.config.choices[token]:
|
|
393
|
+
self._add_annotation_error(
|
|
394
|
+
annotation,
|
|
395
|
+
annotation_errors.InvalidChoice,
|
|
396
|
+
(choice, token, self.config.choices[token])
|
|
397
|
+
)
|
|
398
|
+
elif choice in found_valid_choices:
|
|
399
|
+
self._add_annotation_error(annotation, annotation_errors.DuplicateChoiceValue, (choice,))
|
|
400
|
+
else:
|
|
401
|
+
found_valid_choices.append(choice)
|
|
402
|
+
else:
|
|
403
|
+
self._add_annotation_error(
|
|
404
|
+
annotation,
|
|
405
|
+
annotation_errors.MissingChoiceValue,
|
|
406
|
+
(token, self.config.choices[token])
|
|
407
|
+
)
|
|
408
|
+
return None
|
|
409
|
+
|
|
410
|
+
def _get_group_children(self):
|
|
411
|
+
"""
|
|
412
|
+
Create a list of all annotation tokens that are part of a group.
|
|
413
|
+
|
|
414
|
+
Returns:
|
|
415
|
+
List of annotation tokens that are configured to be in groups
|
|
416
|
+
"""
|
|
417
|
+
group_children = []
|
|
418
|
+
|
|
419
|
+
for group in self.config.groups:
|
|
420
|
+
group_children.extend(self.config.groups[group])
|
|
421
|
+
|
|
422
|
+
return group_children
|
|
423
|
+
|
|
424
|
+
def _get_group_for_token(self, token):
|
|
425
|
+
"""
|
|
426
|
+
Find out which group, if any, an annotation token belongs to.
|
|
427
|
+
|
|
428
|
+
Args:
|
|
429
|
+
token: Annotation token to search for
|
|
430
|
+
|
|
431
|
+
Returns:
|
|
432
|
+
The group name, or None if it doesn't belong to a group.
|
|
433
|
+
"""
|
|
434
|
+
for group in self.config.groups:
|
|
435
|
+
if token in self.config.groups[group]:
|
|
436
|
+
return group
|
|
437
|
+
return None
|
|
438
|
+
|
|
439
|
+
def check_results(self, all_results):
|
|
440
|
+
"""
|
|
441
|
+
Spin through all search results, confirm that they all match configuration.
|
|
442
|
+
|
|
443
|
+
If errors are found they are added to self.errors.
|
|
444
|
+
|
|
445
|
+
Args:
|
|
446
|
+
all_results: Dict of annotations found in search()
|
|
447
|
+
|
|
448
|
+
Returns:
|
|
449
|
+
Boolean indicating whether or not any errors were found
|
|
450
|
+
"""
|
|
451
|
+
self.echo.pprint(all_results, indent=3, verbosity_level=2)
|
|
452
|
+
|
|
453
|
+
# Spin through the search results
|
|
454
|
+
for filename in all_results:
|
|
455
|
+
for annotations in self.iter_groups(all_results[filename]):
|
|
456
|
+
self.check_group(annotations)
|
|
457
|
+
return not self.errors
|
|
458
|
+
|
|
459
|
+
def iter_groups(self, annotations):
|
|
460
|
+
"""
|
|
461
|
+
Iterate on groups of annotations.
|
|
462
|
+
|
|
463
|
+
Annotations are considered as a group when they all have the same `line_number` and optional
|
|
464
|
+
`extra['object_id']`. The line number points to the beginning of the annotation group. The `object_id` is set
|
|
465
|
+
mostly for annotations parsed from a safelist.
|
|
466
|
+
|
|
467
|
+
Yield:
|
|
468
|
+
annotations (annotation list)
|
|
469
|
+
"""
|
|
470
|
+
current_group = []
|
|
471
|
+
current_line_number = None
|
|
472
|
+
current_object_id = None
|
|
473
|
+
for annotation in annotations:
|
|
474
|
+
line_number = annotation["line_number"]
|
|
475
|
+
object_id = annotation.get("extra", {}).get("object_id")
|
|
476
|
+
line_number_changed = line_number != current_line_number
|
|
477
|
+
object_id_changed = object_id != current_object_id
|
|
478
|
+
if line_number_changed or object_id_changed:
|
|
479
|
+
if current_group:
|
|
480
|
+
yield current_group
|
|
481
|
+
current_group.clear()
|
|
482
|
+
current_group.append(annotation)
|
|
483
|
+
current_line_number = line_number
|
|
484
|
+
current_object_id = object_id
|
|
485
|
+
|
|
486
|
+
if current_group:
|
|
487
|
+
yield current_group
|
|
488
|
+
|
|
489
|
+
def check_group(self, annotations):
|
|
490
|
+
"""
|
|
491
|
+
Perform several linting checks on a group of annotations.
|
|
492
|
+
|
|
493
|
+
The following checks are performed:
|
|
494
|
+
|
|
495
|
+
- Choice fields should have a valid value
|
|
496
|
+
- Annotation tokens are valid
|
|
497
|
+
- There is no duplicate
|
|
498
|
+
- All non-optional tokens are present
|
|
499
|
+
"""
|
|
500
|
+
found_tokens = set()
|
|
501
|
+
group_tokens = []
|
|
502
|
+
group_name = None
|
|
503
|
+
for annotation in annotations:
|
|
504
|
+
token = annotation["annotation_token"]
|
|
505
|
+
if not group_name:
|
|
506
|
+
group_name = self._get_group_for_token(token)
|
|
507
|
+
if group_name:
|
|
508
|
+
group_tokens = self.config.groups[group_name]
|
|
509
|
+
|
|
510
|
+
# Check if choice field
|
|
511
|
+
self._check_results_choices(annotation)
|
|
512
|
+
|
|
513
|
+
# Check token belongs to group
|
|
514
|
+
if group_name:
|
|
515
|
+
if token not in group_tokens:
|
|
516
|
+
self._add_annotation_error(
|
|
517
|
+
annotation,
|
|
518
|
+
annotation_errors.InvalidToken,
|
|
519
|
+
(
|
|
520
|
+
token,
|
|
521
|
+
group_name,
|
|
522
|
+
group_tokens
|
|
523
|
+
)
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
# Check for duplicates
|
|
527
|
+
if token in found_tokens:
|
|
528
|
+
self._add_annotation_error(
|
|
529
|
+
annotation,
|
|
530
|
+
annotation_errors.DuplicateToken,
|
|
531
|
+
(token,)
|
|
532
|
+
)
|
|
533
|
+
if group_name:
|
|
534
|
+
found_tokens.add(token)
|
|
535
|
+
|
|
536
|
+
# Check for missing tokens
|
|
537
|
+
for token in group_tokens:
|
|
538
|
+
if token not in self.config.optional_groups:
|
|
539
|
+
if token not in found_tokens:
|
|
540
|
+
self._add_annotation_error(
|
|
541
|
+
annotations[0],
|
|
542
|
+
annotation_errors.MissingToken,
|
|
543
|
+
(token,)
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
def _add_annotation_error(self, annotation, error_type, args=None):
|
|
547
|
+
"""
|
|
548
|
+
Add an error message to self.errors, formatted nicely.
|
|
549
|
+
|
|
550
|
+
Args:
|
|
551
|
+
annotation: A single annotation dict found in search()
|
|
552
|
+
error_type (annotation_errors.AnnotationErrorType): error type from which the error message will be
|
|
553
|
+
generated.
|
|
554
|
+
args (tuple): arguments for error message formatting.
|
|
555
|
+
"""
|
|
556
|
+
args = args or tuple()
|
|
557
|
+
error_message = error_type.message % args
|
|
558
|
+
location = annotation.get("extra", {}).get("object_id", annotation["line_number"])
|
|
559
|
+
message = "{}::{}: {}".format(annotation['filename'], location, error_message)
|
|
560
|
+
self.annotation_errors.append((annotation, error_type, args))
|
|
561
|
+
self._add_error(message)
|
|
562
|
+
|
|
563
|
+
def _add_error(self, message):
|
|
564
|
+
"""
|
|
565
|
+
Add an error message to self.errors.
|
|
566
|
+
|
|
567
|
+
Args:
|
|
568
|
+
message: Custom error message to be added
|
|
569
|
+
"""
|
|
570
|
+
self.errors.append(message)
|
|
571
|
+
|
|
572
|
+
@abstractmethod
|
|
573
|
+
def search(self):
|
|
574
|
+
"""
|
|
575
|
+
Walk the source tree, send known file types to extensions.
|
|
576
|
+
|
|
577
|
+
Returns:
|
|
578
|
+
Dict of {filename: annotations} for all files with found annotations.
|
|
579
|
+
"""
|
|
580
|
+
|
|
581
|
+
def _format_results_for_report(self, all_results):
|
|
582
|
+
"""
|
|
583
|
+
Format the given results dict for reporting purposes.
|
|
584
|
+
|
|
585
|
+
Args:
|
|
586
|
+
all_results: Dict of all results found in a search
|
|
587
|
+
|
|
588
|
+
Returns:
|
|
589
|
+
Dict of results arranged for reporting
|
|
590
|
+
"""
|
|
591
|
+
formatted_results = {}
|
|
592
|
+
current_group_id = 0
|
|
593
|
+
for filename in all_results:
|
|
594
|
+
self.echo.echo_vv(f"report_format: formatting {filename}")
|
|
595
|
+
formatted_results[filename] = []
|
|
596
|
+
for annotation_group in self.iter_groups(all_results[filename]):
|
|
597
|
+
current_group_id += 1
|
|
598
|
+
for annotation_index, annotation in enumerate(annotation_group):
|
|
599
|
+
token = annotation['annotation_token']
|
|
600
|
+
self.echo.echo_vvv(f"report_format: formatting annotation token {token}")
|
|
601
|
+
if annotation_index == 0:
|
|
602
|
+
self.echo.echo_vv('Starting group id {} for "{}" token "{}", line {}'.format(
|
|
603
|
+
current_group_id, annotation_group, token, annotation["line_number"])
|
|
604
|
+
)
|
|
605
|
+
self.echo.echo_vv(
|
|
606
|
+
"report_format: Adding {} to group id {}".format(
|
|
607
|
+
token,
|
|
608
|
+
current_group_id
|
|
609
|
+
)
|
|
610
|
+
)
|
|
611
|
+
annotation["report_group_id"] = current_group_id
|
|
612
|
+
formatted_results[filename].append(annotation)
|
|
613
|
+
|
|
614
|
+
return formatted_results
|
|
615
|
+
|
|
616
|
+
def report(self, all_results, report_prefix=''):
|
|
617
|
+
"""
|
|
618
|
+
Generate the YAML report of all search results.
|
|
619
|
+
|
|
620
|
+
Args:
|
|
621
|
+
all_results: Dict of found annotations, indexed by filename
|
|
622
|
+
report_prefix: Prefix to add to report filename
|
|
623
|
+
|
|
624
|
+
Returns:
|
|
625
|
+
Filename of generated report
|
|
626
|
+
"""
|
|
627
|
+
self.echo.echo_vv(yaml.dump(all_results, default_flow_style=False))
|
|
628
|
+
|
|
629
|
+
now = datetime.datetime.utcnow()
|
|
630
|
+
report_filename = os.path.join(self.config.report_path, '{}{}.yaml'.format(
|
|
631
|
+
report_prefix, now.strftime('%Y%m%d-%H%M%S')
|
|
632
|
+
))
|
|
633
|
+
|
|
634
|
+
formatted_results = self._format_results_for_report(all_results)
|
|
635
|
+
|
|
636
|
+
self.echo(f"Generating report to {report_filename}")
|
|
637
|
+
|
|
638
|
+
try:
|
|
639
|
+
os.makedirs(self.config.report_path)
|
|
640
|
+
except OSError as e: # pragma: no cover
|
|
641
|
+
if e.errno != errno.EEXIST:
|
|
642
|
+
raise
|
|
643
|
+
|
|
644
|
+
with open(report_filename, 'w+') as report_file:
|
|
645
|
+
yaml.safe_dump(formatted_results, report_file, default_flow_style=False)
|
|
646
|
+
|
|
647
|
+
return report_filename
|