code-annotations 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_annotations/__init__.py +6 -0
- code_annotations/annotation_errors.py +64 -0
- code_annotations/base.py +647 -0
- code_annotations/cli.py +271 -0
- code_annotations/contrib/README.rst +8 -0
- code_annotations/contrib/config/__init__.py +14 -0
- code_annotations/contrib/config/feature_toggle_annotations.yaml +29 -0
- code_annotations/contrib/config/openedx_events_annotations.yaml +20 -0
- code_annotations/contrib/config/setting_annotations.yaml +13 -0
- code_annotations/contrib/sphinx/extensions/__init__.py +3 -0
- code_annotations/contrib/sphinx/extensions/base.py +53 -0
- code_annotations/contrib/sphinx/extensions/featuretoggles.py +137 -0
- code_annotations/contrib/sphinx/extensions/openedx_events.py +182 -0
- code_annotations/contrib/sphinx/extensions/settings.py +139 -0
- code_annotations/exceptions.py +9 -0
- code_annotations/extensions/__init__.py +3 -0
- code_annotations/extensions/base.py +166 -0
- code_annotations/extensions/javascript.py +20 -0
- code_annotations/extensions/python.py +20 -0
- code_annotations/find_django.py +460 -0
- code_annotations/find_static.py +96 -0
- code_annotations/generate_docs.py +165 -0
- code_annotations/helpers.py +182 -0
- code_annotations/report_templates/html/annotation.tpl +14 -0
- code_annotations/report_templates/html/annotation_data.tpl +8 -0
- code_annotations/report_templates/html/annotation_list.tpl +27 -0
- code_annotations/report_templates/html/base.tpl +92 -0
- code_annotations/report_templates/rst/annotation.tpl +13 -0
- code_annotations/report_templates/rst/annotation_data.tpl +8 -0
- code_annotations/report_templates/rst/annotation_group.tpl +2 -0
- code_annotations/report_templates/rst/annotation_list.tpl +34 -0
- code_annotations/report_templates/rst/base.tpl +41 -0
- code_annotations-3.1.0.dist-info/METADATA +82 -0
- code_annotations-3.1.0.dist-info/RECORD +39 -0
- code_annotations-3.1.0.dist-info/WHEEL +5 -0
- code_annotations-3.1.0.dist-info/entry_points.txt +6 -0
- code_annotations-3.1.0.dist-info/licenses/LICENSE.txt +202 -0
- code_annotations-3.1.0.dist-info/licenses/NOTICE.txt +13 -0
- code_annotations-3.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Annotation searcher for Django model comment searching Django introspection.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
import django
|
|
10
|
+
import yaml
|
|
11
|
+
from django.apps import apps
|
|
12
|
+
from django.db import models
|
|
13
|
+
|
|
14
|
+
from code_annotations.base import BaseSearch
|
|
15
|
+
from code_annotations.helpers import clean_annotation, fail, get_annotation_regex
|
|
16
|
+
|
|
17
|
+
DEFAULT_SAFELIST_FILE_PATH = ".annotation_safe_list.yml"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DjangoSearch(BaseSearch):
|
|
21
|
+
"""
|
|
22
|
+
Handles Django model comment searching for annotations.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, config, app_name=None):
|
|
26
|
+
"""
|
|
27
|
+
Initialize for DjangoSearch.
|
|
28
|
+
"""
|
|
29
|
+
super().__init__(config)
|
|
30
|
+
self.local_models, self.non_local_models, total, annotation_eligible = (
|
|
31
|
+
self.get_models_requiring_annotations(app_name)
|
|
32
|
+
)
|
|
33
|
+
self.model_counts = {
|
|
34
|
+
"total": total,
|
|
35
|
+
"annotated": 0,
|
|
36
|
+
"unannotated": 0,
|
|
37
|
+
"annotation_eligible": len(annotation_eligible),
|
|
38
|
+
"not_annotation_eligible": total - len(annotation_eligible),
|
|
39
|
+
"safelisted": 0,
|
|
40
|
+
}
|
|
41
|
+
self.uncovered_model_ids = set()
|
|
42
|
+
self.echo.echo_vvv(
|
|
43
|
+
"Local models:\n "
|
|
44
|
+
+ "\n ".join([str(m) for m in self.local_models])
|
|
45
|
+
+ "\n"
|
|
46
|
+
)
|
|
47
|
+
self.echo.echo_vvv(
|
|
48
|
+
"Non-local models:\n "
|
|
49
|
+
+ "\n ".join([str(m) for m in self.non_local_models])
|
|
50
|
+
+ "\n"
|
|
51
|
+
)
|
|
52
|
+
self.echo.echo_vv(
|
|
53
|
+
"The following models require annotations:\n "
|
|
54
|
+
+ "\n ".join(annotation_eligible)
|
|
55
|
+
+ "\n"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
def _increment_count(self, count_type, incr_by=1):
|
|
59
|
+
self.model_counts[count_type] += incr_by
|
|
60
|
+
|
|
61
|
+
def seed_safelist(self):
|
|
62
|
+
"""
|
|
63
|
+
Seed a new safelist file with all non-local models that need to be vetted.
|
|
64
|
+
"""
|
|
65
|
+
if os.path.exists(self.config.safelist_path):
|
|
66
|
+
fail(f"{self.config.safelist_path} already exists, not overwriting.")
|
|
67
|
+
|
|
68
|
+
self.echo(
|
|
69
|
+
"Found {} non-local models requiring annotations. Adding them to safelist.".format(
|
|
70
|
+
len(self.non_local_models)
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
safelist_data = {
|
|
75
|
+
self.get_model_id(model): {} for model in self.non_local_models
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
with open(self.config.safelist_path, "w") as safelist_file:
|
|
79
|
+
safelist_comment = """
|
|
80
|
+
# This is a Code Annotations automatically-generated Django model safelist file.
|
|
81
|
+
# These models must be annotated as follows in order to be counted in the coverage report.
|
|
82
|
+
# See https://code-annotations.readthedocs.io/en/latest/safelist.html for more information.
|
|
83
|
+
#
|
|
84
|
+
# fake_app_1.FakeModelName:
|
|
85
|
+
# ".. no_pii:": "This model has no PII"
|
|
86
|
+
# fake_app_2.FakeModel2:
|
|
87
|
+
# ".. choice_annotation:": foo, bar, baz
|
|
88
|
+
|
|
89
|
+
"""
|
|
90
|
+
safelist_file.write(safelist_comment.lstrip())
|
|
91
|
+
yaml.safe_dump(
|
|
92
|
+
safelist_data, stream=safelist_file, default_flow_style=False
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
self.echo(
|
|
96
|
+
f'Successfully created safelist file "{self.config.safelist_path}".',
|
|
97
|
+
fg="red",
|
|
98
|
+
)
|
|
99
|
+
self.echo("Now, you need to:", fg="red")
|
|
100
|
+
self.echo(
|
|
101
|
+
" 1) Make sure that any un-annotated models in the safelist are annotated, and",
|
|
102
|
+
fg="red",
|
|
103
|
+
)
|
|
104
|
+
self.echo(" 2) Annotate any LOCAL models (see --list_local_models).", fg="red")
|
|
105
|
+
|
|
106
|
+
def list_local_models(self):
|
|
107
|
+
"""
|
|
108
|
+
Dump a list of models in the local code tree that are annotation eligible to stdout.
|
|
109
|
+
"""
|
|
110
|
+
if self.local_models:
|
|
111
|
+
self.echo(
|
|
112
|
+
"Listing {} local models requiring annotations:".format(
|
|
113
|
+
len(self.local_models)
|
|
114
|
+
)
|
|
115
|
+
)
|
|
116
|
+
self.echo.pprint(
|
|
117
|
+
sorted([self.get_model_id(model) for model in self.local_models]),
|
|
118
|
+
indent=4,
|
|
119
|
+
)
|
|
120
|
+
else:
|
|
121
|
+
self.echo("No local models requiring annotations.")
|
|
122
|
+
|
|
123
|
+
def _append_model_annotations(self, model_type, model_id, query, model_annotations):
|
|
124
|
+
"""
|
|
125
|
+
Find the given model's annotations in the file and add them to model_annotations.
|
|
126
|
+
|
|
127
|
+
Args:
|
|
128
|
+
model_type: The type of model we're searching
|
|
129
|
+
model_id: The text representation of the model name from get_model_id
|
|
130
|
+
query: The regex to run to find annotations in the docstring
|
|
131
|
+
model_annotations: The running list of found annotations in search() that we are to add to
|
|
132
|
+
"""
|
|
133
|
+
# Read in the source file to get the line number
|
|
134
|
+
filename = inspect.getsourcefile(model_type)
|
|
135
|
+
with open(filename) as file_handle:
|
|
136
|
+
txt = file_handle.read()
|
|
137
|
+
|
|
138
|
+
# Get the line number by counting newlines + 1 (for the first line).
|
|
139
|
+
# Note that this is the line number of the beginning of the comment, not the
|
|
140
|
+
# annotation token itself. We find based on the entire code content of the model
|
|
141
|
+
# as that seems to be the only way to be sure we're getting the correct line number.
|
|
142
|
+
# It is slow and should be replaced if we can find a better way that is accurate.
|
|
143
|
+
line = txt.count("\n", 0, txt.find(inspect.getsource(model_type))) + 1
|
|
144
|
+
|
|
145
|
+
for inner_match in query.finditer(model_type.__doc__):
|
|
146
|
+
try:
|
|
147
|
+
annotation_token = inner_match.group("token")
|
|
148
|
+
annotation_data = inner_match.group("data")
|
|
149
|
+
except IndexError as error: # pragma: no cover
|
|
150
|
+
raise ValueError(
|
|
151
|
+
'{}: Could not find "data" or "token" groups. Found: {}'.format(
|
|
152
|
+
self.get_model_id(model_type), inner_match.groupdict()
|
|
153
|
+
)
|
|
154
|
+
) from error
|
|
155
|
+
annotation_token, annotation_data = clean_annotation(
|
|
156
|
+
annotation_token, annotation_data
|
|
157
|
+
)
|
|
158
|
+
model_annotations.append(
|
|
159
|
+
{
|
|
160
|
+
"found_by": "django",
|
|
161
|
+
"filename": filename,
|
|
162
|
+
"line_number": line,
|
|
163
|
+
"annotation_token": annotation_token,
|
|
164
|
+
"annotation_data": annotation_data,
|
|
165
|
+
"extra": {
|
|
166
|
+
"object_id": model_id,
|
|
167
|
+
"full_comment": model_type.__doc__.strip(),
|
|
168
|
+
},
|
|
169
|
+
}
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
def _append_safelisted_model_annotations(
|
|
173
|
+
self, safelisted_models, model_id, model_annotations
|
|
174
|
+
):
|
|
175
|
+
"""
|
|
176
|
+
Append the safelisted annotations for the given model id to model_annotations.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
safelisted_models: The dict of models and their annotations loaded from the safelist
|
|
180
|
+
model_id: The text representation of the model name from get_model_id
|
|
181
|
+
model_annotations: The running list of found annotations in search() that we are to add to
|
|
182
|
+
"""
|
|
183
|
+
for annotation in safelisted_models[model_id]:
|
|
184
|
+
comment = safelisted_models[model_id][annotation]
|
|
185
|
+
model_annotations.append(
|
|
186
|
+
{
|
|
187
|
+
"found_by": "safelist",
|
|
188
|
+
"filename": self.config.safelist_path,
|
|
189
|
+
"line_number": 0,
|
|
190
|
+
"annotation_token": annotation.strip(),
|
|
191
|
+
"annotation_data": comment.strip(),
|
|
192
|
+
"extra": {
|
|
193
|
+
"object_id": model_id,
|
|
194
|
+
"full_comment": str(safelisted_models[model_id]),
|
|
195
|
+
},
|
|
196
|
+
}
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
def _read_safelist(self):
|
|
200
|
+
"""
|
|
201
|
+
Read the safelist and return the found models and their annotations.
|
|
202
|
+
|
|
203
|
+
Returns:
|
|
204
|
+
The Python representation of the safelist
|
|
205
|
+
"""
|
|
206
|
+
if os.path.exists(self.config.safelist_path):
|
|
207
|
+
self.echo(f"Found safelist at {self.config.safelist_path}. Reading.\n")
|
|
208
|
+
with open(self.config.safelist_path) as safelist_file:
|
|
209
|
+
safelisted_models = yaml.safe_load(safelist_file)
|
|
210
|
+
self._increment_count("safelisted", len(safelisted_models))
|
|
211
|
+
|
|
212
|
+
if safelisted_models:
|
|
213
|
+
self.echo.echo_vv(
|
|
214
|
+
" Safelisted models:\n " + "\n ".join(safelisted_models)
|
|
215
|
+
)
|
|
216
|
+
else:
|
|
217
|
+
self.echo.echo_vv(" No safelisted models found.\n")
|
|
218
|
+
|
|
219
|
+
return safelisted_models
|
|
220
|
+
else:
|
|
221
|
+
raise Exception(
|
|
222
|
+
"Safelist not found! Generate one with the --seed_safelist command."
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
def search(self):
|
|
226
|
+
"""
|
|
227
|
+
Introspect the configured Django model docstrings for annotations.
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
Dict of all found annotations keyed by filename
|
|
231
|
+
"""
|
|
232
|
+
safelisted_models = self._read_safelist()
|
|
233
|
+
annotation_tokens = self.config.annotation_tokens
|
|
234
|
+
annotation_regexes = self.config.annotation_regexes
|
|
235
|
+
query = get_annotation_regex(annotation_regexes)
|
|
236
|
+
|
|
237
|
+
annotated_models = {}
|
|
238
|
+
|
|
239
|
+
self.echo.echo_vv("Searching models and their parent classes...")
|
|
240
|
+
|
|
241
|
+
# Walk all models and their parents looking for annotations
|
|
242
|
+
for model in self.local_models.union(self.non_local_models):
|
|
243
|
+
model_id = self.get_model_id(model)
|
|
244
|
+
self.echo.echo_vv(" " + model_id)
|
|
245
|
+
hierarchy = inspect.getmro(model)
|
|
246
|
+
model_annotations = []
|
|
247
|
+
|
|
248
|
+
# If any annotations exist in the docstring add them to annotated_models
|
|
249
|
+
for obj in hierarchy:
|
|
250
|
+
if obj.__doc__ is not None:
|
|
251
|
+
if any(anno in obj.__doc__ for anno in annotation_tokens):
|
|
252
|
+
self.echo.echo_vvv(
|
|
253
|
+
" "
|
|
254
|
+
+ DjangoSearch.get_model_id(obj)
|
|
255
|
+
+ " has annotations."
|
|
256
|
+
)
|
|
257
|
+
self._append_model_annotations(
|
|
258
|
+
obj, model_id, query, model_annotations
|
|
259
|
+
)
|
|
260
|
+
else:
|
|
261
|
+
# Don't use get_model_id here, as this could be a base class below Model
|
|
262
|
+
self.echo.echo_vvv(" " + str(obj) + " has no annotations.")
|
|
263
|
+
|
|
264
|
+
# If there are any annotations in the model, format them
|
|
265
|
+
if model_annotations:
|
|
266
|
+
self.echo.echo_vv(
|
|
267
|
+
" {} has {} total annotations".format(
|
|
268
|
+
model_id, len(model_annotations)
|
|
269
|
+
)
|
|
270
|
+
)
|
|
271
|
+
self._increment_count("annotated")
|
|
272
|
+
if model_id in safelisted_models:
|
|
273
|
+
self._add_error(
|
|
274
|
+
f"{model_id} is annotated, but also in the safelist."
|
|
275
|
+
)
|
|
276
|
+
self.format_file_results(annotated_models, [model_annotations])
|
|
277
|
+
|
|
278
|
+
# The model is not in the safelist and is not annotated
|
|
279
|
+
elif model_id not in safelisted_models:
|
|
280
|
+
self._increment_count("unannotated")
|
|
281
|
+
self.uncovered_model_ids.add(model_id)
|
|
282
|
+
self.echo.echo_vv(f" {model_id} has no annotations")
|
|
283
|
+
|
|
284
|
+
# Otherwise it is not annotated and in the safelist
|
|
285
|
+
else:
|
|
286
|
+
if not safelisted_models[model_id]:
|
|
287
|
+
self.uncovered_model_ids.add(model_id)
|
|
288
|
+
self.echo.echo_vv(f" {model_id} is in the safelist.")
|
|
289
|
+
self._add_error(
|
|
290
|
+
f"{model_id} is in the safelist but has no annotations!"
|
|
291
|
+
)
|
|
292
|
+
else:
|
|
293
|
+
self._increment_count("annotated")
|
|
294
|
+
|
|
295
|
+
self._append_safelisted_model_annotations(
|
|
296
|
+
safelisted_models, model_id, model_annotations
|
|
297
|
+
)
|
|
298
|
+
self.format_file_results(annotated_models, [model_annotations])
|
|
299
|
+
|
|
300
|
+
return annotated_models
|
|
301
|
+
|
|
302
|
+
def check_coverage(self):
|
|
303
|
+
"""
|
|
304
|
+
Perform checking of coverage percentage based on stats collected in setup and search.
|
|
305
|
+
|
|
306
|
+
Returns:
|
|
307
|
+
Bool indicating whether or not the number of annotated models covers a percentage
|
|
308
|
+
of total annotation eligible models greater than or equal to the configured
|
|
309
|
+
coverage_target.
|
|
310
|
+
"""
|
|
311
|
+
self.echo("\nModel coverage report")
|
|
312
|
+
self.echo("-" * 40)
|
|
313
|
+
self.echo("Found {total} total models.".format(**self.model_counts))
|
|
314
|
+
self.echo(
|
|
315
|
+
"{annotation_eligible} were eligible for annotation, {annotated} were annotated.".format(
|
|
316
|
+
**self.model_counts
|
|
317
|
+
)
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
if self.model_counts["annotation_eligible"] > 0:
|
|
321
|
+
pct = (
|
|
322
|
+
float(self.model_counts["annotated"])
|
|
323
|
+
/ float(self.model_counts["annotation_eligible"])
|
|
324
|
+
* 100.0
|
|
325
|
+
)
|
|
326
|
+
pct = round(pct, 1)
|
|
327
|
+
else:
|
|
328
|
+
pct = 100.0
|
|
329
|
+
|
|
330
|
+
self.echo(f"Coverage is {pct}%\n")
|
|
331
|
+
|
|
332
|
+
if self.uncovered_model_ids:
|
|
333
|
+
displayed_uncovereds = list(self.uncovered_model_ids)
|
|
334
|
+
displayed_uncovereds.sort()
|
|
335
|
+
self.echo(
|
|
336
|
+
"Coverage found {} uncovered models:\n ".format(
|
|
337
|
+
len(self.uncovered_model_ids)
|
|
338
|
+
)
|
|
339
|
+
+ "\n ".join(displayed_uncovereds)
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
if pct < float(self.config.coverage_target):
|
|
343
|
+
self.echo(
|
|
344
|
+
"\nCoverage threshold not met! Needed {}, actually {}!".format(
|
|
345
|
+
self.config.coverage_target, pct
|
|
346
|
+
)
|
|
347
|
+
)
|
|
348
|
+
return False
|
|
349
|
+
|
|
350
|
+
return True
|
|
351
|
+
|
|
352
|
+
@staticmethod
|
|
353
|
+
def requires_annotations(model):
|
|
354
|
+
"""
|
|
355
|
+
Return true if the given model actually requires annotations, according to PLAT-2344.
|
|
356
|
+
"""
|
|
357
|
+
# Anything inheriting from django.models.Model will have a ._meta attribute. Our tests
|
|
358
|
+
# inherit from object, which doesn't have it, and will fail below. This is a quick way
|
|
359
|
+
# to early out on both.
|
|
360
|
+
if not hasattr(model, "_meta"):
|
|
361
|
+
return False
|
|
362
|
+
|
|
363
|
+
return (
|
|
364
|
+
issubclass(model, models.Model)
|
|
365
|
+
and not (model is models.Model)
|
|
366
|
+
and not model._meta.abstract
|
|
367
|
+
and not model._meta.proxy
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
@staticmethod
|
|
371
|
+
def is_non_local(model):
|
|
372
|
+
"""
|
|
373
|
+
Determine if the given model is non-local to the current IDA.
|
|
374
|
+
|
|
375
|
+
Non-local models are all installed models which are not "local", by
|
|
376
|
+
definition. "Local" models are installed models which are defined in code
|
|
377
|
+
which physically lives in the current codebase, where "current codebase"
|
|
378
|
+
ostensibly refers to the code providing the currently active django
|
|
379
|
+
project.
|
|
380
|
+
|
|
381
|
+
Args:
|
|
382
|
+
model (django.db.models.Model): A model to check.
|
|
383
|
+
|
|
384
|
+
Returns:
|
|
385
|
+
bool: True if the given model is non-local.
|
|
386
|
+
"""
|
|
387
|
+
# If the model _was_ local to the current IDA repository, it should be
|
|
388
|
+
# defined somewhere under sys.prefix + '/src/' or in a path that points to
|
|
389
|
+
# the current checked-out code. On Posix systems according to our testing,
|
|
390
|
+
# non-local packages get installed to paths containing either
|
|
391
|
+
# "site-packages" or "dist-packages".
|
|
392
|
+
non_local_path_prefixes = []
|
|
393
|
+
for path in sys.path:
|
|
394
|
+
if "dist-packages" in path or "site-packages" in path:
|
|
395
|
+
non_local_path_prefixes.append(path)
|
|
396
|
+
model_source_path = inspect.getsourcefile(model)
|
|
397
|
+
return model_source_path.startswith(tuple(non_local_path_prefixes))
|
|
398
|
+
|
|
399
|
+
@staticmethod
|
|
400
|
+
def get_model_id(model):
|
|
401
|
+
"""
|
|
402
|
+
Construct the django standard model identifier in "app_label.ModelClassName" notation.
|
|
403
|
+
|
|
404
|
+
Args:
|
|
405
|
+
model (django.db.models.Model): A model for which to create an identifier.
|
|
406
|
+
|
|
407
|
+
Returns:
|
|
408
|
+
str: identifier string for the given model.
|
|
409
|
+
"""
|
|
410
|
+
return f"{model._meta.app_label}.{model._meta.object_name}"
|
|
411
|
+
|
|
412
|
+
@staticmethod
|
|
413
|
+
def setup_django():
|
|
414
|
+
"""
|
|
415
|
+
Prepare to make django library function calls.
|
|
416
|
+
|
|
417
|
+
On behalf of the current Django project in the current working directory,
|
|
418
|
+
setup/load the django framework, specified settings, and apps therein.
|
|
419
|
+
This should be called before calling any django submodule functions which
|
|
420
|
+
expect apps to be loaded.
|
|
421
|
+
|
|
422
|
+
This function is idempotent.
|
|
423
|
+
"""
|
|
424
|
+
if sys.path[0] != "": # pragma: no cover
|
|
425
|
+
sys.path.insert(0, "")
|
|
426
|
+
django.setup()
|
|
427
|
+
|
|
428
|
+
@staticmethod
|
|
429
|
+
def get_models_requiring_annotations(app_name=None):
|
|
430
|
+
"""
|
|
431
|
+
Determine all local and non-local models via django model introspection.
|
|
432
|
+
|
|
433
|
+
Note that non-local models returned may contain 1st party models (authored by
|
|
434
|
+
edX). This is a compromise in accuracy in order to simplify the generation
|
|
435
|
+
of this list, and also to ease the transition from zero to 100% annotations
|
|
436
|
+
in edX satellite repositories.
|
|
437
|
+
"""
|
|
438
|
+
DjangoSearch.setup_django()
|
|
439
|
+
local_models = set()
|
|
440
|
+
non_local_models = set()
|
|
441
|
+
annotation_eligible_models = []
|
|
442
|
+
total_models = 0
|
|
443
|
+
|
|
444
|
+
for app in apps.get_app_configs():
|
|
445
|
+
if app_name and not app.name.endswith(app_name):
|
|
446
|
+
continue
|
|
447
|
+
|
|
448
|
+
for root_model in app.get_models():
|
|
449
|
+
total_models += 1
|
|
450
|
+
if DjangoSearch.requires_annotations(root_model):
|
|
451
|
+
if DjangoSearch.is_non_local(root_model):
|
|
452
|
+
non_local_models.add(root_model)
|
|
453
|
+
else:
|
|
454
|
+
local_models.add(root_model)
|
|
455
|
+
|
|
456
|
+
annotation_eligible_models.append(
|
|
457
|
+
DjangoSearch.get_model_id(root_model)
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
return local_models, non_local_models, total_models, annotation_eligible_models
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Annotation searcher for static comment searching via Stevedore plugins.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
from code_annotations.base import BaseSearch
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class StaticSearch(BaseSearch):
|
|
11
|
+
"""
|
|
12
|
+
Handles static code searching for annotations.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
def search_extension(self, ext, file_handle, file_extensions_map, filename_extension):
|
|
16
|
+
"""
|
|
17
|
+
Execute a search on the given file using the given extension.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
ext: Extension to execute the search on
|
|
21
|
+
file_handle: An open file handle search
|
|
22
|
+
file_extensions_map: Dict mapping of extension names to configured filename extensions
|
|
23
|
+
filename_extension: The filename extension of the file being searched
|
|
24
|
+
|
|
25
|
+
Returns:
|
|
26
|
+
Tuple of (extension name, list of found annotation dicts)
|
|
27
|
+
"""
|
|
28
|
+
# Only search this file if we are configured for its extension
|
|
29
|
+
if filename_extension in file_extensions_map[ext.name]:
|
|
30
|
+
# Reset the read handle to the beginning of the file in case another
|
|
31
|
+
# extension moved it
|
|
32
|
+
file_handle.seek(0)
|
|
33
|
+
|
|
34
|
+
ext_results = ext.obj.search(file_handle)
|
|
35
|
+
|
|
36
|
+
if ext_results:
|
|
37
|
+
return ext.name, ext_results
|
|
38
|
+
|
|
39
|
+
return ext.name, None
|
|
40
|
+
|
|
41
|
+
def _search_one_file(self, full_name, known_extensions, file_extensions_map, all_results):
|
|
42
|
+
"""
|
|
43
|
+
Perform an annotation search on a single file, using all extensions it is configured for.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
full_name: Complete filename
|
|
47
|
+
known_extensions: List of all file name extensions we are configured to work on
|
|
48
|
+
file_extensions_map: Mapping of file name extensions to Stevedore extensions
|
|
49
|
+
all_results: A dict of annotations returned from search()
|
|
50
|
+
"""
|
|
51
|
+
filename_extension = os.path.splitext(full_name)[1][1:]
|
|
52
|
+
|
|
53
|
+
if filename_extension not in known_extensions:
|
|
54
|
+
self.echo.echo_vvv(
|
|
55
|
+
f"{filename_extension} is not a known extension, skipping ({full_name})."
|
|
56
|
+
)
|
|
57
|
+
return
|
|
58
|
+
|
|
59
|
+
self.echo.echo_vvv(full_name)
|
|
60
|
+
|
|
61
|
+
# TODO: This should probably be a generator so we don't have to store all results in memory
|
|
62
|
+
with open(full_name) as file_handle:
|
|
63
|
+
# Call search_extension on all loaded extensions
|
|
64
|
+
results = self.config.mgr.map(self.search_extension, file_handle, file_extensions_map, filename_extension)
|
|
65
|
+
|
|
66
|
+
# Strip out plugin name, as it's already in the annotation
|
|
67
|
+
results = [r for _, r in results]
|
|
68
|
+
|
|
69
|
+
# Format and add the results to our running full set
|
|
70
|
+
self.format_file_results(all_results, results)
|
|
71
|
+
|
|
72
|
+
def search(self):
|
|
73
|
+
"""
|
|
74
|
+
Walk the source tree, send known file types to extensions.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
Dict of all found annotations keyed by filename
|
|
78
|
+
"""
|
|
79
|
+
# Index the results by extension name
|
|
80
|
+
file_extensions_map = {}
|
|
81
|
+
known_extensions = set()
|
|
82
|
+
for extension_name in self.config.extensions:
|
|
83
|
+
file_extensions_map[extension_name] = self.config.extensions[extension_name]
|
|
84
|
+
known_extensions.update(self.config.extensions[extension_name])
|
|
85
|
+
|
|
86
|
+
all_results = {}
|
|
87
|
+
|
|
88
|
+
if os.path.isfile(self.config.source_path):
|
|
89
|
+
self._search_one_file(self.config.source_path, known_extensions, file_extensions_map, all_results)
|
|
90
|
+
else:
|
|
91
|
+
for root, _, files in os.walk(self.config.source_path):
|
|
92
|
+
for filename in files:
|
|
93
|
+
full_name = os.path.join(root, filename)
|
|
94
|
+
self._search_one_file(full_name, known_extensions, file_extensions_map, all_results)
|
|
95
|
+
|
|
96
|
+
return all_results
|