code-annotations 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. code_annotations/__init__.py +6 -0
  2. code_annotations/annotation_errors.py +64 -0
  3. code_annotations/base.py +647 -0
  4. code_annotations/cli.py +271 -0
  5. code_annotations/contrib/README.rst +8 -0
  6. code_annotations/contrib/config/__init__.py +14 -0
  7. code_annotations/contrib/config/feature_toggle_annotations.yaml +29 -0
  8. code_annotations/contrib/config/openedx_events_annotations.yaml +20 -0
  9. code_annotations/contrib/config/setting_annotations.yaml +13 -0
  10. code_annotations/contrib/sphinx/extensions/__init__.py +3 -0
  11. code_annotations/contrib/sphinx/extensions/base.py +53 -0
  12. code_annotations/contrib/sphinx/extensions/featuretoggles.py +137 -0
  13. code_annotations/contrib/sphinx/extensions/openedx_events.py +182 -0
  14. code_annotations/contrib/sphinx/extensions/settings.py +139 -0
  15. code_annotations/exceptions.py +9 -0
  16. code_annotations/extensions/__init__.py +3 -0
  17. code_annotations/extensions/base.py +166 -0
  18. code_annotations/extensions/javascript.py +20 -0
  19. code_annotations/extensions/python.py +20 -0
  20. code_annotations/find_django.py +460 -0
  21. code_annotations/find_static.py +96 -0
  22. code_annotations/generate_docs.py +165 -0
  23. code_annotations/helpers.py +182 -0
  24. code_annotations/report_templates/html/annotation.tpl +14 -0
  25. code_annotations/report_templates/html/annotation_data.tpl +8 -0
  26. code_annotations/report_templates/html/annotation_list.tpl +27 -0
  27. code_annotations/report_templates/html/base.tpl +92 -0
  28. code_annotations/report_templates/rst/annotation.tpl +13 -0
  29. code_annotations/report_templates/rst/annotation_data.tpl +8 -0
  30. code_annotations/report_templates/rst/annotation_group.tpl +2 -0
  31. code_annotations/report_templates/rst/annotation_list.tpl +34 -0
  32. code_annotations/report_templates/rst/base.tpl +41 -0
  33. code_annotations-3.1.0.dist-info/METADATA +82 -0
  34. code_annotations-3.1.0.dist-info/RECORD +39 -0
  35. code_annotations-3.1.0.dist-info/WHEEL +5 -0
  36. code_annotations-3.1.0.dist-info/entry_points.txt +6 -0
  37. code_annotations-3.1.0.dist-info/licenses/LICENSE.txt +202 -0
  38. code_annotations-3.1.0.dist-info/licenses/NOTICE.txt +13 -0
  39. code_annotations-3.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,460 @@
1
+ """
2
+ Annotation searcher for Django model comment searching Django introspection.
3
+ """
4
+
5
+ import inspect
6
+ import os
7
+ import sys
8
+
9
+ import django
10
+ import yaml
11
+ from django.apps import apps
12
+ from django.db import models
13
+
14
+ from code_annotations.base import BaseSearch
15
+ from code_annotations.helpers import clean_annotation, fail, get_annotation_regex
16
+
17
+ DEFAULT_SAFELIST_FILE_PATH = ".annotation_safe_list.yml"
18
+
19
+
20
+ class DjangoSearch(BaseSearch):
21
+ """
22
+ Handles Django model comment searching for annotations.
23
+ """
24
+
25
+ def __init__(self, config, app_name=None):
26
+ """
27
+ Initialize for DjangoSearch.
28
+ """
29
+ super().__init__(config)
30
+ self.local_models, self.non_local_models, total, annotation_eligible = (
31
+ self.get_models_requiring_annotations(app_name)
32
+ )
33
+ self.model_counts = {
34
+ "total": total,
35
+ "annotated": 0,
36
+ "unannotated": 0,
37
+ "annotation_eligible": len(annotation_eligible),
38
+ "not_annotation_eligible": total - len(annotation_eligible),
39
+ "safelisted": 0,
40
+ }
41
+ self.uncovered_model_ids = set()
42
+ self.echo.echo_vvv(
43
+ "Local models:\n "
44
+ + "\n ".join([str(m) for m in self.local_models])
45
+ + "\n"
46
+ )
47
+ self.echo.echo_vvv(
48
+ "Non-local models:\n "
49
+ + "\n ".join([str(m) for m in self.non_local_models])
50
+ + "\n"
51
+ )
52
+ self.echo.echo_vv(
53
+ "The following models require annotations:\n "
54
+ + "\n ".join(annotation_eligible)
55
+ + "\n"
56
+ )
57
+
58
+ def _increment_count(self, count_type, incr_by=1):
59
+ self.model_counts[count_type] += incr_by
60
+
61
+ def seed_safelist(self):
62
+ """
63
+ Seed a new safelist file with all non-local models that need to be vetted.
64
+ """
65
+ if os.path.exists(self.config.safelist_path):
66
+ fail(f"{self.config.safelist_path} already exists, not overwriting.")
67
+
68
+ self.echo(
69
+ "Found {} non-local models requiring annotations. Adding them to safelist.".format(
70
+ len(self.non_local_models)
71
+ )
72
+ )
73
+
74
+ safelist_data = {
75
+ self.get_model_id(model): {} for model in self.non_local_models
76
+ }
77
+
78
+ with open(self.config.safelist_path, "w") as safelist_file:
79
+ safelist_comment = """
80
+ # This is a Code Annotations automatically-generated Django model safelist file.
81
+ # These models must be annotated as follows in order to be counted in the coverage report.
82
+ # See https://code-annotations.readthedocs.io/en/latest/safelist.html for more information.
83
+ #
84
+ # fake_app_1.FakeModelName:
85
+ # ".. no_pii:": "This model has no PII"
86
+ # fake_app_2.FakeModel2:
87
+ # ".. choice_annotation:": foo, bar, baz
88
+
89
+ """
90
+ safelist_file.write(safelist_comment.lstrip())
91
+ yaml.safe_dump(
92
+ safelist_data, stream=safelist_file, default_flow_style=False
93
+ )
94
+
95
+ self.echo(
96
+ f'Successfully created safelist file "{self.config.safelist_path}".',
97
+ fg="red",
98
+ )
99
+ self.echo("Now, you need to:", fg="red")
100
+ self.echo(
101
+ " 1) Make sure that any un-annotated models in the safelist are annotated, and",
102
+ fg="red",
103
+ )
104
+ self.echo(" 2) Annotate any LOCAL models (see --list_local_models).", fg="red")
105
+
106
+ def list_local_models(self):
107
+ """
108
+ Dump a list of models in the local code tree that are annotation eligible to stdout.
109
+ """
110
+ if self.local_models:
111
+ self.echo(
112
+ "Listing {} local models requiring annotations:".format(
113
+ len(self.local_models)
114
+ )
115
+ )
116
+ self.echo.pprint(
117
+ sorted([self.get_model_id(model) for model in self.local_models]),
118
+ indent=4,
119
+ )
120
+ else:
121
+ self.echo("No local models requiring annotations.")
122
+
123
+ def _append_model_annotations(self, model_type, model_id, query, model_annotations):
124
+ """
125
+ Find the given model's annotations in the file and add them to model_annotations.
126
+
127
+ Args:
128
+ model_type: The type of model we're searching
129
+ model_id: The text representation of the model name from get_model_id
130
+ query: The regex to run to find annotations in the docstring
131
+ model_annotations: The running list of found annotations in search() that we are to add to
132
+ """
133
+ # Read in the source file to get the line number
134
+ filename = inspect.getsourcefile(model_type)
135
+ with open(filename) as file_handle:
136
+ txt = file_handle.read()
137
+
138
+ # Get the line number by counting newlines + 1 (for the first line).
139
+ # Note that this is the line number of the beginning of the comment, not the
140
+ # annotation token itself. We find based on the entire code content of the model
141
+ # as that seems to be the only way to be sure we're getting the correct line number.
142
+ # It is slow and should be replaced if we can find a better way that is accurate.
143
+ line = txt.count("\n", 0, txt.find(inspect.getsource(model_type))) + 1
144
+
145
+ for inner_match in query.finditer(model_type.__doc__):
146
+ try:
147
+ annotation_token = inner_match.group("token")
148
+ annotation_data = inner_match.group("data")
149
+ except IndexError as error: # pragma: no cover
150
+ raise ValueError(
151
+ '{}: Could not find "data" or "token" groups. Found: {}'.format(
152
+ self.get_model_id(model_type), inner_match.groupdict()
153
+ )
154
+ ) from error
155
+ annotation_token, annotation_data = clean_annotation(
156
+ annotation_token, annotation_data
157
+ )
158
+ model_annotations.append(
159
+ {
160
+ "found_by": "django",
161
+ "filename": filename,
162
+ "line_number": line,
163
+ "annotation_token": annotation_token,
164
+ "annotation_data": annotation_data,
165
+ "extra": {
166
+ "object_id": model_id,
167
+ "full_comment": model_type.__doc__.strip(),
168
+ },
169
+ }
170
+ )
171
+
172
+ def _append_safelisted_model_annotations(
173
+ self, safelisted_models, model_id, model_annotations
174
+ ):
175
+ """
176
+ Append the safelisted annotations for the given model id to model_annotations.
177
+
178
+ Args:
179
+ safelisted_models: The dict of models and their annotations loaded from the safelist
180
+ model_id: The text representation of the model name from get_model_id
181
+ model_annotations: The running list of found annotations in search() that we are to add to
182
+ """
183
+ for annotation in safelisted_models[model_id]:
184
+ comment = safelisted_models[model_id][annotation]
185
+ model_annotations.append(
186
+ {
187
+ "found_by": "safelist",
188
+ "filename": self.config.safelist_path,
189
+ "line_number": 0,
190
+ "annotation_token": annotation.strip(),
191
+ "annotation_data": comment.strip(),
192
+ "extra": {
193
+ "object_id": model_id,
194
+ "full_comment": str(safelisted_models[model_id]),
195
+ },
196
+ }
197
+ )
198
+
199
+ def _read_safelist(self):
200
+ """
201
+ Read the safelist and return the found models and their annotations.
202
+
203
+ Returns:
204
+ The Python representation of the safelist
205
+ """
206
+ if os.path.exists(self.config.safelist_path):
207
+ self.echo(f"Found safelist at {self.config.safelist_path}. Reading.\n")
208
+ with open(self.config.safelist_path) as safelist_file:
209
+ safelisted_models = yaml.safe_load(safelist_file)
210
+ self._increment_count("safelisted", len(safelisted_models))
211
+
212
+ if safelisted_models:
213
+ self.echo.echo_vv(
214
+ " Safelisted models:\n " + "\n ".join(safelisted_models)
215
+ )
216
+ else:
217
+ self.echo.echo_vv(" No safelisted models found.\n")
218
+
219
+ return safelisted_models
220
+ else:
221
+ raise Exception(
222
+ "Safelist not found! Generate one with the --seed_safelist command."
223
+ )
224
+
225
+ def search(self):
226
+ """
227
+ Introspect the configured Django model docstrings for annotations.
228
+
229
+ Returns:
230
+ Dict of all found annotations keyed by filename
231
+ """
232
+ safelisted_models = self._read_safelist()
233
+ annotation_tokens = self.config.annotation_tokens
234
+ annotation_regexes = self.config.annotation_regexes
235
+ query = get_annotation_regex(annotation_regexes)
236
+
237
+ annotated_models = {}
238
+
239
+ self.echo.echo_vv("Searching models and their parent classes...")
240
+
241
+ # Walk all models and their parents looking for annotations
242
+ for model in self.local_models.union(self.non_local_models):
243
+ model_id = self.get_model_id(model)
244
+ self.echo.echo_vv(" " + model_id)
245
+ hierarchy = inspect.getmro(model)
246
+ model_annotations = []
247
+
248
+ # If any annotations exist in the docstring add them to annotated_models
249
+ for obj in hierarchy:
250
+ if obj.__doc__ is not None:
251
+ if any(anno in obj.__doc__ for anno in annotation_tokens):
252
+ self.echo.echo_vvv(
253
+ " "
254
+ + DjangoSearch.get_model_id(obj)
255
+ + " has annotations."
256
+ )
257
+ self._append_model_annotations(
258
+ obj, model_id, query, model_annotations
259
+ )
260
+ else:
261
+ # Don't use get_model_id here, as this could be a base class below Model
262
+ self.echo.echo_vvv(" " + str(obj) + " has no annotations.")
263
+
264
+ # If there are any annotations in the model, format them
265
+ if model_annotations:
266
+ self.echo.echo_vv(
267
+ " {} has {} total annotations".format(
268
+ model_id, len(model_annotations)
269
+ )
270
+ )
271
+ self._increment_count("annotated")
272
+ if model_id in safelisted_models:
273
+ self._add_error(
274
+ f"{model_id} is annotated, but also in the safelist."
275
+ )
276
+ self.format_file_results(annotated_models, [model_annotations])
277
+
278
+ # The model is not in the safelist and is not annotated
279
+ elif model_id not in safelisted_models:
280
+ self._increment_count("unannotated")
281
+ self.uncovered_model_ids.add(model_id)
282
+ self.echo.echo_vv(f" {model_id} has no annotations")
283
+
284
+ # Otherwise it is not annotated and in the safelist
285
+ else:
286
+ if not safelisted_models[model_id]:
287
+ self.uncovered_model_ids.add(model_id)
288
+ self.echo.echo_vv(f" {model_id} is in the safelist.")
289
+ self._add_error(
290
+ f"{model_id} is in the safelist but has no annotations!"
291
+ )
292
+ else:
293
+ self._increment_count("annotated")
294
+
295
+ self._append_safelisted_model_annotations(
296
+ safelisted_models, model_id, model_annotations
297
+ )
298
+ self.format_file_results(annotated_models, [model_annotations])
299
+
300
+ return annotated_models
301
+
302
+ def check_coverage(self):
303
+ """
304
+ Perform checking of coverage percentage based on stats collected in setup and search.
305
+
306
+ Returns:
307
+ Bool indicating whether or not the number of annotated models covers a percentage
308
+ of total annotation eligible models greater than or equal to the configured
309
+ coverage_target.
310
+ """
311
+ self.echo("\nModel coverage report")
312
+ self.echo("-" * 40)
313
+ self.echo("Found {total} total models.".format(**self.model_counts))
314
+ self.echo(
315
+ "{annotation_eligible} were eligible for annotation, {annotated} were annotated.".format(
316
+ **self.model_counts
317
+ )
318
+ )
319
+
320
+ if self.model_counts["annotation_eligible"] > 0:
321
+ pct = (
322
+ float(self.model_counts["annotated"])
323
+ / float(self.model_counts["annotation_eligible"])
324
+ * 100.0
325
+ )
326
+ pct = round(pct, 1)
327
+ else:
328
+ pct = 100.0
329
+
330
+ self.echo(f"Coverage is {pct}%\n")
331
+
332
+ if self.uncovered_model_ids:
333
+ displayed_uncovereds = list(self.uncovered_model_ids)
334
+ displayed_uncovereds.sort()
335
+ self.echo(
336
+ "Coverage found {} uncovered models:\n ".format(
337
+ len(self.uncovered_model_ids)
338
+ )
339
+ + "\n ".join(displayed_uncovereds)
340
+ )
341
+
342
+ if pct < float(self.config.coverage_target):
343
+ self.echo(
344
+ "\nCoverage threshold not met! Needed {}, actually {}!".format(
345
+ self.config.coverage_target, pct
346
+ )
347
+ )
348
+ return False
349
+
350
+ return True
351
+
352
+ @staticmethod
353
+ def requires_annotations(model):
354
+ """
355
+ Return true if the given model actually requires annotations, according to PLAT-2344.
356
+ """
357
+ # Anything inheriting from django.models.Model will have a ._meta attribute. Our tests
358
+ # inherit from object, which doesn't have it, and will fail below. This is a quick way
359
+ # to early out on both.
360
+ if not hasattr(model, "_meta"):
361
+ return False
362
+
363
+ return (
364
+ issubclass(model, models.Model)
365
+ and not (model is models.Model)
366
+ and not model._meta.abstract
367
+ and not model._meta.proxy
368
+ )
369
+
370
+ @staticmethod
371
+ def is_non_local(model):
372
+ """
373
+ Determine if the given model is non-local to the current IDA.
374
+
375
+ Non-local models are all installed models which are not "local", by
376
+ definition. "Local" models are installed models which are defined in code
377
+ which physically lives in the current codebase, where "current codebase"
378
+ ostensibly refers to the code providing the currently active django
379
+ project.
380
+
381
+ Args:
382
+ model (django.db.models.Model): A model to check.
383
+
384
+ Returns:
385
+ bool: True if the given model is non-local.
386
+ """
387
+ # If the model _was_ local to the current IDA repository, it should be
388
+ # defined somewhere under sys.prefix + '/src/' or in a path that points to
389
+ # the current checked-out code. On Posix systems according to our testing,
390
+ # non-local packages get installed to paths containing either
391
+ # "site-packages" or "dist-packages".
392
+ non_local_path_prefixes = []
393
+ for path in sys.path:
394
+ if "dist-packages" in path or "site-packages" in path:
395
+ non_local_path_prefixes.append(path)
396
+ model_source_path = inspect.getsourcefile(model)
397
+ return model_source_path.startswith(tuple(non_local_path_prefixes))
398
+
399
+ @staticmethod
400
+ def get_model_id(model):
401
+ """
402
+ Construct the django standard model identifier in "app_label.ModelClassName" notation.
403
+
404
+ Args:
405
+ model (django.db.models.Model): A model for which to create an identifier.
406
+
407
+ Returns:
408
+ str: identifier string for the given model.
409
+ """
410
+ return f"{model._meta.app_label}.{model._meta.object_name}"
411
+
412
+ @staticmethod
413
+ def setup_django():
414
+ """
415
+ Prepare to make django library function calls.
416
+
417
+ On behalf of the current Django project in the current working directory,
418
+ setup/load the django framework, specified settings, and apps therein.
419
+ This should be called before calling any django submodule functions which
420
+ expect apps to be loaded.
421
+
422
+ This function is idempotent.
423
+ """
424
+ if sys.path[0] != "": # pragma: no cover
425
+ sys.path.insert(0, "")
426
+ django.setup()
427
+
428
+ @staticmethod
429
+ def get_models_requiring_annotations(app_name=None):
430
+ """
431
+ Determine all local and non-local models via django model introspection.
432
+
433
+ Note that non-local models returned may contain 1st party models (authored by
434
+ edX). This is a compromise in accuracy in order to simplify the generation
435
+ of this list, and also to ease the transition from zero to 100% annotations
436
+ in edX satellite repositories.
437
+ """
438
+ DjangoSearch.setup_django()
439
+ local_models = set()
440
+ non_local_models = set()
441
+ annotation_eligible_models = []
442
+ total_models = 0
443
+
444
+ for app in apps.get_app_configs():
445
+ if app_name and not app.name.endswith(app_name):
446
+ continue
447
+
448
+ for root_model in app.get_models():
449
+ total_models += 1
450
+ if DjangoSearch.requires_annotations(root_model):
451
+ if DjangoSearch.is_non_local(root_model):
452
+ non_local_models.add(root_model)
453
+ else:
454
+ local_models.add(root_model)
455
+
456
+ annotation_eligible_models.append(
457
+ DjangoSearch.get_model_id(root_model)
458
+ )
459
+
460
+ return local_models, non_local_models, total_models, annotation_eligible_models
@@ -0,0 +1,96 @@
1
+ """
2
+ Annotation searcher for static comment searching via Stevedore plugins.
3
+ """
4
+
5
+ import os
6
+
7
+ from code_annotations.base import BaseSearch
8
+
9
+
10
+ class StaticSearch(BaseSearch):
11
+ """
12
+ Handles static code searching for annotations.
13
+ """
14
+
15
+ def search_extension(self, ext, file_handle, file_extensions_map, filename_extension):
16
+ """
17
+ Execute a search on the given file using the given extension.
18
+
19
+ Args:
20
+ ext: Extension to execute the search on
21
+ file_handle: An open file handle search
22
+ file_extensions_map: Dict mapping of extension names to configured filename extensions
23
+ filename_extension: The filename extension of the file being searched
24
+
25
+ Returns:
26
+ Tuple of (extension name, list of found annotation dicts)
27
+ """
28
+ # Only search this file if we are configured for its extension
29
+ if filename_extension in file_extensions_map[ext.name]:
30
+ # Reset the read handle to the beginning of the file in case another
31
+ # extension moved it
32
+ file_handle.seek(0)
33
+
34
+ ext_results = ext.obj.search(file_handle)
35
+
36
+ if ext_results:
37
+ return ext.name, ext_results
38
+
39
+ return ext.name, None
40
+
41
+ def _search_one_file(self, full_name, known_extensions, file_extensions_map, all_results):
42
+ """
43
+ Perform an annotation search on a single file, using all extensions it is configured for.
44
+
45
+ Args:
46
+ full_name: Complete filename
47
+ known_extensions: List of all file name extensions we are configured to work on
48
+ file_extensions_map: Mapping of file name extensions to Stevedore extensions
49
+ all_results: A dict of annotations returned from search()
50
+ """
51
+ filename_extension = os.path.splitext(full_name)[1][1:]
52
+
53
+ if filename_extension not in known_extensions:
54
+ self.echo.echo_vvv(
55
+ f"{filename_extension} is not a known extension, skipping ({full_name})."
56
+ )
57
+ return
58
+
59
+ self.echo.echo_vvv(full_name)
60
+
61
+ # TODO: This should probably be a generator so we don't have to store all results in memory
62
+ with open(full_name) as file_handle:
63
+ # Call search_extension on all loaded extensions
64
+ results = self.config.mgr.map(self.search_extension, file_handle, file_extensions_map, filename_extension)
65
+
66
+ # Strip out plugin name, as it's already in the annotation
67
+ results = [r for _, r in results]
68
+
69
+ # Format and add the results to our running full set
70
+ self.format_file_results(all_results, results)
71
+
72
+ def search(self):
73
+ """
74
+ Walk the source tree, send known file types to extensions.
75
+
76
+ Returns:
77
+ Dict of all found annotations keyed by filename
78
+ """
79
+ # Index the results by extension name
80
+ file_extensions_map = {}
81
+ known_extensions = set()
82
+ for extension_name in self.config.extensions:
83
+ file_extensions_map[extension_name] = self.config.extensions[extension_name]
84
+ known_extensions.update(self.config.extensions[extension_name])
85
+
86
+ all_results = {}
87
+
88
+ if os.path.isfile(self.config.source_path):
89
+ self._search_one_file(self.config.source_path, known_extensions, file_extensions_map, all_results)
90
+ else:
91
+ for root, _, files in os.walk(self.config.source_path):
92
+ for filename in files:
93
+ full_name = os.path.join(root, filename)
94
+ self._search_one_file(full_name, known_extensions, file_extensions_map, all_results)
95
+
96
+ return all_results