coretrace-python-analyzer 0.22.0__py3-none-any.whl → 0.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coretrace_python/__init__.py +1 -1
- coretrace_python/abstract/module_constants.py +121 -0
- coretrace_python/abstract/strings.py +12 -42
- coretrace_python/bundled/models/python_stdlib/python_stdlib.py +27 -0
- coretrace_python/bundled/secrets/config_secrets/config_secrets.py +2 -1
- coretrace_python/bundled/security/code_injection/code_injection.py +85 -4
- coretrace_python/bundled/security/code_injection/plugin.toml +1 -1
- coretrace_python/bundled/syntax/unsafe_xml/plugin.toml +1 -1
- coretrace_python/bundled/syntax/unsafe_xml/unsafe_xml.py +92 -12
- coretrace_python/cache.py +14 -14
- coretrace_python/cli.py +33 -5
- coretrace_python/engine.py +197 -60
- coretrace_python/findings/context.py +57 -0
- coretrace_python/findings/refutation.py +210 -27
- coretrace_python/frontend/parser.py +7 -1
- coretrace_python/interprocedural/callgraph.py +14 -1
- coretrace_python/interprocedural/summaries.py +91 -19
- coretrace_python/ir/defuse.py +26 -1
- coretrace_python/ir/lowering.py +2 -1
- coretrace_python/ir/model.py +5 -2
- coretrace_python/ir/printer.py +4 -1
- coretrace_python/plugins/__init__.py +7 -1
- coretrace_python/plugins/api.py +16 -3
- coretrace_python/plugins/detectors.py +21 -3
- coretrace_python/plugins/secrets.py +17 -11
- coretrace_python/reporters/sarif.py +15 -3
- coretrace_python/semantic/symbols.py +2 -1
- coretrace_python/taint/engine.py +114 -27
- coretrace_python/taint/models.py +27 -3
- {coretrace_python_analyzer-0.22.0.dist-info → coretrace_python_analyzer-0.23.0.dist-info}/METADATA +1 -1
- {coretrace_python_analyzer-0.22.0.dist-info → coretrace_python_analyzer-0.23.0.dist-info}/RECORD +35 -33
- {coretrace_python_analyzer-0.22.0.dist-info → coretrace_python_analyzer-0.23.0.dist-info}/WHEEL +0 -0
- {coretrace_python_analyzer-0.22.0.dist-info → coretrace_python_analyzer-0.23.0.dist-info}/entry_points.txt +0 -0
- {coretrace_python_analyzer-0.22.0.dist-info → coretrace_python_analyzer-0.23.0.dist-info}/licenses/LICENSE +0 -0
- {coretrace_python_analyzer-0.22.0.dist-info → coretrace_python_analyzer-0.23.0.dist-info}/licenses/NOTICE +0 -0
coretrace_python/engine.py
CHANGED
|
@@ -9,6 +9,8 @@ from __future__ import annotations
|
|
|
9
9
|
import concurrent.futures
|
|
10
10
|
import hashlib
|
|
11
11
|
import multiprocessing
|
|
12
|
+
import sys
|
|
13
|
+
import tempfile
|
|
12
14
|
from collections import Counter
|
|
13
15
|
from collections.abc import Callable, Iterable, Mapping, Sequence
|
|
14
16
|
from dataclasses import dataclass, field, replace
|
|
@@ -18,6 +20,7 @@ from typing import Any, ClassVar
|
|
|
18
20
|
|
|
19
21
|
from coretrace_python import __version__
|
|
20
22
|
from coretrace_python.abstract import ConstantPropagation, HeapAnalysis, RangeAnalysis
|
|
23
|
+
from coretrace_python.abstract.module_constants import ModuleCollectionsAnalysis
|
|
21
24
|
from coretrace_python.abstract.strings import ModuleStringsAnalysis
|
|
22
25
|
from coretrace_python.analysis import (
|
|
23
26
|
AnalysisContext,
|
|
@@ -29,10 +32,8 @@ from coretrace_python.cache import (
|
|
|
29
32
|
CachedModule,
|
|
30
33
|
ProjectCache,
|
|
31
34
|
decode,
|
|
32
|
-
decode_index,
|
|
33
35
|
directory_fingerprint,
|
|
34
36
|
encode,
|
|
35
|
-
encode_index,
|
|
36
37
|
fingerprint,
|
|
37
38
|
module_keys,
|
|
38
39
|
)
|
|
@@ -71,6 +72,7 @@ from coretrace_python.findings import (
|
|
|
71
72
|
Finding,
|
|
72
73
|
Severity,
|
|
73
74
|
)
|
|
75
|
+
from coretrace_python.findings.context import code_context, code_origin, path_in_project
|
|
74
76
|
from coretrace_python.findings.refutation import RefutationAnalysis
|
|
75
77
|
from coretrace_python.findings.suppressions import partition
|
|
76
78
|
from coretrace_python.frontend import HIRBuildError, ParseError, build_hir
|
|
@@ -157,6 +159,7 @@ ALL_ANALYSES: tuple[AnyAnalysis, ...] = (
|
|
|
157
159
|
ConstantPropagation,
|
|
158
160
|
RangeAnalysis,
|
|
159
161
|
ModuleStringsAnalysis,
|
|
162
|
+
ModuleCollectionsAnalysis,
|
|
160
163
|
HeapAnalysis,
|
|
161
164
|
CallGraphAnalysis,
|
|
162
165
|
SummaryAnalysis,
|
|
@@ -324,8 +327,11 @@ def check(source: SourceFile, plugin_roots: Sequence[Path]) -> tuple[Finding, ..
|
|
|
324
327
|
return analyze_file(source, plugin_roots).findings
|
|
325
328
|
|
|
326
329
|
|
|
327
|
-
def analyze_file(source: SourceFile, plugin_roots: Sequence[Path]) -> FileAnalysis:
|
|
328
|
-
"""One file's findings and coverage, with every plugin found under ``plugin_roots``.
|
|
330
|
+
def analyze_file(source: SourceFile, plugin_roots: Sequence[Path], root: Path | None = None) -> FileAnalysis:
|
|
331
|
+
"""One file's findings and coverage, with every plugin found under ``plugin_roots``.
|
|
332
|
+
The context of its findings is judged on its path from ``root``, the root of its
|
|
333
|
+
report: by default its own directory, as the command line reports a file checked
|
|
334
|
+
alone. A source that is not a file is judged on its name as given."""
|
|
329
335
|
|
|
330
336
|
manager = _register_all(build_hir(source))
|
|
331
337
|
registry = load_plugins(plugin_roots, manager)
|
|
@@ -336,7 +342,10 @@ def analyze_file(source: SourceFile, plugin_roots: Sequence[Path]) -> FileAnalys
|
|
|
336
342
|
manager.provide(ClearingAnalysis, table.clearing())
|
|
337
343
|
manager.provide(ProjectSummaries, SummaryIndex())
|
|
338
344
|
manager.provide(RegisteredRoutes, _routes_of(manager))
|
|
339
|
-
|
|
345
|
+
reference = None if source.path is None else root or source.path.resolve().parent
|
|
346
|
+
findings, supported = _check_module(manager, tuple(loaded.plugin for loaded in registry), reference)
|
|
347
|
+
path = path_in_project(source.source_id, reference)
|
|
348
|
+
findings = tuple(_situated(f, path if f.span.source_id == source.source_id else None) for f in findings)
|
|
340
349
|
functions = len(analyzable_functions(manager.module))
|
|
341
350
|
coverage = Coverage((FileCoverage(str(source.source_id), "analysed", functions, len(supported)),))
|
|
342
351
|
kept, suppressed = partition(findings, lambda sid: source.text if sid == source.source_id else None)
|
|
@@ -360,6 +369,20 @@ def resolve_dependencies(root: Path, sources: SourceManager) -> DependencyGraph:
|
|
|
360
369
|
return graph
|
|
361
370
|
|
|
362
371
|
|
|
372
|
+
# ``ProcessPoolExecutor`` refuses more workers on Windows, where it waits on at most 63
|
|
373
|
+
# handles.
|
|
374
|
+
_WINDOWS_POOL_LIMIT = 61
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def process_pool_size(jobs: int, platform: str | None = None) -> int:
|
|
378
|
+
"""The processes a pool asked for ``jobs`` holds: no more than ``platform`` (this one
|
|
379
|
+
by default) allows."""
|
|
380
|
+
|
|
381
|
+
if (platform or sys.platform) == "win32":
|
|
382
|
+
return min(jobs, _WINDOWS_POOL_LIMIT)
|
|
383
|
+
return jobs
|
|
384
|
+
|
|
385
|
+
|
|
363
386
|
def analyze_project(
|
|
364
387
|
root: Path,
|
|
365
388
|
plugin_roots: Sequence[Path] = (),
|
|
@@ -493,9 +516,30 @@ def analyze_project(
|
|
|
493
516
|
module_plugins = tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin))
|
|
494
517
|
suffixes = suffix_index(affected, analysable)
|
|
495
518
|
pool = None
|
|
519
|
+
spooled: tempfile.TemporaryDirectory[str] | None = None
|
|
520
|
+
# Where finished modules are kept for the next waves: the cache, else a directory of
|
|
521
|
+
# this run's own. Workers read from it what a component imports (#245).
|
|
522
|
+
spool = cache
|
|
496
523
|
if jobs > 1:
|
|
524
|
+
if spool is None:
|
|
525
|
+
spooled = tempfile.TemporaryDirectory(prefix="coretrace-")
|
|
526
|
+
spool = ProjectCache(Path(spooled.name))
|
|
527
|
+
project = _Project(
|
|
528
|
+
root,
|
|
529
|
+
tuple(plugin_roots),
|
|
530
|
+
tuple(plugins),
|
|
531
|
+
advisory_paths,
|
|
532
|
+
_encode_routes(routes),
|
|
533
|
+
tuple(sorted(escaped)),
|
|
534
|
+
template_calls,
|
|
535
|
+
tuple(sorted(analysable)),
|
|
536
|
+
spool.directory,
|
|
537
|
+
)
|
|
497
538
|
pool = concurrent.futures.ProcessPoolExecutor(
|
|
498
|
-
max_workers=jobs,
|
|
539
|
+
max_workers=process_pool_size(jobs),
|
|
540
|
+
mp_context=multiprocessing.get_context("spawn"),
|
|
541
|
+
initializer=_start_worker,
|
|
542
|
+
initargs=(project,),
|
|
499
543
|
)
|
|
500
544
|
try:
|
|
501
545
|
for wave in graph.schedule():
|
|
@@ -505,7 +549,9 @@ def analyze_project(
|
|
|
505
549
|
for component in pending:
|
|
506
550
|
batch = {name: analysable[name] for name in sorted(component)}
|
|
507
551
|
computed.update(
|
|
508
|
-
_analyse_managers(
|
|
552
|
+
_analyse_managers(
|
|
553
|
+
batch, _seed(results, graph, component), module_plugins, affected, suffixes, root
|
|
554
|
+
)
|
|
509
555
|
)
|
|
510
556
|
for manager in batch.values():
|
|
511
557
|
manager.run(ResultsEvicted)
|
|
@@ -514,29 +560,27 @@ def analyze_project(
|
|
|
514
560
|
pool.submit(
|
|
515
561
|
_analyse_batch,
|
|
516
562
|
_Batch(
|
|
517
|
-
root,
|
|
518
|
-
tuple(plugin_roots),
|
|
519
|
-
tuple(plugins),
|
|
520
563
|
{name: _path_of(files[name]) for name in sorted(component)},
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
template_calls,
|
|
526
|
-
project=tuple(sorted(analysable)),
|
|
564
|
+
{
|
|
565
|
+
name: (import_name, keys[name])
|
|
566
|
+
for name, import_name in _imported_names(graph, component, results).items()
|
|
567
|
+
},
|
|
527
568
|
),
|
|
528
569
|
)
|
|
529
570
|
for component in pending
|
|
530
571
|
]
|
|
531
572
|
for future in futures:
|
|
532
573
|
computed.update({name: decode(data) for name, data in future.result().items()})
|
|
574
|
+
store = cache if pool is None else spool
|
|
533
575
|
for name, entry in computed.items():
|
|
534
|
-
if
|
|
535
|
-
|
|
576
|
+
if store is not None:
|
|
577
|
+
store.store(keys[name], entry)
|
|
536
578
|
results.update(computed)
|
|
537
579
|
finally:
|
|
538
580
|
if pool is not None:
|
|
539
581
|
pool.shutdown()
|
|
582
|
+
if spooled is not None:
|
|
583
|
+
spooled.cleanup()
|
|
540
584
|
|
|
541
585
|
index = _index(results, {name: name for name in results})
|
|
542
586
|
call_graphs: dict[str, CallGraph] = {}
|
|
@@ -568,6 +612,8 @@ def analyze_project(
|
|
|
568
612
|
for plugin in all_plugins:
|
|
569
613
|
if isinstance(plugin, ProjectPlugin):
|
|
570
614
|
findings.extend(plugin.analyze_project(context))
|
|
615
|
+
# The context comes first, so that a refiner can read it.
|
|
616
|
+
findings = [_situated(finding, path_in_project(finding.span.source_id, root)) for finding in findings]
|
|
571
617
|
for plugin in all_plugins:
|
|
572
618
|
if isinstance(plugin, ProjectPlugin):
|
|
573
619
|
findings = list(apply_refinement(plugin, findings, plugin.refine(context, tuple(findings))))
|
|
@@ -652,6 +698,24 @@ def _located(path: Path, root: Path) -> str:
|
|
|
652
698
|
return str(path)
|
|
653
699
|
|
|
654
700
|
|
|
701
|
+
def _situated(finding: Finding, path: str | None) -> Finding:
|
|
702
|
+
"""``finding`` with the context (``code_context``) and the origin (``code_origin``) of
|
|
703
|
+
its code in its metadata, judged on ``path``, its file's path from the root of the
|
|
704
|
+
report. A context the rule gave the finding stays: it is the rule's (#244). The origin
|
|
705
|
+
is told apart from the context, so vendored test code is both (#243)."""
|
|
706
|
+
|
|
707
|
+
if path is None:
|
|
708
|
+
return finding
|
|
709
|
+
added: dict[str, str] = {}
|
|
710
|
+
context = code_context(path)
|
|
711
|
+
if context is not None and "context" not in finding.metadata:
|
|
712
|
+
added["context"] = context
|
|
713
|
+
origin = code_origin(path)
|
|
714
|
+
if origin is not None:
|
|
715
|
+
added["origin"] = origin
|
|
716
|
+
return replace(finding, metadata={**finding.metadata, **added}) if added else finding
|
|
717
|
+
|
|
718
|
+
|
|
655
719
|
def _sourced(finding: Finding, origins: Origins) -> Finding:
|
|
656
720
|
"""``finding`` with the plugin or advisory file its advisory comes from, when it
|
|
657
721
|
cites one."""
|
|
@@ -705,12 +769,20 @@ def _ambiguity_notes(graph: ModuleGraph, shared: Mapping[str, list[str]], root:
|
|
|
705
769
|
|
|
706
770
|
|
|
707
771
|
def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: frozenset[str]) -> SummaryIndex:
|
|
708
|
-
"""The summaries a component starts from
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
772
|
+
"""The summaries a component starts from (``_imported_names``)."""
|
|
773
|
+
|
|
774
|
+
return _index(results, _imported_names(graph, component, results))
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _imported_names(
|
|
778
|
+
graph: ModuleGraph, component: frozenset[str], results: Mapping[str, CachedModule]
|
|
779
|
+
) -> dict[str, str]:
|
|
780
|
+
"""The modules whose summaries a component starts from, each with the import name the
|
|
781
|
+
component's code refers to it by: every module it imports, transitively, all final
|
|
782
|
+
by the time its wave runs. Of several files with one import name, only the one a
|
|
783
|
+
member imports directly, resolved in its own root, goes under that name: the others
|
|
784
|
+
are reached through summaries that inline them already, and a name no member's root
|
|
785
|
+
resolved is followed into neither file."""
|
|
714
786
|
|
|
715
787
|
wanted: set[str] = set()
|
|
716
788
|
pending = list(component)
|
|
@@ -728,7 +800,7 @@ def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: fr
|
|
|
728
800
|
chosen = modules if len(modules) == 1 else [name for name in modules if name in direct]
|
|
729
801
|
if len(chosen) == 1:
|
|
730
802
|
names[chosen[0]] = import_name
|
|
731
|
-
return
|
|
803
|
+
return names
|
|
732
804
|
|
|
733
805
|
|
|
734
806
|
def _index(results: Mapping[str, CachedModule], names: Mapping[str, str]) -> SummaryIndex:
|
|
@@ -750,6 +822,7 @@ def _analyse_managers(
|
|
|
750
822
|
plugins: tuple[Plugin, ...],
|
|
751
823
|
affected: Affected,
|
|
752
824
|
suffixes: SuffixIndex,
|
|
825
|
+
root: Path,
|
|
753
826
|
) -> dict[str, CachedModule]:
|
|
754
827
|
"""Analyse one component: iterate its summaries to a fixpoint over ``seed`` (§21),
|
|
755
828
|
then extract what the rest of the run needs from each module. Each member registers
|
|
@@ -778,61 +851,124 @@ def _analyse_managers(
|
|
|
778
851
|
for manager in managers.values():
|
|
779
852
|
manager.run(ProjectSummariesUpdated)
|
|
780
853
|
manager.provide(ProjectSummaries, index)
|
|
781
|
-
return {name: _analyse_module(manager, plugins, affected, suffixes) for name, manager in managers.items()}
|
|
854
|
+
return {name: _analyse_module(manager, plugins, affected, suffixes, root) for name, manager in managers.items()}
|
|
782
855
|
|
|
783
856
|
|
|
784
857
|
@dataclass(frozen=True)
|
|
785
|
-
class
|
|
786
|
-
"""
|
|
787
|
-
|
|
788
|
-
|
|
858
|
+
class _Project:
|
|
859
|
+
"""What every worker process needs of the project, whatever the component it
|
|
860
|
+
analyses: it rebuilds the configuration from the project root and the plugin roots
|
|
861
|
+
once, when the pool starts it (``_start_worker``), and reads the finished modules
|
|
862
|
+
from ``spool``, a ``ProjectCache`` directory the parent writes after each wave."""
|
|
789
863
|
|
|
790
864
|
root: Path
|
|
791
865
|
plugin_roots: tuple[Path, ...]
|
|
792
866
|
plugins: tuple[Plugin, ...]
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
template_calls: TemplateCalls = field(default_factory=TemplateCalls)
|
|
799
|
-
# Every analysable module of the project, not only the component's: a curated
|
|
867
|
+
advisory_paths: tuple[Path, ...]
|
|
868
|
+
routes: tuple[tuple[str, str, str, int], ...]
|
|
869
|
+
escaped: tuple[str, ...]
|
|
870
|
+
template_calls: TemplateCalls
|
|
871
|
+
# Every analysable module of the project, not only a component's: a curated
|
|
800
872
|
# entry-point suffix matches derived symbols of any project module.
|
|
801
|
-
|
|
873
|
+
modules: tuple[str, ...]
|
|
874
|
+
spool: Path
|
|
802
875
|
|
|
803
876
|
|
|
804
|
-
|
|
877
|
+
@dataclass(frozen=True)
|
|
878
|
+
class _Batch:
|
|
879
|
+
"""One component handed to a worker process: its files under the engine's module
|
|
880
|
+
names, and the modules its summaries start from, each with its import name and the
|
|
881
|
+
key of its entry in the spool. Only these travel, not the summaries themselves."""
|
|
882
|
+
|
|
883
|
+
paths: Mapping[str, Path]
|
|
884
|
+
imported: Mapping[str, tuple[str, str]]
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
@dataclass
|
|
888
|
+
class _Worker:
|
|
889
|
+
"""The configuration a worker process prepared once, and the finished modules it has
|
|
890
|
+
read from the spool so far, by key."""
|
|
891
|
+
|
|
892
|
+
managed: Callable[[AnalysisManager], None]
|
|
893
|
+
plugins: tuple[Plugin, ...]
|
|
894
|
+
affected: Affected
|
|
895
|
+
suffixes: SuffixIndex
|
|
896
|
+
spool: ProjectCache
|
|
897
|
+
root: Path
|
|
898
|
+
read: dict[str, CachedModule] = field(default_factory=dict)
|
|
899
|
+
|
|
900
|
+
def entries(self, imported: Mapping[str, tuple[str, str]]) -> dict[str, CachedModule]:
|
|
901
|
+
found: dict[str, CachedModule] = {}
|
|
902
|
+
for name, (_, key) in imported.items():
|
|
903
|
+
entry = self.read.get(key)
|
|
904
|
+
if entry is None:
|
|
905
|
+
entry = self.spool.load(key)
|
|
906
|
+
if entry is None:
|
|
907
|
+
raise RuntimeError(f"the summaries of {name} are missing from {self.spool.directory}")
|
|
908
|
+
self.read[key] = entry
|
|
909
|
+
found[name] = entry
|
|
910
|
+
return found
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
# The state of a worker process, set by ``_start_worker`` when the pool starts it.
|
|
914
|
+
_WORKER: _Worker | None = None
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def _start_worker(project: _Project) -> None:
|
|
918
|
+
"""Prepare this worker process for every component it will analyse (#245)."""
|
|
919
|
+
|
|
920
|
+
global _WORKER
|
|
805
921
|
sources = SourceManager()
|
|
806
|
-
|
|
807
|
-
registry = load_plugins(
|
|
808
|
-
all_plugins: tuple[Plugin, ...] = (*(loaded.plugin for loaded in registry), *
|
|
809
|
-
dependencies = resolve_dependencies(
|
|
922
|
+
probe = _register_all(build_hir(sources.add_source("<empty>", "")))
|
|
923
|
+
registry = load_plugins(project.plugin_roots, probe)
|
|
924
|
+
all_plugins: tuple[Plugin, ...] = (*(loaded.plugin for loaded in registry), *project.plugins)
|
|
925
|
+
dependencies = resolve_dependencies(project.root, sources)
|
|
810
926
|
file_advisories: list[tuple[Path, tuple[Advisory, ...]]] = []
|
|
811
|
-
for path in
|
|
927
|
+
for path in project.advisory_paths:
|
|
812
928
|
try:
|
|
813
929
|
file_advisories.append((path, load_advisories(path)))
|
|
814
930
|
except AdvisoryFileError:
|
|
815
931
|
continue
|
|
816
|
-
advisories, _ = _merge_advisories(_contributions(registry,
|
|
932
|
+
advisories, _ = _merge_advisories(_contributions(registry, project.plugins, project.root, file_advisories))
|
|
817
933
|
affected = affected_symbols(dependencies, advisories)
|
|
818
|
-
models = plugin_models(all_plugins,
|
|
934
|
+
models = plugin_models(all_plugins, project.root).extended(
|
|
819
935
|
*advisory_sinks(affected), suffixes=advisory_suffix_sinks(affected)
|
|
820
936
|
)
|
|
821
|
-
routes = _decode_routes(
|
|
822
|
-
escaped = frozenset(
|
|
937
|
+
routes = _decode_routes(project.routes)
|
|
938
|
+
escaped = frozenset(project.escaped)
|
|
823
939
|
members = models.members_by_class()
|
|
824
|
-
|
|
940
|
+
signatures = models.signatures()
|
|
941
|
+
clearing = models.clearing(escaped)
|
|
942
|
+
|
|
943
|
+
def managed(manager: AnalysisManager) -> None:
|
|
825
944
|
manager.provide(SecurityModelAnalysis, models)
|
|
826
945
|
manager.provide(MembersAnalysis, members)
|
|
827
|
-
manager.provide(SignaturesAnalysis,
|
|
946
|
+
manager.provide(SignaturesAnalysis, signatures)
|
|
828
947
|
manager.provide(DependencyAnalysis, dependencies)
|
|
829
948
|
manager.provide(RegisteredRoutes, routes)
|
|
830
949
|
manager.provide(EscapedTemplates, escaped)
|
|
831
|
-
manager.provide(ClearingAnalysis,
|
|
832
|
-
manager.provide(TemplateCallsAnalysis,
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
950
|
+
manager.provide(ClearingAnalysis, clearing)
|
|
951
|
+
manager.provide(TemplateCallsAnalysis, project.template_calls)
|
|
952
|
+
|
|
953
|
+
_WORKER = _Worker(
|
|
954
|
+
managed,
|
|
955
|
+
tuple(p for p in all_plugins if not isinstance(p, ProjectPlugin)),
|
|
956
|
+
affected,
|
|
957
|
+
suffix_index(affected, project.modules),
|
|
958
|
+
ProjectCache(project.spool),
|
|
959
|
+
project.root,
|
|
960
|
+
)
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
def _analyse_batch(batch: _Batch) -> dict[str, dict[str, Any]]:
|
|
964
|
+
worker = _WORKER
|
|
965
|
+
assert worker is not None, "the pool starts every worker with _start_worker"
|
|
966
|
+
sources = SourceManager()
|
|
967
|
+
managers = {name: _register_all(build_hir(sources.load_file(path))) for name, path in batch.paths.items()}
|
|
968
|
+
for manager in managers.values():
|
|
969
|
+
worker.managed(manager)
|
|
970
|
+
seed = _index(worker.entries(batch.imported), {name: import_name for name, (import_name, _) in batch.imported.items()})
|
|
971
|
+
results = _analyse_managers(managers, seed, worker.plugins, worker.affected, worker.suffixes, worker.root)
|
|
836
972
|
return {name: encode(entry) for name, entry in results.items()}
|
|
837
973
|
|
|
838
974
|
|
|
@@ -882,12 +1018,12 @@ def _configuration_key(
|
|
|
882
1018
|
|
|
883
1019
|
|
|
884
1020
|
def _analyse_module(
|
|
885
|
-
manager: AnalysisManager, plugins: tuple[Plugin, ...], affected: Affected, suffixes: SuffixIndex
|
|
1021
|
+
manager: AnalysisManager, plugins: tuple[Plugin, ...], affected: Affected, suffixes: SuffixIndex, root: Path
|
|
886
1022
|
) -> CachedModule:
|
|
887
1023
|
"""One module's findings, summaries, call sites and symbol reads: what the cache
|
|
888
1024
|
keeps (§11)."""
|
|
889
1025
|
|
|
890
|
-
findings, supported = _check_module(manager, plugins)
|
|
1026
|
+
findings, supported = _check_module(manager, plugins, root)
|
|
891
1027
|
graph = manager.get(CallGraphAnalysis)
|
|
892
1028
|
correlated: list[Finding] = []
|
|
893
1029
|
if affected:
|
|
@@ -924,10 +1060,11 @@ def _summaries_of(manager: AnalysisManager) -> dict[str, FunctionSummary]:
|
|
|
924
1060
|
|
|
925
1061
|
|
|
926
1062
|
def _check_module(
|
|
927
|
-
manager: AnalysisManager, plugins: tuple[Plugin, ...]
|
|
1063
|
+
manager: AnalysisManager, plugins: tuple[Plugin, ...], root: Path | None
|
|
928
1064
|
) -> tuple[tuple[Finding, ...], tuple[nodes.Function, ...]]:
|
|
929
1065
|
"""Findings of the module plugins plus notes for unsupported functions, and the
|
|
930
|
-
functions that could be analysed
|
|
1066
|
+
functions that could be analysed; ``root`` is the root of the analysis the plugins
|
|
1067
|
+
read a file's path in the project from (``PluginContext.path``)."""
|
|
931
1068
|
|
|
932
1069
|
supported: list[nodes.Function] = []
|
|
933
1070
|
notes: list[Finding] = []
|
|
@@ -947,7 +1084,7 @@ def _check_module(
|
|
|
947
1084
|
)
|
|
948
1085
|
else:
|
|
949
1086
|
supported.append(function)
|
|
950
|
-
findings = run_plugins(manager, plugins, tuple(supported))
|
|
1087
|
+
findings = run_plugins(manager, plugins, tuple(supported), root)
|
|
951
1088
|
return (*findings, *notes), tuple(supported)
|
|
952
1089
|
|
|
953
1090
|
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Where the code of a finding lives, by its path in the project, as two separate facts:
|
|
2
|
+
its context, test code a project runs to check itself and does not ship (#244), and its
|
|
3
|
+
origin, vendored code, someone else's kept in the project (#243). A vendored project's
|
|
4
|
+
tests are both. Both are shown in the finding's metadata; what they change, if
|
|
5
|
+
anything, is the rule's decision, or a refiner plugin's."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path, PurePosixPath
|
|
11
|
+
|
|
12
|
+
TEST = "test"
|
|
13
|
+
VENDORED = "vendored"
|
|
14
|
+
|
|
15
|
+
_VENDORED_DIRECTORIES = frozenset({"third_party", "third-party", "vendor", "vendored"})
|
|
16
|
+
|
|
17
|
+
_TEST_DIRECTORIES = frozenset({"test", "tests", "testing", "fixtures", "__tests__"})
|
|
18
|
+
_TEST_FILE = re.compile(r"^(test_.*|.*_test|conftest)\.py$")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def is_test_code(path: str) -> bool:
|
|
22
|
+
"""Whether ``path`` is test code or a fixture: under a test directory, or named like
|
|
23
|
+
a test module."""
|
|
24
|
+
|
|
25
|
+
parts = PurePosixPath(path).parts
|
|
26
|
+
return bool(parts) and (
|
|
27
|
+
any(part in _TEST_DIRECTORIES for part in parts[:-1]) or _TEST_FILE.match(parts[-1]) is not None
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def code_context(path: str) -> str | None:
|
|
32
|
+
"""The context a finding at ``path`` carries: ``"test"`` for test code, ``None`` for
|
|
33
|
+
anything else. ``path`` is relative to the project root, so where the project sits
|
|
34
|
+
does not count."""
|
|
35
|
+
|
|
36
|
+
return TEST if is_test_code(path) else None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def code_origin(path: str) -> str | None:
|
|
40
|
+
"""The origin a finding at ``path`` carries: ``"vendored"`` under a ``third_party``,
|
|
41
|
+
``third-party``, ``vendor`` or ``vendored`` directory of the project, ``None`` for the
|
|
42
|
+
project's own code."""
|
|
43
|
+
|
|
44
|
+
return VENDORED if any(part in _VENDORED_DIRECTORIES for part in PurePosixPath(path).parts[:-1]) else None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def path_in_project(source_id: object, root: Path | None) -> str | None:
|
|
48
|
+
"""The path of the file ``source_id`` names from ``root``, the root of the analysis,
|
|
49
|
+
as the classifications above read it; ``None`` for a file outside ``root``. Without a
|
|
50
|
+
root, the source is taken as named: a source that is no file has no other path."""
|
|
51
|
+
|
|
52
|
+
if root is None:
|
|
53
|
+
return Path(str(source_id)).as_posix()
|
|
54
|
+
try:
|
|
55
|
+
return Path(str(source_id)).resolve().relative_to(root.resolve()).as_posix()
|
|
56
|
+
except ValueError:
|
|
57
|
+
return None
|