coretrace-python-analyzer 0.18.0__py3-none-any.whl → 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coretrace_python/__init__.py +1 -1
- coretrace_python/abstract/__init__.py +2 -0
- coretrace_python/abstract/heap.py +4 -1
- coretrace_python/cache.py +14 -7
- coretrace_python/dependency/advisories.py +4 -0
- coretrace_python/dependency/correlation.py +16 -3
- coretrace_python/dependency/graph.py +4 -1
- coretrace_python/engine.py +95 -28
- coretrace_python/findings/coverage.py +6 -4
- coretrace_python/interprocedural/__init__.py +2 -0
- coretrace_python/interprocedural/callgraph.py +67 -7
- coretrace_python/interprocedural/modulegraph.py +103 -9
- coretrace_python/semantic/scopes.py +2 -2
- coretrace_python/taint/__init__.py +2 -0
- coretrace_python/taint/engine.py +106 -9
- coretrace_python/taint/templates.py +279 -29
- {coretrace_python_analyzer-0.18.0.dist-info → coretrace_python_analyzer-0.20.0.dist-info}/METADATA +1 -1
- {coretrace_python_analyzer-0.18.0.dist-info → coretrace_python_analyzer-0.20.0.dist-info}/RECORD +22 -22
- {coretrace_python_analyzer-0.18.0.dist-info → coretrace_python_analyzer-0.20.0.dist-info}/WHEEL +0 -0
- {coretrace_python_analyzer-0.18.0.dist-info → coretrace_python_analyzer-0.20.0.dist-info}/entry_points.txt +0 -0
- {coretrace_python_analyzer-0.18.0.dist-info → coretrace_python_analyzer-0.20.0.dist-info}/licenses/LICENSE +0 -0
- {coretrace_python_analyzer-0.18.0.dist-info → coretrace_python_analyzer-0.20.0.dist-info}/licenses/NOTICE +0 -0
coretrace_python/__init__.py
CHANGED
|
@@ -8,6 +8,7 @@ from coretrace_python.abstract.constants import (
|
|
|
8
8
|
from coretrace_python.abstract.heap import (
|
|
9
9
|
ATTRIBUTES,
|
|
10
10
|
ELEMENTS,
|
|
11
|
+
KEYS,
|
|
11
12
|
MUTATORS,
|
|
12
13
|
AbstractObject,
|
|
13
14
|
AliasSet,
|
|
@@ -24,6 +25,7 @@ from coretrace_python.abstract.values import AbstractValue, Truth
|
|
|
24
25
|
__all__ = [
|
|
25
26
|
"ATTRIBUTES",
|
|
26
27
|
"ELEMENTS",
|
|
28
|
+
"KEYS",
|
|
27
29
|
"MUTATORS",
|
|
28
30
|
"AbstractObject",
|
|
29
31
|
"AbstractValue",
|
|
@@ -6,7 +6,8 @@ globals and imported symbols by name, and the fields loaded from an object by fi
|
|
|
6
6
|
Each value points to an ``AliasSet`` of objects, computed by a flow-insensitive
|
|
7
7
|
points-to fixpoint over the SSA form; each object has two ``HeapLocation`` fields,
|
|
8
8
|
``elements`` for its items and ``attributes`` for its attributes, field-insensitive
|
|
9
|
-
within each
|
|
9
|
+
within each, and a third, ``keys``, for the keys of a mapping, which only the taint
|
|
10
|
+
engine fills. Taint and dependence analyses key their states by these locations, so a
|
|
10
11
|
store, a mutating method call or a load on any alias reads and writes the same place.
|
|
11
12
|
"""
|
|
12
13
|
|
|
@@ -47,6 +48,8 @@ from coretrace_python.source import SourceSpan
|
|
|
47
48
|
|
|
48
49
|
ELEMENTS = "elements"
|
|
49
50
|
ATTRIBUTES = "attributes"
|
|
51
|
+
# The keys of a mapping, stored by the taint engine; the points-to solver never fills it.
|
|
52
|
+
KEYS = "keys"
|
|
50
53
|
|
|
51
54
|
# Method names that store their arguments into the receiver's elements.
|
|
52
55
|
MUTATORS = frozenset(
|
coretrace_python/cache.py
CHANGED
|
@@ -42,7 +42,7 @@ from coretrace_python.interprocedural import (
|
|
|
42
42
|
from coretrace_python.semantic.symbols import SymbolId
|
|
43
43
|
from coretrace_python.source import SourceId, SourceSpan
|
|
44
44
|
|
|
45
|
-
CACHE_FORMAT =
|
|
45
|
+
CACHE_FORMAT = 12
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
@dataclass(frozen=True)
|
|
@@ -134,7 +134,7 @@ class ProjectCache:
|
|
|
134
134
|
def encode(module: CachedModule) -> dict[str, Any]:
|
|
135
135
|
return {
|
|
136
136
|
"format": CACHE_FORMAT,
|
|
137
|
-
"functions": [[f.name, _encode_span(f.span), f.entry_point] for f in module.functions],
|
|
137
|
+
"functions": [[f.name, _encode_span(f.span), f.entry_point, list(f.aliases)] for f in module.functions],
|
|
138
138
|
"summaries": {name: _encode_summary(s) for name, s in module.summaries.items()},
|
|
139
139
|
"sites": [_encode_site(site) for site in module.sites],
|
|
140
140
|
"findings": [_encode_finding(finding) for finding in module.findings],
|
|
@@ -196,8 +196,13 @@ def _decode_span(data: Any) -> SourceSpan:
|
|
|
196
196
|
|
|
197
197
|
|
|
198
198
|
def _decode_function(data: Any) -> ModuleFunction:
|
|
199
|
-
name, span, entry_point = data
|
|
200
|
-
return ModuleFunction(
|
|
199
|
+
name, span, entry_point, aliases = data
|
|
200
|
+
return ModuleFunction(
|
|
201
|
+
_string(name),
|
|
202
|
+
_decode_span(span),
|
|
203
|
+
None if entry_point is None else _string(entry_point),
|
|
204
|
+
tuple(_string(alias) for alias in aliases),
|
|
205
|
+
)
|
|
201
206
|
|
|
202
207
|
|
|
203
208
|
def _decode_read(data: Any) -> SymbolRead:
|
|
@@ -363,6 +368,7 @@ def _encode_arguments(arguments: Arguments) -> dict[str, Any]:
|
|
|
363
368
|
"positional": list(arguments.positional),
|
|
364
369
|
"keywords": [[name, value] for name, value in arguments.keywords],
|
|
365
370
|
"unpacked": arguments.unpacked,
|
|
371
|
+
"keyword_unpacked": arguments.keyword_unpacked,
|
|
366
372
|
}
|
|
367
373
|
|
|
368
374
|
|
|
@@ -370,11 +376,12 @@ def _decode_arguments(data: Mapping[str, Any]) -> Arguments:
|
|
|
370
376
|
def denoted(value: Any) -> str | None:
|
|
371
377
|
return None if value is None else _string(value)
|
|
372
378
|
|
|
373
|
-
unpacked = data["unpacked"]
|
|
374
|
-
if not isinstance(unpacked, bool):
|
|
375
|
-
raise TypeError(f"expected
|
|
379
|
+
unpacked, keyword_unpacked = data["unpacked"], data["keyword_unpacked"]
|
|
380
|
+
if not isinstance(unpacked, bool) or not isinstance(keyword_unpacked, bool):
|
|
381
|
+
raise TypeError(f"expected booleans, got {unpacked!r} and {keyword_unpacked!r}")
|
|
376
382
|
return Arguments(
|
|
377
383
|
tuple(denoted(value) for value in data["positional"]),
|
|
378
384
|
tuple((_string(name), denoted(value)) for name, value in data["keywords"]),
|
|
379
385
|
unpacked,
|
|
386
|
+
keyword_unpacked,
|
|
380
387
|
)
|
|
@@ -263,4 +263,8 @@ def _condition(entry: Mapping[str, Any]) -> Condition:
|
|
|
263
263
|
(condition.argument is None and condition.position is None) or condition.values or condition.default
|
|
264
264
|
):
|
|
265
265
|
raise ValueError("a host condition names the URL argument, by keyword or position, and nothing else")
|
|
266
|
+
if condition.kind == "keyword_name" and (
|
|
267
|
+
condition.argument is not None or condition.position is not None or condition.values or condition.default
|
|
268
|
+
):
|
|
269
|
+
raise ValueError("a keyword_name condition carries its text and nothing else")
|
|
266
270
|
return condition
|
|
@@ -23,7 +23,7 @@ from coretrace_python.findings import Confidence, Finding, Severity
|
|
|
23
23
|
from coretrace_python.findings.refutation import Status, Verdict, Verdicts
|
|
24
24
|
from coretrace_python.interprocedural import Arguments, CallSite, ExternalSymbol
|
|
25
25
|
from coretrace_python.semantic.symbols import SymbolId
|
|
26
|
-
from coretrace_python.taint import Sink, TaintFlow, TaintKind
|
|
26
|
+
from coretrace_python.taint import KEYWORD_NAMES, Sink, TaintFlow, TaintKind
|
|
27
27
|
from coretrace_python.taint.urls import UrlProof, redirects_disabled
|
|
28
28
|
|
|
29
29
|
Affected = Mapping[SymbolId, tuple[Advisory, ...]]
|
|
@@ -72,13 +72,25 @@ def check_conditions(
|
|
|
72
72
|
a ``host`` condition, what ``url`` proves of the value the attacker's input is passed
|
|
73
73
|
in (``passed_as``). A host the URL's constant text fixes rules the call out when the
|
|
74
74
|
call disables redirects; otherwise a redirect may still lead to a host the attacker
|
|
75
|
-
controls, and the condition stays pending with that uncertainty.
|
|
75
|
+
controls, and the condition stays pending with that uncertainty. A ``keyword_name``
|
|
76
|
+
condition is met when the attacker's input is passed as the keys of a ``**`` mapping,
|
|
77
|
+
pending when the call expands a mapping whose keys are not established, and
|
|
78
|
+
contradicted by a call writing every keyword name."""
|
|
76
79
|
|
|
77
80
|
if entry is None:
|
|
78
81
|
return ConditionCheck()
|
|
79
82
|
met: list[Condition] = []
|
|
80
83
|
pending: list[Condition] = []
|
|
81
84
|
for condition in entry.conditions:
|
|
85
|
+
if condition.kind == "keyword_name":
|
|
86
|
+
if any(keyword == KEYWORD_NAMES for _, keyword in passed_as):
|
|
87
|
+
met.append(condition)
|
|
88
|
+
elif arguments is None or arguments.keyword_unpacked:
|
|
89
|
+
pending.append(condition)
|
|
90
|
+
else:
|
|
91
|
+
names = ", ".join(sorted({name for name, _ in arguments.keywords}))
|
|
92
|
+
return ConditionCheck(tuple(met), tuple(pending), condition, names or None)
|
|
93
|
+
continue
|
|
82
94
|
if condition.kind == "host":
|
|
83
95
|
origin = url.origin if url is not None and _passed_in(condition, passed_as) else None
|
|
84
96
|
if origin is None:
|
|
@@ -119,7 +131,8 @@ def ruled_out(module: str, site: CallSite, check: ConditionCheck) -> str:
|
|
|
119
131
|
python.yaml.load(Loader=python.yaml.SafeLoader)``."""
|
|
120
132
|
|
|
121
133
|
assert check.contradicted is not None and isinstance(site.target, ExternalSymbol)
|
|
122
|
-
|
|
134
|
+
contradicted = check.contradicted
|
|
135
|
+
argument = contradicted.argument or ("keywords" if contradicted.kind == "keyword_name" else f"#{contradicted.position}")
|
|
123
136
|
passed = f"{argument} absent" if check.passed is None else f"{argument}={check.passed}"
|
|
124
137
|
return f"{module}:{site.location.start_line} {site.target.symbol}({passed})"
|
|
125
138
|
|
|
@@ -154,7 +154,10 @@ class Condition:
|
|
|
154
154
|
``default`` says an absent argument means a vulnerable value. A ``host`` condition
|
|
155
155
|
says the attacker must choose the host of the URL passed as ``argument`` (or at
|
|
156
156
|
``position``); the engine decides it from what the URL's constant text proves. A
|
|
157
|
-
``
|
|
157
|
+
``keyword_name`` condition says the attacker must choose a keyword name of the call,
|
|
158
|
+
as an ORM alias expanded from a mapping; the engine decides it from the keys of what
|
|
159
|
+
the call expands with ``**``. A ``semantic`` condition cannot be checked and is
|
|
160
|
+
reported as pending review."""
|
|
158
161
|
|
|
159
162
|
kind: str
|
|
160
163
|
text: str
|
coretrace_python/engine.py
CHANGED
|
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|
|
9
9
|
import concurrent.futures
|
|
10
10
|
import hashlib
|
|
11
11
|
import multiprocessing
|
|
12
|
+
from collections import Counter
|
|
12
13
|
from collections.abc import Callable, Iterable, Mapping, Sequence
|
|
13
14
|
from dataclasses import dataclass, field, replace
|
|
14
15
|
from pathlib import Path
|
|
@@ -88,6 +89,7 @@ from coretrace_python.interprocedural import (
|
|
|
88
89
|
TemplateCallsAnalysis,
|
|
89
90
|
build_module_graph,
|
|
90
91
|
discover_sources,
|
|
92
|
+
name_modules,
|
|
91
93
|
project_symbol,
|
|
92
94
|
)
|
|
93
95
|
from coretrace_python.ir import PYIR_SCHEMA_VERSION
|
|
@@ -371,7 +373,11 @@ def analyze_project(
|
|
|
371
373
|
strongly connected components of the module graph, imports first; with ``jobs``
|
|
372
374
|
above one the components of a wave are analysed in that many processes (§29).
|
|
373
375
|
``advisory_files`` add to the ``advisories.json`` at ``root``; ``policy_file``
|
|
374
|
-
replaces the ``coretrace-policy.toml`` there.
|
|
376
|
+
replaces the ``coretrace-policy.toml`` there.
|
|
377
|
+
|
|
378
|
+
Every module is known by the name ``name_modules`` gives its file: its import name,
|
|
379
|
+
or its path from the root when another file has that import name. Such files carry
|
|
380
|
+
an ``ambiguous-module`` note and are covered as ``ambiguous``."""
|
|
375
381
|
|
|
376
382
|
if jobs < 1:
|
|
377
383
|
raise ValueError("jobs must be at least 1")
|
|
@@ -398,18 +404,18 @@ def analyze_project(
|
|
|
398
404
|
files: dict[str, SourceFile] = {}
|
|
399
405
|
unreadable: list[tuple[Path, str]] = []
|
|
400
406
|
coverage: list[FileCoverage] = []
|
|
401
|
-
|
|
407
|
+
named = name_modules(root, discover_sources(root, sources, unreadable))
|
|
402
408
|
for path, reason in unreadable:
|
|
403
409
|
findings.append(_note("syntax-error", f"{path}: {reason}", sources.add_source(str(path), ""), 1))
|
|
404
410
|
coverage.append(FileCoverage(str(path), "unreadable", 0, 0))
|
|
405
|
-
for source in
|
|
411
|
+
for name, source in named.items():
|
|
406
412
|
try:
|
|
407
|
-
modules[
|
|
413
|
+
modules[name] = build_hir(source)
|
|
408
414
|
except (ParseError, HIRBuildError) as error:
|
|
409
415
|
findings.append(_note("syntax-error", str(error), source, 1))
|
|
410
416
|
coverage.append(FileCoverage(str(source.source_id), "syntax-error", 0, 0))
|
|
411
417
|
continue
|
|
412
|
-
files[
|
|
418
|
+
files[name] = source
|
|
413
419
|
|
|
414
420
|
managers = {name: _register_all(module) for name, module in modules.items()}
|
|
415
421
|
probe = next(iter(managers.values()), None) or _register_all(build_hir(sources.add_source("<empty>", "")))
|
|
@@ -439,6 +445,8 @@ def analyze_project(
|
|
|
439
445
|
graph = build_module_graph(
|
|
440
446
|
{name: files[name] for name in analysable}, {name: modules[name] for name in analysable}, imports
|
|
441
447
|
)
|
|
448
|
+
shared = _shared_names(graph)
|
|
449
|
+
findings.extend(_ambiguity_notes(graph, shared, root))
|
|
442
450
|
routes: dict[SymbolId, EntryPoint] = {}
|
|
443
451
|
for name in sorted(analysable):
|
|
444
452
|
for symbol, registered in _routes_of(analysable[name]).items():
|
|
@@ -447,7 +455,8 @@ def analyze_project(
|
|
|
447
455
|
templates = project_templates(root)
|
|
448
456
|
clearing = models.clearing(escaped)
|
|
449
457
|
template_calls = models.template_calls(
|
|
450
|
-
templates.filters,
|
|
458
|
+
templates.filters,
|
|
459
|
+
request_processor((modules[name], analysable[name].get(ScopeAnalysis)) for name in sorted(analysable)),
|
|
451
460
|
)
|
|
452
461
|
for manager in analysable.values():
|
|
453
462
|
manager.provide(RegisteredRoutes, routes)
|
|
@@ -514,15 +523,16 @@ def analyze_project(
|
|
|
514
523
|
if pool is not None:
|
|
515
524
|
pool.shutdown()
|
|
516
525
|
|
|
517
|
-
index =
|
|
526
|
+
index = _index(results, {name: name for name in results})
|
|
518
527
|
call_graphs: dict[str, CallGraph] = {}
|
|
519
528
|
functions: dict[str, tuple[ModuleFunction, ...]] = {}
|
|
520
529
|
for name in sorted(analysable):
|
|
521
530
|
entry = results[name]
|
|
522
531
|
findings.extend(entry.findings)
|
|
523
532
|
unsupported = sum(1 for f in entry.findings if f.rule_id == "unsupported-syntax")
|
|
533
|
+
status = "ambiguous" if files[name].module_name in shared else "analysed"
|
|
524
534
|
coverage.append(
|
|
525
|
-
FileCoverage(str(files[name].source_id),
|
|
535
|
+
FileCoverage(str(files[name].source_id), status, len(entry.functions), len(entry.functions) - unsupported)
|
|
526
536
|
)
|
|
527
537
|
functions[name] = entry.functions
|
|
528
538
|
sites: dict[str, list[CallSite]] = {function.name: [] for function in entry.functions}
|
|
@@ -644,24 +654,76 @@ def _advisory_paths(root: Path, advisory_files: Sequence[Path]) -> tuple[Path, .
|
|
|
644
654
|
return (*([default] if default.is_file() else []), *advisory_files)
|
|
645
655
|
|
|
646
656
|
|
|
657
|
+
def _shared_names(graph: ModuleGraph) -> dict[str, list[str]]:
|
|
658
|
+
"""The import names several analysable files have, each with those files' modules."""
|
|
659
|
+
|
|
660
|
+
by_name: dict[str, list[str]] = {}
|
|
661
|
+
for name in graph.modules:
|
|
662
|
+
by_name.setdefault(graph.source(name).module_name, []).append(name)
|
|
663
|
+
return {import_name: names for import_name, names in by_name.items() if len(names) > 1}
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _ambiguity_notes(graph: ModuleGraph, shared: Mapping[str, list[str]], root: Path) -> list[Finding]:
|
|
667
|
+
"""One note on each file whose import name another file shares, since a symbol of
|
|
668
|
+
that name may denote a function of either, and one on each importer whose import of
|
|
669
|
+
such a name no root resolved, since calls through it are not followed."""
|
|
670
|
+
|
|
671
|
+
def located(name: str) -> str:
|
|
672
|
+
return _located(_path_of(graph.source(name)), root)
|
|
673
|
+
|
|
674
|
+
notes: list[Finding] = []
|
|
675
|
+
for import_name, names in sorted(shared.items()):
|
|
676
|
+
for name in names:
|
|
677
|
+
others = ", ".join(located(other) for other in names if other != name)
|
|
678
|
+
message = (
|
|
679
|
+
f"{located(name)} is the module '{import_name}' like {others}: a symbol "
|
|
680
|
+
f"python.{import_name}.* may name a function of either, and an import of "
|
|
681
|
+
f"'{import_name}' resolves only from its own root"
|
|
682
|
+
)
|
|
683
|
+
notes.append(_note("ambiguous-module", message, graph.source(name), 1))
|
|
684
|
+
for name in graph.modules:
|
|
685
|
+
for import_name in sorted(graph.unresolved(name)):
|
|
686
|
+
candidates = " or ".join(located(other) for other in shared[import_name])
|
|
687
|
+
message = f"'{import_name}' imported here is {candidates}: calls into it are not followed"
|
|
688
|
+
notes.append(_note("ambiguous-module", message, graph.source(name), 1))
|
|
689
|
+
return notes
|
|
690
|
+
|
|
691
|
+
|
|
647
692
|
def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: frozenset[str]) -> SummaryIndex:
|
|
648
693
|
"""The summaries a component starts from: those of every module it imports,
|
|
649
|
-
transitively, all final by the time its wave runs
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
694
|
+
transitively, all final by the time its wave runs, each under the import name the
|
|
695
|
+
component's code refers to it by. Of several files with one import name, only the
|
|
696
|
+
one a member imports directly, resolved in its own root, goes under that name: the
|
|
697
|
+
others are reached through summaries that inline them already, and a name no
|
|
698
|
+
member's root resolved is followed into neither file."""
|
|
699
|
+
|
|
700
|
+
wanted: set[str] = set()
|
|
701
|
+
pending = list(component)
|
|
702
|
+
while pending:
|
|
703
|
+
for imported in graph.imports(pending.pop()):
|
|
704
|
+
if imported in results and imported not in wanted:
|
|
705
|
+
wanted.add(imported)
|
|
706
|
+
pending.append(imported)
|
|
707
|
+
direct = {imported for member in component for imported in graph.imports(member)}
|
|
708
|
+
by_import: dict[str, list[str]] = {}
|
|
709
|
+
for name in wanted:
|
|
710
|
+
by_import.setdefault(graph.source(name).module_name, []).append(name)
|
|
711
|
+
names: dict[str, str] = {}
|
|
712
|
+
for import_name, modules in by_import.items():
|
|
713
|
+
chosen = modules if len(modules) == 1 else [name for name in modules if name in direct]
|
|
714
|
+
if len(chosen) == 1:
|
|
715
|
+
names[chosen[0]] = import_name
|
|
716
|
+
return _index(results, names)
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def _index(results: Mapping[str, CachedModule], names: Mapping[str, str]) -> SummaryIndex:
|
|
720
|
+
"""The summaries of the modules in ``names``, each function under the project symbol
|
|
721
|
+
of the module name given for its module."""
|
|
722
|
+
|
|
661
723
|
return SummaryIndex(
|
|
662
724
|
{
|
|
663
|
-
project_symbol(
|
|
664
|
-
for name in sorted(
|
|
725
|
+
project_symbol(module, function): summary
|
|
726
|
+
for name, module in sorted(names.items())
|
|
665
727
|
for function, summary in results[name].summaries.items()
|
|
666
728
|
}
|
|
667
729
|
)
|
|
@@ -674,8 +736,12 @@ def _analyse_managers(
|
|
|
674
736
|
affected: Affected,
|
|
675
737
|
) -> dict[str, CachedModule]:
|
|
676
738
|
"""Analyse one component: iterate its summaries to a fixpoint over ``seed`` (§21),
|
|
677
|
-
then extract what the rest of the run needs from each module.
|
|
739
|
+
then extract what the rest of the run needs from each module. Each member registers
|
|
740
|
+
its functions under its import name, what the others' code refers to it by; a name
|
|
741
|
+
two members share, a cycle across roots, is registered by neither."""
|
|
678
742
|
|
|
743
|
+
counts = Counter(manager.module.name for manager in managers.values())
|
|
744
|
+
names = {name: manager.module.name for name, manager in managers.items() if counts[manager.module.name] == 1}
|
|
679
745
|
index = seed
|
|
680
746
|
for manager in managers.values():
|
|
681
747
|
manager.provide(ProjectSummaries, index)
|
|
@@ -684,9 +750,9 @@ def _analyse_managers(
|
|
|
684
750
|
{
|
|
685
751
|
**seed.summaries,
|
|
686
752
|
**{
|
|
687
|
-
project_symbol(
|
|
688
|
-
for name,
|
|
689
|
-
for function, summary in _summaries_of(
|
|
753
|
+
project_symbol(module, function): summary
|
|
754
|
+
for name, module in names.items()
|
|
755
|
+
for function, summary in _summaries_of(managers[name]).items()
|
|
690
756
|
},
|
|
691
757
|
}
|
|
692
758
|
)
|
|
@@ -701,8 +767,9 @@ def _analyse_managers(
|
|
|
701
767
|
|
|
702
768
|
@dataclass(frozen=True)
|
|
703
769
|
class _Batch:
|
|
704
|
-
"""One component handed to a worker process
|
|
705
|
-
|
|
770
|
+
"""One component handed to a worker process, its files under the engine's module
|
|
771
|
+
names: it rebuilds the configuration from the project root and the plugin roots, so
|
|
772
|
+
only paths and the imported summaries travel."""
|
|
706
773
|
|
|
707
774
|
root: Path
|
|
708
775
|
plugin_roots: tuple[Path, ...]
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"""Analysis coverage: which files and functions a run actually looked at.
|
|
2
2
|
|
|
3
3
|
"No findings" only means something when the reader knows what was analysed. Each file
|
|
4
|
-
is ``analysed``,
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
is ``analysed``, ``ambiguous`` (analysed, but another file has its Python module name,
|
|
5
|
+
so a symbol of that name may denote a function of either and an import of it resolves
|
|
6
|
+
only from its own root), a ``syntax-error`` (the frontend rejected it) or ``unreadable``
|
|
7
|
+
(it could not be decoded); analysed and ambiguous files count their functions and how
|
|
8
|
+
many lowered. The templates the project names but the run could not read are listed,
|
|
9
|
+
each where the project names it: they may call any template filter.
|
|
8
10
|
"""
|
|
9
11
|
|
|
10
12
|
from __future__ import annotations
|
|
@@ -19,6 +19,7 @@ from coretrace_python.interprocedural.modulegraph import (
|
|
|
19
19
|
build_module_graph,
|
|
20
20
|
discover_files,
|
|
21
21
|
discover_sources,
|
|
22
|
+
name_modules,
|
|
22
23
|
project_symbol,
|
|
23
24
|
)
|
|
24
25
|
from coretrace_python.interprocedural.summaries import (
|
|
@@ -73,5 +74,6 @@ __all__ = [
|
|
|
73
74
|
"cleared_by",
|
|
74
75
|
"discover_files",
|
|
75
76
|
"discover_sources",
|
|
77
|
+
"name_modules",
|
|
76
78
|
"project_symbol",
|
|
77
79
|
]
|
|
@@ -18,6 +18,7 @@ from typing import ClassVar
|
|
|
18
18
|
from coretrace_python.analysis import Analysis, AnalysisContext, AnyAnalysis
|
|
19
19
|
from coretrace_python.cfg import CFGError
|
|
20
20
|
from coretrace_python.hir import nodes
|
|
21
|
+
from coretrace_python.ir.defuse import DefUse, def_use
|
|
21
22
|
from coretrace_python.ir.lowering import (
|
|
22
23
|
MODULE_BODY,
|
|
23
24
|
LoweringError,
|
|
@@ -26,6 +27,7 @@ from coretrace_python.ir.lowering import (
|
|
|
26
27
|
)
|
|
27
28
|
from coretrace_python.ir.model import (
|
|
28
29
|
Await,
|
|
30
|
+
BuildDict,
|
|
29
31
|
Call,
|
|
30
32
|
Constant,
|
|
31
33
|
FunctionIR,
|
|
@@ -68,11 +70,15 @@ class Arguments:
|
|
|
68
70
|
(``python.yaml.FullLoader``), a constant as Python writes it (``True``, ``'/static'``),
|
|
69
71
|
or None for anything else (a parameter, an expression). ``unpacked`` says the call
|
|
70
72
|
unpacks ``*args`` or ``**kwargs``, so an argument it does not give explicitly may
|
|
71
|
-
still be given; after ``*args`` no position is known, and ``positional`` is empty.
|
|
73
|
+
still be given; after ``*args`` no position is known, and ``positional`` is empty.
|
|
74
|
+
``keyword_unpacked`` says the call expands a ``**mapping`` whose keys it does not
|
|
75
|
+
write, so a keyword name may be anything; a literal with constant keys is written out
|
|
76
|
+
as its keywords, and ``unpacked`` stays true for it only when ``*args`` is also given."""
|
|
72
77
|
|
|
73
78
|
positional: tuple[str | None, ...] = ()
|
|
74
79
|
keywords: tuple[tuple[str, str | None], ...] = ()
|
|
75
80
|
unpacked: bool = False
|
|
81
|
+
keyword_unpacked: bool = False
|
|
76
82
|
|
|
77
83
|
def given(self, keyword: str | None, position: int | None = None) -> tuple[bool, str | None] | None:
|
|
78
84
|
"""Whether the call gives an argument, by ``keyword`` or at ``position``, and what it
|
|
@@ -132,12 +138,16 @@ class SignaturesAnalysis(Analysis[tuple[Signature, ...]]):
|
|
|
132
138
|
@dataclass(frozen=True)
|
|
133
139
|
class ModuleFunction:
|
|
134
140
|
"""One function of a module as a project plugin sees it: the name the call graph
|
|
135
|
-
gives it (``Class.method``, ``outer.inner``, ``<module>``), where it is,
|
|
136
|
-
of the entry point it is (``http`` for a route, ``argv`` for a command), if any
|
|
141
|
+
gives it (``Class.method``, ``outer.inner``, ``<module>``), where it is, the label
|
|
142
|
+
of the entry point it is (``http`` for a route, ``argv`` for a command), if any, and
|
|
143
|
+
the module-level ``aliases`` bound to it by assigning one name to another
|
|
144
|
+
(``main = actual``), transitively and in source order; a method or a nested
|
|
145
|
+
function has none."""
|
|
137
146
|
|
|
138
147
|
name: str
|
|
139
148
|
span: SourceSpan
|
|
140
149
|
entry_point: str | None = None
|
|
150
|
+
aliases: tuple[str, ...] = ()
|
|
141
151
|
|
|
142
152
|
|
|
143
153
|
class CallGraph:
|
|
@@ -241,8 +251,45 @@ def derive_symbols(
|
|
|
241
251
|
# passed as an argument is not recorded, so call sites stay small in the cache.
|
|
242
252
|
MAX_CONSTANT_LENGTH = 64
|
|
243
253
|
|
|
254
|
+
_DICT = SymbolId("python.builtins.dict")
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _keyword_entries(
|
|
258
|
+
value: Value, symbols: Mapping[Value, SymbolId], defs: Mapping[Value, Instruction], uses: DefUse
|
|
259
|
+
) -> tuple[tuple[str, Value], ...] | None:
|
|
260
|
+
"""The keywords a ``**value`` gives when every name is established: the string keys
|
|
261
|
+
of a dict literal, the keywords of ``dict(...)``, through what they unpack in turn;
|
|
262
|
+
None when a key is computed, the mapping is built elsewhere, or anything but the
|
|
263
|
+
call uses it, since a store or a mutating call may add a key the literal does not
|
|
264
|
+
write."""
|
|
265
|
+
|
|
266
|
+
if len(uses.uses(value)) != 1:
|
|
267
|
+
return None
|
|
268
|
+
made = defs.get(value)
|
|
269
|
+
entries: list[tuple[str, Value]] = []
|
|
270
|
+
if isinstance(made, BuildDict):
|
|
271
|
+
for key, item in made.items:
|
|
272
|
+
constant = defs.get(key)
|
|
273
|
+
if not isinstance(constant, Constant) or not isinstance(constant.value, str):
|
|
274
|
+
return None
|
|
275
|
+
entries.append((constant.value, item))
|
|
276
|
+
nested = made.unpacked
|
|
277
|
+
elif isinstance(made, Call) and symbols.get(made.callee) == _DICT and not made.arguments and not made.starred:
|
|
278
|
+
nested = tuple(v for name, v in made.keywords if name is None)
|
|
279
|
+
entries.extend((name, v) for name, v in made.keywords if name is not None)
|
|
280
|
+
else:
|
|
281
|
+
return None
|
|
282
|
+
for inner in nested:
|
|
283
|
+
found = _keyword_entries(inner, symbols, defs, uses)
|
|
284
|
+
if found is None:
|
|
285
|
+
return None
|
|
286
|
+
entries.extend(found)
|
|
287
|
+
return tuple(entries)
|
|
288
|
+
|
|
244
289
|
|
|
245
|
-
def _arguments(
|
|
290
|
+
def _arguments(
|
|
291
|
+
call: Call, symbols: Mapping[Value, SymbolId], defs: Mapping[Value, Instruction], uses: DefUse
|
|
292
|
+
) -> Arguments:
|
|
246
293
|
def denoted(value: Value) -> str | None:
|
|
247
294
|
symbol = symbols.get(value)
|
|
248
295
|
if symbol is not None:
|
|
@@ -253,10 +300,22 @@ def _arguments(call: Call, symbols: Mapping[Value, SymbolId], defs: Mapping[Valu
|
|
|
253
300
|
written = repr(made.value)
|
|
254
301
|
return written if len(written) <= MAX_CONSTANT_LENGTH else None
|
|
255
302
|
|
|
303
|
+
keywords: list[tuple[str, str | None]] = []
|
|
304
|
+
keyword_unpacked = False
|
|
305
|
+
for name, value in call.keywords:
|
|
306
|
+
if name is not None:
|
|
307
|
+
keywords.append((name, denoted(value)))
|
|
308
|
+
continue
|
|
309
|
+
entries = _keyword_entries(value, symbols, defs, uses)
|
|
310
|
+
if entries is None:
|
|
311
|
+
keyword_unpacked = True
|
|
312
|
+
else:
|
|
313
|
+
keywords.extend((key, denoted(item)) for key, item in entries)
|
|
256
314
|
return Arguments(
|
|
257
315
|
() if call.starred else tuple(denoted(value) for value in call.arguments),
|
|
258
|
-
tuple(
|
|
259
|
-
bool(call.starred) or
|
|
316
|
+
tuple(keywords),
|
|
317
|
+
bool(call.starred) or keyword_unpacked,
|
|
318
|
+
keyword_unpacked,
|
|
260
319
|
)
|
|
261
320
|
|
|
262
321
|
|
|
@@ -476,6 +535,7 @@ class CallGraphAnalysis(Analysis[CallGraph]):
|
|
|
476
535
|
table=table,
|
|
477
536
|
)
|
|
478
537
|
defs = {i.result: i for block in ssa.blocks for i in block.instructions if i.result is not None}
|
|
538
|
+
uses = def_use(ssa)
|
|
479
539
|
found: list[CallSite] = []
|
|
480
540
|
for block in ssa.blocks:
|
|
481
541
|
for instruction in block.instructions:
|
|
@@ -485,7 +545,7 @@ class CallGraphAnalysis(Analysis[CallGraph]):
|
|
|
485
545
|
name,
|
|
486
546
|
instruction.location,
|
|
487
547
|
targets.get(instruction.callee, UnknownTarget()),
|
|
488
|
-
_arguments(instruction, symbols[name], defs),
|
|
548
|
+
_arguments(instruction, symbols[name], defs, uses),
|
|
489
549
|
)
|
|
490
550
|
)
|
|
491
551
|
sites[name] = tuple(found)
|