coretrace-python-analyzer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coretrace_python/__init__.py +4 -0
- coretrace_python/__main__.py +4 -0
- coretrace_python/abstract/__init__.py +45 -0
- coretrace_python/abstract/constants.py +226 -0
- coretrace_python/abstract/heap.py +252 -0
- coretrace_python/abstract/ranges.py +285 -0
- coretrace_python/abstract/values.py +56 -0
- coretrace_python/analysis/__init__.py +31 -0
- coretrace_python/analysis/manager.py +165 -0
- coretrace_python/analysis/provider.py +73 -0
- coretrace_python/bundled/dependency/dependency_policy/dependency_policy.py +46 -0
- coretrace_python/bundled/dependency/dependency_policy/plugin.toml +9 -0
- coretrace_python/bundled/dependency/reachable_vulnerability/plugin.toml +9 -0
- coretrace_python/bundled/dependency/reachable_vulnerability/reachable_vulnerability.py +56 -0
- coretrace_python/bundled/dependency/sample_advisories/plugin.toml +9 -0
- coretrace_python/bundled/dependency/sample_advisories/sample_advisories.py +109 -0
- coretrace_python/bundled/dependency/vulnerable_dependency/plugin.toml +9 -0
- coretrace_python/bundled/dependency/vulnerable_dependency/vulnerable_dependency.py +43 -0
- coretrace_python/bundled/models/cli/cli_models.py +35 -0
- coretrace_python/bundled/models/cli/plugin.toml +9 -0
- coretrace_python/bundled/models/credentials/credential_models.py +43 -0
- coretrace_python/bundled/models/credentials/plugin.toml +9 -0
- coretrace_python/bundled/models/django/django_models.py +123 -0
- coretrace_python/bundled/models/django/plugin.toml +9 -0
- coretrace_python/bundled/models/fastapi/fastapi_models.py +29 -0
- coretrace_python/bundled/models/fastapi/plugin.toml +9 -0
- coretrace_python/bundled/models/flask/flask_models.py +53 -0
- coretrace_python/bundled/models/flask/plugin.toml +9 -0
- coretrace_python/bundled/models/http_clients/http_client_models.py +43 -0
- coretrace_python/bundled/models/http_clients/plugin.toml +9 -0
- coretrace_python/bundled/models/python_stdlib/plugin.toml +9 -0
- coretrace_python/bundled/models/python_stdlib/python_stdlib.py +68 -0
- coretrace_python/bundled/models/sqlalchemy/plugin.toml +9 -0
- coretrace_python/bundled/models/sqlalchemy/sqlalchemy_models.py +47 -0
- coretrace_python/bundled/secrets/config_secrets/config_secrets.py +38 -0
- coretrace_python/bundled/secrets/config_secrets/plugin.toml +9 -0
- coretrace_python/bundled/secrets/hardcoded_secrets/hardcoded_secrets.py +19 -0
- coretrace_python/bundled/secrets/hardcoded_secrets/plugin.toml +9 -0
- coretrace_python/bundled/security/command_injection/command_injection.py +17 -0
- coretrace_python/bundled/security/command_injection/plugin.toml +9 -0
- coretrace_python/bundled/security/insecure_deserialization/insecure_deserialization.py +17 -0
- coretrace_python/bundled/security/insecure_deserialization/plugin.toml +9 -0
- coretrace_python/bundled/security/open_redirect/open_redirect.py +17 -0
- coretrace_python/bundled/security/open_redirect/plugin.toml +9 -0
- coretrace_python/bundled/security/path_traversal/path_traversal.py +17 -0
- coretrace_python/bundled/security/path_traversal/plugin.toml +9 -0
- coretrace_python/bundled/security/plaintext_credentials/plaintext_credentials.py +21 -0
- coretrace_python/bundled/security/plaintext_credentials/plugin.toml +9 -0
- coretrace_python/bundled/security/sql_injection/plugin.toml +9 -0
- coretrace_python/bundled/security/sql_injection/sql_injection.py +17 -0
- coretrace_python/bundled/security/ssrf/plugin.toml +9 -0
- coretrace_python/bundled/security/ssrf/ssrf.py +17 -0
- coretrace_python/bundled/security/xss/plugin.toml +9 -0
- coretrace_python/bundled/security/xss/xss.py +17 -0
- coretrace_python/bundled/syntax/dangerous_eval/dangerous_eval.py +19 -0
- coretrace_python/bundled/syntax/dangerous_eval/plugin.toml +9 -0
- coretrace_python/bundled/syntax/flask_debug/flask_debug.py +63 -0
- coretrace_python/bundled/syntax/flask_debug/plugin.toml +9 -0
- coretrace_python/bundled/syntax/missing_timeout/missing_timeout.py +51 -0
- coretrace_python/bundled/syntax/missing_timeout/plugin.toml +9 -0
- coretrace_python/bundled/syntax/weak_crypto/plugin.toml +9 -0
- coretrace_python/bundled/syntax/weak_crypto/weak_crypto.py +19 -0
- coretrace_python/cache.py +310 -0
- coretrace_python/cfg/__init__.py +46 -0
- coretrace_python/cfg/builder.py +589 -0
- coretrace_python/cfg/dominance.py +183 -0
- coretrace_python/cfg/model.py +166 -0
- coretrace_python/cli.py +216 -0
- coretrace_python/dataflow/__init__.py +29 -0
- coretrace_python/dataflow/lattice.py +78 -0
- coretrace_python/dataflow/solver.py +96 -0
- coretrace_python/dependency/__init__.py +44 -0
- coretrace_python/dependency/advisories.py +168 -0
- coretrace_python/dependency/correlation.py +87 -0
- coretrace_python/dependency/graph.py +274 -0
- coretrace_python/dependency/policy.py +65 -0
- coretrace_python/dependency/sbom.py +65 -0
- coretrace_python/engine.py +688 -0
- coretrace_python/findings/__init__.py +13 -0
- coretrace_python/findings/coverage.py +45 -0
- coretrace_python/findings/model.py +49 -0
- coretrace_python/findings/refutation.py +427 -0
- coretrace_python/frontend/__init__.py +18 -0
- coretrace_python/frontend/ast_adapter.py +447 -0
- coretrace_python/frontend/parser.py +23 -0
- coretrace_python/hir/__init__.py +5 -0
- coretrace_python/hir/nodes.py +584 -0
- coretrace_python/hir/visitors.py +36 -0
- coretrace_python/interprocedural/__init__.py +49 -0
- coretrace_python/interprocedural/callgraph.py +291 -0
- coretrace_python/interprocedural/modulegraph.py +218 -0
- coretrace_python/interprocedural/summaries.py +463 -0
- coretrace_python/ir/__init__.py +5 -0
- coretrace_python/ir/defuse.py +74 -0
- coretrace_python/ir/lowering.py +575 -0
- coretrace_python/ir/model.py +481 -0
- coretrace_python/ir/printer.py +201 -0
- coretrace_python/ir/ssa.py +277 -0
- coretrace_python/plugins/__init__.py +58 -0
- coretrace_python/plugins/api.py +153 -0
- coretrace_python/plugins/detectors.py +114 -0
- coretrace_python/plugins/loader.py +84 -0
- coretrace_python/plugins/manifest.py +112 -0
- coretrace_python/plugins/registry.py +34 -0
- coretrace_python/plugins/secrets.py +330 -0
- coretrace_python/reporters/__init__.py +22 -0
- coretrace_python/reporters/json_format.py +48 -0
- coretrace_python/reporters/report.py +23 -0
- coretrace_python/reporters/sarif.py +70 -0
- coretrace_python/reporters/text.py +24 -0
- coretrace_python/semantic/__init__.py +9 -0
- coretrace_python/semantic/identity.py +39 -0
- coretrace_python/semantic/imports.py +131 -0
- coretrace_python/semantic/scopes.py +473 -0
- coretrace_python/semantic/symbols.py +87 -0
- coretrace_python/source/__init__.py +13 -0
- coretrace_python/source/manager.py +81 -0
- coretrace_python/source/model.py +57 -0
- coretrace_python/taint/__init__.py +55 -0
- coretrace_python/taint/engine.py +800 -0
- coretrace_python/taint/models.py +317 -0
- coretrace_python/taint/routes.py +99 -0
- coretrace_python_analyzer-0.1.0.dist-info/METADATA +74 -0
- coretrace_python_analyzer-0.1.0.dist-info/RECORD +126 -0
- coretrace_python_analyzer-0.1.0.dist-info/WHEEL +4 -0
- coretrace_python_analyzer-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Worklist solver over control-flow graphs (architecture §37 dataflow/worklist, solver).
|
|
2
|
+
|
|
3
|
+
A problem sees, for each block, the states arriving on its executable incoming edges
|
|
4
|
+
(keyed by the block they come from, or ``ENTRY`` for the initial state) and returns the
|
|
5
|
+
states it sends along outgoing edges. Edges a problem does not return are pruned: the
|
|
6
|
+
blocks behind them stay unreached unless another edge reaches them.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from enum import Enum
|
|
14
|
+
from types import MappingProxyType
|
|
15
|
+
from typing import ClassVar, Generic, TypeVar
|
|
16
|
+
|
|
17
|
+
from coretrace_python.cfg import CFG, BlockId
|
|
18
|
+
|
|
19
|
+
S = TypeVar("S")
|
|
20
|
+
|
|
21
|
+
ENTRY = BlockId("<entry>")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Direction(Enum):
|
|
25
|
+
FORWARD = "forward"
|
|
26
|
+
BACKWARD = "backward"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DataflowProblem(ABC, Generic[S]):
|
|
30
|
+
direction: ClassVar[Direction] = Direction.FORWARD
|
|
31
|
+
|
|
32
|
+
@abstractmethod
|
|
33
|
+
def initial(self) -> S:
|
|
34
|
+
"""State entering the entry block (forward) or leaving the exits (backward)."""
|
|
35
|
+
|
|
36
|
+
@abstractmethod
|
|
37
|
+
def join(self, a: S, b: S) -> S:
|
|
38
|
+
"""Combine the states of two incoming edges."""
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
def flow(self, cfg: CFG, block: BlockId, incoming: Mapping[BlockId, S]) -> Mapping[BlockId, S]:
|
|
42
|
+
"""States sent to the next blocks in the flow direction; omitted edges are pruned."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Solution(Generic[S]):
|
|
46
|
+
def __init__(self, problem: DataflowProblem[S], edges: Mapping[BlockId, Mapping[BlockId, S]]):
|
|
47
|
+
self._problem = problem
|
|
48
|
+
self._incoming: Mapping[BlockId, Mapping[BlockId, S]] = MappingProxyType(
|
|
49
|
+
{block: MappingProxyType(dict(found)) for block, found in edges.items()}
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def incoming(self, block: BlockId) -> Mapping[BlockId, S]:
|
|
53
|
+
return self._incoming.get(block, MappingProxyType({}))
|
|
54
|
+
|
|
55
|
+
def reached(self, block: BlockId) -> bool:
|
|
56
|
+
return bool(self._incoming.get(block))
|
|
57
|
+
|
|
58
|
+
def state(self, block: BlockId) -> S:
|
|
59
|
+
"""Join of every state arriving at ``block``; raises for an unreached block."""
|
|
60
|
+
|
|
61
|
+
states = list(self.incoming(block).values())
|
|
62
|
+
if not states:
|
|
63
|
+
raise KeyError(block)
|
|
64
|
+
result = states[0]
|
|
65
|
+
for other in states[1:]:
|
|
66
|
+
result = self._problem.join(result, other)
|
|
67
|
+
return result
|
|
68
|
+
|
|
69
|
+
def edge(self, source: BlockId, target: BlockId) -> S:
|
|
70
|
+
return self._incoming[target][source]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def solve(problem: DataflowProblem[S], cfg: CFG) -> Solution[S]:
|
|
74
|
+
forward = problem.direction is Direction.FORWARD
|
|
75
|
+
order = {block: position for position, block in enumerate(cfg.blocks)}
|
|
76
|
+
if forward:
|
|
77
|
+
starts = [cfg.entry]
|
|
78
|
+
else:
|
|
79
|
+
starts = [b for b in cfg.blocks if b in cfg.reachable() and not cfg.successors(b)]
|
|
80
|
+
|
|
81
|
+
incoming: dict[BlockId, dict[BlockId, S]] = {start: {ENTRY: problem.initial()} for start in starts}
|
|
82
|
+
worklist = list(starts)
|
|
83
|
+
queued = set(starts)
|
|
84
|
+
while worklist:
|
|
85
|
+
worklist.sort(key=lambda block: order[block], reverse=True)
|
|
86
|
+
block = worklist.pop()
|
|
87
|
+
queued.discard(block)
|
|
88
|
+
for target, state in problem.flow(cfg, block, MappingProxyType(incoming[block])).items():
|
|
89
|
+
edges = incoming.setdefault(target, {})
|
|
90
|
+
if block in edges and edges[block] == state:
|
|
91
|
+
continue
|
|
92
|
+
edges[block] = state
|
|
93
|
+
if target not in queued:
|
|
94
|
+
worklist.append(target)
|
|
95
|
+
queued.add(target)
|
|
96
|
+
return Solution(problem, incoming)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Dependency resolution and advisories (architecture §26)."""
|
|
2
|
+
|
|
3
|
+
from coretrace_python.dependency.advisories import (
|
|
4
|
+
ADVISORY_FILE,
|
|
5
|
+
AdvisoryFileError,
|
|
6
|
+
dump_advisories,
|
|
7
|
+
import_osv,
|
|
8
|
+
load_advisories,
|
|
9
|
+
read_osv,
|
|
10
|
+
)
|
|
11
|
+
from coretrace_python.dependency.graph import (
|
|
12
|
+
DEPENDENCY_FILES,
|
|
13
|
+
Advisory,
|
|
14
|
+
DependencyAnalysis,
|
|
15
|
+
DependencyGraph,
|
|
16
|
+
Requirement,
|
|
17
|
+
Version,
|
|
18
|
+
normalize,
|
|
19
|
+
parse_dependencies,
|
|
20
|
+
)
|
|
21
|
+
from coretrace_python.dependency.policy import POLICY_FILE, Policy, apply_policy, load_policy
|
|
22
|
+
from coretrace_python.dependency.sbom import render_sbom
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"ADVISORY_FILE",
|
|
26
|
+
"DEPENDENCY_FILES",
|
|
27
|
+
"POLICY_FILE",
|
|
28
|
+
"Advisory",
|
|
29
|
+
"AdvisoryFileError",
|
|
30
|
+
"DependencyAnalysis",
|
|
31
|
+
"DependencyGraph",
|
|
32
|
+
"Policy",
|
|
33
|
+
"Requirement",
|
|
34
|
+
"Version",
|
|
35
|
+
"apply_policy",
|
|
36
|
+
"dump_advisories",
|
|
37
|
+
"import_osv",
|
|
38
|
+
"load_advisories",
|
|
39
|
+
"load_policy",
|
|
40
|
+
"normalize",
|
|
41
|
+
"parse_dependencies",
|
|
42
|
+
"read_osv",
|
|
43
|
+
"render_sbom",
|
|
44
|
+
]
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Local advisory files and the OSV import (architecture §26).
|
|
2
|
+
|
|
3
|
+
The analysis never touches the network. ``import_osv`` converts the records of a public
|
|
4
|
+
OSV dump into ``Advisory`` values, keeping the PyPI ecosystem and turning each range of
|
|
5
|
+
events into a version specifier; ``dump_advisories`` writes them as a small JSON file
|
|
6
|
+
that a project keeps at its root as ``advisories.json`` or passes with ``--advisories``.
|
|
7
|
+
OSV records name no affected APIs, so imported advisories feed the requirement checks
|
|
8
|
+
and the SBOM; a file completed by hand with ``affected_symbols`` also feeds the
|
|
9
|
+
reachability and correlation checks.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import zipfile
|
|
16
|
+
from collections.abc import Iterable, Iterator, Mapping
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from coretrace_python.dependency.graph import Advisory, normalize
|
|
21
|
+
from coretrace_python.findings import Severity
|
|
22
|
+
from coretrace_python.semantic.symbols import SymbolId
|
|
23
|
+
|
|
24
|
+
ADVISORY_FILE = "advisories.json"
|
|
25
|
+
ADVISORY_SCHEMA = 1
|
|
26
|
+
|
|
27
|
+
_SEVERITIES = {
|
|
28
|
+
"LOW": Severity.LOW,
|
|
29
|
+
"MODERATE": Severity.MEDIUM,
|
|
30
|
+
"MEDIUM": Severity.MEDIUM,
|
|
31
|
+
"HIGH": Severity.HIGH,
|
|
32
|
+
"CRITICAL": Severity.CRITICAL,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class AdvisoryFileError(Exception):
|
|
37
|
+
"""An advisory or policy file could not be read."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# --------------------------------------------------------------------------- OSV
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def read_osv(path: Path) -> Iterator[Mapping[str, Any]]:
|
|
44
|
+
"""The records of an OSV dump: one JSON file (a record or a list), a directory of
|
|
45
|
+
JSON files, or a zip archive of them."""
|
|
46
|
+
|
|
47
|
+
if path.is_dir():
|
|
48
|
+
for file in sorted(path.glob("*.json")):
|
|
49
|
+
yield from _records(json.loads(file.read_text(encoding="utf-8")))
|
|
50
|
+
elif path.suffix == ".zip":
|
|
51
|
+
with zipfile.ZipFile(path) as archive:
|
|
52
|
+
for name in sorted(archive.namelist()):
|
|
53
|
+
if name.endswith(".json"):
|
|
54
|
+
yield from _records(json.loads(archive.read(name).decode("utf-8")))
|
|
55
|
+
else:
|
|
56
|
+
yield from _records(json.loads(path.read_text(encoding="utf-8")))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _records(data: Any) -> Iterator[Mapping[str, Any]]:
|
|
60
|
+
if isinstance(data, list):
|
|
61
|
+
for item in data:
|
|
62
|
+
if isinstance(item, Mapping):
|
|
63
|
+
yield item
|
|
64
|
+
elif isinstance(data, Mapping):
|
|
65
|
+
yield data
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def import_osv(records: Iterable[Mapping[str, Any]]) -> tuple[Advisory, ...]:
|
|
69
|
+
"""The PyPI advisories of OSV records, one per affected range."""
|
|
70
|
+
|
|
71
|
+
advisories: list[Advisory] = []
|
|
72
|
+
for record in records:
|
|
73
|
+
identifier = record.get("id")
|
|
74
|
+
if not isinstance(identifier, str):
|
|
75
|
+
continue
|
|
76
|
+
summary = str(record.get("summary") or "").strip()
|
|
77
|
+
if not summary:
|
|
78
|
+
lines = str(record.get("details") or "").strip().splitlines()
|
|
79
|
+
summary = lines[0] if lines else identifier
|
|
80
|
+
aliases = tuple(str(a) for a in record.get("aliases") or [])
|
|
81
|
+
severity = _severity(record)
|
|
82
|
+
for affected in record.get("affected") or []:
|
|
83
|
+
package = (affected.get("package") or {}) if isinstance(affected, Mapping) else {}
|
|
84
|
+
if str(package.get("ecosystem", "")).lower() != "pypi" or not package.get("name"):
|
|
85
|
+
continue
|
|
86
|
+
name = normalize(str(package["name"]))
|
|
87
|
+
for specifier in _specifiers(affected.get("ranges") or []):
|
|
88
|
+
advisories.append(Advisory(identifier, name, specifier, summary, severity, (), aliases))
|
|
89
|
+
return tuple(advisories)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _severity(record: Mapping[str, Any]) -> Severity:
|
|
93
|
+
specific = record.get("database_specific") or {}
|
|
94
|
+
label = specific.get("severity") if isinstance(specific, Mapping) else None
|
|
95
|
+
return _SEVERITIES.get(str(label).upper(), Severity.MEDIUM)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _specifiers(ranges: Iterable[Mapping[str, Any]]) -> list[str]:
|
|
99
|
+
found: list[str] = []
|
|
100
|
+
for entry in ranges:
|
|
101
|
+
if entry.get("type") not in ("ECOSYSTEM", "SEMVER"):
|
|
102
|
+
continue
|
|
103
|
+
introduced: str | None = None
|
|
104
|
+
for event in entry.get("events") or []:
|
|
105
|
+
if "introduced" in event:
|
|
106
|
+
introduced = str(event["introduced"])
|
|
107
|
+
elif "fixed" in event or "last_affected" in event:
|
|
108
|
+
bound = f"<{event['fixed']}" if "fixed" in event else f"<={event['last_affected']}"
|
|
109
|
+
found.append(_clause(introduced, bound))
|
|
110
|
+
introduced = None
|
|
111
|
+
if introduced is not None:
|
|
112
|
+
found.append(_clause(introduced, None))
|
|
113
|
+
return found
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _clause(introduced: str | None, bound: str | None) -> str:
|
|
117
|
+
parts: list[str] = []
|
|
118
|
+
if introduced is not None and introduced != "0":
|
|
119
|
+
parts.append(f">={introduced}")
|
|
120
|
+
if bound is not None:
|
|
121
|
+
parts.append(bound)
|
|
122
|
+
return ",".join(parts) if parts else ">=0"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# --------------------------------------------------------------------------- local file
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def dump_advisories(advisories: Iterable[Advisory]) -> str:
|
|
129
|
+
document = {
|
|
130
|
+
"schema": ADVISORY_SCHEMA,
|
|
131
|
+
"advisories": [
|
|
132
|
+
{
|
|
133
|
+
"id": a.id,
|
|
134
|
+
"package": a.package,
|
|
135
|
+
"vulnerable": a.vulnerable,
|
|
136
|
+
"summary": a.summary,
|
|
137
|
+
"severity": a.severity.value,
|
|
138
|
+
"affected_symbols": [str(s) for s in a.affected_symbols],
|
|
139
|
+
"aliases": list(a.aliases),
|
|
140
|
+
}
|
|
141
|
+
for a in advisories
|
|
142
|
+
],
|
|
143
|
+
}
|
|
144
|
+
return json.dumps(document, indent=2) + "\n"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def load_advisories(path: Path) -> tuple[Advisory, ...]:
|
|
148
|
+
try:
|
|
149
|
+
document = json.loads(path.read_text(encoding="utf-8"))
|
|
150
|
+
if document.get("schema") != ADVISORY_SCHEMA:
|
|
151
|
+
raise AdvisoryFileError(f"{path}: unsupported advisory schema {document.get('schema')!r}")
|
|
152
|
+
return tuple(_advisory(entry) for entry in document["advisories"])
|
|
153
|
+
except AdvisoryFileError:
|
|
154
|
+
raise
|
|
155
|
+
except (OSError, ValueError, KeyError, TypeError, AttributeError) as error:
|
|
156
|
+
raise AdvisoryFileError(f"{path}: {error}") from error
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _advisory(entry: Mapping[str, Any]) -> Advisory:
|
|
160
|
+
return Advisory(
|
|
161
|
+
str(entry["id"]),
|
|
162
|
+
normalize(str(entry["package"])),
|
|
163
|
+
str(entry["vulnerable"]),
|
|
164
|
+
str(entry["summary"]),
|
|
165
|
+
Severity(entry["severity"]),
|
|
166
|
+
tuple(SymbolId(str(s)) for s in entry.get("affected_symbols") or []),
|
|
167
|
+
tuple(str(a) for a in entry.get("aliases") or []),
|
|
168
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Correlation engine (architecture §27).
|
|
2
|
+
|
|
3
|
+
A package required in a vulnerable version, an API the advisory affects, a call to that
|
|
4
|
+
API reachable in the project, and attacker-controlled data reaching that call: the
|
|
5
|
+
affected APIs become sinks of the ``ADVISORY`` taint kind, so the shared taint engine,
|
|
6
|
+
the function summaries and the refutation engine do the work, and the flows they leave
|
|
7
|
+
are correlated here into one high-confidence ``exploitable-vulnerability`` finding.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Iterable, Mapping
|
|
13
|
+
|
|
14
|
+
from coretrace_python.dependency.graph import Advisory, DependencyGraph
|
|
15
|
+
from coretrace_python.findings import Confidence, Finding, Severity
|
|
16
|
+
from coretrace_python.findings.refutation import Status, Verdicts
|
|
17
|
+
from coretrace_python.semantic.symbols import SymbolId
|
|
18
|
+
from coretrace_python.taint import Sink, TaintFlow, TaintKind
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def affected_symbols(
|
|
22
|
+
dependencies: DependencyGraph, advisories: Iterable[Advisory]
|
|
23
|
+
) -> Mapping[SymbolId, Advisory]:
|
|
24
|
+
"""The APIs affected by advisories whose package is required in a vulnerable version."""
|
|
25
|
+
|
|
26
|
+
affected: dict[SymbolId, Advisory] = {}
|
|
27
|
+
for requirement in dependencies.requirements:
|
|
28
|
+
for advisory in advisories:
|
|
29
|
+
if advisory.affects(requirement):
|
|
30
|
+
for symbol in advisory.affected_symbols:
|
|
31
|
+
affected.setdefault(symbol, advisory)
|
|
32
|
+
return affected
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def advisory_sinks(affected: Mapping[SymbolId, Advisory]) -> tuple[Sink, ...]:
|
|
36
|
+
return tuple(Sink(symbol, TaintKind.ADVISORY) for symbol in affected)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def correlate(
|
|
40
|
+
function: str,
|
|
41
|
+
flows: Iterable[TaintFlow],
|
|
42
|
+
verdicts: Verdicts | None,
|
|
43
|
+
affected: Mapping[SymbolId, Advisory],
|
|
44
|
+
) -> tuple[Finding, ...]:
|
|
45
|
+
"""Exploitable-vulnerability findings for the non-refuted ADVISORY flows of a function."""
|
|
46
|
+
|
|
47
|
+
findings: list[Finding] = []
|
|
48
|
+
for flow in flows:
|
|
49
|
+
if not flow.kinds & TaintKind.ADVISORY:
|
|
50
|
+
continue
|
|
51
|
+
advisory = affected.get(flow.sink.symbol)
|
|
52
|
+
if advisory is None:
|
|
53
|
+
continue
|
|
54
|
+
verdict = verdicts.verdict(flow) if verdicts is not None else None
|
|
55
|
+
if verdict is not None and verdict.status is Status.REFUTED:
|
|
56
|
+
continue
|
|
57
|
+
hotspot = verdict is not None and verdict.status is Status.HOTSPOT
|
|
58
|
+
message = (
|
|
59
|
+
f"{advisory.id}: {flow.source.label} input reaches {flow.sink.symbol}, affected in "
|
|
60
|
+
f"the required {advisory.package} {advisory.vulnerable}: {advisory.summary}"
|
|
61
|
+
)
|
|
62
|
+
metadata = {
|
|
63
|
+
"advisory": advisory.id,
|
|
64
|
+
"package": advisory.package,
|
|
65
|
+
"symbol": str(flow.sink.symbol),
|
|
66
|
+
"source": str(flow.source.symbol),
|
|
67
|
+
"source_label": flow.source.label,
|
|
68
|
+
"verdict": "hotspot" if hotspot else "vulnerability",
|
|
69
|
+
}
|
|
70
|
+
if verdict is not None:
|
|
71
|
+
metadata["evidence"] = verdict.evidence
|
|
72
|
+
if flow.through is not None and flow.sink_location is not None:
|
|
73
|
+
message += f" through {flow.through}"
|
|
74
|
+
metadata["through"] = flow.through
|
|
75
|
+
metadata["sink_line"] = str(flow.sink_location.start_line)
|
|
76
|
+
findings.append(
|
|
77
|
+
Finding(
|
|
78
|
+
rule_id="exploitable-vulnerability",
|
|
79
|
+
message=message,
|
|
80
|
+
severity=Severity.CRITICAL,
|
|
81
|
+
confidence=Confidence.MEDIUM if hotspot else Confidence.HIGH,
|
|
82
|
+
span=flow.location,
|
|
83
|
+
function=function,
|
|
84
|
+
metadata=metadata,
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
return tuple(findings)
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
"""Dependency resolution (architecture §26).
|
|
2
|
+
|
|
3
|
+
Requirements declared in ``requirements.txt``, ``pyproject.toml`` (PEP 621 and Poetry)
|
|
4
|
+
and pinned in ``poetry.lock`` or ``uv.lock`` become a ``DependencyGraph``. Versions are
|
|
5
|
+
compared with a small PEP 440 subset (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``,
|
|
6
|
+
``~=`` and Poetry's ``^``), enough to decide whether a requirement may allow a version an
|
|
7
|
+
advisory marks as vulnerable. Nothing is downloaded.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
import tomllib
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import PurePath
|
|
17
|
+
from types import MappingProxyType
|
|
18
|
+
from typing import Any, ClassVar
|
|
19
|
+
|
|
20
|
+
from coretrace_python.analysis import Analysis, AnalysisContext
|
|
21
|
+
from coretrace_python.findings import Severity
|
|
22
|
+
from coretrace_python.semantic.symbols import SymbolId
|
|
23
|
+
from coretrace_python.source import SourceFile, SourceId, SourceSpan
|
|
24
|
+
|
|
25
|
+
_REQUIREMENT = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*(\[[^\]]*\])?\s*(.*)$")
|
|
26
|
+
_CLAUSE = re.compile(r"^(===|==|!=|<=|>=|~=|<|>|\^)\s*([0-9][0-9A-Za-z.*+!-]*)$")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def normalize(name: str) -> str:
|
|
30
|
+
return re.sub(r"[-_.]+", "-", name).lower()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True, order=True)
|
|
34
|
+
class Version:
|
|
35
|
+
parts: tuple[int, ...]
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def parse(cls, text: str) -> Version:
|
|
39
|
+
parts: list[int] = []
|
|
40
|
+
for piece in text.strip().split("."):
|
|
41
|
+
digits = re.match(r"\d+", piece)
|
|
42
|
+
if digits is None:
|
|
43
|
+
break
|
|
44
|
+
parts.append(int(digits.group()))
|
|
45
|
+
if digits.end() != len(piece):
|
|
46
|
+
break
|
|
47
|
+
return cls(tuple(parts) or (0,))
|
|
48
|
+
|
|
49
|
+
def padded(self, length: int) -> tuple[int, ...]:
|
|
50
|
+
return self.parts + (0,) * (length - len(self.parts))
|
|
51
|
+
|
|
52
|
+
def satisfies(self, specifier: str) -> bool:
|
|
53
|
+
for clause in [c.strip() for c in specifier.split(",") if c.strip()]:
|
|
54
|
+
match = _CLAUSE.match(clause)
|
|
55
|
+
if match is None:
|
|
56
|
+
continue
|
|
57
|
+
operator, wanted = match.group(1), Version.parse(match.group(2))
|
|
58
|
+
if not self._satisfies_clause(operator, wanted, match.group(2)):
|
|
59
|
+
return False
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
def _satisfies_clause(self, operator: str, wanted: Version, raw: str) -> bool:
|
|
63
|
+
length = max(len(self.parts), len(wanted.parts))
|
|
64
|
+
mine, theirs = self.padded(length), wanted.padded(length)
|
|
65
|
+
if operator in ("==", "==="):
|
|
66
|
+
if raw.endswith(".*"):
|
|
67
|
+
prefix = wanted.parts
|
|
68
|
+
return self.parts[: len(prefix)] == prefix
|
|
69
|
+
return mine == theirs
|
|
70
|
+
if operator == "!=":
|
|
71
|
+
return mine != theirs
|
|
72
|
+
if operator == "<":
|
|
73
|
+
return mine < theirs
|
|
74
|
+
if operator == "<=":
|
|
75
|
+
return mine <= theirs
|
|
76
|
+
if operator == ">":
|
|
77
|
+
return mine > theirs
|
|
78
|
+
if operator == ">=":
|
|
79
|
+
return mine >= theirs
|
|
80
|
+
if operator == "~=":
|
|
81
|
+
ceiling = list(wanted.parts[:-1]) if len(wanted.parts) > 1 else [wanted.parts[0]]
|
|
82
|
+
ceiling[-1] += 1
|
|
83
|
+
return mine >= theirs and self.padded(len(ceiling)) < tuple(ceiling)
|
|
84
|
+
if operator == "^":
|
|
85
|
+
ceiling = [0] * len(wanted.parts)
|
|
86
|
+
for index, part in enumerate(wanted.parts):
|
|
87
|
+
if part != 0 or index == len(wanted.parts) - 1:
|
|
88
|
+
ceiling[index] = part + 1
|
|
89
|
+
break
|
|
90
|
+
return mine >= theirs and self.padded(len(ceiling)) < tuple(ceiling)
|
|
91
|
+
return True
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _lower_bounds(specifier: str) -> list[Version]:
|
|
95
|
+
bounds: list[Version] = []
|
|
96
|
+
for clause in [c.strip() for c in specifier.split(",") if c.strip()]:
|
|
97
|
+
match = _CLAUSE.match(clause)
|
|
98
|
+
if match is not None and match.group(1) in (">=", ">", "~=", "^", "==", "==="):
|
|
99
|
+
bounds.append(Version.parse(match.group(2)))
|
|
100
|
+
return bounds
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass(frozen=True)
|
|
104
|
+
class Requirement:
|
|
105
|
+
name: str
|
|
106
|
+
specifier: str
|
|
107
|
+
span: SourceSpan
|
|
108
|
+
pinned: Version | None = None
|
|
109
|
+
optional: bool = False
|
|
110
|
+
|
|
111
|
+
@classmethod
|
|
112
|
+
def parse(cls, text: str, source_id: SourceId, line: int, optional: bool = False) -> Requirement | None:
|
|
113
|
+
cleaned = text.split("#", 1)[0].split(";", 1)[0].strip()
|
|
114
|
+
match = _REQUIREMENT.match(cleaned)
|
|
115
|
+
if match is None or not match.group(1):
|
|
116
|
+
return None
|
|
117
|
+
specifier = ",".join(part.strip() for part in match.group(3).split(",") if part.strip())
|
|
118
|
+
pinned = None
|
|
119
|
+
clauses = [c for c in specifier.split(",") if c]
|
|
120
|
+
if len(clauses) == 1 and clauses[0].startswith("==") and not clauses[0].endswith(".*"):
|
|
121
|
+
pinned = Version.parse(clauses[0].lstrip("="))
|
|
122
|
+
return cls(normalize(match.group(1)), specifier, SourceSpan(source_id, line, 1), pinned, optional)
|
|
123
|
+
|
|
124
|
+
def may_match(self, vulnerable: str) -> bool:
|
|
125
|
+
"""Whether some version this requirement allows is in the ``vulnerable`` range."""
|
|
126
|
+
|
|
127
|
+
if self.pinned is not None:
|
|
128
|
+
return self.pinned.satisfies(vulnerable)
|
|
129
|
+
candidates = [Version((0,)), *_lower_bounds(self.specifier), *_lower_bounds(vulnerable)]
|
|
130
|
+
return any(v.satisfies(self.specifier) and v.satisfies(vulnerable) for v in candidates)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass(frozen=True)
|
|
134
|
+
class Advisory:
|
|
135
|
+
id: str
|
|
136
|
+
package: str
|
|
137
|
+
vulnerable: str
|
|
138
|
+
summary: str
|
|
139
|
+
severity: Severity
|
|
140
|
+
affected_symbols: tuple[SymbolId, ...] = ()
|
|
141
|
+
aliases: tuple[str, ...] = ()
|
|
142
|
+
|
|
143
|
+
def affects(self, requirement: Requirement) -> bool:
|
|
144
|
+
return requirement.name == normalize(self.package) and requirement.may_match(self.vulnerable)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class DependencyGraph:
|
|
148
|
+
def __init__(self, requirements: Mapping[str, Requirement] | None = None, errors: tuple[str, ...] = ()) -> None:
|
|
149
|
+
self._requirements = MappingProxyType(dict(sorted((requirements or {}).items())))
|
|
150
|
+
self.errors = errors
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def names(self) -> tuple[str, ...]:
|
|
154
|
+
return tuple(self._requirements)
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def requirements(self) -> tuple[Requirement, ...]:
|
|
158
|
+
return tuple(self._requirements.values())
|
|
159
|
+
|
|
160
|
+
def requirement(self, name: str) -> Requirement | None:
|
|
161
|
+
return self._requirements.get(normalize(name))
|
|
162
|
+
|
|
163
|
+
def merge(self, other: DependencyGraph) -> DependencyGraph:
|
|
164
|
+
merged = dict(self._requirements)
|
|
165
|
+
for name, requirement in other._requirements.items():
|
|
166
|
+
current = merged.get(name)
|
|
167
|
+
if current is None:
|
|
168
|
+
merged[name] = requirement
|
|
169
|
+
continue
|
|
170
|
+
merged[name] = Requirement(
|
|
171
|
+
name,
|
|
172
|
+
current.specifier or requirement.specifier,
|
|
173
|
+
current.span if current.specifier else requirement.span,
|
|
174
|
+
requirement.pinned or current.pinned,
|
|
175
|
+
current.optional and requirement.optional,
|
|
176
|
+
)
|
|
177
|
+
return DependencyGraph(merged, self.errors + other.errors)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def parse_dependencies(source: SourceFile) -> DependencyGraph:
|
|
181
|
+
"""The requirements declared or pinned by one dependency file; other files are empty."""
|
|
182
|
+
|
|
183
|
+
name = source.path.name if source.path is not None else PurePath(str(source.source_id)).name
|
|
184
|
+
if name.startswith("requirements") and name.endswith(".txt"):
|
|
185
|
+
return _parse_requirements_txt(source)
|
|
186
|
+
if name == "pyproject.toml":
|
|
187
|
+
return _parse_toml(source, _pyproject_requirements)
|
|
188
|
+
if name in ("poetry.lock", "uv.lock"):
|
|
189
|
+
return _parse_toml(source, _lock_requirements)
|
|
190
|
+
return DependencyGraph()
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _parse_requirements_txt(source: SourceFile) -> DependencyGraph:
|
|
194
|
+
found: dict[str, Requirement] = {}
|
|
195
|
+
for number, line in enumerate(source.text.splitlines(), start=1):
|
|
196
|
+
stripped = line.strip()
|
|
197
|
+
if not stripped or stripped.startswith(("#", "-")):
|
|
198
|
+
continue
|
|
199
|
+
requirement = Requirement.parse(stripped, source.source_id, number)
|
|
200
|
+
if requirement is not None:
|
|
201
|
+
found[requirement.name] = requirement
|
|
202
|
+
return DependencyGraph(found)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _parse_toml(source: SourceFile, extract: Any) -> DependencyGraph:
|
|
206
|
+
try:
|
|
207
|
+
data = tomllib.loads(source.text)
|
|
208
|
+
except tomllib.TOMLDecodeError as error:
|
|
209
|
+
return DependencyGraph(errors=(f"{source.source_id}: {error}",))
|
|
210
|
+
found: dict[str, Requirement] = {}
|
|
211
|
+
for requirement in extract(data, source):
|
|
212
|
+
found[requirement.name] = requirement
|
|
213
|
+
return DependencyGraph(found)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _line_of(source: SourceFile, key: str, default: int = 1) -> int:
|
|
217
|
+
for number, line in enumerate(source.text.splitlines(), start=1):
|
|
218
|
+
if key in line:
|
|
219
|
+
return number
|
|
220
|
+
return default
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _pyproject_requirements(data: Mapping[str, Any], source: SourceFile) -> list[Requirement]:
|
|
224
|
+
found: list[Requirement] = []
|
|
225
|
+
project = data.get("project", {})
|
|
226
|
+
for text in project.get("dependencies", []) or []:
|
|
227
|
+
requirement = Requirement.parse(text, source.source_id, _line_of(source, text))
|
|
228
|
+
if requirement is not None:
|
|
229
|
+
found.append(requirement)
|
|
230
|
+
for group in (project.get("optional-dependencies", {}) or {}).values():
|
|
231
|
+
for text in group or []:
|
|
232
|
+
requirement = Requirement.parse(text, source.source_id, _line_of(source, text), True)
|
|
233
|
+
if requirement is not None:
|
|
234
|
+
found.append(requirement)
|
|
235
|
+
poetry = data.get("tool", {}).get("poetry", {})
|
|
236
|
+
for section, optional in (("dependencies", False), ("dev-dependencies", True)):
|
|
237
|
+
for name, spec in (poetry.get(section, {}) or {}).items():
|
|
238
|
+
if normalize(name) == "python":
|
|
239
|
+
continue
|
|
240
|
+
specifier = spec.get("version", "") if isinstance(spec, dict) else str(spec)
|
|
241
|
+
if specifier == "*":
|
|
242
|
+
specifier = ""
|
|
243
|
+
requirement = Requirement.parse(
|
|
244
|
+
f"{name}{specifier}", source.source_id, _line_of(source, name), optional
|
|
245
|
+
)
|
|
246
|
+
if requirement is not None:
|
|
247
|
+
found.append(requirement)
|
|
248
|
+
return found
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _lock_requirements(data: Mapping[str, Any], source: SourceFile) -> list[Requirement]:
|
|
252
|
+
found: list[Requirement] = []
|
|
253
|
+
for package in data.get("package", []) or []:
|
|
254
|
+
name, version = package.get("name"), package.get("version")
|
|
255
|
+
if not isinstance(name, str) or not isinstance(version, str):
|
|
256
|
+
continue
|
|
257
|
+
line = _line_of(source, f'name = "{name}"')
|
|
258
|
+
found.append(
|
|
259
|
+
Requirement(normalize(name), "", SourceSpan(source.source_id, line, 1), Version.parse(version))
|
|
260
|
+
)
|
|
261
|
+
return found
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class DependencyAnalysis(Analysis[DependencyGraph]):
|
|
265
|
+
"""The project's dependency graph, provided by the engine; empty for a lone file."""
|
|
266
|
+
|
|
267
|
+
name: ClassVar[str] = "dependency.graph"
|
|
268
|
+
|
|
269
|
+
@classmethod
|
|
270
|
+
def compute(cls, ctx: AnalysisContext) -> DependencyGraph:
|
|
271
|
+
return DependencyGraph()
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
DEPENDENCY_FILES = ("pyproject.toml", "poetry.lock", "uv.lock")
|