coretrace-python-analyzer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coretrace_python/__init__.py +4 -0
- coretrace_python/__main__.py +4 -0
- coretrace_python/abstract/__init__.py +45 -0
- coretrace_python/abstract/constants.py +226 -0
- coretrace_python/abstract/heap.py +252 -0
- coretrace_python/abstract/ranges.py +285 -0
- coretrace_python/abstract/values.py +56 -0
- coretrace_python/analysis/__init__.py +31 -0
- coretrace_python/analysis/manager.py +165 -0
- coretrace_python/analysis/provider.py +73 -0
- coretrace_python/bundled/dependency/dependency_policy/dependency_policy.py +46 -0
- coretrace_python/bundled/dependency/dependency_policy/plugin.toml +9 -0
- coretrace_python/bundled/dependency/reachable_vulnerability/plugin.toml +9 -0
- coretrace_python/bundled/dependency/reachable_vulnerability/reachable_vulnerability.py +56 -0
- coretrace_python/bundled/dependency/sample_advisories/plugin.toml +9 -0
- coretrace_python/bundled/dependency/sample_advisories/sample_advisories.py +109 -0
- coretrace_python/bundled/dependency/vulnerable_dependency/plugin.toml +9 -0
- coretrace_python/bundled/dependency/vulnerable_dependency/vulnerable_dependency.py +43 -0
- coretrace_python/bundled/models/cli/cli_models.py +35 -0
- coretrace_python/bundled/models/cli/plugin.toml +9 -0
- coretrace_python/bundled/models/credentials/credential_models.py +43 -0
- coretrace_python/bundled/models/credentials/plugin.toml +9 -0
- coretrace_python/bundled/models/django/django_models.py +123 -0
- coretrace_python/bundled/models/django/plugin.toml +9 -0
- coretrace_python/bundled/models/fastapi/fastapi_models.py +29 -0
- coretrace_python/bundled/models/fastapi/plugin.toml +9 -0
- coretrace_python/bundled/models/flask/flask_models.py +53 -0
- coretrace_python/bundled/models/flask/plugin.toml +9 -0
- coretrace_python/bundled/models/http_clients/http_client_models.py +43 -0
- coretrace_python/bundled/models/http_clients/plugin.toml +9 -0
- coretrace_python/bundled/models/python_stdlib/plugin.toml +9 -0
- coretrace_python/bundled/models/python_stdlib/python_stdlib.py +68 -0
- coretrace_python/bundled/models/sqlalchemy/plugin.toml +9 -0
- coretrace_python/bundled/models/sqlalchemy/sqlalchemy_models.py +47 -0
- coretrace_python/bundled/secrets/config_secrets/config_secrets.py +38 -0
- coretrace_python/bundled/secrets/config_secrets/plugin.toml +9 -0
- coretrace_python/bundled/secrets/hardcoded_secrets/hardcoded_secrets.py +19 -0
- coretrace_python/bundled/secrets/hardcoded_secrets/plugin.toml +9 -0
- coretrace_python/bundled/security/command_injection/command_injection.py +17 -0
- coretrace_python/bundled/security/command_injection/plugin.toml +9 -0
- coretrace_python/bundled/security/insecure_deserialization/insecure_deserialization.py +17 -0
- coretrace_python/bundled/security/insecure_deserialization/plugin.toml +9 -0
- coretrace_python/bundled/security/open_redirect/open_redirect.py +17 -0
- coretrace_python/bundled/security/open_redirect/plugin.toml +9 -0
- coretrace_python/bundled/security/path_traversal/path_traversal.py +17 -0
- coretrace_python/bundled/security/path_traversal/plugin.toml +9 -0
- coretrace_python/bundled/security/plaintext_credentials/plaintext_credentials.py +21 -0
- coretrace_python/bundled/security/plaintext_credentials/plugin.toml +9 -0
- coretrace_python/bundled/security/sql_injection/plugin.toml +9 -0
- coretrace_python/bundled/security/sql_injection/sql_injection.py +17 -0
- coretrace_python/bundled/security/ssrf/plugin.toml +9 -0
- coretrace_python/bundled/security/ssrf/ssrf.py +17 -0
- coretrace_python/bundled/security/xss/plugin.toml +9 -0
- coretrace_python/bundled/security/xss/xss.py +17 -0
- coretrace_python/bundled/syntax/dangerous_eval/dangerous_eval.py +19 -0
- coretrace_python/bundled/syntax/dangerous_eval/plugin.toml +9 -0
- coretrace_python/bundled/syntax/flask_debug/flask_debug.py +63 -0
- coretrace_python/bundled/syntax/flask_debug/plugin.toml +9 -0
- coretrace_python/bundled/syntax/missing_timeout/missing_timeout.py +51 -0
- coretrace_python/bundled/syntax/missing_timeout/plugin.toml +9 -0
- coretrace_python/bundled/syntax/weak_crypto/plugin.toml +9 -0
- coretrace_python/bundled/syntax/weak_crypto/weak_crypto.py +19 -0
- coretrace_python/cache.py +310 -0
- coretrace_python/cfg/__init__.py +46 -0
- coretrace_python/cfg/builder.py +589 -0
- coretrace_python/cfg/dominance.py +183 -0
- coretrace_python/cfg/model.py +166 -0
- coretrace_python/cli.py +216 -0
- coretrace_python/dataflow/__init__.py +29 -0
- coretrace_python/dataflow/lattice.py +78 -0
- coretrace_python/dataflow/solver.py +96 -0
- coretrace_python/dependency/__init__.py +44 -0
- coretrace_python/dependency/advisories.py +168 -0
- coretrace_python/dependency/correlation.py +87 -0
- coretrace_python/dependency/graph.py +274 -0
- coretrace_python/dependency/policy.py +65 -0
- coretrace_python/dependency/sbom.py +65 -0
- coretrace_python/engine.py +688 -0
- coretrace_python/findings/__init__.py +13 -0
- coretrace_python/findings/coverage.py +45 -0
- coretrace_python/findings/model.py +49 -0
- coretrace_python/findings/refutation.py +427 -0
- coretrace_python/frontend/__init__.py +18 -0
- coretrace_python/frontend/ast_adapter.py +447 -0
- coretrace_python/frontend/parser.py +23 -0
- coretrace_python/hir/__init__.py +5 -0
- coretrace_python/hir/nodes.py +584 -0
- coretrace_python/hir/visitors.py +36 -0
- coretrace_python/interprocedural/__init__.py +49 -0
- coretrace_python/interprocedural/callgraph.py +291 -0
- coretrace_python/interprocedural/modulegraph.py +218 -0
- coretrace_python/interprocedural/summaries.py +463 -0
- coretrace_python/ir/__init__.py +5 -0
- coretrace_python/ir/defuse.py +74 -0
- coretrace_python/ir/lowering.py +575 -0
- coretrace_python/ir/model.py +481 -0
- coretrace_python/ir/printer.py +201 -0
- coretrace_python/ir/ssa.py +277 -0
- coretrace_python/plugins/__init__.py +58 -0
- coretrace_python/plugins/api.py +153 -0
- coretrace_python/plugins/detectors.py +114 -0
- coretrace_python/plugins/loader.py +84 -0
- coretrace_python/plugins/manifest.py +112 -0
- coretrace_python/plugins/registry.py +34 -0
- coretrace_python/plugins/secrets.py +330 -0
- coretrace_python/reporters/__init__.py +22 -0
- coretrace_python/reporters/json_format.py +48 -0
- coretrace_python/reporters/report.py +23 -0
- coretrace_python/reporters/sarif.py +70 -0
- coretrace_python/reporters/text.py +24 -0
- coretrace_python/semantic/__init__.py +9 -0
- coretrace_python/semantic/identity.py +39 -0
- coretrace_python/semantic/imports.py +131 -0
- coretrace_python/semantic/scopes.py +473 -0
- coretrace_python/semantic/symbols.py +87 -0
- coretrace_python/source/__init__.py +13 -0
- coretrace_python/source/manager.py +81 -0
- coretrace_python/source/model.py +57 -0
- coretrace_python/taint/__init__.py +55 -0
- coretrace_python/taint/engine.py +800 -0
- coretrace_python/taint/models.py +317 -0
- coretrace_python/taint/routes.py +99 -0
- coretrace_python_analyzer-0.1.0.dist-info/METADATA +74 -0
- coretrace_python_analyzer-0.1.0.dist-info/RECORD +126 -0
- coretrace_python_analyzer-0.1.0.dist-info/WHEEL +4 -0
- coretrace_python_analyzer-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Plugin manifests (architecture §14, §33).
|
|
2
|
+
|
|
3
|
+
A manifest is a ``plugin.toml`` file next to the plugin module::
|
|
4
|
+
|
|
5
|
+
name = "sql-injection"
|
|
6
|
+
version = "1.0.0"
|
|
7
|
+
plugin_api = ">=1,<2"
|
|
8
|
+
requires = ["semantic.symbols", "analysis.taint"]
|
|
9
|
+
provides = ["vulnerability.sql-injection"]
|
|
10
|
+
|
|
11
|
+
[entrypoint]
|
|
12
|
+
module = "sql_injection"
|
|
13
|
+
class = "SQLInjectionPlugin"
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
import tomllib
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ManifestError(Exception):
|
|
25
|
+
"""A manifest that cannot be read, or that disagrees with its plugin."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
_CONSTRAINT = re.compile(r"^(>=|<=|==|>|<)?(\d+)$")
|
|
29
|
+
_COMPARE = {
|
|
30
|
+
"==": lambda actual, wanted: actual == wanted,
|
|
31
|
+
">=": lambda actual, wanted: actual >= wanted,
|
|
32
|
+
"<=": lambda actual, wanted: actual <= wanted,
|
|
33
|
+
">": lambda actual, wanted: actual > wanted,
|
|
34
|
+
"<": lambda actual, wanted: actual < wanted,
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class VersionRange:
|
|
40
|
+
"""Integer API version constraints such as ``">=1,<2"`` or ``"1"``."""
|
|
41
|
+
|
|
42
|
+
spec: str
|
|
43
|
+
constraints: tuple[tuple[str, int], ...]
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def parse(cls, spec: str) -> VersionRange:
|
|
47
|
+
constraints: list[tuple[str, int]] = []
|
|
48
|
+
for part in spec.split(","):
|
|
49
|
+
match = _CONSTRAINT.match(part.strip())
|
|
50
|
+
if match is None:
|
|
51
|
+
raise ManifestError(f"invalid plugin_api range: {spec!r}")
|
|
52
|
+
operator, number = match.groups()
|
|
53
|
+
constraints.append((operator or "==", int(number)))
|
|
54
|
+
return cls(spec, tuple(constraints))
|
|
55
|
+
|
|
56
|
+
def contains(self, version: int) -> bool:
|
|
57
|
+
return all(_COMPARE[operator](version, wanted) for operator, wanted in self.constraints)
|
|
58
|
+
|
|
59
|
+
def __str__(self) -> str:
|
|
60
|
+
return self.spec
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class Entrypoint:
|
|
65
|
+
module: str
|
|
66
|
+
class_name: str
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class PluginManifest:
|
|
71
|
+
name: str
|
|
72
|
+
version: str
|
|
73
|
+
plugin_api: VersionRange
|
|
74
|
+
requires: tuple[str, ...]
|
|
75
|
+
provides: tuple[str, ...]
|
|
76
|
+
entrypoint: Entrypoint
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def load_manifest(path: Path) -> PluginManifest:
|
|
80
|
+
try:
|
|
81
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
82
|
+
except (OSError, tomllib.TOMLDecodeError) as error:
|
|
83
|
+
raise ManifestError(f"{path}: {error}") from error
|
|
84
|
+
|
|
85
|
+
def string(key: str) -> str:
|
|
86
|
+
value = data.get(key)
|
|
87
|
+
if not isinstance(value, str) or not value:
|
|
88
|
+
raise ManifestError(f"{path}: missing or invalid field '{key}'")
|
|
89
|
+
return value
|
|
90
|
+
|
|
91
|
+
def strings(key: str) -> tuple[str, ...]:
|
|
92
|
+
value = data.get(key)
|
|
93
|
+
if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
|
|
94
|
+
raise ManifestError(f"{path}: missing or invalid field '{key}'")
|
|
95
|
+
return tuple(value)
|
|
96
|
+
|
|
97
|
+
entrypoint = data.get("entrypoint")
|
|
98
|
+
if not isinstance(entrypoint, dict):
|
|
99
|
+
raise ManifestError(f"{path}: missing or invalid field 'entrypoint'")
|
|
100
|
+
module = entrypoint.get("module")
|
|
101
|
+
class_name = entrypoint.get("class")
|
|
102
|
+
if not isinstance(module, str) or not isinstance(class_name, str):
|
|
103
|
+
raise ManifestError(f"{path}: entrypoint needs 'module' and 'class'")
|
|
104
|
+
|
|
105
|
+
return PluginManifest(
|
|
106
|
+
name=string("name"),
|
|
107
|
+
version=string("version"),
|
|
108
|
+
plugin_api=VersionRange.parse(string("plugin_api")),
|
|
109
|
+
requires=strings("requires"),
|
|
110
|
+
provides=strings("provides"),
|
|
111
|
+
entrypoint=Entrypoint(module, class_name),
|
|
112
|
+
)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Registry of loaded plugins, indexed by name and by provided capability."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
|
|
7
|
+
from coretrace_python.plugins.loader import LoadedPlugin
|
|
8
|
+
from coretrace_python.plugins.manifest import ManifestError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PluginRegistry:
|
|
12
|
+
def __init__(self) -> None:
|
|
13
|
+
self._by_name: dict[str, LoadedPlugin] = {}
|
|
14
|
+
self._by_capability: dict[str, list[LoadedPlugin]] = {}
|
|
15
|
+
|
|
16
|
+
def add(self, loaded: LoadedPlugin) -> None:
|
|
17
|
+
name = loaded.manifest.name
|
|
18
|
+
if name in self._by_name:
|
|
19
|
+
raise ManifestError(f"plugin {name!r} is already registered")
|
|
20
|
+
self._by_name[name] = loaded
|
|
21
|
+
for capability in loaded.manifest.provides:
|
|
22
|
+
self._by_capability.setdefault(capability, []).append(loaded)
|
|
23
|
+
|
|
24
|
+
def plugin(self, name: str) -> LoadedPlugin:
|
|
25
|
+
return self._by_name[name]
|
|
26
|
+
|
|
27
|
+
def providers(self, capability: str) -> tuple[LoadedPlugin, ...]:
|
|
28
|
+
return tuple(self._by_capability.get(capability, ()))
|
|
29
|
+
|
|
30
|
+
def __iter__(self) -> Iterator[LoadedPlugin]:
|
|
31
|
+
return iter(self._by_name.values())
|
|
32
|
+
|
|
33
|
+
def __len__(self) -> int:
|
|
34
|
+
return len(self._by_name)
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""Secret detection base (architecture §25 plugins/secrets).
|
|
2
|
+
|
|
3
|
+
Secrets are string literals of the PyHIR. ``literals`` walks every string literal with
|
|
4
|
+
the name it is bound to (assignment target, keyword argument, dictionary key) and the
|
|
5
|
+
enclosing function; ``SecretDetector`` reports at most one finding per literal: a
|
|
6
|
+
provider pattern first (``hardcoded-secret``), then a credential-like name with a real
|
|
7
|
+
value (``hardcoded-credential``), then a high-entropy token on its own
|
|
8
|
+
(``high-entropy-string``). Messages and metadata carry a redacted preview, never the
|
|
9
|
+
secret. Only Python sources are scanned; configuration files are not.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
import tomllib
|
|
18
|
+
from collections import Counter
|
|
19
|
+
from collections.abc import Iterator, Sequence
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import ClassVar
|
|
23
|
+
|
|
24
|
+
from coretrace_python.findings import Confidence, Finding, Severity
|
|
25
|
+
from coretrace_python.hir import nodes
|
|
26
|
+
from coretrace_python.hir.visitors import Node, children
|
|
27
|
+
from coretrace_python.interprocedural import discover_files
|
|
28
|
+
from coretrace_python.plugins.api import Plugin, PluginContext
|
|
29
|
+
from coretrace_python.source import SourceId, SourceSpan, decode_text
|
|
30
|
+
|
|
31
|
+
Literal = tuple[str, str | None, SourceSpan, str | None]
|
|
32
|
+
|
|
33
|
+
_HEX = re.compile(r"^[0-9a-fA-F]+$")
|
|
34
|
+
_TOKEN = re.compile(r"^[A-Za-z0-9+/=_-]+$")
|
|
35
|
+
_PLACEHOLDER = re.compile(r"^(<.*>|\$\{.*\}|\{\{.*\}\}|%.*%|\.{3,}|(.)\2*)$")
|
|
36
|
+
_PLACEHOLDER_WORDS = frozenset(
|
|
37
|
+
{"", "changeme", "change_me", "password", "passwd", "secret", "token", "example", "xxx", "todo", "none", "null"}
|
|
38
|
+
)
|
|
39
|
+
_NOT_CREDENTIAL_SUFFIXES = ("_name", "_field", "_file", "_path", "_url", "_id", "_env", "_var", "_header", "_param")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def shannon_entropy(text: str) -> float:
|
|
43
|
+
"""Bits of information per character of ``text``."""
|
|
44
|
+
|
|
45
|
+
if not text:
|
|
46
|
+
return 0.0
|
|
47
|
+
counts = Counter(text)
|
|
48
|
+
length = len(text)
|
|
49
|
+
return -sum(count / length * math.log2(count / length) for count in counts.values())
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def literals(module: nodes.Module) -> Iterator[Literal]:
|
|
53
|
+
"""Every string literal of the module as ``(value, bound name, span, function)``."""
|
|
54
|
+
|
|
55
|
+
for statement in module.body:
|
|
56
|
+
yield from _walk(statement, None, None)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _walk(node: Node, name: str | None, function: str | None) -> Iterator[Literal]:
|
|
60
|
+
if isinstance(node, nodes.Constant):
|
|
61
|
+
if isinstance(node.value, str):
|
|
62
|
+
yield node.value, name, node.span, function
|
|
63
|
+
return
|
|
64
|
+
if isinstance(node, nodes.Function):
|
|
65
|
+
for child in children(node):
|
|
66
|
+
yield from _walk(child, None, node.name)
|
|
67
|
+
return
|
|
68
|
+
if isinstance(node, nodes.Class):
|
|
69
|
+
for child in children(node):
|
|
70
|
+
yield from _walk(child, None, None)
|
|
71
|
+
return
|
|
72
|
+
if isinstance(node, nodes.Assign):
|
|
73
|
+
yield from _walk(node.value, _bound_name(node.target), function)
|
|
74
|
+
return
|
|
75
|
+
if isinstance(node, nodes.Keyword):
|
|
76
|
+
yield from _walk(node.value, node.name, function)
|
|
77
|
+
return
|
|
78
|
+
if isinstance(node, nodes.Parameter):
|
|
79
|
+
if node.default is not None:
|
|
80
|
+
yield from _walk(node.default, node.name, function)
|
|
81
|
+
return
|
|
82
|
+
if isinstance(node, nodes.Call) and _is_environment_lookup(node):
|
|
83
|
+
# ``os.getenv("APP_TOKEN", "fallback")``: the fallback is bound to the variable name.
|
|
84
|
+
variable, fallback = node.arguments[0], node.arguments[1]
|
|
85
|
+
assert isinstance(variable, nodes.Constant) and isinstance(variable.value, str)
|
|
86
|
+
yield from _walk(variable, None, function)
|
|
87
|
+
yield from _walk(fallback, variable.value, function)
|
|
88
|
+
for keyword in node.keywords:
|
|
89
|
+
yield from _walk(keyword, None, function)
|
|
90
|
+
return
|
|
91
|
+
if isinstance(node, nodes.Dict):
|
|
92
|
+
for key, value in node.items:
|
|
93
|
+
# A constant key names the value; it is not a value itself.
|
|
94
|
+
if isinstance(key, nodes.Constant):
|
|
95
|
+
bound = key.value if isinstance(key.value, str) else None
|
|
96
|
+
else:
|
|
97
|
+
bound = None
|
|
98
|
+
if key is not None:
|
|
99
|
+
yield from _walk(key, None, function)
|
|
100
|
+
yield from _walk(value, bound, function)
|
|
101
|
+
return
|
|
102
|
+
for child in children(node):
|
|
103
|
+
yield from _walk(child, None, function)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _is_environment_lookup(call: nodes.Call) -> bool:
|
|
107
|
+
callee = call.callee
|
|
108
|
+
return (
|
|
109
|
+
isinstance(callee, nodes.Attribute)
|
|
110
|
+
and callee.name in ("getenv", "get")
|
|
111
|
+
and len(call.arguments) >= 2
|
|
112
|
+
and isinstance(call.arguments[0], nodes.Constant)
|
|
113
|
+
and isinstance(call.arguments[0].value, str)
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _bound_name(target: nodes.Target) -> str | None:
|
|
118
|
+
"""The name an assignment binds: ``name``, ``obj.attr`` or ``mapping['key']``."""
|
|
119
|
+
|
|
120
|
+
if isinstance(target, nodes.Name):
|
|
121
|
+
return target.identifier
|
|
122
|
+
if isinstance(target, nodes.Attribute):
|
|
123
|
+
return target.name
|
|
124
|
+
if isinstance(target, nodes.Subscript) and isinstance(target.key, nodes.Constant):
|
|
125
|
+
return target.key.value if isinstance(target.key.value, str) else None
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass(frozen=True)
|
|
130
|
+
class SecretPattern:
|
|
131
|
+
"""A provider-specific secret format."""
|
|
132
|
+
|
|
133
|
+
provider: str
|
|
134
|
+
regex: str
|
|
135
|
+
|
|
136
|
+
def matches(self, text: str) -> bool:
|
|
137
|
+
return re.search(self.regex, text) is not None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
DEFAULT_PATTERNS: tuple[SecretPattern, ...] = (
|
|
141
|
+
SecretPattern("aws", r"\b(AKIA|ASIA)[0-9A-Z]{16}\b"),
|
|
142
|
+
SecretPattern("github", r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"),
|
|
143
|
+
SecretPattern("github", r"\bgithub_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59}\b"),
|
|
144
|
+
SecretPattern("slack", r"\bxox[abpr]-[0-9]{10,}-[0-9A-Za-z-]{10,}"),
|
|
145
|
+
SecretPattern("stripe", r"\b[sr]k_(live|test)_[0-9A-Za-z]{24,}\b"),
|
|
146
|
+
SecretPattern("google", r"\bAIza[0-9A-Za-z_-]{35}\b"),
|
|
147
|
+
SecretPattern("private-key", r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
|
|
148
|
+
SecretPattern("jwt", r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"),
|
|
149
|
+
SecretPattern("sendgrid", r"\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b"),
|
|
150
|
+
SecretPattern("twilio", r"\bSK[0-9a-fA-F]{32}\b"),
|
|
151
|
+
SecretPattern(
|
|
152
|
+
"url",
|
|
153
|
+
r"://[^/\s:@'\"]+:[^/\s@'\"]+@"
|
|
154
|
+
r"|[?&](?:password|passwd|pwd|token|api_key|apikey|secret|access_key)=[^&\s'\"]{3,}",
|
|
155
|
+
),
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
DEFAULT_CREDENTIAL_NAMES: tuple[str, ...] = (
|
|
159
|
+
"password",
|
|
160
|
+
"passwd",
|
|
161
|
+
"pwd",
|
|
162
|
+
"secret",
|
|
163
|
+
"token",
|
|
164
|
+
"api_key",
|
|
165
|
+
"apikey",
|
|
166
|
+
"api-key",
|
|
167
|
+
"private_key",
|
|
168
|
+
"access_key",
|
|
169
|
+
"credential",
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
CONFIG_SUFFIXES = frozenset({".env", ".yaml", ".yml", ".toml", ".json", ".ini", ".cfg", ".properties", ".conf"})
|
|
173
|
+
_PAIR = re.compile(r"^\s*(?:-\s+)?([A-Za-z_][\w.-]*)\s*[:=]\s*(.*?)\s*$")
|
|
174
|
+
_MAX_CONFIG_BYTES = 8_000_000
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def config_literals(root: Path) -> Iterator[Literal]:
|
|
178
|
+
"""Every string value of the configuration files under ``root`` with the key it is
|
|
179
|
+
bound to: ``.env``, YAML, TOML, JSON, INI and properties files, decoded by their
|
|
180
|
+
byte order mark. Python files are left to ``literals``."""
|
|
181
|
+
|
|
182
|
+
for path in discover_files(root):
|
|
183
|
+
if not (path.suffix in CONFIG_SUFFIXES or path.name.startswith(".env")):
|
|
184
|
+
continue
|
|
185
|
+
try:
|
|
186
|
+
data = path.read_bytes()
|
|
187
|
+
if len(data) > _MAX_CONFIG_BYTES:
|
|
188
|
+
continue
|
|
189
|
+
text = decode_text(data)
|
|
190
|
+
except (OSError, UnicodeDecodeError):
|
|
191
|
+
continue
|
|
192
|
+
source = SourceId(str(path))
|
|
193
|
+
if path.suffix == ".json":
|
|
194
|
+
yield from _structured(source, text, _load_json(text))
|
|
195
|
+
elif path.suffix == ".toml":
|
|
196
|
+
yield from _structured(source, text, _load_toml(text))
|
|
197
|
+
else:
|
|
198
|
+
yield from _pairs(source, text)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _load_json(text: str) -> object:
|
|
202
|
+
try:
|
|
203
|
+
return json.loads(text)
|
|
204
|
+
except ValueError:
|
|
205
|
+
return None
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _load_toml(text: str) -> object:
|
|
209
|
+
try:
|
|
210
|
+
return tomllib.loads(text)
|
|
211
|
+
except tomllib.TOMLDecodeError:
|
|
212
|
+
return None
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _structured(source: SourceId, text: str, data: object) -> Iterator[Literal]:
|
|
216
|
+
lines = text.splitlines()
|
|
217
|
+
|
|
218
|
+
def line_of(key: str) -> int:
|
|
219
|
+
for number, line in enumerate(lines, start=1):
|
|
220
|
+
if re.match(rf'^\s*"?{re.escape(key)}"?\s*[:=]', line) or f'"{key}"' in line:
|
|
221
|
+
return number
|
|
222
|
+
return 1
|
|
223
|
+
|
|
224
|
+
def walk(node: object, key: str | None) -> Iterator[Literal]:
|
|
225
|
+
if isinstance(node, dict):
|
|
226
|
+
for name, value in node.items():
|
|
227
|
+
yield from walk(value, str(name))
|
|
228
|
+
elif isinstance(node, list):
|
|
229
|
+
for item in node:
|
|
230
|
+
yield from walk(item, key)
|
|
231
|
+
elif isinstance(node, str) and key is not None:
|
|
232
|
+
yield node, key, SourceSpan(source, line_of(key), 1), None
|
|
233
|
+
|
|
234
|
+
yield from walk(data, None)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _pairs(source: SourceId, text: str) -> Iterator[Literal]:
|
|
238
|
+
for number, line in enumerate(text.splitlines(), start=1):
|
|
239
|
+
stripped = line.strip()
|
|
240
|
+
if not stripped or stripped[0] in "#;[":
|
|
241
|
+
continue
|
|
242
|
+
match = _PAIR.match(line)
|
|
243
|
+
if match is None:
|
|
244
|
+
continue
|
|
245
|
+
key, value = match.group(1), match.group(2)
|
|
246
|
+
if value.startswith("#"):
|
|
247
|
+
continue
|
|
248
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
|
|
249
|
+
value = value[1:-1]
|
|
250
|
+
if value:
|
|
251
|
+
yield value, key, SourceSpan(source, number, 1), None
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def redacted(value: str) -> str:
|
|
255
|
+
preview = value[:4] if len(value) > 8 else value[:1]
|
|
256
|
+
return f"{preview}… ({len(value)} characters)"
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def is_placeholder(value: str) -> bool:
|
|
260
|
+
lowered = value.strip().lower()
|
|
261
|
+
return len(lowered) < 4 or lowered in _PLACEHOLDER_WORDS or _PLACEHOLDER.match(lowered) is not None
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class SecretDetector(Plugin):
|
|
265
|
+
"""Report hardcoded secrets among the module's string literals."""
|
|
266
|
+
|
|
267
|
+
patterns: ClassVar[tuple[SecretPattern, ...]] = ()
|
|
268
|
+
credential_names: ClassVar[tuple[str, ...]] = ()
|
|
269
|
+
entropy_threshold: ClassVar[float] = 4.5
|
|
270
|
+
hex_entropy_threshold: ClassVar[float] = 3.5
|
|
271
|
+
minimum_length: ClassVar[int] = 32
|
|
272
|
+
|
|
273
|
+
def analyze(self, ctx: PluginContext) -> Sequence[Finding]:
|
|
274
|
+
findings: list[Finding] = []
|
|
275
|
+
for value, name, span, function in literals(ctx.module):
|
|
276
|
+
finding = self.judge(value, name, span, function)
|
|
277
|
+
if finding is not None:
|
|
278
|
+
findings.append(finding)
|
|
279
|
+
return findings
|
|
280
|
+
|
|
281
|
+
def judge(self, value: str, name: str | None, span: SourceSpan, function: str | None) -> Finding | None:
|
|
282
|
+
where = f"in {name}" if name is not None else "in a string literal"
|
|
283
|
+
for pattern in self.patterns:
|
|
284
|
+
if pattern.matches(value):
|
|
285
|
+
return Finding(
|
|
286
|
+
"hardcoded-secret",
|
|
287
|
+
f"Hardcoded {pattern.provider} secret {where}: {redacted(value)}",
|
|
288
|
+
Severity.HIGH,
|
|
289
|
+
Confidence.HIGH,
|
|
290
|
+
span,
|
|
291
|
+
function,
|
|
292
|
+
{"provider": pattern.provider, "name": name or "", "length": str(len(value))},
|
|
293
|
+
)
|
|
294
|
+
if name is not None and self.is_credential_name(name) and not is_placeholder(value):
|
|
295
|
+
return Finding(
|
|
296
|
+
"hardcoded-credential",
|
|
297
|
+
f"Hardcoded credential {where}: {redacted(value)}",
|
|
298
|
+
Severity.HIGH,
|
|
299
|
+
Confidence.MEDIUM,
|
|
300
|
+
span,
|
|
301
|
+
function,
|
|
302
|
+
{"name": name, "length": str(len(value))},
|
|
303
|
+
)
|
|
304
|
+
entropy = self.entropy_of(value)
|
|
305
|
+
if entropy is not None:
|
|
306
|
+
return Finding(
|
|
307
|
+
"high-entropy-string",
|
|
308
|
+
f"High-entropy string {where}: {redacted(value)}",
|
|
309
|
+
Severity.MEDIUM,
|
|
310
|
+
Confidence.LOW,
|
|
311
|
+
span,
|
|
312
|
+
function,
|
|
313
|
+
{"name": name or "", "length": str(len(value)), "entropy": f"{entropy:.2f}"},
|
|
314
|
+
)
|
|
315
|
+
return None
|
|
316
|
+
|
|
317
|
+
def is_credential_name(self, name: str) -> bool:
|
|
318
|
+
lowered = name.lower()
|
|
319
|
+
if lowered.endswith(_NOT_CREDENTIAL_SUFFIXES):
|
|
320
|
+
return False
|
|
321
|
+
return any(word in lowered for word in self.credential_names)
|
|
322
|
+
|
|
323
|
+
def entropy_of(self, value: str) -> float | None:
|
|
324
|
+
"""The entropy of ``value`` when it looks like an opaque token, else ``None``."""
|
|
325
|
+
|
|
326
|
+
if len(value) < self.minimum_length or _TOKEN.match(value) is None:
|
|
327
|
+
return None
|
|
328
|
+
entropy = shannon_entropy(value)
|
|
329
|
+
threshold = self.hex_entropy_threshold if _HEX.match(value) else self.entropy_threshold
|
|
330
|
+
return entropy if entropy >= threshold else None
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Reporters render normalized findings and never run an analysis (architecture §28)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Mapping
|
|
6
|
+
from types import MappingProxyType
|
|
7
|
+
|
|
8
|
+
from coretrace_python.reporters.json_format import render_json
|
|
9
|
+
from coretrace_python.reporters.report import Report
|
|
10
|
+
from coretrace_python.reporters.sarif import render_sarif
|
|
11
|
+
from coretrace_python.reporters.text import render_text
|
|
12
|
+
|
|
13
|
+
FORMATS: Mapping[str, Callable[[Report], str]] = MappingProxyType(
|
|
14
|
+
{"text": render_text, "json": render_json, "sarif": render_sarif}
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def render(format_name: str, report: Report) -> str:
|
|
19
|
+
return FORMATS[format_name](report)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
__all__ = ["FORMATS", "Report", "render", "render_json", "render_sarif", "render_text"]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""JSON reporter: versioned normalized finding records."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
from coretrace_python.findings import FINDING_SCHEMA_VERSION, Finding
|
|
8
|
+
from coretrace_python.reporters.report import Report
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def finding_record(finding: Finding) -> dict[str, object]:
|
|
12
|
+
span = finding.span
|
|
13
|
+
return {
|
|
14
|
+
"rule_id": finding.rule_id,
|
|
15
|
+
"message": finding.message,
|
|
16
|
+
"severity": finding.severity.value,
|
|
17
|
+
"confidence": finding.confidence.value,
|
|
18
|
+
"location": {
|
|
19
|
+
"path": str(span.source_id),
|
|
20
|
+
"line": span.start_line,
|
|
21
|
+
"column": span.start_column,
|
|
22
|
+
"end_line": span.end_line,
|
|
23
|
+
"end_column": span.end_column,
|
|
24
|
+
},
|
|
25
|
+
"function": finding.function,
|
|
26
|
+
"metadata": dict(finding.metadata),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def render_json(report: Report) -> str:
|
|
31
|
+
document = {
|
|
32
|
+
"schema_version": FINDING_SCHEMA_VERSION,
|
|
33
|
+
"tool": {"name": report.tool_name, "version": report.tool_version},
|
|
34
|
+
"findings": [finding_record(finding) for finding in report.findings],
|
|
35
|
+
}
|
|
36
|
+
if report.coverage is not None:
|
|
37
|
+
coverage = report.coverage
|
|
38
|
+
document["coverage"] = {
|
|
39
|
+
"files": coverage.files,
|
|
40
|
+
"files_analysed": coverage.files_analysed,
|
|
41
|
+
"functions": coverage.functions,
|
|
42
|
+
"functions_analysed": coverage.functions_analysed,
|
|
43
|
+
"details": [
|
|
44
|
+
{"path": d.path, "status": d.status, "functions": d.functions, "analysed": d.analysed}
|
|
45
|
+
for d in coverage.details
|
|
46
|
+
],
|
|
47
|
+
}
|
|
48
|
+
return json.dumps(document, indent=2) + "\n"
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""The normalized report every reporter renders (architecture §23, §28)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
from coretrace_python.findings import Coverage, Finding
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _order(finding: Finding) -> tuple[str, int, int, str]:
|
|
11
|
+
span = finding.span
|
|
12
|
+
return (str(span.source_id), span.start_line, span.start_column, finding.rule_id)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Report:
|
|
17
|
+
findings: tuple[Finding, ...]
|
|
18
|
+
tool_name: str
|
|
19
|
+
tool_version: str
|
|
20
|
+
coverage: Coverage | None = None
|
|
21
|
+
|
|
22
|
+
def __post_init__(self) -> None:
|
|
23
|
+
object.__setattr__(self, "findings", tuple(sorted(self.findings, key=_order)))
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""SARIF 2.1.0 reporter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
from coretrace_python.findings import Finding, Severity
|
|
8
|
+
from coretrace_python.reporters.report import Report
|
|
9
|
+
|
|
10
|
+
SARIF_VERSION = "2.1.0"
|
|
11
|
+
SARIF_SCHEMA = "https://json.schemastore.org/sarif-2.1.0.json"
|
|
12
|
+
|
|
13
|
+
_LEVELS = {
|
|
14
|
+
Severity.CRITICAL: "error",
|
|
15
|
+
Severity.HIGH: "error",
|
|
16
|
+
Severity.MEDIUM: "warning",
|
|
17
|
+
Severity.LOW: "note",
|
|
18
|
+
Severity.INFO: "note",
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _result(finding: Finding, rule_index: int) -> dict[str, object]:
|
|
23
|
+
span = finding.span
|
|
24
|
+
region: dict[str, int] = {"startLine": span.start_line, "startColumn": span.start_column}
|
|
25
|
+
if span.end_line is not None and span.end_column is not None:
|
|
26
|
+
region["endLine"] = span.end_line
|
|
27
|
+
region["endColumn"] = span.end_column
|
|
28
|
+
return {
|
|
29
|
+
"ruleId": finding.rule_id,
|
|
30
|
+
"ruleIndex": rule_index,
|
|
31
|
+
"level": _LEVELS[finding.severity],
|
|
32
|
+
"message": {"text": finding.message},
|
|
33
|
+
"locations": [
|
|
34
|
+
{
|
|
35
|
+
"physicalLocation": {
|
|
36
|
+
"artifactLocation": {"uri": str(span.source_id)},
|
|
37
|
+
"region": region,
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def render_sarif(report: Report) -> str:
|
|
45
|
+
rule_ids: list[str] = []
|
|
46
|
+
for finding in report.findings:
|
|
47
|
+
if finding.rule_id not in rule_ids:
|
|
48
|
+
rule_ids.append(finding.rule_id)
|
|
49
|
+
document = {
|
|
50
|
+
"$schema": SARIF_SCHEMA,
|
|
51
|
+
"version": SARIF_VERSION,
|
|
52
|
+
"runs": [
|
|
53
|
+
{
|
|
54
|
+
"tool": {
|
|
55
|
+
"driver": {
|
|
56
|
+
"name": report.tool_name,
|
|
57
|
+
"version": report.tool_version,
|
|
58
|
+
"rules": [
|
|
59
|
+
{"id": rule_id, "shortDescription": {"text": rule_id}}
|
|
60
|
+
for rule_id in rule_ids
|
|
61
|
+
],
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
"results": [
|
|
65
|
+
_result(finding, rule_ids.index(finding.rule_id)) for finding in report.findings
|
|
66
|
+
],
|
|
67
|
+
}
|
|
68
|
+
],
|
|
69
|
+
}
|
|
70
|
+
return json.dumps(document, indent=2) + "\n"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Plain-text reporter: one ``path:line:column`` line per finding and a summary."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from coretrace_python.reporters.report import Report
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def render_text(report: Report) -> str:
|
|
9
|
+
lines = []
|
|
10
|
+
for finding in report.findings:
|
|
11
|
+
span = finding.span
|
|
12
|
+
line = (
|
|
13
|
+
f"{span.source_id}:{span.start_line}:{span.start_column}: "
|
|
14
|
+
f"{finding.severity.value} {finding.rule_id}: {finding.message}"
|
|
15
|
+
)
|
|
16
|
+
if finding.function is not None:
|
|
17
|
+
line += f" [{finding.function}]"
|
|
18
|
+
lines.append(line)
|
|
19
|
+
count = len(report.findings)
|
|
20
|
+
summary = "no findings" if count == 0 else f"{count} finding{'s' if count > 1 else ''}"
|
|
21
|
+
lines.append(summary)
|
|
22
|
+
if report.coverage is not None:
|
|
23
|
+
lines.append(report.coverage.summary())
|
|
24
|
+
return "\n".join(lines) + "\n"
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Semantic analyses computed from PyHIR: scopes, imports and symbols."""
|
|
2
|
+
|
|
3
|
+
from coretrace_python.semantic.imports import ImportAnalysis
|
|
4
|
+
from coretrace_python.semantic.scopes import ScopeAnalysis
|
|
5
|
+
from coretrace_python.semantic.symbols import SymbolAnalysis
|
|
6
|
+
|
|
7
|
+
SEMANTIC_ANALYSES = (ScopeAnalysis, ImportAnalysis, SymbolAnalysis)
|
|
8
|
+
|
|
9
|
+
__all__ = ["SEMANTIC_ANALYSES", "ImportAnalysis", "ScopeAnalysis", "SymbolAnalysis"]
|