attackmap 0.6.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. attackmap/__init__.py +2 -0
  2. attackmap/analyzer.py +449 -0
  3. attackmap/analyzer_contracts.py +194 -0
  4. attackmap/analyzers.py +1003 -0
  5. attackmap/anomalies.py +612 -0
  6. attackmap/asset_model.py +245 -0
  7. attackmap/attack_taxonomy.py +133 -0
  8. attackmap/auth_protocol.py +702 -0
  9. attackmap/authz.py +1695 -0
  10. attackmap/bench.py +467 -0
  11. attackmap/cli.py +1474 -0
  12. attackmap/config_scanner.py +389 -0
  13. attackmap/context_pack.py +82 -0
  14. attackmap/contracts.py +201 -0
  15. attackmap/control_model.py +280 -0
  16. attackmap/crossrepo.py +356 -0
  17. attackmap/crypto.py +240 -0
  18. attackmap/cve.py +717 -0
  19. attackmap/defensive_review.py +647 -0
  20. attackmap/detection_opportunities.py +1221 -0
  21. attackmap/detections/__init__.py +491 -0
  22. attackmap/detections/bola.yml +57 -0
  23. attackmap/detections/cloud.yml +609 -0
  24. attackmap/detections/injection.yml +167 -0
  25. attackmap/detections/secrets.yml +48 -0
  26. attackmap/detections/sigma.py +511 -0
  27. attackmap/detections/ssrf.yml +98 -0
  28. attackmap/detections/weakness.yml +62 -0
  29. attackmap/detections/webhook.yml +45 -0
  30. attackmap/detections/workflow.yml +121 -0
  31. attackmap/diagrams.py +237 -0
  32. attackmap/diff.py +301 -0
  33. attackmap/exploitability.py +493 -0
  34. attackmap/fingerprint.py +91 -0
  35. attackmap/fleet.py +370 -0
  36. attackmap/gitref.py +129 -0
  37. attackmap/graph.py +26 -0
  38. attackmap/handler_auth.py +639 -0
  39. attackmap/handler_span.py +62 -0
  40. attackmap/hunt_harness.py +505 -0
  41. attackmap/insights.py +701 -0
  42. attackmap/llm_review.py +846 -0
  43. attackmap/lockfiles.py +521 -0
  44. attackmap/mass_assignment.py +543 -0
  45. attackmap/md.py +175 -0
  46. attackmap/merge.py +333 -0
  47. attackmap/models.py +1284 -0
  48. attackmap/plugins_lock.py +114 -0
  49. attackmap/popular_packages.py +193 -0
  50. attackmap/progress.py +258 -0
  51. attackmap/recon_models.py +31 -0
  52. attackmap/recon_to_analysis.py +149 -0
  53. attackmap/redact.py +220 -0
  54. attackmap/report.py +424 -0
  55. attackmap/review_eval.py +335 -0
  56. attackmap/review_json.py +308 -0
  57. attackmap/review_prompts.py +796 -0
  58. attackmap/route_auth_fusion.py +474 -0
  59. attackmap/safe_fs.py +285 -0
  60. attackmap/sarif.py +373 -0
  61. attackmap/sbom.py +418 -0
  62. attackmap/scanner.py +1379 -0
  63. attackmap/sdk/__init__.py +278 -0
  64. attackmap/sdk/contracts.py +17 -0
  65. attackmap/sdk/fs.py +156 -0
  66. attackmap/sdk/models.py +59 -0
  67. attackmap/secrets_history.py +161 -0
  68. attackmap/security_overlay.py +71 -0
  69. attackmap/srcpaths.py +301 -0
  70. attackmap/suggest.py +394 -0
  71. attackmap/supply_chain.py +688 -0
  72. attackmap/suppress.py +740 -0
  73. attackmap/taint.py +1540 -0
  74. attackmap/taint_flow.py +1419 -0
  75. attackmap/taint_sinks.py +431 -0
  76. attackmap/taint_sinks.yaml +308 -0
  77. attackmap/taint_sources.py +396 -0
  78. attackmap/taxonomy.py +465 -0
  79. attackmap/threat_model.py +3441 -0
  80. attackmap/topology.py +316 -0
  81. attackmap/triage.py +151 -0
  82. attackmap/weaknesses.py +365 -0
  83. attackmap/webhardening.py +479 -0
  84. attackmap/webhook_verification.py +1067 -0
  85. attackmap/workflow_scanner.py +964 -0
  86. attackmap-0.6.4.dist-info/METADATA +226 -0
  87. attackmap-0.6.4.dist-info/RECORD +91 -0
  88. attackmap-0.6.4.dist-info/WHEEL +5 -0
  89. attackmap-0.6.4.dist-info/entry_points.txt +2 -0
  90. attackmap-0.6.4.dist-info/licenses/LICENSE +21 -0
  91. attackmap-0.6.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,278 @@
1
+ """Public, stable contract surface for AttackMap analyzers.
2
+
3
+ Anything you can import from ``attackmap.sdk`` is part of the stable analyzer
4
+ contract. Anything you cannot is an internal implementation detail of the
5
+ ``attackmap`` package and may change without a major-version bump.
6
+
7
+ ## What an analyzer is
8
+
9
+ An analyzer is a Python object that satisfies :class:`AnalyzerProtocol`:
10
+
11
+ - exposes ``metadata: AnalyzerMetadata``
12
+ - exposes a ``name`` property
13
+ - implements ``detect(root) -> bool`` — does this repo look like something
14
+ the analyzer should run on?
15
+ - implements ``analyze(root) -> AnalyzerResult`` — extract structured
16
+ signals (routes, external calls, hints, …) from the repo
17
+
18
+ External analyzers register themselves via the ``attackmap.analyzers``
19
+ Python entry-point group. See the README of any official analyzer (e.g.
20
+ ``attackmap-analyzer-python``) for a working example.
21
+
22
+ ## Responsibilities
23
+
24
+ Analyzers emit **structured signals** — small, evidence-bearing data points
25
+ with a file/line citation. They do *not* generate findings, attack paths,
26
+ or reports. Specifically:
27
+
28
+ | Owned by analyzers | Owned by core |
29
+ |---|---|
30
+ | Detecting whether they should run | Discovering and loading analyzers |
31
+ | Extracting routes, external calls, databases | Merging results across analyzers |
32
+ | Emitting auth/secret/service/edge/protocol/framework/entrypoint hints | Building the system graph |
33
+ | Surface-level normalization (path canonicalization, etc.) | Generating findings + severity |
34
+ | | Generating attack paths and threat-model output |
35
+ | | Rendering CLI / JSON / markdown reports |
36
+
37
+ If you find yourself wanting to add finding generation or report rendering
38
+ to an analyzer, the answer is almost always "emit a richer signal and let
39
+ core do the reasoning."
40
+
41
+ ## Merge semantics
42
+
43
+ When multiple analyzers run against the same repo, their results are
44
+ merged by `attackmap.analyzers.merge_analyzer_results`. The schema is
45
+ declared in `attackmap.merge.MERGE_SCHEMA`; the rules are:
46
+
47
+ - **Order**: first-seen wins. Analyzers (built-in and plugin alike) run
48
+ in ``(metadata.priority, name)`` order, so the lower priority value wins
49
+ a duplicate signal.
50
+ - **Opt-in**: an analyzer with ``enabled_by_default=False`` runs only when
51
+ selected with ``--module``; ``detect()`` still gates it.
52
+ - **Dedup**: each list field has a stable tuple key, e.g. routes are
53
+ deduped by ``(path, method, file)`` and auth hints by ``(hint, file)``.
54
+ - **Route auth** (#256): a plugin that knows a route's guard sets
55
+ ``Route.auth`` to ``"required"`` or ``"anonymous"`` (plus ``guards`` and
56
+ ``guard_evidence``). Core trusts it over its own regex resolution and the
57
+ ±40-line auth-hint window; ``"unknown"`` (the default) keeps the old
58
+ behavior. On duplicate route keys a known state fills in an unknown one.
59
+ - **Languages**: union, sorted for stable display.
60
+ - **Files scanned**: summed.
61
+
62
+ Two analyzers that both emit a route for the same ``(path, method, file)``
63
+ produce one route in the merged result — whichever was emitted first.
64
+
65
+ ## App attack surface (#188, #190)
66
+
67
+ A platform plugin (the Apple analyzers: Swift and Objective-C) reports
68
+ app-side issues as :class:`AppSurfaceIssue` items in
69
+ ``ScanResult.app_surface_issues``: App Transport Security exceptions, custom
70
+ URL-scheme handlers, weakening entitlements, WKWebView script built from
71
+ strings, secrets in UserDefaults or on the pasteboard, unvalidated XPC peers,
72
+ and classes, selectors or libraries resolved from runtime strings. The
73
+ ``kind`` should be one of :data:`APP_SURFACE_ISSUE_KINDS` (it is typed ``str``:
74
+ a record with a kind this core doesn't know is dropped and noted under
75
+ ``analyzer_errors``, the rest of the result is kept); ``severity`` and
76
+ ``confidence`` are per instance. Records are merged by ``(kind, file, line,
77
+ evidence_text)``; on a duplicate the more severe, then more confident, wins. Core turns each kind into one aggregated finding (rule id = the
78
+ kind, dashed, e.g. ``apple-ats-insecure-loads``) with CWE/OWASP taxonomy.
79
+ Entry points such as URL schemes, universal links and XPC services also go in
80
+ ``entrypoint_hints`` so they show up in the surface inventory.
81
+
82
+ ## Contributing taint sinks (#240)
83
+
84
+ An analyzer can teach core's taint pass new sink signatures (a framework's
85
+ file-serving helper, an ORM's raw-query escape hatch) without touching core.
86
+ Register an entry point in the ``attackmap.taint_sinks`` group
87
+ (:data:`TAINT_SINK_PLUGIN_GROUP`) that resolves to a list of sink entries, a
88
+ callable returning one, or a path to a YAML file with a ``sinks:`` list::
89
+
90
+ [project.entry-points."attackmap.taint_sinks"]
91
+ myframework = "attackmap_analyzer_myframework.sinks:SINKS"
92
+
93
+ SINKS = [
94
+ {
95
+ "id": "myframework-raw-sql", # unique; prefix with your plugin
96
+ "kind": "sql_execute", # one of TAINT_SINK_KINDS
97
+ "langs": ["python"],
98
+ "regex": r"\\bRaw\\.sql\\s*\\(",
99
+ "gate": "tainted", # or "any"
100
+ "args": [0], # or "all" / "receiver" / "rest"
101
+ "cwe": 89, # required
102
+ },
103
+ ]
104
+
105
+ Entries use the same schema as core's ``taint_sinks.yaml`` and must carry a
106
+ ``cwe``. Check them in your tests with :func:`validate_taint_sink`; at scan
107
+ time an invalid entry is skipped with a warning instead of failing the scan.
108
+ Plugins reuse core's sink kinds (findings, CWE taxonomy and exploitability are
109
+ keyed on them) and core still decides reachability, flow and severity.
110
+
111
+ ## Cloud / IaC misconfigurations
112
+
113
+ IaC plugins (Terraform, Kubernetes / Helm, compose, Dockerfile) report
114
+ misconfigurations as :class:`CloudMisconfig` records in
115
+ ``ScanResult.cloud_misconfigs``, not as auth or entrypoint hints, which core
116
+ treats as recon (an ``AuthHint`` next to a route even counts as an auth
117
+ signal). ``kind`` is one of :data:`CLOUD_MISCONFIG_KINDS`; each kind is a core
118
+ rule id (``privileged_container`` → ``privileged-container``) with CWE/OWASP
119
+ ids. Core emits one finding per (kind, severity): instances keep their own
120
+ severity in SARIF and in the ``--fail-on-new-high`` gate, and a kind's usual
121
+ severity keeps the plain title (other tiers add "(low-severity instances)")::
122
+
123
+ CloudMisconfig(
124
+ kind="host_path_mount", file="k8s/agent.yaml", line=31,
125
+ resource="DaemonSet/agent container agent", platform="kubernetes",
126
+ severity="high", confidence=0.9,
127
+ escalation_path="hostPath / → node root → kubelet credentials",
128
+ evidence_text="path: /",
129
+ )
130
+
131
+ Set ``severity`` per instance (a ``docker.sock`` mount is worse than
132
+ ``/var/log``), lower ``confidence`` for settings you inferred from a default
133
+ rather than read, and put ``for_each``/``count`` or module provenance in
134
+ ``context``. Report literal values only: a templated value
135
+ (``{{ .Values.x }}``, ``var.x``) is not a misconfiguration you can see.
136
+
137
+ ``platform`` is one of :data:`CLOUD_MISCONFIG_PLATFORMS`. The free-text fields
138
+ are redacted, folded to one line and capped (evidence, context and escalation
139
+ path 300 characters, resource 200). Records merge on (kind, file, line,
140
+ resource, evidence); for a true duplicate the more severe report wins.
141
+
142
+ ``kind`` is typed ``str``: core drops records with a kind it doesn't know (a
143
+ plugin built against a newer core), keeps the rest of the result and notes the
144
+ drop under ``analyzer_errors`` (``--strict-analyzers`` fails instead). A new kind needs a
145
+ core release (finding spec + taxonomy entry). A plugin that must also run on
146
+ cores older than this record should feature-detect it (``try: from
147
+ attackmap.sdk import CloudMisconfig`` / ``hasattr(ScanResult,
148
+ "cloud_misconfigs")``) and keep emitting its old hints there, because an older
149
+ core ignores the records.
150
+ ## Weakness and supply-chain records (#189, #191)
151
+
152
+ Some bug classes are decided inside one function and don't fit the
153
+ route-seeded taint pass, e.g. C/C++ memory-safety sinks (``strcpy`` from a
154
+ traced input, a non-literal ``printf`` format, ``system`` on a built
155
+ command). An analyzer may emit those as :class:`CodeWeakness` records in
156
+ ``result.code_weaknesses``. ``kind`` must be one of
157
+ :data:`CODE_WEAKNESS_KINDS`; core owns the finding title, mitigation and
158
+ taxonomy for each kind. For the native kinds (``unbounded_copy``,
159
+ ``format_string``, ``native_command_exec``, ``tainted_copy_length``,
160
+ ``alloc_size_overflow``) set ``severity`` and ``confidence`` per instance
161
+ from whether an input reaches the sink in the same function, and name that
162
+ input in ``source_kind`` / ``source_line``. Core groups the instances into
163
+ one finding per (kind, severity tier), like :class:`CloudMisconfig`. ``kind``
164
+ (and ``DependencyHint.ecosystem``) is typed ``str``: core drops a record with
165
+ a value it doesn't know and notes it under ``analyzer_errors``.
166
+
167
+ Manifest-level supply-chain risks an analyzer understands better than core
168
+ (a CMake ``FetchContent`` ``GIT_TAG`` on a branch) go in
169
+ ``result.supply_chain_issues`` as :class:`SupplyChainIssue` records, using
170
+ the kinds core already reports (``mutable_vcs_dependency``, ...).
171
+
172
+ ## Versioning
173
+
174
+ The names exported below are stable across minor releases of the
175
+ ``attackmap`` package. Breaking changes to this surface only happen on
176
+ a major-version bump; deprecations get one release of overlap.
177
+ """
178
+
179
+ from __future__ import annotations
180
+
181
+ from .contracts import (
182
+ AnalyzerMetadata,
183
+ AnalyzerProtocol,
184
+ AnalyzerRepositoryModule,
185
+ AnalyzerResult,
186
+ normalize_analyzer_metadata,
187
+ )
188
+ from ..safe_fs import contained_file, is_contained, read_repo_text, walk_repo
189
+ from . import fs
190
+ from .fs import DEFAULT_SKIP_DIRS, iter_repo_files, line_of, line_snippet, read_source, rel
191
+ from ..taint_sinks import PLUGIN_GROUP as TAINT_SINK_PLUGIN_GROUP
192
+ from ..taint_sinks import SINK_KINDS as TAINT_SINK_KINDS
193
+ from ..taint_sinks import validate_sink_entry as validate_taint_sink
194
+ from .models import (
195
+ CLOUD_MISCONFIG_KINDS,
196
+ CLOUD_MISCONFIG_PLATFORMS,
197
+ APP_SURFACE_ISSUE_KINDS,
198
+ CODE_WEAKNESS_KINDS,
199
+ DEPENDENCY_ECOSYSTEMS,
200
+ AppSurfaceIssue,
201
+ AppSurfaceIssueKind,
202
+ AuthHint,
203
+ CloudMisconfig,
204
+ CloudMisconfigKind,
205
+ CloudMisconfigPlatform,
206
+ CodeWeakness,
207
+ DatabaseHint,
208
+ DependencyEcosystem,
209
+ DependencyHint,
210
+ EdgeHint,
211
+ EntrypointHint,
212
+ ExternalCall,
213
+ FrameworkHint,
214
+ ProtocolHint,
215
+ Route,
216
+ ScanResult,
217
+ SecretHint,
218
+ ServiceHint,
219
+ SupplyChainIssue,
220
+ )
221
+
222
+ __all__ = [
223
+ "AnalyzerResult",
224
+ "AnalyzerMetadata",
225
+ "AnalyzerRepositoryModule",
226
+ "AnalyzerProtocol",
227
+ "normalize_analyzer_metadata",
228
+ "Route",
229
+ "ExternalCall",
230
+ "DatabaseHint",
231
+ "AuthHint",
232
+ "ServiceHint",
233
+ "EdgeHint",
234
+ "EntrypointHint",
235
+ "ProtocolHint",
236
+ "FrameworkHint",
237
+ "SecretHint",
238
+ "ScanResult",
239
+ "DependencyHint",
240
+ # Package ecosystems a DependencyHint may carry (#255).
241
+ "DependencyEcosystem",
242
+ "DEPENDENCY_ECOSYSTEMS",
243
+ # Cloud / IaC / container misconfigurations (terraform#4, iac#1); core
244
+ # turns each kind into a finding.
245
+ "CloudMisconfig",
246
+ "CloudMisconfigKind",
247
+ "CLOUD_MISCONFIG_KINDS",
248
+ "CloudMisconfigPlatform",
249
+ "CLOUD_MISCONFIG_PLATFORMS",
250
+ # App attack-surface issues a platform plugin may emit (#188, #190).
251
+ "AppSurfaceIssue",
252
+ "AppSurfaceIssueKind",
253
+ "APP_SURFACE_ISSUE_KINDS",
254
+ # Function-local weakness and supply-chain records (#189, #191).
255
+ "CodeWeakness",
256
+ "CODE_WEAKNESS_KINDS",
257
+ "SupplyChainIssue",
258
+ # Repo-confined filesystem helpers (#234): walk and read the scanned repo
259
+ # without following symlinks out of it. Plugins should use these instead
260
+ # of Path.rglob / read_text.
261
+ "walk_repo",
262
+ "is_contained",
263
+ "contained_file",
264
+ "read_repo_text",
265
+ # Shared plugin walker/reader (#253): prunes by repo-relative dir name,
266
+ # skips AttackMap output, caps size, never raises on unreadable files.
267
+ "fs",
268
+ "DEFAULT_SKIP_DIRS",
269
+ "iter_repo_files",
270
+ "read_source",
271
+ "rel",
272
+ "line_of",
273
+ "line_snippet",
274
+ # Plugin-contributed taint sinks (#240).
275
+ "TAINT_SINK_PLUGIN_GROUP",
276
+ "TAINT_SINK_KINDS",
277
+ "validate_taint_sink",
278
+ ]
@@ -0,0 +1,17 @@
1
+ from __future__ import annotations
2
+
3
+ from ..analyzer_contracts import (
4
+ AnalyzerMetadata,
5
+ AnalyzerProtocol,
6
+ AnalyzerRepositoryModule,
7
+ AnalyzerResult,
8
+ normalize_analyzer_metadata,
9
+ )
10
+
11
+ __all__ = [
12
+ "AnalyzerResult",
13
+ "AnalyzerMetadata",
14
+ "AnalyzerRepositoryModule",
15
+ "AnalyzerProtocol",
16
+ "normalize_analyzer_metadata",
17
+ ]
attackmap/sdk/fs.py ADDED
@@ -0,0 +1,156 @@
1
+ """Shared repo walking and reading for analyzer plugins (#253).
2
+
3
+ Every plugin used to copy the same ``root.rglob(...)`` plus
4
+ ``any(part in SKIP_DIRS for part in path.parts)`` walk. That pattern:
5
+
6
+ - matched skip dirs against *absolute* path parts, so a repo checked out under
7
+ ``/build/...`` or ``~/src/out/...`` silently yielded no files at all;
8
+ - didn't prune, so it still descended all of ``node_modules`` and ``target``;
9
+ - followed symlinks out of the repo;
10
+ - let one unreadable file raise out of ``analyze()``;
11
+ - dropped latin-1/cp1252 sources without saying so.
12
+
13
+ Use :func:`iter_repo_files` and :func:`read_source` instead. They prune by
14
+ *repo-relative* directory name, never follow symlinks out of the repo, skip
15
+ AttackMap's own output directories, cap file size, and never raise on an
16
+ unreadable file.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from collections.abc import Callable, Collection, Iterator
22
+ from pathlib import Path
23
+
24
+ from ..safe_fs import is_contained, max_file_bytes, walk_repo
25
+ from ..srcpaths import SKIP_DIRS, is_test_file, line_number
26
+
27
+ # Directories no analyzer should descend into. Plugins extend it for their
28
+ # ecosystem, e.g. ``DEFAULT_SKIP_DIRS | {"bin", "obj"}`` for .NET.
29
+ DEFAULT_SKIP_DIRS: frozenset[str] = frozenset(SKIP_DIRS | {"vendor", "bower_components", ".mypy_cache", ".pytest_cache"})
30
+
31
+ _BINARY_SNIFF_BYTES = 8192
32
+
33
+
34
+ def iter_repo_files(
35
+ root: str | Path,
36
+ *,
37
+ suffixes: Collection[str] | None = None,
38
+ names: Collection[str] | None = None,
39
+ skip_dirs: Collection[str] = DEFAULT_SKIP_DIRS,
40
+ max_bytes: int | None = None,
41
+ include_tests: bool = True,
42
+ on_skip: Callable[[Path, str], None] | None = None,
43
+ ) -> Iterator[Path]:
44
+ """Yield the repo's regular files, sorted and depth-first.
45
+
46
+ - ``suffixes`` (e.g. ``{".go"}``, case-insensitive) and ``names`` (exact
47
+ file names, e.g. ``{"go.mod"}``) filter the files; a file matching
48
+ either is yielded. With neither, every file is.
49
+ - ``skip_dirs`` are matched against directory *names inside the repo* and
50
+ pruned, so the walk never enters them, and directories above ``root``
51
+ never count.
52
+ - Files larger than ``max_bytes`` (default: the core per-file cap,
53
+ ``ATTACKMAP_MAX_FILE_BYTES``) are skipped.
54
+ - ``include_tests=False`` drops test/spec/fixture files (see
55
+ ``ATTACKMAP_INCLUDE_TESTS``).
56
+ - ``on_skip(path, reason)`` is called for each skipped symlink,
57
+ oversized file or directory excluded by an AttackMap output marker, so
58
+ a plugin can report what it didn't analyze.
59
+ """
60
+ root_path = Path(root)
61
+ wanted_suffixes = {s.lower() for s in suffixes} if suffixes is not None else None
62
+ wanted_names = set(names) if names is not None else None
63
+ skip = frozenset(skip_dirs)
64
+ cap = max_bytes if max_bytes is not None else max_file_bytes()
65
+
66
+ def _symlink(path: Path) -> None:
67
+ if on_skip is not None:
68
+ on_skip(path, "symlink not followed")
69
+
70
+ def _pruned(path: Path) -> None:
71
+ if on_skip is not None:
72
+ on_skip(path, "directory skipped (output marker)")
73
+
74
+ for path in walk_repo(root_path, prune=skip.__contains__, on_symlink=_symlink, on_prune=_pruned):
75
+ if wanted_suffixes is not None or wanted_names is not None:
76
+ if not (
77
+ (wanted_suffixes is not None and path.suffix.lower() in wanted_suffixes)
78
+ or (wanted_names is not None and path.name in wanted_names)
79
+ ):
80
+ continue
81
+ if not include_tests and is_test_file(rel(path, root_path)):
82
+ continue
83
+ try:
84
+ oversized = path.stat().st_size > cap
85
+ except OSError:
86
+ continue
87
+ if oversized:
88
+ if on_skip is not None:
89
+ on_skip(path, "oversized file skipped")
90
+ continue
91
+ yield path
92
+
93
+
94
+ def read_source(path: str | Path, *, root: str | Path | None = None) -> str | None:
95
+ """Read a source file as text, or return None if it can't be read.
96
+
97
+ Decodes UTF-8 (with or without a BOM), then falls back to cp1252 and finally
98
+ latin-1, so legacy-encoded sources are analyzed rather than dropped.
99
+ Returns None for unreadable files (``OSError``), binary files (a NUL byte
100
+ in the first 8 KiB) and, when ``root`` is given, paths outside it.
101
+ """
102
+ file_path = Path(path)
103
+ if root is not None:
104
+ if not file_path.is_absolute():
105
+ file_path = Path(root) / file_path
106
+ if not is_contained(root, file_path):
107
+ return None
108
+ try:
109
+ data = file_path.read_bytes()
110
+ except OSError:
111
+ return None
112
+ if b"\x00" in data[:_BINARY_SNIFF_BYTES]:
113
+ return None
114
+ for encoding in ("utf-8-sig", "cp1252"):
115
+ try:
116
+ return data.decode(encoding)
117
+ except UnicodeDecodeError:
118
+ continue
119
+ return data.decode("latin-1")
120
+
121
+
122
+ def rel(path: str | Path, root: str | Path) -> str:
123
+ """``path`` relative to ``root``, POSIX-style (``src/app.py``)."""
124
+ try:
125
+ return Path(path).relative_to(Path(root)).as_posix()
126
+ except ValueError:
127
+ return Path(path).as_posix()
128
+
129
+
130
+ def line_of(content: str, offset: int) -> int:
131
+ """1-indexed line of a character offset (e.g. ``match.start()``)."""
132
+ return line_number(content, offset)
133
+
134
+
135
+ def line_snippet(content: str, line: int, max_len: int = 200) -> str:
136
+ """The stripped text of 1-indexed ``line``, truncated to ``max_len``.
137
+
138
+ Lines are split on ``\n`` only, matching :func:`line_of`; ``str.splitlines``
139
+ would also break on form feeds and other separators common in legacy C and
140
+ put the snippet on the wrong line.
141
+ """
142
+ lines = content.split("\n")
143
+ if line < 1 or line > len(lines):
144
+ return ""
145
+ text = lines[line - 1].strip()
146
+ return text if len(text) <= max_len else text[: max_len - 1] + "…"
147
+
148
+
149
+ __all__ = [
150
+ "DEFAULT_SKIP_DIRS",
151
+ "iter_repo_files",
152
+ "read_source",
153
+ "rel",
154
+ "line_of",
155
+ "line_snippet",
156
+ ]
@@ -0,0 +1,59 @@
1
+ from __future__ import annotations
2
+
3
+ from ..models import (
4
+ CLOUD_MISCONFIG_KINDS,
5
+ CLOUD_MISCONFIG_PLATFORMS,
6
+ CloudMisconfig,
7
+ CloudMisconfigKind,
8
+ CloudMisconfigPlatform,
9
+ APP_SURFACE_ISSUE_KINDS,
10
+ DEPENDENCY_ECOSYSTEMS,
11
+ AppSurfaceIssue,
12
+ AppSurfaceIssueKind,
13
+ DependencyEcosystem,
14
+ DependencyHint,
15
+ CODE_WEAKNESS_KINDS,
16
+ CodeWeakness,
17
+ SupplyChainIssue,
18
+ )
19
+ from ..recon_models import (
20
+ AuthHint,
21
+ DatabaseHint,
22
+ EdgeHint,
23
+ EntrypointHint,
24
+ ExternalCall,
25
+ FrameworkHint,
26
+ ProtocolHint,
27
+ Route,
28
+ ScanResult,
29
+ SecretHint,
30
+ ServiceHint,
31
+ )
32
+
33
+ __all__ = [
34
+ "Route",
35
+ "ExternalCall",
36
+ "DatabaseHint",
37
+ "AuthHint",
38
+ "ServiceHint",
39
+ "EdgeHint",
40
+ "EntrypointHint",
41
+ "ProtocolHint",
42
+ "FrameworkHint",
43
+ "SecretHint",
44
+ "ScanResult",
45
+ "DependencyHint",
46
+ "DependencyEcosystem",
47
+ "DEPENDENCY_ECOSYSTEMS",
48
+ "CloudMisconfig",
49
+ "CloudMisconfigKind",
50
+ "CLOUD_MISCONFIG_KINDS",
51
+ "CloudMisconfigPlatform",
52
+ "CLOUD_MISCONFIG_PLATFORMS",
53
+ "AppSurfaceIssue",
54
+ "AppSurfaceIssueKind",
55
+ "APP_SURFACE_ISSUE_KINDS",
56
+ "CodeWeakness",
57
+ "CODE_WEAKNESS_KINDS",
58
+ "SupplyChainIssue",
59
+ ]
@@ -0,0 +1,161 @@
1
+ """Opt-in git-history secret scan (#252, ``--secrets-history N``).
2
+
3
+ Most leaked credentials live in history, not the working tree: a key is
4
+ committed, noticed, and "removed" in a later commit while every clone keeps
5
+ it. This pass runs the same detectors as the working-tree scan over the
6
+ lines *added* by the last ``N`` commits (all refs), and reports for each
7
+ secret the commit that introduced it and whether it is still in ``HEAD``.
8
+
9
+ Safety:
10
+ - Off by default; only ``--secrets-history`` runs it.
11
+ - Bounded: at most ``MAX_COMMITS`` commits, ``MAX_OUTPUT_BYTES`` of patch
12
+ text, ``MAX_LINE_CHARS`` per line and a wall-clock timeout. Hitting a bound
13
+ is recorded as a scan limitation, never silently.
14
+ - Git is invoked through ``gitref.hardened_git_command``: no hooks, pager,
15
+ fsmonitor, external diff, textconv or lazy fetch, and system/global config
16
+ ignored, so a hostile repo can't execute code through its git config.
17
+ - Raw secret values exist only in memory while matching (to fingerprint and
18
+ test ``still_in_head``); every emitted record is masked (#235).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import hashlib
24
+ import subprocess
25
+ import time
26
+ from pathlib import Path
27
+
28
+ from .gitref import hardened_git_command, hardened_git_env
29
+ from .models import SecretHistoryHit
30
+ from .redact import mask_secret
31
+ from .scanner import _line_snippet, iter_secret_matches
32
+ from .srcpaths import in_skipped_dir, is_test_file, is_vendored_file
33
+
34
+ DEFAULT_COMMITS = 100
35
+ MAX_COMMITS = 5000
36
+ MAX_OUTPUT_BYTES = 64 * 1024 * 1024
37
+ MAX_LINE_CHARS = 4096
38
+ TIMEOUT_SECONDS = 120.0
39
+ _COMMIT_MARKER = "\x1ecommit "
40
+
41
+
42
+ def _run(root: Path, *args: str) -> subprocess.CompletedProcess[bytes]:
43
+ return subprocess.run(
44
+ hardened_git_command(root, *args),
45
+ capture_output=True,
46
+ check=False,
47
+ stdin=subprocess.DEVNULL,
48
+ env=hardened_git_env(),
49
+ timeout=30,
50
+ )
51
+
52
+
53
+ def _head_text(root: Path, prefix: str, rel: str, cache: dict[str, str | None]) -> str | None:
54
+ if rel not in cache:
55
+ result = _run(root, "cat-file", "blob", f"HEAD:{prefix}{rel}")
56
+ cache[rel] = result.stdout.decode("utf-8", errors="replace") if result.returncode == 0 else None
57
+ return cache[rel]
58
+
59
+
60
+ def scan_secret_history(
61
+ root: str | Path,
62
+ commits: int = DEFAULT_COMMITS,
63
+ *,
64
+ max_output_bytes: int = MAX_OUTPUT_BYTES,
65
+ timeout: float = TIMEOUT_SECONDS,
66
+ ) -> tuple[list[SecretHistoryHit], list[str]]:
67
+ """Scan the patches of the last ``commits`` commits under ``root``.
68
+
69
+ Returns ``(hits, limitations)``. A directory that isn't a git checkout
70
+ yields no hits and one limitation line.
71
+ """
72
+ root_path = Path(root).resolve()
73
+ commits = max(1, min(int(commits), MAX_COMMITS))
74
+ probe = _run(root_path, "rev-parse", "--show-prefix")
75
+ if probe.returncode != 0:
76
+ return [], [f"--secrets-history skipped: {root_path} is not a git repository"]
77
+ prefix = probe.stdout.decode("utf-8", errors="replace").strip()
78
+
79
+ proc = subprocess.Popen(
80
+ hardened_git_command(
81
+ root_path, "log", "--all", "-n", str(commits), "-p", "-U0",
82
+ "--no-color", "--no-ext-diff", "--no-textconv", "--no-renames",
83
+ "--diff-filter=AM", f"--format=format:{_COMMIT_MARKER}%H",
84
+ "--", ".",
85
+ ),
86
+ stdout=subprocess.PIPE,
87
+ stderr=subprocess.DEVNULL,
88
+ stdin=subprocess.DEVNULL,
89
+ env=hardened_git_env(),
90
+ )
91
+ limitations: list[str] = []
92
+ # fingerprint -> [kind, file, line_text_masked, introduced_in, literal]
93
+ found: dict[tuple[str, str, str], list] = {}
94
+ commit = ""
95
+ current: str | None = None
96
+ read = 0
97
+ seen_commits: set[str] = set()
98
+ started = time.monotonic()
99
+ assert proc.stdout is not None
100
+ try:
101
+ for raw in proc.stdout:
102
+ read += len(raw)
103
+ if read > max_output_bytes:
104
+ limitations.append(f"--secrets-history stopped after {max_output_bytes} bytes of patch output")
105
+ break
106
+ if time.monotonic() - started > timeout:
107
+ limitations.append(f"--secrets-history stopped after {timeout:.0f}s")
108
+ break
109
+ line = raw.decode("utf-8", errors="replace").rstrip("\n")
110
+ if line.startswith(_COMMIT_MARKER):
111
+ commit = line[len(_COMMIT_MARKER):].strip()
112
+ seen_commits.add(commit)
113
+ current = None
114
+ continue
115
+ if line.startswith("+++ "):
116
+ target = line[4:]
117
+ current = target[2:] if target.startswith("b/") else None
118
+ if current is not None and prefix and current.startswith(prefix):
119
+ current = current[len(prefix):]
120
+ if current is not None and in_skipped_dir(Path(current)):
121
+ current = None
122
+ continue
123
+ if current is None or not line.startswith("+") or len(line) > MAX_LINE_CHARS:
124
+ continue
125
+ added = line[1:]
126
+ first_party = not is_test_file(current) and not is_vendored_file(current)
127
+ for kind, literal, offset, _confidence in iter_secret_matches(
128
+ added, entropy=False, generic=first_party, urls=first_party
129
+ ):
130
+ fingerprint = (kind, current, hashlib.sha256(literal.encode("utf-8")).hexdigest())
131
+ snippet = _line_snippet(added, offset).replace(literal, mask_secret(literal))
132
+ # `git log` is newest first, so the last sighting is the oldest:
133
+ # keep overwriting `introduced_in`.
134
+ found[fingerprint] = [kind, current, snippet, commit, literal]
135
+ finally:
136
+ proc.stdout.close()
137
+ if proc.poll() is None:
138
+ proc.kill()
139
+ proc.wait()
140
+
141
+ head_cache: dict[str, str | None] = {}
142
+ hits: list[SecretHistoryHit] = []
143
+ for kind, rel, snippet, introduced, literal in found.values():
144
+ head = _head_text(root_path, prefix, rel, head_cache)
145
+ hits.append(
146
+ SecretHistoryHit(
147
+ name=mask_secret(literal) if kind != "pem_private_key" else literal,
148
+ kind=kind,
149
+ file=rel,
150
+ introduced_in=introduced,
151
+ still_in_head=bool(head) and literal in head,
152
+ evidence_text=snippet,
153
+ )
154
+ )
155
+ found.clear()
156
+ hits.sort(key=lambda h: (h.still_in_head, h.file, h.kind, h.introduced_in))
157
+ if len(seen_commits) >= commits:
158
+ limitations.append(
159
+ f"--secrets-history scanned the last {commits} commit(s); older history was not checked"
160
+ )
161
+ return hits, limitations