hackscan 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hackscan/__init__.py +3 -0
- hackscan/analyzers/__init__.py +1 -0
- hackscan/analyzers/ast_pass.py +154 -0
- hackscan/analyzers/suppressions.py +63 -0
- hackscan/analyzers/taint_pass.py +808 -0
- hackscan/cli.py +326 -0
- hackscan/config.py +184 -0
- hackscan/core/__init__.py +1 -0
- hackscan/core/dedupe.py +147 -0
- hackscan/core/fingerprint.py +76 -0
- hackscan/core/models.py +270 -0
- hackscan/core/pipeline.py +164 -0
- hackscan/core/redact.py +84 -0
- hackscan/core/taxonomy.py +111 -0
- hackscan/importers/__init__.py +1 -0
- hackscan/importers/common.py +198 -0
- hackscan/importers/gitleaks.py +104 -0
- hackscan/importers/runner.py +161 -0
- hackscan/importers/sarif.py +164 -0
- hackscan/plugins/__init__.py +5 -0
- hackscan/plugins/base.py +261 -0
- hackscan/plugins/loader.py +98 -0
- hackscan/plugins/rules/__init__.py +1 -0
- hackscan/plugins/rules/cmdi.py +80 -0
- hackscan/plugins/rules/codei.py +50 -0
- hackscan/plugins/rules/sqli.py +127 -0
- hackscan/plugins/rules/weak_crypto.py +53 -0
- hackscan/plugins/scopes.py +263 -0
- hackscan/sarif/__init__.py +1 -0
- hackscan/sarif/generator.py +205 -0
- hackscan/taint/__init__.py +1 -0
- hackscan/taint/sanitizers.py +37 -0
- hackscan/taint/sources.py +99 -0
- hackscan-0.1.0.dist-info/METADATA +181 -0
- hackscan-0.1.0.dist-info/RECORD +38 -0
- hackscan-0.1.0.dist-info/WHEEL +4 -0
- hackscan-0.1.0.dist-info/entry_points.txt +2 -0
- hackscan-0.1.0.dist-info/licenses/LICENSE +21 -0
hackscan/cli.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Command-line interface.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 = no failing findings, 1 = `--fail-on` threshold hit,
|
|
4
|
+
2 = usage, configuration or plugin error, an incomplete scan (a file could not be
|
|
5
|
+
analyzed, unless `--allow-incomplete`), or (with `--strict-tools`) an external tool error.
|
|
6
|
+
An incomplete scan takes precedence over findings: its results cannot be trusted to be
|
|
7
|
+
complete, so it must not look like a pass or an ordinary failure.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import click
|
|
17
|
+
|
|
18
|
+
from hackscan import __version__
|
|
19
|
+
from hackscan.config import ConfigError, HackScanConfig, load, parse_import_format, parse_tools
|
|
20
|
+
from hackscan.core.models import OWN_SOURCE, Finding, Severity, Status
|
|
21
|
+
from hackscan.core.pipeline import ScanResult, scan
|
|
22
|
+
from hackscan.plugins.loader import PluginError, resolve_plugins
|
|
23
|
+
|
|
24
|
+
EXIT_OK, EXIT_FINDINGS, EXIT_ERROR = 0, 1, 2
|
|
25
|
+
SEVERITIES = [s.value for s in Severity]
|
|
26
|
+
TOOL_FAILURES = {"missing", "failed", "timeout", "partial"}
|
|
27
|
+
REPORT_SIGNATURES = ('"name": "HackScan"', '"name": "hackscan"', "HackScan ")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@click.group()
|
|
31
|
+
@click.version_option(__version__, prog_name="hackscan")
|
|
32
|
+
def main() -> None:
|
|
33
|
+
"""HackScan: Python SAST orchestrator and verifier."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@main.command("scan")
|
|
37
|
+
@click.argument("path", type=click.Path(exists=True, path_type=Path), default=".")
|
|
38
|
+
@click.option("--format", "fmt", type=click.Choice(["text", "json", "sarif"]), default="text")
|
|
39
|
+
@click.option("-o", "--output", type=click.Path(dir_okay=False, path_type=Path))
|
|
40
|
+
@click.option("--severity", type=click.Choice(SEVERITIES), help="Minimum severity to report.")
|
|
41
|
+
@click.option("--min-confidence", type=click.IntRange(0, 100), help="Minimum confidence.")
|
|
42
|
+
@click.option(
|
|
43
|
+
"--fail-on",
|
|
44
|
+
type=click.Choice(SEVERITIES),
|
|
45
|
+
help="Exit 1 if an open finding is at least this severe.",
|
|
46
|
+
)
|
|
47
|
+
@click.option("--ignore", multiple=True, help="Glob of paths to skip (repeatable).")
|
|
48
|
+
@click.option("--show-suppressed", is_flag=True, default=None, help="Include suppressed findings.")
|
|
49
|
+
@click.option(
|
|
50
|
+
"--plugins",
|
|
51
|
+
type=click.Path(exists=True, file_okay=False, path_type=Path),
|
|
52
|
+
help="Directory of custom rule plugins.",
|
|
53
|
+
)
|
|
54
|
+
@click.option("--with", "with_tools", help="Run external tools: semgrep,bandit,gitleaks.")
|
|
55
|
+
@click.option(
|
|
56
|
+
"--import",
|
|
57
|
+
"imports",
|
|
58
|
+
multiple=True,
|
|
59
|
+
metavar="FORMAT=FILE",
|
|
60
|
+
help="Import a report: sarif|semgrep|bandit|codeql|gitleaks=FILE (repeatable).",
|
|
61
|
+
)
|
|
62
|
+
@click.option("--tool-timeout", type=click.IntRange(0), help="Seconds per tool (0 = none).")
|
|
63
|
+
@click.option(
|
|
64
|
+
"--strict-tools", is_flag=True, default=None, help="Exit 2 if an external tool fails."
|
|
65
|
+
)
|
|
66
|
+
@click.option(
|
|
67
|
+
"--sarif-omit-suppressed",
|
|
68
|
+
is_flag=True,
|
|
69
|
+
default=False,
|
|
70
|
+
help="Leave suppressed findings out of SARIF (recommended for GitHub uploads).",
|
|
71
|
+
)
|
|
72
|
+
@click.option(
|
|
73
|
+
"--allow-incomplete",
|
|
74
|
+
is_flag=True,
|
|
75
|
+
default=None,
|
|
76
|
+
help="Do not exit 2 when some files cannot be analyzed (they are still reported).",
|
|
77
|
+
)
|
|
78
|
+
@click.option("--no-taint", is_flag=True, default=False, help="Skip the taint pass.")
|
|
79
|
+
@click.option("--jobs", type=click.IntRange(0), help="Worker processes (0 = automatic).")
|
|
80
|
+
@click.option(
|
|
81
|
+
"--config",
|
|
82
|
+
"config_path",
|
|
83
|
+
type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
|
84
|
+
help="Use this config file instead of discovering .hackscan.yml.",
|
|
85
|
+
)
|
|
86
|
+
@click.option("-q", "--quiet", is_flag=True, help="Only print findings (no header/summary).")
|
|
87
|
+
def scan_command(
|
|
88
|
+
path: Path,
|
|
89
|
+
fmt: str,
|
|
90
|
+
output: Path | None,
|
|
91
|
+
quiet: bool,
|
|
92
|
+
config_path: Path | None,
|
|
93
|
+
no_taint: bool,
|
|
94
|
+
with_tools: str | None,
|
|
95
|
+
imports: tuple[str, ...],
|
|
96
|
+
sarif_omit_suppressed: bool,
|
|
97
|
+
**options,
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Scan PATH (a directory or a Python file) for vulnerabilities."""
|
|
100
|
+
if output is not None:
|
|
101
|
+
_check_output(output, path)
|
|
102
|
+
try:
|
|
103
|
+
config = load(path, config_path).with_overrides(
|
|
104
|
+
severity=Severity(options["severity"]) if options["severity"] else None,
|
|
105
|
+
fail_on=Severity(options["fail_on"]) if options["fail_on"] else None,
|
|
106
|
+
min_confidence=options["min_confidence"],
|
|
107
|
+
ignore=tuple(options["ignore"]) or None,
|
|
108
|
+
show_suppressed=options["show_suppressed"],
|
|
109
|
+
plugins=options["plugins"].resolve() if options["plugins"] else None,
|
|
110
|
+
tool_timeout=options["tool_timeout"],
|
|
111
|
+
strict_tools=options["strict_tools"],
|
|
112
|
+
allow_incomplete=options["allow_incomplete"],
|
|
113
|
+
jobs=options["jobs"],
|
|
114
|
+
taint=False if no_taint else None,
|
|
115
|
+
with_tools=parse_tools(with_tools, click.BadParameter) if with_tools else None,
|
|
116
|
+
imports=_parse_imports(imports) if imports else None,
|
|
117
|
+
)
|
|
118
|
+
result = scan(path, config)
|
|
119
|
+
plugins = resolve_plugins(config.plugins)
|
|
120
|
+
except (ConfigError, PluginError, click.BadParameter) as exc:
|
|
121
|
+
click.echo(f"hackscan: error: {exc}", err=True)
|
|
122
|
+
sys.exit(EXIT_ERROR)
|
|
123
|
+
|
|
124
|
+
if fmt == "sarif":
|
|
125
|
+
from hackscan.sarif.generator import export_sarif
|
|
126
|
+
|
|
127
|
+
shown = _visible(result.findings, config, include_suppressed=True)
|
|
128
|
+
log = export_sarif(_with(result, shown), plugins, omit_suppressed=sarif_omit_suppressed)
|
|
129
|
+
text = json.dumps(log, indent=2)
|
|
130
|
+
elif fmt == "json":
|
|
131
|
+
shown = _visible(result.findings, config, config.show_suppressed)
|
|
132
|
+
text = json.dumps(_json(result, shown), indent=2)
|
|
133
|
+
else:
|
|
134
|
+
shown = _visible(result.findings, config, config.show_suppressed)
|
|
135
|
+
text = _text(result, shown, config, quiet)
|
|
136
|
+
|
|
137
|
+
if output is not None:
|
|
138
|
+
try:
|
|
139
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
140
|
+
output.write_text(text + "\n", encoding="utf-8")
|
|
141
|
+
except OSError as exc:
|
|
142
|
+
click.echo(f"hackscan: error: cannot write {output}: {exc}", err=True)
|
|
143
|
+
sys.exit(EXIT_ERROR)
|
|
144
|
+
if not quiet:
|
|
145
|
+
click.echo(f"hackscan: wrote {fmt} report to {output}", err=True)
|
|
146
|
+
else:
|
|
147
|
+
click.echo(text)
|
|
148
|
+
|
|
149
|
+
if result.errors and not config.allow_incomplete:
|
|
150
|
+
click.echo(
|
|
151
|
+
f"hackscan: error: scan incomplete, {len(result.errors)} file(s) or report(s) "
|
|
152
|
+
"could not be analyzed (see errors above; --allow-incomplete to accept)",
|
|
153
|
+
err=True,
|
|
154
|
+
)
|
|
155
|
+
sys.exit(EXIT_ERROR)
|
|
156
|
+
if config.strict_tools and any(r.status in TOOL_FAILURES for r in result.tool_runs):
|
|
157
|
+
click.echo("hackscan: error: an external tool failed (--strict-tools)", err=True)
|
|
158
|
+
sys.exit(EXIT_ERROR)
|
|
159
|
+
if config.fail_on is not None and _failing(result.findings, config):
|
|
160
|
+
sys.exit(EXIT_FINDINGS)
|
|
161
|
+
sys.exit(EXIT_OK)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@main.command("rules")
|
|
165
|
+
@click.option("--plugins", type=click.Path(exists=True, file_okay=False, path_type=Path))
|
|
166
|
+
def rules_command(plugins: Path | None) -> None:
|
|
167
|
+
"""List the detection rules."""
|
|
168
|
+
try:
|
|
169
|
+
loaded = resolve_plugins(plugins)
|
|
170
|
+
except PluginError as exc:
|
|
171
|
+
click.echo(f"hackscan: error: {exc}", err=True)
|
|
172
|
+
sys.exit(EXIT_ERROR)
|
|
173
|
+
for p in loaded:
|
|
174
|
+
cwe = ", ".join(p.cwe) or "-"
|
|
175
|
+
click.echo(f"{p.rule_id:<16} {p.severity.value:<8} {cwe:<10} {p.description}")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
# -- helpers ------------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _check_output(output: Path, target: Path) -> None:
|
|
182
|
+
"""A report never overwrites anything but a previous HackScan report.
|
|
183
|
+
|
|
184
|
+
New files are fine unless they would be Python source; an existing file is only
|
|
185
|
+
replaced if it already is a HackScan report (text, JSON or SARIF), so `-o
|
|
186
|
+
pyproject.toml` or `-o app.py` cannot destroy project files.
|
|
187
|
+
"""
|
|
188
|
+
resolved = output.resolve()
|
|
189
|
+
refuse = resolved.suffix in {".py", ".pyw", ".pyi"} or resolved == target.resolve()
|
|
190
|
+
if not refuse and resolved.exists():
|
|
191
|
+
refuse = resolved.is_dir() or not _is_hackscan_report(resolved)
|
|
192
|
+
if refuse:
|
|
193
|
+
click.echo(
|
|
194
|
+
f"hackscan: error: refusing to write the report to {output} "
|
|
195
|
+
"(existing file is not a HackScan report)",
|
|
196
|
+
err=True,
|
|
197
|
+
)
|
|
198
|
+
sys.exit(EXIT_ERROR)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _is_hackscan_report(path: Path) -> bool:
|
|
202
|
+
try:
|
|
203
|
+
with path.open("r", encoding="utf-8", errors="replace") as fh:
|
|
204
|
+
head = fh.read(4096)
|
|
205
|
+
except OSError:
|
|
206
|
+
return False
|
|
207
|
+
return head.startswith("HackScan ") or any(sig in head for sig in REPORT_SIGNATURES[:2])
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _parse_imports(values: tuple[str, ...]) -> tuple[tuple[str, Path], ...]:
|
|
211
|
+
out = []
|
|
212
|
+
for value in values:
|
|
213
|
+
fmt, sep, file = value.partition("=")
|
|
214
|
+
if not sep or not file:
|
|
215
|
+
raise click.BadParameter(f"--import expects FORMAT=FILE, got {value!r}")
|
|
216
|
+
report = Path(file)
|
|
217
|
+
if not report.is_file():
|
|
218
|
+
raise click.BadParameter(f"--import report not found: {file}")
|
|
219
|
+
out.append((parse_import_format(fmt, click.BadParameter), report.resolve()))
|
|
220
|
+
return tuple(out)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _visible(
|
|
224
|
+
findings: list[Finding], config: HackScanConfig, include_suppressed: bool
|
|
225
|
+
) -> list[Finding]:
|
|
226
|
+
return [
|
|
227
|
+
f
|
|
228
|
+
for f in findings
|
|
229
|
+
if f.severity.rank >= config.severity.rank
|
|
230
|
+
and f.confidence >= config.min_confidence
|
|
231
|
+
and (include_suppressed or f.status is not Status.SUPPRESSED)
|
|
232
|
+
]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _failing(findings: list[Finding], config: HackScanConfig) -> list[Finding]:
|
|
236
|
+
"""Open (candidate/confirmed) findings at or above --fail-on; suppressed never count."""
|
|
237
|
+
assert config.fail_on is not None
|
|
238
|
+
return [
|
|
239
|
+
f
|
|
240
|
+
for f in findings
|
|
241
|
+
if f.status is not Status.SUPPRESSED
|
|
242
|
+
and f.severity.rank >= config.fail_on.rank
|
|
243
|
+
and f.confidence >= config.min_confidence
|
|
244
|
+
]
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _with(result: ScanResult, findings: list[Finding]) -> ScanResult:
|
|
248
|
+
return ScanResult(
|
|
249
|
+
root=result.root,
|
|
250
|
+
findings=findings,
|
|
251
|
+
files_scanned=result.files_scanned,
|
|
252
|
+
duration_seconds=result.duration_seconds,
|
|
253
|
+
errors=result.errors,
|
|
254
|
+
warnings=result.warnings,
|
|
255
|
+
tool_runs=result.tool_runs,
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _json(result: ScanResult, findings: list[Finding]) -> dict:
|
|
260
|
+
return {
|
|
261
|
+
"tool": {"name": "hackscan", "version": __version__},
|
|
262
|
+
"root": result.root.as_posix(),
|
|
263
|
+
"stats": {
|
|
264
|
+
"files_scanned": result.files_scanned,
|
|
265
|
+
"duration_seconds": round(result.duration_seconds, 3),
|
|
266
|
+
"findings": len(findings),
|
|
267
|
+
"by_status": {s.value: sum(f.status is s for f in findings) for s in Status},
|
|
268
|
+
},
|
|
269
|
+
"findings": [f.to_dict() for f in findings],
|
|
270
|
+
"errors": result.errors,
|
|
271
|
+
"warnings": result.warnings,
|
|
272
|
+
"tool_runs": [vars(r) for r in result.tool_runs],
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _text(result: ScanResult, findings: list[Finding], config: HackScanConfig, quiet: bool) -> str:
|
|
277
|
+
lines: list[str] = []
|
|
278
|
+
if not quiet:
|
|
279
|
+
lines.append(
|
|
280
|
+
f"HackScan {__version__} - scanned {result.files_scanned} files "
|
|
281
|
+
f"in {result.duration_seconds:.2f}s"
|
|
282
|
+
)
|
|
283
|
+
lines.append("")
|
|
284
|
+
ordered = sorted(
|
|
285
|
+
findings,
|
|
286
|
+
key=lambda f: (-f.severity.rank, f.status is not Status.CONFIRMED, f.sort_key()),
|
|
287
|
+
)
|
|
288
|
+
for f in ordered:
|
|
289
|
+
loc = f"{f.location.path}:{f.location.start_line}:{f.location.start_column}"
|
|
290
|
+
status = f.status.value
|
|
291
|
+
if f.status is Status.SUPPRESSED:
|
|
292
|
+
status = f"suppressed ({f.suppression})"
|
|
293
|
+
lines.append(
|
|
294
|
+
f"{f.severity.value.upper():<8} {status:<10} {loc} {f.rule_id} [{f.confidence}%]"
|
|
295
|
+
)
|
|
296
|
+
lines.append(f" {f.message}")
|
|
297
|
+
if f.snippet:
|
|
298
|
+
lines.append(f" | {f.snippet.splitlines()[0].strip()}")
|
|
299
|
+
lines.extend(
|
|
300
|
+
f" > {e.message}" for e in f.evidence if e.kind in {"taint_step", "taint_verdict"}
|
|
301
|
+
)
|
|
302
|
+
if f.sources != (OWN_SOURCE,):
|
|
303
|
+
lines.append(f" sources: {', '.join(f.sources)}")
|
|
304
|
+
if quiet:
|
|
305
|
+
return "\n".join(lines)
|
|
306
|
+
counts = {s: sum(f.status is s for f in findings) for s in Status}
|
|
307
|
+
hidden = sum(f.status is Status.SUPPRESSED for f in result.findings) - counts[Status.SUPPRESSED]
|
|
308
|
+
if not findings:
|
|
309
|
+
lines.append("No findings.")
|
|
310
|
+
lines.append("")
|
|
311
|
+
summary = (
|
|
312
|
+
f"Summary: {counts[Status.CONFIRMED]} confirmed, {counts[Status.CANDIDATE]} candidates"
|
|
313
|
+
)
|
|
314
|
+
if counts[Status.SUPPRESSED]:
|
|
315
|
+
summary += f", {counts[Status.SUPPRESSED]} suppressed shown"
|
|
316
|
+
if hidden:
|
|
317
|
+
summary += f", {hidden} suppressed hidden (--show-suppressed)"
|
|
318
|
+
lines.append(summary)
|
|
319
|
+
if result.errors:
|
|
320
|
+
lines.append(f"INCOMPLETE: {len(result.errors)} file(s) could not be analyzed")
|
|
321
|
+
lines.extend(f"error: {e}" for e in result.errors)
|
|
322
|
+
lines.extend(f"warning: {w}" for w in result.warnings)
|
|
323
|
+
if config.fail_on is not None:
|
|
324
|
+
failing = len(_failing(result.findings, config))
|
|
325
|
+
lines.append(f"fail-on {config.fail_on.value}: {failing} open finding(s) at or above")
|
|
326
|
+
return "\n".join(lines)
|
hackscan/config.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Configuration: `.hackscan.yml` discovery, validation and CLI overrides.
|
|
2
|
+
|
|
3
|
+
Discovery is hierarchical: every `.hackscan.yml` from the filesystem root down to the
|
|
4
|
+
scan root is loaded and merged, nearer files overriding farther ones (lists such as
|
|
5
|
+
`ignore` are concatenated). CLI options override the merged file configuration.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass, field, replace
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
from hackscan.core.models import Severity
|
|
17
|
+
|
|
18
|
+
CONFIG_NAME = ".hackscan.yml"
|
|
19
|
+
# Keys accepted in .hackscan.yml (and only these: anything else is an error, so a typo
|
|
20
|
+
# never silently disables a setting).
|
|
21
|
+
CONFIG_KEYS = frozenset(
|
|
22
|
+
{
|
|
23
|
+
"ignore",
|
|
24
|
+
"severity",
|
|
25
|
+
"min-confidence",
|
|
26
|
+
"fail-on",
|
|
27
|
+
"show-suppressed",
|
|
28
|
+
"plugins",
|
|
29
|
+
"taint",
|
|
30
|
+
"with",
|
|
31
|
+
"import",
|
|
32
|
+
"tool-timeout",
|
|
33
|
+
"strict-tools",
|
|
34
|
+
"allow-incomplete",
|
|
35
|
+
"jobs",
|
|
36
|
+
}
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
DEFAULT_IGNORES = (
|
|
40
|
+
".git",
|
|
41
|
+
".hg",
|
|
42
|
+
".svn",
|
|
43
|
+
".venv",
|
|
44
|
+
"venv",
|
|
45
|
+
"env",
|
|
46
|
+
".env",
|
|
47
|
+
".tox",
|
|
48
|
+
".nox",
|
|
49
|
+
"node_modules",
|
|
50
|
+
"build",
|
|
51
|
+
"dist",
|
|
52
|
+
"site-packages",
|
|
53
|
+
"__pycache__",
|
|
54
|
+
".mypy_cache",
|
|
55
|
+
".pytest_cache",
|
|
56
|
+
".ruff_cache",
|
|
57
|
+
".eggs",
|
|
58
|
+
"*.egg-info",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
KNOWN_TOOLS = ("semgrep", "bandit", "gitleaks")
|
|
62
|
+
IMPORT_FORMATS = ("sarif", "semgrep", "bandit", "codeql", "gitleaks")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ConfigError(Exception):
|
|
66
|
+
pass
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class HackScanConfig:
|
|
71
|
+
ignore: tuple[str, ...] = ()
|
|
72
|
+
severity: Severity = Severity.LOW # minimum severity reported
|
|
73
|
+
min_confidence: int = 0
|
|
74
|
+
fail_on: Severity | None = None
|
|
75
|
+
show_suppressed: bool = False
|
|
76
|
+
plugins: Path | None = None
|
|
77
|
+
taint: bool = True
|
|
78
|
+
with_tools: tuple[str, ...] = ()
|
|
79
|
+
imports: tuple[tuple[str, Path], ...] = () # (format, report file)
|
|
80
|
+
tool_timeout: int = 300
|
|
81
|
+
strict_tools: bool = False
|
|
82
|
+
allow_incomplete: bool = False # files that cannot be analyzed do not fail the run
|
|
83
|
+
jobs: int = 0 # 0 = automatic
|
|
84
|
+
sources: tuple[Path, ...] = field(default=(), compare=False) # config files loaded
|
|
85
|
+
|
|
86
|
+
def with_overrides(self, **overrides: Any) -> HackScanConfig:
|
|
87
|
+
"""Apply CLI overrides; `None` means "not given". `ignore` is appended."""
|
|
88
|
+
values = {k: v for k, v in overrides.items() if v is not None}
|
|
89
|
+
if "ignore" in values:
|
|
90
|
+
values["ignore"] = (*self.ignore, *values["ignore"])
|
|
91
|
+
return replace(self, **values)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def discover(scan_root: Path) -> list[Path]:
|
|
95
|
+
"""Config files from the filesystem root down to `scan_root` (nearest last)."""
|
|
96
|
+
start = scan_root if scan_root.is_dir() else scan_root.parent
|
|
97
|
+
found = [d / CONFIG_NAME for d in [start.resolve(), *start.resolve().parents]]
|
|
98
|
+
return [p for p in reversed(found) if p.is_file()]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def load(scan_root: Path, explicit: Path | None = None) -> HackScanConfig:
|
|
102
|
+
paths = [explicit] if explicit is not None else discover(scan_root)
|
|
103
|
+
config = HackScanConfig()
|
|
104
|
+
for path in paths:
|
|
105
|
+
config = _merge(config, _parse(path), path)
|
|
106
|
+
return config
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _parse(path: Path) -> dict[str, Any]:
|
|
110
|
+
try:
|
|
111
|
+
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
112
|
+
except (OSError, yaml.YAMLError) as exc:
|
|
113
|
+
raise ConfigError(f"{path}: cannot read config: {exc}") from exc
|
|
114
|
+
if not isinstance(data, dict):
|
|
115
|
+
raise ConfigError(f"{path}: top level must be a mapping")
|
|
116
|
+
unknown = sorted(set(data) - CONFIG_KEYS)
|
|
117
|
+
if unknown:
|
|
118
|
+
raise ConfigError(f"{path}: unknown option(s): {', '.join(unknown)}")
|
|
119
|
+
return data
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _merge(config: HackScanConfig, data: dict[str, Any], path: Path) -> HackScanConfig:
|
|
123
|
+
def err(message: str) -> ConfigError:
|
|
124
|
+
return ConfigError(f"{path}: {message}")
|
|
125
|
+
|
|
126
|
+
values: dict[str, Any] = {}
|
|
127
|
+
if "ignore" in data:
|
|
128
|
+
ignore = data["ignore"]
|
|
129
|
+
if not isinstance(ignore, list) or not all(isinstance(i, str) for i in ignore):
|
|
130
|
+
raise err("`ignore` must be a list of glob strings")
|
|
131
|
+
values["ignore"] = (*config.ignore, *ignore)
|
|
132
|
+
for key in ("severity", "fail-on"):
|
|
133
|
+
if key in data:
|
|
134
|
+
try:
|
|
135
|
+
values[key.replace("-", "_")] = Severity(str(data[key]).lower())
|
|
136
|
+
except ValueError:
|
|
137
|
+
options = ", ".join(s.value for s in Severity)
|
|
138
|
+
raise err(f"`{key}` must be one of: {options}") from None
|
|
139
|
+
if "min-confidence" in data:
|
|
140
|
+
value = data["min-confidence"]
|
|
141
|
+
if not isinstance(value, int) or not 0 <= value <= 100:
|
|
142
|
+
raise err("`min-confidence` must be an integer 0-100")
|
|
143
|
+
values["min_confidence"] = value
|
|
144
|
+
for key in ("show-suppressed", "taint", "strict-tools", "allow-incomplete"):
|
|
145
|
+
if key in data:
|
|
146
|
+
if not isinstance(data[key], bool):
|
|
147
|
+
raise err(f"`{key}` must be true or false")
|
|
148
|
+
values[key.replace("-", "_")] = data[key]
|
|
149
|
+
for key in ("tool-timeout", "jobs"):
|
|
150
|
+
if key in data:
|
|
151
|
+
if not isinstance(data[key], int) or data[key] < 0:
|
|
152
|
+
raise err(f"`{key}` must be a non-negative integer")
|
|
153
|
+
values[key.replace("-", "_")] = data[key]
|
|
154
|
+
if "plugins" in data:
|
|
155
|
+
values["plugins"] = (path.parent / str(data["plugins"])).resolve()
|
|
156
|
+
if "with" in data:
|
|
157
|
+
values["with_tools"] = parse_tools(data["with"], err)
|
|
158
|
+
if "import" in data:
|
|
159
|
+
items = data["import"]
|
|
160
|
+
if not isinstance(items, dict):
|
|
161
|
+
raise err("`import` must map format to report path, e.g. {codeql: results.sarif}")
|
|
162
|
+
values["imports"] = tuple(
|
|
163
|
+
(parse_import_format(fmt, err), (path.parent / str(p)).resolve())
|
|
164
|
+
for fmt, p in items.items()
|
|
165
|
+
)
|
|
166
|
+
return replace(config, sources=(*config.sources, path), **values)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def parse_tools(value: Any, err=ConfigError) -> tuple[str, ...]:
|
|
170
|
+
items = value.split(",") if isinstance(value, str) else value
|
|
171
|
+
if not isinstance(items, (list, tuple)):
|
|
172
|
+
raise err("`with` must be a list or comma-separated string of tools")
|
|
173
|
+
tools = tuple(dict.fromkeys(str(t).strip().lower() for t in items if str(t).strip()))
|
|
174
|
+
unknown = [t for t in tools if t not in KNOWN_TOOLS]
|
|
175
|
+
if unknown:
|
|
176
|
+
raise err(f"unknown tool(s) {', '.join(unknown)}; supported: {', '.join(KNOWN_TOOLS)}")
|
|
177
|
+
return tools
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def parse_import_format(value: Any, err=ConfigError) -> str:
|
|
181
|
+
fmt = str(value).strip().lower()
|
|
182
|
+
if fmt not in IMPORT_FORMATS:
|
|
183
|
+
raise err(f"unknown import format {fmt!r}; supported: {', '.join(IMPORT_FORMATS)}")
|
|
184
|
+
return fmt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Core data model, taxonomy, fingerprinting and dedupe."""
|
hackscan/core/dedupe.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Cross-source dedupe with a deterministic, order-independent merge policy.
|
|
2
|
+
|
|
3
|
+
Pipeline order: collect (own engine + importers) -> `merge_findings` -> `assign_ids`.
|
|
4
|
+
|
|
5
|
+
Clustering is not transitive. Invariant: every pair of findings in a cluster overlaps.
|
|
6
|
+
Findings are visited from most to least precise (smallest region first, own engine breaks
|
|
7
|
+
ties) and each joins the first cluster in which it overlaps *every* member; otherwise it
|
|
8
|
+
starts a new cluster. Two findings that do not overlap each other (distinct sinks) can
|
|
9
|
+
therefore never be fused, whatever a third, broader report covers.
|
|
10
|
+
|
|
11
|
+
Overlap alone is not identity: nested sinks such as `eval(eval(x))` overlap. Two findings
|
|
12
|
+
that share a source but record different sinks are distinct reports from the same producer
|
|
13
|
+
and never merge, so a merge never reports fewer findings than any single source did.
|
|
14
|
+
|
|
15
|
+
A finding may join a cluster only when it shares `vuln_class` and path with its members,
|
|
16
|
+
their regions overlap (column-aware), and both identities are grounded
|
|
17
|
+
(see `taxonomy.is_mergeable_class`).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import math
|
|
23
|
+
from collections import defaultdict
|
|
24
|
+
from collections.abc import Iterable
|
|
25
|
+
from dataclasses import replace
|
|
26
|
+
|
|
27
|
+
from hackscan.core.fingerprint import normalize_expression, sink_text
|
|
28
|
+
from hackscan.core.models import OWN_SOURCE, Finding, Region, Status
|
|
29
|
+
from hackscan.core.taxonomy import is_mergeable_class, normalize_cwes
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def merge_findings(findings: Iterable[Finding]) -> list[Finding]:
|
|
33
|
+
"""Merge duplicates across sources. Output is sorted by `Finding.sort_key`."""
|
|
34
|
+
buckets: dict[tuple[str, str], list[Finding]] = defaultdict(list)
|
|
35
|
+
singles: list[Finding] = []
|
|
36
|
+
for f in findings:
|
|
37
|
+
if is_mergeable_class(f.vuln_class, f.cwe):
|
|
38
|
+
buckets[(f.vuln_class, f.location.path)].append(f)
|
|
39
|
+
else:
|
|
40
|
+
singles.append(f)
|
|
41
|
+
|
|
42
|
+
merged: list[Finding] = list(singles)
|
|
43
|
+
for members in buckets.values():
|
|
44
|
+
for cluster in _anchor_clusters(members):
|
|
45
|
+
merged.append(_merge_cluster(cluster) if len(cluster) > 1 else cluster[0])
|
|
46
|
+
merged.sort(key=Finding.sort_key)
|
|
47
|
+
return merged
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _precision_key(f: Finding) -> tuple:
|
|
51
|
+
"""Total order: most precise region first; own engine breaks ties, then full content.
|
|
52
|
+
|
|
53
|
+
Precision must come first: a broad anchor would absorb separate sinks it spans.
|
|
54
|
+
"""
|
|
55
|
+
return (
|
|
56
|
+
f.location.span,
|
|
57
|
+
f.location.start_pos,
|
|
58
|
+
OWN_SOURCE not in f.sources,
|
|
59
|
+
f.canonical_json(),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _anchor_clusters(members: list[Finding]) -> list[list[Finding]]:
|
|
64
|
+
clusters: list[list[Finding]] = []
|
|
65
|
+
for f in sorted(members, key=_precision_key):
|
|
66
|
+
for cluster in clusters:
|
|
67
|
+
if all(_compatible(m, f) for m in cluster):
|
|
68
|
+
cluster.append(f)
|
|
69
|
+
break
|
|
70
|
+
else:
|
|
71
|
+
clusters.append([f])
|
|
72
|
+
return clusters
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _compatible(a: Finding, b: Finding) -> bool:
|
|
76
|
+
if not a.location.overlaps(b.location):
|
|
77
|
+
return False
|
|
78
|
+
if set(a.sources) & set(b.sources):
|
|
79
|
+
return normalize_expression(sink_text(a)) == normalize_expression(sink_text(b))
|
|
80
|
+
return True
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _primary(cluster: list[Finding]) -> Finding:
|
|
84
|
+
"""Own-engine finding wins; otherwise the lexicographically smallest rule id."""
|
|
85
|
+
own = [f for f in cluster if OWN_SOURCE in f.sources]
|
|
86
|
+
pool = own or cluster
|
|
87
|
+
return min(pool, key=lambda f: (f.rule_id, f.canonical_json()))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _merge_cluster(cluster: list[Finding]) -> Finding:
|
|
91
|
+
primary = _primary(cluster)
|
|
92
|
+
ordered = sorted(cluster, key=lambda f: f.canonical_json())
|
|
93
|
+
status = max((f.status for f in cluster), key=lambda s: s.rank)
|
|
94
|
+
suppression = None
|
|
95
|
+
if status is Status.SUPPRESSED:
|
|
96
|
+
suppression = ";".join(sorted({f.suppression for f in cluster if f.suppression}))
|
|
97
|
+
|
|
98
|
+
rules = {f.rule_id for f in cluster} | {r for f in cluster for r in f.related_rules}
|
|
99
|
+
rules.discard(primary.rule_id)
|
|
100
|
+
|
|
101
|
+
# Concatenation (duplicates kept: each contributor's evidence stands on its own).
|
|
102
|
+
evidence = sorted(
|
|
103
|
+
(e for f in cluster for e in f.evidence),
|
|
104
|
+
key=lambda e: (e.producer, e.kind, e.message, e.region.sort_key() if e.region else ()),
|
|
105
|
+
)
|
|
106
|
+
fix = primary.fix or next((f.fix for f in ordered if f.fix is not None), None)
|
|
107
|
+
sink = primary.sink or next((f.sink for f in _by_precedence(cluster) if f.sink), "")
|
|
108
|
+
function = primary.function or next(
|
|
109
|
+
(f.function for f in _by_precedence(cluster) if f.function), None
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
return replace(
|
|
113
|
+
primary,
|
|
114
|
+
location=_union(f.location for f in cluster),
|
|
115
|
+
severity=max((f.severity for f in cluster), key=lambda s: s.rank),
|
|
116
|
+
confidence=max(f.confidence for f in cluster),
|
|
117
|
+
status=status,
|
|
118
|
+
suppression=suppression,
|
|
119
|
+
sources=tuple(sorted({s for f in cluster for s in f.sources})),
|
|
120
|
+
related_rules=tuple(sorted(rules)),
|
|
121
|
+
cwe=normalize_cwes(c for f in cluster for c in f.cwe),
|
|
122
|
+
evidence=tuple(evidence),
|
|
123
|
+
fix=fix,
|
|
124
|
+
sink=sink,
|
|
125
|
+
function=function,
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _by_precedence(cluster: list[Finding]) -> list[Finding]:
|
|
130
|
+
"""Own-engine contributors first, then by rule id and content."""
|
|
131
|
+
return sorted(
|
|
132
|
+
cluster, key=lambda f: (OWN_SOURCE not in f.sources, f.rule_id, f.canonical_json())
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _union(regions: Iterable[Region]) -> Region:
|
|
137
|
+
"""Smallest region covering every contributor (using effective, exclusive ends)."""
|
|
138
|
+
regions = list(regions)
|
|
139
|
+
start_line, start_column = min(r.start_pos for r in regions)
|
|
140
|
+
end_line, end_column = max(r.end_pos for r in regions)
|
|
141
|
+
return Region(
|
|
142
|
+
path=regions[0].path,
|
|
143
|
+
start_line=start_line,
|
|
144
|
+
start_column=int(start_column),
|
|
145
|
+
end_line=end_line,
|
|
146
|
+
end_column=None if end_column == math.inf else int(end_column),
|
|
147
|
+
)
|