hackscan 0.1.0__py3-none-any.whl

hackscan/core/taxonomy.py

@@ -0,0 +1,111 @@
+"""Canonical vulnerability classes, mapped from rule ids and CWEs.
+
+`vuln_class` is the vendor-independent identity used for fingerprinting and dedupe, so
+the same issue reported by HackScan, Semgrep and Bandit lands in the same class.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+
+SQLI = "sqli"
+CMDI = "cmdi"
+CODEI = "codei"
+WEAK_CRYPTO = "weak_crypto"
+SECRET = "secret"
+
+KNOWN_CLASSES = frozenset({SQLI, CMDI, CODEI, WEAK_CRYPTO, SECRET})
+
+# Classes the own taint engine models; only these may be suppressed by taint.
+TAINT_CLASSES = frozenset({SQLI, CMDI, CODEI})
+
+OTHER_PREFIX = "other:"
+
+_CWE_TO_CLASS = {
+    "CWE-89": SQLI,
+    "CWE-564": SQLI,
+    "CWE-77": CMDI,
+    "CWE-78": CMDI,
+    "CWE-88": CMDI,
+    "CWE-94": CODEI,
+    "CWE-95": CODEI,
+    "CWE-327": WEAK_CRYPTO,
+    "CWE-328": WEAK_CRYPTO,
+    "CWE-916": WEAK_CRYPTO,
+    "CWE-259": SECRET,
+    "CWE-321": SECRET,
+    "CWE-798": SECRET,
+}
+
+# Exact rule ids (tool-qualified) that map without needing a CWE.
+_RULE_TO_CLASS = {
+    # Bandit
+    "bandit:B608": SQLI,
+    "bandit:B602": CMDI,
+    "bandit:B604": CMDI,
+    "bandit:B605": CMDI,
+    "bandit:B609": CMDI,
+    "bandit:B102": CODEI,
+    "bandit:B307": CODEI,
+    "bandit:B303": WEAK_CRYPTO,
+    "bandit:B324": WEAK_CRYPTO,
+    "bandit:B105": SECRET,
+    "bandit:B106": SECRET,
+    "bandit:B107": SECRET,
+}
+
+# Own rules use the HS-<CLASS>-NNN scheme.
+_OWN_RULE_PREFIX = {
+    "HS-SQLI-": SQLI,
+    "HS-CMDI-": CMDI,
+    "HS-CODEI-": CODEI,
+    "HS-CRYPTO-": WEAK_CRYPTO,
+}
+
+_CWE_RE = re.compile(r"CWE[-_ ]?(\d+)", re.IGNORECASE)
+
+
+def normalize_cwe(raw: str) -> str | None:
+    """`'cwe-89: SQL Injection'` -> `'CWE-89'`; returns None if no CWE number is present."""
+    match = _CWE_RE.search(raw)
+    return f"CWE-{int(match.group(1))}" if match else None
+
+
+def normalize_cwes(raws: Iterable[str]) -> tuple[str, ...]:
+    """Normalize, dedupe and sort CWE ids (numeric order)."""
+    found = {c for c in (normalize_cwe(r) for r in raws) if c}
+    return tuple(sorted(found, key=lambda c: int(c.split("-")[1])))
+
+
+def classify(rule_id: str, cwes: Iterable[str] = ()) -> str:
+    """Map a rule id and its CWEs to a canonical class.
+
+    Precedence: own rule prefix, then exact tool rule, then gitleaks (always secrets),
+    then CWE. Unmappable findings get `other:<lowest CWE>` or `other:<rule_id>`.
+    """
+    for prefix, cls in _OWN_RULE_PREFIX.items():
+        if rule_id.startswith(prefix):
+            return cls
+    if rule_id in _RULE_TO_CLASS:
+        return _RULE_TO_CLASS[rule_id]
+    if rule_id.startswith("gitleaks:"):
+        return SECRET
+    normalized = normalize_cwes(cwes)
+    for cwe in normalized:
+        if cwe in _CWE_TO_CLASS:
+            return _CWE_TO_CLASS[cwe]
+    if normalized:
+        return f"{OTHER_PREFIX}{normalized[0]}"
+    return f"{OTHER_PREFIX}{rule_id}"
+
+
+def is_mergeable_class(vuln_class: str, cwes: Iterable[str]) -> bool:
+    """Findings only merge across sources when their identity is grounded.
+
+    Known classes are always mergeable. `other:` classes merge only when backed by a
+    CWE; `other:<rule_id>` (no CWE) never merges with anything else.
+    """
+    if vuln_class in KNOWN_CLASSES:
+        return True
+    return bool(normalize_cwes(cwes))

hackscan/importers/__init__.py

@@ -0,0 +1 @@
+"""Importers for third-party scanner reports."""

hackscan/importers/common.py

@@ -0,0 +1,198 @@
+"""Shared importer helpers: path/region normalization and enrichment from source.
+
+Per the Finding contract, `sink` and `function` are derived from the parsed file (not
+from a tool's display snippet), so fingerprints do not depend on which tool reported.
+"""
+
+from __future__ import annotations
+
+import ast
+import io
+import tokenize
+import warnings
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+from urllib.parse import unquote, urlparse
+
+from hackscan.analyzers.ast_pass import _qualnames
+from hackscan.core.models import Region, Severity
+from hackscan.plugins.base import FileContext
+
+
+class ReportError(Exception):
+    """A third-party report could not be read or run."""
+
+    pass
+
+
+@dataclass
+class ImportResult:
+    tool: str
+    findings: list = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    secrets: set[str] = field(default_factory=set)  # values to redact from all output
+
+
+def normalize_path(raw: str, root: Path, base_uri: str | None = None) -> str | None:
+    """Tool-reported path/URI -> POSIX path relative to `root`, or None if outside it."""
+    if raw.startswith("file:"):
+        parsed = urlparse(raw)
+        if parsed.netloc not in ("", "localhost"):
+            return None  # file://other-host/share/... is not a local path
+        raw = unquote(parsed.path)
+        if len(raw) >= 3 and raw[0] == "/" and raw[2] == ":":  # file:///C:/x -> C:/x
+            raw = raw[1:]
+    else:
+        raw = unquote(raw)
+        if base_uri:
+            base = normalize_path(base_uri, root) if base_uri.startswith("file:") else None
+            if base is not None:
+                raw = str(PurePosixPath(base) / raw)
+    candidate = Path(raw)
+    if not candidate.is_absolute():
+        candidate = root / candidate
+    try:
+        rel = candidate.resolve().relative_to(root.resolve())
+    except ValueError:
+        return None
+    return rel.as_posix()
+
+
+def make_region(
+    path: str,
+    start_line: int | None,
+    start_column: int | None = None,
+    end_line: int | None = None,
+    end_column: int | None = None,
+) -> Region:
+    """Region with the importer normalization rules (PROJECT.md):
+    - missing end line -> start line; missing end column -> same as start (a point),
+      so a column-precise report never silently widens to the rest of the line;
+    - missing start column -> whole line;
+    - invalid values are clamped rather than rejected (non-numbers raise ValueError).
+    """
+    line = max(1, int(start_line or 1))
+    end = max(line, int(end_line or line))
+    col = int(start_column) if start_column and int(start_column) > 0 else 1
+    end_col = int(end_column) if end_column and int(end_column) > 0 else None
+    if start_column is None or int(start_column) <= 0:
+        end_col = None  # whole line
+    elif end_col is None and end == line:
+        end_col = col  # point at the reported start
+    if end_col is not None and end == line and end_col < col:
+        end_col = col
+    return Region(path=path, start_line=line, start_column=col, end_line=end, end_column=end_col)
+
+
+SEVERITY_FROM_LEVEL = {
+    "error": Severity.HIGH,
+    "warning": Severity.MEDIUM,
+    "note": Severity.LOW,
+    "none": Severity.LOW,
+}
+
+
+def severity_from_score(score: float) -> Severity:
+    """CVSS-like `security-severity` (CodeQL, Semgrep) -> Severity."""
+    if score >= 9.0:
+        return Severity.CRITICAL
+    if score >= 7.0:
+        return Severity.HIGH
+    if score >= 4.0:
+        return Severity.MEDIUM
+    return Severity.LOW
+
+
+class SourceIndex:
+    """Lazily parsed project files for enriching imported findings."""
+
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self._cache: dict[str, tuple[FileContext, dict[int, str]] | None] = {}
+
+    def context(self, rel_path: str) -> tuple[FileContext, dict[int, str]] | None:
+        if rel_path not in self._cache:
+            self._cache[rel_path] = self._load(rel_path)
+        return self._cache[rel_path]
+
+    def _load(self, rel_path: str) -> tuple[FileContext, dict[int, str]] | None:
+        path = self.root / rel_path
+        if path.suffix != ".py":
+            return None
+        try:
+            data = path.read_bytes()
+            encoding, _ = tokenize.detect_encoding(io.BytesIO(data).readline)
+            source = data.decode(encoding)
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", SyntaxWarning)
+                tree = ast.parse(source)
+        except (OSError, SyntaxError, UnicodeDecodeError, LookupError, ValueError):
+            return None
+        return FileContext(path=rel_path, source=source, tree=tree), _qualnames(tree)
+
+    def line_text(self, rel_path: str, line: int) -> str:
+        loaded = self.context(rel_path)
+        if loaded is not None:
+            lines = loaded[0].lines
+            return lines[line - 1] if 0 < line <= len(lines) else ""
+        try:
+            lines = (self.root / rel_path).read_text(encoding="utf-8", errors="replace")
+            split = lines.splitlines()
+            return split[line - 1] if 0 < line <= len(split) else ""
+        except OSError:
+            return ""
+
+    def enrich(self, region: Region) -> tuple[str, str | None]:
+        """(sink text, enclosing function qualname) for a region in a Python file.
+
+        The sink is the innermost call that starts on the region's first line and covers
+        its start column (or the first call on that line for whole-line regions).
+        """
+        loaded = self.context(region.path)
+        if loaded is None:
+            return "", None
+        ctx, qualnames = loaded
+        best: ast.AST | None = None
+        for node in ast.walk(ctx.tree):
+            if not isinstance(node, ast.Call) or node.lineno != region.start_line:
+                continue
+            col = ctx.char_column(node.lineno, node.col_offset)
+            end = (
+                ctx.char_column(node.end_lineno, node.end_col_offset)
+                if node.end_lineno == node.lineno and node.end_col_offset is not None
+                else 10**9
+            )
+            if region.end_column is not None and not (col <= region.start_column < end):
+                continue
+            if best is None or (col, -end) > _span_key(ctx, best):
+                best = node
+        if best is None:
+            line_text = (
+                ctx.lines[region.start_line - 1] if region.start_line <= len(ctx.lines) else ""
+            )
+            func = _function_at_line(ctx, qualnames, region.start_line)
+            return line_text.strip(), func
+        func_node = ctx.enclosing_function(best)
+        return ctx.segment(best), qualnames.get(id(func_node)) if func_node else None
+
+
+def _span_key(ctx: FileContext, node: ast.AST) -> tuple[int, int]:
+    col = ctx.char_column(node.lineno, node.col_offset)
+    end = (
+        ctx.char_column(node.end_lineno, node.end_col_offset)
+        if node.end_lineno == node.lineno and node.end_col_offset is not None
+        else 10**9
+    )
+    return col, -end
+
+
+def _function_at_line(ctx: FileContext, qualnames: dict[int, str], line: int) -> str | None:
+    best = None
+    for node in ast.walk(ctx.tree):
+        if (
+            isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+            and node.lineno <= line <= (node.end_lineno or node.lineno)
+            and (best is None or node.lineno >= best.lineno)
+        ):
+            best = node  # innermost: the latest-starting function containing the line
+    return qualnames.get(id(best)) if best is not None else None

hackscan/importers/gitleaks.py

@@ -0,0 +1,104 @@
+"""Gitleaks JSON report import (`gitleaks detect --report-format json`).
+
+Secrets are never copied into findings: the matched text is redacted in the snippet and
+the secret value itself is dropped.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from hackscan.core.models import Evidence, Finding, Severity
+from hackscan.core.taxonomy import SECRET
+from hackscan.importers.common import (
+    ImportResult,
+    ReportError,
+    SourceIndex,
+    make_region,
+    normalize_path,
+)
+
+REDACTED = "<REDACTED>"
+SOURCE = "gitleaks"
+
+
+def load_gitleaks(path: Path) -> list[dict[str, Any]]:
+    try:
+        text = path.read_text(encoding="utf-8-sig").strip()
+        data = json.loads(text) if text else []
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise ReportError(f"{path}: not a readable Gitleaks JSON report: {exc}") from exc
+    if not isinstance(data, list):
+        raise ReportError(f"{path}: Gitleaks report must be a JSON array")
+    return data
+
+
+def import_gitleaks(items: list[dict[str, Any]], root: Path, index: SourceIndex) -> ImportResult:
+    result = ImportResult(tool=SOURCE)
+    for item in items:
+        if not isinstance(item, dict):
+            result.warnings.append("gitleaks: skipped non-object entry")
+            continue
+        secret = str(item.get("Secret") or "")
+        if secret and secret != "REDACTED":
+            result.secrets.add(secret)
+        try:
+            _convert(item, root, index, result)
+        except (TypeError, ValueError) as exc:
+            result.warnings.append(f"gitleaks: skipped malformed entry: {exc}")
+    return result
+
+
+def _convert(item: dict[str, Any], root: Path, index: SourceIndex, result: ImportResult) -> None:
+    raw_path = item.get("File") or item.get("file")
+    if not raw_path:
+        result.warnings.append("gitleaks: skipped entry without File")
+        return
+    rel = normalize_path(str(raw_path), root)
+    if rel is None:
+        result.warnings.append(f"gitleaks: {raw_path} is outside the scan root; skipped")
+        return
+    region = make_region(
+        rel,
+        item.get("StartLine"),
+        item.get("StartColumn"),
+        item.get("EndLine"),
+        # Gitleaks columns are inclusive; SARIF/Region end columns are exclusive.
+        (item.get("EndColumn") or 0) + 1 if item.get("EndColumn") else None,
+    )
+    secret = str(item.get("Secret") or "")
+    rule = str(item.get("RuleID") or "generic")
+    description = str(item.get("Description") or f"Secret detected ({rule})")
+    if secret:
+        description = description.replace(secret, REDACTED)
+    line = index.line_text(rel, region.start_line)
+    snippet = _redact(line or str(item.get("Match") or ""), secret)
+    result.findings.append(
+        Finding(
+            id="",
+            vuln_class=SECRET,
+            rule_id=f"{SOURCE}:{rule}",
+            severity=Severity.HIGH,
+            location=region,
+            message=description,
+            snippet=snippet,
+            # A secret's identity is its location and rule, never its value. No `=`/`:` so
+            # secret-class redaction leaves it intact (it feeds the fingerprint).
+            sink=f"{rule}@{rel}@L{region.start_line}",
+            function=None,
+            cwe=("CWE-798",),
+            sources=(SOURCE,),
+            confidence=70,
+            evidence=(Evidence(SOURCE, "tool_message", description),),
+        )
+    )
+
+
+def _redact(text: str, secret: str) -> str:
+    """Replace the secret; if it is unknown or absent from `text`, keep `text` only when
+    no secret was reported at all (otherwise the safest snippet is none)."""
+    if secret and secret in text:
+        return text.replace(secret, REDACTED)
+    return text if not secret else REDACTED

hackscan/importers/runner.py

@@ -0,0 +1,161 @@
+"""Run external scanners (`--with`) and load bring-your-own reports (`--import`).
+
+Run mode never fails the scan by default: a missing tool, a timeout or unusable output
+becomes a warning and that source is skipped (`--strict-tools` makes it an error).
+A non-zero exit code with valid output is accepted (scanners exit 1 on findings).
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from hackscan.importers.common import ImportResult, ReportError, SourceIndex
+from hackscan.importers.gitleaks import import_gitleaks, load_gitleaks
+from hackscan.importers.sarif import import_sarif, load_sarif
+
+
+@dataclass
+class ToolRun:
+    tool: str
+    status: str  # "ok", "partial", "missing", "failed", "timeout", "imported"
+    detail: str = ""
+    findings: int = 0
+
+
+@dataclass
+class ExternalResults:
+    findings: list = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    # Requested reports that could not be imported: the scan is incomplete.
+    errors: list[str] = field(default_factory=list)
+    runs: list[ToolRun] = field(default_factory=list)
+    secrets: set[str] = field(default_factory=set)
+
+    def add(self, result: ImportResult, run: ToolRun) -> None:
+        self.findings.extend(result.findings)
+        self.warnings.extend(result.warnings)
+        self.secrets |= result.secrets
+        run.findings = len(result.findings)
+        self.runs.append(run)
+
+
+# Exit codes meaning "ran to completion" (1 = findings reported for semgrep/bandit).
+OK_EXIT_CODES = {"semgrep": {0, 1}, "bandit": {0, 1}, "gitleaks": {0}}
+
+
+def _commands(tool: str, target: Path, out: Path) -> list[str]:
+    if tool == "semgrep":
+        return [
+            "semgrep",
+            "scan",
+            "--config",
+            "p/python",
+            "--sarif",
+            "--output",
+            str(out),
+            "--quiet",
+            "--disable-version-check",
+            "--metrics",
+            "off",
+            str(target),
+        ]
+    if tool == "bandit":
+        return ["bandit", "-r", "-q", "-f", "sarif", "-o", str(out), str(target)]
+    if tool == "gitleaks":
+        return [
+            "gitleaks",
+            "detect",
+            "--no-git",
+            "--no-banner",
+            "--report-format",
+            "json",
+            "--report-path",
+            str(out),
+            "--source",
+            str(target),
+            "--exit-code",
+            "0",
+        ]
+    raise ValueError(f"unknown tool {tool}")
+
+
+def collect(
+    root: Path,
+    target: Path,
+    with_tools: tuple[str, ...],
+    imports: tuple[tuple[str, Path], ...],
+    timeout: int,
+    index: SourceIndex | None = None,
+) -> ExternalResults:
+    index = index or SourceIndex(root)
+    results = ExternalResults()
+    for tool in with_tools:
+        _run_tool(tool, root, target, timeout, index, results)
+    for fmt, report in imports:
+        try:
+            result = _load(fmt, report, root, index)
+        except ReportError as exc:
+            results.errors.append(f"import {fmt}={report}: {exc}")
+            results.runs.append(ToolRun(fmt, "failed", str(exc)))
+            continue
+        results.add(result, ToolRun(result.tool, "imported", str(report)))
+    return results
+
+
+def _load(fmt: str, report: Path, root: Path, index: SourceIndex) -> ImportResult:
+    if fmt == "gitleaks":
+        return import_gitleaks(load_gitleaks(report), root, index)
+    label = None if fmt == "sarif" else fmt
+    return import_sarif(load_sarif(report), root, index, label=label)
+
+
+def _run_tool(
+    tool: str, root: Path, target: Path, timeout: int, index: SourceIndex, results: ExternalResults
+) -> None:
+    executable = shutil.which(tool)
+    if executable is None:
+        results.warnings.append(f"{tool}: not found on PATH; skipped")
+        results.runs.append(ToolRun(tool, "missing", "not on PATH"))
+        return
+    with tempfile.TemporaryDirectory(prefix="hackscan-") as tmp:
+        out = Path(tmp) / ("report.json" if tool == "gitleaks" else "report.sarif")
+        command = _commands(tool, target, out)
+        command[0] = executable
+        try:
+            proc = subprocess.run(
+                command, capture_output=True, text=True, timeout=timeout or None, check=False
+            )
+        except subprocess.TimeoutExpired:
+            results.warnings.append(f"{tool}: timed out after {timeout}s; skipped")
+            results.runs.append(ToolRun(tool, "timeout", f"{timeout}s"))
+            return
+        except OSError as exc:
+            results.warnings.append(f"{tool}: could not run: {exc}; skipped")
+            results.runs.append(ToolRun(tool, "failed", str(exc)))
+            return
+        if not out.is_file() or out.stat().st_size == 0:
+            detail = (proc.stderr or proc.stdout).strip().splitlines()[-1:] or ["no output"]
+            results.warnings.append(
+                f"{tool}: produced no report (exit {proc.returncode}): {detail[0]}"
+            )
+            results.runs.append(ToolRun(tool, "failed", f"exit {proc.returncode}"))
+            return
+        try:
+            result = _load(tool, out, root, index)
+        except ReportError as exc:
+            results.warnings.append(f"{tool}: unusable output: {exc}")
+            results.runs.append(ToolRun(tool, "failed", "unusable output"))
+            return
+        if proc.returncode not in OK_EXIT_CODES[tool]:
+            # The tool reported an error but left output: keep what it found, flag the
+            # run so it is visible and `--strict-tools` fails.
+            results.warnings.append(
+                f"{tool}: exited with error code {proc.returncode}; results may be incomplete"
+            )
+            results.add(result, ToolRun(tool, "partial", f"exit {proc.returncode}"))
+            return
+        results.add(result, ToolRun(tool, "ok", f"exit {proc.returncode}"))