hackscan 0.1.0__py3-none-any.whl

hackscan/core/fingerprint.py

@@ -0,0 +1,76 @@
+"""Stable, vendor-independent finding ids.
+
+id = H(vuln_class, path, enclosing function, normalized sink expression)
+
+The sink expression comes from `Finding.sink` (text taken from the source file), falling
+back to `snippet` only when no sink was recorded. Line numbers are deliberately excluded
+so ids survive unrelated edits that shift code.
+Identical sinks in the same function are disambiguated by occurrence order.
+"""
+
+from __future__ import annotations
+
+import ast
+import hashlib
+import re
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import replace
+from functools import lru_cache
+
+from hackscan.core.models import Finding
+
+ID_PREFIX = "hs1-"  # bump if the fingerprint recipe changes
+_HEX_LEN = 32
+_WS_RE = re.compile(r"\s+")
+
+
+@lru_cache(maxsize=65536)
+def normalize_expression(code: str) -> str:
+    """Canonical form of a code snippet: formatting- and comment-insensitive when parseable."""
+    text = code.strip()
+    if not text:
+        return ""
+    for mode in ("eval", "exec"):
+        try:
+            return ast.unparse(ast.parse(text, mode=mode))
+        except SyntaxError:
+            continue
+    return _WS_RE.sub(" ", text)
+
+
+def compute_fingerprint(
+    vuln_class: str, path: str, function: str | None, sink_expression: str, occurrence: int = 0
+) -> str:
+    parts = [vuln_class, path, function or "<module>", normalize_expression(sink_expression)]
+    if occurrence:
+        parts.append(f"#{occurrence}")
+    digest = hashlib.sha256("\x1f".join(parts).encode("utf-8")).hexdigest()
+    return ID_PREFIX + digest[:_HEX_LEN]
+
+
+def assign_ids(findings: Iterable[Finding]) -> list[Finding]:
+    """Return findings with `id` set. Input order does not affect the result."""
+    groups: dict[str, list[Finding]] = defaultdict(list)
+    for f in findings:
+        base = compute_fingerprint(f.vuln_class, f.location.path, f.function, sink_text(f))
+        groups[base].append(f)
+
+    result: list[Finding] = []
+    for base, members in groups.items():
+        members.sort(key=lambda f: (*f.location.sort_key(), f.canonical_json()))
+        for occurrence, f in enumerate(members):
+            new_id = (
+                base
+                if occurrence == 0
+                else compute_fingerprint(
+                    f.vuln_class, f.location.path, f.function, sink_text(f), occurrence
+                )
+            )
+            result.append(replace(f, id=new_id))
+    result.sort(key=Finding.sort_key)
+    return result
+
+
+def sink_text(f: Finding) -> str:
+    return f.sink or f.snippet

hackscan/core/models.py

@@ -0,0 +1,270 @@
+"""Finding schema v1. Frozen at M0: changes require bumping SCHEMA_VERSION.
+
+Every pass and importer produces or transforms `Finding` objects. Passes never delete
+findings; they return updated copies (see `dataclasses.replace`).
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from dataclasses import dataclass
+from enum import Enum
+from functools import cached_property
+from typing import Any
+
+SCHEMA_VERSION = 1
+
+OWN_SOURCE = "hackscan"
+
+
+class Severity(str, Enum):
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+    @property
+    def rank(self) -> int:
+        return _SEVERITY_RANK[self]
+
+
+_SEVERITY_RANK = {Severity.LOW: 0, Severity.MEDIUM: 1, Severity.HIGH: 2, Severity.CRITICAL: 3}
+
+
+class Status(str, Enum):
+    CANDIDATE = "candidate"
+    CONFIRMED = "confirmed"
+    SUPPRESSED = "suppressed"
+
+    @property
+    def rank(self) -> int:
+        """Merge precedence: confirmed > candidate > suppressed."""
+        return _STATUS_RANK[self]
+
+
+_STATUS_RANK = {Status.SUPPRESSED: 0, Status.CANDIDATE: 1, Status.CONFIRMED: 2}
+
+
+@dataclass(frozen=True)
+class Region:
+    """Source location. Lines and columns are 1-based (SARIF convention).
+
+    `path` is a POSIX path relative to the scan root. `end_column` is exclusive (SARIF
+    `endColumn`); None means "to end of line".
+    """
+
+    path: str
+    start_line: int
+    start_column: int = 1
+    end_line: int | None = None
+    end_column: int | None = None
+
+    def __post_init__(self) -> None:
+        if self.start_line < 1 or self.start_column < 1:
+            raise ValueError(f"Region lines/columns are 1-based: {self}")
+        if "\\" in self.path:
+            raise ValueError(f"Region path must be POSIX-style: {self.path!r}")
+        if self.end_line is None:
+            object.__setattr__(self, "end_line", self.start_line)
+        if self.end_line < self.start_line:
+            raise ValueError(f"Region ends before it starts: {self}")
+        if self.end_column is not None and (
+            self.end_column < 1
+            or (self.end_line == self.start_line and self.end_column < self.start_column)
+        ):
+            raise ValueError(f"Region end column is invalid: {self}")
+
+    @property
+    def start_pos(self) -> tuple[int, float]:
+        return (self.start_line, self.start_column)
+
+    @property
+    def end_pos(self) -> tuple[int, float]:
+        """Exclusive end position; whole-line when `end_column` is None."""
+        if self.end_column is None:
+            return (self.end_line, math.inf)
+        # Zero-width regions still cover the character they point at.
+        if self.end_line == self.start_line and self.end_column == self.start_column:
+            return (self.end_line, self.end_column + 1)
+        return (self.end_line, self.end_column)
+
+    @property
+    def span(self) -> tuple[int, float]:
+        """Ordering heuristic preferring precise regions: (extra lines, column delta).
+
+        Only used to pick visit order; correctness of dedupe does not depend on it.
+        """
+        return (self.end_line - self.start_line, self.end_pos[1] - self.start_column)
+
+    def overlaps(self, other: Region) -> bool:
+        """Position overlap (line and column aware) within the same file."""
+        return (
+            self.path == other.path
+            and self.start_pos < other.end_pos
+            and other.start_pos < self.end_pos
+        )
+
+    def sort_key(self) -> tuple[str, int, int]:
+        return (self.path, self.start_line, self.start_column)
+
+
+@dataclass(frozen=True)
+class Evidence:
+    """One piece of supporting evidence, tagged with the producer (pass or tool)."""
+
+    producer: str
+    kind: str  # e.g. "taint_step", "tool_message", "llm_rationale"
+    message: str
+    region: Region | None = None
+
+
+@dataclass(frozen=True)
+class FixEdit:
+    region: Region
+    replacement: str
+
+
+@dataclass(frozen=True)
+class Fix:
+    description: str
+    edits: tuple[FixEdit, ...]
+    producer: str
+
+
+@dataclass(frozen=True)
+class Finding:
+    id: str
+    vuln_class: str
+    rule_id: str
+    severity: Severity
+    location: Region
+    message: str
+    snippet: str = ""
+    cwe: tuple[str, ...] = ()
+    related_rules: tuple[str, ...] = ()
+    sources: tuple[str, ...] = (OWN_SOURCE,)
+    status: Status = Status.CANDIDATE
+    suppression: str | None = None
+    confidence: int = 50
+    evidence: tuple[Evidence, ...] = ()
+    fix: Fix | None = None
+    # Enclosing function qualname. Like `sink`, producers derive it from the parsed file
+    # (importers look it up by location), so it does not depend on which tool reported.
+    function: str | None = None
+    # Source text of the sink expression at `location`, used for fingerprinting.
+    # Producers set it from the parsed file (own engine: the sink AST node; importers:
+    # the file text covered by the region), never from a tool-provided display snippet.
+    sink: str = ""
+
+    def __post_init__(self) -> None:
+        if not 0 <= self.confidence <= 100:
+            raise ValueError(f"confidence must be 0-100, got {self.confidence}")
+        if (self.status is Status.SUPPRESSED) != (self.suppression is not None):
+            raise ValueError("suppression reason is required iff status is suppressed")
+
+    def sort_key(self) -> tuple[str, int, int, str, str, str]:
+        """Deterministic output ordering: path, line, column, rule, id, then full content."""
+        return (*self.location.sort_key(), self.rule_id, self.id, self.canonical_json())
+
+    def canonical_json(self) -> str:
+        """Stable serialization; a total-order tie-breaker for equal-looking findings."""
+        return self._canonical_json
+
+    @cached_property
+    def _canonical_json(self) -> str:
+        # Safe to cache: Finding is frozen, and cached_property writes to __dict__
+        # directly (not a dataclass field, so eq/hash/replace are unaffected).
+        return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":"))
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": SCHEMA_VERSION,
+            "id": self.id,
+            "vuln_class": self.vuln_class,
+            "rule_id": self.rule_id,
+            "related_rules": list(self.related_rules),
+            "cwe": list(self.cwe),
+            "severity": self.severity.value,
+            "confidence": self.confidence,
+            "status": self.status.value,
+            "suppression": self.suppression,
+            "sources": list(self.sources),
+            "message": self.message,
+            "location": _region_to_dict(self.location),
+            "function": self.function,
+            "sink": self.sink,
+            "snippet": self.snippet,
+            "evidence": [
+                {
+                    "producer": e.producer,
+                    "kind": e.kind,
+                    "message": e.message,
+                    "region": _region_to_dict(e.region) if e.region else None,
+                }
+                for e in self.evidence
+            ],
+            "fix": None
+            if self.fix is None
+            else {
+                "description": self.fix.description,
+                "producer": self.fix.producer,
+                "edits": [
+                    {"region": _region_to_dict(ed.region), "replacement": ed.replacement}
+                    for ed in self.fix.edits
+                ],
+            },
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Finding:
+        version = data.get("schema_version")
+        if version != SCHEMA_VERSION:
+            raise ValueError(f"unsupported Finding schema_version {version!r}")
+        fix = data.get("fix")
+        return cls(
+            id=data["id"],
+            vuln_class=data["vuln_class"],
+            rule_id=data["rule_id"],
+            related_rules=tuple(data.get("related_rules", ())),
+            cwe=tuple(data.get("cwe", ())),
+            severity=Severity(data["severity"]),
+            confidence=data["confidence"],
+            status=Status(data["status"]),
+            suppression=data.get("suppression"),
+            sources=tuple(data["sources"]),
+            message=data["message"],
+            location=Region(**data["location"]),
+            function=data.get("function"),
+            sink=data.get("sink", ""),
+            snippet=data.get("snippet", ""),
+            evidence=tuple(
+                Evidence(
+                    producer=e["producer"],
+                    kind=e["kind"],
+                    message=e["message"],
+                    region=Region(**e["region"]) if e.get("region") else None,
+                )
+                for e in data.get("evidence", ())
+            ),
+            fix=None
+            if fix is None
+            else Fix(
+                description=fix["description"],
+                producer=fix["producer"],
+                edits=tuple(
+                    FixEdit(region=Region(**ed["region"]), replacement=ed["replacement"])
+                    for ed in fix["edits"]
+                ),
+            ),
+        )
+
+
+def _region_to_dict(region: Region) -> dict[str, Any]:
+    return {
+        "path": region.path,
+        "start_line": region.start_line,
+        "start_column": region.start_column,
+        "end_line": region.end_line,
+        "end_column": region.end_column,
+    }

hackscan/core/pipeline.py

@@ -0,0 +1,164 @@
+"""Scan pipeline: discover -> analyze (passes 1-2, parallel) -> import -> merge -> ids.
+
+Pipeline order per PROJECT.md: collect (own engine + importers) -> `merge_findings` ->
+`assign_ids`. Output is deterministic regardless of worker scheduling.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+import time
+from collections.abc import Iterator
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from hackscan.analyzers.ast_pass import FileResult, analyze_file
+from hackscan.config import DEFAULT_IGNORES, HackScanConfig
+from hackscan.core.dedupe import merge_findings
+from hackscan.core.fingerprint import assign_ids
+from hackscan.core.models import Finding
+from hackscan.core.redact import redact_findings, redact_messages
+from hackscan.importers.common import SourceIndex
+from hackscan.importers.runner import ToolRun, collect
+from hackscan.plugins.base import RulePlugin
+from hackscan.plugins.loader import resolve_plugins
+
+PARALLEL_THRESHOLD = 16  # below this many files, process start-up costs more than it saves
+
+
+@dataclass
+class ScanResult:
+    root: Path
+    findings: list[Finding]
+    files_scanned: int
+    duration_seconds: float
+    errors: list[str] = field(default_factory=list)  # files we could not analyze
+    warnings: list[str] = field(default_factory=list)  # external tool problems
+    tool_runs: list[ToolRun] = field(default_factory=list)
+
+
+def scan(target: Path, config: HackScanConfig) -> ScanResult:
+    started = time.perf_counter()
+    target = target.resolve()
+    root = target if target.is_dir() else target.parent
+    plugins = resolve_plugins(config.plugins)
+    files = list(discover_files(target, root, config.ignore))
+
+    results = _analyze(files, root, plugins, config)
+    findings: list[Finding] = []
+    errors: list[str] = []
+    for result in results:
+        findings.extend(result.findings)
+        errors.extend(result.errors)
+
+    external = collect(
+        root,
+        target,
+        config.with_tools,
+        config.imports,
+        config.tool_timeout,
+        SourceIndex(root),
+    )
+    # Imported findings obey the same ignores as discovery (defaults included).
+    findings.extend(
+        f for f in external.findings if not is_ignored_path(f.location.path, config.ignore)
+    )
+
+    final = assign_ids(redact_findings(merge_findings(findings), external.secrets))
+    return ScanResult(
+        root=root,
+        findings=final,
+        files_scanned=len(files),
+        duration_seconds=time.perf_counter() - started,
+        errors=redact_messages([*sorted(errors), *external.errors], external.secrets),
+        warnings=redact_messages(external.warnings, external.secrets),
+        tool_runs=external.runs,
+    )
+
+
+# -- discovery ----------------------------------------------------------------------------
+
+
+def discover_files(target: Path, root: Path, ignore: tuple[str, ...]) -> Iterator[Path]:
+    """Python files under `target`, skipping default and configured ignores (sorted)."""
+    if target.is_file():
+        if target.suffix in (".py", ".pyw") and not _is_ignored(target.name, ignore):
+            yield target
+        return
+    for dirpath, dirnames, filenames in os.walk(target):
+        current = Path(dirpath)
+        rel_dir = current.relative_to(root).as_posix()
+        dirnames[:] = sorted(
+            d
+            for d in dirnames
+            if not _is_ignored(f"{rel_dir}/{d}" if rel_dir != "." else d, ignore, is_dir=True)
+        )
+        for name in sorted(filenames):
+            if not name.endswith((".py", ".pyw")):
+                continue
+            rel = f"{rel_dir}/{name}" if rel_dir != "." else name
+            if not _is_ignored(rel, ignore):
+                yield current / name
+
+
+def is_ignored_path(rel_path: str, patterns: tuple[str, ...]) -> bool:
+    """Whether a file path is excluded: any directory component matching a default ignore
+    (`.venv`, `node_modules`, ...) or a configured pattern."""
+    parts = rel_path.split("/")
+    if any(fnmatch.fnmatch(part, p) for part in parts[:-1] for p in DEFAULT_IGNORES):
+        return True
+    return _is_ignored(rel_path, patterns)
+
+
+def _is_ignored(rel_path: str, patterns: tuple[str, ...], is_dir: bool = False) -> bool:
+    """Default ignores match any path component; configured globs match the relative path
+    (`tests/*`, `**/migrations/**`) or any single component (`legacy`)."""
+    parts = rel_path.split("/")
+    if is_dir and any(fnmatch.fnmatch(parts[-1], p) for p in DEFAULT_IGNORES):
+        return True
+    for pattern in patterns:
+        pattern = pattern.strip().rstrip("/")
+        if not pattern:
+            continue
+        if fnmatch.fnmatch(rel_path, pattern) or fnmatch.fnmatch(rel_path, pattern + "/*"):
+            return True
+        if pattern.startswith("**/") and fnmatch.fnmatch(rel_path, pattern[3:]):
+            return True
+        if "/" not in pattern and any(fnmatch.fnmatch(part, pattern) for part in parts):
+            return True
+    return False
+
+
+# -- analysis -----------------------------------------------------------------------------
+
+_WORKER_STATE: dict = {}
+
+
+def _worker_init(plugins_dir: str | None, taint: bool) -> None:
+    _WORKER_STATE["plugins"] = resolve_plugins(Path(plugins_dir) if plugins_dir else None)
+    _WORKER_STATE["taint"] = taint
+
+
+def _worker_analyze(args: tuple[str, str]) -> FileResult:
+    path, root = args
+    return analyze_file(
+        Path(path), Path(root), _WORKER_STATE["plugins"], taint=_WORKER_STATE["taint"]
+    )
+
+
+def _analyze(
+    files: list[Path], root: Path, plugins: list[RulePlugin], config: HackScanConfig
+) -> list[FileResult]:
+    jobs = config.jobs or min(8, os.cpu_count() or 1)
+    if jobs <= 1 or len(files) < PARALLEL_THRESHOLD:
+        return [analyze_file(f, root, plugins, taint=config.taint) for f in files]
+    plugins_dir = str(config.plugins) if config.plugins else None
+    with ProcessPoolExecutor(
+        max_workers=jobs, initializer=_worker_init, initargs=(plugins_dir, config.taint)
+    ) as pool:
+        chunk = max(1, len(files) // (jobs * 4))
+        return list(
+            pool.map(_worker_analyze, [(str(f), str(root)) for f in files], chunksize=chunk)
+        )

hackscan/core/redact.py

@@ -0,0 +1,84 @@
+"""Secret redaction for every user-facing field of a finding.
+
+Applied after merging, before ids are assigned, so no output format (text, JSON, SARIF)
+can carry a secret regardless of which tool reported it or which contributor became the
+primary of a merged finding.
+
+- Known secret values (e.g. Gitleaks `Secret`) are replaced wherever they appear.
+- Findings of class `secret` never carry tool-provided text: their message and evidence
+  become generic, and in snippet/sink every string literal and every assigned value
+  (`token = abc`, `password: abc`) is replaced. A tool may describe a secret in any
+  form (quoted, unquoted, inside a message), so content-based redaction is not enough.
+- `redact_messages` applies known-secret replacement to warnings and errors too.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import replace
+
+from hackscan.core.models import Finding
+from hackscan.core.taxonomy import SECRET
+
+REDACTED = "<REDACTED>"
+MIN_SECRET_LENGTH = 4  # shorter "secrets" would redact ordinary text
+_LITERAL_RE = re.compile(r"""(?P<q>['"])(?:\\.|(?!(?P=q)).)+(?P=q)""")
+# `key = value` / `key: value` / `key=value` (value up to end of line or comment).
+_ASSIGNED_RE = re.compile(r"(?P<key>[=:]\s*)(?P<value>[^\s#'\"][^#]*?)(?P<tail>\s*(?:#.*)?)$")
+
+
+def redact_text(text: str, secrets: Iterable[str], literals: bool) -> str:
+    for secret in sorted(set(secrets), key=len, reverse=True):
+        if len(secret) >= MIN_SECRET_LENGTH:
+            text = text.replace(secret, REDACTED)
+    if literals:
+        text = _LITERAL_RE.sub(lambda m: f"{m.group('q')}{REDACTED}{m.group('q')}", text)
+    return text
+
+
+def redact_code(text: str, secrets: Iterable[str]) -> str:
+    """Secret-class snippet/sink: literals and assigned values removed, line by line."""
+    lines = []
+    for line in redact_text(text, secrets, literals=True).split("\n"):
+        lines.append(
+            _ASSIGNED_RE.sub(lambda m: f"{m.group('key')}{REDACTED}{m.group('tail')}", line)
+        )
+    return "\n".join(lines)
+
+
+def redact_messages(messages: Iterable[str], secrets: Iterable[str]) -> list[str]:
+    secrets = list(secrets)
+    return [redact_text(m, secrets, literals=False) for m in messages]
+
+
+def redact_findings(findings: list[Finding], secrets: Iterable[str]) -> list[Finding]:
+    secrets = [s for s in set(secrets) if len(s) >= MIN_SECRET_LENGTH]
+    out = []
+    for f in findings:
+        if f.vuln_class == SECRET:
+            generic = f"Possible hard-coded secret reported by {', '.join(f.sources)}."
+            new = replace(
+                f,
+                message=generic,
+                snippet=redact_code(f.snippet, secrets),
+                sink=redact_code(f.sink, secrets),
+                evidence=tuple(
+                    replace(e, message=f"{e.producer} reported a secret ({f.rule_id}).")
+                    for e in f.evidence
+                ),
+            )
+        else:
+
+            def clean(text: str) -> str:
+                return redact_text(text, secrets, literals=False)
+
+            new = replace(
+                f,
+                message=clean(f.message),
+                snippet=clean(f.snippet),
+                sink=clean(f.sink),
+                evidence=tuple(replace(e, message=clean(e.message)) for e in f.evidence),
+            )
+        out.append(new if new != f else f)
+    return out