huntkit 0.3.0__py3-none-any.whl

huntkit/scanner/runner.py

@@ -0,0 +1,239 @@
+r"""Active scanning — drive the vuln/fuzz toolchain over gathered URLs.
+
+The recon pipeline *discovers* assets; :func:`run_scan` is the active phase
+that runs the ``SCAN``-category plugins (nuclei, dalfox, …) over the live URLs
+recon found and records their findings.
+
+It deliberately reuses the recon pipeline's machinery rather than inventing a
+second one:
+
+  - **same plugin contract** — a scanner is just a ``ToolPlugin`` whose
+    category is :data:`Category.SCAN`; a new one is one file, picked up with
+    no change here;
+  - **same concurrency model** — every available scanner runs on a thread
+    pool, results merged once on the main thread;
+  - **same safety model** — targets are read from the workspace's already
+    scope-filtered result files *and* re-checked against the program scope
+    here, so the active phase can never reach a host recon would have refused.
+
+Only STDIN scanners (those that take the whole URL list at once, e.g. nuclei,
+dalfox) run in this batch. Per-URL fuzzers (ffuf, arjun) have a different
+shape and are driven separately; they are reported as such, not silently
+dropped.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from typing import Optional
+from urllib.parse import urlparse
+
+from ..core.config import Config
+from ..core.runner import CommandRunner
+from ..core.workspace import Workspace
+from ..plugins import (
+    Capability,
+    Category,
+    InputMode,
+    PluginContext,
+    PluginResult,
+    ToolPlugin,
+    get_registry,
+)
+from ..plugins.registry import PluginRegistry
+
+# Files the scanner reads its candidate URLs from, in order. Both are written
+# by recon and are already scope-filtered at discovery; we re-filter anyway.
+TARGET_FILES = ["recon/live.txt", "urls/urls.txt"]
+
+# Where a scan writes. Per-tool raw output plus one combined JSON.
+FINDINGS_JSON = "scans/findings.json"
+SCAN_STATE = "scan"
+
+EventFn = Callable[[dict], None]
+
+
+@dataclass
+class ScanOutcome:
+    tool: str
+    findings: list[str] = field(default_factory=list)
+    ran: bool = False
+    skipped: bool = False
+    reason: str = ""
+
+    @property
+    def count(self) -> int:
+        return len(self.findings)
+
+
+@dataclass
+class ScanSummary:
+    program: str
+    targets: int = 0
+    outcomes: list[ScanOutcome] = field(default_factory=list)
+
+    @property
+    def total_findings(self) -> int:
+        return sum(o.count for o in self.outcomes if o.ran)
+
+    @property
+    def ran(self) -> list[str]:
+        return [o.tool for o in self.outcomes if o.ran]
+
+
+def scan_plugins(registry: PluginRegistry) -> list[ToolPlugin]:
+    """The batch scanners: SCAN-category plugins that consume a URL list via stdin."""
+    return [p for p in registry.by_category(Category.SCAN) if p.input_mode is InputMode.STDIN]
+
+
+def scan_targets(ws: Workspace) -> list[str]:
+    """Collect candidate URLs from the workspace, deduped and scope-checked.
+
+    A URL survives only if its host is in scope for the program. With no scope
+    set the workspace treats everything as in scope, but recon always sets at
+    least ``*.<seed>`` — so the active phase stays bounded either way.
+    """
+    seen: dict[str, None] = {}
+    for rel in TARGET_FILES:
+        for url in ws.read_lines(rel):
+            host = _host_of(url)
+            if host and ws.in_scope(host):
+                seen[url] = None
+    return sorted(seen)
+
+
+def _host_of(url: str) -> Optional[str]:
+    """Hostname of a URL, tolerating bare ``host`` / ``host:port`` entries."""
+    parsed = urlparse(url if "://" in url else f"//{url}", scheme="http")
+    return parsed.hostname
+
+
+class Scanner:
+    def __init__(
+        self,
+        ws: Workspace,
+        config: Config,
+        *,
+        runner: Optional[CommandRunner] = None,
+        registry: Optional[PluginRegistry] = None,
+        on_event: Optional[EventFn] = None,
+        threads: Optional[int] = None,
+        use_cache: bool = False,
+    ) -> None:
+        self.ws = ws
+        self.config = config
+        self.runner = runner or CommandRunner(config)
+        self.registry = registry if registry is not None else get_registry()
+        self._emit: EventFn = on_event or (lambda _e: None)
+        self.threads = threads or config.general.threads
+        self.use_cache = use_cache
+
+    def run(self, *, tools: Optional[list[str]] = None) -> ScanSummary:
+        targets = scan_targets(self.ws)
+        summary = ScanSummary(program=self.ws.program, targets=len(targets))
+
+        chosen = scan_plugins(self.registry)
+        if tools:
+            wanted = {t.lower() for t in tools}
+            chosen = [p for p in chosen if p.name in wanted]
+
+        self.ws.state.start(SCAN_STATE)
+        self._emit(
+            {"event": "scan_start", "targets": len(targets), "tools": [p.name for p in chosen]}
+        )
+
+        # no targets: record and stop — nothing to feed the scanners
+        if not targets:
+            self.ws.state.skip(SCAN_STATE)
+            self._emit({"event": "scan_done", "targets": 0, "findings": 0, "ran": []})
+            return summary
+
+        ctx = PluginContext(
+            config=self.config,
+            runner=self.runner,
+            inputs=targets,
+            extra={"use_cache": self.use_cache},
+        )
+
+        results: dict[str, PluginResult] = {}
+        with ThreadPoolExecutor(max_workers=max(1, min(self.threads, len(chosen) or 1))) as pool:
+            futures = {pool.submit(p.execute, ctx): p.name for p in chosen}
+            for fut in as_completed(futures):
+                name = futures[fut]
+                try:
+                    results[name] = fut.result()
+                except Exception as exc:  # a crashing scanner must not sink the run
+                    results[name] = PluginResult(
+                        name,
+                        Capability.FINDING,
+                        skipped=True,
+                        reason=f"error: {exc}",
+                    )
+
+        # merge + persist on the main thread (no write races)
+        for plugin in chosen:
+            summary.outcomes.append(self._finalize(plugin.name, results.get(plugin.name)))
+
+        self._write_combined(summary)
+        self.ws.state.set_count("findings", summary.total_findings)
+        self.ws.state.done(
+            SCAN_STATE, targets=len(targets), findings=summary.total_findings, ran=summary.ran
+        )
+        self._emit(
+            {
+                "event": "scan_done",
+                "targets": len(targets),
+                "findings": summary.total_findings,
+                "ran": summary.ran,
+            }
+        )
+        return summary
+
+    def _finalize(self, name: str, result: Optional[PluginResult]) -> ScanOutcome:
+        if result is None or result.skipped:
+            reason = result.reason if result is not None else "not run"
+            self._emit({"event": "plugin_skip", "plugin": name, "reason": reason})
+            return ScanOutcome(tool=name, skipped=True, reason=reason)
+
+        findings = [f for f in result.items if f.strip()]
+        self.ws.path("scans", f"{name}.txt").write_text(
+            "\n".join(findings) + ("\n" if findings else ""), encoding="utf-8"
+        )
+        self._emit({"event": "plugin_done", "plugin": name, "found": len(findings)})
+        return ScanOutcome(tool=name, findings=findings, ran=True)
+
+    def _write_combined(self, summary: ScanSummary) -> None:
+        payload = {
+            "program": summary.program,
+            "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            "targets": summary.targets,
+            "tools": {o.tool: o.findings for o in summary.outcomes if o.ran},
+        }
+        self.ws.path(FINDINGS_JSON).write_text(json.dumps(payload, indent=2), encoding="utf-8")
+
+
+def run_scan(
+    ws: Workspace,
+    config: Config,
+    *,
+    tools: Optional[list[str]] = None,
+    runner: Optional[CommandRunner] = None,
+    registry: Optional[PluginRegistry] = None,
+    on_event: Optional[EventFn] = None,
+    threads: Optional[int] = None,
+    use_cache: bool = False,
+) -> ScanSummary:
+    """Convenience wrapper: build a :class:`Scanner` and run it once."""
+    return Scanner(
+        ws,
+        config,
+        runner=runner,
+        registry=registry,
+        on_event=on_event,
+        threads=threads,
+        use_cache=use_cache,
+    ).run(tools=tools)

huntkit/secrets/__init__.py

@@ -0,0 +1,34 @@
+"""Secret & JS-endpoint scanner — mine gathered content for leaks.
+
+A pure feature layer (depends only on ``utils``): it matches a declarative
+pattern catalog against text the operator already collected and returns
+redacted findings. Offline; never touches a target.
+"""
+
+from __future__ import annotations
+
+from .patterns import (
+    SECRET_PATTERNS,
+    SecretPattern,
+    extract_endpoints,
+    shannon_entropy,
+)
+from .scanner import (
+    SecretHit,
+    endpoints_from_paths,
+    redact,
+    scan_paths,
+    scan_text,
+)
+
+__all__ = [
+    "SECRET_PATTERNS",
+    "SecretHit",
+    "SecretPattern",
+    "endpoints_from_paths",
+    "extract_endpoints",
+    "redact",
+    "scan_paths",
+    "scan_text",
+    "shannon_entropy",
+]

huntkit/secrets/patterns.py

@@ -0,0 +1,172 @@
+"""Secret-pattern catalog + JS endpoint extraction.
+
+A declarative set of regexes for high-signal credential leaks — the same
+"add a row, not a branch" ethos as the signal and playbook catalogs. Each
+pattern names the capture group holding the secret so the scanner can redact
+exactly that span, and an optional Shannon-entropy floor to drop low-entropy
+false positives on the generic patterns.
+
+Pure and offline: it matches text the operator already gathered (JS bundles,
+config dumps, responses) and never touches a target.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass
+
+from ..utils.severity import Severity
+
+
+@dataclass(frozen=True)
+class SecretPattern:
+    id: str
+    title: str
+    severity: Severity
+    regex: re.Pattern
+    group: int = 0  # capture group holding the secret value
+    min_entropy: float = 0.0  # skip matches below this Shannon entropy (0 = off)
+
+
+def shannon_entropy(s: str) -> float:
+    """Bits-per-character Shannon entropy — high for random keys, low for words."""
+    if not s:
+        return 0.0
+    n = len(s)
+    return -sum((c / n) * math.log2(c / n) for c in Counter(s).values())
+
+
+def _c(pattern: str, flags: int = 0) -> re.Pattern:
+    return re.compile(pattern, flags)
+
+
+# Ordered most-specific (unique prefixes) → most-generic (keyword + entropy).
+SECRET_PATTERNS: tuple[SecretPattern, ...] = (
+    SecretPattern(
+        "aws-access-key", "AWS access key id", Severity.CRITICAL, _c(r"\b(AKIA[0-9A-Z]{16})\b"), 1
+    ),
+    SecretPattern(
+        "aws-secret-key",
+        "AWS secret access key",
+        Severity.CRITICAL,
+        _c(r"(?i)aws.{0,24}?['\"]([A-Za-z0-9/+=]{40})['\"]"),
+        1,
+        4.0,
+    ),
+    SecretPattern(
+        "gcp-api-key", "Google API key", Severity.HIGH, _c(r"\b(AIza[0-9A-Za-z\-_]{35})\b"), 1
+    ),
+    SecretPattern(
+        "gcp-oauth",
+        "Google OAuth access token",
+        Severity.HIGH,
+        _c(r"\b(ya29\.[0-9A-Za-z\-_]{20,})"),
+    ),
+    SecretPattern(
+        "github-token", "GitHub token", Severity.CRITICAL, _c(r"\b(gh[opsu]_[0-9A-Za-z]{36})\b"), 1
+    ),
+    SecretPattern(
+        "github-pat",
+        "GitHub fine-grained PAT",
+        Severity.CRITICAL,
+        _c(r"\b(github_pat_[0-9A-Za-z_]{82})\b"),
+        1,
+    ),
+    SecretPattern(
+        "slack-token", "Slack token", Severity.HIGH, _c(r"\b(xox[baprs]-[0-9A-Za-z-]{10,48})\b"), 1
+    ),
+    SecretPattern(
+        "slack-webhook",
+        "Slack incoming webhook",
+        Severity.MEDIUM,
+        _c(r"(https://hooks\.slack\.com/services/T[0-9A-Za-z_]+/B[0-9A-Za-z_]+/[0-9A-Za-z_]+)"),
+        1,
+    ),
+    SecretPattern(
+        "stripe-live",
+        "Stripe live secret key",
+        Severity.CRITICAL,
+        _c(r"\b(sk_live_[0-9A-Za-z]{24,})\b"),
+        1,
+    ),
+    SecretPattern(
+        "stripe-test",
+        "Stripe test secret key",
+        Severity.LOW,
+        _c(r"\b(sk_test_[0-9A-Za-z]{24,})\b"),
+        1,
+    ),
+    SecretPattern(
+        "openai-key",
+        "OpenAI API key",
+        Severity.HIGH,
+        _c(r"\b(sk-(?:proj-)?[A-Za-z0-9_\-]{32,})\b"),
+        1,
+        3.5,
+    ),
+    SecretPattern("twilio-key", "Twilio API key", Severity.HIGH, _c(r"\b(SK[0-9a-fA-F]{32})\b"), 1),
+    SecretPattern(
+        "mailgun-key", "Mailgun API key", Severity.HIGH, _c(r"\b(key-[0-9a-zA-Z]{32})\b"), 1
+    ),
+    SecretPattern(
+        "npm-token", "npm access token", Severity.HIGH, _c(r"\b(npm_[0-9A-Za-z]{36})\b"), 1
+    ),
+    SecretPattern(
+        "private-key",
+        "Private key block",
+        Severity.CRITICAL,
+        _c(r"(-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----)"),
+        1,
+    ),
+    SecretPattern(
+        "jwt",
+        "JSON Web Token",
+        Severity.MEDIUM,
+        _c(r"\b(eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})\b"),
+        1,
+    ),
+    SecretPattern(
+        "generic-secret",
+        "Hardcoded secret / key",
+        Severity.MEDIUM,
+        _c(
+            r"(?i)(?:api[_-]?key|secret|password|passwd|access[_-]?token|auth[_-]?token)"
+            r"['\"]?\s*[:=]\s*['\"]([0-9A-Za-z\-_./+=]{12,})['\"]"
+        ),
+        1,
+        3.2,
+    ),
+)
+
+
+# JS endpoint / path extraction — quoted paths and absolute URLs in script text.
+ENDPOINT_RE = re.compile(r"""['"`](/[\w.\-/?=&%:@~]{2,}|https?://[\w.\-/?=&%:@~]+)['"`]""")
+
+# Static-asset extensions that are noise when hunting for API endpoints.
+_NOISE_EXT = (
+    ".css",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".svg",
+    ".webp",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".ico",
+    ".map",
+)
+
+
+def extract_endpoints(text: str) -> list[str]:
+    """Pull API-ish paths and URLs out of JS text, de-duped, asset noise dropped."""
+    found: list[str] = []
+    for m in ENDPOINT_RE.finditer(text):
+        ep = m.group(1)
+        if ep.lower().rsplit("?", 1)[0].endswith(_NOISE_EXT):
+            continue
+        found.append(ep)
+    return list(dict.fromkeys(found))

huntkit/secrets/scanner.py

@@ -0,0 +1,133 @@
+"""Scan gathered text for leaked secrets and JS endpoints.
+
+The scanner matches the pattern catalog against text or files and returns typed
+:class:`SecretHit` findings. Every finding's value is **redacted** on the way
+out — enough to locate it in the source, never enough to re-leak it — so the
+scanner's output (tables, JSON) is safe to store and share.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..utils.severity import Severity
+from .patterns import SECRET_PATTERNS, extract_endpoints, shannon_entropy
+
+# Binary / asset extensions never worth scanning as text.
+_SKIP_EXT = {
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".webp",
+    ".ico",
+    ".bmp",
+    ".svg",
+    ".pdf",
+    ".zip",
+    ".gz",
+    ".tar",
+    ".bz2",
+    ".7z",
+    ".rar",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".eot",
+    ".mp4",
+    ".mp3",
+    ".wasm",
+    ".exe",
+    ".dll",
+    ".so",
+    ".dylib",
+    ".class",
+    ".pyc",
+}
+_MAX_BYTES = 8 * 1024 * 1024  # skip files larger than 8 MB
+
+
+@dataclass(frozen=True)
+class SecretHit:
+    id: str
+    title: str
+    severity: Severity
+    source: str
+    line: int
+    match: str  # redacted — safe to print and store
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "title": self.title,
+            "severity": self.severity.label,
+            "source": self.source,
+            "line": self.line,
+            "match": self.match,
+        }
+
+
+def redact(value: str) -> str:
+    """Mask the middle of a secret: keep a few edge chars to locate it."""
+    if len(value) <= 8:
+        return "*" * len(value)
+    return f"{value[:4]}…{value[-4:]}  ({len(value)} chars)"
+
+
+def scan_text(text: str, *, source: str = "-") -> list[SecretHit]:
+    """Find secrets in ``text``, de-duped per (pattern, source, line, value)."""
+    hits: list[SecretHit] = []
+    seen: set[tuple] = set()
+    for lineno, line in enumerate(text.splitlines(), 1):
+        for pat in SECRET_PATTERNS:
+            for m in pat.regex.finditer(line):
+                value = m.group(pat.group)
+                if pat.min_entropy and shannon_entropy(value) < pat.min_entropy:
+                    continue
+                key = (pat.id, source, lineno, value)
+                if key in seen:
+                    continue
+                seen.add(key)
+                hits.append(
+                    SecretHit(pat.id, pat.title, pat.severity, source, lineno, redact(value))
+                )
+    return hits
+
+
+def _readable(path: Path) -> str | None:
+    if path.suffix.lower() in _SKIP_EXT:
+        return None
+    try:
+        if path.stat().st_size > _MAX_BYTES:
+            return None
+        return path.read_text(encoding="utf-8", errors="ignore")
+    except OSError:
+        return None
+
+
+def _walk(paths: list[Path]):
+    """Yield (path, text) for every readable file under the given paths."""
+    for raw in paths:
+        p = Path(raw)
+        files = sorted(f for f in p.rglob("*") if f.is_file()) if p.is_dir() else [p]
+        for f in files:
+            text = _readable(f)
+            if text is not None:
+                yield f, text
+
+
+def scan_paths(paths: list[Path]) -> list[SecretHit]:
+    """Scan files and directories (recursively) for secrets."""
+    hits: list[SecretHit] = []
+    for path, text in _walk([Path(p) for p in paths]):
+        hits.extend(scan_text(text, source=str(path)))
+    return hits
+
+
+def endpoints_from_paths(paths: list[Path]) -> list[str]:
+    """Extract JS endpoints across files and directories, de-duped in order."""
+    found: list[str] = []
+    for _path, text in _walk([Path(p) for p in paths]):
+        found.extend(extract_endpoints(text))
+    return list(dict.fromkeys(found))

huntkit/shots/__init__.py

@@ -0,0 +1,29 @@
+"""Screenshots layer — a visual of every live URL, for triage and the report.
+
+Drives a headless screenshotter (``gowitness``) over the scope-filtered live
+URLs and records a manifest (``scans/screenshots.json``) that the reporting
+layer turns into an embedded gallery. Depends only on ``core``; the capture
+backend is injectable, so the orchestration is tested without a browser.
+"""
+
+from __future__ import annotations
+
+from .runner import (
+    SHOTS_DIR,
+    SHOTS_JSON,
+    Shot,
+    ShotSummary,
+    gowitness_capture,
+    run_shots,
+    shot_targets,
+)
+
+__all__ = [
+    "SHOTS_DIR",
+    "SHOTS_JSON",
+    "Shot",
+    "ShotSummary",
+    "gowitness_capture",
+    "run_shots",
+    "shot_targets",
+]