huntkit 0.3.0__py3-none-any.whl

huntkit/history/diff.py

@@ -0,0 +1,95 @@
+"""Diff two snapshots — what appeared and what disappeared since last run.
+
+The delta is the whole point of re-running a program: new subdomains to probe,
+freshly-live URLs, and above all *new findings*. :func:`diff_snapshots`
+compares two :class:`~huntkit.history.store.Snapshot` objects per category;
+:func:`current_vs` compares a saved baseline against the workspace as it stands
+right now, so ``huntkit diff`` answers "what changed since the baseline?".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ..core.workspace import Workspace
+from .store import CATEGORIES, Snapshot, capture
+
+
+@dataclass
+class CategoryDiff:
+    name: str
+    added: list[str] = field(default_factory=list)
+    removed: list[str] = field(default_factory=list)
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.added or self.removed)
+
+
+@dataclass
+class DiffReport:
+    program: str
+    base_id: str
+    base_time: str
+    new_id: str
+    new_time: str
+    categories: list[CategoryDiff] = field(default_factory=list)
+
+    def by_name(self, name: str) -> CategoryDiff:
+        for c in self.categories:
+            if c.name == name:
+                return c
+        return CategoryDiff(name)
+
+    @property
+    def changed(self) -> list[CategoryDiff]:
+        return [c for c in self.categories if c.changed]
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.changed)
+
+    @property
+    def total_added(self) -> int:
+        return sum(len(c.added) for c in self.categories)
+
+    @property
+    def total_removed(self) -> int:
+        return sum(len(c.removed) for c in self.categories)
+
+    @property
+    def new_findings(self) -> list[str]:
+        return self.by_name("findings").added
+
+
+def diff_snapshots(old: Snapshot, new: Snapshot, program: str) -> DiffReport:
+    """Per-category added/removed between two snapshots (old → new)."""
+    cats: list[CategoryDiff] = []
+    for name in CATEGORIES:
+        old_set = set(old.get(name))
+        new_set = set(new.get(name))
+        cats.append(
+            CategoryDiff(
+                name=name,
+                added=sorted(new_set - old_set),
+                removed=sorted(old_set - new_set),
+            )
+        )
+    return DiffReport(
+        program=program,
+        base_id=old.id,
+        base_time=old.generated,
+        new_id=new.id,
+        new_time=new.generated,
+        categories=cats,
+    )
+
+
+def current_vs(ws: Workspace, baseline: Snapshot) -> tuple[DiffReport, Snapshot]:
+    """Diff the workspace as it stands now against ``baseline``.
+
+    Returns the report and the freshly-captured (still unsaved) snapshot, so a
+    caller can persist it as the next baseline if asked.
+    """
+    now = capture(ws)
+    return diff_snapshots(baseline, now, ws.program), now

huntkit/history/store.py

@@ -0,0 +1,133 @@
+"""Snapshots — freeze a program's result set so later runs can be diffed.
+
+Bug bounty is not one pass; it is the same scope re-run over weeks. The signal
+that matters is the *delta*: a subdomain that appeared yesterday, a URL that
+went live, a finding that is new since the last sweep. This layer captures the
+current state of a workspace's result files into a timestamped snapshot under
+``history/`` so :mod:`huntkit.history.diff` can compare two points in time.
+
+Pure and offline — it only reads artifacts already on disk.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..core.workspace import Workspace
+
+HISTORY_DIR = "history"
+_FINDINGS_JSON = "scans/findings.json"
+
+# The result sets a snapshot freezes, in pipeline order. Each text category
+# maps to a deduped result file; ``findings`` is flattened from the scan JSON.
+CATEGORIES: tuple[str, ...] = ("subdomains", "resolved", "live", "ports", "urls", "findings")
+_FILE_SOURCES: dict[str, str] = {
+    "subdomains": "recon/subdomains.txt",
+    "resolved": "recon/resolved.txt",
+    "live": "recon/live.txt",
+    "ports": "recon/ports.txt",
+    "urls": "urls/urls.txt",
+}
+
+
+@dataclass
+class Snapshot:
+    """One frozen point in time: the sorted, deduped items per category."""
+
+    id: str  # sortable stamp, e.g. 20261027-163000
+    generated: str  # ISO-8601 UTC
+    items: dict[str, list[str]] = field(default_factory=dict)
+
+    @property
+    def counts(self) -> dict[str, int]:
+        return {cat: len(self.items.get(cat, [])) for cat in CATEGORIES}
+
+    @property
+    def total(self) -> int:
+        return sum(len(v) for v in self.items.values())
+
+    def get(self, category: str) -> list[str]:
+        return self.items.get(category, [])
+
+    def to_dict(self) -> dict:
+        return {"id": self.id, "generated": self.generated, "items": self.items}
+
+    @classmethod
+    def from_dict(cls, data: dict) -> Snapshot:
+        raw = data.get("items", {})
+        items = {cat: list(raw.get(cat, [])) for cat in CATEGORIES}
+        return cls(id=data.get("id", ""), generated=data.get("generated", ""), items=items)
+
+
+def _findings(ws: Workspace) -> list[str]:
+    """Flatten the combined scan findings into a sorted, deduped list."""
+    path = ws.path(_FINDINGS_JSON)
+    if not path.exists():
+        return []
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (ValueError, OSError):
+        return []
+    seen: set[str] = set()
+    for lines in data.get("tools", {}).values():
+        seen.update(ln for ln in lines if ln.strip())
+    return sorted(seen)
+
+
+def capture(ws: Workspace) -> Snapshot:
+    """Freeze the workspace's current result sets into an unsaved snapshot."""
+    items: dict[str, list[str]] = {}
+    for cat, rel in _FILE_SOURCES.items():
+        items[cat] = sorted(dict.fromkeys(ws.read_lines(rel)))
+    items["findings"] = _findings(ws)
+    return Snapshot(
+        id=time.strftime("%Y%m%d-%H%M%S", time.gmtime()),
+        generated=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        items=items,
+    )
+
+
+def save(ws: Workspace, snap: Snapshot) -> Path:
+    """Persist a snapshot under ``history/<id>.json`` and return its path."""
+    outdir = Path(ws.path(HISTORY_DIR))
+    outdir.mkdir(parents=True, exist_ok=True)
+    path = outdir / f"{snap.id}.json"
+    path.write_text(json.dumps(snap.to_dict(), indent=2), encoding="utf-8")
+    return path
+
+
+def load(path: Path) -> Snapshot:
+    return Snapshot.from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
+
+
+def list_snapshots(ws: Workspace) -> list[Snapshot]:
+    """All saved snapshots, oldest first (sorted by id)."""
+    outdir = Path(ws.path(HISTORY_DIR))
+    if not outdir.exists():
+        return []
+    snaps = []
+    for p in outdir.glob("*.json"):
+        try:
+            snaps.append(load(p))
+        except (ValueError, OSError):
+            continue
+    return sorted(snaps, key=lambda s: s.id)
+
+
+def latest(ws: Workspace) -> Snapshot | None:
+    snaps = list_snapshots(ws)
+    return snaps[-1] if snaps else None
+
+
+def find(ws: Workspace, snapshot_id: str) -> Snapshot | None:
+    """Look up a snapshot by exact id, or None."""
+    path = Path(ws.path(HISTORY_DIR)) / f"{snapshot_id}.json"
+    if not path.exists():
+        return None
+    try:
+        return load(path)
+    except (ValueError, OSError):
+        return None

huntkit/intel/__init__.py

@@ -0,0 +1,48 @@
+"""Intelligence layer — turn recon output into scored, prioritised leads.
+
+The engine reads a workspace's recon artifacts, runs them through a declarative
+:mod:`~huntkit.intel.signals` catalog, scores each host, and rolls the signals
+up into prioritised attack paths. It's a feature layer (like the pipeline):
+it builds on ``core`` and ``utils`` but is independent of the plugin registry,
+so it stays pure and easy to test.
+"""
+
+from __future__ import annotations
+
+from .engine import (
+    AttackPath,
+    HostIntel,
+    IntelReport,
+    Priority,
+    analyze,
+    save_report,
+)
+from .signals import (
+    Rule,
+    Severity,
+    Signal,
+    scan_signals,
+    signals_from_findings,
+    signals_from_labels,
+    signals_from_params,
+    signals_from_paths,
+    signals_from_ports,
+)
+
+__all__ = [
+    "AttackPath",
+    "HostIntel",
+    "IntelReport",
+    "Priority",
+    "Rule",
+    "Severity",
+    "Signal",
+    "analyze",
+    "save_report",
+    "scan_signals",
+    "signals_from_findings",
+    "signals_from_labels",
+    "signals_from_params",
+    "signals_from_paths",
+    "signals_from_ports",
+]

huntkit/intel/engine.py

@@ -0,0 +1,233 @@
+"""Intelligence engine — recon output -> scored, prioritised attack surface.
+
+Reads a workspace's recon artifacts, runs them through the signal catalog,
+groups the findings per host, scores each host Low -> Critical, and rolls the
+signals up into prioritised **attack paths** (which bug-class playbook to run,
+and where). Pure analysis over data already on disk — it never touches the
+target.
+
+The layout paths it reads mirror :data:`huntkit.pipeline.STAGE_FILES`; they are
+duplicated here as small constants so the intelligence layer stays independent
+of the pipeline (and trivially testable without the plugin registry).
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+from enum import IntEnum
+from pathlib import Path
+
+from ..core.workspace import Workspace
+from ..knowledge.playbooks import titles as _playbook_titles
+from ..utils import filesystem as fs
+from .signals import Severity, Signal, scan_signals, url_host
+
+# Workspace-relative recon artifacts the engine consumes.
+_LIVE = "recon/live.txt"
+_URLS = "urls/urls.txt"
+_PORTS = "recon/ports.txt"
+_SUBS = "recon/subdomains.txt"
+_RESOLVED = "recon/resolved.txt"
+# Active-scan output (written by the scanner layer); optional. Named here as a
+# constant rather than imported, so intel stays independent of the scanner.
+_FINDINGS = "scans/findings.json"
+
+# Human names for the bug-class playbook ids. The knowledge base owns the
+# playbook catalog, so names come from there — one source of truth, and an
+# attack path in a report links straight to `huntkit playbook <id>`.
+PLAYBOOK_NAMES: dict[str, str] = _playbook_titles()
+
+
+class Priority(IntEnum):
+    LOW = 1
+    MEDIUM = 2
+    HIGH = 3
+    CRITICAL = 4
+
+    @property
+    def label(self) -> str:
+        return self.name.capitalize()
+
+    @property
+    def style(self) -> str:  # matches the theme keys in utils.terminal
+        return self.name.lower()
+
+
+def _priority(signals: list[Signal]) -> Priority:
+    """Host priority = worst single signal, nudged up by breadth of surface.
+
+    A Critical signal makes the host Critical outright; otherwise a High
+    signal (or enough stacked medium/low ones) makes it High, and so on. The
+    aggregate thresholds mean a host with many medium issues still rises.
+    """
+    if not signals:
+        return Priority.LOW
+    top = max(s.severity for s in signals)
+    total = sum(int(s.severity) for s in signals)
+    if top >= Severity.CRITICAL:
+        return Priority.CRITICAL
+    if top >= Severity.HIGH or total >= 8:
+        return Priority.HIGH
+    if top >= Severity.MEDIUM or total >= 4:
+        return Priority.MEDIUM
+    return Priority.LOW
+
+
+@dataclass
+class HostIntel:
+    host: str
+    signals: list[Signal]
+
+    @property
+    def score(self) -> int:
+        return sum(int(s.severity) for s in self.signals)
+
+    @property
+    def priority(self) -> Priority:
+        return _priority(self.signals)
+
+    @property
+    def playbooks(self) -> list[str]:
+        """Suggested playbooks, most-severe signal first, de-duplicated."""
+        ordered: list[str] = []
+        for s in sorted(self.signals, key=lambda x: -int(x.severity)):
+            for pb in s.playbooks:
+                if pb not in ordered:
+                    ordered.append(pb)
+        return ordered
+
+    def ranked_signals(self) -> list[Signal]:
+        return sorted(self.signals, key=lambda s: (-int(s.severity), s.category, s.id))
+
+    def to_dict(self) -> dict:
+        return {
+            "host": self.host,
+            "priority": self.priority.label,
+            "score": self.score,
+            "playbooks": self.playbooks,
+            "signals": [s.to_dict() for s in self.ranked_signals()],
+        }
+
+
+@dataclass
+class AttackPath:
+    """One bug-class playbook and everywhere the recon points it."""
+
+    playbook: str
+    name: str
+    severity: Severity  # worst contributing signal
+    hosts: list[str]
+    signals: list[Signal]
+
+    def to_dict(self) -> dict:
+        return {
+            "playbook": self.playbook,
+            "name": self.name,
+            "severity": self.severity.label,
+            "hosts": self.hosts,
+            "signal_count": len(self.signals),
+        }
+
+
+@dataclass
+class IntelReport:
+    program: str
+    hosts: list[HostIntel]
+    generated: float = field(default_factory=time.time)
+
+    @property
+    def signals(self) -> list[Signal]:
+        return [s for h in self.hosts for s in h.signals]
+
+    @property
+    def summary(self) -> dict[str, int]:
+        """Host counts per priority, always highest-first and fully populated."""
+        counts = Counter(h.priority for h in self.hosts)
+        return {p.label: counts.get(p, 0) for p in sorted(Priority, reverse=True)}
+
+    def attack_paths(self) -> list[AttackPath]:
+        groups: dict[str, list[Signal]] = defaultdict(list)
+        for s in self.signals:
+            for pb in s.playbooks:
+                groups[pb].append(s)
+        paths = [
+            AttackPath(
+                playbook=pb,
+                name=PLAYBOOK_NAMES.get(pb, pb),
+                severity=max(s.severity for s in sigs),
+                hosts=sorted({s.host for s in sigs}),
+                signals=sigs,
+            )
+            for pb, sigs in groups.items()
+        ]
+        paths.sort(key=lambda p: (-int(p.severity), -len(p.hosts), p.playbook))
+        return paths
+
+    def to_dict(self) -> dict:
+        return {
+            "program": self.program,
+            "generated": self.generated,
+            "summary": self.summary,
+            "hosts": [h.to_dict() for h in self.hosts],
+            "attack_paths": [p.to_dict() for p in self.attack_paths()],
+        }
+
+
+def _host_pool(ws: Workspace, live: list[str]) -> list[str]:
+    """Every hostname worth label-analysis: subdomains, resolved, live hosts."""
+    pool: set[str] = set(ws.read_lines(_SUBS)) | set(ws.read_lines(_RESOLVED))
+    for url in live:
+        host = url_host(url)
+        if host:
+            pool.add(host)
+    return sorted(h.lower() for h in pool if h.strip())
+
+
+def _read_findings(ws: Workspace) -> dict[str, list[str]]:
+    """Scanner findings keyed by tool, or empty if no scan has run / file is bad."""
+    path = ws.path(_FINDINGS)
+    if not path.exists():
+        return {}
+    try:
+        data = json.loads(fs.read_text(path))
+    except (ValueError, OSError):
+        return {}
+    tools = data.get("tools") if isinstance(data, dict) else None
+    if not isinstance(tools, dict):
+        return {}
+    return {
+        str(tool): [str(x) for x in lines]
+        for tool, lines in tools.items()
+        if isinstance(lines, list)
+    }
+
+
+def analyze(ws: Workspace) -> IntelReport:
+    """Score a workspace's recon surface into a prioritised intel report."""
+    live = ws.read_lines(_LIVE)
+    urls = ws.read_lines(_URLS)
+    ports = ws.read_lines(_PORTS)
+    hosts = _host_pool(ws, live)
+    url_pool = list(dict.fromkeys(live + urls))  # dedupe, keep order
+    findings = _read_findings(ws)
+
+    signals = scan_signals(ports=ports, urls=url_pool, hosts=hosts, findings=findings)
+
+    by_host: dict[str, list[Signal]] = defaultdict(list)
+    for s in signals:
+        by_host[s.host].append(s)
+
+    intel_hosts = [HostIntel(host=h, signals=sigs) for h, sigs in by_host.items()]
+    intel_hosts.sort(key=lambda h: (-int(h.priority), -h.score, h.host))
+    return IntelReport(program=ws.program, hosts=intel_hosts)
+
+
+def save_report(ws: Workspace, report: IntelReport) -> Path:
+    """Persist the report as scans/intel.json for later reporting; record count."""
+    path = ws.path("scans", "intel.json")
+    fs.write_text(path, json.dumps(report.to_dict(), indent=2))
+    ws.state.set_count("intel_signals", len(report.signals))
+    return path