huntkit 0.3.0__py3-none-any.whl

huntkit/core/runner.py

@@ -0,0 +1,117 @@
+"""CommandRunner — the configured, logged execution layer.
+
+Sits on top of `utils.process.execute` and adds everything a plugin should
+not have to think about: binary resolution, per-tool timeouts and extra
+args, proxy/UA environment, retries with backoff, structured logging, and
+optional result caching.
+
+Plugins build an argv and hand it here; they never call subprocess directly.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Sequence
+from typing import Optional
+
+from ..utils import process
+from ..utils.process import ProcResult
+from .config import Config
+from .logger import get_logger
+
+log = get_logger("runner")
+
+
+class ToolNotFound(RuntimeError):
+    """Raised when a required binary is not on PATH and not pinned in config."""
+
+
+class CommandRunner:
+    def __init__(self, config: Optional[Config] = None, cache=None) -> None:
+        self.config = config or Config()
+        self.cache = cache  # optional core.cache.Cache; injected by the app
+
+    # ---- binary resolution ----------------------------------------------
+    def resolve(self, tool: str) -> Optional[str]:
+        """Return the binary path for `tool`, honouring a pinned config path."""
+        pinned = self.config.tool(tool).path
+        if pinned:
+            return pinned
+        return process.which(tool)
+
+    def available(self, tool: str) -> bool:
+        return self.resolve(tool) is not None
+
+    # ---- execution -------------------------------------------------------
+    def run(
+        self,
+        tool: str,
+        args: Sequence[str],
+        *,
+        stdin_data: Optional[str] = None,
+        timeout: Optional[int] = None,
+        retry: Optional[int] = None,
+        cache_key: Optional[str] = None,
+    ) -> ProcResult:
+        """Run a registered tool with `args`.
+
+        `tool` is the binary name (resolved via config/PATH); `args` are the
+        arguments after it. Per-tool `extra_args` from config are appended.
+        """
+        binary = self.resolve(tool)
+        if binary is None:
+            raise ToolNotFound(f"'{tool}' not found on PATH — install it or set tools.{tool}.path")
+
+        tool_cfg = self.config.tool(tool)
+        argv = [binary, *[str(a) for a in args], *tool_cfg.extra_args]
+        eff_timeout = timeout or tool_cfg.timeout or self.config.general.timeout
+        eff_retry = self.config.general.retry if retry is None else retry
+
+        if cache_key and self.cache is not None:
+            cached = self.cache.get(cache_key)
+            if cached is not None:
+                log.debug("cache hit: %s", cache_key)
+                return ProcResult(0, cached, "", argv, timed_out=False)
+
+        result = self._run_with_retry(argv, stdin_data, eff_timeout, eff_retry, tool)
+
+        if cache_key and self.cache is not None and result.ok:
+            self.cache.set(cache_key, result.stdout)
+        return result
+
+    def _run_with_retry(
+        self,
+        argv: list[str],
+        stdin_data: Optional[str],
+        timeout: int,
+        retry: int,
+        tool: str,
+    ) -> ProcResult:
+        env = self._build_env()
+        last: Optional[ProcResult] = None
+        for attempt in range(retry + 1):
+            log.debug("exec (try %d/%d): %s", attempt + 1, retry + 1, " ".join(argv))
+            result = process.execute(argv, timeout=timeout, stdin_data=stdin_data, env=env)
+            log.debug(
+                "→ code=%s dur=%.1fs lines=%d", result.code, result.duration, len(result.lines)
+            )
+            # retry only transient failures (timeout), not clean non-zero exits
+            if result.ok or not result.timed_out or attempt == retry:
+                return result
+            last = result
+            backoff = 1.5 * (attempt + 1)
+            log.warning("%s timed out; retrying in %.1fs", tool, backoff)
+            time.sleep(backoff)
+        return last or process.execute(argv, timeout=timeout, stdin_data=stdin_data, env=env)
+
+    def _build_env(self) -> Optional[dict]:
+        """Inject proxy/UA into the child environment when configured."""
+        import os
+
+        http = self.config.http
+        if not http.proxy:
+            return None
+        env = dict(os.environ)
+        env["HTTP_PROXY"] = http.proxy
+        env["HTTPS_PROXY"] = http.proxy
+        return env

huntkit/core/state.py

@@ -0,0 +1,154 @@
+"""Scan state, checkpoints, and resume support.
+
+Every pipeline stage records its status so an interrupted run can resume
+where it left off and re-run only failed steps. Persisted atomically as JSON
+so a crash mid-write can't corrupt it.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Any, Optional
+
+from ..utils import filesystem as fs
+
+
+class StageStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    DONE = "done"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+@dataclass
+class StageRecord:
+    name: str
+    status: StageStatus = StageStatus.PENDING
+    started: Optional[float] = None
+    finished: Optional[float] = None
+    error: Optional[str] = None
+    meta: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def duration(self) -> Optional[float]:
+        if self.started and self.finished:
+            return self.finished - self.started
+        return None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "status": self.status.value,
+            "started": self.started,
+            "finished": self.finished,
+            "error": self.error,
+            "meta": self.meta,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> StageRecord:
+        return cls(
+            name=data["name"],
+            status=StageStatus(data.get("status", "pending")),
+            started=data.get("started"),
+            finished=data.get("finished"),
+            error=data.get("error"),
+            meta=data.get("meta", {}),
+        )
+
+
+class StateStore:
+    """Persistent stage tracker for one workspace."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.stages: dict[str, StageRecord] = {}
+        self.counts: dict[str, int] = {}
+        self.created = time.time()
+        self.load()
+
+    # ---- persistence -----------------------------------------------------
+    def load(self) -> None:
+        import json
+
+        if not self.path.exists():
+            return
+        try:
+            data = json.loads(fs.read_text(self.path))
+        except (ValueError, OSError):
+            return
+        self.created = data.get("created", self.created)
+        self.counts = data.get("counts", {})
+        self.stages = {
+            name: StageRecord.from_dict(rec) for name, rec in data.get("stages", {}).items()
+        }
+
+    def save(self) -> None:
+        import json
+
+        payload = {
+            "created": self.created,
+            "updated": time.time(),
+            "counts": self.counts,
+            "stages": {name: rec.to_dict() for name, rec in self.stages.items()},
+        }
+        fs.write_text(self.path, json.dumps(payload, indent=2))
+
+    # ---- stage lifecycle -------------------------------------------------
+    def _get(self, stage: str) -> StageRecord:
+        if stage not in self.stages:
+            self.stages[stage] = StageRecord(name=stage)
+        return self.stages[stage]
+
+    def status(self, stage: str) -> StageStatus:
+        return self._get(stage).status
+
+    def is_done(self, stage: str) -> bool:
+        return self.status(stage) == StageStatus.DONE
+
+    def start(self, stage: str) -> None:
+        rec = self._get(stage)
+        rec.status = StageStatus.RUNNING
+        rec.started = time.time()
+        rec.error = None
+        self.save()
+
+    def done(self, stage: str, **meta: Any) -> None:
+        rec = self._get(stage)
+        rec.status = StageStatus.DONE
+        rec.finished = time.time()
+        rec.meta.update(meta)
+        self.save()
+
+    def fail(self, stage: str, error: str) -> None:
+        rec = self._get(stage)
+        rec.status = StageStatus.FAILED
+        rec.finished = time.time()
+        rec.error = error
+        self.save()
+
+    def skip(self, stage: str) -> None:
+        rec = self._get(stage)
+        rec.status = StageStatus.SKIPPED
+        self.save()
+
+    # ---- resume queries --------------------------------------------------
+    def pending(self, all_stages: list[str]) -> list[str]:
+        """Stages not yet completed — the resume set."""
+        return [s for s in all_stages if not self.is_done(s)]
+
+    def failed(self) -> list[str]:
+        return [s for s, r in self.stages.items() if r.status == StageStatus.FAILED]
+
+    def set_count(self, key: str, value: int) -> None:
+        self.counts[key] = value
+        self.save()
+
+    def reset(self) -> None:
+        """Clear all stage state (fresh scan)."""
+        self.stages.clear()
+        self.save()

huntkit/core/workspace.py

@@ -0,0 +1,145 @@
+"""Per-engagement workspace: layout, scope, deduped results, and state.
+
+A workspace is one folder per program under the configured workspace root
+(``~/.huntkit`` by default, or ``$HUNTKIT_HOME``). It owns three things:
+
+  - **layout**   a stable set of subdirectories (recon/urls/scans/…)
+  - **scope**    in/out patterns kept as plain, greppable text files, plus
+                 an :meth:`in_scope` guard so nothing is scanned off-scope
+  - **state**    a :class:`~huntkit.core.state.StateStore` for stage tracking,
+                 counts, and resume
+
+It sits in ``core`` because it depends only on the foundation (state, config,
+utils) — never on plugins or features — so anything above can build on it.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from ..utils import filesystem as fs
+from ..utils.validators import sanitize_filename
+from .config import Config
+from .state import StateStore
+
+# Subdirectories created inside every workspace.
+LAYOUT = ["scope", "recon", "urls", "scans", "notes", "reports"]
+
+_SCOPE_IN = "scope/in_scope.txt"
+_SCOPE_OUT = "scope/out_scope.txt"
+
+
+def default_base(config: Config | None = None) -> Path:
+    """Where workspaces live: $HUNTKIT_HOME, else config root, else ~/.huntkit."""
+    env = os.environ.get("HUNTKIT_HOME")
+    if env:
+        return Path(env).expanduser()
+    if config is not None:
+        return config.workspace_root
+    return Path.home() / ".huntkit"
+
+
+class Workspace:
+    def __init__(self, program: str, root: Path) -> None:
+        self.program = program
+        self.root = root
+        self.state = StateStore(root / "state.json")
+
+    # ---- construction ----------------------------------------------------
+    @classmethod
+    def open(
+        cls,
+        program: str,
+        *,
+        config: Config | None = None,
+        base: Path | None = None,
+    ) -> Workspace:
+        base = base or default_base(config)
+        root = base / sanitize_filename(program)
+        ws = cls(program=program, root=root)
+        ws._ensure_layout()
+        return ws
+
+    def _ensure_layout(self) -> None:
+        self.root.mkdir(parents=True, exist_ok=True)
+        for sub in LAYOUT:
+            (self.root / sub).mkdir(exist_ok=True)
+        # persist an initial state file so the workspace is discoverable by
+        # `workspace list` immediately after creation
+        if not self.state.path.exists():
+            self.state.save()
+
+    # ---- paths -----------------------------------------------------------
+    def path(self, *parts: str) -> Path:
+        """A path inside the workspace, guarded against traversal."""
+        return fs.safe_join(self.root, *parts)
+
+    # ---- scope -----------------------------------------------------------
+    def set_scope(self, in_scope: list[str], out_scope: list[str]) -> None:
+        fs.write_text(self.path(_SCOPE_IN), "\n".join(in_scope) + ("\n" if in_scope else ""))
+        fs.write_text(self.path(_SCOPE_OUT), "\n".join(out_scope) + ("\n" if out_scope else ""))
+
+    @property
+    def scope_in(self) -> list[str]:
+        return fs.read_lines(self.path(_SCOPE_IN))
+
+    @property
+    def scope_out(self) -> list[str]:
+        return fs.read_lines(self.path(_SCOPE_OUT))
+
+    def has_scope(self) -> bool:
+        return bool(self.scope_in or self.scope_out)
+
+    def in_scope(self, host: str) -> bool:
+        """True if ``host`` matches an in-scope pattern and no out-of-scope one.
+
+        Patterns accept a leading ``*.`` wildcard (``*.example.com`` matches
+        the apex and any subdomain). With no in-scope patterns defined,
+        everything not explicitly excluded is considered in scope.
+        """
+        if any(_match(host, p) for p in self.scope_out):
+            return False
+        ins = self.scope_in
+        if not ins:
+            return True
+        return any(_match(host, p) for p in ins)
+
+    def filter_scope(self, hosts: list[str]) -> list[str]:
+        return [h for h in hosts if self.in_scope(h)]
+
+    # ---- deduped result sets --------------------------------------------
+    def append_unique(self, relpath: str, items: list[str]) -> int:
+        """Merge items into a result file, sorted & unique; return new count."""
+        return fs.append_unique(self.path(relpath), items)
+
+    def read_lines(self, relpath: str) -> list[str]:
+        return fs.read_lines(self.path(relpath))
+
+    def count(self, relpath: str) -> int:
+        return fs.count_lines(self.path(relpath))
+
+    def exists(self, relpath: str) -> bool:
+        return self.path(relpath).exists()
+
+    def __repr__(self) -> str:  # pragma: no cover - cosmetic
+        return f"<Workspace {self.program} at {self.root}>"
+
+
+# --------------------------------------------------------------------------
+# Helpers
+# --------------------------------------------------------------------------
+def _match(host: str, pattern: str) -> bool:
+    host = host.lower().strip()
+    pattern = pattern.lower().strip()
+    if pattern.startswith("*."):
+        suffix = pattern[1:]  # ".example.com"
+        return host == pattern[2:] or host.endswith(suffix)
+    return host == pattern
+
+
+def list_workspaces(base: Path | None = None, config: Config | None = None) -> list[str]:
+    base = base or default_base(config)
+    if not base.exists():
+        return []
+    return sorted(p.name for p in base.iterdir() if p.is_dir() and (p / "state.json").exists())

huntkit/enrich/__init__.py

@@ -0,0 +1,39 @@
+"""Enrichment layer — asset-correlation signals over the recon surface.
+
+Two enrichers that turn a gathered surface into leads *elsewhere*:
+
+  - **favicon** — the Shodan-convention favicon hash (``http.favicon.hash:``),
+    the pivot from one icon to every host serving it;
+  - **asn** — the ASN / BGP prefix a host is announced from, the network block
+    to sweep for sibling assets.
+
+Depends only on ``core`` (workspace/scope) and ``utils``; both the HTTP fetch
+and the ASN resolve/lookup are injectable, so the layer is offline in tests.
+"""
+
+from __future__ import annotations
+
+from .asn import AsnResult, asn_for, parse_cymru
+from .engine import (
+    ENRICH_JSON,
+    EnrichReport,
+    enrich_hosts,
+    enrich_origins,
+    run_enrich,
+)
+from .favicon import FaviconResult, favicon_hash, favicon_url, hash_favicon
+
+__all__ = [
+    "ENRICH_JSON",
+    "AsnResult",
+    "EnrichReport",
+    "FaviconResult",
+    "asn_for",
+    "enrich_hosts",
+    "enrich_origins",
+    "favicon_hash",
+    "favicon_url",
+    "hash_favicon",
+    "parse_cymru",
+    "run_enrich",
+]

huntkit/enrich/asn.py

@@ -0,0 +1,139 @@
+"""ASN enrichment — map an in-scope host to the network that announces it.
+
+Knowing the ASN and BGP prefix behind a host turns a single name into a block
+of IP space to sweep: sibling origins, admin panels, and dev boxes frequently
+live in the same prefix as the flagship site. HuntKit resolves the host to an
+IP locally, then asks Team Cymru's IP-to-ASN WHOIS service for the mapping.
+
+> That WHOIS call is the one place this layer sends data off the box — the
+> *resolved IP* of an in-scope host is sent to ``whois.cymru.com``. It is a
+> public, well-known lookup service, but it is third-party egress, so the CLI
+> names it before running and both the resolver and the lookup are injectable
+> (the tests never touch the network).
+"""
+
+from __future__ import annotations
+
+import socket
+from collections.abc import Callable
+from dataclasses import dataclass
+
+ResolveFn = Callable[[str], str]  # host -> IP, raises on failure
+LookupFn = Callable[[str], str]  # IP -> raw Cymru WHOIS text, raises on failure
+
+_CYMRU_HOST = "whois.cymru.com"
+_CYMRU_PORT = 43
+_TIMEOUT = 10
+
+
+@dataclass
+class AsnResult:
+    """One host's network origin, as announced in BGP."""
+
+    host: str
+    ip: str = ""
+    asn: str = ""
+    prefix: str = ""
+    cc: str = ""
+    registry: str = ""
+    as_name: str = ""
+    error: str = ""
+
+    @property
+    def ok(self) -> bool:
+        return bool(self.asn)
+
+    def to_dict(self) -> dict:
+        return {
+            "host": self.host,
+            "ip": self.ip,
+            "asn": self.asn,
+            "prefix": self.prefix,
+            "cc": self.cc,
+            "registry": self.registry,
+            "as_name": self.as_name,
+            "error": self.error,
+        }
+
+
+def default_resolve(host: str) -> str:
+    """Resolve a hostname to an IPv4 address (stdlib; no third party)."""
+    return socket.gethostbyname(host)
+
+
+def default_lookup(ip: str) -> str:
+    """Query Team Cymru's WHOIS IP-to-ASN service for ``ip``.
+
+    Opens a short-lived TCP connection to port 43 and sends a single verbose
+    query. This is the layer's only outbound call to a non-target host.
+    """
+    query = f" -v {ip}\r\n".encode()
+    with socket.create_connection((_CYMRU_HOST, _CYMRU_PORT), timeout=_TIMEOUT) as sock:
+        sock.sendall(query)
+        chunks = []
+        while True:
+            chunk = sock.recv(4096)
+            if not chunk:
+                break
+            chunks.append(chunk)
+    return b"".join(chunks).decode("utf-8", "replace")
+
+
+def parse_cymru(text: str) -> dict[str, str]:
+    """Parse the first data row of a Team Cymru verbose WHOIS response.
+
+    The verbose format is a header row then pipe-delimited data:
+    ``AS | IP | BGP Prefix | CC | Registry | Allocated | AS Name``.
+    Returns an empty dict when no data row is present.
+    """
+    for line in text.splitlines():
+        if "|" not in line:
+            continue
+        cols = [c.strip() for c in line.split("|")]
+        # a data row leads with the numeric AS; the header row ("AS | IP | …")
+        # and any noise fail that test and are skipped
+        if len(cols) < 7 or not cols[0].isdigit():
+            continue
+        return {
+            "asn": cols[0],
+            "ip": cols[1],
+            "prefix": cols[2],
+            "cc": cols[3],
+            "registry": cols[4],
+            "as_name": cols[6],
+        }
+    return {}
+
+
+def asn_for(
+    host: str,
+    *,
+    resolve: ResolveFn = default_resolve,
+    lookup: LookupFn = default_lookup,
+) -> AsnResult:
+    """Resolve ``host`` and look up its ASN, never raising.
+
+    Any failure (unresolvable name, WHOIS timeout, unparsable reply) is
+    recorded on the result so one bad host cannot sink a batch.
+    """
+    try:
+        ip = resolve(host)
+    except Exception as exc:  # noqa: BLE001 — a resolution failure is data
+        return AsnResult(host=host, error=f"resolve: {exc or type(exc).__name__}")
+    try:
+        raw = lookup(ip)
+    except Exception as exc:  # noqa: BLE001 — a lookup failure is data
+        return AsnResult(host=host, ip=ip, error=f"lookup: {exc or type(exc).__name__}")
+
+    row = parse_cymru(raw)
+    if not row:
+        return AsnResult(host=host, ip=ip, error="no ASN in response")
+    return AsnResult(
+        host=host,
+        ip=row.get("ip", ip),
+        asn=row["asn"],
+        prefix=row.get("prefix", ""),
+        cc=row.get("cc", ""),
+        registry=row.get("registry", ""),
+        as_name=row.get("as_name", ""),
+    )