shadowcat 2.0.0__py3-none-any.whl

agent/core/phantom.py

@@ -0,0 +1,2327 @@
+"""PHANTOM — Probabilistic Hallucination-Anchored Navigation.
+
+Stage 1: Fingerprint Crystallization.
+
+The goal of this stage is to collapse a noisy set of HTTP signals (headers,
+cookies, body snippets from a handful of well-known paths) into a single
+compact ``FingerprintVector`` that downstream PHANTOM stages can reason over
+without re-issuing the underlying requests. One concurrent burst of recon +
+one cheap LLM call replaces dozens of agent-driven probes.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from collections.abc import Awaitable, Callable
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import httpx
+from pydantic import BaseModel, Field, ValidationError, field_validator
+
+from agent.core.profiling import span
+
+LLMCallable = Callable[[str], Awaitable[str]]
+
+# Same async callable shape, but with an optional ``json_schema`` kwarg the
+# integrator can plumb into the provider's strict structured-output mode
+# (OpenAI ``response_format={"type":"json_schema",...}``, Anthropic tool-use
+# forcing, vLLM/llama.cpp guided JSON). Used by :func:`generate_mam` to
+# constrain MAM output against :data:`COMPACT_MAM_JSON_SCHEMA`. If the
+# supplied callable doesn't accept the kwarg, :func:`generate_mam`
+# transparently falls back to a plain ``(prompt,) -> str`` call — so older
+# test stubs and integrators that haven't migrated still work.
+StructuredLLMCallable = Callable[..., Awaitable[str]]
+
+DEFAULT_RECON_PATHS: tuple[str, ...] = (
+    "/",
+    "/robots.txt",
+    "/favicon.ico",
+    "/.well-known/security.txt",
+    "/sitemap.xml",
+)
+
+_HEADERS_OF_INTEREST: tuple[str, ...] = (
+    "server",
+    "x-powered-by",
+    "x-aspnet-version",
+    "x-aspnetmvc-version",
+    "x-generator",
+    "x-drupal-cache",
+    "x-runtime",
+    "via",
+    "set-cookie",
+    "content-type",
+    "www-authenticate",
+)
+
+_BODY_SNIPPET_CHARS = 2048
+_PER_REQUEST_TIMEOUT_SECONDS = 3.0
+_FAVICON_HASH_BYTES = 4096
+
+
+class FingerprintVector(BaseModel):
+    """Crystallized view of a target's tech stack.
+
+    Every field is a free-form short string (the LLM picks the vocabulary) so
+    that downstream stages can do cheap string matching without a fixed
+    taxonomy. ``overall_confidence`` is the model's self-reported credence in
+    the *combination* of fields, not an average — a single high-confidence
+    field paired with several unknowns should still pull this down.
+    """
+
+    framework: str = Field(
+        description="Server-side framework (e.g. 'Django', 'Express', 'ASP.NET', 'unknown')."
+    )
+    version_guess: str = Field(
+        description="Best-effort version string for the framework. 'unknown' if no signal."
+    )
+    frontend: str = Field(
+        description="Frontend stack (e.g. 'React', 'Vue', 'server-rendered', 'unknown')."
+    )
+    auth_pattern: str = Field(
+        description="Auth scheme hint (e.g. 'session-cookie', 'JWT', 'Basic', 'none-observed')."
+    )
+    likely_orm: str = Field(
+        description="ORM / persistence hint (e.g. 'Django ORM', 'ActiveRecord', 'Prisma', 'unknown')."
+    )
+    overall_confidence: float = Field(
+        ge=0.0,
+        le=1.0,
+        description="Self-reported confidence in the overall fingerprint, in [0.0, 1.0].",
+    )
+
+    @field_validator("framework", "version_guess", "frontend", "auth_pattern", "likely_orm")
+    @classmethod
+    def _non_empty(cls, v: str) -> str:
+        # Empty strings would silently degrade downstream string matching; force
+        # the LLM (and any test stub) to commit to at least 'unknown'.
+        if not v or not v.strip():
+            return "unknown"
+        return v.strip()
+
+
+class PathSignal(BaseModel):
+    """Single-path result inside the raw signal dict.
+
+    Kept as a model rather than a free-form dict so the crystallization prompt
+    has a stable shape regardless of which paths timed out.
+    """
+
+    path: str
+    status: int | None = None
+    headers: dict[str, str] = Field(default_factory=dict)
+    set_cookie: list[str] = Field(default_factory=list)
+    body_snippet: str = ""
+    body_length: int = 0
+    error: str | None = None
+
+
+async def gather_raw_signals(
+    target_url: str,
+    *,
+    paths: tuple[str, ...] = DEFAULT_RECON_PATHS,
+    timeout_seconds: float = _PER_REQUEST_TIMEOUT_SECONDS,
+    client: httpx.AsyncClient | None = None,
+) -> dict[str, Any]:
+    """Fetch ``paths`` concurrently from ``target_url`` and return raw signals.
+
+    Each path is fetched in parallel via :func:`asyncio.gather`. A failure on
+    one path (timeout, connection refused, non-2xx, etc.) never blocks the
+    others — the failed entry simply carries an ``error`` field.
+
+    The optional ``client`` parameter lets callers reuse a long-lived
+    :class:`httpx.AsyncClient` (and its connection pool). When ``None``, a
+    one-shot client is created and closed before return.
+    """
+    if not target_url:
+        raise ValueError("target_url must be a non-empty URL")
+
+    parsed = urlparse(target_url)
+    if not parsed.scheme or not parsed.netloc:
+        raise ValueError(f"target_url must include scheme and host (got {target_url!r})")
+    base = f"{parsed.scheme}://{parsed.netloc}"
+
+    owns_client = False
+    if client is not None:
+        http = client
+    else:
+        from agent.core.backend import get_shared_http_client
+
+        http = get_shared_http_client()
+
+    try:
+        with span(f"phantom.gather_raw_signals [{len(paths)} paths, base={base}]"):
+            coros = [_fetch_one(http, base, p, timeout_seconds) for p in paths]
+            results = await asyncio.gather(*coros, return_exceptions=False)
+    finally:
+        if owns_client:
+            await http.aclose()
+
+    return {
+        "target_url": target_url,
+        "base": base,
+        "paths": [r.model_dump() for r in results],
+    }
+
+
+async def _fetch_one(
+    http: httpx.AsyncClient,
+    base: str,
+    path: str,
+    timeout_seconds: float,
+) -> PathSignal:
+    """Single-path fetch with hard timeout and graceful failure."""
+    url = urljoin(base + "/", path.lstrip("/"))
+    try:
+        resp = await asyncio.wait_for(
+            http.get(url),
+            timeout=timeout_seconds,
+        )
+    except TimeoutError:
+        return PathSignal(path=path, error="timeout")
+    except httpx.TimeoutException:
+        return PathSignal(path=path, error="timeout")
+    except httpx.RequestError as e:
+        return PathSignal(path=path, error=f"{e.__class__.__name__}: {e}")
+
+    headers = {k: v for k, v in resp.headers.items() if k.lower() in _HEADERS_OF_INTEREST}
+    set_cookie = (
+        list(resp.headers.get_list("set-cookie")) if hasattr(resp.headers, "get_list") else []
+    )
+
+    body_snippet = _safe_text_snippet(resp, path)
+
+    return PathSignal(
+        path=path,
+        status=resp.status_code,
+        headers=headers,
+        set_cookie=set_cookie,
+        body_snippet=body_snippet,
+        body_length=len(resp.content),
+    )
+
+
+def _safe_text_snippet(resp: httpx.Response, path: str) -> str:
+    """Return a short textual snippet suited to the path.
+
+    Binary paths (favicon.ico) get a hex digest of the leading bytes — useful
+    for favicon-hash style fingerprinting later — rather than mojibake text.
+    """
+    ctype = (resp.headers.get("content-type") or "").lower()
+    if path.endswith(".ico") or ("image" in ctype) or ("octet-stream" in ctype):
+        head = resp.content[:_FAVICON_HASH_BYTES]
+        return f"<binary {len(resp.content)}B, leading-sha={_short_hex(head)}>"
+    text = resp.text
+    if len(text) > _BODY_SNIPPET_CHARS:
+        return text[:_BODY_SNIPPET_CHARS] + "…[truncated]"
+    return text
+
+
+def _short_hex(data: bytes) -> str:
+    import hashlib
+
+    return hashlib.sha256(data).hexdigest()[:16]
+
+
+# ---------------------------------------------------------------------------
+# Crystallization: raw signals -> FingerprintVector via a fast/cheap LLM
+# ---------------------------------------------------------------------------
+
+
+_CRYSTALLIZATION_SYSTEM_RULES = """\
+You are a web-stack fingerprinting assistant. You will receive raw HTTP recon
+signals (headers, Set-Cookie values, and short body snippets) from a small
+set of well-known paths on a single target.
+
+Your only job is to output a STRICT JSON object — no prose, no markdown
+fences, no commentary — that matches this schema exactly:
+
+{
+  "framework":          "<server-side framework or 'unknown'>",
+  "version_guess":      "<version string or 'unknown'>",
+  "frontend":           "<frontend stack or 'unknown'>",
+  "auth_pattern":       "<auth hint or 'none-observed'>",
+  "likely_orm":         "<ORM hint or 'unknown'>",
+  "overall_confidence": <float in [0.0, 1.0]>
+}
+
+Rules:
+  - Every string field MUST be present. Use 'unknown' when there is no signal.
+  - overall_confidence reflects credence in the WHOLE fingerprint, not any
+    single field. If most fields are 'unknown', confidence must be low.
+  - Do not invent versions. If no version string is visible, return 'unknown'.
+  - Do not output anything outside the JSON object.
+"""
+
+
+def build_crystallization_prompt(raw_signals: dict[str, Any]) -> str:
+    """Construct the user-message prompt for the crystallization LLM call.
+
+    The prompt embeds ``raw_signals`` as compact JSON and re-states the
+    schema. Kept pure (no I/O) so it can be unit-tested deterministically and
+    snapshot-compared.
+    """
+    serialized = json.dumps(raw_signals, separators=(",", ":"), default=str)
+    return (
+        f"{_CRYSTALLIZATION_SYSTEM_RULES}\n"
+        f"--- BEGIN RAW SIGNALS (JSON) ---\n"
+        f"{serialized}\n"
+        f"--- END RAW SIGNALS ---\n"
+        f"Return the JSON object now."
+    )
+
+
+_JSON_OBJECT_RE = re.compile(r"\{.*\}", re.DOTALL)
+
+
+def parse_fingerprint_response(response_text: str) -> FingerprintVector:
+    """Parse the LLM's JSON output into a validated ``FingerprintVector``.
+
+    Tolerant of common LLM output quirks (leading/trailing prose, ```json
+    fences) by extracting the first balanced ``{...}`` block before parsing.
+    Raises :class:`ValueError` with the original text included if parsing or
+    schema validation fails — the caller can decide whether to retry or fall
+    back to a default low-confidence vector.
+    """
+    if not response_text or not response_text.strip():
+        raise ValueError("Empty response from crystallization LLM")
+
+    candidate = response_text.strip()
+    if not candidate.startswith("{"):
+        match = _JSON_OBJECT_RE.search(candidate)
+        if match is None:
+            raise ValueError(f"No JSON object found in LLM response: {response_text!r}")
+        candidate = match.group(0)
+
+    try:
+        payload = json.loads(candidate)
+    except json.JSONDecodeError as e:
+        raise ValueError(
+            f"Crystallization LLM returned invalid JSON ({e}): {response_text!r}"
+        ) from e
+
+    try:
+        return FingerprintVector.model_validate(payload)
+    except ValidationError as e:
+        raise ValueError(
+            f"Crystallization LLM output failed schema validation: {e}\nRaw payload: {payload!r}"
+        ) from e
+
+
+async def crystallize_fingerprint(
+    raw_signals: dict[str, Any],
+    llm_call: LLMCallable,
+) -> FingerprintVector:
+    """Turn raw signals into a ``FingerprintVector`` via one LLM call.
+
+    ``llm_call`` is an injected async callable ``(prompt: str) -> str`` so the
+    concrete backend (Haiku via Anthropic, an OpenRouter-routed cheap model,
+    or a unit-test stub) is selected at the integration site rather than
+    hardwired here. This mirrors the project's existing ``AgentBackend``
+    injection pattern.
+    """
+    prompt = build_crystallization_prompt(raw_signals)
+    response_text = await llm_call(prompt)
+    return parse_fingerprint_response(response_text)
+
+
+# ---------------------------------------------------------------------------
+# Stage 2: Code Mirroring — FingerprintVector -> MirroredApplicationManifest
+#
+# Given a crystallized fingerprint, a reasoning LLM is asked to hallucinate
+# the most probable internal architecture: which routes likely exist, which
+# default framework misconfigurations are likely exposed, and which
+# vulnerability classes each predicted endpoint might harbor. The output is
+# a budget the agent can spend on targeted probes, instead of brute-forcing
+# wordlists.
+# ---------------------------------------------------------------------------
+
+
+class PredictedEndpoint(BaseModel):
+    """One hypothesized route on the target.
+
+    ``probability`` is the LLM's credence that this exact path+method exists
+    and returns ``expected_status``; downstream stages use it to rank probe
+    order. ``rationale`` must reference the fingerprint signal that justified
+    the guess so a human reviewer can sanity-check (or veto) the prediction.
+    """
+
+    path: str = Field(description="Absolute URL path beginning with '/' (e.g. '/api/v1/users').")
+    method: str = Field(
+        description="HTTP method in upper case (GET, POST, PUT, PATCH, DELETE, etc.)."
+    )
+    expected_status: int = Field(
+        ge=100,
+        le=599,
+        description="Most likely HTTP status returned by an unauthenticated probe.",
+    )
+    probability: float = Field(
+        ge=0.0,
+        le=1.0,
+        description="Credence that this endpoint exists as predicted, in [0.0, 1.0].",
+    )
+    rationale: str = Field(
+        description=(
+            "Short justification tied to a fingerprint signal or framework convention "
+            "(e.g. 'Laravel default debug route', 'Django admin convention')."
+        )
+    )
+    potential_vulnerability_class: str | None = Field(
+        default=None,
+        description=(
+            "Suspected vulnerability class if probed naively (e.g. 'IDOR', 'SSRF', "
+            "'info-disclosure', 'auth-bypass'). None if no specific class is implied."
+        ),
+    )
+
+    @field_validator("path")
+    @classmethod
+    def _leading_slash(cls, v: str) -> str:
+        v = v.strip()
+        if not v:
+            raise ValueError("path must be non-empty")
+        return v if v.startswith("/") else "/" + v
+
+    @field_validator("method")
+    @classmethod
+    def _upper_method(cls, v: str) -> str:
+        v = v.strip().upper()
+        if not v:
+            raise ValueError("method must be non-empty")
+        return v
+
+    @field_validator("rationale")
+    @classmethod
+    def _non_empty_rationale(cls, v: str) -> str:
+        if not v or not v.strip():
+            raise ValueError("rationale must be non-empty (every prediction needs justification)")
+        return v.strip()
+
+    @field_validator("potential_vulnerability_class")
+    @classmethod
+    def _normalize_vuln_class(cls, v: str | None) -> str | None:
+        # Models routinely emit "none"/"n/a"/"" instead of a true JSON null;
+        # collapse those to None so downstream filters work uniformly.
+        if v is None:
+            return None
+        stripped = v.strip()
+        if not stripped or stripped.lower() in {"none", "n/a", "na", "null", "unknown"}:
+            return None
+        return stripped
+
+
+class MirroredApplicationManifest(BaseModel):
+    """Hallucinated mirror of the target application's internal surface."""
+
+    predicted_endpoints: list[PredictedEndpoint] = Field(
+        default_factory=list,
+        description="Endpoints the LLM expects to exist, ranked by `probability`.",
+    )
+    global_assumptions: list[str] = Field(
+        default_factory=list,
+        description=(
+            "Framework- or deployment-wide assumptions underlying the predictions "
+            "(e.g. 'Assumes default Laravel Sanctum configuration', "
+            "'Assumes APP_DEBUG=true based on stack trace')."
+        ),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Stage 2 (Compact): same MAM contract, ~half the output tokens.
+#
+# The original schema forces a non-empty rationale per endpoint. On a complex
+# target the model emits 60-120 endpoints; rationale + verbose field names
+# cost ~2-3k OUTPUT tokens per MAM call, and output tokens dominate Stage-2
+# latency. The compact wire format below shortens every field name and drops
+# the mandatory rationale. Expansion back into the full PredictedEndpoint
+# shape happens server-side in :func:`expand_compact`, so downstream stages
+# (rank_probes, cheap_sweep, Stage 4) see the same MirroredApplicationManifest
+# type they always did — only the LLM-facing wire format changes.
+# ---------------------------------------------------------------------------
+
+
+class PredictedEndpointCompact(BaseModel):
+    """Compact wire format the LLM emits in :func:`generate_mam`.
+
+    Field names are single letters because output tokens are the dominant
+    cost of Stage 2; ``"p"`` over ``"path"`` saves ~3 tokens per endpoint
+    times ~60 endpoints/target = ~180 tokens per call, every call.
+    """
+
+    p: str = Field(description="Absolute path starting with '/'")
+    m: str = Field(default="GET", description="HTTP method, upper case")
+    s: int = Field(default=200, ge=100, le=599, description="Expected HTTP status code")
+    c: float = Field(ge=0.0, le=1.0, description="Confidence/probability in [0.0, 1.0]")
+    v: str | None = Field(
+        default=None,
+        description="Vulnerability class label (kebab/snake) or null",
+    )
+
+    @field_validator("p")
+    @classmethod
+    def _leading_slash(cls, v: str) -> str:
+        v = v.strip()
+        if not v:
+            raise ValueError("p must be non-empty")
+        return v if v.startswith("/") else "/" + v
+
+    @field_validator("m")
+    @classmethod
+    def _upper_method(cls, v: str) -> str:
+        return (v or "").strip().upper() or "GET"
+
+    @field_validator("v")
+    @classmethod
+    def _normalize_vuln(cls, v: str | None) -> str | None:
+        # Models routinely emit "none"/"unknown"/"" instead of true JSON null
+        # even when the schema declares the field nullable; collapse those to
+        # None so downstream severity scoring works uniformly.
+        if v is None:
+            return None
+        stripped = v.strip()
+        if not stripped or stripped.lower() in {"none", "n/a", "na", "null", "unknown"}:
+            return None
+        return stripped
+
+
+class CompactMirroredManifest(BaseModel):
+    """Compact analogue of :class:`MirroredApplicationManifest`."""
+
+    endpoints: list[PredictedEndpointCompact] = Field(default_factory=list)
+    assumptions: list[str] = Field(default_factory=list)
+
+
+# JSON Schema passed to provider strict-output modes. Hand-written (not
+# Pydantic-derived) so it satisfies OpenAI's strict-mode constraints exactly:
+# every object has ``additionalProperties: false`` and lists every property
+# in ``required``. Models can still emit JSON null for ``v`` thanks to the
+# ``["string","null"]`` union — that's OpenAI's official nullable shape in
+# strict mode.
+COMPACT_MAM_JSON_SCHEMA: dict[str, Any] = {
+    "name": "MirroredApplicationManifest",
+    "strict": True,
+    "schema": {
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["endpoints", "assumptions"],
+        "properties": {
+            "endpoints": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "additionalProperties": False,
+                    "required": ["p", "m", "s", "c", "v"],
+                    "properties": {
+                        "p": {
+                            "type": "string",
+                            "description": "Absolute path starting with /",
+                        },
+                        "m": {
+                            "type": "string",
+                            "enum": [
+                                "GET",
+                                "POST",
+                                "PUT",
+                                "PATCH",
+                                "DELETE",
+                                "HEAD",
+                                "OPTIONS",
+                            ],
+                        },
+                        "s": {"type": "integer", "minimum": 100, "maximum": 599},
+                        "c": {"type": "number", "minimum": 0.0, "maximum": 1.0},
+                        "v": {
+                            "type": ["string", "null"],
+                            "description": (
+                                "Vulnerability class label "
+                                "(e.g. 'info-disclosure', 'IDOR', 'SSRF', "
+                                "'auth-bypass', 'rce') or null."
+                            ),
+                        },
+                    },
+                },
+            },
+            "assumptions": {
+                "type": "array",
+                "items": {"type": "string"},
+            },
+        },
+    },
+}
+
+
+def expand_compact(
+    compact: CompactMirroredManifest,
+    *,
+    source: str = "llm-compact",
+) -> MirroredApplicationManifest:
+    """Expand compact wire format into a full :class:`MirroredApplicationManifest`.
+
+    The original schema requires a non-empty ``rationale`` per endpoint — we
+    trade that field for tokens in compact mode, so synthesize a short
+    ``source`` placeholder ("llm-compact" / "deterministic") that downstream
+    debuggers can grep on without forcing the model to re-generate
+    justifications it didn't need to produce.
+    """
+    endpoints = [
+        PredictedEndpoint(
+            path=ep.p,
+            method=ep.m,
+            expected_status=ep.s,
+            probability=ep.c,
+            rationale=source,
+            potential_vulnerability_class=ep.v,
+        )
+        for ep in compact.endpoints
+    ]
+    return MirroredApplicationManifest(
+        predicted_endpoints=endpoints,
+        global_assumptions=compact.assumptions,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Deterministic rulebook
+#
+# Framework conventions are constants — there's no value in burning LLM tokens
+# to re-derive them every call, and a rulebook gives us EXHAUSTIVE coverage
+# of the boring-but-high-value endpoints (the ones CTF designers love to use
+# precisely because they look low-signal to a generation model).
+#
+# ALWAYS_PROBE is the framework-agnostic floor (dotfiles, common admin/API
+# routes, backup leaks). FRAMEWORK_RULES is keyed by a case-insensitive
+# substring of FingerprintVector.framework so a fingerprint of "Spring Boot 2.7"
+# still matches the "spring" pack.
+#
+# Rationale strings start with "static:" so a debugger reading a merged MAM
+# can immediately tell which endpoints came from this table vs the LLM.
+# Probabilities reflect "how often is this *actually* reachable in the wild",
+# NOT severity — severity weighting happens in rank_probes via
+# potential_vulnerability_class.
+# ---------------------------------------------------------------------------
+
+
+ALWAYS_PROBE: tuple[PredictedEndpoint, ...] = (
+    PredictedEndpoint(
+        path="/.git/config",
+        method="GET",
+        expected_status=200,
+        probability=0.15,
+        rationale="static: VCS leak",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/.git/HEAD",
+        method="GET",
+        expected_status=200,
+        probability=0.15,
+        rationale="static: VCS leak",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/.env",
+        method="GET",
+        expected_status=200,
+        probability=0.10,
+        rationale="static: env leak",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/.DS_Store",
+        method="GET",
+        expected_status=200,
+        probability=0.05,
+        rationale="static: macOS metadata",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/robots.txt",
+        method="GET",
+        expected_status=200,
+        probability=0.70,
+        rationale="static: convention",
+        potential_vulnerability_class=None,
+    ),
+    PredictedEndpoint(
+        path="/sitemap.xml",
+        method="GET",
+        expected_status=200,
+        probability=0.40,
+        rationale="static: convention",
+        potential_vulnerability_class=None,
+    ),
+    PredictedEndpoint(
+        path="/.well-known/security.txt",
+        method="GET",
+        expected_status=200,
+        probability=0.10,
+        rationale="static: convention",
+        potential_vulnerability_class=None,
+    ),
+    PredictedEndpoint(
+        path="/server-status",
+        method="GET",
+        expected_status=200,
+        probability=0.05,
+        rationale="static: Apache mod_status",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/server-info",
+        method="GET",
+        expected_status=200,
+        probability=0.05,
+        rationale="static: Apache mod_info",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/phpinfo.php",
+        method="GET",
+        expected_status=200,
+        probability=0.05,
+        rationale="static: PHP info leak",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/swagger.json",
+        method="GET",
+        expected_status=200,
+        probability=0.15,
+        rationale="static: OpenAPI doc",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/swagger-ui.html",
+        method="GET",
+        expected_status=200,
+        probability=0.10,
+        rationale="static: Swagger UI",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/openapi.json",
+        method="GET",
+        expected_status=200,
+        probability=0.10,
+        rationale="static: OpenAPI doc",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/graphql",
+        method="GET",
+        expected_status=200,
+        probability=0.10,
+        rationale="static: GraphQL endpoint",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/admin",
+        method="GET",
+        expected_status=200,
+        probability=0.30,
+        rationale="static: admin convention",
+        potential_vulnerability_class="auth-bypass",
+    ),
+    PredictedEndpoint(
+        path="/login",
+        method="GET",
+        expected_status=200,
+        probability=0.40,
+        rationale="static: auth convention",
+        potential_vulnerability_class=None,
+    ),
+    PredictedEndpoint(
+        path="/api",
+        method="GET",
+        expected_status=200,
+        probability=0.40,
+        rationale="static: API convention",
+        potential_vulnerability_class=None,
+    ),
+    PredictedEndpoint(
+        path="/api/v1",
+        method="GET",
+        expected_status=200,
+        probability=0.30,
+        rationale="static: API convention",
+        potential_vulnerability_class=None,
+    ),
+    PredictedEndpoint(
+        path="/backup",
+        method="GET",
+        expected_status=200,
+        probability=0.05,
+        rationale="static: backup leak",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/backup.zip",
+        method="GET",
+        expected_status=200,
+        probability=0.05,
+        rationale="static: backup leak",
+        potential_vulnerability_class="info-disclosure",
+    ),
+    PredictedEndpoint(
+        path="/backup.tar.gz",
+        method="GET",
+        expected_status=200,
+        probability=0.03,
+        rationale="static: backup leak",
+        potential_vulnerability_class="info-disclosure",
+    ),
+)
+
+
+FRAMEWORK_RULES: dict[str, tuple[PredictedEndpoint, ...]] = {
+    "django": (
+        PredictedEndpoint(
+            path="/admin/",
+            method="GET",
+            expected_status=200,
+            probability=0.60,
+            rationale="static: Django admin",
+            potential_vulnerability_class="auth-bypass",
+        ),
+        PredictedEndpoint(
+            path="/accounts/login/",
+            method="GET",
+            expected_status=200,
+            probability=0.40,
+            rationale="static: Django auth",
+            potential_vulnerability_class=None,
+        ),
+        PredictedEndpoint(
+            path="/__debug__/",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: django-debug-toolbar leak",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/static/admin/css/base.css",
+            method="GET",
+            expected_status=200,
+            probability=0.50,
+            rationale="static: Django admin CSS",
+            potential_vulnerability_class=None,
+        ),
+    ),
+    "rails": (
+        PredictedEndpoint(
+            path="/rails/info/routes",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Rails dev routes",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/rails/info/properties",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Rails dev props",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/rails/conductor",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: ActionMailbox conductor",
+            potential_vulnerability_class="info-disclosure",
+        ),
+    ),
+    "laravel": (
+        PredictedEndpoint(
+            path="/telescope",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Laravel Telescope",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/horizon",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Laravel Horizon",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/_ignition/execute-solution",
+            method="POST",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Ignition (CVE-2021-3129)",
+            potential_vulnerability_class="rce",
+        ),
+        PredictedEndpoint(
+            path="/storage/logs/laravel.log",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: Laravel log leak",
+            potential_vulnerability_class="info-disclosure",
+        ),
+    ),
+    "spring": (
+        PredictedEndpoint(
+            path="/actuator",
+            method="GET",
+            expected_status=200,
+            probability=0.40,
+            rationale="static: Spring Boot Actuator",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/actuator/env",
+            method="GET",
+            expected_status=200,
+            probability=0.15,
+            rationale="static: Actuator env",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/actuator/heapdump",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Actuator heapdump",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/actuator/health",
+            method="GET",
+            expected_status=200,
+            probability=0.50,
+            rationale="static: Actuator health",
+            potential_vulnerability_class=None,
+        ),
+        PredictedEndpoint(
+            path="/actuator/info",
+            method="GET",
+            expected_status=200,
+            probability=0.40,
+            rationale="static: Actuator info",
+            potential_vulnerability_class=None,
+        ),
+        PredictedEndpoint(
+            path="/actuator/mappings",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Actuator route map",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/actuator/beans",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Actuator beans",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/actuator/configprops",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Actuator config",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/actuator/threaddump",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Actuator threaddump",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/h2-console",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: H2 console (RCE risk)",
+            potential_vulnerability_class="rce",
+        ),
+        PredictedEndpoint(
+            path="/jolokia",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: Jolokia JMX-over-HTTP",
+            potential_vulnerability_class="info-disclosure",
+        ),
+    ),
+    "express": (
+        PredictedEndpoint(
+            path="/static/",
+            method="GET",
+            expected_status=200,
+            probability=0.20,
+            rationale="static: Express static",
+            potential_vulnerability_class=None,
+        ),
+    ),
+    "flask": (
+        PredictedEndpoint(
+            path="/static/",
+            method="GET",
+            expected_status=200,
+            probability=0.40,
+            rationale="static: Flask static",
+            potential_vulnerability_class=None,
+        ),
+        PredictedEndpoint(
+            path="/console",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: Werkzeug debug console (RCE)",
+            potential_vulnerability_class="rce",
+        ),
+    ),
+    "asp.net": (
+        PredictedEndpoint(
+            path="/web.config",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: web.config leak",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/trace.axd",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: ASP.NET trace",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/elmah.axd",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: ELMAH error log",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/default.aspx",
+            method="GET",
+            expected_status=200,
+            probability=0.20,
+            rationale="static: ASP.NET default page",
+            potential_vulnerability_class=None,
+        ),
+    ),
+    "wordpress": (
+        PredictedEndpoint(
+            path="/wp-login.php",
+            method="GET",
+            expected_status=200,
+            probability=0.80,
+            rationale="static: WP login",
+            potential_vulnerability_class="auth-bypass",
+        ),
+        PredictedEndpoint(
+            path="/wp-admin/",
+            method="GET",
+            expected_status=200,
+            probability=0.70,
+            rationale="static: WP admin",
+            potential_vulnerability_class="auth-bypass",
+        ),
+        PredictedEndpoint(
+            path="/wp-content/",
+            method="GET",
+            expected_status=200,
+            probability=0.50,
+            rationale="static: WP content",
+            potential_vulnerability_class=None,
+        ),
+        PredictedEndpoint(
+            path="/wp-json/wp/v2/users",
+            method="GET",
+            expected_status=200,
+            probability=0.30,
+            rationale="static: WP REST user enum",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/xmlrpc.php",
+            method="GET",
+            expected_status=200,
+            probability=0.40,
+            rationale="static: WP XMLRPC",
+            potential_vulnerability_class="auth-bypass",
+        ),
+        PredictedEndpoint(
+            path="/wp-config.php.bak",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: WP config backup",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/readme.html",
+            method="GET",
+            expected_status=200,
+            probability=0.30,
+            rationale="static: WP version disclosure",
+            potential_vulnerability_class="info-disclosure",
+        ),
+    ),
+    "drupal": (
+        PredictedEndpoint(
+            path="/user/login",
+            method="GET",
+            expected_status=200,
+            probability=0.50,
+            rationale="static: Drupal login",
+            potential_vulnerability_class=None,
+        ),
+        PredictedEndpoint(
+            path="/CHANGELOG.txt",
+            method="GET",
+            expected_status=200,
+            probability=0.15,
+            rationale="static: Drupal changelog (version leak)",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/core/CHANGELOG.txt",
+            method="GET",
+            expected_status=200,
+            probability=0.15,
+            rationale="static: Drupal 8+ changelog",
+            potential_vulnerability_class="info-disclosure",
+        ),
+        PredictedEndpoint(
+            path="/update.php",
+            method="GET",
+            expected_status=200,
+            probability=0.10,
+            rationale="static: Drupal update",
+            potential_vulnerability_class=None,
+        ),
+        PredictedEndpoint(
+            path="/sites/default/files/",
+            method="GET",
+            expected_status=200,
+            probability=0.20,
+            rationale="static: Drupal files dir",
+            potential_vulnerability_class=None,
+        ),
+    ),
+    "joomla": (
+        PredictedEndpoint(
+            path="/administrator/",
+            method="GET",
+            expected_status=200,
+            probability=0.50,
+            rationale="static: Joomla admin",
+            potential_vulnerability_class="auth-bypass",
+        ),
+        PredictedEndpoint(
+            path="/administrator/index.php",
+            method="GET",
+            expected_status=200,
+            probability=0.40,
+            rationale="static: Joomla admin login",
+            potential_vulnerability_class="auth-bypass",
+        ),
+        PredictedEndpoint(
+            path="/configuration.php-dist",
+            method="GET",
+            expected_status=200,
+            probability=0.05,
+            rationale="static: Joomla config template",
+            potential_vulnerability_class="info-disclosure",
+        ),
+    ),
+}
+
+
+def deterministic_endpoints(fingerprint: FingerprintVector) -> list[PredictedEndpoint]:
+    """Compute the rulebook-only endpoint list for a given fingerprint.
+
+    Returns ``ALWAYS_PROBE`` first, then every ``FRAMEWORK_RULES`` entry whose
+    key appears as a case-insensitive substring of ``fingerprint.framework``.
+    A single fingerprint can match multiple rule packs in principle, though
+    the keys are designed not to overlap by accident.
+
+    Pure: no I/O, no side effects.
+    """
+    out: list[PredictedEndpoint] = list(ALWAYS_PROBE)
+    fw_norm = (fingerprint.framework or "").lower().strip()
+    if fw_norm and fw_norm != "unknown":
+        for key, rules in FRAMEWORK_RULES.items():
+            if key in fw_norm:
+                out.extend(rules)
+    return out
+
+
+def merge_endpoints(
+    deterministic: list[PredictedEndpoint],
+    llm_manifest: MirroredApplicationManifest,
+) -> MirroredApplicationManifest:
+    """Merge deterministic + LLM predictions, deduped by (method, path).
+
+    The LLM's prediction wins on conflict. Reasoning: the compact prompt
+    explicitly tells the LLM NOT to re-predict deterministic paths, so if it
+    does anyway, that's *target-specific evidence* (e.g. it spotted
+    ``/.git/HEAD`` linked from robots.txt and wants to assign higher
+    probability than our 0.15 default). Throwing that signal away would
+    defeat the point of running the LLM at all on those paths.
+
+    Output is sorted by probability descending for human debug-readability;
+    :func:`rank_probes` re-sorts by score downstream, so ordering here is
+    cosmetic.
+    """
+    by_key: dict[tuple[str, str], PredictedEndpoint] = {
+        (ep.method, ep.path): ep for ep in deterministic
+    }
+    for ep in llm_manifest.predicted_endpoints:
+        by_key[(ep.method, ep.path)] = ep  # LLM wins on conflict
+    merged = sorted(by_key.values(), key=lambda e: e.probability, reverse=True)
+    return MirroredApplicationManifest(
+        predicted_endpoints=merged,
+        global_assumptions=llm_manifest.global_assumptions,
+    )
+
+
+_MIRRORING_SYSTEM_RULES_COMPACT = """\
+You are a web-stack code-mirroring assistant. You receive a crystallized
+fingerprint of one target web application.
+
+A DETERMINISTIC RULESET runs ALONGSIDE you and ALREADY probes the following
+classes of paths. Do NOT re-predict them — they will be deduped out, wasting
+your output budget:
+
+  - VCS/secret leaks:  /.git/*, /.env, /.DS_Store
+  - Conventional docs: /robots.txt, /sitemap.xml, /.well-known/security.txt
+  - Server defaults:   /server-status, /server-info, /phpinfo.php
+  - API docs:          /swagger.json, /swagger-ui.html, /openapi.json, /graphql
+  - Common admin/auth: /admin, /login, /api, /api/v1
+  - Backup leaks:      /backup, /backup.zip, /backup.tar.gz
+  - The framework-specific default-routes pack for the fingerprinted stack
+    (Django admin, Rails dev routes, Laravel telescope/horizon/ignition,
+    Spring Boot actuator family, ASP.NET trace/elmah, WordPress/Drupal/Joomla
+    cores, Flask/Werkzeug debug, etc.)
+
+YOUR job: predict NOVEL, target-specific endpoints the deterministic ruleset
+would miss — application-specific REST resources implied by the fingerprint,
+custom admin paths, debug routes inferred from version-specific defaults that
+aren't already in the rulebook, etc.
+
+Output ONLY a STRICT JSON object matching this exact schema:
+
+{
+  "endpoints": [
+    {"p": "/path", "m": "GET", "s": 200, "c": 0.0, "v": null}
+  ],
+  "assumptions": ["..."]
+}
+
+Field meanings (single-letter keys are intentional — output tokens are the
+bottleneck):
+  p = absolute path starting with '/'
+  m = HTTP method, upper case (GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS)
+  s = expected HTTP status, integer 100..599
+  c = your confidence/probability the path exists, float in [0.0, 1.0]
+      Calibrate strictly: >=0.8 = near-certain, <=0.3 = speculative
+  v = vulnerability class label (short kebab/snake, e.g. 'info-disclosure',
+      'IDOR', 'SSRF', 'auth-bypass', 'rce', 'ssti') or null if none implied
+
+Rules:
+  - Do NOT repeat any path covered by the deterministic ruleset above.
+  - Rank `endpoints` by `c`, descending.
+  - `assumptions` is for framework- or deployment-wide hypotheses you relied
+    on (e.g. 'Assumes APP_DEBUG=true based on stack trace').
+  - Output ONLY the JSON object — no prose, no markdown fences, no preamble.
+"""
+
+
+def build_mirroring_prompt_compact(
+    fingerprint: FingerprintVector,
+    *,
+    correction_context: str | None = None,
+) -> str:
+    """Compact-format counterpart to :func:`build_mirroring_prompt`.
+
+    Pure (no I/O) and snapshot-testable. Same correction-context contract as
+    the legacy builder — when set, the block is appended verbatim so the LLM
+    sees which previous predictions were disproven by real probes.
+    """
+    serialized = json.dumps(fingerprint.model_dump(), separators=(",", ":"), default=str)
+    correction_block = ""
+    if correction_context and correction_context.strip():
+        correction_block = (
+            f"--- BEGIN CORRECTION CONTEXT ---\n"
+            f"{correction_context.strip()}\n"
+            f"--- END CORRECTION CONTEXT ---\n"
+        )
+    return (
+        f"{_MIRRORING_SYSTEM_RULES_COMPACT}\n"
+        f"--- BEGIN FINGERPRINT (JSON) ---\n"
+        f"{serialized}\n"
+        f"--- END FINGERPRINT ---\n"
+        f"{correction_block}"
+        f"Return the JSON object now."
+    )
+
+
+def parse_compact_mirroring_response(response_text: str) -> CompactMirroredManifest:
+    """Parse the LLM's compact-schema JSON output.
+
+    Tolerant of markdown fences and leading/trailing prose so the function
+    works even when ``json_schema`` strict mode is unavailable (e.g. a local
+    vLLM build without guided JSON, or a provider that returns the schema
+    correctly but still wraps it in a ```json fence). Raises
+    :class:`ValueError` with the original text on parse/validation failure so
+    the caller can retry or fall back to an empty manifest.
+    """
+    if not response_text or not response_text.strip():
+        raise ValueError("Empty response from compact-mirroring LLM")
+
+    candidate = _strip_markdown_fence(response_text)
+    if not candidate.startswith("{"):
+        match = _JSON_OBJECT_RE.search(candidate)
+        if match is None:
+            raise ValueError(
+                f"No JSON object found in compact-mirroring LLM response: {response_text!r}"
+            )
+        candidate = match.group(0)
+
+    try:
+        payload = json.loads(candidate)
+    except json.JSONDecodeError as e:
+        raise ValueError(
+            f"Compact-mirroring LLM returned invalid JSON ({e}): {response_text!r}"
+        ) from e
+
+    try:
+        return CompactMirroredManifest.model_validate(payload)
+    except ValidationError as e:
+        raise ValueError(
+            f"Compact-mirroring LLM output failed schema validation: {e}\nRaw payload: {payload!r}"
+        ) from e
+
+
+_MIRRORING_SYSTEM_RULES = """\
+You are an elite security architect performing a code-mirroring exercise. You
+are given a crystallized fingerprint of a single target web application
+(framework, version guess, frontend, auth pattern, ORM, overall confidence).
+
+Your job is to hallucinate the application's most probable internal surface:
+the routes that are likely to exist, the default misconfigurations the stack
+is known to expose, and the vulnerability classes each predicted endpoint
+might harbor under default settings. Treat this as a hypothesis-generation
+step, not a scan — your output drives WHERE the agent probes next.
+
+Concretely, draw on:
+  - Framework conventions and scaffolded routes
+      (e.g. Django: /admin/, /accounts/login/, /__debug__/;
+       Rails: /rails/info/routes, /rails/conductor;
+       Laravel: /telescope, /horizon, /_ignition/execute-solution;
+       Spring Boot: /actuator, /actuator/env, /actuator/heapdump;
+       Express: /api, default error pages exposing stack traces).
+  - Default exposed files and dotfiles
+      (e.g. /.git/config, /.env, /.DS_Store, /server-status, /server-info,
+       /storage/logs/laravel.log, /web.config, /WEB-INF/web.xml).
+  - Common API structures inferred from the auth pattern
+      (e.g. JWT often implies /api/v1/auth/login, /api/v1/auth/refresh;
+       session-cookie often implies /login, /logout, /csrf, /me).
+  - ORM-implied resource routes
+      (e.g. ActiveRecord -> RESTful /users, /posts; Prisma -> often /api/<model>).
+
+Output a STRICT JSON object — no prose, no markdown fences, no preamble, no
+trailing commentary — that matches this schema exactly:
+
+{
+  "predicted_endpoints": [
+    {
+      "path":                          "<absolute path starting with '/'>",
+      "method":                        "<HTTP method in upper case>",
+      "expected_status":               <int 100-599>,
+      "probability":                   <float in [0.0, 1.0]>,
+      "rationale":                     "<one-line justification tied to a fingerprint signal or framework convention>",
+      "potential_vulnerability_class": "<short class name or null>"
+    }
+  ],
+  "global_assumptions": [
+    "<assumption 1>",
+    "<assumption 2>"
+  ]
+}
+
+Rules:
+  - Every predicted_endpoints entry MUST include every field above.
+  - potential_vulnerability_class is null when no specific class is implied;
+    otherwise a short kebab/snake label (e.g. 'info-disclosure', 'IDOR',
+    'SSRF', 'auth-bypass', 'rce').
+  - probability must be calibrated: reserve >=0.8 for routes that are
+    near-guaranteed under the fingerprinted stack's defaults; use <=0.3 for
+    speculative guesses.
+  - Rank predicted_endpoints by probability, descending.
+  - Do not output anything outside the JSON object.
+"""
+
+
+def build_mirroring_prompt(
+    fingerprint: FingerprintVector,
+    *,
+    correction_context: str | None = None,
+) -> str:
+    """Construct the user-message prompt for the code-mirroring LLM call.
+
+    Pure (no I/O). The fingerprint is serialized as compact JSON so the
+    prompt is deterministic and snapshot-testable. When ``correction_context``
+    is supplied (Stage 4 belief update), it is appended as its own block so
+    the LLM can see which previous predictions were disproven by real probes.
+    """
+    serialized = json.dumps(fingerprint.model_dump(), separators=(",", ":"), default=str)
+    correction_block = ""
+    if correction_context and correction_context.strip():
+        correction_block = (
+            f"--- BEGIN CORRECTION CONTEXT ---\n"
+            f"{correction_context.strip()}\n"
+            f"--- END CORRECTION CONTEXT ---\n"
+        )
+    return (
+        f"{_MIRRORING_SYSTEM_RULES}\n"
+        f"--- BEGIN FINGERPRINT (JSON) ---\n"
+        f"{serialized}\n"
+        f"--- END FINGERPRINT ---\n"
+        f"{correction_block}"
+        f"Return the JSON object now."
+    )
+
+
+def _strip_markdown_fence(text: str) -> str:
+    """Remove a single surrounding ```json ... ``` (or ``` ... ```) fence.
+
+    Reasoning models drop into fenced output more often than fingerprinting
+    models, so Stage 2 strips fences explicitly before falling back to the
+    Stage 1 regex extraction.
+    """
+    s = text.strip()
+    if not s.startswith("```"):
+        return s
+    # Drop the opening fence line ("```" or "```json", etc.)
+    first_nl = s.find("\n")
+    if first_nl == -1:
+        return s
+    s = s[first_nl + 1 :]
+    if s.rstrip().endswith("```"):
+        s = s.rstrip()[: -len("```")]
+    return s.strip()
+
+
+def parse_mirroring_response(response_text: str) -> MirroredApplicationManifest:
+    """Parse the LLM's JSON output into a validated manifest.
+
+    Tolerant of markdown fences and leading/trailing prose. Raises
+    :class:`ValueError` with the original text included if parsing or
+    schema validation fails so the caller can retry or fall back to an
+    empty manifest.
+    """
+    if not response_text or not response_text.strip():
+        raise ValueError("Empty response from code-mirroring LLM")
+
+    candidate = _strip_markdown_fence(response_text)
+    if not candidate.startswith("{"):
+        match = _JSON_OBJECT_RE.search(candidate)
+        if match is None:
+            raise ValueError(f"No JSON object found in mirroring LLM response: {response_text!r}")
+        candidate = match.group(0)
+
+    try:
+        payload = json.loads(candidate)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Mirroring LLM returned invalid JSON ({e}): {response_text!r}") from e
+
+    try:
+        return MirroredApplicationManifest.model_validate(payload)
+    except ValidationError as e:
+        raise ValueError(
+            f"Mirroring LLM output failed schema validation: {e}\nRaw payload: {payload!r}"
+        ) from e
+
+
+async def _call_llm_with_optional_schema(
+    llm_call: StructuredLLMCallable,
+    prompt: str,
+    *,
+    json_schema: dict[str, Any] | None,
+) -> str:
+    """Invoke ``llm_call`` with ``json_schema`` when supported, else without.
+
+    Integrators that wire structured-output mode (OpenAI/Anthropic/vLLM)
+    define ``llm_call`` to accept the ``json_schema`` kwarg and pass it
+    through to ``response_format``. Older test stubs and the basic
+    :data:`LLMCallable` shape don't accept the kwarg; for those we catch the
+    ``TypeError`` and re-call without ``json_schema``. The behavioural
+    fallback means callers don't have to special-case "old SDK" vs "new SDK"
+    — they just get whatever level of strictness their stub supports.
+    """
+    if json_schema is None:
+        return await llm_call(prompt)
+    try:
+        return await llm_call(prompt, json_schema=json_schema)
+    except TypeError as exc:
+        # Distinguish "wrong kwarg" from "wrong type altogether" — the latter
+        # is a real bug we don't want to swallow.
+        msg = str(exc)
+        if "json_schema" in msg or "unexpected keyword" in msg:
+            return await llm_call(prompt)
+        raise
+
+
+async def generate_mam(
+    fingerprint: FingerprintVector,
+    llm_call: StructuredLLMCallable,
+    *,
+    correction_context: str | None = None,
+    use_compact_schema: bool = True,
+    include_deterministic: bool = True,
+) -> MirroredApplicationManifest:
+    """Turn a fingerprint into a ``MirroredApplicationManifest``.
+
+    Default fast path (``use_compact_schema=True``, ``include_deterministic=True``):
+      1. Build the compact prompt — the deterministic rulebook is announced
+         as already-handled so the LLM doesn't burn output tokens
+         re-predicting it.
+      2. Call ``llm_call`` with :data:`COMPACT_MAM_JSON_SCHEMA`; the
+         integrator is responsible for plumbing this into the provider's
+         strict-output mode (OpenAI ``response_format``, Anthropic tool-use
+         forcing, vLLM guided JSON). When ``llm_call`` doesn't accept the
+         ``json_schema`` kwarg, we transparently fall back to a plain
+         ``(prompt,) -> str`` call and the response is parsed via the
+         regex-fence-and-extract path in
+         :func:`parse_compact_mirroring_response`.
+      3. Expand compact -> full :class:`MirroredApplicationManifest`.
+      4. Merge with :func:`deterministic_endpoints` for the fingerprint;
+         the LLM wins on (method, path) conflicts. See
+         :func:`merge_endpoints` for the rationale.
+
+    Legacy path (``use_compact_schema=False``): pre-refactor behaviour —
+    calls :func:`build_mirroring_prompt` / :func:`parse_mirroring_response`
+    and (by default) still merges in the deterministic ruleset. Kept so
+    existing tests and integrators that haven't migrated still work.
+
+    Set ``include_deterministic=False`` to skip the rulebook entirely —
+    useful for snapshot tests that want to assert only on LLM output.
+    """
+    if use_compact_schema:
+        prompt = build_mirroring_prompt_compact(fingerprint, correction_context=correction_context)
+        response_text = await _call_llm_with_optional_schema(
+            llm_call, prompt, json_schema=COMPACT_MAM_JSON_SCHEMA
+        )
+        compact = parse_compact_mirroring_response(response_text)
+        llm_manifest = expand_compact(compact, source="llm-compact")
+    else:
+        prompt = build_mirroring_prompt(fingerprint, correction_context=correction_context)
+        response_text = await _call_llm_with_optional_schema(llm_call, prompt, json_schema=None)
+        llm_manifest = parse_mirroring_response(response_text)
+
+    if include_deterministic:
+        return merge_endpoints(deterministic_endpoints(fingerprint), llm_manifest)
+    return llm_manifest
+
+
+# ---------------------------------------------------------------------------
+# Stage 3: Posterior Construction & Probe Ranking
+#
+# Deterministic, LLM-free. Takes a MirroredApplicationManifest and ranks its
+# predicted endpoints by expected value:
+#
+#     score = (probability * severity_weight) / cost_weight
+#
+# This is the budgeting layer — the agent walks the ranked list top-down and
+# stops when it runs out of probe budget, instead of issuing requests in MAM
+# insertion order (which reflects LLM whim, not pentest priority).
+# ---------------------------------------------------------------------------
+
+
+# Severity buckets. Tuples of (substring-to-match-case-insensitively, weight).
+# Order matters: the FIRST matching bucket wins, so the highest-severity
+# patterns are checked first and a vuln_class like "sql-injection-to-rce"
+# correctly collapses to the RCE-tier weight.
+_SEVERITY_BUCKETS: tuple[tuple[tuple[str, ...], float], ...] = (
+    (("rce", "sqli", "command injection"), 10.0),
+    (("xss", "csrf", "traversal"), 7.0),
+    (("disclosure", "leak", "log"), 5.0),
+)
+_DEFAULT_SEVERITY_WEIGHT = 1.0
+
+# Method cost buckets. GET/HEAD are bare reads; the mutating verbs typically
+# need a body and may trigger side effects, so they cost more to probe.
+_CHEAP_METHODS: frozenset[str] = frozenset({"GET", "HEAD"})
+_MUTATING_METHODS: frozenset[str] = frozenset({"POST", "PUT", "PATCH", "DELETE"})
+_CHEAP_METHOD_COST = 1.0
+_MUTATING_METHOD_COST = 1.5
+
+
+def estimate_severity(vuln_class: str | None) -> float:
+    """Map a predicted vulnerability class to a severity weight.
+
+    Case-insensitive substring match against ordered buckets — see
+    ``_SEVERITY_BUCKETS``. Returns the default weight when ``vuln_class`` is
+    ``None`` or matches no bucket; this matches the Stage 2 contract where
+    "no specific class is implied" is encoded as ``None``.
+    """
+    if vuln_class is None:
+        return _DEFAULT_SEVERITY_WEIGHT
+    needle = vuln_class.lower()
+    for patterns, weight in _SEVERITY_BUCKETS:
+        if any(p in needle for p in patterns):
+            return weight
+    return _DEFAULT_SEVERITY_WEIGHT
+
+
+def estimate_cost(method: str) -> float:
+    """Map an HTTP method to a probe cost weight.
+
+    Unknown methods fall back to the cheap-read cost: most exotic verbs
+    (OPTIONS, TRACE, PROPFIND) are still bare requests, and we'd rather not
+    silently demote a high-severity prediction just because the LLM picked
+    an uncommon verb.
+    """
+    m = method.strip().upper()
+    if m in _MUTATING_METHODS:
+        return _MUTATING_METHOD_COST
+    if m in _CHEAP_METHODS:
+        return _CHEAP_METHOD_COST
+    return _CHEAP_METHOD_COST
+
+
+class RankedProbe(PredictedEndpoint):
+    """A ``PredictedEndpoint`` enriched with ranking weights and a final score.
+
+    Inherits every field (and validator) from :class:`PredictedEndpoint` so
+    the ranked list can be consumed anywhere a predicted endpoint is
+    expected, while also carrying the heuristic inputs that produced the
+    score — useful for debugging unexpected orderings.
+    """
+
+    severity_weight: float = Field(
+        ge=0.0,
+        description="Severity weight derived from `potential_vulnerability_class`.",
+    )
+    cost_weight: float = Field(
+        gt=0.0,
+        description="Probe cost weight derived from the HTTP `method`. Strictly > 0.",
+    )
+    score: float = Field(
+        ge=0.0,
+        description="Expected value: (probability * severity_weight) / cost_weight.",
+    )
+
+
+def rank_probes(mam: MirroredApplicationManifest) -> list[RankedProbe]:
+    """Score and rank every predicted endpoint in ``mam`` by expected value.
+
+    Pure: same input always yields the same output, no I/O, no LLM calls.
+    The returned list is sorted by ``score`` descending; ties preserve the
+    MAM's original ordering (Python's sort is stable), which means the LLM's
+    own confidence ranking acts as the tiebreaker.
+    """
+    ranked: list[RankedProbe] = []
+    for ep in mam.predicted_endpoints:
+        severity = estimate_severity(ep.potential_vulnerability_class)
+        cost = estimate_cost(ep.method)
+        score = (ep.probability * severity) / cost
+        ranked.append(
+            RankedProbe(
+                **ep.model_dump(),
+                severity_weight=severity,
+                cost_weight=cost,
+                score=score,
+            )
+        )
+    ranked.sort(key=lambda r: r.score, reverse=True)
+    return ranked
+
+
+# ---------------------------------------------------------------------------
+# Stage 4a: Two-tier probing — cheap exhaustive sweep, then targeted deep probe
+#
+# A single "top-N by score" pass is fast but lossy: it filters using the LLM's
+# self-reported confidence before any real evidence exists. CTF designers
+# routinely hide flags behind endpoints that look low-signal to a generation
+# model (a stray /robots.txt comment, a boring /backup directory). The fix is
+# to flip the order: probe everything cheaply first, then let the *cheap
+# evidence* — not just the LLM score — decide who gets a full deep probe.
+#
+# Tier 1 (cheap sweep):
+#   Concurrent GET against EVERY ranked probe, semaphore-capped to avoid
+#   hammering small CTF VMs, body streamed and truncated to ~4KB. Bounded by a
+#   total wall-clock budget; partial results survive a timeout instead of all
+#   being cancelled by `asyncio.wait_for`. Method is always GET regardless of
+#   probe.method — HEAD has too much per-server variance, and POST/PUT/DELETE
+#   could mutate state on a real target. A 405 from a GET on a POST endpoint
+#   is itself useful signal that the route exists.
+#
+# Selection (pure):
+#   A probe graduates to Tier 2 if ANY of:
+#     - Tier-1 status is in INTERESTING_STATUS_CODES (200/301/302/401/403/405/500/...).
+#     - Tier-1 body matched the flag-shaped regex — direct flag detection
+#       cuts straight to the answer.
+#     - Path matched a keyword pattern (admin/.git/.env/swagger/etc.). This
+#       is a BOOST, not an override — top-N by score already wins below.
+#   Plus the top-N by Stage-3 score regardless (the LLM's bet always gets a
+#   deep probe even if Tier 1 looked uninteresting). Cap at ``max_deep_probes``.
+#
+# Tier 2 (deep probe):
+#   Existing `_execute_probe` using probe.method, same semaphore. Results feed
+#   the unchanged surprise heuristic.
+#
+# Note: this optimizes the *HTTP* leg of PHANTOM. If profiling shows the
+# bottleneck is LLM generation (Stage 1 crystallization or Stage 2 mirroring),
+# the win is elsewhere — measure before tuning these constants.
+# ---------------------------------------------------------------------------
+
+
+_DEFAULT_TOP_N = 10
+_DEFAULT_SURPRISE_TOP_K = 3
+_PROBE_BODY_SNIPPET_CHARS = 200
+
+_DEFAULT_CHEAP_TIMEOUT_SECONDS = 4.0
+_DEFAULT_CHEAP_BUDGET_SECONDS = 15.0
+_DEFAULT_CONCURRENCY = 8
+_DEFAULT_MAX_DEEP_PROBES = 20
+_CHEAP_BODY_BYTES = 4096
+_CHEAP_BODY_SNIPPET_CHARS = 512
+
+# Status codes that historically carry signal on CTF / lab targets. 405 is in
+# here on purpose: a GET against a POST-only endpoint that returns 405 still
+# proves the route exists, which is exactly what we want Tier 2 to verify.
+INTERESTING_STATUS_CODES: frozenset[int] = frozenset(
+    {200, 201, 202, 204, 301, 302, 307, 308, 401, 403, 405, 500, 501, 503}
+)
+
+# High-value path substrings used as a Tier-2 promotion BOOST (not an override
+# of the LLM score). Lower-case, substring match. The list intentionally
+# leans heavy on CTF/HTB defaults — dotfiles, framework debug routes, common
+# leaks. ``flag`` is included for completeness but in practice the cheap-sweep
+# flag-regex check catches the real wins.
+DEFAULT_KEYWORD_PATTERNS: tuple[str, ...] = (
+    "admin",
+    "secret",
+    "api",
+    "login",
+    "flag",
+    "debug",
+    "dev",
+    "test",
+    "backup",
+    "config",
+    ".git",
+    ".env",
+    ".bak",
+    ".old",
+    ".zip",
+    ".sql",
+    ".tar",
+    ".ds_store",
+    "phpinfo",
+    "console",
+    "swagger",
+    "graphql",
+    "actuator",
+    "metrics",
+    "health",
+    "wp-",
+    "phpmyadmin",
+    "web.config",
+    "id_rsa",
+    "private",
+    "internal",
+    "hidden",
+)
+
+# Mirrors ``AgentController.FLAG_PATTERNS`` so a flag visible in a Tier-1
+# snippet immediately promotes that endpoint to Tier 2. Kept local (not
+# imported from controller) to avoid a phantom -> controller dependency.
+_FLAG_SHAPED_RE = re.compile(
+    r"(?:flag|FLAG|HTB|THM|CTF|picoCTF|HackTheBox)\{[^}\s]+\}|\b[a-f0-9]{32}\b",
+    re.IGNORECASE,
+)
+
+
+class CheapProbeResult(BaseModel):
+    """Tier-1 sweep result: a streamed-and-truncated GET against one endpoint.
+
+    Treated as *evidence* feeding the selection step, not a verdict. A status
+    of ``0`` means the request failed before producing a response (timeout,
+    connection refused, TLS error). ``contains_flag_shape`` and
+    ``keyword_hits`` are precomputed here so the pure selector doesn't have
+    to re-scan paths and bodies.
+    """
+
+    probe: RankedProbe = Field(description="The ranked probe that was swept.")
+    actual_status: int = Field(description="HTTP status returned, or 0 on transport error.")
+    content_type: str = Field(default="", description="Response Content-Type header.")
+    body_length: int = Field(default=0, description="Bytes downloaded (capped by the sweep).")
+    body_snippet: str = Field(
+        default="",
+        description=f"First ~{_CHEAP_BODY_SNIPPET_CHARS} chars of the body.",
+    )
+    contains_flag_shape: bool = Field(
+        default=False,
+        description="True iff body_snippet matched the flag-shaped regex.",
+    )
+    keyword_hits: tuple[str, ...] = Field(
+        default_factory=tuple,
+        description="Keyword substrings matched against the (lower-cased) path.",
+    )
+    error: str | None = Field(
+        default=None,
+        description="Bracketed error string when the request failed pre-response.",
+    )
+
+
+async def _cheap_probe_one(
+    http: httpx.AsyncClient,
+    base: str,
+    probe: RankedProbe,
+    *,
+    timeout_seconds: float,
+    body_cap_bytes: int,
+    keyword_patterns: tuple[str, ...],
+) -> CheapProbeResult:
+    """Stream one GET, truncate the body, and classify the result.
+
+    Always uses GET regardless of ``probe.method``: in the cheap sweep we
+    care about "does this route exist and what does it return?", not about
+    semantically mutating the target. A 405 here is itself signal.
+    """
+    url = urljoin(base + "/", probe.path.lstrip("/"))
+    path_lower = probe.path.lower()
+    keyword_hits = tuple(k for k in keyword_patterns if k in path_lower)
+
+    try:
+        async with http.stream("GET", url, timeout=timeout_seconds) as resp:
+            body = bytearray()
+            async for chunk in resp.aiter_bytes():
+                body.extend(chunk)
+                if len(body) >= body_cap_bytes:
+                    break
+            snippet = bytes(body).decode("utf-8", errors="replace")[:_CHEAP_BODY_SNIPPET_CHARS]
+            return CheapProbeResult(
+                probe=probe,
+                actual_status=resp.status_code,
+                content_type=resp.headers.get("content-type", ""),
+                body_length=len(body),
+                body_snippet=snippet,
+                contains_flag_shape=bool(_FLAG_SHAPED_RE.search(snippet)),
+                keyword_hits=keyword_hits,
+            )
+    except (TimeoutError, httpx.TimeoutException):
+        return CheapProbeResult(
+            probe=probe,
+            actual_status=0,
+            keyword_hits=keyword_hits,
+            error="timeout",
+        )
+    except httpx.RequestError as e:
+        return CheapProbeResult(
+            probe=probe,
+            actual_status=0,
+            keyword_hits=keyword_hits,
+            error=f"{e.__class__.__name__}: {e}",
+        )
+
+
+async def cheap_sweep(
+    base: str,
+    probes: list[RankedProbe],
+    http: httpx.AsyncClient,
+    *,
+    timeout_seconds: float = _DEFAULT_CHEAP_TIMEOUT_SECONDS,
+    total_budget_seconds: float = _DEFAULT_CHEAP_BUDGET_SECONDS,
+    concurrency: int = _DEFAULT_CONCURRENCY,
+    body_cap_bytes: int = _CHEAP_BODY_BYTES,
+    keyword_patterns: tuple[str, ...] = DEFAULT_KEYWORD_PATTERNS,
+) -> list[CheapProbeResult]:
+    """Tier 1: cheap, exhaustive GET sweep with semaphore + total wall budget.
+
+    Probes are fanned out under an :class:`asyncio.Semaphore` so that no more
+    than ``concurrency`` requests fly in parallel (CTF VMs and shared HTB
+    boxes routinely fall over under uncapped ``asyncio.gather``). The whole
+    sweep is bounded by ``total_budget_seconds``; tasks still pending at the
+    deadline are cancelled, and the *partial* result list (completed tasks
+    only, in submission order) is returned. Callers can detect a truncated
+    sweep by comparing ``len(result) < len(probes)``.
+
+    Returns ``[]`` when ``probes`` is empty (no I/O performed).
+    """
+    if not probes:
+        return []
+
+    semaphore = asyncio.Semaphore(max(1, concurrency))
+
+    async def _bounded(p: RankedProbe) -> CheapProbeResult:
+        async with semaphore:
+            return await _cheap_probe_one(
+                http,
+                base,
+                p,
+                timeout_seconds=timeout_seconds,
+                body_cap_bytes=body_cap_bytes,
+                keyword_patterns=keyword_patterns,
+            )
+
+    tasks: list[asyncio.Task[CheapProbeResult]] = [asyncio.create_task(_bounded(p)) for p in probes]
+
+    try:
+        # return_exceptions=True so a single rogue task can't blow up the
+        # gather before the budget timer trips.
+        await asyncio.wait_for(
+            asyncio.gather(*tasks, return_exceptions=True),
+            timeout=total_budget_seconds,
+        )
+    except TimeoutError:
+        # wait_for cancels the inner gather, which cascade-cancels every
+        # pending task. Drain them so we don't leave dangling coroutines.
+        for t in tasks:
+            if not t.done():
+                t.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    results: list[CheapProbeResult] = []
+    for t in tasks:
+        if t.done() and not t.cancelled():
+            try:
+                results.append(t.result())
+            except Exception:
+                # _cheap_probe_one is supposed to catch every transport error
+                # and return a CheapProbeResult; any leak is treated as a
+                # non-result rather than killing the whole sweep.
+                continue
+    return results
+
+
+def select_for_deep_probe(
+    cheap_results: list[CheapProbeResult],
+    ranked: list[RankedProbe],
+    *,
+    interesting_status: frozenset[int] = INTERESTING_STATUS_CODES,
+    top_n_forced: int = _DEFAULT_TOP_N,
+    max_deep_probes: int = _DEFAULT_MAX_DEEP_PROBES,
+) -> list[RankedProbe]:
+    """Pick which probes deserve a full Tier-2 probe.
+
+    Pure: no I/O. Inputs are the Tier-1 evidence (``cheap_results``) and the
+    full Stage-3 ranking (``ranked``). A probe is selected if it is in the
+    top ``top_n_forced`` by score OR its Tier-1 result triggered ANY of:
+      - status in ``interesting_status``,
+      - body matched the flag-shaped regex (``contains_flag_shape``),
+      - path matched a keyword pattern (``keyword_hits`` non-empty).
+
+    Returned order preserves the Stage-3 ranking (highest score first), is
+    de-duplicated by (method, path), and is hard-capped at ``max_deep_probes``
+    to bound Tier-2 cost.
+    """
+    by_key: dict[tuple[str, str], CheapProbeResult] = {
+        (r.probe.method, r.probe.path): r for r in cheap_results
+    }
+
+    selected: list[RankedProbe] = []
+    seen: set[tuple[str, str]] = set()
+
+    # Pass 1 — forced top-N. The LLM's bet always gets a deep probe even if
+    # Tier 1 looked uninteresting (handles the case where the response was
+    # interesting in body but not in status: a 200 ratio of nothing useful).
+    for probe in ranked[:top_n_forced]:
+        key = (probe.method, probe.path)
+        if key not in seen:
+            selected.append(probe)
+            seen.add(key)
+            if len(selected) >= max_deep_probes:
+                return selected
+
+    # Pass 2 — evidence-driven promotions in score order.
+    for probe in ranked:
+        if len(selected) >= max_deep_probes:
+            break
+        key = (probe.method, probe.path)
+        if key in seen:
+            continue
+        cheap = by_key.get(key)
+        if cheap is None:
+            # No Tier-1 evidence (e.g. cheap sweep timed out before reaching
+            # this probe) — defer to the LLM's score, which already had its
+            # chance in Pass 1.
+            continue
+        promote = (
+            cheap.actual_status in interesting_status
+            or cheap.contains_flag_shape
+            or bool(cheap.keyword_hits)
+        )
+        if promote:
+            selected.append(probe)
+            seen.add(key)
+
+    return selected
+
+
+# ---------------------------------------------------------------------------
+# Stage 4: Adaptive Verification with Belief Update
+#
+# The full PHANTOM loop. Stage 1-3 produce a ranked probe list deterministically;
+# Stage 4 fires probes against the target, evaluates the results, and — if
+# the surprise heuristic trips — re-prompts the mirroring LLM with a
+# correction context that names the disproven predictions. The new MAM is
+# re-ranked and the next batch fires. Iteration is capped by ``max_iterations``.
+#
+# When ``enable_two_tier=True`` (the default) the per-iteration probe step is
+# the Stage 4a two-tier flow: cheap sweep over the full ranked list, then
+# selection, then Tier-2 deep probes on the selected subset. When False,
+# the legacy single-tier flow runs: top-N by score, fired directly with
+# ``_execute_probe``.
+#
+# Concretely: surprise = "the K highest-probability deep-probe results in
+# this batch all returned a status other than the one predicted". That
+# signal is strong enough to suggest the fingerprint itself misled the
+# mirror, so we hand the evidence back to the LLM rather than just trying
+# more rows from the same (already-disproven) ranking.
+# ---------------------------------------------------------------------------
+
+
+class ProbeResult(BaseModel):
+    """Outcome of firing a single ``RankedProbe`` at the target."""
+
+    probe: RankedProbe = Field(description="The probe that was executed.")
+    actual_status: int = Field(
+        description=(
+            "HTTP status returned by the target, or 0 if the request itself "
+            "failed (timeout, connection refused, TLS error, etc.)."
+        ),
+    )
+    confirmed: bool = Field(
+        description="True iff actual_status == probe.expected_status.",
+    )
+    response_snippet: str = Field(
+        description=(
+            f"First {_PROBE_BODY_SNIPPET_CHARS} characters of the response body, "
+            "or a bracketed error string when the request failed before a response."
+        ),
+    )
+
+
+async def _execute_probe(
+    http: httpx.AsyncClient,
+    base: str,
+    probe: RankedProbe,
+    timeout_seconds: float,
+) -> ProbeResult:
+    """Fire one ranked probe and convert the response into a ``ProbeResult``.
+
+    Network/HTTP failures collapse to ``actual_status=0`` with a bracketed
+    error string in ``response_snippet`` — that way the surprise heuristic
+    uniformly treats "failed to reach" and "reached but wrong status" as
+    'did not match expected_status', without special-casing.
+    """
+    url = urljoin(base + "/", probe.path.lstrip("/"))
+    try:
+        resp = await asyncio.wait_for(
+            http.request(probe.method, url),
+            timeout=timeout_seconds,
+        )
+    except (TimeoutError, httpx.TimeoutException):
+        return ProbeResult(
+            probe=probe,
+            actual_status=0,
+            confirmed=False,
+            response_snippet="<error: timeout>",
+        )
+    except httpx.RequestError as e:
+        return ProbeResult(
+            probe=probe,
+            actual_status=0,
+            confirmed=False,
+            response_snippet=f"<error: {e.__class__.__name__}: {e}>",
+        )
+
+    snippet = (resp.text or "")[:_PROBE_BODY_SNIPPET_CHARS]
+    return ProbeResult(
+        probe=probe,
+        actual_status=resp.status_code,
+        confirmed=(resp.status_code == probe.expected_status),
+        response_snippet=snippet,
+    )
+
+
+def _build_correction_context(failed: list[ProbeResult]) -> str:
+    """Format previously-failed probes for the belief-update re-prompt.
+
+    Lists each failed probe with method, path, expected vs actual status,
+    its prior probability, and the LLM's original rationale — that last
+    field is the lever the LLM needs to notice 'oh, my framework assumption
+    was wrong' rather than just shuffling paths.
+    """
+    lines = [
+        (
+            f"- {r.probe.method} {r.probe.path} -> expected {r.probe.expected_status}, "
+            f"got {r.actual_status} (prior probability {r.probe.probability:.2f}; "
+            f"original rationale: {r.probe.rationale})"
+        )
+        for r in failed
+    ]
+    return (
+        "PREVIOUS ATTEMPT FAILED. The following high-probability predictions did "
+        "NOT return the expected status when probed. Treat this as strong evidence "
+        "that your prior framework or deployment assumptions were wrong — consider "
+        "a different version, a non-default routing prefix, a different framework "
+        "entirely, or that the target is behind a reverse proxy that rewrites paths. "
+        "Do NOT re-predict any endpoint that has already been disproven below.\n\n"
+        "Disproven predictions:\n" + "\n".join(lines)
+    )
+
+
+async def execute_phantom_loop(
+    target_url: str,
+    llm_call: LLMCallable,
+    *,
+    max_iterations: int = 2,
+    top_n: int = _DEFAULT_TOP_N,
+    surprise_top_k: int = _DEFAULT_SURPRISE_TOP_K,
+    timeout_seconds: float = _PER_REQUEST_TIMEOUT_SECONDS,
+    client: httpx.AsyncClient | None = None,
+    enable_two_tier: bool = True,
+    cheap_timeout_seconds: float = _DEFAULT_CHEAP_TIMEOUT_SECONDS,
+    cheap_budget_seconds: float = _DEFAULT_CHEAP_BUDGET_SECONDS,
+    concurrency: int = _DEFAULT_CONCURRENCY,
+    max_deep_probes: int = _DEFAULT_MAX_DEEP_PROBES,
+    keyword_patterns: tuple[str, ...] = DEFAULT_KEYWORD_PATTERNS,
+    include_deterministic: bool = True,
+    use_compact_schema: bool = True,
+) -> list[ProbeResult]:
+    """Run the full PHANTOM pipeline against ``target_url``.
+
+    Pipeline per iteration:
+      1. (iteration 0 only) gather raw signals + crystallize fingerprint.
+      2. ``generate_mam`` — fresh on iteration 0, carrying a correction
+         context on subsequent iterations.
+      3. ``rank_probes`` over the MAM.
+      4. Two-tier probe step (``enable_two_tier=True``):
+           4a. ``cheap_sweep`` GETs the full ranked list (minus already-tried),
+               under a semaphore and total wall budget.
+           4b. ``select_for_deep_probe`` picks the Tier-2 subset (top-N by
+               score plus evidence-driven promotions).
+           4c. ``_execute_probe`` fires the selected probes concurrently
+               (same semaphore) using their predicted method.
+         Legacy single-tier path (``enable_two_tier=False``): take the top-N
+         ranked by score and fire them directly with ``_execute_probe``.
+      5. Surprise check: if the top-K deep-probe results (by probability)
+         from this batch were all unconfirmed, build a correction context
+         and loop. Otherwise return.
+
+    Returns every Tier-2 ``ProbeResult`` collected across all iterations, in
+    the order they were executed. Tier-1 ``CheapProbeResult`` evidence is
+    intentionally NOT in the return — callers that want it should invoke
+    :func:`cheap_sweep` directly.
+
+    ``client`` may be supplied so callers (and tests) can share a single
+    long-lived :class:`httpx.AsyncClient`; when ``None``, a one-shot client
+    is created and reused across Stage 1 recon and Stage 4 probes so that
+    the connection pool warms once.
+
+    ``include_deterministic`` and ``use_compact_schema`` are forwarded to
+    :func:`generate_mam`; leave them ``True`` in production (merge the always-on
+    rulebook; use the compact prompt/schema). Tests that feed a full
+    ``MirroredApplicationManifest`` dump and want to assert only on the LLM's
+    predicted endpoints set both to ``False``.
+    """
+    if max_iterations < 1:
+        raise ValueError("max_iterations must be >= 1")
+    if top_n < 1:
+        raise ValueError("top_n must be >= 1")
+    if surprise_top_k < 1:
+        raise ValueError("surprise_top_k must be >= 1")
+    if concurrency < 1:
+        raise ValueError("concurrency must be >= 1")
+    if max_deep_probes < 1:
+        raise ValueError("max_deep_probes must be >= 1")
+
+    owns_client = False
+    if client is not None:
+        http = client
+    else:
+        from agent.core.backend import get_shared_http_client
+
+        http = get_shared_http_client()
+
+    all_results: list[ProbeResult] = []
+    # Dedupe across iterations: even with the correction context telling the
+    # LLM not to re-predict disproven endpoints, models drift — this is the
+    # belt-and-braces guarantee that we never burn budget probing the same
+    # (method, path) twice.
+    tried: set[tuple[str, str]] = set()
+    correction: str | None = None
+    # One semaphore shared across Tier 1 and Tier 2 within a single iteration
+    # so we never exceed ``concurrency`` in-flight requests against the target.
+    semaphore = asyncio.Semaphore(concurrency)
+
+    try:
+        raw_signals = await gather_raw_signals(
+            target_url, timeout_seconds=timeout_seconds, client=http
+        )
+        base = raw_signals["base"]
+        fingerprint = await crystallize_fingerprint(raw_signals, llm_call)
+
+        # Optional RAG hook for framework CVEs
+        try:
+            if fingerprint.framework and fingerprint.framework.lower() != "unknown":
+                from agent.rag_module.knowledge_base import get_shared_kb
+
+                cves = await get_shared_kb().query_framework_cves(
+                    fingerprint.framework,
+                    fingerprint.version_guess,
+                    top_k=3,
+                )
+                if cves:
+                    rag_context = "\n".join(cves)
+                    if correction:
+                        correction += f"\n\n[RAG Framework Info]:\n{rag_context}"
+                    else:
+                        correction = f"[RAG Framework Info]:\n{rag_context}"
+        except Exception:
+            pass
+
+        for _iteration in range(max_iterations):
+            mam = await generate_mam(
+                fingerprint,
+                llm_call,
+                correction_context=correction,
+                include_deterministic=include_deterministic,
+                use_compact_schema=use_compact_schema,
+            )
+            ranked = rank_probes(mam)
+            ranked = [r for r in ranked if (r.method, r.path) not in tried]
+            if not ranked:
+                # MAM produced nothing new; further iteration cannot help.
+                break
+
+            if enable_two_tier:
+                # Tier 1: cheap exhaustive GET sweep over the whole ranked list.
+                cheap_results = await cheap_sweep(
+                    base,
+                    ranked,
+                    http,
+                    timeout_seconds=cheap_timeout_seconds,
+                    total_budget_seconds=cheap_budget_seconds,
+                    concurrency=concurrency,
+                    keyword_patterns=keyword_patterns,
+                )
+                # Tier 2 selection — top-N by score (the LLM's bet) plus any
+                # cheap-sweep evidence that justifies a deeper look.
+                batch = select_for_deep_probe(
+                    cheap_results,
+                    ranked,
+                    top_n_forced=top_n,
+                    max_deep_probes=max_deep_probes,
+                )
+            else:
+                # Legacy single-tier behaviour, preserved for callers that
+                # explicitly opt out (and to keep older tests honest).
+                batch = ranked[:top_n]
+
+            if not batch:
+                break
+            for r in batch:
+                tried.add((r.method, r.path))
+
+            results = await _execute_batch(http, base, batch, timeout_seconds, semaphore)
+            all_results.extend(results)
+
+            # Surprise heuristic: rank THIS batch by probability (not by score)
+            # and check whether the top-K all failed. Probability is the right
+            # axis here because severity weighting can promote a low-probability
+            # high-impact guess into the top-N — we don't want one such miss
+            # to dominate the belief-update signal.
+            by_prob = sorted(results, key=lambda r: r.probe.probability, reverse=True)
+            top_k = by_prob[:surprise_top_k]
+            if len(top_k) < surprise_top_k or any(r.confirmed for r in top_k):
+                break
+
+            correction = _build_correction_context(top_k)
+    finally:
+        if owns_client:
+            await http.aclose()
+
+    return all_results
+
+
+async def _execute_batch(
+    http: httpx.AsyncClient,
+    base: str,
+    batch: list[RankedProbe],
+    timeout_seconds: float,
+    semaphore: asyncio.Semaphore,
+) -> list[ProbeResult]:
+    """Run Tier-2 probes concurrently under ``semaphore``.
+
+    Extracted so the two-tier and legacy paths share identical concurrency
+    semantics — the only thing that varies between them is how ``batch`` was
+    chosen.
+    """
+
+    async def _bounded(p: RankedProbe) -> ProbeResult:
+        async with semaphore:
+            return await _execute_probe(http, base, p, timeout_seconds)
+
+    coros = [_bounded(p) for p in batch]
+    return await asyncio.gather(*coros, return_exceptions=False)
+
+
+__all__ = [
+    "ALWAYS_PROBE",
+    "COMPACT_MAM_JSON_SCHEMA",
+    "DEFAULT_KEYWORD_PATTERNS",
+    "DEFAULT_RECON_PATHS",
+    "FRAMEWORK_RULES",
+    "INTERESTING_STATUS_CODES",
+    "CheapProbeResult",
+    "CompactMirroredManifest",
+    "FingerprintVector",
+    "LLMCallable",
+    "MirroredApplicationManifest",
+    "PathSignal",
+    "PredictedEndpoint",
+    "PredictedEndpointCompact",
+    "ProbeResult",
+    "RankedProbe",
+    "StructuredLLMCallable",
+    "build_crystallization_prompt",
+    "build_mirroring_prompt",
+    "build_mirroring_prompt_compact",
+    "cheap_sweep",
+    "crystallize_fingerprint",
+    "deterministic_endpoints",
+    "estimate_cost",
+    "estimate_severity",
+    "execute_phantom_loop",
+    "expand_compact",
+    "gather_raw_signals",
+    "generate_mam",
+    "merge_endpoints",
+    "parse_compact_mirroring_response",
+    "parse_fingerprint_response",
+    "parse_mirroring_response",
+    "rank_probes",
+    "select_for_deep_probe",
+]