misterdev 0.2.0__py3-none-any.whl

misterdev/core/verification/claim_verifier.py

@@ -0,0 +1,210 @@
+"""Independent verification of completeness claims before they become tasks.
+
+The completeness analyzer flags features as "incomplete" or files as "stubs"
+from a lossy project overview, so it can mislabel deliberate design — a
+graceful-degradation path, a platform-gated no-op, a parity shim, a documented
+fallback — as unfinished work. Acting on that wastes budget and risks rewriting
+code that was correct by design (observed: a documented wasm degrade-to-empty
+backend planned as a "fix the stub" task).
+
+This gate gives each such CLAIM a second, INDEPENDENT look at the REAL file (and
+whatever evidence the caller assembled — the file body, tests that exercise it,
+the verified build/test state) and drops only the claims it can REFUTE with
+evidence. Anything it confirms, or is unsure about, is KEPT — the burden of proof
+is on dropping, so genuine work is never silently lost on a guess. It mirrors the
+edit-time adversarial critic and the goal-completion judge: advisory and
+best-effort, routed through an independent model when one is configured, and a
+SKIP (keep the claim) on no client, an unparseable verdict, any error, or the
+hard timeout. The decomposer and gates remain the ground truth.
+"""
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import Callable, List, Optional
+
+from misterdev.core.execution.bounded import run_bounded
+from misterdev.core.verification.independent import build_independent_call
+from misterdev.llm.responses import (
+    extract_json_object as _extract_json_object,
+)
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+# A claim is KEPT (acted on) unless the verifier REFUTES it with evidence.
+CONFIRMED = "confirmed"
+REFUTED = "refuted"
+KEPT = "kept"
+
+# A verifier call takes the assembled prompt text and returns the model's text.
+# Injected in tests; defaulted to the project client's generate_code path.
+VerifyCall = Callable[[str], str]
+
+_VERIFY_PROMPT = """A project analyzer claimed the following is INCOMPLETE work \
+that needs implementing. Decide whether that claim is REAL or a FALSE POSITIVE.
+
+It is a FALSE POSITIVE (refute it) when the code is intentional and complete:
+- a documented graceful-degradation or fallback path (returns empty/default BY
+  DESIGN, e.g. a missing-model or no-network contract),
+- a platform-gated no-op (e.g. a wasm/no-filesystem backend), a parity shim, or
+  a deliberate placeholder the design says is correct,
+- a capability that already builds and is covered by passing tests.
+
+It is REAL (confirm it) only when there is a concrete, unmet behavior the code is
+supposed to provide but does not. Judge the SOURCE and tests, not names or docs
+plans. If the evidence does not let you decide, say unsure — do not guess "real".
+
+## Claim ({kind})
+{label}: {description}
+
+## Evidence (real source / tests / verified build+test state)
+{evidence}
+
+Return ONLY a JSON object, no prose or fences:
+{{"real": true|false|null, "reason": "<one sentence citing the evidence>"}}
+where false = false positive (intentional/complete), true = genuinely incomplete,
+null = unsure."""
+
+
+@dataclass(frozen=True)
+class Claim:
+    """One completeness claim to verify.
+
+    ``kind`` is "incomplete" or "stub"; ``label`` is its identity (feature name
+    or file path) used to drop it from the assessment; ``description`` and
+    ``evidence`` are the text shown to the verifier (real file body, tests, and
+    the verified build/test state — assembled by the caller).
+    """
+
+    kind: str
+    label: str
+    description: str = ""
+    evidence: str = ""
+
+
+@dataclass
+class ClaimVerdict:
+    """Outcome for one claim. ``status`` is CONFIRMED / REFUTED / KEPT.
+
+    REFUTED means drop the claim (a false positive). CONFIRMED and KEPT both keep
+    it; KEPT covers "unsure" and every skip/error path, so uncertainty never
+    drops real work. ``reason`` is the model's one-line justification (or the
+    skip reason); ``raw`` is its text.
+    """
+
+    claim: Claim
+    status: str
+    reason: str = ""
+    raw: str = ""
+
+    @property
+    def refuted(self) -> bool:
+        return self.status == REFUTED
+
+
+# Evidence cap per claim so a huge file can't blow up the prompt or cost; the
+# head carries the most signal (module docs, signatures, the flagged symbols).
+_MAX_EVIDENCE_CHARS = 12000
+
+
+def verify_claims(
+    claims: List[Claim],
+    verify_call: Optional[VerifyCall] = None,
+    llm_client=None,
+    model: Optional[str] = None,
+    timeout: float = 45,
+) -> List[ClaimVerdict]:
+    """Verify each claim independently; REFUTE only false positives, else KEEP.
+
+    The model call is performed by ``verify_call`` when supplied (the test seam);
+    otherwise one is built from ``llm_client``. ``model``, when given, routes the
+    judgment through an INDEPENDENT model so it does not share the generator's
+    blind spots (its absence is logged — the same-model verifier is weaker).
+
+    With no callable and no client, every claim is KEPT (no verifier available),
+    so the behavior is byte-identical to not running the gate. ``timeout`` is the
+    hard ceiling PER claim; an unparseable verdict, any error, or a timeout KEEPS
+    that claim. Independent claims fan out across a small thread pool (the
+    analyzers do the same) EXCEPT when an independent ``model`` is configured:
+    routing through it uses ``with_model``, which mutates shared client state and
+    is not thread-safe, so that path stays sequential. Returns one verdict per
+    input claim, in order.
+    """
+    if not claims:
+        return []
+
+    call = verify_call or build_independent_call(
+        llm_client,
+        "You are a precise code reviewer. Return only valid JSON.",
+        model,
+        "Completeness-claim verifier",
+    )
+    if call is None:
+        return [
+            ClaimVerdict(c, KEPT, reason="no LLM verifier available") for c in claims
+        ]
+
+    def judge(claim: Claim) -> ClaimVerdict:
+        prompt = _VERIFY_PROMPT.format(
+            kind=claim.kind,
+            label=claim.label or "(unnamed)",
+            description=(claim.description or "(no description)").strip(),
+            evidence=(claim.evidence or "(no evidence assembled)").strip()[
+                :_MAX_EVIDENCE_CHARS
+            ],
+        )
+
+        def _work() -> ClaimVerdict:
+            try:
+                return _parse_verdict(claim, call(prompt) or "")
+            except Exception as e:  # any model/IO failure is non-fatal -> keep
+                logger.debug(f"Claim verifier unavailable for {claim.label!r}: {e}")
+                return ClaimVerdict(claim, KEPT, reason=f"error: {e}")
+
+        return run_bounded(
+            _work,
+            timeout,
+            ClaimVerdict(claim, KEPT, reason="timed out"),
+            "Completeness-claim verifier",
+        )
+
+    # `model` set -> per-call with_model() is not thread-safe -> sequential.
+    if model or len(claims) == 1:
+        return [judge(c) for c in claims]
+    with ThreadPoolExecutor(max_workers=min(4, len(claims))) as pool:
+        return list(pool.map(judge, claims))
+
+
+def _parse_verdict(claim: Claim, text: str) -> ClaimVerdict:
+    """Parse the verifier's JSON verdict; default to KEEP on any ambiguity.
+
+    Only an explicit ``{"real": false}`` REFUTES (drops) the claim. ``real`` true
+    CONFIRMS it; ``real`` null, a missing/invalid object, or a missing/non-bool
+    ``real`` all KEEP it — the claim is acted on unless the verifier positively
+    refutes it, so uncertainty never silently removes real work.
+    """
+    if not text or not text.strip():
+        return ClaimVerdict(claim, KEPT, reason="empty verdict", raw=text)
+
+    obj = _extract_json_object(text)
+    if obj is None or "real" not in obj:
+        return ClaimVerdict(claim, KEPT, reason="unparseable verdict", raw=text)
+
+    real = obj.get("real")
+    reason = obj.get("reason")
+    reason = reason.strip() if isinstance(reason, str) and reason.strip() else ""
+    if real is False:
+        # Surface a content-free refute honestly (it dropped a claim with no stated
+        # justification) instead of fabricating a confident-sounding rationale.
+        return ClaimVerdict(
+            claim,
+            REFUTED,
+            reason=reason or "refuted without a stated reason",
+            raw=text,
+        )
+    if real is True:
+        return ClaimVerdict(
+            claim, CONFIRMED, reason=reason or "genuinely incomplete", raw=text
+        )
+    # real is null / non-boolean / anything else -> keep (unsure).
+    return ClaimVerdict(claim, KEPT, reason=reason or "unsure", raw=text)

misterdev/core/verification/critic.py

@@ -0,0 +1,324 @@
+"""Optional adversarial edit critic (independent second component).
+
+A single generator shares its own blind spots: a model that misreads a
+requirement, misses an edge case, or "fixes" a symptom instead of the cause will
+not catch that by re-reading its own output — the error and the reviewer are the
+same component. This gate adds a SECOND, deliberately independent component that
+reviews a CANDIDATE edit *before it is applied* and either APPROVES it or returns
+concrete, actionable objections. Objections are fed back to the generator as the
+next attempt's error context, forming a generate -> critique -> regenerate loop.
+
+Independence is the whole point. A critic that shares the generator's model,
+prompt, and training mostly rationalizes the first answer (self-critique is
+weakest exactly when the model is confidently wrong). Configure ``critic.model``
+to a DIFFERENT model for the real benefit; without it the gate still runs on the
+generator's own model with adversarial framing, but the signal is weaker — and
+that weakness is logged so it is never silently assumed away.
+
+The critic is ADVISORY, never authoritative. The deterministic build/test/lint
+gates remain the ground truth; this only catches semantic blind spots earlier
+and more cheaply than a full gate run, and it defers to the gates after
+``max_rejections`` regenerations so an over-zealous critic can't starve the loop.
+
+Mirrors :mod:`misterdev.core.verification.goal_check`: best-effort and run in a
+daemon worker thread with a hard timeout so a slow or unreachable model can NEVER
+block a build. No client, no candidate edit, an unparseable verdict, or a timeout
+is a SKIP (no opinion) that lets the edit proceed untouched.
+"""
+
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable, Dict, List, Optional
+
+from misterdev.core.execution.bounded import run_bounded
+from misterdev.core.verification.goal_check import _extract_json_object
+from misterdev.core.verification.independent import build_independent_call
+from misterdev.logging_setup import setup_logger
+
+logger = setup_logger(__name__)
+
+# Perspective lenses for a multi-member panel. Each member reviews the same
+# change through a different lens so the panel catches failure modes a single
+# reviewer (or N identical reviewers) would share — diversity, not redundancy.
+_LENSES = (
+    "Focus above all on CORRECTNESS: does the logic actually do what the task asks?",
+    "Focus above all on EDGE CASES: empty/null/zero/maximum inputs and concurrency.",
+    "Focus above all on SAFETY: resource leaks, swallowed errors, and security holes.",
+    "Focus above all on REQUIREMENTS: is any acceptance criterion unmet or misread?",
+    "Focus above all on ROOT CAUSE: does this fix the underlying defect, or only "
+    "patch a symptom (a localized workaround that leaves the real cause in place)?",
+    "Focus above all on DUPLICATION (DRY): does this re-implement logic that already "
+    "exists instead of reusing an existing function/type/module?",
+)
+
+# Outcome constants. SKIP means "no opinion" (no client/edit, an unparseable
+# verdict, or a timeout) and must let the edit proceed — never a rejection.
+SKIP = "skip"
+APPROVED = "approved"
+REJECTED = "rejected"
+
+_PROMPT = (
+    "You are an adversarial code reviewer. Your job is to find what is WRONG with "
+    "a candidate change BEFORE it is applied — assume it is flawed and try to "
+    "prove it. Judge only the change shown against the task and acceptance "
+    "criteria; do not rewrite it.\n\n"
+    "## Task\n{task}\n\n"
+    "## Acceptance Criteria\n{criteria}\n\n"
+    "## Candidate Change\n{candidate}\n\n"
+    "Look specifically for: misread or unmet requirements, logic errors, "
+    "unhandled edge cases (empty/null/zero/max, concurrency), resource leaks, "
+    "swallowed errors, and security holes. Also reject if the change fixes a "
+    "SYMPTOM rather than the ROOT CAUSE (a localized patch that leaves the "
+    "underlying defect in place), or if it DUPLICATES logic that already exists "
+    "instead of reusing it (violates DRY). Ignore pure style.\n\n"
+    "Reply with a single JSON object on the first line and nothing else:\n"
+    '{{"approved": true|false, "objections": ["one concrete, actionable problem '
+    'per item"]}}\n'
+    "If approved is true, objections must be an empty list. Only reject for a "
+    "concrete defect you can name — not a vague preference. Keep each objection to "
+    "one sentence."
+)
+
+# Cap the candidate evidence so a large edit can't blow up the prompt or cost;
+# the head of each file carries the signatures and new logic that matter most.
+_MAX_CANDIDATE_CHARS = 16000
+
+# A critic call takes the assembled prompt text and returns the model's text.
+# Injected in tests; defaulted to the project client's generate_code path.
+CriticCall = Callable[[str], str]
+
+
+class CritiqueVerdict:
+    """Outcome of an adversarial edit critique.
+
+    ``status`` is SKIP / APPROVED / REJECTED. ``objections`` lists the concrete
+    problems (empty unless REJECTED). ``raw`` is the model's text (evidence);
+    ``reason`` explains a SKIP.
+    """
+
+    def __init__(
+        self,
+        status: str,
+        objections: Optional[List[str]] = None,
+        raw: str = "",
+        reason: str = "",
+    ):
+        self.status = status
+        self.objections = list(objections or [])
+        self.raw = raw
+        self.reason = reason
+
+    @property
+    def approved(self) -> bool:
+        return self.status == APPROVED
+
+    @property
+    def rejected(self) -> bool:
+        return self.status == REJECTED
+
+    @property
+    def skipped(self) -> bool:
+        return self.status == SKIP
+
+    def __repr__(self) -> str:
+        return (
+            f"CritiqueVerdict(status={self.status!r}, "
+            f"objections={len(self.objections)})"
+        )
+
+
+def run_edit_critic(
+    task_description: Optional[str],
+    acceptance_criteria: Optional[str],
+    candidate_edits: Optional[Dict[str, str]],
+    critic_call: Optional[CriticCall] = None,
+    llm_client=None,
+    critic_model: Optional[str] = None,
+    candidate_diffs: Optional[Dict[str, str]] = None,
+    panel: int = 1,
+    timeout: float = 60,
+) -> CritiqueVerdict:
+    """Critique ``candidate_edits`` against the task and acceptance criteria.
+
+    The model call is performed by ``critic_call`` when supplied (the test seam);
+    otherwise one is built from ``llm_client``. ``critic_model``, when given,
+    selects an INDEPENDENT model so the critic does not share the generator's
+    blind spots (its absence is logged — the same-model critic is weaker).
+
+    When ``candidate_diffs`` is given, the critic reviews the unified diff of each
+    change (what actually changed, with a little context) instead of whole files —
+    sharper signal and far less prompt for a small edit to a large file.
+
+    ``panel`` > 1 runs that many reviewers concurrently, each through a different
+    perspective lens, and the change is rejected only on a MAJORITY of decisive
+    votes (ties approve), which suppresses a lone false rejection.
+
+    With no callable and no client, or with no candidate edit, the critic SKIPs.
+    SKIP also on an unparseable verdict, any critic error, or the hard timeout
+    (never blocks). Only a parsed rejection returns objections.
+    """
+    if not candidate_edits:
+        return CritiqueVerdict(SKIP, reason="no candidate edit to review")
+
+    call = critic_call or _default_critic_call(llm_client, critic_model)
+    if call is None:
+        return CritiqueVerdict(SKIP, reason="no LLM critic available")
+
+    prompt = _PROMPT.format(
+        task=(task_description or "(none given)").strip(),
+        criteria=(acceptance_criteria or "(none given)").strip(),
+        candidate=_render_candidate(candidate_edits, candidate_diffs),
+    )
+    members = max(1, int(panel))
+
+    def _work() -> CritiqueVerdict:
+        try:
+            if members == 1:
+                return _parse_verdict(call(prompt) or "")
+            return _aggregate_panel(_run_panel(call, prompt, members))
+        except Exception as e:  # any model/IO failure is non-fatal -> skip
+            logger.debug(f"Adversarial critic unavailable: {e}")
+            return CritiqueVerdict(SKIP, reason=f"error: {e}")
+
+    return run_bounded(
+        _work, timeout, CritiqueVerdict(SKIP, reason="timed out"), "Adversarial critic"
+    )
+
+
+def _run_panel(
+    call: CriticCall, base_prompt: str, members: int
+) -> List["CritiqueVerdict"]:
+    """Run ``members`` lens-diversified critiques concurrently; return verdicts.
+
+    A member whose call fails contributes a SKIP (abstention) rather than sinking
+    the panel, so one flaky reviewer can't force a decision either way.
+    """
+    prompts = [f"{base_prompt}\n{_LENSES[i % len(_LENSES)]}\n" for i in range(members)]
+
+    def _one(p: str) -> "CritiqueVerdict":
+        try:
+            return _parse_verdict(call(p) or "")
+        except Exception as e:
+            logger.debug(f"Critic panel member failed: {e}")
+            return CritiqueVerdict(SKIP, reason=f"member error: {e}")
+
+    with ThreadPoolExecutor(max_workers=members) as pool:
+        return list(pool.map(_one, prompts))
+
+
+def _aggregate_panel(verdicts: List["CritiqueVerdict"]) -> "CritiqueVerdict":
+    """Combine panel verdicts: reject only on a strict majority of decisive votes.
+
+    Abstentions (SKIP) don't count. A tie approves — the critic is advisory and
+    defers to the real gates, so consensus is required to block. A rejection
+    unions the objections of the rejecting members (deduplicated, order-stable).
+    """
+    decisive = [v for v in verdicts if v.status != SKIP]
+    if not decisive:
+        return CritiqueVerdict(SKIP, reason="panel reached no verdict")
+    rejects = [v for v in decisive if v.status == REJECTED]
+    approves = [v for v in decisive if v.status == APPROVED]
+    if len(rejects) > len(approves):
+        objections: List[str] = []
+        seen = set()
+        for v in rejects:
+            for o in v.objections:
+                if o not in seen:
+                    seen.add(o)
+                    objections.append(o)
+        return CritiqueVerdict(
+            REJECTED,
+            objections=objections,
+            reason=f"panel rejected {len(rejects)}/{len(decisive)}",
+        )
+    return CritiqueVerdict(
+        APPROVED, reason=f"panel approved {len(approves)}/{len(decisive)}"
+    )
+
+
+def _render_candidate(
+    edits: Dict[str, str], diffs: Optional[Dict[str, str]] = None
+) -> str:
+    """Render the candidate as labeled, size-bounded blocks.
+
+    Shows unified diffs when ``diffs`` is provided (sharper, smaller signal —
+    just what changed), otherwise full file contents. Files are shown in a stable
+    (sorted) order and the total is capped so a huge edit can't blow up the
+    prompt; truncation is marked so the critic knows it saw a head, not all.
+    """
+    source = diffs if diffs else edits
+    note = (
+        "(shown as unified diffs — unchanged lines omitted)"
+        if diffs
+        else "(full content of each edited/created file)"
+    )
+    parts: List[str] = [note]
+    budget = _MAX_CANDIDATE_CHARS
+    for path in sorted(source):
+        if budget <= 0:
+            parts.append("... (further files omitted; candidate too large)")
+            break
+        body = source[path] or ""
+        shown = body[:budget]
+        if len(body) > len(shown):
+            shown += "\n... (truncated)"
+        budget -= len(shown)
+        parts.append(f"### {path}\n{shown}")
+    return "\n\n".join(parts)
+
+
+def _parse_verdict(text: str) -> CritiqueVerdict:
+    """Deterministically parse the critic's JSON verdict.
+
+    Tolerates surrounding prose / markdown fences via the balanced-object
+    extractor shared with the goal check. A missing/invalid object or a missing
+    ``approved`` boolean is a SKIP (no opinion), not a rejection — the critic must
+    never block on its own malformed output. A rejection with no listed objection
+    still records one generic objection so the loop never regenerates without
+    telling the generator why.
+    """
+    if not text or not text.strip():
+        return CritiqueVerdict(SKIP, raw=text, reason="empty verdict")
+
+    obj = _extract_json_object(text)
+    if obj is None or "approved" not in obj:
+        return CritiqueVerdict(SKIP, raw=text, reason="unparseable verdict")
+
+    approved = obj.get("approved")
+    if not isinstance(approved, bool):
+        return CritiqueVerdict(SKIP, raw=text, reason="non-boolean 'approved'")
+
+    if approved:
+        return CritiqueVerdict(APPROVED, objections=[], raw=text)
+
+    raw_objections = obj.get("objections")
+    objections: List[str] = []
+    if isinstance(raw_objections, list):
+        for o in raw_objections:
+            if isinstance(o, str) and o.strip():
+                objections.append(o.strip())
+    elif isinstance(raw_objections, str) and raw_objections.strip():
+        objections.append(raw_objections.strip())
+    if not objections:
+        objections = ["change rejected (no specific objection reported)"]
+    return CritiqueVerdict(REJECTED, objections=objections, raw=text)
+
+
+def _default_critic_call(
+    llm_client, critic_model: Optional[str]
+) -> Optional[CriticCall]:
+    """Build a critic call from the project's LLM client, or None if unusable.
+
+    Uses the client's ``generate_code(prompt, system)`` text interface. When
+    ``critic_model`` is given and the client supports ``with_model``, the call is
+    routed through that INDEPENDENT model; otherwise it runs on the generator's
+    own model and the weaker independence is logged once. Tolerant of client
+    shape so an absent/limited client degrades to SKIP rather than raising. No
+    network is touched until the returned callable is invoked in the worker.
+    """
+    system = (
+        "You are a precise adversarial code reviewer. Output only the requested "
+        "JSON object. Reject only for a concrete, nameable defect."
+    )
+    return build_independent_call(
+        llm_client, system, critic_model, "Adversarial critic"
+    )