debugerai 0.2.0__py3-none-any.whl

debugai/detectors.py

@@ -0,0 +1,206 @@
+"""Layer 2 — Failure Classification Rules (Architecture §5).
+
+Five deterministic detectors. Each takes the signal vector + thresholds and
+returns a DetectorResult. All detectors run (§5.2); results are ranked by
+confidence into primary + secondary. Gate patterns prevent nonsensical
+multi-classification.
+
+Detector bases are tuned to the doc's worked example: Scenario A (similarity
+0.41, entity 0.17, overlap 0.12) → retrieval failure 0.95 = 0.70 base + 0.15
+(entity) + 0.10 (overlap).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from debugai.schema import CaptureRecord
+from debugai.signals import SignalVector
+from debugai.thresholds import Thresholds
+
+# Failure type identifiers (also used by the fix-agent registry later).
+CONTEXT_OVERFLOW = "context_overflow"
+RETRIEVAL_FAILURE = "retrieval_failure"
+ENTITY_GAP = "entity_gap"
+HALLUCINATION = "hallucination"
+PROMPT_BRITTLENESS = "prompt_brittleness"
+
+SEVERITY = {
+    CONTEXT_OVERFLOW: "critical",
+    RETRIEVAL_FAILURE: "critical",
+    ENTITY_GAP: "warning",
+    HALLUCINATION: "critical",
+    PROMPT_BRITTLENESS: "warning",
+}
+
+_GATED_BASE = 0.70  # base confidence for a critical gated detector that fires
+
+
+@dataclass
+class DetectorResult:
+    failure: str
+    fired: bool
+    confidence: float
+    severity: str
+    root_cause: str = ""
+    fix: str = ""  # deterministic fix hint (Layer-3 fallback)
+    evidence: dict = field(default_factory=dict)
+
+    def clamp(self) -> "DetectorResult":
+        self.confidence = round(max(0.0, min(self.confidence, 1.0)), 4)
+        return self
+
+
+# --------------------------------------------------------------------------- #
+# 1. Context overflow — Critical | checked 1st
+# --------------------------------------------------------------------------- #
+def detect_context_overflow(s: SignalVector, rec: CaptureRecord, t: Thresholds) -> DetectorResult:
+    fired = s.context_ratio > t.context_length_ratio_max
+    conf = _GATED_BASE
+    if s.token_ratio > t.token_usage_high:
+        conf += 0.15
+    if s.latency_ms > t.latency_high_ms:
+        conf += 0.10
+    if s.overlap < t.overlap_low:
+        conf += 0.10
+    return DetectorResult(
+        failure=CONTEXT_OVERFLOW,
+        fired=fired,
+        confidence=conf,
+        severity=SEVERITY[CONTEXT_OVERFLOW],
+        root_cause=(
+            f"Prompt fills {s.context_ratio:.0%} of the context window "
+            f"(> {t.context_length_ratio_max:.0%}); content is likely truncated."
+        ),
+        fix="Reduce retrieved chunks to the top-N most relevant, summarise prior "
+        "conversation history, or move to a larger-context model.",
+        evidence={"context_ratio": s.context_ratio, "token_ratio": s.token_ratio,
+                  "latency_ms": s.latency_ms},
+    ).clamp()
+
+
+# --------------------------------------------------------------------------- #
+# 2. Retrieval failure — Critical | checked 2nd
+# --------------------------------------------------------------------------- #
+def detect_retrieval_failure(s: SignalVector, rec: CaptureRecord, t: Thresholds) -> DetectorResult:
+    fired = s.similarity < t.similarity_min
+    conf = _GATED_BASE
+    if s.entity_coverage < t.entity_coverage_min:
+        conf += 0.15
+    if s.overlap < t.overlap_very_low:
+        conf += 0.10
+    return DetectorResult(
+        failure=RETRIEVAL_FAILURE,
+        fired=fired,
+        confidence=conf,
+        severity=SEVERITY[RETRIEVAL_FAILURE],
+        root_cause=(
+            f"Mean retrieval similarity {s.similarity:.2f} is below "
+            f"{t.similarity_min:.2f}; the retriever returned irrelevant chunks."
+        ),
+        fix="Re-chunk source documents with an entity-aware strategy, tune the "
+        "retriever / embedding model, or expand the knowledge base.",
+        evidence={"similarity": s.similarity, "entity_coverage": s.entity_coverage,
+                  "overlap": s.overlap},
+    ).clamp()
+
+
+# --------------------------------------------------------------------------- #
+# 3. Entity gap — Warning | checked 3rd
+# --------------------------------------------------------------------------- #
+def detect_entity_gap(s: SignalVector, rec: CaptureRecord, t: Thresholds) -> DetectorResult:
+    fired = (
+        s.similarity >= t.similarity_min
+        and s.entity_coverage < t.entity_coverage_min
+        and s.contradiction < t.contradiction_min
+        and s.variance <= t.variance_min
+    )
+    conf = 0.60 + min(0.08 * s.entities_missing, 0.30)
+    return DetectorResult(
+        failure=ENTITY_GAP,
+        fired=fired,
+        confidence=conf,
+        severity=SEVERITY[ENTITY_GAP],
+        root_cause=(
+            f"Retrieval is healthy but {s.entities_missing} entity(ies) in the "
+            "answer are absent from the retrieved context — a knowledge-base hole."
+        ),
+        fix="Check whether the missing entities exist elsewhere in the corpus with "
+        "different chunking; if not, flag the knowledge-base gap for human review.",
+        evidence={"entity_coverage": s.entity_coverage,
+                  "entities_missing": s.entities_missing},
+    ).clamp()
+
+
+# --------------------------------------------------------------------------- #
+# 4. Hallucination — Critical | checked 4th
+# --------------------------------------------------------------------------- #
+def detect_hallucination(s: SignalVector, rec: CaptureRecord, t: Thresholds) -> DetectorResult:
+    score = 0.0
+    if s.similarity >= t.similarity_min:  # gate
+        if s.entity_coverage < t.entity_coverage_hallucination:
+            score += 0.35
+        if s.contradiction > t.contradiction_min:
+            score += 0.30
+        if s.variance > t.variance_min:
+            score += 0.20
+        if t.overlap_very_low <= s.overlap <= 0.70:
+            score += 0.10
+        # High overlap (≥ 0.70) means the output largely repeats the context —
+        # strong grounding evidence that weighs against hallucination.
+        if s.overlap >= 0.70:
+            score -= 0.15
+    fired = s.similarity >= t.similarity_min and score >= t.hallucination_fire
+    return DetectorResult(
+        failure=HALLUCINATION,
+        fired=fired,
+        confidence=score,
+        severity=SEVERITY[HALLUCINATION],
+        root_cause=(
+            "Retrieval succeeded but the output is not grounded in it "
+            f"(fabrication score {score:.2f}): low entity coverage / contradiction "
+            "/ instability indicate invented content."
+        ),
+        fix="Add grounding constraints to the system prompt (answer only from "
+        "provided context; cite sources; say 'not found' when unsupported).",
+        evidence={"entity_coverage": s.entity_coverage, "contradiction": s.contradiction,
+                  "variance": s.variance, "overlap": s.overlap},
+    ).clamp()
+
+
+# --------------------------------------------------------------------------- #
+# 5. Prompt brittleness — Warning | checked 5th
+# --------------------------------------------------------------------------- #
+def detect_prompt_brittleness(s: SignalVector, rec: CaptureRecord, t: Thresholds) -> DetectorResult:
+    gate = (
+        s.similarity >= t.similarity_min
+        and s.entity_coverage >= t.entity_coverage_min
+        and s.contradiction < t.contradiction_min
+    )
+    fired = gate and s.variance > t.variance_min
+    conf = 0.60
+    if rec.temperature is not None and rec.temperature > t.temperature_high:
+        conf += 0.15
+    return DetectorResult(
+        failure=PROMPT_BRITTLENESS,
+        fired=fired,
+        confidence=conf,
+        severity=SEVERITY[PROMPT_BRITTLENESS],
+        root_cause=(
+            f"All grounding signals are healthy but output variance {s.variance:.2f} "
+            "is high — the same prompt yields inconsistent answers."
+        ),
+        fix="Lower the sampling temperature, add an explicit output-format template, "
+        "and insert few-shot examples to pin down the expected response shape.",
+        evidence={"variance": s.variance, "temperature": rec.temperature},
+    ).clamp()
+
+
+# Priority order matters (§5.2): earlier detectors gate later ones.
+DETECTORS = [
+    detect_context_overflow,
+    detect_retrieval_failure,
+    detect_entity_gap,
+    detect_hallucination,
+    detect_prompt_brittleness,
+]

debugai/diagnosis.py

@@ -0,0 +1,64 @@
+"""Diagnosis pipeline (Architecture §5.2 / §7.3).
+
+Runs all five detectors, ranks the ones that fired by confidence, and returns a
+primary diagnosis plus secondary issues. Gate patterns in the detectors prevent
+nonsensical combinations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from debugai.detectors import DETECTORS, DetectorResult
+from debugai.schema import CaptureRecord
+from debugai.signals import SignalVector
+from debugai.thresholds import DEFAULT_THRESHOLDS, Thresholds
+
+
+@dataclass
+class Diagnosis:
+    healthy: bool
+    primary: DetectorResult | None
+    secondary: list[DetectorResult] = field(default_factory=list)
+    signals: SignalVector | None = None
+
+    def to_dict(self) -> dict:
+        def fmt(r: DetectorResult | None) -> dict | None:
+            if r is None:
+                return None
+            return {
+                "failure": r.failure,
+                "confidence": r.confidence,
+                "severity": r.severity,
+                "root_cause": r.root_cause,
+                "fix": r.fix,
+                "evidence": r.evidence,
+            }
+
+        return {
+            "healthy": self.healthy,
+            "primary": fmt(self.primary),
+            "secondary": [fmt(r) for r in self.secondary],
+            "signals": self.signals.to_dict() if self.signals else None,
+        }
+
+
+def diagnose(
+    signals: SignalVector,
+    rec: CaptureRecord,
+    thresholds: Thresholds = DEFAULT_THRESHOLDS,
+) -> Diagnosis:
+    """Classify a signal vector into primary + secondary failures."""
+    results = [detector(signals, rec, thresholds) for detector in DETECTORS]
+    fired = [r for r in results if r.fired]
+    # Rank by confidence; ties broken by detector priority (stable sort order).
+    fired.sort(key=lambda r: r.confidence, reverse=True)
+
+    if not fired:
+        return Diagnosis(healthy=True, primary=None, secondary=[], signals=signals)
+    return Diagnosis(
+        healthy=False,
+        primary=fired[0],
+        secondary=fired[1:],
+        signals=signals,
+    )

debugai/explainer.py

@@ -0,0 +1,105 @@
+"""Layer 3 — LLM Explainer (Architecture §2.2, §8.2).
+
+The ONLY diagnosis-path layer that calls an LLM. It translates the structured,
+deterministic diagnosis into a human-readable explanation + fix, calibrating
+language to confidence and variance type (§2.3 step 5).
+
+Fail-open design: if no API key is configured (or the SDK is missing), we fall
+back to a deterministic template built from the detector's own root_cause / fix
+strings. The deterministic system always has the final say (§8.2).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+
+from debugai.diagnosis import Diagnosis
+
+log = logging.getLogger("debugai.explainer")
+
+# Small, fast, cheap model is right for an advisory explanation layer.
+DEFAULT_MODEL = os.environ.get("DEBUGAI_EXPLAINER_MODEL", "claude-haiku-4-5-20251001")
+
+_SYSTEM = (
+    "You are DebugAI's explanation layer. A deterministic engine has already "
+    "diagnosed why an LLM application's output failed. Your ONLY job is to turn "
+    "the structured diagnosis into a crisp, developer-facing explanation and a "
+    "concrete fix. Rules: (1) Never contradict the diagnosis — it is ground "
+    "truth. (2) Calibrate certainty to the confidence score: state high-"
+    "confidence findings plainly, hedge low-confidence ones. (3) Be specific — "
+    "never say 'add more context'. (4) Keep it under 120 words. Respond as JSON: "
+    '{"explanation": "...", "fix": "..."}.'
+)
+
+
+def _deterministic(diag: Diagnosis) -> dict:
+    if diag.healthy or diag.primary is None:
+        return {
+            "explanation": "No failure detected — all signals are within healthy "
+            "ranges.",
+            "fix": "",
+            "model": "deterministic",
+        }
+    p = diag.primary
+    secondary = ", ".join(r.failure for r in diag.secondary)
+    explanation = p.root_cause
+    if secondary:
+        explanation += f" Secondary issues also detected: {secondary}."
+    return {"explanation": explanation, "fix": p.fix, "model": "deterministic"}
+
+
+def _client():
+    """Return an Anthropic client, or None if unavailable (fail open)."""
+    if not os.environ.get("ANTHROPIC_API_KEY"):
+        return None
+    try:
+        import anthropic
+
+        return anthropic.Anthropic(timeout=30.0, max_retries=2)
+    except Exception as e:  # pragma: no cover - environment dependent
+        log.warning("Anthropic client unavailable (%s); using deterministic explain", e)
+        return None
+
+
+def explain(diag: Diagnosis, model: str = DEFAULT_MODEL) -> dict:
+    """Produce {explanation, fix, model} for a diagnosis."""
+    if diag.healthy or diag.primary is None:
+        return _deterministic(diag)
+
+    client = _client()
+    if client is None:
+        return _deterministic(diag)
+
+    payload = {
+        "primary": {
+            "failure": diag.primary.failure,
+            "confidence": diag.primary.confidence,
+            "severity": diag.primary.severity,
+            "root_cause": diag.primary.root_cause,
+            "deterministic_fix_hint": diag.primary.fix,
+            "evidence": diag.primary.evidence,
+        },
+        "secondary": [
+            {"failure": r.failure, "confidence": r.confidence} for r in diag.secondary
+        ],
+        "signals": diag.signals.to_dict() if diag.signals else {},
+    }
+    try:
+        msg = client.messages.create(
+            model=model,
+            max_tokens=400,
+            system=_SYSTEM,
+            messages=[{"role": "user", "content": json.dumps(payload)}],
+        )
+        text = "".join(b.text for b in msg.content if getattr(b, "type", "") == "text")
+        parsed = json.loads(text)
+        return {
+            "explanation": parsed.get("explanation", diag.primary.root_cause),
+            "fix": parsed.get("fix", diag.primary.fix),
+            "model": model,
+        }
+    except Exception as e:  # pragma: no cover - network dependent
+        log.warning("LLM explain failed (%s); falling back to deterministic", e)
+        return _deterministic(diag)

debugai/integrations/__init__.py

@@ -0,0 +1,5 @@
+"""Framework integrations for DebugAI."""
+
+from debugai.integrations.langchain import DebugAICallbackHandler
+
+__all__ = ["DebugAICallbackHandler"]

debugai/integrations/langchain.py

@@ -0,0 +1,109 @@
+"""LangChain integration — diagnose a RAG/LLM chain automatically.
+
+Attach the callback handler to any LangChain run; it captures the retrieved
+documents and the LLM prompt/output, then runs ``analyze()`` on the result and
+hands the diagnosis to your sink:
+
+    from debugai.integrations import DebugAICallbackHandler
+    handler = DebugAICallbackHandler(on_diagnosis=lambda d: print(d["primary"]))
+    chain.invoke(question, config={"callbacks": [handler]})
+
+Works whether or not ``langchain`` is installed — if the LangChain base class is
+importable we subclass it (so LangChain dispatches the events); otherwise the
+handler is still a usable plain object you can drive directly (and in tests).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable
+
+from debugai.analyze import analyze
+
+log = logging.getLogger("debugai.integrations.langchain")
+
+# Subclass the real base when available so LangChain routes callbacks to us.
+try:  # langchain-core (current)
+    from langchain_core.callbacks import BaseCallbackHandler as _Base
+except Exception:  # pragma: no cover - optional dep
+    try:  # older monolithic langchain
+        from langchain.callbacks.base import BaseCallbackHandler as _Base
+    except Exception:
+        _Base = object
+
+
+def _doc_text(doc: Any) -> str:
+    return getattr(doc, "page_content", None) or (doc.get("page_content", "") if isinstance(doc, dict) else str(doc))
+
+
+def _gen_text(response: Any) -> str:
+    """Pull the generated text out of a LangChain LLMResult (LLM or chat)."""
+    gens = getattr(response, "generations", None)
+    if not gens:
+        return ""
+    first = gens[0][0]
+    text = getattr(first, "text", "") or ""
+    if not text:  # chat models carry it on .message.content
+        msg = getattr(first, "message", None)
+        content = getattr(msg, "content", "")
+        text = content if isinstance(content, str) else " ".join(
+            p.get("text", "") for p in content if isinstance(p, dict)
+        ) if isinstance(content, list) else str(content or "")
+    return text
+
+
+class DebugAICallbackHandler(_Base):
+    """Captures retrieval + generation from a LangChain run and diagnoses it."""
+
+    def __init__(self, on_diagnosis: Callable[[dict], None] | None = None,
+                 system_prompt: str = "", judge: bool = False,
+                 explain_with_llm: bool = False):
+        super().__init__()
+        self._on_diagnosis = on_diagnosis
+        self._system_prompt = system_prompt
+        self._judge = judge
+        self._explain = explain_with_llm
+        self.last: dict | None = None        # most recent diagnosis, for inspection
+        self._prompt: str = ""
+        self._chunks: list[str] = []
+
+    # --- LangChain callback hooks -----------------------------------------
+    def on_retriever_end(self, documents, **kwargs) -> None:
+        try:
+            self._chunks = [_doc_text(d) for d in (documents or [])]
+        except Exception as e:  # never break the chain
+            log.warning("retriever capture failed: %s", e)
+
+    def on_llm_start(self, serialized, prompts, **kwargs) -> None:
+        if prompts:
+            self._prompt = prompts[-1]
+
+    def on_chat_model_start(self, serialized, messages, **kwargs) -> None:
+        # messages: list[list[BaseMessage]]; grab the last human message's text.
+        try:
+            flat = messages[-1] if messages else []
+            for m in reversed(flat):
+                content = getattr(m, "content", "")
+                if content:
+                    self._prompt = content if isinstance(content, str) else str(content)
+                    break
+        except Exception as e:
+            log.warning("chat-start capture failed: %s", e)
+
+    def on_llm_end(self, response, **kwargs) -> None:
+        output = _gen_text(response)
+        if not (self._prompt or output):
+            return
+        try:
+            self.last = analyze(
+                prompt=self._prompt or "(unknown)", output=output,
+                system_prompt=self._system_prompt,
+                chunks=self._chunks or None,
+                explain_with_llm=self._explain, judge=self._judge,
+            )
+            if self._on_diagnosis is not None:
+                self._on_diagnosis(self.last)
+        except Exception as e:  # diagnosis must never break the user's chain
+            log.warning("DebugAI analyze failed: %s", e)
+        finally:
+            self._chunks = []  # reset retrieval for the next call

debugai/judge.py

@@ -0,0 +1,171 @@
+"""Instruction-adherence judge — diagnoses *behavioural* / prompt-following
+failures that the deterministic grounding signals can't see.
+
+Some failures aren't about retrieval or hallucination at all — e.g. a Socratic
+tutor that reveals the answer in the first turn or re-asks the same guiding
+question. These are violations of the system prompt's own rules. We detect them
+with an LLM-as-judge (OpenAI by default), with a deterministic heuristic
+fallback so the feature still works without an API key.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from dataclasses import asdict, dataclass, field
+
+log = logging.getLogger("debugai.judge")
+
+INSTRUCTION_VIOLATION = "instruction_violation"
+DEFAULT_JUDGE_MODEL = os.environ.get("DEBUGAI_JUDGE_MODEL", "gpt-5.5")
+
+_SENT_RE = re.compile(r"[.!?]+")
+_WORD_RE = re.compile(r"[A-Za-z0-9']+")
+
+
+@dataclass
+class Violation:
+    rule: str
+    severity: str = "warning"   # warning | critical
+    evidence: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class InstructionDiagnosis:
+    healthy: bool
+    confidence: float
+    violations: list[Violation] = field(default_factory=list)
+    model: str = "heuristic"
+
+    def to_dict(self) -> dict:
+        return {
+            "healthy": self.healthy,
+            "confidence": self.confidence,
+            "model": self.model,
+            "violations": [v.to_dict() for v in self.violations],
+        }
+
+
+# --------------------------------------------------------------------------- #
+# Public entry point
+# --------------------------------------------------------------------------- #
+def judge_instructions(system_prompt: str, user_prompt: str, output: str,
+                       model: str | None = None) -> InstructionDiagnosis:
+    """Evaluate an assistant ``output`` against the rules in its ``system_prompt``.
+
+    Uses the OpenAI judge when ``OPENAI_API_KEY`` is set; otherwise a deterministic
+    heuristic check. Returns the violations found (empty → healthy)."""
+    if not (system_prompt or "").strip():
+        return InstructionDiagnosis(healthy=True, confidence=0.0, model="n/a")
+    model = model or DEFAULT_JUDGE_MODEL
+    if os.environ.get("OPENAI_API_KEY"):
+        try:
+            return _openai_judge(system_prompt, user_prompt, output, model)
+        except Exception as e:  # pragma: no cover - network dependent
+            log.warning("OpenAI judge failed (%s); using heuristic fallback", e)
+    return _heuristic_judge(system_prompt, user_prompt, output)
+
+
+# --------------------------------------------------------------------------- #
+# LLM-as-judge (OpenAI)
+# --------------------------------------------------------------------------- #
+_JUDGE_SYSTEM = (
+    "You are a strict evaluator of an AI assistant's adherence to its own system "
+    "prompt. You are given the assistant's SYSTEM PROMPT (which contains its rules) "
+    "and the assistant's RESPONSE to a student. Identify every rule the response "
+    "violates — especially pedagogy rules such as revealing too much of the answer "
+    "early, asking more than one question, asking a question that merely restates a "
+    "previous one, or paraphrasing the student. Respond ONLY as JSON: "
+    '{"violations": [{"rule": "<short rule description>", "severity": '
+    '"critical|warning", "evidence": "<quote/why>"}], "confidence": <0..1>}. '
+    "If the response fully complies, return an empty violations list."
+)
+
+
+def _openai_judge(system_prompt: str, user_prompt: str, output: str,
+                  model: str) -> InstructionDiagnosis:
+    from openai import OpenAI
+
+    client = OpenAI(timeout=30.0, max_retries=2)
+    payload = (
+        f"SYSTEM PROMPT (rules):\n{system_prompt}\n\n"
+        f"STUDENT MESSAGE:\n{user_prompt}\n\n"
+        f"ASSISTANT RESPONSE:\n{output}"
+    )
+    resp = client.chat.completions.create(
+        model=model,
+        response_format={"type": "json_object"},
+        messages=[{"role": "system", "content": _JUDGE_SYSTEM},
+                  {"role": "user", "content": payload}],
+    )
+    data = json.loads(resp.choices[0].message.content or "{}")
+    violations = [
+        Violation(rule=v.get("rule", "rule"), severity=v.get("severity", "warning"),
+                  evidence=v.get("evidence", ""))
+        for v in data.get("violations", [])
+    ]
+    conf = float(data.get("confidence", 0.8 if violations else 0.0))
+    return InstructionDiagnosis(
+        healthy=not violations, confidence=round(conf, 4),
+        violations=violations, model=f"openai:{model}",
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Deterministic heuristic fallback (no API key)
+# --------------------------------------------------------------------------- #
+def _sentences(text: str) -> list[str]:
+    return [s.strip() for s in _SENT_RE.split(text or "") if s.strip()]
+
+
+def _jaccard(a: str, b: str) -> float:
+    sa = {w.lower() for w in _WORD_RE.findall(a or "")}
+    sb = {w.lower() for w in _WORD_RE.findall(b or "")}
+    return len(sa & sb) / len(sa | sb) if (sa or sb) else 0.0
+
+
+def _heuristic_judge(system_prompt: str, user_prompt: str, output: str) -> InstructionDiagnosis:
+    """Catches the most common Socratic-tutor violations without an LLM.
+
+    These map to typical rules: exactly one question, don't reveal the answer
+    early, don't open by paraphrasing the student."""
+    sysl = (system_prompt or "").lower()
+    violations: list[Violation] = []
+    qn = (output or "").count("?")
+    words = _WORD_RE.findall(output or "")
+
+    # Rule: exactly one leading question.
+    if "one question" in sysl or "socratic" in sysl or "leading question" in sysl:
+        if qn == 0:
+            violations.append(Violation("No leading question — the turn should advance with one question.",
+                                        "critical", "0 question marks in the response."))
+        elif qn > 1:
+            violations.append(Violation("More than one question in a single turn.",
+                                        "warning", f"{qn} question marks found."))
+
+    # Rule: don't reveal the full solution in the first response (length/declarative heuristic).
+    if "socratic" in sysl or "do not give away" in sysl or "not give away" in sysl or "leading question" in sysl:
+        declarative = [s for s in _sentences(output) if "?" not in s]
+        decl_words = sum(len(_WORD_RE.findall(s)) for s in declarative)
+        if len(words) > 90 or decl_words > 70:
+            violations.append(Violation("Reveals too much — long explanation given before the student reasons.",
+                                        "critical", f"{decl_words} words of explanation before the question."))
+
+    # Rule: never open by paraphrasing the student.
+    first = _sentences(output)[0] if _sentences(output) else ""
+    if first and _jaccard(first, user_prompt) > 0.5:
+        violations.append(Violation("Opens by paraphrasing the student's message.",
+                                    "warning", "First sentence closely mirrors the student input."))
+
+    # Confidence scales with count + severity.
+    if not violations:
+        return InstructionDiagnosis(healthy=True, confidence=0.0, model="heuristic")
+    crit = sum(1 for v in violations if v.severity == "critical")
+    conf = min(0.6 + 0.15 * len(violations) + 0.1 * crit, 0.95)
+    return InstructionDiagnosis(healthy=False, confidence=round(conf, 4),
+                                violations=violations, model="heuristic")