counterfix 0.0.1__py3-none-any.whl

counterfix/__init__.py

@@ -0,0 +1,41 @@
+"""counterfix: find why an LLM call failed and get a fix that was tested to work."""
+
+from counterfix.adapters import GuardedClient, get_diagnosis, guard, record_from_openai
+from counterfix.core import (
+    CallRecord,
+    Config,
+    Detector,
+    Diagnosis,
+    Evidence,
+    FailureType,
+    Finding,
+    Message,
+    Prescription,
+    PrescriptionKind,
+    SkipCheck,
+    Usage,
+    run_detectors,
+)
+
+__version__ = "0.0.1"
+
+__all__ = [
+    "CallRecord",
+    "Config",
+    "Detector",
+    "Diagnosis",
+    "Evidence",
+    "FailureType",
+    "Finding",
+    "GuardedClient",
+    "Message",
+    "Prescription",
+    "PrescriptionKind",
+    "SkipCheck",
+    "Usage",
+    "__version__",
+    "get_diagnosis",
+    "guard",
+    "record_from_openai",
+    "run_detectors",
+]

counterfix/adapters/__init__.py

@@ -0,0 +1,6 @@
+"""Adapters that plug counterfix into LLM clients and frameworks."""
+
+from counterfix.adapters._attach import get_diagnosis
+from counterfix.adapters.openai_compat import GuardedClient, guard, record_from_openai
+
+__all__ = ["GuardedClient", "get_diagnosis", "guard", "record_from_openai"]

counterfix/adapters/_attach.py

@@ -0,0 +1,40 @@
+"""Attaching a Diagnosis to a provider's response object without changing its type."""
+
+from __future__ import annotations
+
+from collections import OrderedDict
+from typing import Any
+
+from counterfix.core.diagnosis import Diagnosis
+
+ATTR = "diagnosis"
+_FALLBACK_LIMIT = 2048
+# Objects that refuse new attributes (slots, frozen models) are tracked here by id,
+# together with the object itself so the id cannot be reused while it is stored.
+_fallback: OrderedDict[int, tuple[Any, Diagnosis]] = OrderedDict()
+
+
+def attach(response: Any, diagnosis: Diagnosis) -> None:
+    """Attach ``diagnosis`` as ``response.diagnosis`` when possible, else remember it."""
+    for setter in (setattr, object.__setattr__):
+        try:
+            setter(response, ATTR, diagnosis)
+            if getattr(response, ATTR, None) is diagnosis:
+                return
+        except Exception:
+            continue
+    _fallback[id(response)] = (response, diagnosis)
+    _fallback.move_to_end(id(response))
+    while len(_fallback) > _FALLBACK_LIMIT:
+        _fallback.popitem(last=False)
+
+
+def get_diagnosis(response: Any) -> Diagnosis | None:
+    """Return the Diagnosis counterfix attached to ``response``, or None."""
+    value = getattr(response, ATTR, None)
+    if isinstance(value, Diagnosis):
+        return value
+    stored = _fallback.get(id(response))
+    if stored is not None and stored[0] is response:
+        return stored[1]
+    return None

counterfix/adapters/openai_compat.py

@@ -0,0 +1,206 @@
+"""``guard()`` for OpenAI-compatible clients.
+
+Works with the official ``openai`` SDK and with any provider that exposes the same
+``client.chat.completions.create(...)`` shape: OpenAI, Groq, Ollama, vLLM, Together,
+OpenRouter and others. The wrapped client keeps the same API; each response gains a
+``.diagnosis`` attribute (or use :func:`counterfix.get_diagnosis`).
+"""
+
+from __future__ import annotations
+
+import inspect
+import logging
+import time
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from counterfix.adapters._attach import attach
+from counterfix.core.config import Config
+from counterfix.core.detector import Detector, run_detectors
+from counterfix.core.diagnosis import Diagnosis
+from counterfix.core.record import CallRecord, Message, Usage
+
+logger = logging.getLogger("counterfix")
+
+_CREATE_PATH = ("chat", "completions", "create")
+
+
+def _get(obj: Any, key: str, default: Any = None) -> Any:
+    """Read ``key`` from a dict or an object attribute."""
+    if obj is None:
+        return default
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    return getattr(obj, key, default)
+
+
+def _to_dict(obj: Any) -> dict[str, Any]:
+    """Best-effort conversion of SDK objects to plain dicts."""
+    if isinstance(obj, dict):
+        return dict(obj)
+    dump = getattr(obj, "model_dump", None)
+    if callable(dump):
+        try:
+            result = dump()
+            if isinstance(result, dict):
+                return result
+        except Exception:
+            pass
+    try:
+        return dict(vars(obj))
+    except TypeError:
+        return {"value": repr(obj)}
+
+
+def _usage(raw: Any) -> Usage:
+    """Token usage, including reasoning tokens when the provider reports them."""
+    usage = Usage.model_validate(_to_dict(raw))
+    details = _get(raw, "completion_tokens_details")
+    reasoning_tokens = _get(details, "reasoning_tokens")
+    if isinstance(reasoning_tokens, int):
+        usage.reasoning_tokens = reasoning_tokens
+    return usage
+
+
+def record_from_openai(
+    request: dict[str, Any],
+    response: Any,
+    *,
+    latency_ms: float | None = None,
+    provider: str | None = None,
+) -> CallRecord:
+    """Build a :class:`CallRecord` from chat-completions kwargs and the response."""
+    messages = [Message.model_validate(_to_dict(m)) for m in request.get("messages") or []]
+    params = {k: v for k, v in request.items() if k not in ("messages", "model")}
+
+    choices = _get(response, "choices") or []
+    first = choices[0] if len(choices) > 0 else None
+    message = _get(first, "message")
+    raw_tool_calls = _get(message, "tool_calls")
+    usage_raw = _get(response, "usage")
+    usage = _usage(usage_raw) if usage_raw is not None else None
+    # Reasoning models: Groq/OpenAI-style "reasoning", vLLM/DeepSeek-style "reasoning_content".
+    reasoning = _get(message, "reasoning") or _get(message, "reasoning_content")
+
+    return CallRecord(
+        provider=provider,
+        model=_get(response, "model") or request.get("model"),
+        messages=messages,
+        params=params,
+        response_text=_get(message, "content"),
+        reasoning_text=reasoning if isinstance(reasoning, str) else None,
+        finish_reason=_get(first, "finish_reason"),
+        tool_calls=[_to_dict(t) for t in raw_tool_calls] if raw_tool_calls else None,
+        usage=usage,
+        latency_ms=latency_ms,
+        metadata={"n_choices": len(choices)},
+    )
+
+
+class _Guard:
+    """Shared state for one guarded client."""
+
+    def __init__(self, config: Config, detectors: Sequence[Detector], provider: str | None):
+        self.config = config
+        self.detectors = list(detectors)
+        self.provider = provider
+
+    def diagnose(self, request: dict[str, Any], response: Any, latency_ms: float) -> None:
+        """Build the record, run detectors, attach the result. Never raises if fail_open."""
+        try:
+            record = record_from_openai(
+                request, response, latency_ms=latency_ms, provider=self.provider
+            )
+            diagnosis: Diagnosis = run_detectors(record, self.detectors, self.config)
+            attach(response, diagnosis)
+            if self.config.on_diagnosis is not None:
+                self.config.on_diagnosis(record, diagnosis)
+        except Exception as exc:
+            if not self.config.fail_open:
+                raise
+            logger.warning("counterfix could not diagnose this call: %s", exc)
+
+    def wrap_create(self, create: Callable[..., Any]) -> Callable[..., Any]:
+        guard = self
+
+        def guarded_create(*args: Any, **kwargs: Any) -> Any:
+            if not guard.config.enabled:
+                return create(*args, **kwargs)
+            if kwargs.get("stream"):
+                # Streams are passed through untouched for now; buffering comes later.
+                return create(*args, **kwargs)
+            start = time.perf_counter()
+            result = create(*args, **kwargs)  # the user's own errors propagate unchanged
+            if inspect.isawaitable(result):
+                return guard._finish_async(result, kwargs, start)
+            guard.diagnose(kwargs, result, (time.perf_counter() - start) * 1000)
+            return result
+
+        guarded_create.__doc__ = create.__doc__
+        guarded_create.__wrapped__ = create  # type: ignore[attr-defined]
+        return guarded_create
+
+    async def _finish_async(self, pending: Any, kwargs: dict[str, Any], start: float) -> Any:
+        result = await pending
+        self.diagnose(kwargs, result, (time.perf_counter() - start) * 1000)
+        return result
+
+
+class GuardedClient:
+    """A transparent proxy: everything behaves like the wrapped client, except that
+    ``chat.completions.create`` also produces a diagnosis."""
+
+    __slots__ = ("_cf_guard", "_cf_path", "_cf_target")
+
+    def __init__(self, target: Any, guard: _Guard, path: tuple[str, ...] = ()):
+        object.__setattr__(self, "_cf_target", target)
+        object.__setattr__(self, "_cf_guard", guard)
+        object.__setattr__(self, "_cf_path", path)
+
+    def __getattr__(self, name: str) -> Any:
+        target = object.__getattribute__(self, "_cf_target")
+        guard = object.__getattribute__(self, "_cf_guard")
+        path = (*object.__getattribute__(self, "_cf_path"), name)
+        value = getattr(target, name)
+        if path == _CREATE_PATH and callable(value):
+            return guard.wrap_create(value)
+        if path == _CREATE_PATH[: len(path)]:
+            return GuardedClient(value, guard, path)
+        return value
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        setattr(object.__getattribute__(self, "_cf_target"), name, value)
+
+    def __repr__(self) -> str:
+        return f"counterfix.guard({object.__getattribute__(self, '_cf_target')!r})"
+
+    @property
+    def unwrapped(self) -> Any:
+        """The original client."""
+        return object.__getattribute__(self, "_cf_target")
+
+
+def guard(
+    client: Any,
+    *,
+    config: Config | None = None,
+    detectors: Sequence[Detector] | None = None,
+    provider: str | None = None,
+) -> Any:
+    """Wrap an OpenAI-compatible client so every chat completion is checked.
+
+    >>> from openai import OpenAI          # doctest: +SKIP
+    >>> client = guard(OpenAI())           # doctest: +SKIP
+    >>> r = client.chat.completions.create(model="gpt-4o-mini", messages=[...])  # doctest: +SKIP
+    >>> r.diagnosis.summary()              # doctest: +SKIP
+
+    ``detectors`` defaults to the built-in live detectors. ``provider`` is an optional
+    label stored on each record (e.g. "groq", "ollama").
+    """
+    if isinstance(client, GuardedClient):
+        return client
+    if detectors is None:
+        from counterfix.detectors import default_detectors
+
+        detectors = default_detectors()
+    return GuardedClient(client, _Guard(config or Config(), detectors, provider))

counterfix/core/__init__.py

@@ -0,0 +1,23 @@
+"""Core, provider-free building blocks."""
+
+from counterfix.core.config import Config
+from counterfix.core.detector import Detector, SkipCheck, run_detectors
+from counterfix.core.diagnosis import Diagnosis, Evidence, Finding, Prescription
+from counterfix.core.record import CallRecord, Message, Usage
+from counterfix.core.types import FailureType, PrescriptionKind
+
+__all__ = [
+    "CallRecord",
+    "Config",
+    "Detector",
+    "Diagnosis",
+    "Evidence",
+    "FailureType",
+    "Finding",
+    "Message",
+    "Prescription",
+    "PrescriptionKind",
+    "SkipCheck",
+    "Usage",
+    "run_detectors",
+]

counterfix/core/config.py

@@ -0,0 +1,41 @@
+"""Runtime settings for counterfix."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from counterfix.core.diagnosis import Diagnosis
+from counterfix.core.record import CallRecord
+
+DiagnosisHook = Callable[[CallRecord, Diagnosis], None]
+
+
+class Config(BaseModel):
+    """Settings shared by the live guard and the diagnosis engine.
+
+    The defaults are deliberately safe: live mode only flags failures, never retries,
+    makes no extra LLM calls, and never lets an internal error reach the host app.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    mode: Literal["flag", "retry"] = "flag"
+    """'flag' (default) attaches a diagnosis only; 'retry' may re-run with a known fix."""
+
+    enabled: bool = True
+    """Set False to turn counterfix off without removing the wrapper."""
+
+    fail_open: bool = True
+    """If True, internal errors are logged and recorded, never raised."""
+
+    max_extra_calls: int = Field(default=0, ge=0)
+    """Extra LLM calls allowed per live call. 0 means live mode never calls a model."""
+
+    judge_model: str | None = None
+    """Model used by LLM-judge detectors (diagnose mode). None until configured."""
+
+    on_diagnosis: DiagnosisHook | None = None
+    """Optional callback run after every diagnosis, e.g. to log or store it."""

counterfix/core/detector.py

@@ -0,0 +1,61 @@
+"""The detector interface and the runner that applies detectors to a call."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterable, Sequence
+from typing import Protocol, runtime_checkable
+
+from counterfix.core.config import Config
+from counterfix.core.diagnosis import Diagnosis, Finding
+from counterfix.core.record import CallRecord
+from counterfix.core.types import FailureType
+
+logger = logging.getLogger("counterfix")
+
+
+class SkipCheck(Exception):
+    """Raised by a detector that cannot run on this call (e.g. missing capability)."""
+
+
+@runtime_checkable
+class Detector(Protocol):
+    """Anything with a name, the failure types it covers, and a ``check`` method.
+
+    ``check`` returns the findings for one call (an empty list when the call looks
+    fine) or raises :class:`SkipCheck` when it cannot judge this call.
+    """
+
+    name: str
+    covers: tuple[FailureType, ...]
+
+    def check(self, record: CallRecord, config: Config) -> list[Finding]: ...
+
+
+def run_detectors(
+    record: CallRecord,
+    detectors: Iterable[Detector],
+    config: Config,
+) -> Diagnosis:
+    """Run each detector on the record and collect a :class:`Diagnosis`.
+
+    A detector that raises is recorded under ``errors`` and the others still run,
+    so one broken check never hides the rest or breaks the host application.
+    """
+    diagnosis = Diagnosis(record_id=record.id)
+    for detector in detectors:
+        name = getattr(detector, "name", type(detector).__name__)
+        try:
+            findings: Sequence[Finding] = detector.check(record, config)
+        except SkipCheck as skip:
+            diagnosis.skipped[name] = str(skip) or "not applicable"
+            continue
+        except Exception as exc:
+            if not config.fail_open:
+                raise
+            logger.warning("counterfix detector %s failed: %s", name, exc)
+            diagnosis.errors[name] = f"{type(exc).__name__}: {exc}"
+            continue
+        diagnosis.checked.append(name)
+        diagnosis.findings.extend(findings)
+    return diagnosis

counterfix/core/diagnosis.py

@@ -0,0 +1,74 @@
+"""What counterfix returns: findings, their evidence, and prescriptions."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from counterfix.core.types import FailureType, PrescriptionKind
+
+
+class Evidence(BaseModel):
+    """Why a detector believes a failure happened."""
+
+    detector: str
+    detail: str
+    data: dict[str, Any] = Field(default_factory=dict)
+
+
+class Prescription(BaseModel):
+    """A suggested change. ``verified`` is True only after a counterfactual re-run passed."""
+
+    kind: PrescriptionKind
+    description: str
+    patch: dict[str, Any] = Field(default_factory=dict)
+    verified: bool = False
+
+
+class Finding(BaseModel):
+    """One suspected failure in a call."""
+
+    failure_type: FailureType
+    confidence: float = Field(ge=0.0, le=1.0)
+    evidence: list[Evidence] = Field(default_factory=list)
+    prescription: Prescription | None = None
+    inferred: bool = False  # True when the signal was indirect (e.g. no stop reason)
+
+
+class Diagnosis(BaseModel):
+    """The result of checking one call.
+
+    ``checked`` lists the detectors that ran, ``skipped`` those that could not run
+    (with the reason), and ``errors`` any internal errors. Internal errors never
+    reach the user's application; they are recorded here instead.
+    """
+
+    record_id: str
+    findings: list[Finding] = Field(default_factory=list)
+    checked: list[str] = Field(default_factory=list)
+    skipped: dict[str, str] = Field(default_factory=dict)
+    errors: dict[str, str] = Field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        """True when no failure was found."""
+        return not self.findings
+
+    @property
+    def top(self) -> Finding | None:
+        """The most confident finding, or None."""
+        if not self.findings:
+            return None
+        return max(self.findings, key=lambda f: f.confidence)
+
+    def summary(self) -> str:
+        """A one-line human summary."""
+        if self.ok:
+            note = f" ({len(self.errors)} internal errors)" if self.errors else ""
+            return f"no failure found by {len(self.checked)} checks{note}"
+        parts = [
+            f"{f.failure_type.value} {f.failure_type.label} ({f.confidence:.0%})"
+            for f in sorted(self.findings, key=lambda f: -f.confidence)
+        ]
+        return "; ".join(parts)

counterfix/core/record.py

@@ -0,0 +1,119 @@
+"""The provider-neutral record of one LLM call.
+
+Everything inside counterfix works on a :class:`CallRecord`. Adapters convert each
+provider's request and response into one, so the core never imports a provider SDK.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class Message(BaseModel):
+    """One chat message. ``content`` may be text, a list of parts, or None."""
+
+    model_config = ConfigDict(extra="allow")
+
+    role: str
+    content: Any = None
+
+    @property
+    def text(self) -> str:
+        """The message's text, joining text parts when content is a list."""
+        if self.content is None:
+            return ""
+        if isinstance(self.content, str):
+            return self.content
+        if isinstance(self.content, list):
+            parts: list[str] = []
+            for part in self.content:
+                if isinstance(part, str):
+                    parts.append(part)
+                elif isinstance(part, dict) and isinstance(part.get("text"), str):
+                    parts.append(part["text"])
+            return "\n".join(parts)
+        return str(self.content)
+
+
+class Usage(BaseModel):
+    """Token counts, when the provider returns them."""
+
+    model_config = ConfigDict(extra="allow")
+
+    prompt_tokens: int | None = None
+    completion_tokens: int | None = None
+    total_tokens: int | None = None
+    reasoning_tokens: int | None = None
+
+
+class CallRecord(BaseModel):
+    """A single LLM call: what was asked, with which settings, and what came back."""
+
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+    provider: str | None = None
+    model: str | None = None
+    messages: list[Message] = Field(default_factory=list)
+    params: dict[str, Any] = Field(default_factory=dict)
+
+    response_text: str | None = None
+    reasoning_text: str | None = None
+    """Hidden "thinking" returned by reasoning models, when the provider exposes it."""
+    finish_reason: str | None = None
+    tool_calls: list[dict[str, Any]] | None = None
+    usage: Usage | None = None
+    latency_ms: float | None = None
+
+    error: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def system_text(self) -> str:
+        """All system messages joined, or an empty string."""
+        return "\n".join(m.text for m in self.messages if m.role in ("system", "developer"))
+
+    @property
+    def prompt_text(self) -> str:
+        """The text of the last user message, or an empty string."""
+        for message in reversed(self.messages):
+            if message.role == "user":
+                return message.text
+        return ""
+
+    @property
+    def answer_text(self) -> str:
+        """The visible answer, with inline reasoning removed.
+
+        Handles the tag styles models use for inline thinking: <think> (Qwen, DeepSeek),
+        <thought> (Gemma) and <thinking>. An unclosed tag (cut off mid-thought) removes
+        everything after it.
+        """
+        return _THINK.sub("", self.response_text or "").strip()
+
+    @property
+    def has_reasoning(self) -> bool:
+        """True when the model spent tokens on reasoning (separate field or inline tags)."""
+        return bool(
+            self.reasoning_text
+            or (self.usage is not None and (self.usage.reasoning_tokens or 0) > 0)
+            or _THINK_OPEN.search(self.response_text or "")
+        )
+
+    @property
+    def max_tokens(self) -> int | None:
+        """The output-token limit the call used, if one was set."""
+        for key in ("max_tokens", "max_completion_tokens", "max_output_tokens"):
+            value = self.params.get(key)
+            if isinstance(value, int):
+                return value
+        return None
+
+
+_TAGS = r"(think|thought|thinking)"
+_THINK = re.compile(rf"<{_TAGS}>.*?(</\1>|$)", re.DOTALL | re.IGNORECASE)
+_THINK_OPEN = re.compile(rf"<{_TAGS}>", re.IGNORECASE)

counterfix/core/types.py

@@ -0,0 +1,66 @@
+"""Failure types and small shared enums."""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class FailureType(str, Enum):
+    """The failure taxonomy (v1: F1 to F9) plus non-model categories.
+
+    ``INFRASTRUCTURE``, ``PROVIDER_FILTER`` and ``EMPTY_OUTPUT`` are kept apart from
+    the model failure types so that API errors or provider filtering are never
+    counted as the model's fault.
+    """
+
+    TRUNCATION = "F1"
+    FORMAT_BREAK = "F2"
+    OVER_REFUSAL = "F3"
+    HALLUCINATION = "F4"
+    FALSE_PREMISE = "F5"
+    DROPPED_INSTRUCTION = "F6"
+    PROMPT_FRAGILITY = "F7"
+    COMPUTATION = "F8"
+    AMBIGUITY = "F9"
+
+    INFRASTRUCTURE = "INFRA"
+    PROVIDER_FILTER = "FILTER"
+    EMPTY_OUTPUT = "EMPTY"
+    UNCLASSIFIED = "UNCLASSIFIED"
+
+    @property
+    def is_model_failure(self) -> bool:
+        """True for F1 to F9, the failures that belong to the model's behaviour."""
+        return self.value.startswith("F") and self.value[1:].isdigit()
+
+    @property
+    def label(self) -> str:
+        return _LABELS[self]
+
+
+_LABELS: dict[FailureType, str] = {
+    FailureType.TRUNCATION: "Truncation",
+    FailureType.FORMAT_BREAK: "Format break (JSON, schema)",
+    FailureType.OVER_REFUSAL: "Over-refusal",
+    FailureType.HALLUCINATION: "Hallucination from missing knowledge",
+    FailureType.FALSE_PREMISE: "False-premise acceptance",
+    FailureType.DROPPED_INSTRUCTION: "Dropped instruction",
+    FailureType.PROMPT_FRAGILITY: "Prompt fragility",
+    FailureType.COMPUTATION: "Computation or token-level error",
+    FailureType.AMBIGUITY: "Ambiguous request",
+    FailureType.INFRASTRUCTURE: "Infrastructure error (not a model failure)",
+    FailureType.PROVIDER_FILTER: "Provider content filter (not a model failure)",
+    FailureType.EMPTY_OUTPUT: "Empty output",
+    FailureType.UNCLASSIFIED: "Unclassified failure",
+}
+
+
+class PrescriptionKind(str, Enum):
+    """What kind of change a prescription asks the user to make."""
+
+    PARAM_CHANGE = "param_change"  # e.g. raise max_tokens, enable JSON mode
+    PROMPT_CHANGE = "prompt_change"  # e.g. restructure, add "check premises"
+    ADD_CONTEXT = "add_context"  # e.g. add retrieval
+    USE_TOOL = "use_tool"  # e.g. route arithmetic to a code tool
+    ASK_USER = "ask_user"  # e.g. ask a clarifying question (F9)
+    ESCALATE = "escalate"  # e.g. larger model, abstain; no prompt-level fix

counterfix/detectors/__init__.py

@@ -0,0 +1,17 @@
+"""Built-in detectors.
+
+Phase 1 ships the detector interface only. The cheap live detectors (F1 truncation,
+F2 format, F3 refusal, plus empty output and provider filtering) arrive in Phase 2.
+"""
+
+from __future__ import annotations
+
+from counterfix.core.detector import Detector
+
+
+def default_detectors() -> list[Detector]:
+    """The detectors ``guard()`` uses when none are passed. Empty until Phase 2."""
+    return []
+
+
+__all__ = ["default_detectors"]

counterfix-0.0.1.dist-info/METADATA

@@ -0,0 +1,143 @@
+Metadata-Version: 2.5
+Name: counterfix
+Version: 0.0.1
+Summary: Find why an LLM call failed and get a fix that was tested to work.
+Project-URL: Homepage, https://github.com/KANSADWALA/CounterFix
+Project-URL: Issues, https://github.com/KANSADWALA/CounterFix/issues
+Author: Shubham Kansadwala
+License-Expression: MIT
+License-File: LICENSE
+Keywords: counterfactual,debugging,evaluation,hallucination,llm,reliability
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: json-repair>=0.30
+Requires-Dist: jsonschema>=4.20
+Requires-Dist: pydantic>=2.7
+Requires-Dist: tenacity>=8.2
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40; extra == 'anthropic'
+Provides-Extra: cache
+Requires-Dist: diskcache>=5.6; extra == 'cache'
+Provides-Extra: cli
+Requires-Dist: rich>=13.7; extra == 'cli'
+Requires-Dist: typer>=0.12; extra == 'cli'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest-recording>=0.13; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: jev
+Requires-Dist: typesafe-sdk>=0.7; extra == 'jev'
+Provides-Extra: langchain
+Requires-Dist: langchain-core>=0.3; extra == 'langchain'
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.50; extra == 'litellm'
+Provides-Extra: llamaindex
+Requires-Dist: llama-index-core>=0.11; extra == 'llamaindex'
+Provides-Extra: openai
+Requires-Dist: openai>=1.40; extra == 'openai'
+Provides-Extra: otel
+Requires-Dist: opentelemetry-sdk>=1.27; extra == 'otel'
+Provides-Extra: report
+Requires-Dist: jinja2>=3.1; extra == 'report'
+Provides-Extra: semantic
+Requires-Dist: torch>=2.2; extra == 'semantic'
+Requires-Dist: transformers>=4.44; extra == 'semantic'
+Description-Content-Type: text/markdown
+
+# counterfix
+
+**Find why an LLM call failed, and get a fix that was tested to work.**
+
+Most LLM evaluation tools tell you *that* a model failed. counterfix is built to tell you
+*why*, and to hand back a fix it has already checked. When a call fails, it changes one
+thing at a time (adds context, removes a false premise, rewords the prompt, gives the
+model a tool, raises `max_tokens`) and re-runs the call. The change that fixes the
+failure is both the diagnosis and the prescription. We call this **counterfactual
+diagnosis**.
+
+> **Status: pre-alpha (v0.0.1).** The core records, the `guard()` wrapper and the seed
+> failure set are in place. Detectors and the counterfactual engine are being built.
+> APIs will change.
+
+## What it plugs into
+
+counterfix is middleware: it sits between your code and any text-in, text-out LLM.
+
+```python
+from openai import OpenAI
+from counterfix import guard
+
+client = guard(OpenAI())  # same API as before
+response = client.chat.completions.create(
+    model="gpt-4o-mini",
+    messages=[{"role": "user", "content": "Why did Einstein win his third Nobel Prize?"}],
+)
+print(response.choices[0].message.content)
+print(response.diagnosis.summary())  # what counterfix found
+```
+
+The same wrapper works with any OpenAI-compatible endpoint, for example Groq or a local
+Ollama server:
+
+```python
+client = guard(OpenAI(base_url="http://localhost:11434/v1", api_key="ollama"))
+```
+
+Design rules:
+
+- **Fail open.** If counterfix itself errors, your call still returns normally.
+- **Your errors stay yours.** Exceptions from the provider reach your code unchanged.
+- **Safe defaults.** Live mode only flags failures and makes no extra LLM calls.
+- **Light core.** The core depends on 4 small libraries; providers and heavy tools are extras.
+
+## Failure types (v1)
+
+| Code | Failure |
+| --- | --- |
+| F1 | Truncation |
+| F2 | Format break (JSON, schema) |
+| F3 | Over-refusal |
+| F4 | Hallucination from missing knowledge |
+| F5 | False-premise acceptance |
+| F6 | Dropped instruction |
+| F7 | Prompt fragility |
+| F8 | Computation or token-level error |
+| F9 | Ambiguous request |
+
+API errors, rate limits and provider content filters are tracked separately, so they are
+never counted as the model's fault.
+
+## Install (development)
+
+```bash
+git clone https://github.com/KANSADWALA/CounterFix.git
+cd CounterFix
+pip install -e ".[dev]"
+pytest
+```
+
+Optional extras: `openai`, `anthropic`, `litellm`, `langchain`, `llamaindex`, `otel`,
+`semantic`, `cache`, `cli`, `report`, `jev`.
+
+## Roadmap
+
+1. Foundations and seed failure set ✅
+2. Core records and the `guard()` plug ✅
+3. Cheap live detectors (F1 to F3) and the prompt linter
+4. LLM-judge and sampling detectors
+5. Diagnose mode and HTML report
+6. Counterfactual engine with verified fixes
+7. Model failure profiles
+8. More plugs: decorator, Anthropic, LiteLLM, LangChain, OpenTelemetry
+9. Evaluation on public benchmarks
+10. Release and paper
+
+## License
+
+MIT

counterfix-0.0.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+counterfix/__init__.py,sha256=WRIy8CRT8dGLfCV5sC9Mp8JtFjVAqd_SJrq8Zbztqok,768
+counterfix/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+counterfix/adapters/__init__.py,sha256=v1HMre8Ma3K0C8WhzVd623tjCo0_4Cbytz8KPkLp7_o,288
+counterfix/adapters/_attach.py,sha256=Y5aBsrid0ki6Zpby-eKRVrcuyJfgAFAJnZXTWkDxAxQ,1435
+counterfix/adapters/openai_compat.py,sha256=aWFZuV702dI-gF_pFr8tpKc9f6td2Up2SpVT8IC7ge0,7869
+counterfix/core/__init__.py,sha256=ygkjZBEiTJxqtkaTuwc07l8D3Qu-gpJ4YPiaJU_U4RI,605
+counterfix/core/config.py,sha256=1jENGUwsL9V8K99P1f7Imx1uNuSTRqQgx2I3i-js85w,1399
+counterfix/core/detector.py,sha256=BydbM_8oGgmlCduN3RtSrAcUyDENSmWjqUBrHUxpBgs,2067
+counterfix/core/diagnosis.py,sha256=R2Rwn60ci2JtJogbxIGOzg-bsRVj3c--3c1tQLAFOl0,2367
+counterfix/core/record.py,sha256=0suFx5r53CFn6Wwpyb4hhopSL1ABzCS2p2SO_eVpIck,3989
+counterfix/core/types.py,sha256=DHgJOvMzoA40c8tCgMX2YTWTM9KQrzr3quLLnDvmmzQ,2369
+counterfix/detectors/__init__.py,sha256=SAjOJ2h0ad6h61DC7VdXb0i7NJk3wfelRw5x096YdRc,456
+counterfix-0.0.1.dist-info/METADATA,sha256=8wUSncTn5lAz2IN4r5lOe8RC6SL9FeIgDKno_XiTyJc,4965
+counterfix-0.0.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+counterfix-0.0.1.dist-info/licenses/LICENSE,sha256=85-Dud9Oh3wzc3TUrRb0OAG4sHuVYYmm9SL5jIP4Xos,1075
+counterfix-0.0.1.dist-info/RECORD,,

counterfix-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

counterfix-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shubham Kansadwala
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.