agentrec 0.2.0__py3-none-any.whl

agentrec/__init__.py

@@ -0,0 +1,94 @@
+from importlib.metadata import PackageNotFoundError, version as _version
+
+from .capture import CapturedChunk, CapturedInteraction, CapturedRequest
+from .comparators import (
+    Comparator,
+    ComparisonResult,
+    EmbeddingComparator,
+    ExactMatchComparator,
+    FuzzyComparator,
+    JudgeComparator,
+    build_comparators,
+)
+from .keying import Fingerprint, default_key, fingerprint, fingerprint_of
+from .migration import (
+    CategoryBreakdown,
+    MigrationReport,
+    RowResult,
+    TokenTotals,
+    annotate_corpus,
+    migration_id_for,
+    run_migration,
+)
+from .providers import (
+    Conversation,
+    DecodedResponse,
+    DecodeError,
+    MissingAPIKeyError,
+    ProviderAdapter,
+    UnsupportedRequestError,
+    conversation_of,
+    decode_interaction,
+)
+from .report import render_console, render_html, render_markdown
+from .session import DynamicTransport, async_client, cassette
+from .store import FileStore, InMemoryStore, InteractionStore
+from .transport import AutoTransport, RecordingTransport, ReplayTransport
+
+try:
+    __version__ = _version("agentrec")
+except PackageNotFoundError:  # running from a source tree without an install
+    __version__ = "0.0.0.dev0"
+
+__all__ = [
+    "__version__",
+    # Data
+    "CapturedChunk",
+    "CapturedInteraction",
+    "CapturedRequest",
+    # Keying
+    "Fingerprint",
+    "default_key",
+    "fingerprint",
+    "fingerprint_of",
+    # Stores
+    "FileStore",
+    "InMemoryStore",
+    "InteractionStore",
+    # Low-level transports
+    "AutoTransport",
+    "RecordingTransport",
+    "ReplayTransport",
+    # High-level facade
+    "DynamicTransport",
+    "async_client",
+    "cassette",
+    # Providers
+    "Conversation",
+    "DecodedResponse",
+    "DecodeError",
+    "MissingAPIKeyError",
+    "ProviderAdapter",
+    "UnsupportedRequestError",
+    "conversation_of",
+    "decode_interaction",
+    # Comparators
+    "Comparator",
+    "ComparisonResult",
+    "ExactMatchComparator",
+    "FuzzyComparator",
+    "EmbeddingComparator",
+    "JudgeComparator",
+    "build_comparators",
+    # Migration report
+    "CategoryBreakdown",
+    "MigrationReport",
+    "RowResult",
+    "TokenTotals",
+    "annotate_corpus",
+    "migration_id_for",
+    "run_migration",
+    "render_console",
+    "render_html",
+    "render_markdown",
+]

agentrec/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

agentrec/capture.py

@@ -0,0 +1,46 @@
+"""
+Storage-agnostic data structures for a captured HTTP interaction.
+
+Storage backends (the JSON-cassette FileStore today, others later) consume
+CapturedInteraction without the transports knowing about them.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Tuple
+
+
+@dataclass
+class CapturedChunk:
+    """One raw byte frame from an HTTP response body."""
+
+    data: bytes
+    # Seconds elapsed since the first chunk arrived.  Stored so replay can
+    # optionally simulate realistic pacing; instant replay ignores it.
+    timestamp_offset: float = 0.0
+
+
+@dataclass
+class CapturedRequest:
+    method: str
+    url: str
+    # Raw (bytes, bytes) header pairs — provider-neutral, no decoding here.
+    headers: List[Tuple[bytes, bytes]]
+    content: bytes
+
+
+@dataclass
+class CapturedInteraction:
+    """Complete record of one request/response exchange."""
+
+    request: CapturedRequest
+    response_status: int
+    response_headers: List[Tuple[bytes, bytes]]
+    # extensions minus transport-specific keys like "network_stream"
+    response_extensions: dict
+    chunks: List[CapturedChunk] = field(default_factory=list)
+    # Provenance for the corpus: provider, model, semantic_key, recorded_at.
+    # Free-form so new fields (e.g. a future migration-report needs) drop in
+    # without a schema change.  Populated by RecordingTransport; empty when an
+    # interaction is hand-built or loaded from a pre-metadata cassette.
+    metadata: Dict[str, Any] = field(default_factory=dict)

agentrec/cli.py

@@ -0,0 +1,173 @@
+"""
+Command-line interface: ``python -m agentrec <command>`` (or ``agentrec ...``).
+
+migrate   Run the corpus against a target model (records new responses into
+          the corpus) and write a Markdown/HTML migration report.
+report    Re-render the report fully offline from already-recorded cassettes;
+          only the offline comparators (exact, fuzzy) are allowed.
+annotate  Backfill human-readable summary blocks and fingerprint metadata
+          into existing cassettes.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from .comparators import OFFLINE_COMPARATOR_NAMES, build_comparators
+from .migration import annotate_corpus, run_migration
+from .report import default_report_basename, render_console, render_html, render_markdown
+from .store import FileStore
+
+
+def _add_report_args(parser: argparse.ArgumentParser, *, default_compare: str) -> None:
+    parser.add_argument("--corpus", default="corpus", help="corpus directory (default: corpus)")
+    parser.add_argument("--target", required=True, help="target model id, e.g. claude-haiku-4-5")
+    parser.add_argument(
+        "--compare",
+        default=default_compare,
+        help=f"comma-separated comparators or 'all' (default: {default_compare})",
+    )
+    parser.add_argument(
+        "--target-provider",
+        default=None,
+        help="override the provider inferred from the target model id",
+    )
+    parser.add_argument("--judge-model", default="claude-opus-4-8", help="model for the judge comparator")
+    parser.add_argument(
+        "--embedding-model", default="text-embedding-3-small", help="model for the embedding comparator"
+    )
+    parser.add_argument("--fuzzy-threshold", type=float, default=0.8)
+    parser.add_argument("--embedding-threshold", type=float, default=0.8)
+    parser.add_argument(
+        "--max-tokens", type=int, default=4096,
+        help="max_tokens for target requests when the baseline did not set one",
+    )
+    parser.add_argument("--filter", default=None, help="only baselines whose id contains this substring")
+    parser.add_argument(
+        "--concurrency", type=int, default=8,
+        help="rows scored in parallel (default: 8)",
+    )
+    parser.add_argument(
+        "--format", choices=("md", "html", "both"), default="both", help="report format(s) to write"
+    )
+    parser.add_argument(
+        "--out", default=None,
+        help="output base path (extension added per format; default: migration-report__<target>__<timestamp>)",
+    )
+    parser.add_argument(
+        "--strict", action="store_true",
+        help="exit with code 1 if any comparison failed or errored (CI gate)",
+    )
+
+
+def _parse(argv: Optional[List[str]]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="agentrec", description="Record/replay LLM corpus tooling and migration reports."
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    migrate = sub.add_parser(
+        "migrate", help="run corpus prompts against a target model and write a migration report"
+    )
+    _add_report_args(migrate, default_compare="exact,fuzzy")
+
+    report = sub.add_parser(
+        "report", help="re-render a migration report offline from recorded cassettes"
+    )
+    _add_report_args(report, default_compare="exact,fuzzy")
+
+    annotate = sub.add_parser(
+        "annotate", help="backfill summary blocks and metadata into existing cassettes"
+    )
+    annotate.add_argument("--corpus", default="corpus", help="corpus directory (default: corpus)")
+
+    return parser.parse_args(argv)
+
+
+def _write_reports(args: argparse.Namespace, report) -> List[Path]:
+    base = args.out or default_report_basename(args.target)
+    base_path = Path(base)
+    if base_path.suffix.lower() in (".md", ".html"):
+        base_path = base_path.with_suffix("")
+    written: List[Path] = []
+    if args.format in ("md", "both"):
+        path = base_path.with_suffix(".md")
+        path.write_text(render_markdown(report), encoding="utf-8")
+        written.append(path)
+    if args.format in ("html", "both"):
+        path = base_path.with_suffix(".html")
+        path.write_text(render_html(report), encoding="utf-8")
+        written.append(path)
+    return written
+
+
+async def _run_report_command(args: argparse.Namespace, *, offline: bool) -> int:
+    if offline:
+        requested = (
+            list(OFFLINE_COMPARATOR_NAMES)
+            if args.compare.strip().lower() == "all"
+            else [name.strip() for name in args.compare.split(",") if name.strip()]
+        )
+        online = [name for name in requested if name not in OFFLINE_COMPARATOR_NAMES]
+        if online:
+            print(
+                f"report (offline) supports only {', '.join(OFFLINE_COMPARATOR_NAMES)}; "
+                f"drop: {', '.join(online)} (use `agentrec migrate` for live comparators)",
+                file=sys.stderr,
+            )
+            return 2
+        args.compare = ",".join(requested)
+
+    comparators = build_comparators(
+        args.compare,
+        judge_model=args.judge_model,
+        embedding_model=args.embedding_model,
+        fuzzy_threshold=args.fuzzy_threshold,
+        embedding_threshold=args.embedding_threshold,
+    )
+    store = FileStore(args.corpus)
+    report = await run_migration(
+        store,
+        args.target,
+        comparators,
+        target_provider=args.target_provider,
+        offline=offline,
+        max_tokens_default=args.max_tokens,
+        filter_substr=args.filter,
+        concurrency=args.concurrency,
+    )
+    written = _write_reports(args, report)
+    print(render_console(report))
+    for path in written:
+        print(f"Report written: {path}")
+    if args.strict and not report.all_passed:
+        return 1
+    return 0
+
+
+async def _run_annotate(args: argparse.Namespace) -> int:
+    annotated = await annotate_corpus(FileStore(args.corpus))
+    print(f"Annotated {len(annotated)} cassette(s) in {args.corpus}")
+    return 0
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = _parse(argv)
+    try:
+        if args.command == "migrate":
+            return asyncio.run(_run_report_command(args, offline=False))
+        if args.command == "report":
+            return asyncio.run(_run_report_command(args, offline=True))
+        if args.command == "annotate":
+            return asyncio.run(_run_annotate(args))
+    except (ValueError, LookupError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agentrec/comparators.py

@@ -0,0 +1,271 @@
+"""
+Comparators score a baseline response against a target-model response.
+
+The migration runner evaluates *all* selected comparators per prompt in one
+pass, so a single ``migrate`` run can report exact-match, fuzzy similarity,
+embedding similarity and judge verdicts side by side.
+
+* ``exact``     — normalized string equality.  The right metric for
+                  classification-style outputs ("positive" vs "Positive ").
+* ``fuzzy``     — ``difflib.SequenceMatcher`` ratio; offline, dependency-free.
+* ``embedding`` — cosine similarity of OpenAI embeddings (live API call).
+* ``judge``     — an LLM scores semantic equivalence (live API call).
+
+A comparator failure (missing API key, malformed judge reply) degrades to an
+errored :class:`ComparisonResult` on that row — it never crashes the run.
+"""
+from __future__ import annotations
+
+import difflib
+import json
+import math
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Sequence
+
+import httpx
+
+from .providers import Conversation, DecodedResponse, adapter_for_model, adapter_for_provider
+
+OFFLINE_COMPARATOR_NAMES = ("exact", "fuzzy")
+ALL_COMPARATOR_NAMES = ("exact", "fuzzy", "embedding", "judge")
+
+_WHITESPACE = re.compile(r"\s+")
+
+
+def _normalize(text: str) -> str:
+    return _WHITESPACE.sub(" ", text).strip().casefold()
+
+
+def _clip(text: str, limit: int = 6000) -> str:
+    if len(text) <= limit:
+        return text
+    return text[:limit] + " …[truncated]"
+
+
+@dataclass(frozen=True)
+class ComparisonResult:
+    comparator: str
+    score: float  # 0.0–1.0
+    passed: Optional[bool]  # None when pass/fail is not meaningful
+    detail: str = ""
+    error: bool = False  # True when the comparator itself failed
+
+
+class Comparator(ABC):
+    """Scores how well *target* preserves the behaviour of *baseline*."""
+
+    name: str
+
+    @abstractmethod
+    async def compare(
+        self, prompt: str, baseline: DecodedResponse, target: DecodedResponse
+    ) -> ComparisonResult:
+        ...
+
+
+class ExactMatchComparator(Comparator):
+    name = "exact"
+
+    async def compare(self, prompt, baseline, target) -> ComparisonResult:
+        matched = _normalize(baseline.text) == _normalize(target.text)
+        return ComparisonResult(
+            comparator=self.name,
+            score=1.0 if matched else 0.0,
+            passed=matched,
+            detail="normalized texts match" if matched else "normalized texts differ",
+        )
+
+
+class FuzzyComparator(Comparator):
+    name = "fuzzy"
+
+    def __init__(self, threshold: float = 0.8) -> None:
+        self._threshold = threshold
+
+    async def compare(self, prompt, baseline, target) -> ComparisonResult:
+        score = difflib.SequenceMatcher(
+            None, _normalize(baseline.text), _normalize(target.text)
+        ).ratio()
+        return ComparisonResult(
+            comparator=self.name,
+            score=score,
+            passed=score >= self._threshold,
+            detail=f"sequence similarity {score:.2f} (threshold {self._threshold})",
+        )
+
+
+def cosine_similarity(a: Sequence[float], b: Sequence[float]) -> float:
+    dot = sum(x * y for x, y in zip(a, b))
+    norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
+    if norm == 0:
+        return 0.0
+    return dot / norm
+
+
+class _HttpComparator(Comparator):
+    """Shared plumbing for comparators that call an API via httpx."""
+
+    def __init__(self, http: Optional[httpx.AsyncClient] = None) -> None:
+        self._http = http
+
+    async def _post(self, url: str, headers: Dict[str, str], body: dict) -> httpx.Response:
+        if self._http is not None:
+            return await self._http.post(url, headers=headers, json=body)
+        async with httpx.AsyncClient(timeout=60.0) as client:
+            return await client.post(url, headers=headers, json=body)
+
+
+class EmbeddingComparator(_HttpComparator):
+    name = "embedding"
+
+    embeddings_url = "https://api.openai.com/v1/embeddings"
+
+    def __init__(
+        self,
+        http: Optional[httpx.AsyncClient] = None,
+        *,
+        model: str = "text-embedding-3-small",
+        threshold: float = 0.8,
+    ) -> None:
+        super().__init__(http)
+        self._model = model
+        self._threshold = threshold
+
+    async def compare(self, prompt, baseline, target) -> ComparisonResult:
+        headers = {
+            "Authorization": f"Bearer {adapter_for_provider('openai').api_key()}",
+            "Content-Type": "application/json",
+        }
+        body = {"model": self._model, "input": [_clip(baseline.text), _clip(target.text)]}
+        response = await self._post(self.embeddings_url, headers, body)
+        response.raise_for_status()
+        data = sorted(response.json()["data"], key=lambda item: item["index"])
+        score = max(0.0, min(1.0, cosine_similarity(data[0]["embedding"], data[1]["embedding"])))
+        return ComparisonResult(
+            comparator=self.name,
+            score=score,
+            passed=score >= self._threshold,
+            detail=f"cosine similarity {score:.2f} via {self._model} (threshold {self._threshold})",
+        )
+
+
+_JUDGE_SYSTEM = (
+    "You compare two AI assistant responses to the same prompt and judge whether "
+    "the candidate response is an acceptable substitute for the baseline response. "
+    "Judge semantic equivalence of the substantive content, not style or length. "
+    'Reply with ONLY a JSON object: {"equivalent": true|false, "score": 0.0-1.0, '
+    '"reason": "<one sentence>"}'
+)
+
+_JUDGE_TEMPLATE = """<prompt>
+{prompt}
+</prompt>
+
+<baseline_response>
+{baseline}
+</baseline_response>
+
+<candidate_response>
+{target}
+</candidate_response>"""
+
+
+def _first_json_object(text: str) -> dict:
+    decoder = json.JSONDecoder()
+    for start in range(len(text)):
+        if text[start] != "{":
+            continue
+        try:
+            obj, _ = decoder.raw_decode(text, start)
+        except ValueError:
+            continue
+        if isinstance(obj, dict):
+            return obj
+    raise ValueError(f"no JSON object found in judge reply: {text[:200]!r}")
+
+
+class JudgeComparator(_HttpComparator):
+    name = "judge"
+
+    def __init__(
+        self,
+        http: Optional[httpx.AsyncClient] = None,
+        *,
+        judge_model: str = "claude-opus-4-8",
+    ) -> None:
+        super().__init__(http)
+        self._judge_model = judge_model
+
+    async def compare(self, prompt, baseline, target) -> ComparisonResult:
+        adapter = adapter_for_model(self._judge_model)
+        conversation = Conversation(
+            system=_JUDGE_SYSTEM,
+            messages=[
+                {
+                    "role": "user",
+                    "content": _JUDGE_TEMPLATE.format(
+                        prompt=_clip(prompt),
+                        baseline=_clip(baseline.text),
+                        target=_clip(target.text),
+                    ),
+                }
+            ],
+            # No sampling params: the newest judge models reject them.
+            max_tokens=1024,
+        )
+        url, headers, body = adapter.build_request(conversation, self._judge_model)
+        response = await self._post(url, headers, body)
+        response.raise_for_status()
+        decoded = adapter.decode_response(await response.aread(), is_sse=False)
+        verdict = _first_json_object(decoded.text)
+
+        equivalent = bool(verdict.get("equivalent"))
+        raw_score = verdict.get("score")
+        score = float(raw_score) if isinstance(raw_score, (int, float)) else (1.0 if equivalent else 0.0)
+        score = max(0.0, min(1.0, score))
+        return ComparisonResult(
+            comparator=self.name,
+            score=score,
+            passed=equivalent,
+            detail=str(verdict.get("reason", "")) or f"judge {self._judge_model} verdict",
+        )
+
+
+def build_comparators(
+    spec: str,
+    *,
+    judge_model: str = "claude-opus-4-8",
+    embedding_model: str = "text-embedding-3-small",
+    fuzzy_threshold: float = 0.8,
+    embedding_threshold: float = 0.8,
+    http: Optional[httpx.AsyncClient] = None,
+) -> List[Comparator]:
+    """Parse a ``--compare`` spec like ``"exact,judge"`` or ``"all"``."""
+    names = (
+        list(ALL_COMPARATOR_NAMES)
+        if spec.strip().lower() == "all"
+        else [name.strip().lower() for name in spec.split(",") if name.strip()]
+    )
+    seen: List[str] = []
+    for name in names:
+        if name not in ALL_COMPARATOR_NAMES:
+            raise ValueError(
+                f"unknown comparator {name!r}; expected any of "
+                f"{', '.join(ALL_COMPARATOR_NAMES)} or 'all'"
+            )
+        if name not in seen:
+            seen.append(name)
+    if not seen:
+        raise ValueError("no comparators selected")
+
+    factories = {
+        "exact": lambda: ExactMatchComparator(),
+        "fuzzy": lambda: FuzzyComparator(threshold=fuzzy_threshold),
+        "embedding": lambda: EmbeddingComparator(
+            http, model=embedding_model, threshold=embedding_threshold
+        ),
+        "judge": lambda: JudgeComparator(http, judge_model=judge_model),
+    }
+    return [factories[name]() for name in seen]