agentinputlab 0.1.0a1__py3-none-any.whl

agentinputlab/backends/retrieval/memory.py

@@ -0,0 +1,119 @@
+"""Small deterministic retrievers for fixtures and controlled experiments."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from agentinputlab.contracts.errors import ValidationError
+from agentinputlab.contracts.serialization import content_hash, freeze_mapping
+
+from .base import Document, RetrievalResult, Retriever
+
+_TOKEN = re.compile(r"[\w]+", re.UNICODE)
+
+
+def default_tokenize(text: str) -> tuple[str, ...]:
+    return tuple(token.lower() for token in _TOKEN.findall(text))
+
+
+@dataclass(slots=True)
+class InMemoryBM25Retriever(Retriever):
+    documents: Iterable[Document]
+    retriever_id: str = "memory-bm25"
+    k1: float = 1.2
+    b: float = 0.75
+    tokenizer: Callable[[str], tuple[str, ...]] = default_tokenize
+    tokenizer_id: str = "agentinputlab.default_tokenize.v1"
+    _documents: tuple[Document, ...] = field(init=False, repr=False)
+    _term_counts: tuple[Counter[str], ...] = field(init=False, repr=False)
+    _doc_freq: Counter[str] = field(init=False, repr=False)
+    _avgdl: float = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        if self.tokenizer is not default_tokenize and self.tokenizer_id == (
+            "agentinputlab.default_tokenize.v1"
+        ):
+            raise ValidationError("custom BM25 tokenizer requires an explicit tokenizer_id")
+        if not self.tokenizer_id:
+            raise ValidationError("BM25 tokenizer_id must be non-empty")
+        self._documents = tuple(self.documents)
+        self._term_counts = tuple(
+            Counter(self.tokenizer(f"{doc.title} {doc.text}")) for doc in self._documents
+        )
+        self._doc_freq = Counter()
+        for counts in self._term_counts:
+            self._doc_freq.update(counts.keys())
+        self._avgdl = (
+            sum(sum(counts.values()) for counts in self._term_counts) / len(self._term_counts)
+            if self._term_counts
+            else 0.0
+        )
+
+    def retrieve(self, query: str, *, top_k: int = 5) -> tuple[RetrievalResult, ...]:
+        if top_k < 1:
+            return ()
+        query_terms = self.tokenizer(query)
+        scored: list[tuple[float, str, int]] = []
+        n_docs = len(self._documents)
+        for index, counts in enumerate(self._term_counts):
+            length = sum(counts.values())
+            score = 0.0
+            for term in query_terms:
+                tf = counts.get(term, 0)
+                if not tf:
+                    continue
+                df = self._doc_freq[term]
+                idf = math.log(1 + (n_docs - df + 0.5) / (df + 0.5))
+                norm = tf + self.k1 * (1 - self.b + self.b * length / max(self._avgdl, 1e-9))
+                score += idf * (tf * (self.k1 + 1)) / norm
+            scored.append((score, self._documents[index].doc_id, index))
+        scored.sort(key=lambda item: (-item[0], item[1]))
+        results = []
+        for rank, (score, _, index) in enumerate(scored[:top_k], start=1):
+            results.append(
+                RetrievalResult(
+                    document=self._documents[index],
+                    score=score,
+                    rank=rank,
+                    features={"bm25_score": score},
+                )
+            )
+        return tuple(results)
+
+    def fingerprint(self) -> Mapping[str, Any]:
+        return {
+            "retriever_id": self.retriever_id,
+            "implementation": "agentinputlab.memory_bm25.v1",
+            "corpus_hash": content_hash(self._documents),
+            "document_count": len(self._documents),
+            "k1": self.k1,
+            "b": self.b,
+            "tokenizer_id": self.tokenizer_id,
+        }
+
+
+@dataclass(slots=True)
+class FunctionRetriever(Retriever):
+    fn: Callable[[str, int], Iterable[RetrievalResult]]
+    retriever_id: str = "function"
+    identity: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.identity:
+            raise ValidationError("function retriever requires explicit stable identity")
+        if not self.identity.get("implementation") or not self.identity.get("revision"):
+            raise ValidationError(
+                "function retriever identity requires implementation and revision"
+            )
+        self.identity = freeze_mapping(self.identity)
+
+    def retrieve(self, query: str, *, top_k: int = 5) -> tuple[RetrievalResult, ...]:
+        return tuple(self.fn(query, top_k))
+
+    def fingerprint(self) -> Mapping[str, Any]:
+        return {"retriever_id": self.retriever_id, **dict(self.identity)}

agentinputlab/backends/retrieval/sqlite_fts.py

@@ -0,0 +1,155 @@
+"""Disk-backed, deterministic SQLite FTS5 BM25 for mounted large corpora.
+
+This is an explicit lexical retrieval profile, not a claim to reproduce a
+different BM25 implementation's rankings. Building streams the corpus once;
+concurrent queries use independent read-only connections.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sqlite3
+import tempfile
+from collections.abc import Iterable, Mapping
+from contextlib import closing
+from pathlib import Path
+from typing import Any
+
+from agentinputlab.contracts import ArtifactError, ExternalDependencyError, ValidationError
+from agentinputlab.contracts.serialization import canonical_json, content_hash
+
+from .base import Document, RetrievalResult
+
+
+class SQLiteFTS5Retriever:
+    retriever_id = "sqlite.fts5.bm25.unicode61.v1"
+
+    def __init__(self, path: str | Path, *, corpus_identity: Mapping[str, Any]):
+        self.path = Path(path).expanduser().absolute()
+        self.identity = json.loads(canonical_json(corpus_identity))
+        if not self.identity:
+            raise ValidationError("disk corpus identity is required")
+        if self.path.is_symlink() or not self.path.is_file():
+            raise ArtifactError("FTS index must be a regular existing file")
+        try:
+            with closing(self._connection()) as db:
+                row = db.execute("SELECT value FROM metadata WHERE key=?", ("binding",)).fetchone()
+                if row is None:
+                    raise ArtifactError("FTS index has no corpus binding")
+                metadata = json.loads(row[0])
+        except (sqlite3.DatabaseError, json.JSONDecodeError) as exc:
+            raise ArtifactError("FTS index is invalid or incomplete") from exc
+        if (
+            metadata["corpus_identity"] != self.identity
+            or metadata["retriever_id"] != self.retriever_id
+        ):
+            raise ArtifactError("FTS index identity differs")
+        self.count = metadata["document_count"]
+
+    def _connection(self):
+        return sqlite3.connect(self.path.as_uri() + "?mode=ro", uri=True, timeout=30)
+
+    @classmethod
+    def build(
+        cls, path: str | Path, documents: Iterable[Document], *, corpus_identity: Mapping[str, Any]
+    ):
+        target = Path(path).expanduser().absolute()
+        if target.exists():
+            return cls(target, corpus_identity=corpus_identity)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        fd, name = tempfile.mkstemp(prefix=".fts-build-", suffix=".sqlite", dir=target.parent)
+        os.close(fd)
+        temporary = Path(name)
+        try:
+            with closing(sqlite3.connect(temporary)) as db:
+                db.execute("PRAGMA journal_mode=OFF")
+                db.execute("PRAGMA synchronous=OFF")
+                db.execute(
+                    "CREATE TABLE documents (doc_id TEXT UNIQUE NOT NULL,title TEXT NOT NULL,text TEXT NOT NULL,revision TEXT NOT NULL)"
+                )
+                db.execute(
+                    "CREATE VIRTUAL TABLE passages USING fts5(title,text,content='documents',content_rowid='rowid',tokenize='unicode61')"
+                )
+                count = 0
+                for doc in documents:
+                    if not isinstance(doc, Document):
+                        raise ValidationError("FTS corpus requires Documents")
+                    cursor = db.execute(
+                        "INSERT INTO documents VALUES (?,?,?,?)",
+                        (doc.doc_id, doc.title, doc.text, doc.corpus_revision),
+                    )
+                    db.execute(
+                        "INSERT INTO passages(rowid,title,text) VALUES (?,?,?)",
+                        (cursor.lastrowid, doc.title, doc.text),
+                    )
+                    count += 1
+                    if count % 10000 == 0:
+                        db.commit()
+                if not count:
+                    raise ValidationError("FTS corpus cannot be empty")
+                db.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY,value TEXT NOT NULL)")
+                binding = {
+                    "retriever_id": cls.retriever_id,
+                    "corpus_identity": json.loads(canonical_json(corpus_identity)),
+                    "document_count": count,
+                }
+                db.execute(
+                    "INSERT INTO metadata VALUES (?,?)", ("binding", canonical_json(binding))
+                )
+                db.execute("INSERT INTO passages(passages) VALUES ('optimize')")
+                db.commit()
+            # A concurrent builder cannot replace an existing index.
+            os.link(temporary, target)
+        except sqlite3.IntegrityError as exc:
+            raise ArtifactError("FTS corpus contains duplicate document IDs") from exc
+        except sqlite3.OperationalError as exc:
+            if "no such module: fts5" in str(exc):
+                raise ExternalDependencyError("SQLite must be compiled with FTS5 support") from exc
+            raise ArtifactError(f"FTS index build failed: {exc}") from exc
+        finally:
+            temporary.unlink(missing_ok=True)
+        return cls(target, corpus_identity=corpus_identity)
+
+    def retrieve(self, query: str, *, top_k: int = 5) -> tuple[RetrievalResult, ...]:
+        if isinstance(top_k, bool) or not isinstance(top_k, int) or top_k < 1:
+            raise ValidationError("top_k must be positive")
+        if not isinstance(query, str):
+            raise ValidationError("query must be text")
+        terms = list(dict.fromkeys(re.findall(r"\w+", query, flags=re.UNICODE)))[:256]
+        if not terms:
+            return ()
+        expression = " OR ".join('"' + term + '"' for term in terms)
+        with closing(self._connection()) as db:
+            rows = db.execute(
+                "SELECT d.doc_id,d.title,d.text,d.revision,bm25(passages) AS score "
+                "FROM passages JOIN documents d ON d.rowid=passages.rowid WHERE passages MATCH ? "
+                "ORDER BY score ASC,d.doc_id ASC LIMIT ?",
+                (expression, top_k),
+            ).fetchall()
+        return tuple(
+            RetrievalResult(
+                Document(doc_id=r[0], title=r[1], text=r[2], corpus_revision=r[3]),
+                score=-r[4],
+                rank=i + 1,
+            )
+            for i, r in enumerate(rows)
+        )
+
+    def fingerprint(self):
+        return {
+            "retriever_id": self.retriever_id,
+            "sqlite_version": sqlite3.sqlite_version,
+            "corpus_identity": self.identity,
+            "corpus_hash": content_hash(self.identity),
+            "document_count": self.count,
+            "tokenizer": "unicode61",
+            "query_operator": "OR",
+            "max_unique_query_terms": 256,
+            "title_weight": 1.0,
+            "text_weight": 1.0,
+        }
+
+
+__all__ = ["SQLiteFTS5Retriever"]

agentinputlab/backends/user/__init__.py

@@ -0,0 +1,15 @@
+"""Injected and replayable user-response backends."""
+
+from .base import UserRequest, UserResponder, UserResponse
+from .callable import CallableUserResponder, UserClient
+from .replay import ReplayTurn, ReplayUserResponder
+
+__all__ = [
+    "CallableUserResponder",
+    "ReplayTurn",
+    "ReplayUserResponder",
+    "UserClient",
+    "UserRequest",
+    "UserResponder",
+    "UserResponse",
+]

agentinputlab/backends/user/base.py

@@ -0,0 +1,66 @@
+"""User-response backend contract for clarification and interactive benchmarks."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+from agentinputlab.contracts.errors import ValidationError
+from agentinputlab.contracts.serialization import content_hash
+
+
+@dataclass(frozen=True, slots=True)
+class UserRequest:
+    request_id: str
+    case_id: str
+    turn_index: int
+    prompt: str
+    state_hash: str = ""
+    choices: tuple[str, ...] = ()
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "choices", tuple(self.choices))
+        if not self.request_id:
+            raise ValidationError("user request_id must be non-empty")
+        if not self.case_id:
+            raise ValidationError("user case_id must be non-empty")
+        if self.turn_index < 0:
+            raise ValidationError("user turn_index must be non-negative")
+
+    @property
+    def prompt_hash(self) -> str:
+        return content_hash(
+            {
+                "prompt": self.prompt,
+                "choices": self.choices,
+                "state_hash": self.state_hash,
+            }
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class UserResponse:
+    request_id: str
+    text: str
+    responder_id: str = ""
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.request_id:
+            raise ValidationError("user response request_id must be non-empty")
+
+
+@runtime_checkable
+class UserResponder(Protocol):
+    responder_id: str
+
+    def respond(self, request: UserRequest) -> UserResponse:
+        """Return one response without consulting evaluator-only data."""
+
+    def fingerprint(self) -> Mapping[str, Any]:
+        """Return stable, secret-free identity fields."""
+
+
+__all__ = ["UserRequest", "UserResponder", "UserResponse"]

agentinputlab/backends/user/callable.py

@@ -0,0 +1,54 @@
+"""User responder using an explicitly injected callable."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from agentinputlab.contracts.errors import ValidationError
+from agentinputlab.contracts.serialization import content_hash, freeze_mapping
+
+from .base import UserRequest, UserResponder, UserResponse
+
+UserClient = Callable[[UserRequest], UserResponse | str]
+
+
+@dataclass(slots=True)
+class CallableUserResponder(UserResponder):
+    client: UserClient
+    responder_id: str = "callable-user"
+    revision: str = "unversioned"
+    identity: Mapping[str, Any] = field(default_factory=dict)
+    network: bool = False
+
+    def __post_init__(self) -> None:
+        if not self.revision or self.revision == "unversioned":
+            raise ValidationError("callable user responder requires an explicit stable revision")
+        if not self.identity:
+            raise ValidationError("callable user responder requires explicit identity fields")
+        self.identity = freeze_mapping(self.identity)
+
+    def respond(self, request: UserRequest) -> UserResponse:
+        value = self.client(request)
+        if isinstance(value, UserResponse):
+            if value.request_id != request.request_id:
+                raise ValidationError("callable user response request_id does not match request")
+            return value
+        return UserResponse(
+            request_id=request.request_id,
+            text=str(value),
+            responder_id=self.responder_id,
+        )
+
+    def fingerprint(self) -> Mapping[str, Any]:
+        return {
+            "backend": "agentinputlab.callable_user.v1",
+            "responder_id": self.responder_id,
+            "revision": self.revision,
+            "identity_hash": content_hash(self.identity),
+            "network": self.network,
+        }
+
+
+__all__ = ["CallableUserResponder", "UserClient"]

agentinputlab/backends/user/replay.py

@@ -0,0 +1,89 @@
+"""Strict, prompt-bound replay user responder."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from agentinputlab.contracts.errors import ValidationError
+from agentinputlab.contracts.serialization import content_hash, freeze_mapping
+
+from .base import UserRequest, UserResponder, UserResponse
+
+
+@dataclass(frozen=True, slots=True)
+class ReplayTurn:
+    case_id: str
+    turn_index: int
+    response: str
+    request_id: str | None = None
+    prompt_hash: str | None = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    @property
+    def key(self) -> str:
+        return self.request_id or f"{self.case_id}:{self.turn_index}"
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "metadata", freeze_mapping(self.metadata))
+        if not self.case_id:
+            raise ValidationError("replay user case_id must be non-empty")
+        if self.turn_index < 0:
+            raise ValidationError("replay user turn_index must be non-negative")
+        if self.request_id == "":
+            raise ValidationError("replay user request_id must be null or non-empty")
+
+
+@dataclass(slots=True)
+class ReplayUserResponder(UserResponder):
+    turns: Iterable[ReplayTurn]
+    responder_id: str = "replay-user"
+    revision: str = "fixture-v1"
+    _turns: tuple[ReplayTurn, ...] = field(init=False, repr=False)
+    _by_key: Mapping[str, ReplayTurn] = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        turns = tuple(self.turns)
+        by_key = {turn.key: turn for turn in turns}
+        if len(by_key) != len(turns):
+            raise ValidationError("replay user script contains duplicate turn keys")
+        self._turns = turns
+        self._by_key = by_key
+
+    def respond(self, request: UserRequest) -> UserResponse:
+        turn = self._by_key.get(request.request_id)
+        if turn is None:
+            turn = self._by_key.get(f"{request.case_id}:{request.turn_index}")
+        if turn is None:
+            raise ValidationError(
+                f"replay user script has no response for {request.request_id} "
+                f"({request.case_id}:{request.turn_index})"
+            )
+        if turn.case_id != request.case_id or turn.turn_index != request.turn_index:
+            raise ValidationError("replay user turn identity does not match request")
+        if turn.prompt_hash is not None and turn.prompt_hash != request.prompt_hash:
+            raise ValidationError("replay user prompt hash does not match request")
+        return UserResponse(
+            request_id=request.request_id,
+            text=turn.response,
+            responder_id=self.responder_id,
+            metadata={"replay": True, **dict(turn.metadata)},
+        )
+
+    def fingerprint(self) -> Mapping[str, Any]:
+        ordered = sorted(
+            self._turns,
+            key=lambda item: (item.case_id, item.turn_index, item.request_id or ""),
+        )
+        return {
+            "backend": "agentinputlab.replay_user.v1",
+            "responder_id": self.responder_id,
+            "revision": self.revision,
+            "turn_count": len(ordered),
+            "script_hash": content_hash(ordered),
+            "network": False,
+        }
+
+
+__all__ = ["ReplayTurn", "ReplayUserResponder"]

agentinputlab/backends/vector.py

@@ -0,0 +1,38 @@
+"""Dense embedding and vector-search runtime contracts.
+
+These intentionally match the lightweight seams used by paper-faithful HyDE
+adapters.  They remain distinct from :class:`EmbeddingBackend`, whose request
+and response objects carry provider usage metadata.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Protocol, runtime_checkable
+
+from .retrieval import RetrievalResult
+
+
+@runtime_checkable
+class DenseEmbedder(Protocol):
+    embedder_id: str
+
+    def embed(self, texts: Sequence[str]) -> Sequence[Sequence[float]]:
+        """Encode texts in the provided order."""
+
+    def fingerprint(self) -> Mapping[str, object]:
+        """Return the model/tokenizer/revision identity."""
+
+
+@runtime_checkable
+class VectorSearcher(Protocol):
+    searcher_id: str
+
+    def search(self, vector: Sequence[float], *, top_k: int) -> Sequence[RetrievalResult]:
+        """Search a fixed dense index with an already-computed query vector."""
+
+    def fingerprint(self) -> Mapping[str, object]:
+        """Return the corpus/index/metric identity."""
+
+
+__all__ = ["DenseEmbedder", "VectorSearcher"]