contrastset 0.1.0__py3-none-any.whl

contrastset/__init__.py

@@ -0,0 +1,3 @@
+"""contrastset: build and check contrast sets for interpretability experiments."""
+
+__version__ = "0.1.0"

contrastset/adapters/__init__.py

@@ -0,0 +1,14 @@
+"""Adapters: the only way contrastset talks to a model.
+
+Both the subject model and the generator model are reached through the same tiny
+`Adapter` interface, so any backend that can turn a prompt into text works.
+"""
+
+from contrastset.adapters.anthropic import AnthropicAdapter
+from contrastset.adapters.base import Adapter, AdapterError, Scorer, make_adapter
+from contrastset.adapters.fake import FakeAdapter
+from contrastset.adapters.hf import HFAdapter, Score
+from contrastset.adapters.ollama import OllamaAdapter
+from contrastset.adapters.openai import OpenAIAdapter
+
+__all__ = ["Adapter", "AnthropicAdapter", "AdapterError", "FakeAdapter", "HFAdapter", "Score", "Scorer", "OllamaAdapter", "OpenAIAdapter", "make_adapter"]

contrastset/adapters/anthropic.py

@@ -0,0 +1,63 @@
+"""Claude models through Anthropic's official SDK (`pip install contrastset[anthropic]`).
+
+Meant for the generator model, not the subject: the API exposes no raw completions
+and no internals. Two things differ from the other adapters and are recorded in
+describe(), so provenance says it:
+    - no temperature: the Messages API rejects temperature/top_p/top_k since
+      2026-10-21, so `temperature` is ignored;
+    - no seed: the API has none, so generator output is not reproducible from the
+      seed. The pairs file keeps every generated text; that file is the record.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contrastset.adapters.base import AdapterError
+
+
+class AnthropicAdapter:
+    def __init__(self, model: str, *, effort: str = "low", system: str | None = None,
+                 client: Any = None) -> None:
+        if client is None:
+            try:
+                import anthropic
+            except ImportError as exc:
+                raise AdapterError("the anthropic adapter needs `pip install contrastset[anthropic]`") from exc
+            client = anthropic.Anthropic()  # credentials from ANTHROPIC_API_KEY or an `ant auth login` profile
+        self.model = model
+        self.effort = effort  # how much the model thinks before answering: low | medium | high | xhigh | max
+        self.system = system
+        self._client = client
+
+    def complete(
+        self,
+        prompt: str,
+        *,
+        n: int = 1,
+        temperature: float = 0.0,  # ignored, see module docstring
+        seed: int | None = None,  # ignored, see module docstring
+        max_tokens: int = 4096,
+    ) -> list[str]:
+        out = []
+        for _ in range(n):
+            kwargs: dict[str, Any] = {
+                "model": self.model,
+                "max_tokens": max_tokens,
+                "messages": [{"role": "user", "content": prompt}],
+                "output_config": {"effort": self.effort},
+            }
+            if self.system:
+                kwargs["system"] = self.system
+            try:
+                response = self._client.messages.create(**kwargs)
+            except Exception as exc:  # SDK errors carry status and message; surface them as-is
+                raise AdapterError(f"anthropic {self.model!r}: {exc}") from exc
+            if response.stop_reason == "refusal":
+                raise AdapterError(f"anthropic {self.model!r} declined the request: {response.stop_details}")
+            out.append("".join(block.text for block in response.content if block.type == "text"))
+        return out
+
+    def describe(self) -> dict[str, Any]:
+        return {"adapter": "anthropic", "model": self.model, "effort": self.effort,
+                "temperature_supported": False, "seed_supported": False}

contrastset/adapters/base.py

@@ -0,0 +1,78 @@
+"""The adapter interface."""
+
+from __future__ import annotations
+
+from typing import Any, Literal, Protocol, runtime_checkable
+
+
+class AdapterError(RuntimeError):
+    pass
+
+
+# The contrastset sampling order. Where an adapter samples itself (hf), it follows
+# this exactly; where a server samples (ollama), the adapter records the server's
+# settings and version, since the server's order is the server's. Matches
+# llama.cpp, so HF and Ollama grades are comparable. Truncating before the
+# temperature matters for top-p: after a temperature below 1 the distribution is
+# sharper and the same top-p keeps fewer tokens.
+SAMPLING_ORDER = "top_k -> top_p (both on untempered probabilities) -> temperature -> draw"
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    """Turns a prompt into n completions.
+
+    Seeding contract: completion i is sampled with seed `seed + i`, one request per
+    completion. That makes every sample individually reproducible and works with
+    servers that ignore the OpenAI `n` parameter (Ollama does). With temperature 0
+    (greedy decoding: always take the most likely next token) the seed is irrelevant.
+    """
+
+    def complete(
+        self,
+        prompt: str,
+        *,
+        n: int = 1,
+        temperature: float = 0.0,
+        seed: int | None = None,
+        max_tokens: int = 64,
+    ) -> list[str]: ...
+
+    def describe(self) -> dict[str, Any]:
+        """Identify the model for provenance: adapter kind, model name, endpoint, and
+        whatever the server reports about the exact weights (digest, quantisation)."""
+        ...
+
+
+@runtime_checkable
+class Scorer(Protocol):
+    """An adapter that can also score text: log P(continuation | prompt). Needs model
+    internals (logits), so only in-process adapters (hf) implement it."""
+
+    def score(self, prompt: str, continuation: str) -> Any: ...
+
+
+def sample_seed(seed: int | None, i: int) -> int | None:
+    return None if seed is None else seed + i
+
+
+def make_adapter(kind: str, model: str, base_url: str | None = None, **options: Any) -> Adapter:
+    """Build an adapter by name: "ollama", "openai" (raw completions), "openai-chat" or
+    "anthropic" or "hf"."""
+    from contrastset.adapters.anthropic import AnthropicAdapter
+    from contrastset.adapters.hf import HFAdapter
+    from contrastset.adapters.ollama import OllamaAdapter
+    from contrastset.adapters.openai import OpenAIAdapter
+
+    if kind == "ollama":
+        return OllamaAdapter(model, base_url=base_url or OllamaAdapter.default_url, **options)
+    if kind in ("openai", "openai-chat"):
+        if base_url is None:
+            raise AdapterError(f"adapter {kind!r} needs a base URL, e.g. http://localhost:8000/v1")
+        mode: Literal["completions", "chat"] = "chat" if kind == "openai-chat" else "completions"
+        return OpenAIAdapter(model, base_url=base_url, mode=mode, **options)
+    if kind == "anthropic":
+        return AnthropicAdapter(model, **options)
+    if kind == "hf":
+        return HFAdapter(model, **options)
+    raise AdapterError(f"unknown adapter {kind!r}; expected ollama, openai, openai-chat, anthropic or hf")

contrastset/adapters/fake.py

@@ -0,0 +1,60 @@
+"""A deterministic stand-in model for tests. No network, no weights."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from contrastset.adapters.base import sample_seed
+
+
+class FakeAdapter:
+    """Answers from a script.
+
+    `script` maps a substring of the prompt to the answers the fake model gives;
+    the first key found in the prompt wins. Greedy calls (temperature 0) return the
+    first answer; sampled calls return answers[seed % len(answers)], so a script like
+    ["Germany", "Germany", "France"] yields a known share of each answer. Alternatively
+    pass a function (prompt, temperature, seed) -> str. Every call is logged in `calls`.
+    """
+
+    def __init__(
+        self,
+        script: dict[str, list[str]] | Callable[[str, float, int | None], str],
+        *,
+        default: str = "I don't know.",
+        name: str = "fake",
+    ) -> None:
+        self.script = script
+        self.default = default
+        self.name = name
+        self.calls: list[dict[str, Any]] = []
+
+    def _one(self, prompt: str, temperature: float, seed: int | None) -> str:
+        if callable(self.script):
+            return self.script(prompt, temperature, seed)
+        for key, answers in self.script.items():
+            if key in prompt:
+                if temperature == 0 or seed is None:
+                    return answers[0]
+                return answers[seed % len(answers)]
+        return self.default
+
+    def complete(
+        self,
+        prompt: str,
+        *,
+        n: int = 1,
+        temperature: float = 0.0,
+        seed: int | None = None,
+        max_tokens: int = 64,
+    ) -> list[str]:
+        out = []
+        for i in range(n):
+            s = sample_seed(seed, i)
+            self.calls.append({"prompt": prompt, "temperature": temperature, "seed": s, "max_tokens": max_tokens})
+            out.append(self._one(prompt, temperature, s))
+        return out
+
+    def describe(self) -> dict[str, Any]:
+        return {"adapter": "fake", "model": self.name}