dev-double 0.1.0__py3-none-any.whl

dev_double/providers/needle/decision/record_tool.py

@@ -0,0 +1,75 @@
+"""Extraction as one Needle tool call: every field becomes an argument
+of a ``record`` tool, and the call's arguments are mapped back to typed values.
+
+pydantic is imported lazily, like needle, so this module imports without either.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal, Optional
+
+from dev_double.core.decision.confidence import DIGITS
+from dev_double.core.decision.t_extract import EMPTY, TExtractField, TFieldValue
+from dev_double.core.decision.tracker import Tracker
+from dev_double.core.decision.value_parsing import coerce_json_value
+
+_PY_TYPES: dict[str, type] = {"string": str, "number": float, "integer": int, "boolean": bool}
+
+RECORD_SYSTEM = (
+    "Extract the described fields from the input. Always call record once. "
+    "Leave out a text or number field the input does not contain."
+)
+
+
+def fields_key(fields: dict[str, TExtractField]) -> tuple:
+    """A hashable key for a field set, to reuse one agent per distinct set."""
+    return tuple((n, f.type, f.description, tuple((f.options or {}).items())) for n, f in fields.items())  # type: ignore[union-attr]
+
+
+def record_model(fields: dict[str, TExtractField]) -> Any:
+    """The tool schema: field names are aliases, so any name (even "due date") works."""
+    from pydantic import Field, create_model
+
+    specs: dict[str, Any] = {}
+    for i, (name, f) in enumerate(fields.items()):
+        description = f.description or name
+        if f.type == "enum":
+            assert isinstance(f.options, dict)
+            kind: Any = Literal[tuple(f.options)]  # type: ignore[valid-type]
+            description += " One of: " + "; ".join(f"{k}: {v}" for k, v in f.options.items())
+        else:
+            kind = _PY_TYPES[f.type]
+        if f.type in ("enum", "boolean"):
+            # Always answerable from the text, so required: optional fields with no
+            # matching span come back empty from Needle.
+            specs[f"f{i}"] = (kind, Field(..., alias=name, description=description))
+        else:
+            specs[f"f{i}"] = (Optional[kind], Field(None, alias=name, description=description))
+    return create_model("record", __doc__="Record the fields found in the input.", **specs)
+
+
+def _coerce(f: TExtractField, raw: Any) -> tuple[Any, Optional[str]]:
+    if f.type == "boolean":
+        return (raw, None) if isinstance(raw, bool) else (None, f"not a boolean: {raw!r}")
+    if f.type == "enum":
+        return (raw, None) if raw in (f.options or {}) else (None, f"not one of the options: {raw!r}")
+    return coerce_json_value(raw, f.type)
+
+
+def value_from_argument(name: str, f: TExtractField, raw: Any, conf: float, tracker: Tracker) -> TFieldValue:
+    """One argument of the call as a field value; the call's confidence applies to every field."""
+    if raw is None:
+        return TFieldValue(None, conf)
+    value, problem = _coerce(f, raw)
+    if problem is not None:
+        tracker.warn(problem, name)
+        return EMPTY
+    if value is None:
+        return TFieldValue(None, conf)
+    if f.type == "enum":
+        rest = round((1 - conf) / (len(f.options or {}) - 1), DIGITS)
+        probs = {k: (conf if k == value else rest) for k in f.options or {}}
+        return TFieldValue(value, conf, probabilities=probs)
+    if f.type == "boolean":
+        return TFieldValue(value, conf, probability=round(conf if value else 1 - conf, DIGITS))
+    return TFieldValue(value, conf)

dev_double/providers/openai/__init__.py

@@ -0,0 +1 @@
+

dev_double/providers/openai/decision/__init__.py

@@ -0,0 +1,3 @@
+from .engine_openai_impl import EngineOpenAIImpl
+
+__all__ = ["EngineOpenAIImpl"]

dev_double/providers/openai/decision/engine_openai_impl.py

@@ -0,0 +1,154 @@
+"""Engine for any server with an OpenAI-style /chat/completions endpoint that
+returns logprobs: Ollama (>= 0.12.11), vLLM, llama.cpp server, LM Studio, or a
+hosted provider your company already approved.
+
+The model is asked to reply with a single label, and the answer is read from
+the probabilities the model gave each candidate token. One short
+completion per question; no text generation, no JSON parsing.
+
+It is also an ``IGenerator``: for extraction fields that need free text
+(names, amounts, dates) it generates one JSON object, and returns the tokens'
+probabilities so each value gets its own confidence.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Optional
+
+import httpx
+
+from dev_double.core.decision.errors import EngineError
+from dev_double.core.decision.i_engine import IEngine
+from dev_double.core.decision.i_generator import IGenerator
+from dev_double.core.decision.label_scoring import label_distribution, label_from_text, one_hot, uniform
+from dev_double.core.decision.t_generate import TGenerateQuery, TGenerateResult
+from dev_double.core.decision.t_label_query import TLabelQuery, TLabelResult, TPosition
+from dev_double.core.decision.t_usage import TUsage
+
+# Enough tokens to get past a leading newline and to tell apart options that
+# share their first token ("re" + "fund" vs "re" + "booking").
+MAX_TOKENS = 8
+TOP_LOGPROBS = 20
+LOW_MASS = 0.5
+
+
+class EngineOpenAIImpl(IEngine, IGenerator):
+    name = "openai"
+
+    def __init__(
+        self,
+        base_url: str,
+        model: str,
+        api_key: Optional[str] = None,
+        timeout: float = 120.0,
+        client: Optional[httpx.AsyncClient] = None,
+    ) -> None:
+        self._model = model
+        self._url = base_url.rstrip("/") + "/chat/completions"
+        self._headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        self._client = client or httpx.AsyncClient(timeout=timeout)
+
+    @property
+    def model(self) -> str:
+        return self._model
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    async def _complete(self, system: str, user: str, **options: object) -> tuple[dict, TUsage]:
+        """One /chat/completions call; returns the first choice and the usage."""
+        body = {
+            "model": self._model,
+            "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
+            "temperature": 0,
+            "logprobs": True,
+            **options,
+        }
+        try:
+            response = await self._client.post(self._url, json=body, headers=self._headers)
+        except httpx.HTTPError as exc:
+            raise EngineError(f"Could not reach model server at {self._url}: {exc}") from exc
+        if response.status_code >= 400:
+            raise EngineError(
+                f"Model server returned {response.status_code}: {response.text[:500]}"
+            )
+
+        data = response.json()
+        try:
+            choice = data["choices"][0]
+        except (KeyError, IndexError) as exc:
+            raise EngineError(f"Unexpected response from model server: {str(data)[:500]}") from exc
+
+        raw_usage = data.get("usage") or {}
+        usage = TUsage(
+            input_tokens=raw_usage.get("prompt_tokens", 0),
+            output_tokens=raw_usage.get("completion_tokens", 0),
+        )
+        return choice, usage
+
+    async def distribution(self, query: TLabelQuery) -> TLabelResult:
+        choice, usage = await self._complete(
+            query.system, query.user, max_tokens=MAX_TOKENS, top_logprobs=TOP_LOGPROBS
+        )
+        content = (choice.get("message") or {}).get("content") or ""
+        positions = ((choice.get("logprobs") or {}).get("content")) or []
+        return self._read(positions, content, query.labels, usage)
+
+    async def generate(self, query: TGenerateQuery) -> TGenerateResult:
+        """Free text (or one JSON object) at temperature 0; confidence is exp(mean
+        logprob) of the generated tokens, which are returned for per-value scoring."""
+        options: dict = {"max_tokens": query.max_tokens, "top_logprobs": 1}
+        if query.json:
+            options["response_format"] = {"type": "json_object"}
+        choice, usage = await self._complete(query.system, query.user, **options)
+        content = (choice.get("message") or {}).get("content") or ""
+        positions = ((choice.get("logprobs") or {}).get("content")) or []
+        tokens = [(p.get("token", ""), float(p["logprob"])) for p in positions if "logprob" in p]
+        warnings: list[str] = []
+        if tokens:
+            confidence = min(1.0, math.exp(sum(lp for _, lp in tokens) / len(tokens)))
+        else:
+            confidence = 1.0
+            warnings.append(
+                "The model server returned no logprobs, so extracted values' confidence is a "
+                "placeholder 1.0. Use a server with logprobs support (e.g. Ollama >= 0.12.11)."
+            )
+        if choice.get("finish_reason") == "length":
+            warnings.append(f"The output was cut off at {query.max_tokens} tokens.")
+        return TGenerateResult(content, confidence, usage, warnings, tokens)
+
+    @staticmethod
+    def _read(raw_positions: list[dict], content: str, labels: list[str], usage: TUsage) -> TLabelResult:
+        warnings: list[str] = []
+        positions = [
+            TPosition(
+                token=pos.get("token", ""),
+                candidates=[(c["token"], c["logprob"]) for c in pos.get("top_logprobs") or []],
+            )
+            for pos in raw_positions
+        ]
+        dist, mass = label_distribution(positions, labels)
+        if mass > 0:
+            if mass < LOW_MASS:
+                warnings.append(
+                    f"Only {mass:.0%} of the model's probability landed on a valid label; "
+                    "this answer is unreliable. Try a larger or instruction-tuned model."
+                )
+            return TLabelResult(dist, usage, warnings)
+
+        # No usable logprobs: fall back to the generated text.
+        if not positions:
+            warnings.append(
+                "The model server returned no logprobs, so probabilities are 0/1 guesses, "
+                "not calibrated. Use a server with logprobs support (e.g. Ollama >= 0.12.11)."
+            )
+        match = label_from_text(content, labels)
+        if match is not None:
+            return TLabelResult(one_hot(labels, match), usage, warnings)
+        warnings.append(
+            f"The model did not answer with a valid label (got {content[:40]!r}); "
+            "returning a uniform distribution. Reasoning models that emit <think> "
+            "first are not supported; use a non-thinking instruct model."
+        )
+        return TLabelResult(uniform(labels), usage, warnings)

dev_double/providers/std/__init__.py

@@ -0,0 +1 @@
+

dev_double/providers/std/decision/__init__.py

@@ -0,0 +1,4 @@
+from .clock_std_impl import ClockStdImpl
+from .id_provider_std_impl import IdProviderStdImpl
+
+__all__ = ["ClockStdImpl", "IdProviderStdImpl"]

dev_double/providers/std/decision/clock_std_impl.py

@@ -0,0 +1,12 @@
+"""The process's monotonic high-resolution clock."""
+
+from __future__ import annotations
+
+import time
+
+from dev_double.core.decision.i_clock import IClock
+
+
+class ClockStdImpl(IClock):
+    def now(self) -> float:
+        return time.perf_counter()

dev_double/providers/std/decision/id_provider_std_impl.py

@@ -0,0 +1,12 @@
+"""Random UUID4 ids."""
+
+from __future__ import annotations
+
+import uuid
+
+from dev_double.core.decision.i_id_provider import IIdProvider
+
+
+class IdProviderStdImpl(IIdProvider):
+    def new_id(self) -> str:
+        return str(uuid.uuid4())

dev_double/providers/systemone/__init__.py

@@ -0,0 +1 @@
+

dev_double/providers/systemone/decision/__init__.py

@@ -0,0 +1,3 @@
+from .decider_system_one_impl import DeciderSystemOneImpl
+
+__all__ = ["DeciderSystemOneImpl"]

dev_double/providers/systemone/decision/decider_system_one_impl.py

@@ -0,0 +1,100 @@
+"""A real System 1 decision model (Kev, Laya) served at ``POST /v1/systemone``.
+
+Each question is sent as-is (no dev-double prompt): these models answer
+typed questions natively, so this implements ``IDecider`` directly.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Optional
+
+import httpx
+
+from dev_double.core.decision.answer_shaping import shape_answer
+from dev_double.core.decision.errors import EngineError
+from dev_double.core.decision.i_decider import IDecider
+from dev_double.core.decision.t_answer import TAnswer
+from dev_double.core.decision.t_input import TInputValue
+from dev_double.core.decision.t_question import TBinaryQuestion, TChoiceQuestion, TQuestion
+from dev_double.core.decision.t_usage import TUsage
+from dev_double.core.decision.tracker import Tracker
+
+PATH = "/v1/systemone"
+
+
+def to_wire(q: TQuestion) -> dict[str, Any]:
+    """A core question in the System 1 wire format."""
+    if isinstance(q, TBinaryQuestion):
+        body: dict[str, Any] = {"type": "noul", "instructions": q.question}
+        criteria = {k: v for k, v in (("true", q.yes), ("false", q.no)) if v}
+        if criteria:
+            body["criteria"] = criteria
+        return body
+    if isinstance(q, TChoiceQuestion):
+        return {"type": "choice", "instructions": q.question, "criteria": q.options}
+    return {"type": "score", "instructions": q.question, "criteria": q.levels}
+
+
+def from_wire(q: TQuestion, answer: dict[str, Any]) -> dict[str, float]:
+    """The wire answer as a distribution over the question's labels."""
+    if isinstance(q, TBinaryQuestion):
+        p = float(answer["noul"])
+        return {"yes": p, "no": 1 - p}
+    return {str(k): float(v) for k, v in answer["probabilities"].items()}
+
+
+class DeciderSystemOneImpl(IDecider):
+    def __init__(
+        self,
+        name: str,
+        base_url: str,
+        api_key: Optional[str] = None,
+        *,
+        model: Optional[str] = None,
+        max_concurrency: int = 1,
+        timeout: float = 120.0,
+        client: Optional[httpx.AsyncClient] = None,
+    ) -> None:
+        self._name = name
+        self._model = model or name
+        self._base_url = base_url.rstrip("/")
+        headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+        self._client = client or httpx.AsyncClient(timeout=timeout)
+        self._headers = headers
+        self._limit = asyncio.Semaphore(max_concurrency)
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def model(self) -> str:
+        return self._model
+
+    async def ask(
+        self, value: TInputValue, question: TQuestion, tracker: Tracker, label: str = ""
+    ) -> TAnswer:
+        payload = {"state": value, "questions": {"q": to_wire(question)}}
+        url = self._base_url + PATH
+        try:
+            async with self._limit:
+                r = await self._client.post(url, json=payload, headers=self._headers)
+        except httpx.HTTPError as exc:
+            raise EngineError(f"Could not reach System 1 server at {url}: {exc}") from exc
+        if r.status_code >= 400:
+            raise EngineError(f"System 1 server returned {r.status_code}: {r.text[:500]}")
+        data = r.json()
+        try:
+            probs = from_wire(question, data["answers"]["q"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise EngineError(f"Unexpected response from System 1 server: {str(data)[:500]}") from exc
+        usage = data.get("usage") or {}
+        tracker.usage.add(
+            TUsage(input_tokens=usage.get("input_tokens", 0), output_tokens=usage.get("output_tokens", 0))
+        )
+        # The server's probabilities are passed through unrounded, as it sent them.
+        return shape_answer(question, probs, round_probs=False)
+
+    async def aclose(self) -> None:
+        await self._client.aclose()