dev-double 0.1.0__py3-none-any.whl

dev_double/core/decision/t_question.py

@@ -0,0 +1,70 @@
+"""The three question types every use case is built from.
+
+- ``binary``: is this statement true? -> probability of "yes"
+- ``choice``: which of these unordered options? -> option + distribution
+- ``scale``:  which of these ordered levels? -> expected level + distribution
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Union
+
+# Probabilities are read from the model's top 20 candidate tokens (the most
+# OpenAI-compatible servers return), so more options than that can't all be seen.
+MAX_OPTIONS = 20
+MAX_LEVELS = 10
+
+
+def check_option_keys(options: dict[str, str]) -> None:
+    """The model answers with an option key, compared ignoring case and
+    punctuation, so keys must stay distinct under that comparison."""
+    seen: dict[str, str] = {}
+    for key in options:
+        norm = "".join(ch for ch in key.lower() if ch.isalnum())
+        if not norm:
+            raise ValueError(f"Option key {key!r} must contain a letter or digit.")
+        if norm in seen:
+            raise ValueError(f"Option keys {seen[norm]!r} and {key!r} are too similar; rename one.")
+        seen[norm] = key
+
+
+def check_options(options: dict[str, str], what: str = "options") -> None:
+    """2 to MAX_OPTIONS options with distinct keys."""
+    if not 2 <= len(options) <= MAX_OPTIONS:
+        raise ValueError(f"`{what}` must have 2 to {MAX_OPTIONS} entries, got {len(options)}.")
+    check_option_keys(options)
+
+
+def check_levels(levels: list[str], what: str = "levels") -> None:
+    """2 to MAX_LEVELS ordered levels."""
+    if not 2 <= len(levels) <= MAX_LEVELS:
+        raise ValueError(f"`{what}` must have 2 to {MAX_LEVELS} entries, got {len(levels)}.")
+
+
+@dataclass(frozen=True)
+class TBinaryQuestion:
+    question: str
+    yes: Optional[str] = None  # what counts as yes
+    no: Optional[str] = None  # what counts as no
+
+
+@dataclass(frozen=True)
+class TChoiceQuestion:
+    question: str
+    options: dict[str, str]  # option key -> description; keys are returned as the answer
+
+    def __post_init__(self) -> None:
+        check_options(self.options)
+
+
+@dataclass(frozen=True)
+class TScaleQuestion:
+    question: str
+    levels: list[str]  # ordered descriptions, lowest first; level i is returned as i
+
+    def __post_init__(self) -> None:
+        check_levels(self.levels)
+
+
+TQuestion = Union[TBinaryQuestion, TChoiceQuestion, TScaleQuestion]

dev_double/core/decision/t_rerank.py

@@ -0,0 +1,36 @@
+"""Order documents by relevance to a query."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .t_meta import TMeta
+
+
+@dataclass(frozen=True)
+class TRerankRequest:
+    query: str
+    documents: list[str]
+    top_n: Optional[int] = None
+    return_documents: bool = False
+
+    def __post_init__(self) -> None:
+        if not self.documents:
+            raise ValueError("`documents` must not be empty.")
+        if self.top_n is not None and self.top_n < 1:
+            raise ValueError("`top_n` must be at least 1.")
+
+
+@dataclass(frozen=True)
+class TRerankResult:
+    index: int
+    relevance_score: float
+    document: Optional[str] = None  # set when return_documents is true
+
+
+@dataclass(frozen=True)
+class TRerankResponse:
+    id: str
+    results: list[TRerankResult]
+    meta: TMeta

dev_double/core/decision/t_route.py

@@ -0,0 +1,28 @@
+"""Model routing: which model (or handler) should take a request."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .t_input import TInputValue
+from .t_meta import TMeta
+from .t_question import check_options
+
+
+@dataclass(frozen=True)
+class TRouteRequest:
+    input: TInputValue
+    routes: Optional[dict[str, str]] = None  # route -> when to use it; None = DEFAULT_ROUTES
+
+    def __post_init__(self) -> None:
+        if self.routes is not None:
+            check_options(self.routes, "routes")
+
+
+@dataclass(frozen=True)
+class TRouteResponse:
+    route: str
+    probabilities: dict[str, float]
+    confidence: float
+    meta: TMeta

dev_double/core/decision/t_usage.py

@@ -0,0 +1,15 @@
+"""Token usage, accumulated across the model calls of one request."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class TUsage:
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+    def add(self, other: "TUsage") -> None:
+        self.input_tokens += other.input_tokens
+        self.output_tokens += other.output_tokens

dev_double/core/decision/tracker.py

@@ -0,0 +1,27 @@
+"""Collects usage, warnings, and latency across the model calls of one request."""
+
+from __future__ import annotations
+
+from .i_clock import IClock
+from .t_meta import TMeta
+from .t_usage import TUsage
+
+
+class Tracker:
+    def __init__(self, clock: IClock) -> None:
+        self._clock = clock
+        self.usage = TUsage()
+        self.warnings: list[str] = []
+        self.started = clock.now()
+
+    def warn(self, message: str, label: str = "") -> None:
+        self.warnings.append(f"{label}: {message}" if label else message)
+
+    def meta(self, engine: str, model: str) -> TMeta:
+        return TMeta(
+            engine=engine,
+            model=model,
+            latency_ms=round((self._clock.now() - self.started) * 1000),
+            usage=self.usage,
+            warnings=list(dict.fromkeys(self.warnings)),
+        )

dev_double/core/decision/use_case_questions.py

@@ -0,0 +1,80 @@
+"""The questions and payloads each use case asks. Wording was tuned by experiment."""
+
+from __future__ import annotations
+
+from .defaults import DEFAULT_OUTCOMES, DEFAULT_POLICIES, DEFAULT_ROUTES, DEFAULT_RUBRIC
+from .t_gate import TGateRequest
+from .t_guard import TGuardRequest
+from .t_judge import TJudgeRequest
+from .t_question import TBinaryQuestion, TChoiceQuestion, TScaleQuestion
+from .t_route import TRouteRequest
+
+
+def route_question(req: TRouteRequest) -> TChoiceQuestion:
+    return TChoiceQuestion(
+        question="Which option is the best fit for handling this request?",
+        options=req.routes or DEFAULT_ROUTES,
+    )
+
+
+def guard_questions(req: TGuardRequest) -> dict[str, TBinaryQuestion]:
+    policies = dict(req.policies or DEFAULT_POLICIES)
+    if req.scope:
+        policies["off_topic"] = f"The input is unrelated to this assistant's purpose: {req.scope}"
+    return {
+        name: TBinaryQuestion(question=f"Does the input do the following? {description}")
+        for name, description in policies.items()
+    }
+
+
+def gate_payload(req: TGateRequest) -> dict:
+    arguments = req.tool_call.arguments
+    payload: dict = {
+        "tool_call": {
+            "name": req.tool_call.name,
+            "arguments": arguments if isinstance(arguments, dict) else str(arguments),
+        }
+    }
+    if req.context is not None:
+        payload["context"] = req.context
+    if req.policy:
+        payload["policy"] = req.policy
+    return payload
+
+
+def gate_question(req: TGateRequest) -> TChoiceQuestion:
+    return TChoiceQuestion(
+        question=(
+            "An AI agent wants to make this tool call. What should happen? "
+            "Check that the call matches what the user asked for, and whether it moves money, "
+            "deletes data, or is costly or irreversible."
+        ),
+        options=req.outcomes or DEFAULT_OUTCOMES,
+    )
+
+
+def judge_payload(req: TJudgeRequest) -> dict:
+    # Task first, output last: the model reads what was asked before judging the answer.
+    payload: dict = {}
+    if req.input is not None:
+        payload["task"] = req.input
+    if req.reference is not None:
+        payload["reference_answer"] = req.reference
+    payload["output"] = req.output
+    return payload
+
+
+def judge_question(req: TJudgeRequest) -> TScaleQuestion:
+    return TScaleQuestion(
+        question=(
+            f"How good is the output as an answer to the task? Criteria: {req.criteria} "
+            "If a reference answer is given, an output that contradicts it is wrong."
+        ),
+        levels=req.levels or DEFAULT_RUBRIC,
+    )
+
+
+def rerank_question(query: str) -> TBinaryQuestion:
+    return TBinaryQuestion(
+        question=f"Is this document relevant to the query below and useful for answering it?\nQuery: {query}",
+    )

dev_double/core/decision/value_parsing.py

@@ -0,0 +1,109 @@
+"""Pure helpers that turn a model's short free-text answer into a typed value.
+
+Models add noise around the value: quotes, a trailing period, currency
+symbols, thousands separators. We strip it rather than reject the answer.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Optional
+
+from .t_extract import TEXT_TYPES, TValue
+
+NONE_MARKERS = frozenset({"none", "n/a", "null", ""})
+_QUOTES = "\"'`"
+_DIGITS = "0123456789"
+_MINUS = "-−"
+_CURRENCY = "$€£¥₹¢"
+
+
+def clean_text(text: str) -> str:
+    """Strip whitespace, surrounding quotes, and one trailing period."""
+    s = text.strip().strip(_QUOTES).strip()
+    if s.endswith("."):
+        s = s[:-1].rstrip()
+    return s.strip(_QUOTES).strip()
+
+
+def _is_negative(before: str) -> bool:
+    """A minus sign right before the number, allowing spaces or a currency symbol between ("-$5")."""
+    for ch in reversed(before):
+        if ch in _MINUS:
+            return True
+        if not (ch.isspace() or ch in _CURRENCY):
+            return False
+    return False
+
+
+def _normalize_separators(run: str) -> str:
+    """Digits with "," and "." as thousands or decimal separators -> a float literal.
+
+    When both appear, the last one is the decimal separator ("1,200.50", "1.200,50").
+    A lone comma is a decimal comma unless exactly three digits follow it ("12,50" vs "1,200").
+    Several dots are thousands separators ("1.200.000").
+    """
+    comma, dot = run.rfind(","), run.rfind(".")
+    if comma >= 0 and dot >= 0:
+        if comma > dot:
+            return run.replace(".", "").replace(",", ".")
+        return run.replace(",", "")
+    if comma >= 0:
+        if run.count(",") == 1 and len(run) - comma - 1 != 3:
+            return run.replace(",", ".")
+        return run.replace(",", "")
+    if run.count(".") > 1:
+        return run.replace(".", "")
+    return run
+
+
+def parse_number(text: str) -> Optional[float]:
+    """The first number in the text, or None if it has none."""
+    start = next((i for i, ch in enumerate(text) if ch in _DIGITS), None)
+    if start is None:
+        return None
+    end = start
+    while end < len(text) and (text[end] in _DIGITS or text[end] in ".,"):
+        end += 1
+    run = text[start:end].rstrip(".,")
+    try:
+        value = float(_normalize_separators(run))
+    except ValueError:
+        return None
+    return -value if _is_negative(text[:start]) else value
+
+
+def coerce_json_value(raw: object, kind: str) -> tuple[TValue, Optional[str]]:
+    """(value, warning) for a JSON value of a string, number, or integer field.
+
+    Numbers pass through directly; anything else is read as text with ``parse_field_text``.
+    """
+    if kind in ("number", "integer") and isinstance(raw, (int, float)) and not isinstance(raw, bool):
+        if kind == "number":
+            return float(raw), None
+        return (int(raw), None) if float(raw).is_integer() else (None, f"{raw!r} is not a whole number; left empty")
+    text = raw if isinstance(raw, str) else json.dumps(raw, ensure_ascii=False)
+    return parse_field_text(text, kind)
+
+
+def parse_field_text(text: str, kind: str) -> tuple[TValue, Optional[str]]:
+    """(value, warning) for a string, number, or integer field.
+
+    "NONE" and similar markers mean the input doesn't contain the value: (None, None).
+    An answer that can't be read as the type gives (None, a warning).
+    """
+    if kind not in TEXT_TYPES:
+        raise ValueError(f"Only {', '.join(TEXT_TYPES)} fields are parsed from text, not {kind!r}.")
+    s = clean_text(text)
+    if s.lower() in NONE_MARKERS:
+        return None, None
+    if kind == "string":
+        return s, None
+    number = parse_number(s)
+    if number is None:
+        return None, f"could not read a number from {s[:40]!r}; left empty"
+    if kind == "integer":
+        if not number.is_integer():
+            return None, f"{s[:40]!r} is not a whole number; left empty"
+        return int(number), None
+    return number, None

dev_double/providers/__init__.py

@@ -0,0 +1 @@
+"""Provider layer: technology-specific implementations of core interfaces. Providers never import each other."""

dev_double/providers/mock/__init__.py

@@ -0,0 +1 @@
+

dev_double/providers/mock/decision/__init__.py

@@ -0,0 +1,5 @@
+from .clock_mock_impl import ClockMockImpl
+from .engine_mock_impl import EngineMockImpl
+from .id_provider_mock_impl import IdProviderMockImpl
+
+__all__ = ["ClockMockImpl", "EngineMockImpl", "IdProviderMockImpl"]

dev_double/providers/mock/decision/clock_mock_impl.py

@@ -0,0 +1,19 @@
+"""A deterministic clock: starts at ``start`` and advances ``step`` seconds per read."""
+
+from __future__ import annotations
+
+from dev_double.core.decision.i_clock import IClock
+
+
+class ClockMockImpl(IClock):
+    def __init__(self, start: float = 0.0, step: float = 0.0) -> None:
+        self._t = start
+        self._step = step
+
+    def now(self) -> float:
+        t = self._t
+        self._t += self._step
+        return t
+
+    def advance(self, seconds: float) -> None:
+        self._t += seconds

dev_double/providers/mock/decision/engine_mock_impl.py

@@ -0,0 +1,78 @@
+"""A model-free engine for CI and plumbing tests.
+
+It scores each label by word overlap between the input and the label's
+description. It is deterministic and instant, and its answers are only
+loosely sensible. Use it to test your integration code, never its quality.
+
+It also "generates" extraction values: for each field, the text after the
+colon on the first input line whose label (before the colon) contains every
+word of the field name, so ``due_date`` finds ``Due date: 2026-10-01``. In JSON
+mode it returns one object with a key per field (null when not found) and
+confidence 1.0; otherwise the first field's value, or NONE with confidence 0.5.
+It returns no token logprobs.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from typing import Optional
+
+from dev_double.core.decision.i_engine import IEngine
+from dev_double.core.decision.i_generator import IGenerator
+from dev_double.core.decision.t_generate import TGenerateQuery, TGenerateResult
+from dev_double.core.decision.t_label_query import TLabelQuery, TLabelResult
+from dev_double.core.decision.t_usage import TUsage
+
+_WORD = re.compile(r"[a-z0-9]+")
+_STOP = frozenset(
+    "a an and are as at be by for from has have in is it its of on or that the this to was "
+    "were will with you your does do not no yes which what how".split()
+)
+
+
+def _words(text: str) -> set[str]:
+    return {w for w in _WORD.findall(text.lower()) if w not in _STOP and len(w) > 2}
+
+
+class EngineMockImpl(IEngine, IGenerator):
+    name = "mock"
+    model = "mock-overlap"
+
+    async def aclose(self) -> None:
+        return None
+
+    async def distribution(self, query: TLabelQuery) -> TLabelResult:
+        input_words = _words(query.input_text)
+        scores = []
+        for description in query.descriptions:
+            desc = _words(description)
+            overlap = len(input_words & desc)
+            scores.append(overlap / (1 + math.sqrt(len(desc))))
+        top = max(scores)
+        exps = [math.exp(4 * (s - top)) for s in scores]
+        total = sum(exps)
+        probs = {label: e / total for label, e in zip(query.labels, exps)}
+        usage = TUsage(
+            input_tokens=len(query.system.split()) + len(query.user.split()),
+            output_tokens=1,
+        )
+        return TLabelResult(probs, usage)
+
+    @staticmethod
+    def _lookup(field: str, input_text: str) -> Optional[str]:
+        wanted = set(_WORD.findall(field.lower()))
+        for line in input_text.splitlines():
+            key, colon, rest = line.partition(":")
+            if colon and wanted and wanted <= set(_WORD.findall(key.lower())):
+                return rest.strip().rstrip(",").strip().strip('"')
+        return None
+
+    async def generate(self, query: TGenerateQuery) -> TGenerateResult:
+        found = {name: self._lookup(name, query.input_text) for name in query.fields}
+        usage = TUsage(input_tokens=len(query.system.split()) + len(query.user.split()), output_tokens=1)
+        if query.json:
+            return TGenerateResult(json.dumps(found), 1.0, usage)
+        first = next(iter(found.values()), None)
+        return TGenerateResult(first, 1.0, usage) if first is not None else TGenerateResult("NONE", 0.5, usage)

dev_double/providers/mock/decision/id_provider_mock_impl.py

@@ -0,0 +1,15 @@
+"""Deterministic ids: ``<prefix>-1``, ``<prefix>-2``, ..."""
+
+from __future__ import annotations
+
+from dev_double.core.decision.i_id_provider import IIdProvider
+
+
+class IdProviderMockImpl(IIdProvider):
+    def __init__(self, prefix: str = "mock") -> None:
+        self._prefix = prefix
+        self._n = 0
+
+    def new_id(self) -> str:
+        self._n += 1
+        return f"{self._prefix}-{self._n}"

dev_double/providers/needle/__init__.py

@@ -0,0 +1 @@
+

dev_double/providers/needle/decision/__init__.py

@@ -0,0 +1,4 @@
+# needle itself is imported lazily when DeciderNeedleImpl is constructed.
+from .decider_needle_impl import DeciderNeedleImpl
+
+__all__ = ["DeciderNeedleImpl"]

dev_double/providers/needle/decision/decider_needle_impl.py

@@ -0,0 +1,117 @@
+"""Cactus Needle, a tiny on-device tool-calling model, as a decider.
+
+Needle isn't a decision model: it fills tool calls. Its docs treat
+classification as extraction with an enum field, so each question becomes one
+tool whose `label` argument may only be one of the allowed labels. Needle
+returns one calibrated confidence for the call, not a distribution, so the
+other labels share the remainder evenly. Rerank uses Needle's embeddings.
+Extraction is native: one ``record`` tool call carries every field (see record_tool).
+
+Needs the optional extra: `pip install "dev-double[needle]"`. ``needle`` is
+imported lazily, so the package imports without it. Telemetry is off by
+default (NEEDLE_TELEMETRY=0, DO_NOT_TRACK=1); the library reads
+NEEDLE3_LIB_PATH itself if you need to point it at a local build.
+"""
+
+from __future__ import annotations
+
+import math
+import os
+from typing import Any, Literal, Optional
+
+from dev_double.core.decision import prompts
+from dev_double.core.decision.answer_shaping import question_labels, shape_answer
+from dev_double.core.decision.confidence import DIGITS
+from dev_double.core.decision.i_decider import IDecider
+from dev_double.core.decision.i_extractor import IExtractor
+from dev_double.core.decision.i_reranker import IReranker
+from dev_double.core.decision.t_answer import TAnswer
+from dev_double.core.decision.t_extract import EMPTY, TExtractRequest, TFieldValue
+from dev_double.core.decision.t_input import TInputValue
+from dev_double.core.decision.t_question import TQuestion
+from dev_double.core.decision.tracker import Tracker
+
+from .record_tool import RECORD_SYSTEM, fields_key, record_model, value_from_argument
+
+
+def _import_needle() -> Any:
+    os.environ.setdefault("NEEDLE_TELEMETRY", "0")
+    os.environ.setdefault("DO_NOT_TRACK", "1")
+    import needle
+
+    return needle
+
+
+def _cosine(a: list[float], b: list[float]) -> float:
+    dot = sum(x * y for x, y in zip(a, b))
+    return dot / (math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b)) or 1)
+
+
+class DeciderNeedleImpl(IDecider, IReranker, IExtractor):
+    name = "needle"
+    model = "needle3"
+
+    def __init__(self) -> None:
+        self._needle = _import_needle()
+        self._agents: dict[tuple, Any] = {}
+        self._extractors: dict[tuple, Any] = {}
+        self._embedder: Optional[Any] = None
+
+    def _agent(self, q: TQuestion, labels: dict[str, str]) -> Any:
+        from pydantic import Field, create_model  # the tool schema Needle expects
+
+        key = (q.question, tuple(labels.items()))
+        if key not in self._agents:
+            options = "; ".join(f"{k}: {v}" for k, v in labels.items())
+            tool = create_model(
+                "answer",
+                __doc__=f"Record the answer to: {q.question}",
+                label=(Literal[tuple(labels)], Field(description=f"Exactly one of: {options}")),
+            )
+            system = f"{q.question}\nAllowed labels: {options}\nAlways call answer with one label."
+            self._agents[key] = self._needle.Needle(tools=[tool], system=system, stateless=True)
+        return self._agents[key]
+
+    async def ask(
+        self, value: TInputValue, question: TQuestion, tracker: Tracker, label: str = ""
+    ) -> TAnswer:
+        labels = question_labels(question)
+        r = self._agent(question, labels).complete(prompts.render_input(value))
+        calls = r.get("function_calls") or r.get("suppressed_calls") or []
+        chosen = (calls[0].get("arguments") or {}).get("label") if calls else None
+        conf = r.get("confidence")
+        conf = 1.0 if conf is None else float(conf)
+        if chosen in labels:
+            rest = (1 - conf) / (len(labels) - 1)
+            probs = {k: (conf if k == chosen else rest) for k in labels}
+        else:
+            tracker.warn("needle made no valid call", label)
+            probs = {k: 1 / len(labels) for k in labels}
+        return shape_answer(question, probs)
+
+    async def extract(self, req: TExtractRequest, tracker: Tracker) -> dict[str, TFieldValue]:
+        key = fields_key(req.fields)
+        if key not in self._extractors:
+            tool = record_model(req.fields)
+            self._extractors[key] = self._needle.Needle(tools=[tool], system=RECORD_SYSTEM, stateless=True)
+        r = self._extractors[key].complete(prompts.render_input(req.input))
+        calls = r.get("function_calls") or r.get("suppressed_calls") or []
+        if not calls:
+            tracker.warn("needle made no valid call")
+            return {name: EMPTY for name in req.fields}
+        args = calls[0].get("arguments") or {}
+        conf = r.get("confidence")
+        conf = 1.0 if conf is None else min(1.0, max(0.0, float(conf)))
+        return {name: value_from_argument(name, f, args.get(name), conf, tracker) for name, f in req.fields.items()}
+
+    async def rerank_scores(self, query: str, documents: list[str], tracker: Tracker) -> list[float]:
+        if self._embedder is None:
+            self._embedder = self._needle.Needle(stateless=True)
+        embed = self._embedder.embed
+        q = embed(query)
+        return [round(_cosine(q, embed(text)), DIGITS) for text in documents]
+
+    async def aclose(self) -> None:
+        agents = [*self._agents.values(), *self._extractors.values()]
+        for agent in [*agents, *([self._embedder] if self._embedder else [])]:
+            agent.close()