dev-double 0.1.0__py3-none-any.whl

dev_double/core/decision/prompts.py

@@ -0,0 +1,130 @@
+"""Turns a question into a single-label prompt.
+
+Every prompt ends by asking for exactly one label (an option name, a digit, or
+yes/no), so the answer sits in the first few tokens, whose probabilities we can read.
+
+Choices are answered by option name, not by letter: small models over-pick
+"B" when options are lettered, which skews the probabilities.
+"""
+
+from __future__ import annotations
+
+import json
+
+from .t_extract import TExtractField
+from .t_generate import TGenerateQuery
+from .t_input import TInputValue
+from .t_label_query import TLabelQuery
+from .t_question import TBinaryQuestion, TChoiceQuestion, TQuestion, TScaleQuestion
+
+SYSTEM = (
+    "You are a precise decision function. Read the input, then answer the question "
+    "with exactly one label from the allowed labels. Output only the label: no "
+    "explanation, no punctuation, no other text."
+)
+
+# Extraction: all free-text fields of a request in one JSON object. Wording
+# chosen by A/B test against a per-field prompt (which small models answered
+# with NONE for values plainly present, or echoed the field description).
+RECORD_SYSTEM = (
+    "You extract data from text into JSON. Output one JSON object with exactly the requested "
+    "keys. Copy values from the text, converting them to the requested format. Use null only "
+    "when the text gives no information for a key. Output only the JSON."
+)
+RECORD_MAX_TOKENS = 512
+
+def render_input(value: TInputValue) -> str:
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False, indent=2)
+
+
+def _frame(question: str, input_text: str, body: str) -> str:
+    # Asking the question before and after the input measurably helps small
+    # models: they read the input knowing what to look for.
+    return f"Question: {question}\n\n<input>\n{input_text}\n</input>\n\n{body}"
+
+
+def binary(input_text: str, q: TBinaryQuestion) -> TLabelQuery:
+    lines = [f"Question: {q.question}"]
+    if q.yes:
+        lines.append(f'Answer "yes" if: {q.yes}')
+    if q.no:
+        lines.append(f'Answer "no" if: {q.no}')
+    lines.append("Reply with exactly one word: yes or no.")
+    return TLabelQuery(
+        system=SYSTEM,
+        user=_frame(q.question, input_text, "\n".join(lines)),
+        labels=["yes", "no"],
+        input_text=input_text,
+        question=q.question,
+        descriptions=[f"{q.question} {q.yes or ''}", q.no or ""],
+    )
+
+
+def choice(input_text: str, q: TChoiceQuestion) -> TLabelQuery:
+    keys = list(q.options)
+    option_lines = [f"- {key}: {q.options[key]}" for key in keys]
+    body = (
+        f"Question: {q.question}\n\nOptions:\n"
+        + "\n".join(option_lines)
+        + f"\n\nReply with exactly one option name: {', '.join(keys)}."
+    )
+    return TLabelQuery(
+        system=SYSTEM,
+        user=_frame(q.question, input_text, body),
+        labels=keys,
+        input_text=input_text,
+        question=q.question,
+        descriptions=[f"{key} {q.options[key]}" for key in keys],
+    )
+
+
+def scale(input_text: str, q: TScaleQuestion) -> TLabelQuery:
+    digits = [str(i) for i in range(len(q.levels))]
+    level_lines = [f"{d}: {text}" for d, text in zip(digits, q.levels)]
+    body = (
+        f"Question: {q.question}\n\nLevels (lowest first):\n"
+        + "\n".join(level_lines)
+        + f"\n\nReply with exactly one digit: {', '.join(digits)}."
+    )
+    return TLabelQuery(
+        system=SYSTEM,
+        user=_frame(q.question, input_text, body),
+        labels=digits,
+        input_text=input_text,
+        question=q.question,
+        descriptions=list(q.levels),
+    )
+
+
+def query_for(input_text: str, q: TQuestion) -> TLabelQuery:
+    """Dispatch on the question type."""
+    if isinstance(q, TBinaryQuestion):
+        return binary(input_text, q)
+    if isinstance(q, TChoiceQuestion):
+        return choice(input_text, q)
+    return scale(input_text, q)
+
+
+def record_max_tokens(n_fields: int) -> int:
+    return min(RECORD_MAX_TOKENS, 48 + 40 * n_fields)
+
+
+def record_key_line(name: str, field: TExtractField) -> str:
+    return f'- "{name}" ({field.type}): {field.description or name.replace("_", " ")}'
+
+
+def extract_record(input_text: str, fields: dict[str, TExtractField]) -> TGenerateQuery:
+    """Free-text fields (string, number, integer) as one JSON object, keys listed
+    before and after the text."""
+    keys = "\n".join(record_key_line(name, f) for name, f in fields.items())
+    user = f"Keys:\n{keys}\n\n<text>\n{input_text}\n</text>\n\nFill in the keys from the text above:\n{keys}\n\nJSON:"
+    return TGenerateQuery(
+        system=RECORD_SYSTEM,
+        user=user,
+        max_tokens=record_max_tokens(len(fields)),
+        json=True,
+        input_text=input_text,
+        fields=tuple(fields),
+    )

dev_double/core/decision/record_parsing.py

@@ -0,0 +1,153 @@
+"""Pure helpers that turn a model's JSON record into typed field values, with a
+per-field confidence read from the tokens that spell each value.
+
+The model may wrap the object in ``` fences or prose; we take the outermost
+{...}. Each value's span in the output is located by a small depth-aware
+scan, and its confidence is exp(mean logprob) of the tokens overlapping it.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from typing import Any, Optional, Sequence
+
+from .confidence import DIGITS
+from .t_extract import EMPTY, TExtractField, TFieldValue
+from .tracker import Tracker
+from .value_parsing import coerce_json_value
+
+
+def parse_object(text: str) -> Optional[dict[str, Any]]:
+    """The outermost {...} in the text as a dict, or None if it isn't a JSON object."""
+    start, end = text.find("{"), text.rfind("}")
+    if start < 0 or end < start:
+        return None
+    try:
+        obj = json.loads(text[start : end + 1])
+    except ValueError:
+        return None
+    return obj if isinstance(obj, dict) else None
+
+
+def _string_end(text: str, i: int) -> int:
+    """Index just past the JSON string starting at text[i] == '"', or -1."""
+    j = i + 1
+    while j < len(text):
+        if text[j] == "\\":
+            j += 2
+            continue
+        if text[j] == '"':
+            return j + 1
+        j += 1
+    return -1
+
+
+def _value_end(text: str, j: int) -> int:
+    """Index of the ',' or '}' that ends the value starting at j (depth 0, outside strings)."""
+    depth = 0
+    while j < len(text):
+        ch = text[j]
+        if ch == '"':
+            end = _string_end(text, j)
+            if end < 0:
+                return len(text)
+            j = end
+            continue
+        if ch in "[{":
+            depth += 1
+        elif ch in "]}":
+            if depth == 0:
+                return j
+            depth -= 1
+        elif ch == "," and depth == 0:
+            return j
+        j += 1
+    return j
+
+
+def value_spans(text: str) -> dict[str, tuple[int, int]]:
+    """Key -> (start, end) of its raw value (e.g. '"Acme"', '12.5', 'null') in the
+    first top-level object of the text. Stops quietly at malformed JSON."""
+    i = text.find("{")
+    if i < 0:
+        return {}
+    spans: dict[str, tuple[int, int]] = {}
+    i += 1
+    while i < len(text):
+        if text[i].isspace() or text[i] == ",":
+            i += 1
+            continue
+        if text[i] != '"':
+            break  # '}' or malformed
+        key_end = _string_end(text, i)
+        if key_end < 0:
+            break
+        try:
+            key = json.loads(text[i:key_end])
+        except ValueError:
+            break
+        j = key_end
+        while j < len(text) and text[j].isspace():
+            j += 1
+        if j >= len(text) or text[j] != ":":
+            break
+        j += 1
+        while j < len(text) and text[j].isspace():
+            j += 1
+        end = _value_end(text, j)
+        stop = end
+        while stop > j and text[stop - 1].isspace():
+            stop -= 1
+        spans[key] = (j, stop)
+        i = end
+    return spans
+
+
+def span_confidence(tokens: Sequence[tuple[str, float]], start: int, end: int) -> Optional[float]:
+    """exp(mean logprob) of the tokens overlapping text[start:end], or None if none do."""
+    logprobs, offset = [], 0
+    for token, logprob in tokens:
+        if offset < end and offset + len(token) > start:
+            logprobs.append(logprob)
+        offset += len(token)
+    if not logprobs:
+        return None
+    return min(1.0, math.exp(sum(logprobs) / len(logprobs)))
+
+
+def parse_record(
+    text: str,
+    tokens: Sequence[tuple[str, float]],
+    fields: dict[str, TExtractField],
+    fallback_confidence: float,
+    tracker: Tracker,
+) -> dict[str, TFieldValue]:
+    """Field name -> value for the text fields in a generated JSON record.
+
+    Missing keys and nulls give None. A value's confidence comes from its own
+    tokens; ``fallback_confidence`` (the whole generation's) is used when there
+    are no tokens or the value's span can't be found in them.
+    """
+    obj = parse_object(text)
+    if obj is None:
+        tracker.warn(f"the model did not return a JSON object (got {text[:40]!r}); text fields left empty")
+        return {name: EMPTY for name in fields}
+    spans = value_spans("".join(token for token, _ in tokens)) if tokens else {}
+    out: dict[str, TFieldValue] = {}
+    for name, field in fields.items():
+        conf = None
+        if name in spans:
+            conf = span_confidence(tokens, *spans[name])
+        conf = round(fallback_confidence if conf is None else conf, DIGITS)
+        raw = obj.get(name)
+        if raw is None:
+            out[name] = TFieldValue(None, conf)
+            continue
+        value, problem = coerce_json_value(raw, field.type)
+        if problem is not None:
+            tracker.warn(problem, name)
+            out[name] = EMPTY
+            continue
+        out[name] = TFieldValue(value, conf)
+    return out

dev_double/core/decision/t_answer.py

@@ -0,0 +1,32 @@
+"""Answers to the three question types."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Union
+
+
+@dataclass(frozen=True)
+class TBinaryAnswer:
+    value: bool
+    probability: float  # probability that the answer is yes
+    confidence: float
+
+
+@dataclass(frozen=True)
+class TChoiceAnswer:
+    value: str
+    probabilities: dict[str, float]
+    confidence: float
+
+
+@dataclass(frozen=True)
+class TScaleAnswer:
+    value: float  # probability-weighted level, from 0 to len(levels) - 1
+    level: int  # the single most likely level
+    probabilities: dict[str, float]
+    legend: dict[str, str]
+    confidence: float
+
+
+TAnswer = Union[TBinaryAnswer, TChoiceAnswer, TScaleAnswer]

dev_double/core/decision/t_classify.py

@@ -0,0 +1,42 @@
+"""Labeling one item or a batch."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .defaults import DEFAULT_CLASSIFY_QUESTION
+from .t_input import TInputValue
+from .t_meta import TMeta
+from .t_question import check_options
+
+
+@dataclass(frozen=True)
+class TClassifyRequest:
+    labels: dict[str, str]
+    input: Optional[TInputValue] = None  # one item...
+    inputs: Optional[list[TInputValue]] = None  # ...or many, labeled in parallel
+    question: str = DEFAULT_CLASSIFY_QUESTION
+
+    def __post_init__(self) -> None:
+        check_options(self.labels, "labels")
+        if (self.input is None) == (self.inputs is None):
+            raise ValueError("Provide exactly one of `input` or `inputs`.")
+        if self.inputs is not None and not self.inputs:
+            raise ValueError("`inputs` must not be empty.")
+
+    def items(self) -> list[TInputValue]:
+        return self.inputs if self.inputs is not None else [self.input]  # type: ignore[list-item]
+
+
+@dataclass(frozen=True)
+class TClassification:
+    label: str
+    probabilities: dict[str, float]
+    confidence: float
+
+
+@dataclass(frozen=True)
+class TClassifyResponse:
+    results: list[TClassification]
+    meta: TMeta

dev_double/core/decision/t_decide.py

@@ -0,0 +1,26 @@
+"""Generic decision: several typed questions about one input."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .t_answer import TAnswer
+from .t_input import TInputValue
+from .t_meta import TMeta
+from .t_question import TQuestion
+
+
+@dataclass(frozen=True)
+class TDecideRequest:
+    input: TInputValue
+    questions: dict[str, TQuestion]
+
+    def __post_init__(self) -> None:
+        if not self.questions:
+            raise ValueError("`questions` must not be empty.")
+
+
+@dataclass(frozen=True)
+class TDecideResponse:
+    answers: dict[str, TAnswer]
+    meta: TMeta

dev_double/core/decision/t_extract.py

@@ -0,0 +1,89 @@
+"""Structured extraction: typed fields pulled out of one input.
+
+Enum and boolean fields are answered as choice and binary questions (real
+probabilities); string, number, and integer fields are generated as text.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Union
+
+from .t_input import TInputValue
+from .t_meta import TMeta
+from .t_question import check_options
+
+FIELD_TYPES = ("string", "number", "integer", "boolean", "enum")
+TEXT_TYPES = ("string", "number", "integer")  # generated as text, then parsed
+MAX_FIELDS = 30
+
+# An extracted value; None when the input doesn't contain the field.
+TValue = Optional[Union[str, float, int, bool]]
+
+
+def _check_probability(p: Optional[float], what: str) -> None:
+    if p is not None and not 0 <= p <= 1:
+        raise ValueError(f"`{what}` must be between 0 and 1, got {p}.")
+
+
+@dataclass(frozen=True)
+class TExtractField:
+    type: str  # one of FIELD_TYPES
+    description: Optional[str] = None
+    # enum only: option -> description, or a list of option names (each its own description).
+    options: Optional[Union[dict[str, str], list[str]]] = None
+
+    def __post_init__(self) -> None:
+        if self.type not in FIELD_TYPES:
+            raise ValueError(f"Field `type` must be one of {', '.join(FIELD_TYPES)}; got {self.type!r}.")
+        if self.type != "enum":
+            if self.options is not None:
+                raise ValueError(f"Only enum fields take `options`, not {self.type} fields.")
+            return
+        if self.options is None:
+            raise ValueError("Enum fields need `options`.")
+        options = self.options
+        if isinstance(options, list):
+            if len(set(options)) != len(options):
+                raise ValueError("Duplicate enum options.")
+            options = {name: name for name in options}
+        check_options(options)
+        object.__setattr__(self, "options", dict(options))
+
+
+@dataclass(frozen=True)
+class TExtractRequest:
+    input: TInputValue
+    fields: dict[str, TExtractField]  # field name -> spec; answers keep this order
+
+    def __post_init__(self) -> None:
+        if not 1 <= len(self.fields) <= MAX_FIELDS:
+            raise ValueError(f"`fields` must have 1 to {MAX_FIELDS} entries, got {len(self.fields)}.")
+        for name in self.fields:
+            if not name.strip():
+                raise ValueError("Field names must not be empty.")
+
+
+@dataclass(frozen=True)
+class TFieldValue:
+    value: TValue
+    confidence: float
+    probabilities: Optional[dict[str, float]] = None  # enum fields only
+    probability: Optional[float] = None  # boolean fields only: probability of true
+
+    def __post_init__(self) -> None:
+        _check_probability(self.confidence, "confidence")
+        _check_probability(self.probability, "probability")
+
+
+EMPTY = TFieldValue(None, 0.0)
+
+
+@dataclass(frozen=True)
+class TExtractResponse:
+    fields: dict[str, TFieldValue]
+    meta: TMeta
+
+    @property
+    def values(self) -> dict[str, TValue]:
+        return {name: f.value for name, f in self.fields.items()}

dev_double/core/decision/t_gate.py

@@ -0,0 +1,36 @@
+"""Tool-call gating: allow, ask a human, or deny."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Optional, Union
+
+from .t_input import TInputValue
+from .t_meta import TMeta
+from .t_question import check_options
+
+
+@dataclass(frozen=True)
+class TToolCall:
+    name: str
+    arguments: Union[dict[str, Any], str] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class TGateRequest:
+    tool_call: TToolCall
+    context: Optional[TInputValue] = None  # what the user asked for, or the recent conversation
+    policy: Optional[str] = None  # rules for tool use, in plain language
+    outcomes: Optional[dict[str, str]] = None  # outcome -> when; None = DEFAULT_OUTCOMES
+
+    def __post_init__(self) -> None:
+        if self.outcomes is not None:
+            check_options(self.outcomes, "outcomes")
+
+
+@dataclass(frozen=True)
+class TGateResponse:
+    decision: str
+    probabilities: dict[str, float]
+    confidence: float
+    meta: TMeta

dev_double/core/decision/t_generate.py

@@ -0,0 +1,37 @@
+"""The generator contract's data: a rendered prompt in, a short text and how
+sure the model was of it out."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .t_usage import TUsage
+
+
+@dataclass(frozen=True)
+class TGenerateQuery:
+    system: str
+    user: str
+    max_tokens: int
+    json: bool = False  # ask for one JSON object (e.g. response_format json_object)
+    # Semantic parts, for engines that don't use the rendered prompt (e.g. mock).
+    input_text: str = ""
+    fields: tuple[str, ...] = ()  # the field names being extracted
+
+    def __post_init__(self) -> None:
+        if self.max_tokens < 1:
+            raise ValueError("`max_tokens` must be at least 1.")
+
+
+@dataclass
+class TGenerateResult:
+    text: str
+    confidence: float  # 0..1 for the whole text, e.g. exp(mean token logprob)
+    usage: TUsage = field(default_factory=TUsage)
+    warnings: list[str] = field(default_factory=list)
+    # The generated tokens as (text, logprob), in order; empty if the server returns none.
+    tokens: list[tuple[str, float]] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        if not 0 <= self.confidence <= 1:
+            raise ValueError(f"`confidence` must be between 0 and 1, got {self.confidence}.")

dev_double/core/decision/t_guard.py

@@ -0,0 +1,38 @@
+"""Input screening: injection, abuse, off-topic use."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .defaults import DEFAULT_GUARD_THRESHOLD
+from .t_input import TInputValue
+from .t_meta import TMeta
+
+
+@dataclass(frozen=True)
+class TGuardRequest:
+    input: TInputValue
+    policies: Optional[dict[str, str]] = None  # policy -> violation; None = DEFAULT_POLICIES
+    scope: Optional[str] = None  # what the assistant is for; adds an off_topic check
+    threshold: float = DEFAULT_GUARD_THRESHOLD  # block when a violation probability reaches this
+
+    def __post_init__(self) -> None:
+        if self.policies is not None and not self.policies:
+            raise ValueError("`policies` must not be empty.")
+        if not 0 <= self.threshold <= 1:
+            raise ValueError("`threshold` must be between 0 and 1.")
+
+
+@dataclass(frozen=True)
+class TGuardCheck:
+    probability: float
+    flagged: bool
+
+
+@dataclass(frozen=True)
+class TGuardResponse:
+    allowed: bool
+    flagged: list[str]
+    checks: dict[str, TGuardCheck]
+    meta: TMeta

dev_double/core/decision/t_input.py

@@ -0,0 +1,8 @@
+"""The content a question is asked about."""
+
+from __future__ import annotations
+
+from typing import Any, Union
+
+# Text, an object, or a list. Objects and lists are shown to the model as JSON.
+TInputValue = Union[str, dict[str, Any], list[Any]]

dev_double/core/decision/t_judge.py

@@ -0,0 +1,35 @@
+"""LLM evals: score an output against a rubric."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from .defaults import DEFAULT_JUDGE_CRITERIA
+from .t_input import TInputValue
+from .t_meta import TMeta
+from .t_question import check_levels
+
+
+@dataclass(frozen=True)
+class TJudgeRequest:
+    output: TInputValue  # the answer being judged
+    input: Optional[TInputValue] = None  # the prompt or task that produced it
+    reference: Optional[TInputValue] = None  # optional reference answer
+    criteria: str = DEFAULT_JUDGE_CRITERIA
+    levels: Optional[list[str]] = None  # ordered rubric, worst first; None = DEFAULT_RUBRIC
+
+    def __post_init__(self) -> None:
+        if self.levels is not None:
+            check_levels(self.levels)
+
+
+@dataclass(frozen=True)
+class TJudgeResponse:
+    score: float  # probability-weighted level, from 0 to len(levels) - 1
+    normalized: float  # score / (len(levels) - 1), from 0 to 1
+    level: int
+    probabilities: dict[str, float]
+    legend: dict[str, str]
+    confidence: float
+    meta: TMeta

dev_double/core/decision/t_label_query.py

@@ -0,0 +1,36 @@
+"""The engine contract's data: a rendered single-label prompt in, a
+distribution over those labels out."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .t_usage import TUsage
+
+
+@dataclass
+class TLabelQuery:
+    """One question, rendered and ready for an engine."""
+
+    system: str
+    user: str
+    labels: list[str]
+    # Semantic parts, for engines that don't use the rendered prompt (e.g. mock).
+    input_text: str
+    question: str
+    descriptions: list[str]
+
+
+@dataclass
+class TLabelResult:
+    probabilities: dict[str, float]
+    usage: TUsage = field(default_factory=TUsage)
+    warnings: list[str] = field(default_factory=list)
+
+
+@dataclass
+class TPosition:
+    """One generated token and the top alternatives the model considered there."""
+
+    token: str
+    candidates: list[tuple[str, float]]  # (token, logprob)

dev_double/core/decision/t_meta.py

@@ -0,0 +1,17 @@
+"""Response metadata shared by every use case."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .t_usage import TUsage
+
+
+@dataclass(frozen=True)
+class TMeta:
+    engine: str
+    model: str
+    latency_ms: int
+    usage: TUsage
+    # Where this answer may be less faithful than a real decision model.
+    warnings: list[str] = field(default_factory=list)