annoscript 0.1.0__py3-none-any.whl

annoscript/anno.py

@@ -0,0 +1,270 @@
+from abc import ABC, abstractmethod
+from typing import Any, List, Dict
+from dataclasses import dataclass
+
+from annoscript._examples import Example
+
+
+def constrain(condition: bool, *, message: str | None = None) -> None:
+    """Declare a hard constraint in a compiled AnnoScript eDSL program.
+
+    The compiler captures the pure ``condition`` expression and its dependencies
+    before Python evaluates the arguments. It may propagate the constraint to
+    related inference operations whose results are still adjustable within the
+    current solving scope. A constraint inside a branch applies only on that
+    execution path.
+
+    Successful execution past this call requires the condition to hold, as
+    established by proof or runtime validation. If a satisfying result cannot
+    be obtained, execution must report failure; retry and repair budgets are
+    execution-policy concerns. ``message`` optionally describes the constraint
+    for failure diagnostics. Constraints remain active in optimized execution.
+
+    This function is an eDSL intrinsic, not a standalone solver or validator.
+    Direct Python calls always raise ``RuntimeError``, even for a true condition:
+    the evaluated argument no longer carries the expression needed to propagate
+    constraints. The eDSL compiler must lower this call to a constraint operation.
+    """
+    raise RuntimeError(
+        "constrain() requires a compiled AnnoScript eDSL program; "
+        "direct Python execution cannot propagate constraints."
+    )
+
+
+@dataclass
+class NoulResponse:
+    noul: float
+
+
+@dataclass
+class ChoiceResponse:
+    """Choice result; the provider guarantees finite confidence in [0, 1]."""
+
+    choice: str
+    confidence: float
+    probabilities: Dict[str, float]
+
+
+@dataclass
+class ScoreResponse:
+    probabilities: Dict[int, float]
+    score: float
+    legend: Dict[int, str]
+    confidence: float
+
+
+@dataclass
+class ExtractResponse:
+    items: List[str]
+
+
+@dataclass
+class GenerateResponse:
+    text: str | None
+
+
+@dataclass
+class GenerateListResponse:
+    items: List[str]
+
+
+class Anno(ABC):
+    """Decision, extraction, and generation operations sharing execution context.
+
+    The runtime supplies the program's system_prompt when constructing the
+    per-record instance. The LLM backend applies it alongside per-call task
+    instructions, including calls from helpers. Decision requests use only their
+    task instructions and requirements. Neither overrides API guarantees or hard
+    constraints.
+
+    All operations accept ``examples``: a list of Example values (or their
+    serialized state/answer objects). None and [] mean no examples. Examples
+    inherit this call's instructions, criteria and constraints, and are checked
+    before dispatch. They teach task semantics, not facts about the current state.
+
+    All operations accept ``requirements``: a list of nonblank strings describing
+    additional task instructions. None and [] mean no extra requirements. LLM
+    requests group identical strings and name only the tasks they apply to;
+    Decision questions include their own requirements in the task instructions.
+    Requirements do not replace API guarantees or hard constraints.
+
+    All operations accept ``field_name``: a nonblank string used as the
+    public task name, including question and batch answer keys. None preserves automatic
+    names. Names must be distinct within a physical request, including automatic
+    aliases; internal call IDs and response types remain unchanged.
+    """
+
+    def __init__(
+        self,
+        state: str | Dict[str, Any] | List[Any],
+        model: str,
+        *,
+        system_prompt: str | None = None,
+    ):
+        self.state = state
+        self.model = model
+        self.system_prompt = system_prompt
+
+    @abstractmethod
+    def noul(
+        self,
+        instructions: str,
+        criteria_false: str | None = None,
+        criteria_true: str | None = None,
+        *,
+        examples: list[Example] | None = None,
+        requirements: list[str] | None = None,
+        field_name: str | None = None,
+    ) -> NoulResponse:
+        """Evaluate a yes/no question using this instance's state and model."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def choice(self, instructions: str, criteria: Dict[str, str], *,
+               examples: list[Example] | None = None,
+               requirements: list[str] | None = None,
+               field_name: str | None = None) -> ChoiceResponse:
+        """Select an option using this instance's state and model.
+
+        A successful result must have ``result.choice in criteria`` and a finite
+        ``result.confidence`` in the inclusive range [0, 1], excluding NaN and
+        infinities. Implementations must enforce these guarantees and report
+        failure if a valid result cannot be produced.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def score(self, instructions: str, criteria: List[str], *,
+              examples: list[Example] | None = None,
+              requirements: list[str] | None = None,
+              field_name: str | None = None) -> ScoreResponse:
+        """Score the state against ordered criteria using this instance's model."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def extract(
+        self,
+        instructions: str,
+        *,
+        source_field: str | None = None,
+        min_items: int = 0,
+        max_items: int,
+        max_length_per_item: int,
+        max_num_tokens: int | None = None,
+        examples: list[Example] | None = None,
+        field_name: str | None = None,
+        requirements: list[str] | None = None,
+    ) -> ExtractResponse:
+        """Extract verbatim passages using this instance's state and model.
+
+        ``source_field`` names a top-level string field in an object state,
+        such as ``source_field="text"``. It is a literal key, not a nested path.
+        String states require ``source_field=None`` and are extracted directly.
+        Object states require an explicit field; list states are unsupported.
+        Other state fields may provide context, but are not extraction sources.
+
+        Each item must be a nonempty, contiguous substring of the source, without
+        rewriting, normalization, or concatenation of separate passages. The
+        number of items must satisfy the inclusive ``min_items`` / ``max_items``
+        bounds. Required ``max_length_per_item`` is a suggested Unicode code-point
+        length, measured by Python's ``len``, and contributes to the token budget
+        unless a positive ``max_num_tokens`` overrides it (including format overhead).
+        In a batch this budget contributes to the shared limit.
+        Exceeding the suggestion does not invalidate an otherwise valid item.
+        Return an empty list when nothing matches and ``min_items`` is zero.
+
+        Implementations must raise ``ValueError`` for a missing, invalid or
+        non-string field, negative item bounds, an inverted item range, or a
+        nonpositive ``max_length_per_item``. If extraction cannot satisfy the
+        constraints, report failure rather than inventing items or returning a
+        successful result that violates the bounds. Matching the source does
+        not by itself establish that an item semantically supports a claim.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def generate(
+        self,
+        instructions: str,
+        *,
+        min_length: int = 0,
+        max_length: int,
+        max_num_tokens: int | None = None,
+        examples: list[Example] | None = None,
+        field_name: str | None = None,
+        nullable: bool = False,
+        requirements: list[str] | None = None,
+    ) -> GenerateResponse:
+        """Generate text using this instance's state and model.
+
+        The result is a string by default. ``nullable=True`` additionally allows
+        ``None`` (JSON null), without converting the literal string "null".
+        Nullable standalone calls and repairs return a JSON string or null scalar;
+        ordinary standalone calls remain raw text. Batches retain their answers
+        wrapper. The result may paraphrase or synthesize the state as directed by
+        ``instructions``. ``min_length`` and
+        required ``max_length`` are suggested Unicode code-point lengths, measured by
+        Python's ``len``. A positive ``min_length`` requires nonempty non-null text,
+        but other deviations from the suggested range are not failures.
+
+        ``max_num_tokens`` is an independent output limit for a standalone
+        generation. When Runtime batches ready calls, it contributes to a shared
+        budget and is not an enforced per-answer token limit. ``None`` uses the
+        client's assumed_avg_num_tokens_per_char (default 2) times max_length,
+        rounded up with a minimum of one token. Suggested lengths are not hard bounds.
+        Exhausting the budget must not silently turn an incomplete generation
+        into a successful response.
+
+        Implementations must raise ``ValueError`` for negative length bounds,
+        an inverted length range, a nonpositive ``max_num_tokens``, or a
+        non-boolean ``nullable``. If valid
+        output cannot be produced, report failure rather than returning a
+        successful result that violates the constraints. These constraints do
+        not guarantee factual accuracy. Branching and recovery remain the
+        caller's responsibility.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def generate_list(
+        self,
+        instructions: str,
+        *,
+        min_items: int = 0,
+        max_items: int,
+        min_length_per_item: int = 0,
+        max_length_per_item: int,
+        max_num_tokens: int | None = None,
+        examples: list[Example] | None = None,
+        field_name: str | None = None,
+        requirements: list[str] | None = None,
+    ) -> GenerateListResponse:
+        """Generate a list of texts using this instance's state and model.
+
+        The result is always a list of strings, with no ``None`` items. Each
+        item may paraphrase or synthesize the state as directed by
+        ``instructions``. The item count must satisfy the inclusive
+        ``min_items`` / ``max_items`` bounds; an empty list is allowed when
+        ``min_items=0``.
+
+        ``min_length_per_item`` and required ``max_length_per_item`` are suggested
+        Unicode code-point lengths for each string, measured by Python's ``len``.
+        A positive minimum still requires nonempty items. Required ``max_items``
+        is a hard count limit. Neither required maximum accepts None.
+
+        ``max_num_tokens`` is a generation budget for the entire list,
+        including formatting, in the model's tokens. In a Runtime batch it
+        contributes to a shared budget, without an independent per-list token
+        limit. ``None`` estimates ceil(assumed_avg_num_tokens_per_char * max_items *
+        max_length_per_item) + 32 + 8 * max_items. Extraction uses the same
+        formula. Item count and nonempty requirements remain strict. Exhausting the budget must not silently
+        turn an incomplete list or item into a successful response.
+
+        Implementations must raise ``ValueError`` for negative item or length
+        bounds, inverted ranges, or a nonpositive ``max_num_tokens``. If valid
+        output cannot be produced, report failure rather than returning a
+        successful result that violates the constraints. These constraints do
+        not guarantee factual accuracy. Branching and recovery remain the
+        caller's responsibility.
+        """
+        raise NotImplementedError

annoscript/backends/__init__.py

@@ -0,0 +1,13 @@
+"""API adapters. Batching and dependency scheduling belong to AnnoRuntime."""
+
+from .base import (
+    BatchResult, LLMBatchPlan, DecisionBatchPlan, InferenceCall, InferenceContext, InferenceOutput,
+    InferenceError, BackendFatalError,
+)
+from .decision import DecisionAPIBackend
+from .llm import LLMAPIBackend
+
+__all__ = [
+    "BatchResult", "LLMBatchPlan", "DecisionBatchPlan", "InferenceCall", "InferenceContext", "InferenceOutput",
+    "InferenceError", "BackendFatalError", "DecisionAPIBackend", "LLMAPIBackend",
+]

annoscript/backends/_batch_wire.py

@@ -0,0 +1,26 @@
+"""Request-local names for model APIs and compact LLM batch responses."""
+
+FORMAT = "short_ids_values_v1"
+
+
+def wire_name(call, fallback):
+    """Resolve an optional public name without replacing runtime identity."""
+    name = call.arguments.get("field_name")
+    if name is None:
+        return fallback
+    if type(name) is not str or not name.strip():
+        raise ValueError("field_name must be a nonblank string or None")
+    return name
+
+
+def batch_calls(calls):
+    """Map wire IDs to unchanged calls; aliases never enter runtime identity."""
+    if len({call.id for call in calls}) != len(calls):
+        raise ValueError("a batch requires distinct calls")
+    identified = {}
+    for index, call in enumerate(calls, 1):
+        name = wire_name(call, f"q{index}")
+        if name in identified:
+            raise ValueError(f"duplicate model-visible field name: {name!r}")
+        identified[name] = call
+    return identified

annoscript/backends/_bounded_grammar.py

@@ -0,0 +1,97 @@
+"""Hard structure/count/nonempty constraints; advisory code-point lengths, with source-isolated Substring rules."""
+
+import json
+
+from .base import InferenceError
+from ._batch_wire import batch_calls
+
+
+# Keep a complete JSON code point in one lexical FSM: a repeated independent
+# json-char rule makes every text token context-sensitive in XGrammar.
+JSON_ATOM = r'(?:[^"\\\x00-\x1f]|\\["\\/bfnrt]|\\u(?:[0-9a-cA-Ce-fE-F][0-9a-fA-F]{3}|[dD][0-7][0-9a-fA-F]{2})|\\u[dD][89aAbB][0-9a-fA-F]{2}\\u[dD][c-fC-F][0-9a-fA-F]{2})'
+
+
+def json_body(nonempty):
+    return "Regex(" + json.dumps(JSON_ATOM + ("+" if nonempty else "*")) + ")"
+
+
+def literal(value):
+    return json.dumps(value, ensure_ascii=False)
+
+
+def repeat(rule, minimum, maximum):
+    if maximum == 0:
+        return '""'
+    return rule + ('{' + str(minimum) + ',' + ('' if maximum is None else str(maximum)) + '}')
+
+
+def ensure_satisfiable(call):
+    a = call.arguments
+    if call.operation == "extract" and not call.extraction_source and a.get("min_items", 0) > 0:
+        raise InferenceError("empty extraction source cannot satisfy min_items",
+                             code="unsatisfiable_constraints", call_id=call.id,
+                             metadata={"dispatch_status": "not_sent"})
+
+
+def grammar(calls, *, batch=False, extract_mode="validated"):
+    definitions = []
+    cache = {}
+
+    def value_rule(call):
+        ensure_satisfiable(call)
+        a = call.arguments
+        key = (call.operation, call.extraction_source, a.get("min_items", 0), a.get("max_items"),
+               bool(a.get("min_length", 0)), bool(a.get("min_length_per_item", 0)),
+               bool(a.get("nullable", False)))
+        if key in cache:
+            return cache[key]
+        prefix = 'value-' + str(len(cache))
+        cache[key] = prefix
+        if call.operation == "generate":
+            body = json_body(bool(a.get('min_length', 0)))
+            string = '"\\\"" ' + body + ' "\\\""'
+            value = '(' + string + ' | "null")' if a.get("nullable", False) else string
+            definitions.append(prefix + ' ::= ' + value)
+            return prefix
+        if call.operation == "extract" and not call.extraction_source:
+            definitions.append(prefix + ' ::= "[]"')
+            return prefix
+        item = prefix + '-string'
+        if call.operation == "extract" and extract_mode == "substring_chars":
+            chunks = [literal(json.dumps(c, ensure_ascii=False)[1:-1]) for c in call.extraction_source]
+            options = ', allow_empty=false'
+            body = prefix + '-body'
+            definitions.append(body + ' ::= Substring(' + ', '.join(chunks) + options + ')')
+        else:
+            minimum = 1 if call.operation == "extract" else a.get('min_length_per_item', 0)
+            body = json_body(bool(minimum))
+        definitions.append(item + ' ::= "\\\"" ' + body + ' "\\\""')
+        low, high = a.get('min_items', 0), a.get('max_items')
+        if high == 0:
+            interior = '""'
+        else:
+            tail = repeat('(\",\" ' + item + ')', max(0, low - 1), None if high is None else high - 1)
+            interior = item + ' ' + tail
+            if low == 0:
+                interior = '(' + interior + ')?'
+        definitions.append(prefix + ' ::= "[" ' + interior + ' "]"')
+        return prefix
+
+    if batch:
+        parts = [literal('{"answers":{')]
+        for i, (identity, call) in enumerate(batch_calls(calls).items()):
+            prefix = (',' if i else '') + json.dumps(identity) + ':'
+            parts.extend([literal(prefix), value_rule(call)])
+        parts.append(literal('}}'))
+        root = ' '.join(parts)
+    else:
+        call, = calls
+        ensure_satisfiable(call)
+        if call.operation == 'generate' and not call.arguments.get("nullable", False):
+            # Standalone generate remains raw text, with no JSON wrapper.
+            root = 'Regex(' + json.dumps(r'[\s\S]' + ('+' if call.arguments.get('min_length', 0) else '*')) + ')' 
+        elif call.operation == 'generate':
+            root = value_rule(call)
+        else:
+            root = literal('{"items":') + ' ' + value_rule(call) + ' ' + literal('}')
+    return 'root ::= ' + root + '\n' + '\n'.join(definitions)

annoscript/backends/_example_layout.py

@@ -0,0 +1,23 @@
+"""Request-local example sharing; never modifies the current inference state."""
+
+import json
+
+
+EXAMPLE_CONTRACT = (
+    "examples 是任务的输入与正确答案示范，只用于理解规则，不是当前 state 的事实。"
+    "answer 是该算子的值，不包含置信度；当前输出仍遵守指定格式。"
+)
+
+
+def group_examples(tasks):
+    """Group inputs by first occurrence; retain every answer, including repeats."""
+    groups = {}
+    for identity, task in tasks.items():
+        for example in task["task"].get("examples", []):
+            state = example["state"]
+            key = json.dumps(state, ensure_ascii=False, allow_nan=False,
+                             sort_keys=True, separators=(",", ":"))
+            if key not in groups:
+                groups[key] = {"state": state, "answers": []}
+            groups[key]["answers"].append((identity, example["answer"]))
+    return list(groups.values())

annoscript/backends/_generation_policy.py

@@ -0,0 +1,116 @@
+"""Opt-in, release-bound Qwen request limits. No network or credential access."""
+
+from functools import lru_cache
+import hashlib
+import json
+import math
+import os
+from annoscript._env import env_value
+from pathlib import Path
+
+from .base import BackendFatalError
+from annoscript._data import thaw
+
+
+SAMPLING = {"temperature": 0, "top_p": 1, "top_k": -1, "min_p": 0,
+            "repetition_penalty": 1, "presence_penalty": 0, "frequency_penalty": 0}
+
+
+def validate_policy(value):
+    if value is None:
+        return None
+    keys = {"sampling", "default_max_tokens", "context_window", "tokenizer", "bounded_outputs"}
+    if type(value) is not dict or set(value) != keys:
+        raise ValueError("invalid generation request_policy")
+    sampling = value["sampling"]
+    if (type(sampling) is not dict or set(sampling) != set(SAMPLING)
+            or any(type(v) not in (int, float) or not math.isfinite(v) for v in sampling.values())
+            or sampling != SAMPLING):
+        raise ValueError("P0 requires explicit greedy sampling without additional penalties")
+    for name in ("default_max_tokens", "context_window"):
+        if type(value[name]) is not int or value[name] < 1:
+            raise ValueError("generation limits must be positive integers")
+    if value["default_max_tokens"] > value["context_window"] or value["bounded_outputs"] is not True:
+        raise ValueError("invalid bounded generation policy")
+    binding = value["tokenizer"]
+    if (type(binding) is not dict or set(binding) != {"revision", "files"}
+            or type(binding["revision"]) is not str or not binding["revision"]
+            or type(binding["files"]) is not dict
+            or not {"tokenizer.json", "tokenizer_config.json", "chat_template.jinja"} <= set(binding["files"])):
+        raise ValueError("tokenizer needs an immutable revision and file digests")
+    for name, digest in binding["files"].items():
+        if (type(name) is not str or Path(name).name != name or name in (".", "..")
+                or type(digest) is not str or len(digest) != 64
+                or any(c not in "0123456789abcdef" for c in digest)):
+            raise ValueError("invalid tokenizer file binding")
+    return json.loads(json.dumps(value))
+
+
+@lru_cache(maxsize=4)
+def _load_tokenizer(directory, serialized_binding):
+    binding = json.loads(serialized_binding)
+    root = Path(directory)
+    for name, digest in binding["files"].items():
+        if hashlib.sha256((root / name).read_bytes()).hexdigest() != digest:
+            raise ValueError("tokenizer artifact digest mismatch")
+    from transformers import AutoTokenizer
+    tokenizer = AutoTokenizer.from_pretrained(root, local_files_only=True, trust_remote_code=False)
+    if tokenizer.chat_template != (root / "chat_template.jinja").read_text():
+        raise ValueError("loaded chat template differs from bound artifact")
+    # Validate the template engine at startup, before any task can send HTTP.
+    tokenizer.apply_chat_template([{"role": "system", "content": ""},
+                                   {"role": "user", "content": "startup check"}],
+                                  tokenize=True, return_dict=False, add_generation_prompt=True, enable_thinking=False)
+    return tokenizer
+
+
+class PinnedTokenCounter:
+    def __init__(self, binding):
+        try:
+            directory = env_value("ANNO_GENERATE_TOKENIZER_DIR")
+            self.tokenizer = _load_tokenizer(directory, json.dumps(binding, sort_keys=True))
+        except Exception:
+            raise BackendFatalError("pinned tokenizer is unavailable or mismatched",
+                                    code="configuration_failed") from None
+
+    def __call__(self, payload):
+        try:
+            ids = self.tokenizer.apply_chat_template(
+                payload["messages"], tokenize=True, return_dict=False, add_generation_prompt=True,
+                **payload.get("chat_template_kwargs", {}))
+            if type(ids) is not list or any(type(token) is not int for token in ids):
+                raise ValueError("template must return one token ID sequence")
+            return len(ids)
+        except Exception:
+            raise BackendFatalError("bound chat template could not tokenize request",
+                                    code="configuration_failed") from None
+
+
+def validate_token_ratio(value):
+    if type(value) not in (int, float) or not math.isfinite(value) or value <= 0:
+        raise ValueError("assumed_avg_num_tokens_per_char must be finite and positive")
+    return value
+
+
+def effective_budget(call, assumed_avg_num_tokens_per_char=2, override=None):
+    """One rule for direct, runtime, batch and repair budgets; no hidden cap."""
+    ratio = validate_token_ratio(assumed_avg_num_tokens_per_char)
+    a = call.arguments
+    # Direct backend callers also pass through the public argument validator.
+    from .base import normalize_arguments
+    normalize_arguments(call.operation, (), thaw(a))
+    if call.operation == "extract" and type(call.extraction_source) is not str:
+        raise ValueError("extract call requires a normalized source_field binding")
+    explicit = a.get("max_num_tokens")
+    if explicit is not None:
+        budget = explicit
+    elif call.operation == "generate":
+        budget = max(1, math.ceil(ratio * a["max_length"]))
+    else:
+        count = a["max_items"]
+        budget = math.ceil(ratio * count * a["max_length_per_item"]) + 32 + 8 * count
+    if override is not None:
+        if type(override) is not int or override < 1:
+            raise ValueError("invalid token override")
+        budget = min(budget, override)
+    return budget

annoscript/backends/_grammar.py

@@ -0,0 +1,83 @@
+"""Compact JSON structure for list operations, independent of business contracts.
+
+JSON syntax and batch ID/value type mapping are constrained here. Runtime validates
+decoded strings, item counts, nonempty requirements and verbatim extraction.
+The grammar is source, not generated from the car example or a third-party repo.
+"""
+
+import json
+from ._batch_wire import batch_calls
+
+
+# Keep the string continuation in the same right-recursive rule. Repeating a
+# separate `character` rule is language-equivalent but makes xgrammar 0.2.3
+# classify tokens against the surrounding parser stack at every decode step
+# with the Qwen3.5 vocabulary. It can dominate decoding and cause HTTP timeouts.
+# This spelling preserves JSON escapes, Unicode and empty strings; it neither
+# narrows extraction semantics nor changes any business/token budget.
+STRING_RULES = r'''
+string ::= "\"" string-body
+string-body ::= "\"" | [^"\\\x00-\x1f] string-body | "\\" escape string-body
+escape ::= ["\\/bfnrt] | "u" hex hex hex hex
+hex ::= [0-9a-fA-F]
+'''.strip()
+
+ITEMS_GRAMMAR = r'''root ::= "{\"items\":[" (string ("," string)*)? "]}"''' + "\n" + STRING_RULES
+
+
+BATCH_GRAMMAR = r'''
+root ::= "{\"answers\":{" (answer ("," answer)*)? "}}"
+answer ::= string ":" (string | items)
+items ::= "[" (string ("," string)*)? "]"
+'''.strip() + "\n" + STRING_RULES
+
+
+def _extract_rules(source, prefix):
+    """Atomic JSON encodings preserve code-point boundaries after JSON decoding.
+
+    Requires the pinned XGrammar allow_empty extension. No Unicode normalization
+    or union of sources is permitted. Runtime still checks counts and lengths.
+    """
+    items, body = prefix + "-items", prefix + "-body"
+    if not source:
+        return items, [items + ' ::= "[]"']
+    chunks = [json.dumps(json.dumps(char, ensure_ascii=False)[1:-1], ensure_ascii=False)
+              for char in source]
+    string = prefix + "-string"
+    return items, [
+        body + " ::= Substring(" + ", ".join(chunks) + ", allow_empty=false)",
+        string + ' ::= "\\\"" ' + body + ' "\\\""',
+        items + ' ::= "[" (' + string + ' ("," ' + string + ')*)? "]"',
+    ]
+
+
+def extract_grammar(source):
+    items, rules = _extract_rules(source, "extract")
+    return 'root ::= "{\\\"items\\\":" ' + items + ' "}"\n' + "\n".join(rules)
+
+
+def batch_grammar(calls, *, extract_mode="validated"):
+    """Bind every requested ID to its operation, once, in request order.
+
+    A generic union allows an extract ID to produce a string, even though
+    that answer is valid JSON. The response parser still validates arbitrary
+    object ordering and all constraints when the provider ignores the grammar.
+    """
+    parts = [json.dumps('{"answers":{')]
+    sources, rules = {}, []
+    for index, (identity, call) in enumerate(batch_calls(calls).items()):
+        prefix = ("," if index else "") + json.dumps(identity) + ':'
+        value_rule = "string" if call.operation == "generate" else "items"
+        if call.operation == "generate" and call.arguments.get("nullable", False):
+            value_rule = '(string | "null")'
+        if call.operation == "extract" and extract_mode == "substring_chars":
+            source = call.extraction_source
+            if source not in sources:
+                rule, definitions = _extract_rules(source, f"source-{len(sources)}")
+                sources[source] = rule
+                rules.extend(definitions)
+            value_rule = sources[source]
+        parts.extend((json.dumps(prefix, ensure_ascii=False), value_rule))
+    parts.append(json.dumps("}}"))
+    grammar = "root ::= " + " ".join(parts) + '\nitems ::= "[" (string ("," string)*)? "]"\n' + STRING_RULES
+    return grammar + ("\n" + "\n".join(rules) if rules else "")

annoscript/backends/_preview.py

@@ -0,0 +1,13 @@
+"""Detached inspection of the exact HTTP body; never includes transport headers."""
+
+import hashlib
+
+from .base import encode_payload
+
+
+def request_preview(payload, *, stage, adapter, url, evidence=None):
+    body = encode_payload(payload)
+    return {"format": "lev-request-preview", "version": 1, "stage": stage,
+            "request_adapter": adapter, "method": "POST", "url": url,
+            "body_sha256": hashlib.sha256(body).hexdigest(), "body_bytes": len(body),
+            "payload": payload, "evidence": evidence or {}}