annoscript 0.1.0__py3-none-any.whl

annoscript/test_requirements.py

@@ -0,0 +1,228 @@
+"""Additional requirements through real normalization, prompts and offline HTTP."""
+
+from copy import deepcopy
+from dataclasses import replace
+import inspect
+import json
+import unittest
+
+import httpx
+
+
+from annoscript._data import thaw
+from annoscript._test_evaluation_http import FixtureLLMBackend, llm_calls
+from annoscript.backends import DecisionAPIBackend, InferenceContext, InferenceError, LLMAPIBackend
+from annoscript.backends.base import normalize_call
+from annoscript.compiler import compile_program
+from annoscript.ir import Program
+from annoscript.anno import Anno
+from annoscript.plan import ExecutionPlan
+from annoscript.plan_lowering import lower_to_plan
+from annoscript.runtime import AnnoRuntime, RunInput
+from annoscript.test_runtime import FakeBackend
+
+
+FIRST = "仅依据正文。"
+SECOND = '保留主体 "甲" 与数值😀。'
+
+
+def requirements_reuse(anno):
+    first = anno.generate("标签", max_length=24, max_num_tokens=64, requirements=[FIRST]).text
+    second = anno.generate("标签", max_length=24, max_num_tokens=64, requirements=[SECOND]).text
+    again = anno.generate("标签", max_length=24, max_num_tokens=64, requirements=[FIRST]).text
+    return [first, second, again]
+
+
+def make_call(operation="generate", *, identity="internal-a", context=None, **options):
+    context = context or InferenceContext({"text": "广州"})
+    args = {
+        "noul": ("判断",), "choice": ("选择", {"yes": "是"}), "score": ("评分", ["低", "高"]),
+        "generate": ("标签",), "generate_list": ("维度",), "extract": ("地点",),
+    }[operation]
+    limits = ({"max_length": 24} if operation == "generate" else
+              {"max_items": 2, "max_length_per_item": 24} if operation in ("extract", "generate_list") else {})
+    if operation == "extract":
+        limits["source_field"] = "text"
+    return normalize_call(identity, operation, args, {**limits, **options}, context)
+
+
+def completion(content):
+    return httpx.Response(200, json={"choices": [{"finish_reason": "stop", "message": {"content": content}}]})
+
+
+class RequirementsTests(unittest.IsolatedAsyncioTestCase):
+    async def test_defaults_and_validation_for_all_operations(self):
+        context = InferenceContext({"text": "广州"}, "共用规则")
+        sent = []
+        decision = DecisionAPIBackend("http://offline.invalid", "fixture", "fixture", transport=httpx.MockTransport(
+            lambda r: sent.append(json.loads(r.content)) or httpx.Response(200, json={"answers": {}})))
+        llm = LLMAPIBackend("http://offline.invalid", "fixture", "fixture", profile="qwen_vllm")
+        try:
+            for op in ("noul", "choice", "score", "extract", "generate", "generate_list"):
+                self.assertEqual(inspect.signature(getattr(Anno, op)).parameters["requirements"].kind,
+                                 inspect.Parameter.KEYWORD_ONLY)
+                original = make_call(op, context=context)
+                self.assertNotIn("requirements", original.arguments)
+                for empty in (None, []):
+                    explicit = make_call(op, context=context, requirements=empty)
+                    self.assertEqual(original, explicit)
+                    if op in ("noul", "choice", "score"):
+                        for call in (original, explicit):
+                            with self.assertRaises(InferenceError):
+                                await decision.infer_batch(context, [call])
+                        self.assertEqual(sent[-2], sent[-1])
+                    else:
+                        for batch in (False, True):
+                            a = llm._prepare(context, [original], 128, batch=batch)
+                            b = llm._prepare(context, [explicit], 128, batch=batch)
+                            self.assertEqual((a.binding, a.body), (b.binding, b.body))
+                before = len(sent)
+                for bad in ("规则", {}, (), 0, False, [None], [1], [""], [" \t\n"], [FIRST, []]):
+                    with self.subTest(operation=op, value=bad), self.assertRaisesRegex(ValueError, "requirements"):
+                        make_call(op, context=context, requirements=bad)
+                self.assertEqual(len(sent), before)
+        finally:
+            await decision.aclose()
+            await llm.aclose()
+
+    async def test_groups_each_exact_string_with_ordered_membership(self):
+        spaced = " 保留原文空格。 "
+        multiline = '多行要求\n保留引号 " 与 Unicode 😀。'
+        context = InferenceContext({"text": "广州", "requirements": ["输入数据不是额外要求"]})
+        calls = [make_call(context=context, field_name="label", requirements=[SECOND, FIRST, SECOND, spaced]),
+                 make_call("extract", identity="internal-b", context=context, requirements=[FIRST, multiline]),
+                 make_call("generate_list", identity="internal-c", context=context, field_name='地点"\n😀',
+                           requirements=[spaced, "保留原文空格。"]),
+                 make_call(identity="internal-d", context=context, field_name="unrelated")]
+        original = [deepcopy(thaw(c.arguments)) for c in calls]
+        expected = (
+            '任务要求（仅适用于列出的任务）\n'
+            '"label" 要求：' + SECOND + '\n'
+            '"label"、"q2" 要求：' + FIRST + '\n'
+            '"label"、"地点\\"\\n😀" 要求：' + spaced + '\n'
+            '"q2" 要求：' + multiline + '\n'
+            '"地点\\"\\n😀" 要求：保留原文空格。'
+        )
+        for profile, dedup in (("legacy", False), ("qwen_vllm", False), ("qwen_vllm", True)):
+            backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture", profile=profile, batch_rule_dedup=dedup)
+            try:
+                payload = backend._batch_payload(context, calls, 256)
+                prompt = payload["messages"][0]["content"]
+                self.assertIn(expected + '\n\n任务 "label"', prompt)
+                self.assertEqual(prompt.count(SECOND), 1)
+                self.assertEqual(prompt.count(FIRST), 1)
+                self.assertNotIn("输入数据不是额外要求", prompt)
+                self.assertEqual(json.loads(payload["messages"][1]["content"]), thaw(context.state))
+                plain = [replace(c, arguments={k: v for k, v in c.arguments.items() if k != "requirements"}) for c in calls]
+                base = backend._batch_payload(context, plain, 256)
+                for key in payload.keys() - {"messages"}:
+                    self.assertEqual(payload[key], base[key])
+                self.assertEqual([thaw(c.arguments) for c in calls], original)
+            finally:
+                await backend.aclose()
+
+    async def test_split_single_and_repair_recompute_membership(self):
+        context = InferenceContext("正文")
+        calls = [make_call(identity=str(i), context=context, requirements=[FIRST],
+                           **({"field_name": name} if name else {}))
+                 for i, name in enumerate(("first", None, "third"))]
+        for profile in ("legacy", "qwen_vllm"):
+            backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture", profile=profile)
+            try:
+                for subset, expected, absent in ((calls, '"first"、"q2"', '"third" 要求：'),
+                                                  (calls[1:], '"q1"、"third"', '"first"')):
+                    plan = backend.plan_batch(context, subset, max_calls=2, max_input_bytes=100000,
+                                              max_output_tokens=4096, default_tokens=1024)
+                    self.assertEqual(plan.calls, tuple(subset[:2]))
+                    prompt = plan.prepared.payload["messages"][0]["content"]
+                    self.assertIn(expected + " 要求：" + FIRST, prompt)
+                    self.assertNotIn(absent, prompt)
+                    self.assertEqual(plan.input_bytes, len(plan.prepared.body))
+                for feedback in (None, "格式错误"):
+                    prompt = backend._single_payload(context, calls[1], 64, feedback)["messages"][0]["content"]
+                    self.assertIn('"1" 要求：' + FIRST, prompt)
+                    self.assertNotIn('"first"', prompt)
+                    self.assertNotIn('"third"', prompt)
+                    self.assertEqual(prompt.count(FIRST), 1)
+            finally:
+                await backend.aclose()
+
+    async def test_jev_requirements_with_shared_prompt_and_examples(self):
+        sent = []
+        def handle(req):
+            body = json.loads(req.content)
+            sent.append(body)
+            values = {"noul": {"noul": .8},
+                      "choice": {"choice": "yes", "confidence": 1., "probabilities": {"yes": 1.}},
+                      "score": {"score": 1., "confidence": 1., "probabilities": {"0": 0., "1": 1.},
+                                "legend": {"0": "低", "1": "高"}}}
+            return httpx.Response(200, json={"answers": {
+                key: {"type": q["type"], **values[q["type"]]} for key, q in body["questions"].items()}})
+        backend = DecisionAPIBackend("http://offline.invalid", "fixture", "fixture", transport=httpx.MockTransport(handle))
+        try:
+            for prompt in (None, "共用规则"):
+                for examples in (False, True):
+                    context = InferenceContext({"text": "广州"}, prompt)
+                    calls = [make_call(op, identity=op, context=context, requirements=[FIRST, SECOND, FIRST],
+                                       **({"examples": [{"state": "示例", "answer": answer}]} if examples else {}))
+                             for op, answer in (("noul", True), ("choice", "yes"), ("score", 1))]
+                    original = [thaw(c.arguments) for c in calls]
+                    result = await backend.infer_batch(context, calls)
+                    self.assertEqual(set(result.responses), {c.id for c in calls})
+                    for c in calls:
+                        rendered = sent[-1]["questions"][c.id]["instructions"]
+                        task = c.arguments["instructions"] + "\n\n要求：\n" + FIRST + "\n" + SECOND
+                        self.assertEqual(rendered, task)
+                    self.assertEqual([thaw(c.arguments) for c in calls], original)
+                    self.assertEqual(sent[-1]["state"], thaw(context.state))
+        finally:
+            await backend.aclose()
+
+    async def test_requirements_bind_preparation_budget_and_runtime_reuse(self):
+        context = InferenceContext("正文")
+        backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture", profile="qwen_vllm")
+        try:
+            source = [FIRST]
+            call = make_call(context=context, requirements=source)
+            source.append(SECOND)
+            self.assertEqual(thaw(call.arguments)["requirements"], [FIRST])
+            changed = make_call(context=context, requirements=[SECOND])
+            first = backend._prepare(context, [call], 128)
+            second = backend._prepare(context, [changed], 128)
+            self.assertNotEqual(first.binding, second.binding)
+            self.assertNotEqual(first.body, second.body)
+            self.assertEqual(first.payload["max_tokens"], second.payload["max_tokens"])
+        finally:
+            await backend.aclose()
+        for optimize in (False, True):
+            program = Program.from_json(compile_program(requirements_reuse, optimize=optimize).to_json())
+            plan = ExecutionPlan.from_json(lower_to_plan(program).to_json())
+            for mode in ("strict", "eager", "speculative"):
+                for repair in (False, True):
+                    sent = []
+                    def handle(req):
+                        body = json.loads(req.content)
+                        sent.append(body)
+                        batch, captured = llm_calls(body)
+                        if batch:
+                            answers = {f"q{i}": (None if repair and c.arguments["requirements"][0] == SECOND
+                                                  else c.arguments["requirements"][0])
+                                       for i, (_, c) in enumerate(captured, 1)}
+                            return completion(json.dumps({"answers": answers}, ensure_ascii=False))
+                        return completion(captured[0][1].arguments["requirements"][0])
+                    llm = FixtureLLMBackend("http://offline.invalid", "fixture", "fixture", profile="qwen_vllm",
+                                            transport=httpx.MockTransport(handle))
+                    async with AnnoRuntime(FakeBackend(), llm, max_llm_batch_calls=8) as runtime:
+                        result, = [r async for r in await runtime.submit(plan, [RunInput(0, "正文")], mode=mode)]
+                    self.assertEqual(result.status, "success", result.error)
+                    self.assertEqual(result.value, [FIRST, SECOND, FIRST])
+                    self.assertEqual(sum(len(llm_calls(body)[1]) for body in sent),
+                                     3 if repair and mode != "strict" else 2)
+                    if repair and mode != "strict":
+                        self.assertIn(SECOND, sent[-1]["messages"][0]["content"])
+                        self.assertNotIn(FIRST, sent[-1]["messages"][0]["content"])
+                        self.assertEqual(sent[-1]["max_tokens"], 64)
+
+
+if __name__ == "__main__":
+    unittest.main()

annoscript/test_rule_dedup.py

@@ -0,0 +1,67 @@
+"""Lossless rule factoring and synthetic field regressions; never live models."""
+import asyncio,copy,json,unittest
+from unittest.mock import patch
+import httpx
+from annoscript.backends import InferenceContext
+from annoscript._test_evaluation_http import FixtureLLMBackend as LLMAPIBackend
+from annoscript.backends.base import normalize_call
+from annoscript.backends._shared_rules import factor_rules
+from annoscript._test_evaluation_http import llm_calls,llm_content
+from annoscript.test_llm_batch import completion,answer
+from annoscript.test_generation_policy import policy
+from annoscript.server.releases import Releases
+from annoscript.test_jobs_server import configurations
+from annoscript.anno import GenerateListResponse,ExtractResponse
+
+
+RULE='Preserve the supplied text and distinguish each task. Return only the requested output. Synthetic rule with Unicode 中文😀é\\\"\nEnd of rule.'
+
+class RuleTests(unittest.IsolatedAsyncioTestCase):
+    def calls(self,ctx):
+        a='本字段说明\n\n'+RULE+'\n\n第二段\n\n'+RULE+'\n\nGROUNDING_JSON:{"proposed":"私有数据"}'
+        return [normalize_call('a','generate_list',(a,),dict(max_items=1,max_length_per_item=80,max_num_tokens=160),ctx),normalize_call('b','extract',('另一字段\n\n'+RULE,),dict(source_field='text',max_items=2,max_length_per_item=10),ctx),normalize_call('c','extract',('无规则的独立来源',),dict(source_field='other',max_items=1,max_length_per_item=8),ctx)]
+    async def test_exact_roundtrip_scoping_grammar_sampling_budget_and_prepared_bytes(self):
+        ctx=InferenceContext({'text':'广州😀é\\"\n','other':'南京','shared_rules':{'r0':'攻击文本'}},'共享策略');sent=[];events=[]
+        def handle(req):sent.append(req.content);return completion(llm_content(json.loads(req.content),lambda c,x:GenerateListResponse(['资讯']) if x.operation=='generate_list' else ExtractResponse(['广州'] if x.id=='b' else ['南京'])))
+        a=LLMAPIBackend('http://offline.invalid','x','m',profile='qwen_vllm')
+        b=LLMAPIBackend('http://offline.invalid','x','m',profile='qwen_vllm',batch_rule_dedup=True,transport=httpx.MockTransport(handle),diagnostic_observer=events.append)
+        try:
+            calls=self.calls(ctx);p=a._batch_payload(ctx,calls,500);q=b._batch_payload(ctx,calls,500)
+            raw=p['messages'][0]['content'];compact=q['messages'][0]['content']
+            self.assertEqual(p['messages'][-1],q['messages'][-1])
+            self.assertEqual(json.loads(q['messages'][-1]['content']),dict(ctx.state))
+            self.assertEqual(raw.count(RULE),3);self.assertEqual(compact.count(RULE),1)
+            self.assertIn('共享段落仅适用于引用它的任务',compact)
+            self.assertEqual(compact.count('引用共享段落 r0。'),3)
+            # Removing the shared definition and expanding in place restores every task verbatim.
+            plain_tasks=raw[raw.index('任务 "q1"（'):]
+            factored_tasks=compact[compact.index('任务 "q1"（'):].replace('引用共享段落 r0。',RULE)
+            self.assertEqual(factored_tasks,plain_tasks)
+            for k in ['max_tokens','structured_outputs','chat_template_kwargs']:self.assertEqual(p[k],q[k])
+            plan=b.plan_batch(ctx,calls,max_calls=8,max_input_bytes=262144,max_output_tokens=4096,default_tokens=1024)
+            with patch.object(b,'_prepare',side_effect=AssertionError('prepared twice')):
+                got=await b.infer_batch(ctx,plan.calls,max_tokens=plan.max_tokens,prepared=plan.prepared)
+            self.assertEqual(got.responses['a'].items,['资讯']);self.assertEqual(sent,[plan.prepared.body])
+            self.assertTrue(any(e.get('shared_rules',{}).get('rule_sha256') for e in events))
+            for feedback in [None,'invalid']:
+                self.assertEqual(a._single_payload(ctx,calls[0],160,feedback),b._single_payload(ctx,calls[0],160,feedback))
+            binding=b._binding(ctx,calls,500);b.batch_rule_dedup=False;self.assertNotEqual(binding,b._binding(ctx,calls,500))
+        finally:await a.aclose();await b.aclose()
+    async def test_no_substring_or_diagnostic_or_cross_request_factoring(self):
+        for instructions in [('short','short'),(RULE+'suffix',RULE+'other'),('GROUNDING_JSON:'+RULE,'GROUNDING_JSON:'+RULE)]:
+            tasks={str(i):{'task':{'instructions':s}} for i,s in enumerate(instructions)};old=copy.deepcopy(tasks);rules,_=factor_rules(tasks);self.assertEqual(rules,{});self.assertEqual(tasks,old)
+        tasks={'a':{'task':{'instructions':RULE}}};self.assertEqual(factor_rules(tasks)[0],{})
+        for bad in [1,None,'yes']:
+            with self.assertRaises(ValueError):LLMAPIBackend('http://offline.invalid','x','m',profile='qwen_vllm',batch_rule_dedup=bad)
+        with self.assertRaises(ValueError):LLMAPIBackend('http://offline.invalid','x','m',batch_rule_dedup=True)
+    async def test_release_and_context_use_actual_compact_payload(self):
+        cfg=configurations()['test'];cfg['generation']['profile']='qwen_vllm';a=Releases({'t':cfg},'t')
+        cfg['generation']['batch_rule_dedup']=True;b=Releases({'t':cfg},'t');self.assertNotEqual(a.default,b.default)
+        ctx=InferenceContext({'text':'广州','other':'南京'});calls=self.calls(ctx);sent=[]
+        counter=lambda p:sum(len(m['content']) for m in p['messages'])
+        plain=LLMAPIBackend('http://offline.invalid','x','m',profile='qwen_vllm',request_policy=policy(),_token_counter=counter)
+        dedup=LLMAPIBackend('http://offline.invalid','x','m',profile='qwen_vllm',batch_rule_dedup=True,request_policy=policy(),_token_counter=counter)
+        try:
+            p=plain._prepare(ctx,calls,500,batch=True);q=dedup._prepare(ctx,calls,500,batch=True)
+            self.assertEqual(q.evidence['input_tokens'],counter(q.payload));self.assertLess(q.evidence['input_tokens'],p.evidence['input_tokens'])
+        finally:await plain.aclose();await dedup.aclose()

annoscript/test_runtime.py

@@ -0,0 +1,314 @@
+"""Independent runtime semantics, bounded scheduling and per-state batching."""
+
+import asyncio
+from collections import Counter
+from contextlib import ExitStack
+from copy import deepcopy
+from dataclasses import replace
+import unittest
+from unittest.mock import patch
+
+from annoscript._data import thaw
+from annoscript.backends import BatchResult, InferenceOutput, BackendFatalError
+from annoscript.compiler import compile_program
+from annoscript.plan import ExecutionPlan
+from annoscript import ir, plan as p
+from annoscript.plan_lowering import lower_to_plan
+from annoscript.runtime import AnnoRuntime, RunInput
+from annoscript.anno import NoulResponse, ChoiceResponse, ScoreResponse, GenerateResponse, constrain
+from annoscript import test_plan as fixtures
+from annoscript import test_plan_constraints as checks
+from annoscript._test_plan_runtime import evaluate
+
+
+def mixed(anno):
+    a = anno.noul("yes?").noul
+    b = anno.choice("which?", {"a": "A", "b": "B"}).choice
+    c = anno.score("score?", ["bad", "good"]).score
+    return [a, b, c]
+
+
+def decision_chain(anno):
+    a = anno.choice("first", {"a": "A", "b": "B"}).choice
+    b = anno.noul("independent").noul
+    return anno.noul(a).noul + b
+
+
+def decision_loop(anno):
+    result = []
+    for item in anno.state:
+        result = result + [anno.noul(item).noul]
+    return result
+
+
+def changed_criteria(anno):
+    criteria = {"a": "before", "b": "other"}
+    alias = criteria
+    first = anno.choice("choose", criteria).choice
+    alias["a"] = "after"
+    second = anno.choice("choose", criteria).choice
+    return [first, second]
+
+
+class FakeBackend:
+    def __init__(self, table=None, fail=()):
+        self.table, self.fail = table, fail
+        self.calls, self.batches = [], []
+        self.closed = False
+
+    def answer(self, call):
+        a = thaw(call.arguments)
+        self.calls.append(fixtures._key(call.operation, (), a))
+        if self.table is not None:
+            result = self.table[self.calls[-1]]
+            if isinstance(result, Exception):
+                raise type(result)(str(result))
+            return deepcopy(result)
+        if a["instructions"] in self.fail:
+            raise ValueError(a["instructions"])
+        if call.operation == "generate":
+            return GenerateResponse(a["instructions"])
+        if call.operation == "noul":
+            return NoulResponse(0.8)
+        if call.operation == "choice":
+            return ChoiceResponse(next(iter(a["criteria"])), 0.9, {k: float(i == 0) for i, k in enumerate(a["criteria"])})
+        if call.operation == "score":
+            return ScoreResponse({i: float(i == 0) for i in range(len(a["criteria"]))}, 0, dict(enumerate(a["criteria"])), 0.9)
+        raise AssertionError(call.operation)
+
+    async def infer(self, context, call):
+        return InferenceOutput(self.answer(call))
+
+    async def infer_batch(self, context, calls):
+        self.batches.append((thaw(context.state), [c.operation for c in calls], [c.arguments["instructions"] for c in calls]))
+        return BatchResult({c.id: self.answer(c) for c in calls}, {"usage": {"input_tokens": 100}})
+
+    async def aclose(self):
+        self.closed = True
+
+
+async def execute(function, state, *, backend=None, mode="eager", **options):
+    backend = backend or FakeBackend()
+    async with AnnoRuntime(backend, backend, trace=True, **options) as runtime:
+        plan = ExecutionPlan.from_json(lower_to_plan(compile_program(function)).to_json())
+        before = plan.to_dict()
+        stream = await runtime.submit(plan, [RunInput(0, state)], mode=mode)
+        result = [r async for r in stream]
+        assert before == plan.to_dict()
+        return result[0], backend, runtime.trace
+
+
+class RuntimeTests(unittest.IsolatedAsyncioTestCase):
+    async def test_all_plan_shapes_against_independent_interpreter(self):
+        cases = [
+            (fixtures.independent, "state"), (fixtures.duplicates, "state"),
+            (fixtures.branch_duplicates, []), (fixtures.branch_duplicates, [1]),
+            (fixtures.unused_branch, []), (fixtures.loop_invariant, [True, False, True]),
+            (fixtures.independent_loop, [str(i) for i in range(40)]),
+            (fixtures.last_value, []), (fixtures.last_value, ["a", "b"]),
+            (fixtures.conditional_last, ["a", "", "b"]),
+            (fixtures.nested_loop, [["a", "b"], [], ["c"]]),
+            (fixtures.local_container, ["a", "b"]), (fixtures.memory_snapshots, "x"),
+            (fixtures.iteration_alias, ["a", "b", "c"]),
+            (fixtures.nested_memory, [["a", "b"], [], ["c"]]),
+            (fixtures.iterator_twice, ["a", "b"]),
+            (fixtures.early_return, []), (fixtures.early_return, [True]),
+            (fixtures.nested_return, [False]), (fixtures.nested_return, [True]),
+            (fixtures.partial_nested_return, []), (fixtures.partial_nested_return, [False]),
+            (checks.checked_memory, "x"), (checks.checked_rebinding, "x"),
+            (checks.checked_shared, "same"), (checks.checked_memory_loop, ["a", "b"]),
+        ]
+        for fn, state in cases:
+            for mode in ("eager", "strict"):
+                with self.subTest(function=fn.__name__, mode=mode, state=state):
+                    plan = lower_to_plan(compile_program(fn))
+                    reference = fixtures.PureAnnoScript(deepcopy(state))
+                    expected = await asyncio.to_thread(evaluate, plan.to_dict(), reference, mode=mode)
+                    result, backend, _ = await execute(fn, state, mode=mode, loop_window=2)
+                    self.assertEqual(result.status, "success", result.error)
+                    self.assertEqual(result.value, expected)
+                    expected_calls = [fixtures._key("generate", (x,), opts) for x, opts in reference.calls]
+                    self.assertEqual(Counter(backend.calls), Counter(expected_calls))
+
+    async def test_mixed_batch_and_usage_once(self):
+        result, backend, trace = await execute(mixed, "shared")
+        self.assertEqual(result.value, [0.8, "a", 0])
+        self.assertEqual(backend.batches[0][1], ["noul", "choice", "score"])
+        self.assertEqual(len(backend.batches), 1)
+        self.assertEqual(sum(e.get("usage", {}).get("input_tokens", 0) for e in trace), 100)
+        _, backend, _ = await execute(mixed, "shared", mode="strict")
+        self.assertEqual([len(b[1]) for b in backend.batches], [1, 1, 1])
+
+    async def test_dependency_forms_later_batch_and_loop_capacity(self):
+        result, backend, _ = await execute(decision_chain, "x")
+        self.assertEqual(result.value, 1.6)
+        self.assertEqual([b[2] for b in backend.batches], [["first", "independent"], ["a"]])
+        result, backend, _ = await execute(decision_loop, [str(i) for i in range(9)], max_batch_questions=3, loop_window=4)
+        self.assertEqual(result.value, [0.8] * 9)
+        self.assertTrue(all(len(b[1]) <= 3 for b in backend.batches))
+        self.assertGreater(max(len(b[1]) for b in backend.batches), 1)
+
+    async def test_mutated_criteria_preserves_call_snapshots(self):
+        for mode in ("eager", "strict"):
+            result, backend, _ = await execute(changed_criteria, "x", mode=mode)
+            self.assertEqual(result.value, ["a", "a"])
+            self.assertEqual(backend.calls, [fixtures._key("choice", ("choose", {"a": value, "b": "other"}), {}) for value in ("before", "after")])
+
+    async def test_inputs_never_share_batches_and_are_snapshotted(self):
+        backend = FakeBackend()
+        async with AnnoRuntime(backend, backend) as runtime:
+            source = {"text": "original"}
+            stream = await runtime.submit(lower_to_plan(compile_program(mixed)), [RunInput(3, source), RunInput(9, source)])
+            source["text"] = "changed"
+            result = [r async for r in stream]
+            self.assertEqual({r.index for r in result}, {3, 9})
+            self.assertEqual(len(backend.batches), 2)
+            self.assertTrue(all(b[0] == {"text": "original"} for b in backend.batches))
+
+    async def test_failure_prefix_and_constraint_paths(self):
+        for fn, state, fail, prefix in [
+            (fixtures.independent, "x", ("a",), ["a"]),
+            (fixtures.expression_failure, "x", (), ["first"]),
+            (fixtures.unpack_failure, [1], (), []),
+            (fixtures.unused_failure, "x", ("bad",), ["bad"]),
+            (fixtures.independent_loop, ["bad", "later"], ("bad",), ["bad"]),
+            (checks.checked_chain, "x", (), ["check"]),
+            (checks.checked_container_truth, "x", (), []),
+        ]:
+            with self.subTest(function=fn.__name__):
+                result, backend, _ = await execute(fn, state, backend=FakeBackend(fail=fail), mode="strict")
+                self.assertEqual(result.status, "error")
+                expected = [fixtures._key("generate", (x,), {}) for x in prefix]
+                self.assertEqual(backend.calls, expected)
+                self.assertIn("task=", result.error.message)
+
+    async def test_eager_failure_cancels_pending_and_no_new_dispatch(self):
+        class Controlled(FakeBackend):
+            async def infer(self, context, call):
+                name = call.arguments["instructions"]
+                self.started.append(name)
+                if name == "check":
+                    await self.other_started.wait()
+                    return InferenceOutput(GenerateResponse("bad"))
+                if name == "independent":
+                    self.other_started.set()
+                    try:
+                        await asyncio.Event().wait()
+                    finally:
+                        self.cancelled.set()
+                self.fail_unexpected = True
+        backend = Controlled()
+        backend.started, backend.other_started, backend.cancelled = [], asyncio.Event(), asyncio.Event()
+        result, _, _ = await execute(checks.checked_chain, "x", backend=backend)
+        self.assertEqual(result.error.code, "constraint_unsatisfied")
+        self.assertCountEqual(backend.started, ["check", "independent"])
+        self.assertTrue(backend.cancelled.is_set())
+
+    async def test_concurrency_limit_close_and_submission_isolation(self):
+        class Controlled(FakeBackend):
+            async def infer(self, context, call):
+                self.active += 1
+                self.peak = max(self.active, self.peak)
+                self.started.set()
+                try:
+                    await self.release.wait()
+                    return InferenceOutput(GenerateResponse(call.arguments["instructions"]))
+                finally:
+                    self.active -= 1
+        backend = Controlled()
+        backend.active = backend.peak = 0
+        backend.started, backend.release = asyncio.Event(), asyncio.Event()
+        async with AnnoRuntime(backend, backend, max_llm_inflight=1) as runtime:
+            plan = lower_to_plan(compile_program(fixtures.independent))
+            first = await runtime.submit(plan, [RunInput(0, "x")])
+            second = await runtime.submit(plan, [RunInput(1, "y")])
+            await asyncio.wait_for(backend.started.wait(), 2)
+            await first.aclose()
+            backend.release.set()
+            result = [r async for r in second]
+            self.assertEqual(result[0].value, ["a", "b"])
+            self.assertEqual(backend.peak, 1)
+            self.assertFalse(backend.closed)
+
+    async def test_active_run_bound_and_local_work_quantum(self):
+        result, backend, trace = await execute(decision_loop, [str(i) for i in range(20)], local_task_budget=3, loop_window=2)
+        self.assertEqual(result.value, [0.8] * 20)
+        self.assertEqual(sum(len(b[1]) for b in backend.batches), 20)
+
+    async def test_success_waits_for_unused_calls_and_checks(self):
+        for function, state in ((checks.checked_constant_return, "x"), (checks.checked_nested_return, [True]),
+                                (checks.checked_unused_loop, ["a", "b"])):
+            class Delayed(FakeBackend):
+                async def infer(self, context, call):
+                    self.started.set()
+                    await self.release.wait()
+                    return InferenceOutput(GenerateResponse("bad"))
+            backend = Delayed()
+            backend.started, backend.release = asyncio.Event(), asyncio.Event()
+            async with AnnoRuntime(backend, backend) as runtime:
+                stream = await runtime.submit(lower_to_plan(compile_program(function)), [RunInput(0, state)])
+                next_result = asyncio.create_task(stream.__anext__())
+                await asyncio.wait_for(backend.started.wait(), 2)
+                self.assertFalse(next_result.done())
+                backend.release.set()
+                result = await asyncio.wait_for(next_result, 2)
+                self.assertEqual(result.error.code, "constraint_unsatisfied")
+
+    async def test_first_observed_error_beats_source_order(self):
+        class Failing(FakeBackend):
+            async def infer(self, context, call):
+                if call.arguments["instructions"] == "a":
+                    self.first_started.set()
+                    await asyncio.Event().wait()
+                await self.first_started.wait()
+                raise ValueError("b observed first")
+        backend = Failing()
+        backend.first_started = asyncio.Event()
+        result, _, _ = await execute(fixtures.independent, "x", backend=backend)
+        self.assertIn("b observed first", result.error.message)
+
+    async def test_active_runs_and_loop_window_before_completion(self):
+        class Delayed(FakeBackend):
+            async def infer(self, context, call):
+                self.started.set()
+                await self.release.wait()
+                return InferenceOutput(GenerateResponse(call.arguments["instructions"]))
+        backend = Delayed()
+        backend.started, backend.release = asyncio.Event(), asyncio.Event()
+        async with AnnoRuntime(backend, backend, max_active_runs=1, loop_window=2, trace=True) as runtime:
+            stream = await runtime.submit(lower_to_plan(compile_program(fixtures.independent_loop)),
+                                          [RunInput(i, ["a", "b", "c", "d"]) for i in range(3)])
+            await asyncio.wait_for(backend.started.wait(), 2)
+            self.assertEqual(sum(e["event"] == "run_started" for e in runtime.trace), 1)
+            self.assertEqual(sum(e["event"] == "infer_ready" for e in runtime.trace), 2)
+            backend.release.set()
+            self.assertEqual(len([r async for r in stream]), 3)
+
+    async def test_deserialized_plan_cannot_write_input_state(self):
+        plan = lower_to_plan(compile_program(fixtures.memory_snapshots))
+        plan = replace(plan, tasks=tuple(replace(t, container=p.BoundExpr(ir.State())) if isinstance(t, p.Store) else t for t in plan.tasks))
+        plan = ExecutionPlan.from_json(plan.to_json())
+        state = {"x": "input"}
+        backend = FakeBackend()
+        async with AnnoRuntime(backend, backend) as runtime:
+            stream = await runtime.submit(plan, [RunInput(0, state)], mode="strict")
+            result = [r async for r in stream]
+            self.assertEqual(result[0].status, "error")
+            self.assertIn("locally constructed", result[0].error.message)
+            self.assertEqual(state, {"x": "input"})
+
+    async def test_invalid_plan_and_empty_submission(self):
+        backend = FakeBackend()
+        async with AnnoRuntime(backend, backend) as runtime:
+            plan = lower_to_plan(compile_program(mixed))
+            stream = await runtime.submit(plan, [])
+            self.assertEqual([r async for r in stream], [])
+            with self.assertRaises(ValueError):
+                await runtime.submit(plan, [RunInput(0, "x"), RunInput(0, "y")])
+            with self.assertRaises(ValueError):
+                await runtime.submit(plan, [RunInput(0, "x")], mode="invalid")
+            self.assertEqual(backend.calls, [])
+
+
+if __name__ == "__main__":
+    unittest.main()