annoscript 0.1.0__py3-none-any.whl

annoscript/test_diagnostics.py

@@ -0,0 +1,97 @@
+"""Opt-in raw diagnostics preserve wire behavior, failure isolation and usage."""
+import asyncio,json,tempfile,threading,unittest
+from pathlib import Path
+from unittest.mock import patch
+import httpx
+from annoscript.backends import DecisionAPIBackend, InferenceContext, InferenceError
+from annoscript._test_evaluation_http import FixtureLLMBackend as LLMAPIBackend
+from annoscript.backends.base import normalize_call
+from annoscript.server.store import Store
+from annoscript.server.releases import Releases
+from annoscript.server.worker import Worker
+from annoscript.test_jobs_server import configurations,payload
+from annoscript.test_llm_batch import mixed_generation,answer,completion
+from annoscript._test_evaluation_http import llm_content,llm_calls
+
+class WireTests(unittest.IsolatedAsyncioTestCase):
+ async def test_raw_before_validation_and_same_requests_and_results(self):
+  for content,status,finish,usage in [('{"items":["广州"]}',200,'stop',{'prompt_tokens':5,'completion_tokens':2}),('{',200,'stop',None),('{"items":["北京"]}',200,'stop',{}),('{"items":[]}',200,'length',{}),('{"error":{"message":"context length"}}',400,None,None)]:
+   bodies=[];outcomes=[];events=[]
+   for enabled in (False,True):
+    def handler(req):
+     bodies.append(req.content)
+     body={'choices':[{'message':{'content':content},'finish_reason':finish}]}
+     if usage is not None:body['usage']=usage
+     return httpx.Response(status,json=body) if status==200 else httpx.Response(status,content=content)
+    b=LLMAPIBackend('https://offline.invalid','secret-fixture','qwen',profile='qwen_vllm',transport=httpx.MockTransport(handler),diagnostic_observer=events.append if enabled else None)
+    context=InferenceContext('广州');call=normalize_call('topic','extract',('地名',),{'max_items': 8, 'max_length_per_item': 58},context)
+    try:
+     result=await b.infer(context,call);outcomes.append(result.response.items)
+    except InferenceError as e:outcomes.append(e.code)
+    finally:await b.aclose()
+   self.assertEqual(bodies[0],bodies[1]);self.assertEqual(outcomes[0],outcomes[1])
+   wire_events=[e for e in events if e['event']!='advisory_lengths']
+   self.assertEqual([e['event'] for e in wire_events],['request','response','outcome'])
+   self.assertEqual(len({e['request_id'] for e in events}),1)
+   self.assertEqual(events[0]['body'],bodies[1]);self.assertTrue(events[1]['body'])
+   self.assertNotIn('secret-fixture',repr(events));self.assertEqual(wire_events[-1]['http_status'],status)
+
+class WorkerTests(unittest.TestCase):
+ def setUp(self):
+  self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup)
+  self.store=Store(Path(self.temp.name)/'tasks.sqlite');self.releases=Releases(configurations(retry_unknown_outcomes=False),'test')
+ def worker(self,handler,**kw):
+  def backend(_):
+   transport=httpx.MockTransport(handler)
+   return DecisionAPIBackend('https://offline.invalid','secret','m',transport=transport),LLMAPIBackend('https://offline.invalid','secret','m',profile='qwen_vllm',transport=transport)
+  return Worker(self.store,self.releases,backend_factory=backend,**kw)
+ def submit(self,key,**kw):return self.store.submit('alice',key,payload(**kw),self.releases)
+ def test_batch_repair_raw_parent_usage_and_runtime_mapping(self):
+  calls=[];audit=[];raw=[]
+  def handler(req):
+   body=json.loads(req.content);calls.append(body);value=llm_content(body,answer)
+   if llm_calls(body)[0]:
+    v=json.loads(value);del v['answers'][next(iter(v['answers']))];value=json.dumps(v)
+   return completion(value)
+  job=self.submit('batch',states=('广州与北京',),program=mixed_generation)
+  asyncio.run(self.worker(handler,request_observer=audit.append,diagnostic_observer=raw.append).execute(self.store.claim('w',self.releases)))
+  self.assertEqual(self.store.status('alice',job['job_id'])['counts']['success'],1)
+  rr=[e for e in raw if e['event']=='request'];self.assertEqual(len(rr),2)
+  self.assertEqual(rr[1]['repair_parent_request_ids'],[rr[0]['request_id']]);self.assertTrue(rr[1]['repair'])
+  self.assertEqual({e['request_id'] for e in audit},{e['request_id'] for e in rr})
+  self.assertTrue(any(e['event']=='trace' and e['plan'] for e in raw))
+ def test_diagnostic_failure_pauses_without_model_retry_and_keeps_usage(self):
+  for phase,expected in [('request',0),('response',1),('outcome',1)]:
+   job=self.submit(phase,states=('one',));calls=[];audit=[]
+   def observe(event):
+    if event['event']==phase:raise OSError('disk fixture')
+   def handler(req):calls.append(req);return httpx.Response(200,json={'choices':[{'finish_reason':'stop','message':{'content':'yes'}}],'usage':{'prompt_tokens':5,'completion_tokens':1}})
+   asyncio.run(self.worker(handler,request_observer=audit.append,diagnostic_observer=observe).execute(self.store.claim('w',self.releases)))
+   self.assertEqual(len(calls),expected)
+   self.assertEqual(self.store.status('alice',job['job_id'])['pause_reason']['code'],'diagnostics_failed')
+   if expected:self.assertEqual([x for x in audit if x['event']=='outcome'][0]['usage']['prompt_tokens'],5)
+ def test_item_failure_completes_and_next_batch_can_be_claimed(self):
+  count=[]
+  for batch in range(2):
+   job=self.submit(str(batch),states=tuple(str(i) for i in range(20)))
+   def handler(req):count.append(req);return httpx.Response(200,content=b'{')
+   worker=self.worker(handler,diagnostic_observer=lambda e:None)
+   for _ in range(20):asyncio.run(worker.execute(self.store.claim('w',self.releases)))
+   self.assertEqual(self.store.status('alice',job['job_id'])['state'],'completed')
+   self.assertEqual(self.store.status('alice',job['job_id'])['counts']['error'],20)
+  self.assertEqual(len(count),40)
+ def test_cancel_during_commit_keeps_raw_response_and_matching_usage(self):
+  self.submit('cancel',states=('one',));entered=threading.Event();release=threading.Event();original=self.store.request_finished;raw=[];audit=[]
+  def finish(*a,**kw):entered.set();release.wait(5);return original(*a,**kw)
+  def handler(req):return httpx.Response(200,json={'choices':[{'finish_reason':'stop','message':{'content':'yes'}}],'usage':{'prompt_tokens':9,'completion_tokens':1}})
+  async def run():
+   with patch.object(self.store,'request_finished',side_effect=finish):
+    task=asyncio.create_task(self.worker(handler,request_observer=audit.append,diagnostic_observer=raw.append).execute(self.store.claim('w',self.releases)))
+    async with asyncio.timeout(5):
+     while not entered.is_set():await asyncio.sleep(.005)
+    task.cancel();release.set()
+    with self.assertRaises(asyncio.CancelledError):await task
+  asyncio.run(run())
+  self.assertEqual(len([e for e in raw if e['event']=='response']),1)
+  self.assertEqual([e for e in audit if e['event']=='outcome'][0]['usage']['prompt_tokens'],9)
+  self.assertEqual([e for e in raw if e['event']=='outcome'][0]['http_status'],200)

annoscript/test_engineering_p0.py

@@ -0,0 +1,191 @@
+"""Engineering P0 regressions: no live models or business-system access."""
+import asyncio
+from dataclasses import replace
+import json
+import time
+import unittest
+from unittest.mock import patch
+
+import httpx
+
+from annoscript._data import thaw
+from annoscript._test_evaluation_http import llm_calls, llm_content
+from annoscript.backends import InferenceContext, BackendFatalError, InferenceError
+from annoscript._test_evaluation_http import FixtureLLMBackend as LLMAPIBackend
+from annoscript.backends.base import normalize_call
+from annoscript.compiler import compile_program
+from annoscript.ir import Program
+from annoscript.plan_lowering import lower_to_plan
+from annoscript.server._plan_cache import PlanCache
+from annoscript.server.worker import ObservedBackend, Worker
+from annoscript.test_generation_policy import policy
+from annoscript.test_llm_batch import answer, completion
+
+
+def identity_program(anno):
+    return anno.state
+
+
+class PlanCacheTests(unittest.IsolatedAsyncioTestCase):
+    async def test_concurrent_miss_cancel_and_release_isolation(self):
+        ir = compile_program(identity_program).to_dict()
+        cache = PlanCache()
+        compiled = []
+        def build(encoded):
+            time.sleep(.025)
+            compiled.append(encoded)
+            return lower_to_plan(Program.from_json(encoded.decode()))
+        try:
+            tasks = [asyncio.create_task(cache.get("a", ir, build)) for _ in range(4)]
+            await asyncio.sleep(.005)
+            tasks[0].cancel()
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            self.assertIsInstance(results[0], asyncio.CancelledError)
+            self.assertEqual(len(compiled), 1)
+            self.assertTrue(all(v[0] is results[1][0] for v in results[1:]))
+            self.assertEqual((await cache.get("a", ir, build))[1]["status"], "hit")
+            await cache.get("b", ir, build)
+            changed = json.loads(json.dumps(ir)); changed["source"]["line"] += 1
+            await cache.get("a", changed, build)
+            self.assertEqual(len(compiled), 3)
+            self.assertEqual(cache.stats["waits"], 3)
+        finally:
+            await cache.aclose()
+
+    async def test_lru_oversize_and_failure_not_cached(self):
+        ir = compile_program(identity_program).to_dict()
+        build = lambda b: lower_to_plan(Program.from_json(b.decode()))
+        for options in ({"max_entries": 1}, {"max_bytes": 1}):
+            cache = PlanCache(**options)
+            try:
+                await cache.get("a", ir, build)
+                await cache.get("b", ir, build)
+                _, event = await cache.get("a", ir, build)
+                self.assertEqual(event["status"], "miss")
+                self.assertLessEqual(len(cache.entries), options.get("max_entries", 32))
+                self.assertLessEqual(cache.bytes, options.get("max_bytes", 64*1024*1024))
+                def fail(b): raise ValueError("bad IR")
+                for _ in range(2):
+                    with self.assertRaises(ValueError): await cache.get("failure", ir, fail)
+                self.assertEqual(cache.stats["failures"], 2)
+            finally:
+                await cache.aclose()
+
+
+class PreparedRequestTests(unittest.IsolatedAsyncioTestCase):
+    def backend(self, events=None, profile="qwen_vllm", window=8192, counter=None):
+        self.sent = []
+        def handle(request):
+            self.sent.append(request)
+            return completion(llm_content(json.loads(request.content), answer))
+        return LLMAPIBackend("https://offline.invalid", "fixture", "fixture", profile=profile,
+            request_policy=policy(window) if profile == "qwen_vllm" else None,
+            _token_counter=counter or (lambda p: 10), transport=httpx.MockTransport(handle),
+            diagnostic_observer=events.append if events is not None else None)
+
+    def calls(self, ctx):
+        return [normalize_call("a", "generate", ("标题",), {"max_length": 8}, ctx),
+                normalize_call("b", "extract", ("地点",),
+                               {"source_field": "text" if isinstance(ctx.state, dict) or hasattr(ctx.state, "keys") else None, "max_items": 2, "max_length_per_item": 8}, ctx)]
+
+    def plan(self, b, ctx, calls):
+        return b.plan_batch(ctx, calls, max_calls=8, max_input_bytes=262144,
+                            max_output_tokens=4096, default_tokens=1024)
+
+    async def test_selected_payload_sent_without_rebuild_count_or_encoding(self):
+        events=[]; b=self.backend(events); ctx=InferenceContext({"text":"广州与北京"}, "业务策略")
+        calls=self.calls(ctx)
+        try:
+            p=self.plan(b,ctx,calls)
+            before=dict(b.preparation_stats)
+            self.assertIs(self.plan(b,ctx,calls),p)
+            with patch.object(b,"_prepare",side_effect=AssertionError("prepared again")):
+                out=await b.infer_batch(ctx,calls,max_tokens=p.max_tokens,prepared=p.prepared)
+            self.assertEqual(out.responses['b'].items,['广州'])
+            self.assertEqual(b.preparation_stats['token_counts'],before['token_counts'])
+            self.assertEqual(b.preparation_stats['encodes'],before['encodes'])
+            self.assertEqual(self.sent[0].content,p.prepared.body)
+            self.assertEqual(next(e['body'] for e in events if e['event']=='request'),self.sent[0].content)
+            self.assertTrue(thaw(p.prepared.payload)['messages'][0]['content'].startswith('业务策略\n'))
+            with self.assertRaises(TypeError): p.prepared.payload['model']='changed'
+        finally: await b.aclose()
+
+    async def test_stale_budget_context_calls_and_config_are_reprepared(self):
+        b=self.backend();ctx=InferenceContext({"text":"广州与北京"});calls=self.calls(ctx)
+        try:
+            p=self.plan(b,ctx,calls)
+            tests=[(ctx,calls,p.max_tokens+1),
+                   (InferenceContext({"text":"广州与北京", "extra": "new"}),calls,p.max_tokens),
+                   (ctx,[replace(calls[0],arguments={**calls[0].arguments,"instructions":"不同标题"}),calls[1]],p.max_tokens)]
+            for c,cs,budget in tests:
+                before=b.preparation_stats['payload_builds']
+                await b.infer_batch(c,cs,max_tokens=budget,prepared=p.prepared)
+                self.assertEqual(b.preparation_stats['payload_builds'],before+1)
+                self.assertEqual(json.loads(self.sent[-1].content)['max_tokens'],budget)
+            b.model="different"
+            self.assertIsNot(self.plan(b,ctx,calls),p)
+        finally: await b.aclose()
+
+    async def test_source_fields_single_repair_batch_and_other_profiles(self):
+        texts=['广州与北京','甲"乙\\丙\n😀e\u0301','', '\\u1234']
+        for profile in ['qwen_vllm','legacy']:
+            for text in texts:
+                for state in [text,{"text":text,"other":"独立来源"}]:
+                    b=self.backend(profile=profile);ctx=InferenceContext(state)
+                    c=normalize_call('e','extract',('原文',),{'source_field':None if isinstance(state,str) else 'text','max_items':2,'max_length_per_item':8},ctx)
+                    try:
+                        for feedback in [None,'bad structure']:
+                            p=b._prepare(ctx,[c],80,repair_feedback=feedback)
+                            content=thaw(p.payload)['messages'][-1]['content']
+                            self.assertEqual(content if isinstance(state,str) else json.loads(content),state)
+                            prompt=thaw(p.payload)['messages'][0]['content']
+                            self.assertIn('输入字符串' if isinstance(state,str) else '输入的顶层字段 "text"',prompt)
+                            self.assertEqual(llm_calls(thaw(p.payload))[1][0][1].extraction_source,text)
+                        other = replace(c,id='other') if isinstance(state,str) else normalize_call('other','extract',('原文',),{'source_field':'other','max_items':1,'max_length_per_item':8},ctx)
+                        p=self.plan(b,ctx,[c,other,replace(c,id='again')])
+                        reconstructed=llm_calls(thaw(p.prepared.payload))[1]
+                        self.assertEqual([v.extraction_source for _,v in reconstructed],[text,text if isinstance(state,str) else '独立来源',text])
+                        content=thaw(p.prepared.payload)['messages'][-1]['content']
+                        self.assertEqual(content if isinstance(state,str) else json.loads(content),state)
+                    finally: await b.aclose()
+
+    async def test_prepared_pause_blocks_http_and_local_rejection_not_sent(self):
+        b=self.backend();ctx=InferenceContext('广州与北京');calls=self.calls(ctx)
+        p=self.plan(b,ctx,calls)
+        allowed=True
+        observed=ObservedBackend(b,[],lambda:True,dispatch_allowed=lambda:allowed)
+        allowed=False
+        try:
+            with self.assertRaises(BackendFatalError):
+                await observed.infer_batch(ctx,calls,max_tokens=p.max_tokens,prepared=p.prepared)
+            self.assertEqual(self.sent,[])
+        finally:await b.aclose()
+        b=self.backend(counter=lambda p:8192)
+        try:
+            with self.assertRaises(InferenceError) as e:await b.infer(ctx,calls[0])
+            self.assertEqual(e.exception.metadata['dispatch_status'],'not_sent')
+            self.assertEqual(self.sent,[])
+        finally:await b.aclose()
+
+
+class WorkerRefillTests(unittest.IsolatedAsyncioTestCase):
+    async def test_completion_refills_before_poll_timeout(self):
+        stop=asyncio.Event();times=[]
+        class Store:
+            n=0
+            def maintain(self):pass
+            def claim(self,*args):
+                if self.n>=2:return None
+                self.n+=1;return self.n
+        class TestWorker(Worker):
+            async def execute(self,claim):
+                times.append(time.monotonic())
+                await asyncio.sleep(.01)
+                if claim==2:stop.set()
+        w=TestWorker(Store(),None,concurrency=1,poll_interval=1)
+        await asyncio.wait_for(w.run(stop=stop),timeout=.5)
+        self.assertEqual(len(times),2)
+        self.assertLess(times[1]-times[0],.2)
+
+
+if __name__ == '__main__': unittest.main()

annoscript/test_environment.py

@@ -0,0 +1,65 @@
+"""Configuration compatibility across real file/process boundaries; no network."""
+
+import asyncio
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from annoscript._env import env_value
+from annoscript.backends.config import backends_from_env
+
+
+class EnvironmentTests(unittest.TestCase):
+    def configured(self, file_values, environment, *, generation_profile=None):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "configuration"
+            path.write_text("\n".join(f"{k}={v}" for k, v in file_values.items()))
+            with patch.dict(os.environ, environment, clear=True):
+                decision, generation = backends_from_env(env_file=path, generation_profile=generation_profile)
+                try:
+                    return decision.model, generation.model, generation.profile
+                finally:
+                    async def close():
+                        await decision.aclose()
+                        await generation.aclose()
+                    asyncio.run(close())
+
+    def baseline(self, prefix):
+        return {f"{prefix}_{stage}_{key}": value
+                for stage in ("DECISION", "GENERATE")
+                for key, value in (("BASE_URL", "https://example.invalid/v1"),
+                                   ("AUTH_TOKEN", "fixture"), ("MODEL", "file-model"))}
+
+    def test_legacy_file_remains_usable(self):
+        self.assertEqual(self.configured(self.baseline("LEV"), {}),
+                         ("file-model", "file-model", "legacy"))
+
+    def test_legacy_process_overrides_canonical_file(self):
+        self.assertEqual(self.configured(self.baseline("ANNO"), {"LEV_GENERATE_MODEL": "process"})[1],
+                         "process")
+
+    def test_canonical_process_overrides_legacy_file(self):
+        self.assertEqual(self.configured(self.baseline("LEV"), {"ANNO_GENERATE_MODEL": "process"})[1],
+                         "process")
+
+    def test_canonical_wins_within_each_layer(self):
+        file = {**self.baseline("LEV"), "ANNO_GENERATE_MODEL": "canonical-file"}
+        self.assertEqual(self.configured(file, {})[1], "canonical-file")
+        self.assertEqual(self.configured(file, {"LEV_GENERATE_MODEL": "old", "ANNO_GENERATE_MODEL": "new"})[1], "new")
+
+    def test_explicit_arguments_override_both_layers(self):
+        file = {**self.baseline("LEV"), "ANNO_GENERATE_PROFILE": "invalid"}
+        self.assertEqual(self.configured(file, {"LEV_GENERATE_PROFILE": "also-invalid"},
+                                         generation_profile="qwen_vllm")[2], "qwen_vllm")
+
+    def test_empty_canonical_value_does_not_restore_legacy_secret(self):
+        with self.assertRaisesRegex(ValueError, "ANNO_GENERATE_AUTH_TOKEN"):
+            self.configured(self.baseline("LEV"), {"ANNO_GENERATE_AUTH_TOKEN": ""})
+        self.assertEqual(env_value("ANNO_SERVICE_TOKEN", environ={"LEV_SERVICE_TOKEN": "old"}), "old")
+        self.assertEqual(env_value("ANNO_SERVICE_TOKEN", environ={"LEV_SERVICE_TOKEN": "old", "ANNO_SERVICE_TOKEN": "new"}), "new")
+
+
+if __name__ == "__main__":
+    unittest.main()

annoscript/test_example_backends.py

@@ -0,0 +1,251 @@
+"""Offline examples through real normalization, HTTP adapters and runtime."""
+
+from copy import deepcopy
+import json
+import unittest
+
+import httpx
+
+
+from annoscript._data import thaw
+from annoscript._test_evaluation_http import http_backends, llm_calls, decision_calls
+
+from annoscript.backends import InferenceContext, InferenceError, DecisionAPIBackend
+from annoscript._test_evaluation_http import FixtureLLMBackend as LLMAPIBackend
+from annoscript.backends.base import normalize_call, validate_response, BatchResult
+from annoscript.backends._bounded_grammar import grammar
+from annoscript.compiler import compile_program
+from annoscript.ir import Program
+from annoscript.anno import Example, ExtractResponse, ChoiceResponse
+from annoscript.plan import ExecutionPlan
+from annoscript.plan_lowering import lower_to_plan
+from annoscript.runtime import AnnoRuntime, RunInput
+from annoscript.test_examples import REVIEW, LAUNCH, captured, constructed, different_examples
+from annoscript.test_generation_policy import policy
+from annoscript.test_llm_batch import completion
+
+
+def call(operation, answer, *, identity="q", examples=True, **options):
+    arguments = {"choice": ("分类", {"正面": "积极", "中性": "中立"}),
+                 "noul": ("是事件吗",), "score": ("评分", ["评价", "资讯"]),
+                 "generate": ("标题",), "extract": ("地点",),
+                 "generate_list": ("标签",)}[operation]
+    limits = ({"max_length": 80} if operation == "generate" else
+              {"max_items": 2, "max_length_per_item": 80} if operation in ("extract", "generate_list") else {})
+    if operation == "extract":
+        limits["source_field"] = "text"
+    if examples:
+        limits["examples"] = [Example(LAUNCH, answer)]
+    return normalize_call(identity, operation, arguments, {**limits, **options}, InferenceContext(REVIEW))
+
+
+class ExampleValidationTests(unittest.TestCase):
+    def test_all_six_operations_and_advisory_lengths(self):
+        for operation, answer in [("choice", "中性"), ("noul", True), ("score", 1),
+                                  ("generate", LAUNCH["text"]), ("generate_list", ["资讯"]),
+                                  ("extract", ["广州"])]:
+            with self.subTest(operation=operation):
+                result = call(operation, answer)
+                self.assertEqual(thaw(result.arguments)["examples"], [{"state": LAUNCH, "answer": answer}])
+        call("generate", LAUNCH["text"], max_length=1)
+        call("generate_list", ["资讯"], max_length_per_item=1)
+        call("extract", ["广州"], max_length_per_item=1)
+
+    def test_invalid_answers_shapes_and_source_bindings(self):
+        cases = [
+            ("choice", "不存在", {}), ("choice", {"choice": "中性", "confidence": .95}, {}),
+            ("noul", .95, {}), ("score", True, {}), ("score", 2, {}), ("score", -1, {}),
+            ("generate", None, {}), ("generate", "", {"min_length": 1}),
+            ("generate_list", [None], {}), ("generate_list", [""], {"min_length_per_item": 1}),
+            ("generate_list", ["资讯"], {"max_items": 0}),
+            ("extract", [], {"min_items": 1}), ("extract", [""], {}),
+            ("extract", ["条目"], {}), ("extract", ["广州上市"], {}),
+            ("extract", ["广州"], {"source_field": "missing"}),
+        ]
+        for operation, answer, options in cases:
+            with self.subTest(operation=operation, answer=answer), self.assertRaises(ValueError):
+                call(operation, answer, **options)
+        for examples in ({}, (), [{"state": LAUNCH}], [{"state": LAUNCH, "answer": [], "criteria": {}}],
+                         [{"state": None, "answer": []}]):
+            with self.subTest(examples=examples), self.assertRaises(ValueError):
+                normalize_call("q", "extract", ("地点",),
+                               {"examples": examples, "source_field": "text",
+                                "max_items": 2, "max_length_per_item": 80}, InferenceContext(REVIEW))
+        c = normalize_call("q", "extract", ("地点",),
+                           {"examples": [Example(LAUNCH["text"], ["广州"])],
+                            "max_items": 2, "max_length_per_item": 80}, InferenceContext(REVIEW["text"]))
+        self.assertEqual(c.extraction_source, REVIEW["text"])
+
+    def test_examples_cannot_expand_current_source_or_reuse_prepared_request(self):
+        a = call("extract", ["广州"])
+        b = call("extract", [])
+        backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture",
+                                profile="qwen_vllm", extract_mode="substring_chars")
+        self.assertEqual(grammar([a], extract_mode="substring_chars"),
+                         grammar([b], extract_mode="substring_chars"))
+        with self.assertRaises(InferenceError):
+            validate_response(a, ExtractResponse(["广州"]))
+        validate_response(a, ExtractResponse(["条目"]))
+        context = InferenceContext(REVIEW)
+        pa = backend._prepare(context, [a], 32)
+        pb = backend._prepare(context, [b], 32)
+        self.assertEqual(pa.binding, backend._prepare(context, [a], 32).binding)
+        self.assertNotEqual(pa.binding, pb.binding)
+        self.assertNotEqual(pa.body, pb.body)
+        self.assertNotIn("广", pa.payload["structured_outputs"]["grammar"])
+
+
+class ExampleBackendTests(unittest.IsolatedAsyncioTestCase):
+
+
+    async def test_missing_and_empty_examples_preserve_wire_for_every_operation(self):
+        operations = [("choice", "中性"), ("noul", True), ("score", 1),
+                      ("generate", "标题"), ("generate_list", []), ("extract", [])]
+        sent = []
+        backend = DecisionAPIBackend("http://offline.invalid", "fixture", "fixture",
+                                     transport=httpx.MockTransport(lambda request: (
+                                         sent.append(json.loads(request.content)) or httpx.Response(200, json={"answers": {}}))))
+        llm = LLMAPIBackend("http://offline.invalid", "fixture", "fixture")
+        context = InferenceContext(REVIEW, "共用规则")
+        try:
+            for operation, answer in operations:
+                original = call(operation, answer, examples=False)
+                for empty in (None, []):
+                    empty_call = normalize_call("q", operation, (), {**thaw(original.arguments), "examples": empty}, context)
+                    self.assertEqual(empty_call, original)
+                    if operation in ("generate", "generate_list", "extract"):
+                        self.assertEqual(llm._single_payload(context, original, 100),
+                                         llm._single_payload(context, empty_call, 100))
+                    else:
+                        for c in (original, empty_call):
+                            with self.assertRaises(InferenceError):
+                                await backend.infer_batch(context, [c])
+                        self.assertEqual(sent[-2], sent[-1])
+        finally:
+            await backend.aclose()
+            await llm.aclose()
+
+    async def test_jev_criteria_examples_mixed_with_plain_question(self):
+        sent = []
+        def handle(request):
+            body = json.loads(request.content)
+            sent.append(body)
+            answers = {}
+            for identity, q in body["questions"].items():
+                if q["type"] == "noul":
+                    answer = {"noul": .9}
+                elif q["type"] == "score":
+                    answer = {"score": 1, "confidence": .9, "legend": {"0": "评价", "1": "资讯"},
+                              "probabilities": {"0": .1, "1": .9}}
+                else:
+                    answer = {"choice": "中性", "confidence": .9, "probabilities": {"正面": .1, "中性": .9}}
+                answers[identity] = {"type": q["type"], **answer}
+            return httpx.Response(200, json={"answers": answers})
+        backend = DecisionAPIBackend("http://offline.invalid", "fixture", "fixture", transport=httpx.MockTransport(handle))
+        calls = [call("choice", "中性", identity="choice"), call("noul", True, identity="noul"),
+                 call("score", 1, identity="score"), call("noul", True, identity="plain", examples=False)]
+        try:
+            await backend.infer_batch(InferenceContext(REVIEW, "共用规则"), calls)
+        finally:
+            await backend.aclose()
+        body = sent[0]
+        self.assertEqual(body["state"], REVIEW)
+        self.assertNotIn("examples", body)
+        for c in calls:
+            self.assertEqual(body["questions"][c.id]["instructions"], c.arguments["instructions"])
+        self.assertEqual(body["questions"]["choice"]["criteria"],
+                         {"正面": "积极", "中性": {"what": "中立", "examples": [LAUNCH]}})
+        self.assertEqual(body["questions"]["noul"]["criteria"], {"true": {"examples": [LAUNCH]}})
+        self.assertEqual(body["questions"]["score"]["criteria"],
+                         ["评价", {"what": "资讯", "examples": [LAUNCH]}])
+        self.assertNotIn("criteria", body["questions"]["plain"])
+
+    async def test_qwen_sharing_single_batch_and_repair_round_trip(self):
+        calls = [call("extract", ["广州"], identity="extract", max_num_tokens=32),
+                 call("generate", LAUNCH["text"], identity="generate", max_num_tokens=32),
+                 call("generate_list", ["资讯"], identity="list", max_num_tokens=32)]
+        context = InferenceContext(REVIEW)
+        backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture", profile="qwen_vllm")
+        bodies = [backend._batch_payload(context, calls, 256)]
+        for c in calls:
+            bodies.extend([backend._single_payload(context, c, 32),
+                           backend._single_payload(context, c, 32, "格式错误")])
+        for body in bodies:
+            self.assertEqual(json.loads(body["messages"][-1]["content"]), REVIEW)
+            self.assertEqual(body["messages"][0]["content"].count(
+                '输入：' + json.dumps(LAUNCH, ensure_ascii=False, separators=(",", ":"))), 1)
+            batched, decoded = llm_calls(body)
+            expected = calls if batched else [next(c for c in calls if c.id == decoded[0][1].id)]
+            for (_, reconstructed), original in zip(decoded, expected):
+                a, b = thaw(reconstructed.arguments), thaw(original.arguments)
+                a.pop("max_num_tokens")
+                b.pop("max_num_tokens")
+                self.assertEqual(a, b)
+        self.assertEqual(thaw(calls[0].arguments)["examples"], [{"state": LAUNCH, "answer": ["广州"]}])
+
+    async def test_examples_affect_split_bytes_context_checks_and_preserve_output_budget(self):
+        context = InferenceContext(REVIEW)
+        calls = [call("generate", LAUNCH["text"], identity=str(i), max_num_tokens=32) for i in range(3)]
+        plain = [call("generate", LAUNCH["text"], identity=str(i), examples=False, max_num_tokens=32) for i in range(3)]
+        def count(body):
+            return 100 * len(llm_calls(body)[1]) if "\n\nExample\n" in body["messages"][0]["content"] else 10
+        config = policy()
+        config.update(context_window=500, default_max_tokens=32)
+        backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture",
+                                profile="qwen_vllm", request_policy=config, _token_counter=count)
+        options = dict(max_calls=8, max_input_bytes=100000, max_output_tokens=400, default_tokens=32)
+        self.assertEqual(len(backend.plan_batch(context, plain, **options).calls), 3)
+        plan = backend.plan_batch(context, calls, **options)
+        self.assertEqual(len(plan.calls), 2)
+        self.assertEqual(plan.max_tokens, 192)
+        with self.subTest("byte limit"):
+            bounded = backend.plan_batch(context, calls, **{**options, "max_input_bytes": plan.input_bytes - 1})
+            self.assertIsNone(bounded)
+        sent = []
+        backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture",
+                                profile="qwen_vllm", request_policy=config,
+                                _token_counter=lambda b: 480 if "\n\nExample\n" in b["messages"][0]["content"] else 10,
+                                transport=httpx.MockTransport(lambda r: sent.append(r) or completion("标题")))
+        try:
+            with self.assertRaises(InferenceError) as caught:
+                await backend.infer(context, calls[0])
+            self.assertEqual(caught.exception.code, "context_length_exceeded")
+            self.assertEqual(sent, [])
+            await backend.infer(context, plain[0])
+            self.assertEqual(json.loads(sent[0].content)["max_tokens"], 32)
+        finally:
+            await backend.aclose()
+
+    async def test_runtime_snapshots_modes_and_different_examples_never_merge(self):
+        class Decision:
+            def __init__(self):
+                self.calls = []
+            async def infer_batch(self, context, calls):
+                self.calls.extend(calls)
+                return BatchResult({c.id: ChoiceResponse(c.arguments["examples"][0]["answer"], .9,
+                                   {"正面": .5, "中性": .5}) for c in calls})
+            async def aclose(self):
+                pass
+        class LLM:
+            async def infer(self, context, call):
+                from annoscript.backends import InferenceOutput
+                self.examples = thaw(call.arguments)["examples"]
+                return InferenceOutput(ExtractResponse([]))
+            async def aclose(self):
+                pass
+        for optimize in (False, True):
+            for mode in ("strict", "eager"):
+                decision, llm = Decision(), LLM()
+                async with AnnoRuntime(decision, llm) as runtime:
+                    for function in (captured, constructed, different_examples):
+                        program = Program.from_json(compile_program(function, optimize=optimize).to_json())
+                        plan = lower_to_plan(program)
+                        plan = ExecutionPlan.from_json(plan.to_json())
+                        result, = [r async for r in await runtime.submit(plan, [RunInput(0, REVIEW)], mode=mode)]
+                        self.assertEqual(result.status, "success", result.error)
+                        if function is different_examples:
+                            self.assertEqual(result.value, ["正面", "中性", "正面"])
+                            self.assertEqual(len(decision.calls), 2)
+                        else:
+                            self.assertEqual(llm.examples[0]["answer"], ["广州"])
+                            self.assertEqual(llm.examples[0]["state"]["text"], LAUNCH["text"])