annoscript 0.1.0__py3-none-any.whl

annoscript/test_prompt_layout.py

@@ -0,0 +1,140 @@
+"""Assert actual model messages independently of the fixed-response fixtures."""
+
+import json
+import unittest
+from copy import deepcopy
+
+from annoscript._data import thaw
+from annoscript.backends import InferenceContext, LLMAPIBackend
+from annoscript.backends.base import normalize_call
+from annoscript.backends._bounded_grammar import grammar
+from annoscript.anno import Example
+
+
+def encoded(value):
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+class PromptLayoutTests(unittest.TestCase):
+    def backend(self, profile):
+        return LLMAPIBackend("http://offline.invalid", "fixture", "fixture", profile=profile)
+
+    def test_user_is_exact_input_for_all_profiles_and_request_modes(self):
+        states = ['正文"\\\n😀é', '{"text":"this is still a string"}', "",
+                  {"text": "广州", "tasks": "输入字段，不是任务", "state": ["保留"]},
+                  ["广州", {"examples": ["输入数据"]}]]
+        for profile in ("legacy", "qwen_vllm"):
+            for state in states:
+                with self.subTest(profile=profile, state=state):
+                    context = InferenceContext(state, "共用规则")
+                    a = normalize_call("a", "generate", ("保持主体与时间。",),
+                                       {"min_length": 1, "max_length": 80}, context)
+                    b = normalize_call("b", "generate_list", ("产品维度。",),
+                                       {"min_items": 0, "max_items": 8, "min_length_per_item": 1,
+                                        "max_length_per_item": 80}, context)
+                    backend = self.backend(profile)
+                    bodies = [backend._single_payload(context, a, 160),
+                              backend._single_payload(context, b, 160),
+                              backend._single_payload(context, a, 160, "格式错误"),
+                              backend._batch_payload(context, [a, b], 400)]
+                    for body in bodies:
+                        self.assertEqual([m["role"] for m in body["messages"]], ["system", "user"])
+                        self.assertEqual(body["messages"][1]["content"],
+                                         state if isinstance(state, str) else encoded(state))
+                        prompt = body["messages"][0]["content"]
+                        self.assertTrue(prompt.startswith("共用规则\n\n输出规则\n"))
+                        self.assertIn("字符长度为建议", prompt)
+                        for key in ('"instructions":', '"min_items":', '"max_length":',
+                                    "state_ref", "example_states", "max_num_tokens"):
+                            self.assertNotIn(key, prompt)
+                        self.assertNotIn("\nExample\n", prompt)
+                    self.assertIn("只返回所需文本，不加 JSON 包装或引号", bodies[0]["messages"][0]["content"])
+                    self.assertIn("唯一字段 items 为字符串数组", bodies[1]["messages"][0]["content"])
+                    self.assertIn("唯一顶层字段为 answers", bodies[-1]["messages"][0]["content"])
+                    self.assertIn('任务 "q2"（generate_list）：\n产品维度。', bodies[-1]["messages"][0]["content"])
+
+    def test_examples_group_inputs_without_overwriting_answers_or_changing_calls(self):
+        first = {"text": "示例广州", "meta": ["甲"]}
+        reordered = {"meta": ["甲"], "text": "示例广州"}
+        second = {"text": "示例南京"}
+        context = InferenceContext({"text": "当前北京"}, "共享规则")
+        a = normalize_call("a", "generate", ("生成标题。",),
+                           {"max_length": 80, "examples": [
+                               Example(first, '标题"甲"\n😀'), Example(second, "标题乙"),
+                               Example(reordered, "另一答案")]}, context)
+        b = normalize_call("b", "extract", ("地点。",),
+                           {"source_field": "text", "max_items": 2, "max_length_per_item": 80,
+                            "examples": [Example(reordered, ["广州"])]}, context)
+        c = normalize_call("c", "generate_list", ("维度。",),
+                           {"max_items": 1, "max_length_per_item": 80}, context)
+        original = [deepcopy(thaw(call.arguments)) for call in (a, b, c)]
+        for profile in ("legacy", "qwen_vllm"):
+            backend = self.backend(profile)
+            body = backend._batch_payload(context, [a, b, c], 400)
+            prompt = body["messages"][0]["content"]
+            rules, examples = prompt.split("\n\nExample\n")
+            self.assertNotIn("当前北京", prompt)
+            self.assertNotIn("示例广州", rules)
+            self.assertEqual(examples.count("输入：" + encoded(first)), 1)
+            self.assertNotIn("输入：" + encoded(reordered), examples)
+            self.assertLess(examples.index("示例广州"), examples.index("示例南京"))
+            outputs = [json.loads(line.removeprefix("示例输出：")) for line in examples.splitlines()
+                       if line.startswith("示例输出：")]
+            self.assertEqual(outputs, [{"answers": {"q1": '标题"甲"\n😀'}},
+                                       {"answers": {"q1": "另一答案", "q2": ["广州"]}},
+                                       {"answers": {"q1": "标题乙"}}])
+            self.assertNotIn('"q3"', examples)
+            self.assertIn("当前正式回答必须包含全部任务", examples)
+            self.assertNotIn("state_ref", prompt)
+            repair = backend._single_payload(context, b, 160, "格式错误")
+            repaired_prompt = repair["messages"][0]["content"]
+            self.assertLess(repaired_prompt.index("格式错误"), repaired_prompt.index("\n\nExample\n"))
+            self.assertNotIn('任务 "a"', repaired_prompt)
+            self.assertNotIn("示例南京", repaired_prompt)
+            self.assertEqual(repair["messages"][1], body["messages"][1])
+            if profile == "qwen_vllm":
+                self.assertEqual(body["structured_outputs"]["grammar"], grammar([a, b, c], batch=True))
+                self.assertEqual(repair["structured_outputs"]["grammar"], grammar([b]))
+            self.assertEqual([thaw(call.arguments) for call in (a, b, c)], original)
+
+    def test_natural_limits_preserve_zero_nonempty_and_source_semantics(self):
+        for profile in ("legacy", "qwen_vllm"):
+            backend = self.backend(profile)
+            for source in (None, '正文"\\\n😀'):
+                context = InferenceContext("北京" if source is None else {source: "北京"})
+                call = normalize_call("e", "extract", ("逐字抽取。",),
+                                      {"source_field": source, "min_items": 1, "max_items": 8,
+                                       "max_length_per_item": 80}, context)
+                prompt = backend._single_payload(context, call, 160)["messages"][0]["content"]
+                self.assertIn("返回 1 至 8 项", prompt)
+                self.assertIn("非空连续原文，不得改写或拼接", prompt)
+                self.assertIn("每项建议不超过 80 字符", prompt)
+                self.assertIn("输入字符串" if source is None else "输入的顶层字段 " + encoded(source), prompt)
+                self.assertNotIn("北京", prompt)
+            context = InferenceContext([])
+            cases = [
+                ("generate", {"min_length": 0, "max_length": 0}, "允许空文本。建议长度为 0 至 0 字符"),
+                ("generate", {"min_length": 3, "max_length": 80}, "必须非空。建议长度为 3 至 80 字符"),
+                ("generate_list", {"max_items": 0, "max_length_per_item": 0}, "返回 0 至 0 项"),
+                ("generate_list", {"max_items": 8, "min_length_per_item": 3, "max_length_per_item": 80},
+                 "每项为非空字符串。每项建议长度为 3 至 80 字符"),
+            ]
+            for operation, options, expected in cases:
+                call = normalize_call("q", operation, ("说明",), options, context)
+                self.assertIn(expected, backend._single_payload(context, call, 160)["messages"][0]["content"])
+
+    def test_profiles_share_messages_and_no_fixture_data_is_sent(self):
+        context = InferenceContext({"text": "当前输入"})
+        calls = [normalize_call("a", "generate", ("标题",),
+                                {"max_length": 80, "examples": [Example(["示例输入"], "示例答案")]}, context),
+                 normalize_call("b", "generate_list", ("维度",),
+                                {"max_items": 8, "max_length_per_item": 80}, context)]
+        a, b = self.backend("legacy"), self.backend("qwen_vllm")
+        for feedback in (None, "错误"):
+            left = a._single_payload(context, calls[0], 160, feedback)
+            right = b._single_payload(context, calls[0], 160, feedback)
+            self.assertEqual(left["messages"], right["messages"])
+            self.assertEqual(set(left), {"model", "messages", "thinking", "max_tokens"})
+            self.assertEqual(set(right), {"model", "messages", "chat_template_kwargs", "max_tokens", "structured_outputs"})
+        self.assertEqual(a._batch_payload(context, calls, 400)["messages"],
+                         b._batch_payload(context, calls, 400)["messages"])

annoscript/test_qwen_backend.py

@@ -0,0 +1,181 @@
+"""Qwen wire/configuration contracts and real Runtime with offline HTTP responses."""
+
+from copy import deepcopy
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import httpx
+
+from annoscript.backends import InferenceContext, InferenceError
+from annoscript._test_evaluation_http import FixtureLLMBackend as LLMAPIBackend
+from annoscript.backends._grammar import ITEMS_GRAMMAR, batch_grammar
+from annoscript.backends._bounded_grammar import grammar
+from annoscript._test_evaluation_http import llm_content, llm_calls
+from annoscript.backends.base import normalize_call
+from annoscript.backends.config import backends_from_env
+from annoscript.client import AnnoClient
+
+
+def completion(content, finish="stop", **message):
+    return httpx.Response(200, json={"choices": [{"finish_reason": finish,
+        "message": {"content": content, **message}}]})
+
+
+class QwenBackendTests(unittest.IsolatedAsyncioTestCase):
+    async def test_profiles_thinking_operations_and_token_budgets(self):
+        context = InferenceContext({"text": "广州"}, "共同规则")
+        for profile in ("legacy", "qwen_vllm"):
+            for disabled in (True, False):
+                for operation in ("generate", "generate_list", "extract"):
+                    for budget in ((None, 7) if operation != "extract" else (None,)):
+                        with self.subTest(profile=profile, disabled=disabled, operation=operation, budget=budget):
+                            requests = []
+
+                            def handle(request):
+                                body = json.loads(request.content)
+                                requests.append(body)
+                                self.assertEqual(str(request.url), "https://llm.invalid/v1/chat/completions")
+                                self.assertEqual(body["model"], "qwen-model-name-is-not-detection")
+                                if disabled and profile == "legacy":
+                                    self.assertEqual(body["thinking"], {"type": "disabled"})
+                                else:
+                                    self.assertNotIn("thinking", body)
+                                if disabled and profile == "qwen_vllm":
+                                    self.assertEqual(body["chat_template_kwargs"], {"enable_thinking": False})
+                                else:
+                                    self.assertNotIn("chat_template_kwargs", body)
+                                if operation == "generate":
+                                    self.assertNotIn("response_format", body)
+                                    self.assertEqual("structured_outputs" in body, profile=="qwen_vllm")
+                                elif profile == "qwen_vllm":
+                                    self.assertEqual(body["structured_outputs"], {"grammar": grammar([normalize_call("q", operation, ("正文",), options, context)])})
+                                    self.assertNotIn("response_format", body)
+                                else:
+                                    self.assertEqual(body["response_format"], {"type": "json_object"})
+                                    self.assertNotIn("structured_outputs", body)
+                                if budget is None:
+                                    self.assertEqual(body["max_tokens"], 1024)
+                                else:
+                                    self.assertEqual(body["max_tokens"], budget)
+                                return completion("广州" if operation == "generate" else '{"items":["广州"]}')
+
+                            options = {"source_field": "text"} if operation == "extract" else {"max_num_tokens": budget}
+                            options.update({"max_length":512} if operation=="generate" else {"max_items":8,"max_length_per_item":58})
+                            backend = LLMAPIBackend("https://llm.invalid/v1", "fixture", "qwen-model-name-is-not-detection",
+                                profile=profile, disable_thinking=disabled, transport=httpx.MockTransport(handle))
+                            try:
+                                await backend.infer(context, normalize_call("q", operation, ("正文",), options, context))
+                                self.assertEqual(len(requests), 1)
+                            finally:
+                                await backend.aclose()
+
+    async def test_http_rejection_is_not_downgraded_or_retried(self):
+        context = InferenceContext("正文")
+        call = normalize_call("q", "generate_list", ("列举",), {'max_items': 8, 'max_length_per_item': 58}, context)
+        for status in (400, 401, 403, 408, 422, 429, 500, 503):
+            with self.subTest(status=status):
+                requests = []
+
+                def handle(request):
+                    requests.append(json.loads(request.content))
+                    return httpx.Response(status, text="private model response", headers={"Retry-After": "9"})
+
+                backend = LLMAPIBackend("https://llm.invalid", "fixture", "model", profile="qwen_vllm",
+                                        transport=httpx.MockTransport(handle))
+                try:
+                    with self.assertRaises(InferenceError) as caught:
+                        await backend.infer(context, call)
+                    self.assertEqual(caught.exception.metadata, {"http_status": status, "retry_after": "9"})
+                    self.assertNotIn("private", str(caught.exception))
+                    self.assertEqual(len(requests), 1)
+                    self.assertIn("grammar", requests[0]["structured_outputs"])
+                    self.assertNotIn("response_format", requests[0])
+                finally:
+                    await backend.aclose()
+
+    async def test_invalid_output_and_truncation_do_not_trigger_repair(self):
+        context = InferenceContext("广州")
+        call = normalize_call("q", "extract", ("地点",), {"min_items": 1, 'max_items': 8, 'max_length_per_item': 58}, context)
+        cases = [
+            ("{}", "stop", {}), ('{"items":["北京"]}', "stop", {}),
+            ('{"items":[]}', "stop", {}), ('{"items":[null]}', "stop", {}),
+            ('{"items":[""]}', "stop", {}), ('{"items":["广州"],"extra":0}', "stop", {}),
+            ('{"items":[', "stop", {}), ('{"items":["广州"]}', "length", {}),
+            (None, "stop", {}), ('{"items":["广州"]}', "stop", {"refusal": "refused"}),
+        ]
+        for content, finish, message in cases:
+            requests = []
+
+            def handle(request):
+                requests.append(request)
+                return completion(content, finish, **message)
+
+            backend = LLMAPIBackend("https://llm.invalid", "fixture", "model", profile="qwen_vllm",
+                                    transport=httpx.MockTransport(handle))
+            try:
+                with self.subTest(content=content, finish=finish), self.assertRaises(InferenceError) as caught:
+                    await backend.infer(context, call)
+                self.assertEqual(caught.exception.code, "incomplete_generation" if finish == "length" else "invalid_response")
+                self.assertEqual(len(requests), 1)
+            finally:
+                await backend.aclose()
+
+    async def test_decoded_strings_are_preserved_and_count_unicode_characters(self):
+        source = ' 中文😀 "引号" \\路径\n换行\t '
+        context = InferenceContext(source)
+        call = normalize_call("q", "extract", ("原文",), {"max_length_per_item": len(source), 'max_items': 8}, context)
+        for ascii_only in (True, False):
+            backend = LLMAPIBackend("https://llm.invalid", "fixture", "model", profile="qwen_vllm",
+                transport=httpx.MockTransport(lambda _: completion(json.dumps({"items": [source]}, ensure_ascii=ascii_only))))
+            try:
+                response = await backend.infer(context, call)
+                self.assertEqual(response.response.items, [source])
+            finally:
+                await backend.aclose()
+
+
+class ConfigurationTests(unittest.TestCase):
+    def test_invalid_configuration_fails_before_any_http(self):
+        with patch("httpx.AsyncClient", side_effect=AssertionError("HTTP must not be constructed")):
+            for profile in (None, "", "Qwen", 1, True, [], {}):
+                with self.subTest(profile=profile), self.assertRaises(ValueError):
+                    LLMAPIBackend("https://llm.invalid", "fixture", "model", profile=profile)
+            for disabled in (None, "false", 0, 1):
+                with self.subTest(disabled=disabled), self.assertRaises(ValueError):
+                    LLMAPIBackend("https://llm.invalid", "fixture", "model", disable_thinking=disabled)
+
+    def test_environment_precedence_and_public_client_forwarding(self):
+        credentials = {f"ANNO_{prefix}_{suffix}": value for prefix in ("DECISION", "GENERATE")
+                       for suffix, value in (("BASE_URL", "https://offline.invalid"),
+                                             ("AUTH_TOKEN", "fixture"), ("MODEL", "model"))}
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory, "backend.env")
+            path.write_text("ANNO_GENERATE_PROFILE=qwen_vllm\n")
+            with patch.dict(os.environ, credentials, clear=True):
+                self.assertEqual(backends_from_env()[1].profile, "legacy")
+                self.assertEqual(backends_from_env(env_file=path)[1].profile, "qwen_vllm")
+                with AnnoClient.from_env(env_file=path, generation_profile="legacy", disable_thinking=False) as client:
+                    self.assertEqual(client.runtime.llm_backend.profile, "legacy")
+                    self.assertFalse(client.runtime.llm_backend.disable_thinking)
+                with patch.dict(os.environ, {"ANNO_GENERATE_PROFILE": "legacy"}):
+                    self.assertEqual(backends_from_env(env_file=path)[1].profile, "legacy")
+                    self.assertEqual(backends_from_env(env_file=path, generation_profile="qwen_vllm")[1].profile, "qwen_vllm")
+                with patch.dict(os.environ, {"ANNO_GENERATE_PROFILE": "invalid"}):
+                    with self.assertRaises(ValueError):
+                        backends_from_env()
+                    self.assertEqual(backends_from_env(generation_profile="legacy")[1].profile, "legacy")
+                for profile in ("", 0, False):
+                    with self.assertRaises(ValueError):
+                        backends_from_env(generation_profile=profile)
+
+
+class CarRuntimeTests(unittest.TestCase):
+    pass
+
+
+if __name__ == "__main__":
+    unittest.main()

annoscript/test_qwen_pilot_regression.py

@@ -0,0 +1,180 @@
+"""Offline regressions for Qwen layout, paused dispatch and uncertain outcomes."""
+import asyncio
+import json
+import threading
+import unittest
+from unittest.mock import patch
+import httpx
+from annoscript.backends import InferenceContext, DecisionAPIBackend, BackendFatalError, InferenceError
+from annoscript._test_evaluation_http import FixtureLLMBackend as LLMAPIBackend
+from annoscript.backends.base import normalize_call
+from annoscript.server.worker import Worker, classification
+from annoscript.server.metrics import RequestMeter
+from annoscript.server.releases import Releases
+from annoscript.test_jobs_server import StoreTests, configurations, payload
+
+
+def steps(anno):
+    first = anno.generate('first', max_length=512).text
+    return anno.generate(first, max_length=512).text
+
+
+def completion(text='next'):
+    return httpx.Response(200, json={'choices':[{'finish_reason':'stop','message':{'content':text}}],
+                                   'usage':{'prompt_tokens':7,'completion_tokens':2}})
+
+
+class LayoutTests(unittest.IsolatedAsyncioTestCase):
+    async def test_single_batch_repair_and_empty_policy_keep_contents_and_budgets(self):
+        for policy in (None, '', '共同规则'):
+            context = InferenceContext({'text':'广州'}, policy)
+            calls = [normalize_call('a','extract',('地点',),{'source_field':'text', 'max_items': 8, 'max_length_per_item': 58},context),
+                     normalize_call('b','generate_list',('维度',),{'max_num_tokens':32, 'max_items': 8, 'max_length_per_item': 58},context)]
+            for mode in ('single','repair','batch'):
+                bodies = {}
+                for profile in ('legacy','qwen_vllm'):
+                    def handle(req):
+                        body=json.loads(req.content);bodies[profile]=body
+                        return completion(json.dumps({'answers':{'q1':['广州'],'q2':['空间']}} if mode=='batch' else {'items':['广州']}))
+                    backend=LLMAPIBackend('https://offline.invalid/v1','fixture','model',profile=profile,
+                                          transport=httpx.MockTransport(handle))
+                    try:
+                        if mode=='batch':await backend.infer_batch(context,calls,max_tokens=2176)
+                        else:await backend.infer(context,calls[0],**({'repair_feedback':'bad answer','max_tokens_override':128} if mode=='repair' else {}))
+                    finally:await backend.aclose()
+                legacy,qwen=bodies['legacy'],bodies['qwen_vllm']
+                self.assertEqual([m['role'] for m in qwen['messages']],['system','user'])
+                self.assertEqual(legacy['messages'],qwen['messages'])
+                self.assertIn('输入的顶层字段 "text"',qwen['messages'][0]['content'])
+                if policy:
+                    self.assertTrue(qwen['messages'][0]['content'].startswith(policy+'\n'))
+                from annoscript._test_evaluation_http import llm_calls
+                a,b=llm_calls(legacy),llm_calls(qwen)
+                self.assertEqual(a[0],b[0])
+                self.assertEqual([(c.state,x.operation,x.arguments) for c,x in a[1]],
+                                 [(c.state,x.operation,x.arguments) for c,x in b[1]])
+                self.assertEqual(qwen.get('max_tokens'),legacy.get('max_tokens'))
+                self.assertEqual(len(legacy['messages']),2)
+
+
+class ProtocolTests(StoreTests):
+    # Inherited store tests remain a separate baseline; pytest collection here
+    # also exercises them against the new policy defaults.
+    def config(self, **policy):
+        self.releases=Releases(configurations(**policy),'test')
+
+    def run_http(self, handler, *, observer=None):
+        def factory(_):
+            return (DecisionAPIBackend('https://offline.invalid','fixture','model',transport=httpx.MockTransport(handler)),
+                    LLMAPIBackend('https://offline.invalid/v1','fixture','model',profile='qwen_vllm',transport=httpx.MockTransport(handler)))
+        return Worker(self.store,self.releases,backend_factory=factory,concurrency=4,request_observer=observer)
+
+    def test_protocol_auth_context_and_retry_classification(self):
+        for code,metadata,expected in [
+            ('backend_protocol_error',{'http_status':400},'pause'),
+            ('authentication_failed',{'http_status':401},'pause'),
+            ('model_http_error',{'http_status':400},'result'),
+            ('model_timeout',{},'pause'),('transport_error',{},'pause'),
+            ('backend_unavailable',{},'retry'),
+            ('model_http_error',{'http_status':429},'retry'),
+            ('model_http_error',{'http_status':503},'retry')]:
+            self.assertEqual(classification(code,metadata,retry_unknown_outcomes=False),expected)
+        self.assertEqual(classification('model_timeout',{}),'retry')
+
+    def test_four_inflight_workers_drain_after_protocol_pause_without_new_dispatch(self):
+        self.config(retry_unknown_outcomes=False)
+        job=self.store.submit('alice','protocol',payload(tuple(str(i) for i in range(8)),program=steps),self.releases)
+        claims=[self.claim() for _ in range(4)];audit=[];calls=[]
+        async def run():
+            started=asyncio.Event();release=asyncio.Event()
+            async def handler(req):
+                body=json.loads(req.content);state=body['messages'][-1]['content'];calls.append(state)
+                if len(calls)==4:started.set()
+                await started.wait()
+                if state=='0':return httpx.Response(400,json={'error':{'message':'System message must be at the beginning.'}})
+                await release.wait()
+                return completion()
+            worker=self.run_http(handler,observer=audit.append)
+            tasks=[asyncio.create_task(worker.execute(c)) for c in claims]
+            async with asyncio.timeout(10):
+                while self.store.status('alice',job['job_id'])['state']!='paused':await asyncio.sleep(.005)
+                self.assertIsNone(self.claim())
+                self.assertTrue(all(self.store.renew(c) for c in claims[1:]))
+                self.assertTrue(all(not self.store.can_dispatch(c) for c in claims[1:]))
+                release.set();await asyncio.gather(*tasks)
+        asyncio.run(run())
+        self.assertEqual(len(calls),4)
+        status=self.store.status('alice',job['job_id'])
+        self.assertEqual(status['pause_reason']['code'],'backend_protocol_error')
+        self.assertEqual(status['counts']['completed'],0)
+        with self.store._tx() as db:
+            rows=db.execute('SELECT http_status,seconds,usage FROM http_requests').fetchall()
+        self.assertEqual(sorted(r['http_status'] for r in rows),[200,200,200,400])
+        self.assertTrue(all(r['seconds'] is not None for r in rows))
+        self.assertEqual(len([x for x in audit if x['event']=='outcome']),4)
+        self.assertTrue(all(x['storage_committed'] for x in audit if x['event']=='outcome'))
+
+    def test_context_error_is_local_and_unknown_timeout_pauses_once(self):
+        for name,unknown in [('context',False),('timeout',True)]:
+            self.config(retry_unknown_outcomes=False)
+            job=self.submit((name,),key=name);calls=[]
+            async def handler(req):
+                calls.append(req)
+                if unknown:raise httpx.ReadTimeout('fixture')
+                return httpx.Response(400,json={'error':{'message':'maximum context length is 8192 tokens'}})
+            asyncio.run(self.run_http(handler).execute(self.claim()))
+            self.assertEqual(len(calls),1)
+            status=self.store.status('alice',job['job_id'])
+            self.assertEqual(status['state'],'paused' if unknown else 'completed')
+            if unknown:self.assertIsNone(self.claim())
+            else:self.assertEqual(self.store.results('alice',job['job_id'])['results'][0]['error']['code'],'model_http_error')
+
+    def test_pause_gate_at_meter_boundary_preserves_existing_response(self):
+        job=self.submit(('one','two'));a,b=self.claim(),self.claim()
+        self.store.request_started(b,'inflight','generation',1)
+        self.store.attempt_failed(a,{'code':'backend_protocol_error','message':'fixture'},pause=True)
+        self.assertFalse(self.store.request_started(b,'blocked','generation',1))
+        self.assertTrue(self.store.request_finished(b,'inflight',seconds=1,http_status=200,usage={'prompt_tokens':7}))
+        async def run():
+            with self.assertRaises(BackendFatalError) as e:await RequestMeter(self.store,b,'generation').start(1)
+            self.assertEqual(e.exception.code,'job_paused')
+        asyncio.run(run())
+
+    def test_observer_failure_never_retries_model(self):
+        for phase,expected in [('intent',0),('outcome',1)]:
+            self.config(retry_unknown_outcomes=False)
+            job=self.submit((phase,),key=phase);calls=[]
+            def observe(event):
+                if event['event']==phase:raise OSError('fixture storage unavailable')
+            def handler(req):calls.append(req);return completion()
+            asyncio.run(self.run_http(handler,observer=observe).execute(self.claim()))
+            self.assertEqual(len(calls),expected)
+            self.assertEqual(self.store.status('alice',job['job_id'])['pause_reason']['code'],'metrics_failed')
+            self.assertIsNone(self.claim())
+
+    def test_cancel_during_commit_still_audits_observed_outcome_once(self):
+        self.submit(('a',));claim=self.claim();audit=[];meter=RequestMeter(self.store,claim,'generation',observer=audit.append)
+        entered=threading.Event();release=threading.Event();original=self.store.request_finished
+        def finish(*args,**kwargs):entered.set();release.wait(5);return original(*args,**kwargs)
+        async def run():
+            identity=await meter.start(1)
+            with patch.object(self.store,'request_finished',side_effect=finish):
+                task=asyncio.create_task(meter.finish(identity,.1,200,{'prompt_tokens':5,'completion_tokens':1}))
+                async with asyncio.timeout(5):
+                    while not entered.is_set():await asyncio.sleep(.005)
+                task.cancel();release.set()
+                with self.assertRaises(asyncio.CancelledError):await task
+        asyncio.run(run())
+        outcomes=[e for e in audit if e['event']=='outcome']
+        self.assertEqual(len(outcomes),1);self.assertTrue(outcomes[0]['storage_committed'])
+        self.assertEqual(outcomes[0]['usage']['prompt_tokens'],5)
+        self.assertIsNotNone(outcomes[0]['finished_unix'])
+
+    def test_policy_boolean_digest_default_and_abandoned_attempt(self):
+        default=self.releases.default
+        self.config(retry_unknown_outcomes=True);self.assertEqual(default,self.releases.default)
+        self.config(retry_unknown_outcomes=False);self.assertNotEqual(default,self.releases.default)
+        for invalid in (0,1,'false',None):
+            with self.assertRaises(ValueError):Releases(configurations(retry_unknown_outcomes=invalid),'test')
+        job=self.submit(('a',));self.claim();self.clock.now+=61
+        self.assertIsNone(self.claim());self.assertEqual(self.store.status('alice',job['job_id'])['state'],'paused')

annoscript/test_release_client.py

@@ -0,0 +1,20 @@
+import hashlib
+import unittest
+
+import httpx
+
+from annoscript.job_protocol import encode
+from annoscript.jobs import AnnoJobClient, RemoteProtocolError
+
+
+class ReleaseTests(unittest.TestCase):
+    def test_digest_available_and_unavailable(self):
+        manifest = {"format": "lev-release", "version": 1, "policy": {"max_inputs": 500}}
+        rid = hashlib.sha256(encode(manifest, canonical=True)).hexdigest()
+        for available in (False, True):
+            payload = {"protocol_version": 1, "release_id": rid, "manifest": manifest, "available": available}
+            with AnnoJobClient("http://offline", transport=httpx.MockTransport(lambda r: httpx.Response(200, json=payload))) as client:
+                self.assertEqual(client.release(rid), payload)
+                payload["manifest"] = {"changed": True}
+                with self.assertRaises(RemoteProtocolError):
+                    client.release(rid)