annoscript 0.1.0__py3-none-any.whl

annoscript/test_runtime_client.py

@@ -0,0 +1,133 @@
+"""The concrete synchronous client and asynchronous runtime share ownership safely."""
+
+import asyncio
+import unittest
+from unittest.mock import patch
+from threading import Event
+
+from annoscript.client import AnnoClient, precondition, require
+from annoscript.runtime import AnnoRuntime
+from annoscript.backends import InferenceOutput, BackendFatalError
+from annoscript.anno import GenerateResponse
+from annoscript.test_runtime import FakeBackend, mixed
+
+
+def echo(anno):
+    return anno.generate(anno.state, max_length=512).text
+
+
+def constant(anno):
+    return [None, "中文"]
+
+
+def check(state):
+    require(state != "invalid", message="invalid record")
+
+
+@precondition(check)
+def checked(anno):
+    return anno.generate(anno.state, max_length=512).text
+
+
+class ClientTests(unittest.TestCase):
+    def client(self, backend=None, **options):
+        backend = backend or FakeBackend()
+        return AnnoClient(AnnoRuntime(backend, backend, **options))
+
+    def test_results_preconditions_order_indices_and_repeated_submissions(self):
+        with self.client() as client:
+            for ordered in (False, True):
+                with client.submit(checked, ["a", "invalid", "b"], ordered=ordered) as stream:
+                    result = list(stream)
+                by_index = {r.index: r for r in result}
+                self.assertEqual(by_index[0].value, "a")
+                self.assertEqual(by_index[1].error.code, "invalid_input")
+                self.assertEqual(by_index[2].value, "b")
+                if ordered:
+                    self.assertEqual([r.index for r in result], [0, 1, 2])
+            self.assertEqual(list(client.submit(constant, [])), [])
+            self.assertEqual(list(client.submit(constant, ["x"]))[0].value, [None, "中文"])
+            self.assertEqual(list(client.submit(echo, [float("nan")]))[0].error.code, "invalid_input")
+        self.assertFalse(client._thread.is_alive())
+        self.assertTrue(client.runtime.closed)
+        client.close()
+
+    def test_checker_defect_prevents_all_model_calls(self):
+        seen = []
+        def broken(state):
+            seen.append(state)
+            if state == "bad":
+                raise RuntimeError("checker defect")
+        with self.client() as client, patch("annoscript._client_common._PRECONDITIONS", {echo: (broken,)}):
+            with self.assertRaisesRegex(RuntimeError, "checker defect"):
+                client.submit(echo, ["first", "bad"])
+            self.assertIsNone(client._thread)
+            self.assertEqual(client.runtime.llm_backend.calls, [])
+        self.assertEqual(seen, ["first", "bad"])
+
+    def test_per_input_failure_isolated_and_fatal_error_closes_stream(self):
+        with self.client(FakeBackend(fail=("bad",))) as client:
+            result = list(client.submit(echo, ["bad", "ok"], ordered=True))
+            self.assertEqual([r.status for r in result], ["error", "success"])
+        class Fatal(FakeBackend):
+            async def infer(self, context, call):
+                raise BackendFatalError("auth failed")
+        with self.client(Fatal()) as client:
+            stream = client.submit(echo, ["a", "b"])
+            with self.assertRaisesRegex(BackendFatalError, "auth failed"):
+                list(stream)
+            self.assertTrue(stream.closed)
+
+    def test_close_stops_queued_work_without_waiting_for_model(self):
+        class Controlled(FakeBackend):
+            async def infer(self, context, call):
+                self.count += 1
+                self.started.set()
+                try:
+                    await asyncio.Event().wait()
+                finally:
+                    self.cancelled.set()
+        backend = Controlled()
+        backend.started, backend.cancelled, backend.count = Event(), Event(), 0
+        with self.client(backend, max_llm_inflight=1) as client:
+            stream = client.submit(echo, ["a", "b", "c"])
+            self.assertTrue(backend.started.wait(2))
+            stream.close()
+            self.assertTrue(backend.cancelled.wait(2))
+            self.assertEqual(backend.count, 1)
+            self.assertFalse(client.runtime.closed)
+            self.assertEqual(list(stream), [])
+
+    def test_client_does_not_execute_program_function(self):
+        with self.client() as client, patch("annoscript.anno.Anno.__init__", side_effect=AssertionError("must not construct AnnoScript")):
+            result = list(client.submit(mixed, ["state"]))
+            self.assertEqual(result[0].value, [0.8, "a", 0])
+
+    def test_runtime_has_one_client_owner(self):
+        with self.client() as client:
+            with self.assertRaisesRegex(ValueError, "unstarted"):
+                AnnoClient(client.runtime)
+
+    def test_results_are_delivered_out_of_order_then_can_be_ordered(self):
+        class Ordered(FakeBackend):
+            async def infer(self, context, call):
+                if call.arguments["instructions"] == "slow":
+                    if self.release is None:
+                        self.release = asyncio.Event()
+                    await self.release.wait()
+                    await asyncio.sleep(0)
+                else:
+                    if self.release is None:
+                        self.release = asyncio.Event()
+                    self.release.set()
+                return InferenceOutput(GenerateResponse(call.arguments["instructions"]))
+        for ordered in (False, True):
+            backend = Ordered()
+            backend.release = None
+            with self.client(backend) as client:
+                result = list(client.submit(echo, ["slow", "fast"], ordered=ordered))
+                self.assertEqual([r.index for r in result], [0, 1] if ordered else [1, 0])
+
+
+if __name__ == "__main__":
+    unittest.main()

annoscript/test_source_fields.py

@@ -0,0 +1,111 @@
+"""Field-bound extraction and simplified car requests; no live services."""
+import json
+import unittest
+from dataclasses import replace
+
+import httpx
+
+
+from annoscript.backends import InferenceContext, InferenceError
+from annoscript._test_evaluation_http import FixtureLLMBackend as LLMAPIBackend
+from annoscript.backends.base import normalize_call, validate_response
+from annoscript.backends._bounded_grammar import grammar
+from annoscript.backends._generation_policy import effective_budget
+from annoscript.client import AnnoClient
+from annoscript.compiler import compile_program, CompileError
+from annoscript.anno import ExtractResponse
+from annoscript.plan_lowering import lower_to_plan
+from annoscript.runtime import AnnoRuntime, RunInput
+from annoscript._test_evaluation_http import llm_content
+from annoscript.test_generation_policy import policy
+from annoscript.test_llm_batch import completion
+
+
+def extract(context, **options):
+    return normalize_call("e", "extract", ("抽取原文",),
+                          {"max_items": 2, "max_length_per_item": 300, **options}, context)
+
+
+def obsolete_source(anno):
+    return anno.extract("原文", source="text", max_items=1, max_length_per_item=20).items
+
+
+class SourceFieldTests(unittest.TestCase):
+    def test_obsolete_api_fails_before_configuration_or_execution(self):
+        with self.assertRaisesRegex(TypeError, "source_layout was removed"):
+            AnnoClient.from_env(source_layout="inline")
+        for optimize in (False, True):
+            with self.assertRaisesRegex(CompileError, "source"):
+                compile_program(obsolete_source, optimize=optimize)
+
+    def test_resolution_snapshot_literal_keys_and_invalid_combinations(self):
+        data = {"text": "甲乙", "other": "丙丁", "nested": {"text": "错误"}}
+        context = InferenceContext(data)
+        call = extract(context, source_field="other", max_num_tokens=100)
+        data["other"] = "已改变"
+        self.assertEqual(call.extraction_source, "丙丁")
+        self.assertNotIn("source", call.arguments)
+        self.assertEqual(effective_budget(call), 100)
+        self.assertEqual(validate_response(call, ExtractResponse(["丙"])).items, ["丙"])
+        with self.assertRaises(InferenceError):
+            validate_response(call, ExtractResponse(["甲"]))
+        self.assertEqual(extract(InferenceContext("甲乙")).extraction_source, "甲乙")
+        self.assertEqual(extract(InferenceContext({"a.b": "literal"}), source_field="a.b").extraction_source, "literal")
+        for state, options in [
+            (data, {}), (data, {"source_field": "missing"}), (data, {"source_field": "nested"}),
+            (data, {"source_field": "nested.text"}), ({"text": 1}, {"source_field": "text"}),
+            ("甲乙", {"source_field": "text"}), ([], {}), (data, {"source_field": True}),
+            (data, {"source": "甲乙"}),
+        ]:
+            with self.subTest(state=state, options=options), self.assertRaises(ValueError):
+                extract(InferenceContext(state), **options)
+        for budget in (0, -1, True, 1.5):
+            with self.assertRaises(ValueError):
+                extract(context, source_field="text", max_num_tokens=budget)
+
+    def test_source_isolation_in_grammar_and_prepared_cache(self):
+        backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture",
+                                profile="qwen_vllm", extract_mode="substring_chars")
+        a, b = InferenceContext({"text": "甲乙"}), InferenceContext({"text": "丙丁"})
+        ca, cb = extract(a, source_field="text"), extract(b, source_field="text")
+        self.assertNotEqual(grammar([ca], extract_mode="substring_chars"),
+                            grammar([cb], extract_mode="substring_chars"))
+        pa, pb = backend._prepare(a, [ca], 100), backend._prepare(b, [cb], 100)
+        self.assertNotEqual(pa.binding, pb.binding)
+        self.assertNotEqual(pa.body, pb.body)
+        with self.assertRaisesRegex(ValueError, "binding differs"):
+            backend._prepare(b, [ca], 100)
+        with self.assertRaises(ValueError):
+            effective_budget(replace(ca, extraction_source=None))
+
+
+class CarWireTests(unittest.IsolatedAsyncioTestCase):
+
+
+    async def test_explicit_extract_budget_avoids_9760_but_long_input_fails_locally(self):
+        context = InferenceContext({"text": "原文"})
+        call = normalize_call("e", "extract", ("抽取",),
+                              {"source_field": "text", "max_items": 16, "max_length_per_item": 300,
+                               "max_num_tokens": 1440}, context)
+        self.assertEqual(effective_budget(call), 1440)
+        for input_tokens, expected_count in ((6752, 1), (6753, 0)):
+            sent = []
+
+            def handle(request):
+                sent.append(request)
+                return completion('{"items":["原文"]}')
+
+            backend = LLMAPIBackend("http://offline.invalid", "fixture", "fixture",
+                                    profile="qwen_vllm", request_policy=policy(),
+                                    _token_counter=lambda p: input_tokens,
+                                    transport=httpx.MockTransport(handle))
+            try:
+                if expected_count:
+                    await backend.infer(context, call)
+                else:
+                    with self.assertRaises(InferenceError) as caught:
+                        await backend.infer(context, call)
+                    self.assertEqual(caught.exception.code, "context_length_exceeded")
+                self.assertEqual(len(sent), expected_count)
+            finally:
+                await backend.aclose()