annoscript 0.1.0__py3-none-any.whl

annoscript/_test_evaluation_http.py

@@ -0,0 +1,117 @@
+"""Deterministic model HTTP responses; never connects to a model service."""
+
+from dataclasses import asdict
+from collections import OrderedDict
+import hashlib
+import json
+
+import httpx
+
+from annoscript._test_synthetic import CannedBackend
+from annoscript.backends import DecisionAPIBackend, LLMAPIBackend as _LLMAPIBackend
+from annoscript.anno import GenerateResponse
+
+
+_CAPTURES = OrderedDict()
+
+
+def _key(body):
+    return hashlib.sha256(json.dumps(body, ensure_ascii=False, allow_nan=False,
+                                    sort_keys=True, separators=(",", ":")).encode()).digest()
+
+
+class FixtureLLMBackend(_LLMAPIBackend):
+    """Test-only capture before transport; never add fixture metadata to the wire.
+
+    Fixed answers use the original immutable calls, not a parser for prose. Tests
+    separately assert messages, grammars and budgets on the actual HTTP requests.
+    The bounded registry keeps captures available after a runtime closes its backend.
+    """
+
+    @staticmethod
+    def _capture(payload, context, calls, batch):
+        key = _key(payload)
+        _CAPTURES[key] = (batch, tuple((context, call) for call in calls))
+        _CAPTURES.move_to_end(key)
+        if len(_CAPTURES) > 4096:
+            _CAPTURES.popitem(last=False)
+        return payload
+
+    def _single_payload(self, context, call, max_tokens, repair_feedback=None):
+        payload = super()._single_payload(context, call, max_tokens, repair_feedback)
+        return self._capture(payload, context, [call], False)
+
+    def _batch_payload(self, context, calls, max_tokens):
+        payload = super()._batch_payload(context, calls, max_tokens)
+        return self._capture(payload, context, calls, True)
+
+
+class FixtureDecisionBackend(DecisionAPIBackend):
+    """Keep original call identity and example order outside the model request."""
+
+    def _batch_payload(self, context, calls):
+        payload = super()._batch_payload(context, calls)
+        return FixtureLLMBackend._capture(payload, context, calls, True)
+
+
+def decision_calls(body):
+    return list(_CAPTURES[_key(body)][1])
+
+
+def llm_calls(body):
+    """Look up test-side calls by the exact request, without parsing model prose."""
+    batch, calls = _CAPTURES[_key(body)]
+    return batch, list(calls)
+
+
+def llm_content(body, answer):
+    batched, calls = llm_calls(body)
+    values = [asdict(answer(context, call))["text" if call.operation == "generate" else "items"]
+              for context, call in calls]
+    if batched:
+        return json.dumps({"answers": {call.arguments.get("field_name", f"q{index}"): value
+                                      for index, ((_, call), value) in enumerate(zip(calls, values), 1)}},
+                          ensure_ascii=False, separators=(",", ":"))
+    call = calls[0][1]
+    if call.operation == "generate":
+        return json.dumps(values[0], ensure_ascii=False) if call.arguments.get("nullable", False) else values[0]
+    return json.dumps({"items": values[0]}, ensure_ascii=False)
+
+
+def llm_wire_id(body, call_id):
+    """Test-side alias lookup, independent of the production mapping helper."""
+    batched, calls = llm_calls(body)
+    assert batched
+    return next(call.arguments.get("field_name", f"q{index}")
+                for index, (_, call) in enumerate(calls, 1) if call.id == call_id)
+
+
+def http_backends(scenarios=(), *, requests=None, request_bytes=None, override=None, profile="qwen_vllm", speculative=False):
+    canned = CannedBackend(scenarios, speculative=speculative)
+    requests = requests if requests is not None else []
+
+    def handle(request):
+        body = json.loads(request.content)
+        requests.append(body)
+        if request_bytes is not None:
+            request_bytes.append(len(request.content))
+        if override is not None:
+            response = override(body)
+            if response is not None:
+                return response
+        if "questions" in body:
+            answers = {}
+            for context, call in decision_calls(body):
+                answer = canned.answer(context, call)
+                identity = call.arguments.get("field_name", call.id)
+                answers[identity] = {"type": call.operation, **asdict(answer)}
+            return httpx.Response(200, json={"answers": answers, "usage": {"input_tokens": 11, "output_tokens": 3}})
+        content = llm_content(body, lambda context, call: GenerateResponse(context.state)
+                              if type(context.state) is str else canned.answer(context, call))
+        return httpx.Response(200, json={"choices": [{"finish_reason": "stop", "message": {"content": content}}],
+                                      "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9,
+                                                "completion_tokens_details": {"reasoning_tokens": 1}}})
+
+    transport = httpx.MockTransport(handle)
+    return (FixtureDecisionBackend("http://offline.invalid", "fixture", "fixture", transport=transport),
+            FixtureLLMBackend("http://offline.invalid", "fixture", "fixture", profile=profile, transport=transport))

annoscript/_test_ir_runtime.py

@@ -0,0 +1,138 @@
+"""Test-only interpreter for serialized IR; validates, never repairs constraints.
+
+Deliberately consumes the JSON representation, independently of compiler classes.
+It is not a supported execution API or a production solver.
+"""
+
+import json
+import operator
+from copy import deepcopy
+
+
+class _Returned(Exception):
+    def __init__(self, value):
+        self.value = value
+
+
+_COMPARE = {
+    "eq": operator.eq, "ne": operator.ne, "lt": operator.lt, "le": operator.le,
+    "gt": operator.gt, "ge": operator.ge, "in": lambda a, b: a in b,
+    "not_in": lambda a, b: a not in b, "is": operator.is_, "is_not": operator.is_not,
+}
+
+
+def evaluate(program, anno):
+    """Evaluate JSON IR against a supplied deterministic fake AnnoScript."""
+    env = {}
+
+    def expr(node):
+        kind = node["kind"]
+        if kind == "literal":
+            return node["value"]
+        if kind == "var":
+            return env[node["name"]]
+        if kind == "state":
+            return anno.state
+        if kind == "get_item":
+            return expr(node["value"])[expr(node["index"])]
+        if kind == "field":
+            return getattr(expr(node["value"]), node["name"])
+        if kind in ("list", "tuple"):
+            items = [expr(item) for item in node["items"]]
+            return items if kind == "list" else tuple(items)
+        if kind == "dict":
+            return {expr(key): expr(value) for key, value in node["entries"]}
+        if kind == "unary":
+            return {"not": operator.not_, "positive": operator.pos, "negative": operator.neg}[node["op"]](expr(node["value"]))
+        if kind == "binary":
+            assert node["op"] == "add"
+            return expr(node["left"]) + expr(node["right"])
+        if kind == "compare":
+            left = expr(node["left"])
+            for op, right_node in zip(node["ops"], node["comparators"]):
+                right = expr(right_node)
+                if not _COMPARE[op](left, right):
+                    return False
+                left = right
+            return True
+        if kind == "bool_op":
+            value = expr(node["values"][0])
+            for item in node["values"][1:]:
+                if (node["op"] == "and" and not value) or (node["op"] == "or" and value):
+                    break
+                value = expr(item)
+            return value
+        if kind == "intrinsic":
+            functions = {"len": len, "int": int, "range": range, "enumerate": enumerate,
+                         "str.strip": str.strip, "json.dumps": json.dumps,
+                         "example": lambda state, answer: deepcopy({"state": state, "answer": answer})}
+            args = [expr(arg) for arg in node["args"]]
+            kwargs = {key: expr(value) for key, value in node["kwargs"]}
+            return functions[node["name"]](*args, **kwargs)
+        if kind == "format_value":
+            value = expr(node["value"])
+            spec = expr(node["spec"]) if node["spec"] is not None else ""
+            conversion = node["conversion"]
+            if conversion != "none":
+                value = {"s": str, "r": repr, "a": ascii}[conversion](value)
+            return format(value, spec)
+        if kind == "format_string":
+            return "".join(expr(part) for part in node["parts"])
+        raise AssertionError(f"Unknown expression {kind}")
+
+    def target(node, value):
+        if node["kind"] == "name_target":
+            env[node["name"]] = value
+        else:
+            values = list(value)
+            if len(values) != len(node["items"]):
+                raise ValueError("Unpack length mismatch")
+            for item, v in zip(node["items"], values):
+                target(item, v)
+
+    def block(body):
+        for node in body:
+            kind = node["kind"]
+            if kind == "assign":
+                env[node["target"]] = expr(node["value"])
+            elif kind == "unpack":
+                target(node["target"], expr(node["value"]))
+            elif kind == "set_item":
+                value, index, item = expr(node["value"]), expr(node["index"]), expr(node["item"])
+                value[index] = item
+            elif kind == "if":
+                block(node["then_body"] if expr(node["condition"]) else node["else_body"])
+            elif kind == "for":
+                for value in expr(node["iterable"]):
+                    target(node["target"], value)
+                    block(node["body"])
+            elif kind == "infer":
+                args = [expr(arg) for arg in node["args"]]
+                kwargs = {key: expr(value) for key, value in node["kwargs"]}
+                env[node["target"]] = getattr(anno, node["operation"])(*args, **kwargs)
+            elif kind == "constrain":
+                valid = expr(node["condition"])
+                message = expr(node["message"]) if node["message"] is not None else None
+                if not valid:
+                    raise ValueError(message or "Constraint failed")
+            elif kind == "return":
+                raise _Returned(expr(node["value"]))
+            else:
+                raise AssertionError(f"Unknown statement {kind}")
+
+    try:
+        block(program["body"])
+    except _Returned as result:
+        return result.value
+    raise AssertionError("Program ended without return")
+
+
+def walk_json(value):
+    """Traverse a decoded tree for structural acceptance checks."""
+    if isinstance(value, dict):
+        yield value
+        for item in value.values():
+            yield from walk_json(item)
+    elif isinstance(value, list):
+        for item in value:
+            yield from walk_json(item)

annoscript/_test_job_process.py

@@ -0,0 +1,166 @@
+"""Loopback-only process fixtures for durable-job acceptance, never a deployment."""
+
+import argparse
+import json
+from pathlib import Path
+
+
+def server_components(folder):
+    from annoscript.server.releases import Releases
+    from annoscript.server.store import Store
+    from annoscript.test_jobs_server import configurations
+    clock = lambda: float(Path(folder, "clock").read_text())
+    return Store(Path(folder, "jobs.sqlite"), clock=clock, lease_seconds=0.4), Releases(configurations(), "test")
+
+
+def serve_api(folder, port):
+    import asyncio
+    import socket
+    import uvicorn
+    from annoscript.server.api import create_app
+    store, releases = server_components(folder)
+    sock = socket.socket()
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    sock.bind(("127.0.0.1", port))
+    sock.listen(128)
+    server = uvicorn.Server(uvicorn.Config(create_app(store, releases, {"offline-token": "acceptance"}),
+                                           log_level="error", access_log=False))
+    async def run():
+        task = asyncio.create_task(server.serve(sockets=[sock]))
+        while not server.started:
+            if task.done():
+                await task
+                raise RuntimeError("test API did not start")
+            await asyncio.sleep(0.01)
+        print(json.dumps({"url": f"http://127.0.0.1:{sock.getsockname()[1]}"}), flush=True)
+        await task
+    asyncio.run(run())
+
+
+def serve_worker(folder):
+    import asyncio
+    from annoscript._data import thaw
+    from annoscript._test_remote_server import cases
+    from annoscript._test_synthetic import CannedBackend
+    from annoscript.backends import InferenceOutput
+    from annoscript.anno import GenerateResponse
+    from annoscript.server.worker import Worker
+    store, releases = server_components(folder)
+    scenarios, _, _, failures = cases()
+    class Backend(CannedBackend):
+        async def infer(self, context, call):
+            state = thaw(context.state)
+            if type(state) is str:
+                if state == "__hold__":
+                    Path(folder, "model-started").touch()
+                    while not Path(folder, "release-model").exists():
+                        await asyncio.sleep(0.01)
+                return InferenceOutput(GenerateResponse(state))
+            return await super().infer(context, call)
+    def factory(manifest):
+        backend = Backend(scenarios, failures, speculative=True)
+        return backend, backend
+    print(json.dumps({"worker": "ready"}), flush=True)
+    asyncio.run(Worker(store, releases, backend_factory=factory, random_source=lambda: 0,
+                       poll_interval=0.01).run())
+
+
+def fixtures(folder):
+    from annoscript._test_remote_server import cases
+    _, inputs, expected, _ = cases()
+    Path(folder, "fixtures.json").write_text(json.dumps({"inputs": inputs[:8], "expected": expected[:8]}, ensure_ascii=False), encoding="utf-8")
+
+
+def run_client(url, folder, action):
+    # Fail fast on accidental import coupling in a real Python 3.10 process.
+    import importlib.abc
+    import sys
+    class Block(importlib.abc.MetaPathFinder):
+        def find_spec(self, fullname, *args):
+            if fullname.startswith(("annoscript.runtime", "annoscript.backends", "annoscript.plan", "annoscript.server", "fastapi", "uvicorn")):
+                raise AssertionError("execution module imported on client: " + fullname)
+    sys.meta_path.insert(0, Block())
+    import httpx
+    from annoscript_examples.request import annotate
+    from annoscript.client import AnnoClient
+    from annoscript.compiler import compile_program
+    from annoscript.jobs import AnnoJobClient
+    from annoscript.test_jobs import generate
+    if action == "matrix":
+        data = json.loads(Path(folder, "fixtures.json").read_text())
+        class DropResponses(httpx.BaseTransport):
+            def __init__(self):
+                self.inner = httpx.HTTPTransport(retries=0)
+                self.dropped = set()
+            def handle_request(self, request):
+                response = self.inner.handle_request(request)
+                kind = "accept" if request.url.path == "/v1/jobs" else "page" if request.url.path.endswith("/results") else None
+                if kind and kind not in self.dropped:
+                    self.dropped.add(kind)
+                    response.read()
+                    response.close()
+                    raise httpx.ReadError("deliberately lost committed HTTP response")
+                return response
+            def close(self):
+                self.inner.close()
+        for mode in ("strict", "eager", "speculative"):
+            for optimize in (False, True):
+                for ordered in (False, True):
+                    with AnnoClient.remote(url, "offline-token", mode=mode, optimize=optimize,
+                                          transport=DropResponses()) as client:
+                        client._remote.session.client.sleep = lambda _: None
+                        results = list(client.submit(annotate, data["inputs"], ordered=ordered))
+                        assert len(results) == len(data["expected"])
+                        if ordered:
+                            assert [r.index for r in results] == list(range(len(results)))
+                        by_index = {r.index: r for r in results}
+                        assert len(by_index) == len(results)
+                        for index, expected in enumerate(data["expected"]):
+                            result = by_index[index]
+                            assert result.status == expected["status"], (index, result)
+                            if result.status == "success":
+                                assert result.value == expected["value"], (index, result)
+                            else:
+                                assert result.error.code == expected["code"], (index, result)
+                        assert by_index[2].value["tags"] == ["sample"]
+        print(json.dumps({"matrix": "12 combinations passed"}), flush=True)
+    elif action == "hold-session":
+        client = AnnoClient.remote(url, "offline-token")
+        stream = client.submit(generate, ["__hold__"])
+        print(json.dumps({"job_id": stream.job["job_id"]}), flush=True)
+        sys.stdin.readline()  # Parent kills the process; no destructor can run.
+        client.close()
+    elif action == "submit-durable":
+        with AnnoJobClient(url, "offline-token") as client:
+            job = client.submit_ir(compile_program(generate), [{"index": 42, "state": "__hold__"}], idempotency_key="durable")
+        print(json.dumps(job), flush=True)
+    elif action == "read-durable":
+        with AnnoJobClient(url, "offline-token") as client:
+            job = client.lookup("durable")
+            page = client.fetch_results(job)
+            assert page["done"] and page["results"][0]["index"] == 42
+            assert page["results"][0]["value"] == "__hold__"
+            assert page["results"][0]["attempts"] == 2
+        print(json.dumps({"recovery": "passed"}), flush=True)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("kind", choices=("api", "worker", "fixtures", "client"))
+    parser.add_argument("folder")
+    parser.add_argument("--url")
+    parser.add_argument("--port", type=int, default=0)
+    parser.add_argument("--action", default="matrix")
+    args = parser.parse_args()
+    if args.kind == "api":
+        serve_api(args.folder, args.port)
+    elif args.kind == "worker":
+        serve_worker(args.folder)
+    elif args.kind == "fixtures":
+        fixtures(args.folder)
+    else:
+        run_client(args.url, args.folder, args.action)
+
+
+if __name__ == "__main__":
+    main()