perd-worker 0.1.0__py3-none-any.whl

perd_worker/__init__.py

@@ -0,0 +1,32 @@
+r"""perd_worker — the SDK workflow developers build against.
+
+Authoring surface:
+
+.. code-block:: python
+
+    from perd_worker import workflow, WorkflowStreamInput, WorkflowStreamOutput
+
+Build time: :func:`perd_worker.compiler.compile_contract` projects the
+registered signatures into the workflow's ``FileDescriptorSet``; its SHA-256
+is the contract identity every other component pins.
+
+Runtime: :class:`perd_worker.runtime.WorkerServicer` serves the uniform
+``perd.v1.WorkerService`` over gRPC.
+"""
+
+from .registry import (
+    WorkflowDefinitionError,
+    WorkflowStreamInput,
+    WorkflowStreamOutput,
+    workflow,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "workflow",
+    "WorkflowStreamInput",
+    "WorkflowStreamOutput",
+    "WorkflowDefinitionError",
+    "__version__",
+]

perd_worker/compiler.py

@@ -0,0 +1,178 @@
+r"""Build-time projection: registered signatures -> protobuf contract.
+
+Runs ONCE, at image build. The output — a serialized ``FileDescriptorSet`` —
+is the workflow's entire public identity:
+
+.. math::
+
+    \mathrm{contract\_hash} = \operatorname{SHA256}\!\left(D\right)
+
+where:
+
+- :math:`D`: the ``FileDescriptorSet`` bytes embedded in the worker image and returned verbatim by ``WorkerService.Describe``
+- :math:`\mathrm{contract\_hash}`: pinned by the store at publish, by the client at resolve, and re-verified by the orchestrator at registration — one value, three enforcement points
+
+Never runs at request time. v1 downloaded protos from GCS and shelled out to
+``protoc`` on first use of every workflow, inside the orchestrator; here the
+runtime path only ever *reads* the embedded bytes.
+
+Field rules (all consumed by ``contracts/tools/manifest_from_descriptor``):
+
+- every field is proto3 ``optional`` — presence-tracked, so a returned
+  :math:`0`, :math:`0.0`, ``""`` or ``False`` is distinguishable from absent,
+  and an omitted defaulted parameter is distinguishable from an explicit zero
+- input-stream items are ``item_1..item_N`` numbered from 1
+- per-call parameters keep their Python names, numbered from 20 upward so
+  item fields and parameter fields can never collide
+- non-streaming results use the single field ``result``; streaming outputs
+  use ``item_1..item_N``
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from .registry import (
+    MODE_BI_DI,
+    MODE_INPUT_STREAM,
+    MODE_OUTPUT_STREAM,
+    SCALAR_TYPES,
+    RegisteredOp,
+    WorkflowRegistry,
+)
+
+#: First field number for per-call parameters; items occupy 1..19.
+PARAM_FIELD_BASE = 20
+
+_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+class CompileError(RuntimeError):
+    """Raised when the registry cannot be projected into a valid contract."""
+
+
+def _pascal(name: str) -> str:
+    return "".join(part[:1].upper() + part[1:] for part in name.split("_") if part)
+
+
+@dataclass(frozen=True, slots=True)
+class CompiledContract:
+    """The build artifact: proto source, descriptor bytes, and identity."""
+
+    workflow_name: str
+    proto_source: str
+    descriptor_set: bytes
+    contract_hash: str
+
+
+def render_proto(workflow_name: str, ops: dict[str, RegisteredOp]) -> str:
+    """Render the registry as a .proto source file."""
+    if not _NAME_RE.match(workflow_name):
+        raise CompileError(
+            f"workflow name '{workflow_name}' must be snake_case "
+            "([a-z][a-z0-9_]*) — it becomes a protobuf package segment."
+        )
+    if not ops:
+        raise CompileError(
+            "No operations registered. Decorate at least one function with "
+            "@workflow.unary / .input_stream / .output_stream / .bi_di."
+        )
+
+    lines = [
+        "// Generated by perd_worker.compiler — DO NOT EDIT.",
+        'syntax = "proto3";',
+        "",
+        f"package perd.workflow.{workflow_name};",
+        "",
+    ]
+
+    rpcs = []
+    for op in ops.values():
+        rpc = _pascal(op.function_name)
+        req, res = f"{rpc}Request", f"{rpc}Response"
+
+        lines.append(f"message {req} {{")
+        for i, t in enumerate(op.input_item_types, start=1):
+            lines.append(f"  optional {SCALAR_TYPES[t]} item_{i} = {i};")
+        if len(op.input_item_types) >= PARAM_FIELD_BASE:
+            raise CompileError(
+                f"{op.function_name}: at most {PARAM_FIELD_BASE - 1} stream "
+                "item fields are supported."
+            )
+        for j, (name, t, _hd, _d) in enumerate(op.params, start=PARAM_FIELD_BASE):
+            if re.fullmatch(r"item_\d+", name):
+                raise CompileError(
+                    f"{op.function_name}: parameter name '{name}' collides "
+                    "with the reserved stream-item naming (item_N)."
+                )
+            lines.append(f"  optional {SCALAR_TYPES[t]} {name} = {j};")
+        lines.append("}")
+        lines.append("")
+
+        lines.append(f"message {res} {{")
+        if op.mode in (MODE_OUTPUT_STREAM, MODE_BI_DI):
+            for i, t in enumerate(op.output_item_types, start=1):
+                lines.append(f"  optional {SCALAR_TYPES[t]} item_{i} = {i};")
+        else:
+            lines.append(
+                f"  optional {SCALAR_TYPES[op.output_item_types[0]]} result = 1;"
+            )
+        lines.append("}")
+        lines.append("")
+
+        in_stream = "stream " if op.mode in (MODE_INPUT_STREAM, MODE_BI_DI) else ""
+        out_stream = "stream " if op.mode in (MODE_OUTPUT_STREAM, MODE_BI_DI) else ""
+        rpcs.append(f"  rpc {rpc}({in_stream}{req}) returns ({out_stream}{res});")
+
+    lines.append("service Workflow {")
+    lines.extend(rpcs)
+    lines.append("}")
+    return "\n".join(lines) + "\n"
+
+
+def compile_contract(
+    workflow_name: str, registry: WorkflowRegistry
+) -> CompiledContract:
+    """Project the registry into a serialized ``FileDescriptorSet``.
+
+    Uses ``grpc_tools.protoc`` in a temp dir — acceptable because this is a
+    build step, executed exactly once per image, never per request.
+    """
+    proto_source = render_proto(workflow_name, registry.ops)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        (root / "workflow.proto").write_text(proto_source)
+        out = root / "descriptor.binpb"
+        result = subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "grpc_tools.protoc",
+                f"-I{root}",
+                f"--descriptor_set_out={out}",
+                "--include_imports",
+                str(root / "workflow.proto"),
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise CompileError(
+                f"protoc rejected the generated schema:\n{result.stderr}\n"
+                f"--- generated source ---\n{proto_source}"
+            )
+        descriptor_set = out.read_bytes()
+
+    return CompiledContract(
+        workflow_name=workflow_name,
+        proto_source=proto_source,
+        descriptor_set=descriptor_set,
+        contract_hash=hashlib.sha256(descriptor_set).hexdigest(),
+    )

perd_worker/registry.py

@@ -0,0 +1,252 @@
+r"""Workflow authoring surface: decorators and stream type markers.
+
+A workflow developer writes ordinary Python and marks entry points:
+
+.. code-block:: python
+
+    from perd_worker import workflow, WorkflowStreamInput, WorkflowStreamOutput
+
+    @workflow.unary
+    def add(a: float, b: float) -> float: ...
+
+    @workflow.bi_di
+    async def train(
+        inputStream: WorkflowStreamInput[float, float, float],
+        batch_size: int = 4,
+    ) -> WorkflowStreamOutput[int, float]: ...
+
+The decorators only *register*; all schema work happens at build time in
+:mod:`perd_worker.compiler`, which projects these signatures into a protobuf
+service and embeds its ``FileDescriptorSet`` in the image. The four mode names
+here are the only spellings that exist anywhere in the system — v1 carried
+three independent normalisation tables plus legacy aliases, which is why they
+drifted.
+
+where:
+
+- :math:`\text{unary}`: one request message, one response message
+- :math:`\text{input\_stream}`: a stream of item messages plus trailing per-call parameters, one response
+- :math:`\text{output\_stream}`: one request, a stream of responses
+- :math:`\text{bi\_di}`: both, fully interleaved
+"""
+
+from __future__ import annotations
+
+import inspect
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+MODE_UNARY = "unary"
+MODE_INPUT_STREAM = "input_stream"
+MODE_OUTPUT_STREAM = "output_stream"
+MODE_BI_DI = "bi_di"
+
+#: Python annotation -> proto scalar type. The ONLY types a workflow
+#: signature may use. Matches manifest_from_descriptor._PROTO_TYPE_NAMES.
+SCALAR_TYPES: dict[type, str] = {
+    float: "double",
+    int: "int64",
+    str: "string",
+    bool: "bool",
+    bytes: "bytes",
+}
+
+
+class _StreamMarker:
+    """Base for the generic stream annotations; carries the item types."""
+
+    item_types: tuple[type, ...] = ()
+
+    def __class_getitem__(cls, item: type | tuple[type, ...]) -> type[_StreamMarker]:
+        types = item if isinstance(item, tuple) else (item,)
+        marker = type(cls.__name__, (cls,), {"item_types": tuple(types)})
+        return marker
+
+
+class WorkflowStreamInput(_StreamMarker):
+    r"""Annotates the (first) parameter that receives the input stream.
+
+    ``WorkflowStreamInput[float, int]`` declares two-field stream items; the
+    generated request message carries them as ``item_1: double, item_2: int64``.
+    """
+
+
+class WorkflowStreamOutput(_StreamMarker):
+    r"""Annotates a streaming return type.
+
+    ``WorkflowStreamOutput[int, float]`` declares two-field output items,
+    carried as ``item_1: int64, item_2: double`` on the response message.
+    """
+
+
+class WorkflowDefinitionError(TypeError):
+    """Raised at registration time for a signature the SDK cannot serve."""
+
+
+@dataclass(frozen=True, slots=True)
+class RegisteredOp:
+    """One decorated function plus everything introspected from it."""
+
+    function_name: str
+    mode: str
+    fn: Callable[..., Any]
+    #: (name, python type, has_default, default) for per-call parameters.
+    params: tuple[tuple[str, type, bool, Any], ...]
+    #: item types of the input stream ('' when the mode has none).
+    input_item_types: tuple[type, ...]
+    #: item types of the output stream / the single result type for
+    #: non-streaming outputs (always at least one entry).
+    output_item_types: tuple[type, ...]
+    docstring: str | None
+
+
+def _check_scalar(t: Any, where: str) -> type:
+    if t not in SCALAR_TYPES:
+        allowed = ", ".join(x.__name__ for x in SCALAR_TYPES)
+        raise WorkflowDefinitionError(
+            f"{where}: type {t!r} is not supported. "
+            f"Workflow signatures may only use: {allowed}."
+        )
+    assert isinstance(t, type)  # SCALAR_TYPES membership guarantees this
+    return t
+
+
+def _resolve_hints(fn: Callable[..., Any]) -> dict[str, Any]:
+    """Resolve annotations to real objects.
+
+    ``fn.__annotations__`` holds *strings* under ``from __future__ import
+    annotations`` (PEP 563), which every modern workflow module uses —
+    evaluate them in the function's own namespace.
+    """
+    import typing
+
+    try:
+        return typing.get_type_hints(fn)
+    except NameError as exc:
+        raise WorkflowDefinitionError(
+            f"{fn.__name__}: annotation references an undefined name ({exc})."
+        ) from exc
+
+
+def _introspect(fn: Callable[..., Any], mode: str) -> RegisteredOp:
+    sig = inspect.signature(fn)
+    hints = _resolve_hints(fn)
+    params: list[tuple[str, type, bool, Any]] = []
+    input_items: tuple[type, ...] = ()
+
+    parameters = list(sig.parameters.values())
+    input_streaming = mode in (MODE_INPUT_STREAM, MODE_BI_DI)
+    output_streaming = mode in (MODE_OUTPUT_STREAM, MODE_BI_DI)
+
+    if input_streaming:
+        if not parameters:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: {mode} requires a first WorkflowStreamInput parameter."
+            )
+        first = parameters[0]
+        ann = hints.get(first.name)
+        if not (isinstance(ann, type) and issubclass(ann, WorkflowStreamInput)):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: first parameter of a {mode} workflow must be "
+                f"annotated WorkflowStreamInput[...], got {ann!r}."
+            )
+        if not ann.item_types:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: WorkflowStreamInput needs at least one item type, "
+                "e.g. WorkflowStreamInput[float]."
+            )
+        input_items = tuple(
+            _check_scalar(t, f"{fn.__name__} stream item {i + 1}")
+            for i, t in enumerate(ann.item_types)
+        )
+        parameters = parameters[1:]
+
+    for p in parameters:
+        if p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: *args/**kwargs are not representable in a "
+                "workflow contract; declare explicit parameters."
+            )
+        if p.name.startswith("_"):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: parameter '{p.name}' is invalid — names "
+                "beginning with '_' are reserved for client call options "
+                "(e.g. _deadline_ms)."
+            )
+        ann = hints.get(p.name)
+        if ann is None:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: parameter '{p.name}' needs a type annotation."
+            )
+        _check_scalar(ann, f"{fn.__name__}.{p.name}")
+        has_default = p.default is not inspect.Parameter.empty
+        params.append((p.name, ann, has_default, p.default if has_default else None))
+
+    ret = hints.get("return")
+    if output_streaming:
+        if not (isinstance(ret, type) and issubclass(ret, WorkflowStreamOutput)):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: a {mode} workflow must return "
+                f"WorkflowStreamOutput[...], got {ret!r}."
+            )
+        out_items = tuple(
+            _check_scalar(t, f"{fn.__name__} output item {i + 1}")
+            for i, t in enumerate(ret.item_types)
+        )
+        if not out_items:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: WorkflowStreamOutput needs at least one item type."
+            )
+    else:
+        if ret is None:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: a return annotation is required."
+            )
+        out_items = (_check_scalar(ret, f"{fn.__name__} return"),)
+
+    return RegisteredOp(
+        function_name=fn.__name__,
+        mode=mode,
+        fn=fn,
+        params=tuple(params),
+        input_item_types=input_items,
+        output_item_types=out_items,
+        docstring=inspect.getdoc(fn),
+    )
+
+
+@dataclass
+class WorkflowRegistry:
+    """Collects decorated operations for one workflow module."""
+
+    ops: dict[str, RegisteredOp] = field(default_factory=dict)
+
+    def _register(self, fn: Callable[..., Any], mode: str) -> Callable[..., Any]:
+        op = _introspect(fn, mode)
+        if op.function_name in self.ops:
+            raise WorkflowDefinitionError(
+                f"Duplicate workflow operation '{op.function_name}'."
+            )
+        self.ops[op.function_name] = op
+        return fn
+
+    # The four decorators. Names are the contract; never alias them.
+    def unary(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a request/response operation."""
+        return self._register(fn, MODE_UNARY)
+
+    def input_stream(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a client-streaming operation (stream in, one result out)."""
+        return self._register(fn, MODE_INPUT_STREAM)
+
+    def output_stream(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a server-streaming operation (one request, stream out)."""
+        return self._register(fn, MODE_OUTPUT_STREAM)
+
+    def bi_di(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a fully bidirectional streaming operation."""
+        return self._register(fn, MODE_BI_DI)
+
+
+#: The module-level registry workflow authors decorate against.
+workflow = WorkflowRegistry()

perd_worker/runtime.py

@@ -0,0 +1,488 @@
+r"""WorkerService runtime: Describe + the uniform Invoke stream.
+
+One gRPC service carries every workflow and every call mode. The per-workflow
+schema lives entirely in the embedded ``FileDescriptorSet``; message classes
+are built in-process from an isolated ``DescriptorPool`` — no protoc, no
+downloads, no generated modules on the runtime path.
+
+Contract guarantees enforced here, each mapped to the v1 defect it kills:
+
+- ``deadline_ms`` is required and finite; ``0`` is rejected with
+  ``INVALID_ARGUMENT`` (v1 provisioned ``GRPC_DEADLINE_SECONDS`` as the
+  string ``none``, disabling every timeout in the chain)
+- input frames carry contiguous ``seq`` from 1; a gap fails the job rather
+  than silently reordering (v1 had no input sequencing at all)
+- input is credit-gated: the worker grants ``InvokeInputCredit`` and the
+  orchestrator may not exceed it (v1 buffered into an unbounded queue)
+- results are presence-tracked messages; a returned :math:`0`, ``""`` or
+  ``False`` arrives set, never elided (v1's ``MessageToDict`` dropped them)
+- a user-code exception becomes ``InvokeFailed(INTERNAL)`` with the message
+  only — never a traceback on the wire
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import inspect
+import logging
+from typing import Any, AsyncIterator
+
+from google.protobuf import descriptor_pb2, descriptor_pool, message_factory
+
+from perd.v1 import common_pb2, worker_pb2, worker_pb2_grpc
+from manifest_from_descriptor import Operation, build_manifest
+
+from .compiler import CompiledContract
+from .registry import (
+    MODE_BI_DI,
+    MODE_INPUT_STREAM,
+    MODE_OUTPUT_STREAM,
+    MODE_UNARY,
+    RegisteredOp,
+    WorkflowRegistry,
+)
+
+logger = logging.getLogger(__name__)
+
+SDK_VERSION = "0.1.0"
+
+#: Input credit granted on start, and the consumption step that triggers a
+#: fresh grant. Bounds worker memory to ~INITIAL_INPUT_CREDIT frames.
+INITIAL_INPUT_CREDIT = 32
+CREDIT_REFRESH_STEP = 16
+
+_MODE_TO_PROTO = {
+    MODE_UNARY: worker_pb2.CALL_MODE_UNARY,
+    MODE_INPUT_STREAM: worker_pb2.CALL_MODE_INPUT_STREAM,
+    MODE_OUTPUT_STREAM: worker_pb2.CALL_MODE_OUTPUT_STREAM,
+    MODE_BI_DI: worker_pb2.CALL_MODE_BI_DI,
+}
+
+
+class _InvokeAbort(Exception):
+    """Internal: terminate the invoke with a typed Failure."""
+
+    def __init__(
+        self, code: common_pb2.FailureCode, message: str, *, retryable: bool = False
+    ):
+        super().__init__(message)
+        self.failure = common_pb2.Failure(
+            code=code, message=message, retryable=retryable
+        )
+
+
+class _Cancelled(_InvokeAbort):
+    def __init__(self, reason: str):
+        super().__init__(
+            common_pb2.FAILURE_CODE_CANCELLED,
+            f"cancelled by client: {reason}" if reason else "cancelled by client",
+        )
+
+
+def _failed(failure: common_pb2.Failure) -> worker_pb2.InvokeResponse:
+    return worker_pb2.InvokeResponse(failed=worker_pb2.InvokeFailed(failure=failure))
+
+
+class WorkerServicer(worker_pb2_grpc.WorkerServiceServicer):
+    """Serves one workflow's operations over the uniform contract."""
+
+    def __init__(self, contract: CompiledContract, registry: WorkflowRegistry):
+        self._contract = contract
+        self._registry = registry
+
+        # Message classes from the embedded descriptor, in an isolated pool.
+        fds = descriptor_pb2.FileDescriptorSet()
+        fds.ParseFromString(contract.descriptor_set)
+        pool = descriptor_pool.DescriptorPool()
+        for file_proto in fds.file:
+            pool.Add(file_proto)
+        pkg = f"perd.workflow.{contract.workflow_name}"
+
+        self._ops_by_rpc: dict[str, tuple[RegisteredOp, type, type, Operation]] = {}
+        for manifest_op in build_manifest(contract.descriptor_set):
+            reg_op = registry.ops[manifest_op.function_name]
+            req_cls = message_factory.GetMessageClass(
+                pool.FindMessageTypeByName(f"{pkg}.{manifest_op.rpc_name}Request")
+            )
+            res_cls = message_factory.GetMessageClass(
+                pool.FindMessageTypeByName(f"{pkg}.{manifest_op.rpc_name}Response")
+            )
+            self._ops_by_rpc[manifest_op.rpc_name] = (
+                reg_op,
+                req_cls,
+                res_cls,
+                manifest_op,
+            )
+
+        self._describe = self._build_describe()
+
+    # ------------------------------------------------------------ Describe
+    def _build_describe(self) -> worker_pb2.DescribeResponse:
+        ops = []
+        for _rpc, (reg_op, _req, _res, m) in sorted(self._ops_by_rpc.items()):
+            ops.append(
+                worker_pb2.OperationSpec(
+                    function_name=m.function_name,
+                    rpc_name=m.rpc_name,
+                    mode=_MODE_TO_PROTO[reg_op.mode],
+                    params=[
+                        worker_pb2.FieldSpec(
+                            name=f.name,
+                            proto_type=f.proto_type,
+                            field_number=f.field_number,
+                            optional=f.optional,
+                        )
+                        for f in m.params
+                    ],
+                    input_stream_fields=[
+                        worker_pb2.FieldSpec(
+                            name=f.name,
+                            proto_type=f.proto_type,
+                            field_number=f.field_number,
+                            optional=f.optional,
+                        )
+                        for f in m.input_stream_fields
+                    ],
+                    output_fields=[
+                        worker_pb2.FieldSpec(
+                            name=f.name,
+                            proto_type=f.proto_type,
+                            field_number=f.field_number,
+                            optional=f.optional,
+                        )
+                        for f in m.output_fields
+                    ],
+                    docstring=reg_op.docstring or "",
+                )
+            )
+
+        fds = descriptor_pb2.FileDescriptorSet()
+        fds.ParseFromString(self._contract.descriptor_set)
+        return worker_pb2.DescribeResponse(
+            contract=worker_pb2.WorkflowContract(
+                ref=common_pb2.WorkflowRef(
+                    workflow_id=self._contract.workflow_name,
+                    version="",
+                    contract_hash=self._contract.contract_hash,
+                ),
+                descriptor_set=fds,
+                operations=ops,
+            ),
+            contract_hash=self._contract.contract_hash,
+            sdk_version=SDK_VERSION,
+        )
+
+    async def Describe(
+        self, request: worker_pb2.DescribeRequest, context: Any
+    ) -> worker_pb2.DescribeResponse:
+        return self._describe
+
+    # -------------------------------------------------------------- Invoke
+    async def Invoke(
+        self,
+        request_iterator: AsyncIterator[worker_pb2.InvokeRequest],
+        context: Any,
+    ) -> AsyncIterator[worker_pb2.InvokeResponse]:
+        outbound: asyncio.Queue[worker_pb2.InvokeResponse | None] = asyncio.Queue()
+
+        async def run() -> None:
+            try:
+                await self._run_invoke(request_iterator, outbound)
+            except _InvokeAbort as abort:
+                await outbound.put(_failed(abort.failure))
+            except asyncio.CancelledError:
+                raise
+            except Exception as exc:  # user code or protocol surprise
+                logger.exception("invoke failed")
+                await outbound.put(
+                    _failed(
+                        common_pb2.Failure(
+                            code=common_pb2.FAILURE_CODE_INTERNAL,
+                            message=str(exc) or exc.__class__.__name__,
+                            retryable=False,
+                        )
+                    )
+                )
+            finally:
+                await outbound.put(None)
+
+        task = asyncio.create_task(run())
+        try:
+            while True:
+                frame = await outbound.get()
+                if frame is None:
+                    break
+                yield frame
+        finally:
+            if not task.done():
+                task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await task
+
+    async def _run_invoke(
+        self,
+        request_iterator: AsyncIterator[worker_pb2.InvokeRequest],
+        outbound: asyncio.Queue[worker_pb2.InvokeResponse | None],
+    ) -> None:
+        first = await anext(aiter(request_iterator), None)
+        if first is None or first.WhichOneof("frame") != "start":
+            raise _InvokeAbort(
+                common_pb2.FAILURE_CODE_INVALID_ARGUMENT,
+                "First frame must be InvokeStart.",
+            )
+        start = first.start
+
+        if start.deadline_ms <= 0:
+            raise _InvokeAbort(
+                common_pb2.FAILURE_CODE_INVALID_ARGUMENT,
+                "deadline_ms must be > 0; there is no 'no deadline'.",
+            )
+
+        entry = self._ops_by_rpc.get(start.rpc_name)
+        if entry is None:
+            known = ", ".join(sorted(self._ops_by_rpc))
+            raise _InvokeAbort(
+                common_pb2.FAILURE_CODE_NOT_FOUND,
+                f"Unknown rpc '{start.rpc_name}'. This worker serves: {known}.",
+            )
+        reg_op, req_cls, res_cls, manifest_op = entry
+
+        if start.mode != _MODE_TO_PROTO[reg_op.mode]:
+            raise _InvokeAbort(
+                common_pb2.FAILURE_CODE_CONTRACT_MISMATCH,
+                f"'{manifest_op.function_name}' is {reg_op.mode}; the caller "
+                f"asserted a different mode. The client contract is stale.",
+            )
+
+        kwargs = self._bind_params(reg_op, req_cls, start.params)
+        input_iter = (
+            self._input_stream(request_iterator, reg_op, req_cls, outbound)
+            if reg_op.mode in (MODE_INPUT_STREAM, MODE_BI_DI)
+            else None
+        )
+
+        try:
+            async with asyncio.timeout(start.deadline_ms / 1000.0):
+                if input_iter is not None:
+                    await self._execute(reg_op, res_cls, kwargs, input_iter, outbound)
+                else:
+                    await self._execute_with_cancel_watch(
+                        request_iterator, reg_op, res_cls, kwargs, outbound
+                    )
+        except TimeoutError:
+            raise _InvokeAbort(
+                common_pb2.FAILURE_CODE_DEADLINE_EXCEEDED,
+                f"deadline of {start.deadline_ms}ms exceeded",
+                retryable=True,
+            ) from None
+
+    async def _execute_with_cancel_watch(
+        self,
+        request_iterator: AsyncIterator[worker_pb2.InvokeRequest],
+        reg_op: RegisteredOp,
+        res_cls: type[Any],
+        kwargs: dict[str, Any],
+        outbound: asyncio.Queue[worker_pb2.InvokeResponse | None],
+    ) -> None:
+        """Run a no-input-stream operation while honouring mid-call cancel.
+
+        For ``input_stream``/``bi_di`` the input loop reads the request stream
+        and sees ``InvokeCancel`` naturally; for ``unary``/``output_stream``
+        nothing else reads it, so cancellation needs this watcher — without it
+        a cancel frame sat unread and the job ran to completion.
+        """
+        exec_task = asyncio.create_task(
+            self._execute(reg_op, res_cls, kwargs, None, outbound)
+        )
+        cancel_task = asyncio.create_task(self._watch_cancel(request_iterator))
+        try:
+            done, _ = await asyncio.wait(
+                {exec_task, cancel_task}, return_when=asyncio.FIRST_COMPLETED
+            )
+            if cancel_task in done and not exec_task.done():
+                reason = cancel_task.result()
+                if reason is not None:
+                    raise _Cancelled(reason)
+                # Client closed its send side without cancelling: normal for
+                # these modes; let the work finish.
+            await exec_task
+        finally:
+            for task in (exec_task, cancel_task):
+                if not task.done():
+                    task.cancel()
+            await asyncio.gather(exec_task, cancel_task, return_exceptions=True)
+
+    async def _watch_cancel(
+        self, request_iterator: AsyncIterator[worker_pb2.InvokeRequest]
+    ) -> str | None:
+        """Return the cancel reason, or None if the stream ends without one."""
+        async for frame in request_iterator:
+            if frame.WhichOneof("frame") == "cancel":
+                return frame.cancel.reason
+        return None
+
+    # ---------------------------------------------------- request plumbing
+    def _bind_params(
+        self, reg_op: RegisteredOp, req_cls: type[Any], params_bytes: bytes
+    ) -> dict[str, Any]:
+        msg = req_cls()
+        if params_bytes:
+            msg.ParseFromString(params_bytes)
+        kwargs: dict[str, Any] = {}
+        missing: list[str] = []
+        for name, _t, has_default, default in reg_op.params:
+            if msg.HasField(name):
+                kwargs[name] = getattr(msg, name)
+            elif has_default:
+                kwargs[name] = default
+            else:
+                missing.append(name)
+        if missing:
+            raise _InvokeAbort(
+                common_pb2.FAILURE_CODE_INVALID_ARGUMENT,
+                f"Missing required parameter(s): {', '.join(missing)}.",
+            )
+        return kwargs
+
+    async def _input_stream(
+        self,
+        request_iterator: AsyncIterator[worker_pb2.InvokeRequest],
+        reg_op: RegisteredOp,
+        req_cls: type[Any],
+        outbound: asyncio.Queue[worker_pb2.InvokeResponse | None],
+    ) -> AsyncIterator[Any]:
+        arity = len(reg_op.input_item_types)
+        names = [f"item_{i}" for i in range(1, arity + 1)]
+        await outbound.put(
+            worker_pb2.InvokeResponse(
+                input_credit=worker_pb2.InvokeInputCredit(
+                    additional=INITIAL_INPUT_CREDIT
+                )
+            )
+        )
+        expected_seq = 1
+        consumed_since_grant = 0
+        async for frame in request_iterator:
+            kind = frame.WhichOneof("frame")
+            if kind == "cancel":
+                raise _Cancelled(frame.cancel.reason)
+            if kind != "input":
+                raise _InvokeAbort(
+                    common_pb2.FAILURE_CODE_INVALID_ARGUMENT,
+                    f"Unexpected mid-stream frame '{kind}'.",
+                )
+            inp = frame.input
+            if inp.final and not inp.payload:
+                return
+            if inp.seq != expected_seq:
+                raise _InvokeAbort(
+                    common_pb2.FAILURE_CODE_INVALID_ARGUMENT,
+                    f"Input sequence gap: expected seq={expected_seq}, "
+                    f"got seq={inp.seq}. Frames were lost or reordered.",
+                )
+            expected_seq += 1
+
+            item_msg = req_cls()
+            item_msg.ParseFromString(inp.payload)
+            values = []
+            for n in names:
+                if not item_msg.HasField(n):
+                    raise _InvokeAbort(
+                        common_pb2.FAILURE_CODE_INVALID_ARGUMENT,
+                        f"Stream item seq={inp.seq} is missing field '{n}'.",
+                    )
+                values.append(getattr(item_msg, n))
+            yield values[0] if arity == 1 else tuple(values)
+
+            consumed_since_grant += 1
+            if consumed_since_grant >= CREDIT_REFRESH_STEP:
+                consumed_since_grant = 0
+                await outbound.put(
+                    worker_pb2.InvokeResponse(
+                        input_credit=worker_pb2.InvokeInputCredit(
+                            additional=CREDIT_REFRESH_STEP
+                        )
+                    )
+                )
+            if inp.final:
+                return
+
+    # ------------------------------------------------------------ execute
+    def _encode_output(
+        self, reg_op: RegisteredOp, res_cls: type[Any], value: Any
+    ) -> Any:
+        msg = res_cls()
+        if reg_op.mode in (MODE_OUTPUT_STREAM, MODE_BI_DI):
+            arity = len(reg_op.output_item_types)
+            items = (value,) if arity == 1 else tuple(value)
+            if len(items) != arity:
+                raise _InvokeAbort(
+                    common_pb2.FAILURE_CODE_INTERNAL,
+                    f"'{reg_op.function_name}' yielded {len(items)} values; "
+                    f"its contract declares {arity}.",
+                )
+            for i, v in enumerate(items, start=1):
+                setattr(msg, f"item_{i}", v)
+        else:
+            msg.result = value
+        return msg
+
+    async def _execute(
+        self,
+        reg_op: RegisteredOp,
+        res_cls: type[Any],
+        kwargs: dict[str, Any],
+        input_iter: AsyncIterator[Any] | None,
+        outbound: asyncio.Queue[worker_pb2.InvokeResponse | None],
+    ) -> None:
+        out_seq = 0
+
+        async def emit(value: Any) -> int:
+            nonlocal out_seq
+            out_seq += 1
+            payload = self._encode_output(reg_op, res_cls, value)
+            await outbound.put(
+                worker_pb2.InvokeResponse(
+                    output=worker_pb2.InvokeOutput(
+                        seq=out_seq, payload=payload.SerializeToString()
+                    )
+                )
+            )
+            return out_seq
+
+        fn = reg_op.fn
+        if reg_op.mode == MODE_UNARY:
+            result = fn(**kwargs)
+            if inspect.isawaitable(result):
+                result = await result
+            final = await emit(result)
+
+        elif reg_op.mode == MODE_OUTPUT_STREAM:
+            produced = fn(**kwargs)
+            if inspect.isawaitable(produced):
+                produced = await produced
+            if hasattr(produced, "__aiter__"):
+                async for value in produced:
+                    await emit(value)
+            else:
+                for value in produced:
+                    await emit(value)
+            final = out_seq
+
+        elif reg_op.mode == MODE_INPUT_STREAM:
+            result = fn(input_iter, **kwargs)
+            if inspect.isawaitable(result):
+                result = await result
+            final = await emit(result)
+
+        else:  # MODE_BI_DI
+            async for value in fn(input_iter, **kwargs):
+                await emit(value)
+            final = out_seq
+
+        await outbound.put(
+            worker_pb2.InvokeResponse(
+                completed=worker_pb2.InvokeCompleted(final_seq=final)
+            )
+        )

perd_worker/serve.py

@@ -0,0 +1,103 @@
+r"""Serve a workflow module over WorkerService — the worker container entrypoint.
+
+.. code-block:: bash
+
+    python -m perd_worker.serve --module my_flows --name my_flows --port 50051
+
+The module is imported, its registry located (the module-level ``workflow``
+object by convention, or ``--factory`` naming a zero-argument callable that
+returns one), the contract compiled, and the uniform ``WorkerService``
+served. The orchestrator registers against ``Describe`` and recomputes the
+contract hash from the descriptor bytes — nothing here is trusted.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import importlib
+import logging
+import sys
+
+import grpc
+
+from perd.v1 import worker_pb2_grpc
+
+from .compiler import compile_contract
+from .registry import WorkflowRegistry
+from .runtime import WorkerServicer
+
+logger = logging.getLogger("perd_worker")
+
+
+def load_registry(module_name: str, factory: str | None) -> WorkflowRegistry:
+    """Import ``module_name`` and return its :class:`WorkflowRegistry`.
+
+    Raises:
+        SystemExit: with a remediation message if no registry is found.
+    """
+    module = importlib.import_module(module_name)
+    if factory is not None:
+        registry = getattr(module, factory)()
+    else:
+        registry = getattr(module, "workflow", None)
+    if not isinstance(registry, WorkflowRegistry):
+        raise SystemExit(
+            f"{module_name!r} does not expose a WorkflowRegistry. Either "
+            "decorate against `from perd_worker import workflow` (the "
+            "module-level registry) or pass --factory <callable> returning one."
+        )
+    return registry
+
+
+async def serve(
+    registry: WorkflowRegistry, name: str, port: int, *, bind_host: str = "0.0.0.0"
+) -> None:
+    contract = compile_contract(name, registry)
+    server = grpc.aio.server()
+    # The generated grpc glue is untyped by design (see the perd.v1 override).
+    worker_pb2_grpc.add_WorkerServiceServicer_to_server(  # type: ignore[no-untyped-call]
+        WorkerServicer(contract, registry), server
+    )
+    bound = server.add_insecure_port(f"{bind_host}:{port}")
+    await server.start()
+    logger.info(
+        "serving workflow %s [%s…] on %s:%d (%d ops)",
+        name,
+        contract.contract_hash[:12],
+        bind_host,
+        bound,
+        len(registry.ops),
+    )
+    await server.wait_for_termination()
+
+
+def main(
+    argv: list[str],
+) -> int:  # pragma: no cover — container entrypoint, subprocess-only
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s"
+    )
+    parser = argparse.ArgumentParser(
+        prog="perd_worker.serve", description="Serve a workflow over WorkerService."
+    )
+    parser.add_argument("--module", required=True, help="python module to import")
+    parser.add_argument(
+        "--name", required=True, help="workflow name (becomes workflow_id)"
+    )
+    parser.add_argument(
+        "--factory",
+        default=None,
+        help="zero-arg callable on the module returning a WorkflowRegistry "
+        "(default: use the module-level `workflow` registry)",
+    )
+    parser.add_argument("--port", type=int, default=50051)
+    parser.add_argument("--bind-host", default="0.0.0.0")
+    args = parser.parse_args(argv)
+    registry = load_registry(args.module, args.factory)
+    asyncio.run(serve(registry, args.name, args.port, bind_host=args.bind_host))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

perd_worker-0.1.0.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: perd-worker
+Version: 0.1.0
+Summary: Author PERD workflows: decorators, contract compiler, WorkerService runtime
+Requires-Python: >=3.11
+Requires-Dist: grpcio-tools>=1.60
+Requires-Dist: grpcio>=1.60
+Requires-Dist: perd-contracts~=0.1.0
+Requires-Dist: protobuf>=6

perd_worker-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+perd_worker/__init__.py,sha256=mQ-3vQDK92nCcHv-I9-FcunLoC_5rT2ojqMeGqjuToM,784
+perd_worker/compiler.py,sha256=lB0-_TIFcZOh9upkWe9xqsFXXaTH3ACMVWMZ35ubbbc,6264
+perd_worker/registry.py,sha256=Afuoct--pu4gVWSZfuadva7YU-6kzSw-wM7jWQ8Z-ME,9165
+perd_worker/runtime.py,sha256=7FDuZu41-yjM2FTiBtIf1YZ5NNXrvij25OFJ02mJh9I,18532
+perd_worker/serve.py,sha256=BX87KMvx_OjMHU00PGeMcR0PZu92y3MecO6xak3MyCM,3505
+perd_worker-0.1.0.dist-info/METADATA,sha256=5ckY0ONaMhrLoJCvtL8XpUK38xiGPjiABKqFDqGuOtc,290
+perd_worker-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+perd_worker-0.1.0.dist-info/RECORD,,

perd_worker-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any