perd-worker 0.1.0__tar.gz

perd_worker-0.1.0/.gitignore

@@ -0,0 +1,24 @@
+__pycache__/
+*.py[cod]
+.venv/
+node_modules/
+.pytest_cache/
+.coverage
+.coverage.*
+# Spike codegen: regenerated by conftest.py on every run.
+spikes/**/echo_pb2.py
+spikes/**/echo_pb2.pyi
+# Next.js build output
+.next/
+out/
+*.tsbuildinfo
+
+# Build artifacts
+dist/
+coverage.xml
+
+# terraform: state and plugin cache never committed; .terraform.lock.hcl IS.
+**/.terraform/
+*.tfplan
+*.tfstate
+*.tfstate.*

perd_worker-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: perd-worker
+Version: 0.1.0
+Summary: Author PERD workflows: decorators, contract compiler, WorkerService runtime
+Requires-Python: >=3.11
+Requires-Dist: grpcio-tools>=1.60
+Requires-Dist: grpcio>=1.60
+Requires-Dist: perd-contracts~=0.1.0
+Requires-Dist: protobuf>=6

perd_worker-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "perd-worker"
+version = "0.1.0"
+description = "Author PERD workflows: decorators, contract compiler, WorkerService runtime"
+requires-python = ">=3.11"
+dependencies = [
+    # Compatible-release pin: the generated perd.v1 modules must match this
+    # wheel; release-py.yml enforces one version across all published dists.
+    "perd-contracts~=0.1.0",
+    "protobuf>=6",
+    "grpcio>=1.60",
+    # grpcio-tools is a build-time need of workflow images (compile_contract
+    # runs protoc when the developer builds), carried as a runtime dep so a
+    # worker image is always able to compile the workflow it ships.
+    "grpcio-tools>=1.60",
+]
+
+[tool.uv.sources]
+perd-contracts = { workspace = true }
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/perd_worker"]

perd_worker-0.1.0/src/perd_worker/__init__.py

@@ -0,0 +1,32 @@
+r"""perd_worker — the SDK workflow developers build against.
+
+Authoring surface:
+
+.. code-block:: python
+
+    from perd_worker import workflow, WorkflowStreamInput, WorkflowStreamOutput
+
+Build time: :func:`perd_worker.compiler.compile_contract` projects the
+registered signatures into the workflow's ``FileDescriptorSet``; its SHA-256
+is the contract identity every other component pins.
+
+Runtime: :class:`perd_worker.runtime.WorkerServicer` serves the uniform
+``perd.v1.WorkerService`` over gRPC.
+"""
+
+from .registry import (
+    WorkflowDefinitionError,
+    WorkflowStreamInput,
+    WorkflowStreamOutput,
+    workflow,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "workflow",
+    "WorkflowStreamInput",
+    "WorkflowStreamOutput",
+    "WorkflowDefinitionError",
+    "__version__",
+]

perd_worker-0.1.0/src/perd_worker/compiler.py

@@ -0,0 +1,178 @@
+r"""Build-time projection: registered signatures -> protobuf contract.
+
+Runs ONCE, at image build. The output — a serialized ``FileDescriptorSet`` —
+is the workflow's entire public identity:
+
+.. math::
+
+    \mathrm{contract\_hash} = \operatorname{SHA256}\!\left(D\right)
+
+where:
+
+- :math:`D`: the ``FileDescriptorSet`` bytes embedded in the worker image and returned verbatim by ``WorkerService.Describe``
+- :math:`\mathrm{contract\_hash}`: pinned by the store at publish, by the client at resolve, and re-verified by the orchestrator at registration — one value, three enforcement points
+
+Never runs at request time. v1 downloaded protos from GCS and shelled out to
+``protoc`` on first use of every workflow, inside the orchestrator; here the
+runtime path only ever *reads* the embedded bytes.
+
+Field rules (all consumed by ``contracts/tools/manifest_from_descriptor``):
+
+- every field is proto3 ``optional`` — presence-tracked, so a returned
+  :math:`0`, :math:`0.0`, ``""`` or ``False`` is distinguishable from absent,
+  and an omitted defaulted parameter is distinguishable from an explicit zero
+- input-stream items are ``item_1..item_N`` numbered from 1
+- per-call parameters keep their Python names, numbered from 20 upward so
+  item fields and parameter fields can never collide
+- non-streaming results use the single field ``result``; streaming outputs
+  use ``item_1..item_N``
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from .registry import (
+    MODE_BI_DI,
+    MODE_INPUT_STREAM,
+    MODE_OUTPUT_STREAM,
+    SCALAR_TYPES,
+    RegisteredOp,
+    WorkflowRegistry,
+)
+
+#: First field number for per-call parameters; items occupy 1..19.
+PARAM_FIELD_BASE = 20
+
+_NAME_RE = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+class CompileError(RuntimeError):
+    """Raised when the registry cannot be projected into a valid contract."""
+
+
+def _pascal(name: str) -> str:
+    return "".join(part[:1].upper() + part[1:] for part in name.split("_") if part)
+
+
+@dataclass(frozen=True, slots=True)
+class CompiledContract:
+    """The build artifact: proto source, descriptor bytes, and identity."""
+
+    workflow_name: str
+    proto_source: str
+    descriptor_set: bytes
+    contract_hash: str
+
+
+def render_proto(workflow_name: str, ops: dict[str, RegisteredOp]) -> str:
+    """Render the registry as a .proto source file."""
+    if not _NAME_RE.match(workflow_name):
+        raise CompileError(
+            f"workflow name '{workflow_name}' must be snake_case "
+            "([a-z][a-z0-9_]*) — it becomes a protobuf package segment."
+        )
+    if not ops:
+        raise CompileError(
+            "No operations registered. Decorate at least one function with "
+            "@workflow.unary / .input_stream / .output_stream / .bi_di."
+        )
+
+    lines = [
+        "// Generated by perd_worker.compiler — DO NOT EDIT.",
+        'syntax = "proto3";',
+        "",
+        f"package perd.workflow.{workflow_name};",
+        "",
+    ]
+
+    rpcs = []
+    for op in ops.values():
+        rpc = _pascal(op.function_name)
+        req, res = f"{rpc}Request", f"{rpc}Response"
+
+        lines.append(f"message {req} {{")
+        for i, t in enumerate(op.input_item_types, start=1):
+            lines.append(f"  optional {SCALAR_TYPES[t]} item_{i} = {i};")
+        if len(op.input_item_types) >= PARAM_FIELD_BASE:
+            raise CompileError(
+                f"{op.function_name}: at most {PARAM_FIELD_BASE - 1} stream "
+                "item fields are supported."
+            )
+        for j, (name, t, _hd, _d) in enumerate(op.params, start=PARAM_FIELD_BASE):
+            if re.fullmatch(r"item_\d+", name):
+                raise CompileError(
+                    f"{op.function_name}: parameter name '{name}' collides "
+                    "with the reserved stream-item naming (item_N)."
+                )
+            lines.append(f"  optional {SCALAR_TYPES[t]} {name} = {j};")
+        lines.append("}")
+        lines.append("")
+
+        lines.append(f"message {res} {{")
+        if op.mode in (MODE_OUTPUT_STREAM, MODE_BI_DI):
+            for i, t in enumerate(op.output_item_types, start=1):
+                lines.append(f"  optional {SCALAR_TYPES[t]} item_{i} = {i};")
+        else:
+            lines.append(
+                f"  optional {SCALAR_TYPES[op.output_item_types[0]]} result = 1;"
+            )
+        lines.append("}")
+        lines.append("")
+
+        in_stream = "stream " if op.mode in (MODE_INPUT_STREAM, MODE_BI_DI) else ""
+        out_stream = "stream " if op.mode in (MODE_OUTPUT_STREAM, MODE_BI_DI) else ""
+        rpcs.append(f"  rpc {rpc}({in_stream}{req}) returns ({out_stream}{res});")
+
+    lines.append("service Workflow {")
+    lines.extend(rpcs)
+    lines.append("}")
+    return "\n".join(lines) + "\n"
+
+
+def compile_contract(
+    workflow_name: str, registry: WorkflowRegistry
+) -> CompiledContract:
+    """Project the registry into a serialized ``FileDescriptorSet``.
+
+    Uses ``grpc_tools.protoc`` in a temp dir — acceptable because this is a
+    build step, executed exactly once per image, never per request.
+    """
+    proto_source = render_proto(workflow_name, registry.ops)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        (root / "workflow.proto").write_text(proto_source)
+        out = root / "descriptor.binpb"
+        result = subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "grpc_tools.protoc",
+                f"-I{root}",
+                f"--descriptor_set_out={out}",
+                "--include_imports",
+                str(root / "workflow.proto"),
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise CompileError(
+                f"protoc rejected the generated schema:\n{result.stderr}\n"
+                f"--- generated source ---\n{proto_source}"
+            )
+        descriptor_set = out.read_bytes()
+
+    return CompiledContract(
+        workflow_name=workflow_name,
+        proto_source=proto_source,
+        descriptor_set=descriptor_set,
+        contract_hash=hashlib.sha256(descriptor_set).hexdigest(),
+    )

perd_worker-0.1.0/src/perd_worker/registry.py

@@ -0,0 +1,252 @@
+r"""Workflow authoring surface: decorators and stream type markers.
+
+A workflow developer writes ordinary Python and marks entry points:
+
+.. code-block:: python
+
+    from perd_worker import workflow, WorkflowStreamInput, WorkflowStreamOutput
+
+    @workflow.unary
+    def add(a: float, b: float) -> float: ...
+
+    @workflow.bi_di
+    async def train(
+        inputStream: WorkflowStreamInput[float, float, float],
+        batch_size: int = 4,
+    ) -> WorkflowStreamOutput[int, float]: ...
+
+The decorators only *register*; all schema work happens at build time in
+:mod:`perd_worker.compiler`, which projects these signatures into a protobuf
+service and embeds its ``FileDescriptorSet`` in the image. The four mode names
+here are the only spellings that exist anywhere in the system — v1 carried
+three independent normalisation tables plus legacy aliases, which is why they
+drifted.
+
+where:
+
+- :math:`\text{unary}`: one request message, one response message
+- :math:`\text{input\_stream}`: a stream of item messages plus trailing per-call parameters, one response
+- :math:`\text{output\_stream}`: one request, a stream of responses
+- :math:`\text{bi\_di}`: both, fully interleaved
+"""
+
+from __future__ import annotations
+
+import inspect
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+MODE_UNARY = "unary"
+MODE_INPUT_STREAM = "input_stream"
+MODE_OUTPUT_STREAM = "output_stream"
+MODE_BI_DI = "bi_di"
+
+#: Python annotation -> proto scalar type. The ONLY types a workflow
+#: signature may use. Matches manifest_from_descriptor._PROTO_TYPE_NAMES.
+SCALAR_TYPES: dict[type, str] = {
+    float: "double",
+    int: "int64",
+    str: "string",
+    bool: "bool",
+    bytes: "bytes",
+}
+
+
+class _StreamMarker:
+    """Base for the generic stream annotations; carries the item types."""
+
+    item_types: tuple[type, ...] = ()
+
+    def __class_getitem__(cls, item: type | tuple[type, ...]) -> type[_StreamMarker]:
+        types = item if isinstance(item, tuple) else (item,)
+        marker = type(cls.__name__, (cls,), {"item_types": tuple(types)})
+        return marker
+
+
+class WorkflowStreamInput(_StreamMarker):
+    r"""Annotates the (first) parameter that receives the input stream.
+
+    ``WorkflowStreamInput[float, int]`` declares two-field stream items; the
+    generated request message carries them as ``item_1: double, item_2: int64``.
+    """
+
+
+class WorkflowStreamOutput(_StreamMarker):
+    r"""Annotates a streaming return type.
+
+    ``WorkflowStreamOutput[int, float]`` declares two-field output items,
+    carried as ``item_1: int64, item_2: double`` on the response message.
+    """
+
+
+class WorkflowDefinitionError(TypeError):
+    """Raised at registration time for a signature the SDK cannot serve."""
+
+
+@dataclass(frozen=True, slots=True)
+class RegisteredOp:
+    """One decorated function plus everything introspected from it."""
+
+    function_name: str
+    mode: str
+    fn: Callable[..., Any]
+    #: (name, python type, has_default, default) for per-call parameters.
+    params: tuple[tuple[str, type, bool, Any], ...]
+    #: item types of the input stream ('' when the mode has none).
+    input_item_types: tuple[type, ...]
+    #: item types of the output stream / the single result type for
+    #: non-streaming outputs (always at least one entry).
+    output_item_types: tuple[type, ...]
+    docstring: str | None
+
+
+def _check_scalar(t: Any, where: str) -> type:
+    if t not in SCALAR_TYPES:
+        allowed = ", ".join(x.__name__ for x in SCALAR_TYPES)
+        raise WorkflowDefinitionError(
+            f"{where}: type {t!r} is not supported. "
+            f"Workflow signatures may only use: {allowed}."
+        )
+    assert isinstance(t, type)  # SCALAR_TYPES membership guarantees this
+    return t
+
+
+def _resolve_hints(fn: Callable[..., Any]) -> dict[str, Any]:
+    """Resolve annotations to real objects.
+
+    ``fn.__annotations__`` holds *strings* under ``from __future__ import
+    annotations`` (PEP 563), which every modern workflow module uses —
+    evaluate them in the function's own namespace.
+    """
+    import typing
+
+    try:
+        return typing.get_type_hints(fn)
+    except NameError as exc:
+        raise WorkflowDefinitionError(
+            f"{fn.__name__}: annotation references an undefined name ({exc})."
+        ) from exc
+
+
+def _introspect(fn: Callable[..., Any], mode: str) -> RegisteredOp:
+    sig = inspect.signature(fn)
+    hints = _resolve_hints(fn)
+    params: list[tuple[str, type, bool, Any]] = []
+    input_items: tuple[type, ...] = ()
+
+    parameters = list(sig.parameters.values())
+    input_streaming = mode in (MODE_INPUT_STREAM, MODE_BI_DI)
+    output_streaming = mode in (MODE_OUTPUT_STREAM, MODE_BI_DI)
+
+    if input_streaming:
+        if not parameters:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: {mode} requires a first WorkflowStreamInput parameter."
+            )
+        first = parameters[0]
+        ann = hints.get(first.name)
+        if not (isinstance(ann, type) and issubclass(ann, WorkflowStreamInput)):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: first parameter of a {mode} workflow must be "
+                f"annotated WorkflowStreamInput[...], got {ann!r}."
+            )
+        if not ann.item_types:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: WorkflowStreamInput needs at least one item type, "
+                "e.g. WorkflowStreamInput[float]."
+            )
+        input_items = tuple(
+            _check_scalar(t, f"{fn.__name__} stream item {i + 1}")
+            for i, t in enumerate(ann.item_types)
+        )
+        parameters = parameters[1:]
+
+    for p in parameters:
+        if p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: *args/**kwargs are not representable in a "
+                "workflow contract; declare explicit parameters."
+            )
+        if p.name.startswith("_"):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: parameter '{p.name}' is invalid — names "
+                "beginning with '_' are reserved for client call options "
+                "(e.g. _deadline_ms)."
+            )
+        ann = hints.get(p.name)
+        if ann is None:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: parameter '{p.name}' needs a type annotation."
+            )
+        _check_scalar(ann, f"{fn.__name__}.{p.name}")
+        has_default = p.default is not inspect.Parameter.empty
+        params.append((p.name, ann, has_default, p.default if has_default else None))
+
+    ret = hints.get("return")
+    if output_streaming:
+        if not (isinstance(ret, type) and issubclass(ret, WorkflowStreamOutput)):
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: a {mode} workflow must return "
+                f"WorkflowStreamOutput[...], got {ret!r}."
+            )
+        out_items = tuple(
+            _check_scalar(t, f"{fn.__name__} output item {i + 1}")
+            for i, t in enumerate(ret.item_types)
+        )
+        if not out_items:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: WorkflowStreamOutput needs at least one item type."
+            )
+    else:
+        if ret is None:
+            raise WorkflowDefinitionError(
+                f"{fn.__name__}: a return annotation is required."
+            )
+        out_items = (_check_scalar(ret, f"{fn.__name__} return"),)
+
+    return RegisteredOp(
+        function_name=fn.__name__,
+        mode=mode,
+        fn=fn,
+        params=tuple(params),
+        input_item_types=input_items,
+        output_item_types=out_items,
+        docstring=inspect.getdoc(fn),
+    )
+
+
+@dataclass
+class WorkflowRegistry:
+    """Collects decorated operations for one workflow module."""
+
+    ops: dict[str, RegisteredOp] = field(default_factory=dict)
+
+    def _register(self, fn: Callable[..., Any], mode: str) -> Callable[..., Any]:
+        op = _introspect(fn, mode)
+        if op.function_name in self.ops:
+            raise WorkflowDefinitionError(
+                f"Duplicate workflow operation '{op.function_name}'."
+            )
+        self.ops[op.function_name] = op
+        return fn
+
+    # The four decorators. Names are the contract; never alias them.
+    def unary(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a request/response operation."""
+        return self._register(fn, MODE_UNARY)
+
+    def input_stream(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a client-streaming operation (stream in, one result out)."""
+        return self._register(fn, MODE_INPUT_STREAM)
+
+    def output_stream(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a server-streaming operation (one request, stream out)."""
+        return self._register(fn, MODE_OUTPUT_STREAM)
+
+    def bi_di(self, fn: Callable[..., Any]) -> Callable[..., Any]:
+        """Register a fully bidirectional streaming operation."""
+        return self._register(fn, MODE_BI_DI)
+
+
+#: The module-level registry workflow authors decorate against.
+workflow = WorkflowRegistry()