velophex 0.2.0__py3-none-any.whl

velophex/worker/framing.py

@@ -0,0 +1,248 @@
+r"""Authenticated framing of the executor protocol (``docs/EXECUTOR-PROTOCOL.md`` section 2).
+
+::
+
+    frame = length (u32 BE, 1..limit) || tag (32 bytes) || payload (UTF-8 JSON)
+    tag   = HMAC-SHA256(key, "VPXEX/1\0" || direction (u8) || sequence (u64 BE)
+                             || length (u32 BE) || payload)
+
+Binding the direction and the per-direction sequence number into the tag is what makes a
+frame impossible to forge, reorder, replay or reflect back at its sender. A reader that has
+rejected one frame stays faulted: after a violation nothing on the channel can be trusted, so
+the session ends and the parent kills the process.
+"""
+
+from __future__ import annotations
+
+import hmac
+import os
+import struct
+import threading
+from base64 import b64decode
+from hashlib import sha256
+from typing import BinaryIO
+
+from velophex.contracts.executor import (
+    ABSOLUTE_MAXIMUM_FRAME_BYTES,
+    DEFAULT_MAXIMUM_FRAME_BYTES,
+    FRAME_HEADER_BYTES,
+    FRAME_LENGTH_BYTES,
+    FRAME_TAG_BYTES,
+    SESSION_KEY_BYTES,
+    SESSION_KEY_ENVIRONMENT_VARIABLE,
+    TAG_LABEL,
+    ExecutorMessage,
+    ExecutorProtocolError,
+    ExecutorProtocolException,
+    decode_message,
+    encode_message,
+)
+
+__all__ = ["FrameReader", "FrameWriter", "compute_tag", "take_session_key_from_environment"]
+
+_HEADER = struct.Struct(">I")
+_BINDING = struct.Struct(">BQI")
+
+
+def compute_tag(session_key: bytes, direction: int, sequence: int, payload: bytes) -> bytes:
+    """The 32-byte tag of one frame."""
+    if len(session_key) != SESSION_KEY_BYTES:
+        raise ValueError(f"The session key must be {SESSION_KEY_BYTES} bytes.")
+
+    mac = hmac.new(session_key, digestmod=sha256)
+    mac.update(TAG_LABEL)
+    mac.update(_BINDING.pack(direction, sequence, len(payload)))
+    mac.update(payload)
+    return mac.digest()
+
+
+def take_session_key_from_environment() -> bytes | None:
+    """Read the session key from the environment and **remove it**.
+
+    Automation code runs in this process, so the key must not be readable once the handler
+    starts: it is the only thing that separates the protocol stream from anything else that
+    could write to stdout.
+    """
+    encoded = os.environ.pop(SESSION_KEY_ENVIRONMENT_VARIABLE, None)
+
+    if not encoded or not encoded.strip():
+        return None
+
+    try:
+        key = b64decode(encoded.strip(), validate=True)
+    except ValueError:
+        return None
+
+    return key if len(key) == SESSION_KEY_BYTES else None
+
+
+class FrameWriter:
+    """Writes authenticated frames.
+
+    Thread-safe, because logs, progress and the terminal message are written from different
+    threads while the handler runs.
+    """
+
+    def __init__(
+        self,
+        stream: BinaryIO,
+        session_key: bytes,
+        direction: int,
+        maximum_payload_bytes: int = DEFAULT_MAXIMUM_FRAME_BYTES,
+    ) -> None:
+        if not 1 <= maximum_payload_bytes <= ABSOLUTE_MAXIMUM_FRAME_BYTES:
+            raise ValueError(f"The frame limit must be between 1 and {ABSOLUTE_MAXIMUM_FRAME_BYTES} bytes.")
+
+        self._stream = stream
+        self._key = session_key
+        self._direction = direction
+        self._maximum = maximum_payload_bytes
+        self._lock = threading.Lock()
+        self._sequence = 0
+
+    @property
+    def frames_written(self) -> int:
+        """How many frames this writer has produced."""
+        return self._sequence
+
+    @property
+    def maximum_payload_bytes(self) -> int:
+        """The largest payload this writer will frame."""
+        return self._maximum
+
+    def build_frame(self, message: ExecutorMessage) -> bytes:
+        """Build the next frame without writing it. Used by the conformance vectors."""
+        with self._lock:
+            return self._build(self.encode(message))
+
+    def write(self, message: ExecutorMessage) -> None:
+        """Encode, authenticate and flush *message*."""
+        self.write_payload(self.encode(message))
+
+    def write_payload(self, payload: bytes) -> None:
+        """Authenticate and flush a payload :meth:`encode` produced.
+
+        Splitting the two lets a caller find out that a message cannot be sent - a value that
+        is not JSON, or a payload over the limit - before anything reaches the stream, and
+        send something else in its place.
+        """
+        if not 1 <= len(payload) <= self._maximum:
+            raise ValueError(f"A payload must be between 1 and {self._maximum} bytes.")
+
+        with self._lock:
+            frame = self._build(payload)
+            self._stream.write(frame)
+            self._stream.flush()
+
+    def encode(self, message: ExecutorMessage) -> bytes:
+        """The payload of *message*.
+
+        Raises:
+            TypeError: The message holds a value that has no JSON form (a ``datetime``, a set).
+            ValueError: The message holds a number JSON cannot carry (NaN, infinity) or is
+                nested too deeply.
+            ExecutorProtocolException: The payload is over the frame limit (``PayloadTooLarge``).
+        """
+        payload = encode_message(message)
+
+        if len(payload) > self._maximum:
+            raise ExecutorProtocolException(
+                ExecutorProtocolError.PAYLOAD_TOO_LARGE,
+                f"The '{message.type_name}' message is {len(payload)} bytes; the limit is {self._maximum} bytes.",
+            )
+
+        return payload
+
+    def _build(self, payload: bytes) -> bytes:
+        tag = compute_tag(self._key, self._direction, self._sequence, payload)
+        self._sequence += 1
+        return _HEADER.pack(len(payload)) + tag + payload
+
+
+class FrameReader:
+    """Reads and verifies frames. Not thread-safe: one reader per direction."""
+
+    def __init__(
+        self,
+        stream: BinaryIO,
+        session_key: bytes,
+        direction: int,
+        maximum_payload_bytes: int = DEFAULT_MAXIMUM_FRAME_BYTES,
+    ) -> None:
+        if not 1 <= maximum_payload_bytes <= ABSOLUTE_MAXIMUM_FRAME_BYTES:
+            raise ValueError(f"The frame limit must be between 1 and {ABSOLUTE_MAXIMUM_FRAME_BYTES} bytes.")
+
+        self._stream = stream
+        self._key = session_key
+        self._direction = direction
+        self._maximum = maximum_payload_bytes
+        self._sequence = 0
+        self._fault: ExecutorProtocolException | None = None
+
+    @property
+    def frames_read(self) -> int:
+        """How many frames this reader has accepted."""
+        return self._sequence
+
+    def read(self) -> ExecutorMessage | None:
+        """The next message, or ``None`` when the stream ends cleanly between frames."""
+        if self._fault is not None:
+            raise ExecutorProtocolException(
+                self._fault.error, "The channel is closed after an earlier protocol violation."
+            )
+
+        header = self._read_exactly(FRAME_HEADER_BYTES, allow_empty=True)
+
+        if header is None:
+            return None
+
+        length = _HEADER.unpack_from(header)[0]
+
+        if length == 0:
+            raise self._faulted(ExecutorProtocolError.INVALID_LENGTH, "A frame declared an empty payload.")
+
+        if length > self._maximum:
+            raise self._faulted(
+                ExecutorProtocolError.PAYLOAD_TOO_LARGE,
+                f"A frame declared a {length}-byte payload; the limit is {self._maximum} bytes.",
+            )
+
+        payload = self._read_exactly(length, allow_empty=False)
+        assert payload is not None
+
+        expected = compute_tag(self._key, self._direction, self._sequence, payload)
+
+        if not hmac.compare_digest(expected, header[FRAME_LENGTH_BYTES : FRAME_LENGTH_BYTES + FRAME_TAG_BYTES]):
+            raise self._faulted(ExecutorProtocolError.AUTHENTICATION_FAILED, "A frame failed authentication.")
+
+        self._sequence += 1
+
+        try:
+            return decode_message(payload)
+        except ExecutorProtocolException as exception:
+            self._fault = exception
+            raise
+
+    def _read_exactly(self, count: int, *, allow_empty: bool) -> bytes | None:
+        buffer = bytearray()
+
+        while len(buffer) < count:
+            chunk = self._stream.read(count - len(buffer))
+
+            if not chunk:
+                if not buffer and allow_empty:
+                    return None
+
+                where = "header" if allow_empty else "payload"
+                raise self._faulted(
+                    ExecutorProtocolError.TRUNCATED_FRAME,
+                    f"The stream ended inside a frame {where}.",
+                )
+
+            buffer.extend(chunk)
+
+        return bytes(buffer)
+
+    def _faulted(self, error: str, message: str) -> ExecutorProtocolException:
+        self._fault = ExecutorProtocolException(error, message)
+        return self._fault

velophex/worker/loader.py

@@ -0,0 +1,105 @@
+"""Finding the handler a ``start`` message names.
+
+``start`` carries a package root and an entry point id; ``project.json`` at the root of that
+package turns the id into a ``module:function`` handler (OD-11), and this module imports it.
+
+The package's own sources are added to ``sys.path`` so ``invoices.main:run`` resolves whether
+the project uses a flat layout or ``src/``. Nothing is executed until the module is imported,
+which is the first moment automation code runs in the process - after the session key has
+already been taken out of the environment.
+"""
+
+from __future__ import annotations
+
+import importlib
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from velophex.contracts.diagnostics import has_errors
+from velophex.contracts.project import EntryPoint, ProjectFiles, ProjectManifest, read_project_manifest
+from velophex.entrypoints import EntryPointDefinition, entrypoint_definition, is_entrypoint
+
+__all__ = ["LoadedEntryPoint", "PackageLoadError", "load_entry_point", "read_project"]
+
+
+class PackageLoadError(Exception):
+    """The package, its ``project.json`` or the handler module could not be loaded."""
+
+
+@dataclass(frozen=True, slots=True)
+class LoadedEntryPoint:
+    """A handler resolved from the package, with the manifest it came from."""
+
+    project: ProjectManifest
+    entry_point: EntryPoint
+    definition: EntryPointDefinition
+
+
+def read_project(package_root: str | Path) -> ProjectManifest:
+    """Read and validate ``project.json`` at *package_root*."""
+    path = Path(package_root) / ProjectFiles.MANIFEST_FILE_NAME
+
+    if not path.is_file():
+        raise PackageLoadError(f"'{package_root}' has no {ProjectFiles.MANIFEST_FILE_NAME}.")
+
+    manifest, diagnostics = read_project_manifest(path.read_bytes())
+
+    if manifest is None or has_errors(diagnostics):
+        reasons = "; ".join(str(diagnostic) for diagnostic in diagnostics if diagnostic.severity.value == "Error")
+        raise PackageLoadError(f"{path} is not a valid project manifest: {reasons}")
+
+    return manifest
+
+
+def load_entry_point(package_root: str | Path, entry_point_id: str | None) -> LoadedEntryPoint:
+    """Resolve *entry_point_id* to an imported, decorated handler."""
+    root = Path(package_root)
+    project = read_project(root)
+    entry = project.find_entry_point(entry_point_id)
+
+    if entry is None:
+        known = ", ".join(item.id for item in project.entry_points) or "none"
+        raise PackageLoadError(f"'{entry_point_id}' is not an entry point of '{project.name}' (it has: {known}).")
+
+    if not entry.handler:
+        raise PackageLoadError(
+            f"Entry point '{entry.id}' has no handler; a Python entry point needs 'module:function'."
+        )
+
+    _extend_path(root, entry.path)
+
+    module_name, _, function_name = entry.handler.partition(":")
+
+    try:
+        module = importlib.import_module(module_name)
+    except ImportError as error:
+        raise PackageLoadError(f"The handler module '{module_name}' could not be imported: {error}") from error
+
+    function = getattr(module, function_name, None)
+
+    if function is None or not callable(function):
+        raise PackageLoadError(f"'{module_name}' has no callable '{function_name}'.")
+
+    if not is_entrypoint(function):
+        raise PackageLoadError(
+            f"'{entry.handler}' is not decorated with @velophex.entrypoint, so it has no input or output schema."
+        )
+
+    return LoadedEntryPoint(project=project, entry_point=entry, definition=entrypoint_definition(function))
+
+
+def _extend_path(root: Path, entry_path: str) -> None:
+    """Put the package's import roots first on ``sys.path``.
+
+    The project root covers a flat layout; ``src/`` covers the layout ``velophex init``
+    creates. The directory holding the entry point file is added last so a single-file
+    automation also imports.
+    """
+    candidates = [root, root / "src", (root / entry_path).parent]
+
+    for candidate in candidates:
+        resolved = str(candidate.resolve())
+
+        if candidate.is_dir() and resolved not in sys.path:
+            sys.path.insert(0, resolved)

velophex/worker/outbox.py

@@ -0,0 +1,147 @@
+"""Everything the worker sends to its parent, and the bridge from :mod:`logging`.
+
+Sending never raises: a parent that has gone away must not fail the automation, and the
+process ends when its stdin closes. The log sequence is monotonic per process, which is what
+the Orchestrator de-duplicates a retried log batch on.
+
+Automation code is expected to use ``logging``; the handler installed here turns each record
+into a ``log`` frame so ``logging.info("...")`` lands on the job's log without the author
+learning a VeloPhex API. ``print`` goes to stderr instead, because stdout is the protocol.
+"""
+
+from __future__ import annotations
+
+import itertools
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from velophex.contracts.executor import ExecutorMessage, ExecutorProtocolException, LogMessage, ProgressMessage
+from velophex.contracts.runtimes import ExecutionPhases, LogLevels
+from velophex.worker.framing import FrameWriter
+
+__all__ = ["ExecutorOutbox", "WorkerLogHandler", "level_name"]
+
+_LEVELS: tuple[tuple[int, str], ...] = (
+    (logging.CRITICAL, LogLevels.CRITICAL),
+    (logging.ERROR, LogLevels.ERROR),
+    (logging.WARNING, LogLevels.WARNING),
+    (logging.INFO, LogLevels.INFORMATION),
+    (logging.DEBUG, LogLevels.DEBUG),
+)
+
+
+def level_name(level: int) -> str:
+    """Map a :mod:`logging` level onto the platform's log levels."""
+    for threshold, name in _LEVELS:
+        if level >= threshold:
+            return name
+
+    return LogLevels.TRACE
+
+
+class ExecutorOutbox:
+    """Writes messages to the parent, numbering log lines as it goes."""
+
+    def __init__(self, writer: FrameWriter) -> None:
+        self._writer = writer
+        self._sequence = itertools.count(1)
+        self.disconnected = False
+
+    @property
+    def maximum_payload_bytes(self) -> int:
+        """The frame limit of the underlying writer."""
+        return self._writer.maximum_payload_bytes
+
+    def encode(self, message: ExecutorMessage) -> bytes:
+        """The payload of *message*; raises like :meth:`FrameWriter.encode`."""
+        return self._writer.encode(message)
+
+    def send(self, message: ExecutorMessage) -> None:
+        """Send *message*, ignoring a parent that has already gone.
+
+        A message that cannot be encoded (a value with no JSON form, or one over the frame
+        limit) is dropped rather than raised: that is a property of the message, not of the
+        channel, so it must not mark the parent as gone and swallow everything after it. The
+        terminal message is checked before it gets here (see ``WorkerSession``), so what can
+        be dropped is telemetry.
+        """
+        if self.disconnected:
+            return
+
+        try:
+            payload = self._writer.encode(message)
+        except (TypeError, ValueError, ExecutorProtocolException):
+            return
+
+        self.send_payload(payload)
+
+    def send_payload(self, payload: bytes) -> None:
+        """Send a payload :meth:`encode` produced, ignoring a parent that has already gone."""
+        if self.disconnected:
+            return
+
+        try:
+            self._writer.write_payload(payload)
+        except (OSError, ValueError):
+            # ValueError is what a closed file raises on write.
+            self.disconnected = True
+
+    def log(
+        self,
+        level: str,
+        message: str,
+        phase: str | None = ExecutionPhases.EXECUTION,
+        sub_phase: str | None = None,
+        details: Any | None = None,
+    ) -> None:
+        """Send one log line."""
+        self.send(
+            LogMessage(
+                sequence=next(self._sequence),
+                timestamp=datetime.now(UTC),
+                level=level,
+                message=message,
+                phase=phase,
+                sub_phase=sub_phase,
+                details=details,
+            )
+        )
+
+    def progress(self, node_id: str, state: str, display_name: str | None = None) -> None:
+        """Report that a step started, completed, faulted or was canceled."""
+        self.send(
+            ProgressMessage(
+                node_id=node_id,
+                state=state,
+                timestamp=datetime.now(UTC),
+                display_name=display_name,
+            )
+        )
+
+
+class WorkerLogHandler(logging.Handler):
+    """A :mod:`logging` handler that forwards records to the parent as ``log`` messages."""
+
+    def __init__(self, outbox: ExecutorOutbox, phase: str = ExecutionPhases.EXECUTION) -> None:
+        super().__init__(level=logging.NOTSET)
+        self._outbox = outbox
+        self._phase = phase
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Forward one record. A failure here must never escape into automation code."""
+        try:
+            details: dict[str, Any] | None = None
+
+            if record.exc_info:
+                details = {"exception": self.format(record)}
+
+            self._outbox.log(
+                level_name(record.levelno),
+                record.getMessage(),
+                phase=self._phase,
+                sub_phase=record.name if record.name != "root" else None,
+                details=details,
+            )
+        except Exception:
+            self.handleError(record)