pycforge 0.15.2__py3-none-any.whl

pycforge/ide/_worker_event_codec.py

@@ -0,0 +1,276 @@
+"""Progress, terminal-result, and worker-failure event codec."""
+
+from __future__ import annotations
+
+from pycforge.converter.core.progress import ConversionProgress
+from pycforge.converter.core.result import ConversionResult
+
+from ._worker_protocol_json import (
+    _contains_surrogate,
+    _decode_bounded,
+    _encode_bounded,
+    _exact_fields,
+    _generation,
+    _object,
+    _receive_frame,
+    _send_frame,
+    _sha256,
+)
+from ._worker_protocol_types import (
+    FAILURE_CLASSIFICATIONS,
+    MAX_EVENT_BYTES,
+    MAX_FAILURE_MESSAGE_CHARS,
+    PROTOCOL_SCHEMA,
+    ByteConnection,
+    WorkerEvent,
+    WorkerProtocolError,
+    WorkerRequest,
+)
+from ._worker_request_codec import _validate_worker_request
+from ._worker_result_data import (
+    _mappings_from_data,
+    _progress_from_data,
+    _result_from_data,
+    _result_to_data,
+    _validate_result_mappings,
+)
+
+
+_ENVELOPE_FIELDS = frozenset(
+    {
+        "schema",
+        "kind",
+        "generation",
+        "bundle_fingerprint",
+        "transport_fingerprint",
+        "progress",
+        "result",
+        "mappings",
+        "failure_classification",
+        "message",
+    }
+)
+
+
+def encode_progress(request: WorkerRequest, progress: ConversionProgress) -> bytes:
+    _validate_worker_request(request)
+    if not isinstance(progress, ConversionProgress):
+        raise WorkerProtocolError("progress event must be ConversionProgress")
+    return _encode_event(
+        request,
+        "progress",
+        progress={
+            "state": progress.state,
+            "stage_id": progress.stage_id,
+            "completed_stages": progress.completed_stages,
+            "total_stages": progress.total_stages,
+        },
+    )
+
+
+def encode_terminal(request: WorkerRequest, result: ConversionResult) -> bytes:
+    _validate_worker_request(request)
+    if not isinstance(result, ConversionResult):
+        raise WorkerProtocolError("terminal event must contain ConversionResult")
+    artifact = result.stage_artifact
+    mappings: list[object] = []
+    if artifact is not None:
+        raw_mappings = artifact.payload.get("source_output_mappings", ())
+        if not isinstance(raw_mappings, (tuple, list)):
+            raise WorkerProtocolError("terminal source/output mappings are malformed")
+        mappings = list(raw_mappings)
+    return _encode_event(
+        request,
+        "terminal",
+        result=_result_to_data(result),
+        mappings=mappings,
+    )
+
+
+def encode_failure(
+    request: WorkerRequest | None,
+    failure_classification: str,
+    message: str,
+) -> bytes:
+    if failure_classification not in FAILURE_CLASSIFICATIONS:
+        raise WorkerProtocolError("unknown worker failure classification")
+    if (
+        not isinstance(message, str)
+        or not message
+        or len(message) > MAX_FAILURE_MESSAGE_CHARS
+        or _contains_surrogate(message)
+    ):
+        raise WorkerProtocolError("worker failure message is malformed")
+    return _encode_event(
+        request,
+        "failure",
+        failure_classification=failure_classification,
+        message=message,
+    )
+
+
+def decode_event(
+    frame: bytes,
+    *,
+    expected: WorkerRequest | None = None,
+    expected_generation: int | None = None,
+    expected_bundle_fingerprint: str | None = None,
+    expected_transport_fingerprint: str | None = None,
+) -> WorkerEvent:
+    value = _decode_bounded(frame, MAX_EVENT_BYTES, "event")
+    data = _object(value, "event envelope")
+    _exact_fields(data, _ENVELOPE_FIELDS, "event envelope")
+    if data["schema"] != PROTOCOL_SCHEMA:
+        raise WorkerProtocolError("incompatible worker event schema")
+    kind = data["kind"]
+    if kind not in {"progress", "terminal", "failure"}:
+        raise WorkerProtocolError("unknown worker event kind")
+
+    identities = (
+        data["generation"],
+        data["bundle_fingerprint"],
+        data["transport_fingerprint"],
+    )
+    if all(item is None for item in identities):
+        if kind != "failure":
+            raise WorkerProtocolError("non-failure event lacks request identity")
+        generation = None
+        bundle_fingerprint = None
+        transport_fingerprint = None
+    elif any(item is None for item in identities):
+        raise WorkerProtocolError("worker event has partial request identity")
+    else:
+        generation = _generation(data["generation"])
+        bundle_fingerprint = _sha256(
+            data["bundle_fingerprint"], "bundle fingerprint"
+        )
+        transport_fingerprint = _sha256(
+            data["transport_fingerprint"], "transport fingerprint"
+        )
+
+    if expected is not None:
+        _validate_worker_request(expected)
+        expected_generation = expected.generation
+        expected_bundle_fingerprint = expected.bundle_fingerprint
+        expected_transport_fingerprint = expected.transport_fingerprint
+    if generation is not None:
+        if (
+            expected_generation is not None
+            and generation != _generation(expected_generation)
+        ):
+            raise WorkerProtocolError("worker event generation mismatch")
+        if (
+            expected_bundle_fingerprint is not None
+            and bundle_fingerprint
+            != _sha256(expected_bundle_fingerprint, "expected bundle fingerprint")
+        ):
+            raise WorkerProtocolError("worker event bundle fingerprint mismatch")
+        if (
+            expected_transport_fingerprint is not None
+            and transport_fingerprint
+            != _sha256(
+                expected_transport_fingerprint,
+                "expected transport fingerprint",
+            )
+        ):
+            raise WorkerProtocolError("worker event transport fingerprint mismatch")
+
+    progress: ConversionProgress | None = None
+    result: ConversionResult | None = None
+    mappings: tuple[dict[str, object], ...] = ()
+    failure_classification: str | None = None
+    message: str | None = None
+
+    if kind == "progress":
+        if any(
+            data[key] is not None
+            for key in ("result", "failure_classification", "message")
+        ) or data["mappings"] != []:
+            raise WorkerProtocolError("progress event contains terminal fields")
+        progress = _progress_from_data(data["progress"])
+    elif kind == "terminal":
+        if (
+            data["progress"] is not None
+            or data["failure_classification"] is not None
+            or data["message"] is not None
+        ):
+            raise WorkerProtocolError("terminal event contains non-terminal fields")
+        result = _result_from_data(data["result"])
+        mappings = _mappings_from_data(data["mappings"])
+        _validate_result_mappings(result, mappings)
+    else:
+        if (
+            data["progress"] is not None
+            or data["result"] is not None
+            or data["mappings"] != []
+        ):
+            raise WorkerProtocolError("failure event contains conversion fields")
+        failure_classification = data["failure_classification"]
+        message = data["message"]
+        if failure_classification not in FAILURE_CLASSIFICATIONS:
+            raise WorkerProtocolError("unknown worker failure classification")
+        if (
+            not isinstance(message, str)
+            or not message
+            or len(message) > MAX_FAILURE_MESSAGE_CHARS
+            or _contains_surrogate(message)
+        ):
+            raise WorkerProtocolError("worker failure message is malformed")
+
+    return WorkerEvent(
+        kind,
+        generation,
+        bundle_fingerprint,
+        transport_fingerprint,
+        progress,
+        result,
+        mappings,
+        failure_classification,
+        message,
+    )
+
+
+def send_event(connection: ByteConnection, frame: bytes) -> None:
+    _send_frame(connection, frame, MAX_EVENT_BYTES, "event")
+
+
+def receive_event(
+    connection: ByteConnection,
+    *,
+    expected: WorkerRequest | None = None,
+) -> WorkerEvent:
+    return decode_event(
+        _receive_frame(connection, MAX_EVENT_BYTES, "event"),
+        expected=expected,
+    )
+
+
+def _encode_event(
+    request: WorkerRequest | None,
+    kind: str,
+    *,
+    progress: object = None,
+    result: object = None,
+    mappings: list[object] | None = None,
+    failure_classification: str | None = None,
+    message: str | None = None,
+) -> bytes:
+    if request is not None:
+        _validate_worker_request(request)
+    data = {
+        "schema": PROTOCOL_SCHEMA,
+        "kind": kind,
+        "generation": None if request is None else request.generation,
+        "bundle_fingerprint": (
+            None if request is None else request.bundle_fingerprint
+        ),
+        "transport_fingerprint": (
+            None if request is None else request.transport_fingerprint
+        ),
+        "progress": progress,
+        "result": result,
+        "mappings": [] if mappings is None else mappings,
+        "failure_classification": failure_classification,
+        "message": message,
+    }
+    return _encode_bounded(data, MAX_EVENT_BYTES, "event")

pycforge/ide/_worker_protocol_json.py

@@ -0,0 +1,197 @@
+"""Strict canonical-JSON parsing and byte-only connection framing."""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from typing import Any
+
+from pycforge.converter.core.fingerprint import canonical_json
+
+from ._worker_protocol_types import (
+    MAX_JSON_DEPTH,
+    ByteConnection,
+    WorkerProtocolError,
+)
+
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+
+def _send_frame(
+    connection: ByteConnection,
+    frame: bytes,
+    maximum: int,
+    role: str,
+) -> None:
+    if not isinstance(frame, bytes) or len(frame) > maximum:
+        raise WorkerProtocolError(f"worker {role} frame exceeds protocol bounds")
+    connection.send_bytes(frame)
+
+
+def _receive_frame(
+    connection: ByteConnection,
+    maximum: int,
+    role: str,
+) -> bytes:
+    try:
+        frame = connection.recv_bytes(maximum)
+    except OSError as exc:
+        raise WorkerProtocolError(
+            f"worker {role} frame exceeds protocol bounds or is unreadable"
+        ) from exc
+    if not isinstance(frame, bytes) or len(frame) > maximum:
+        raise WorkerProtocolError(f"worker {role} frame exceeds protocol bounds")
+    return frame
+
+
+def _encode_bounded(value: object, maximum: int, role: str) -> bytes:
+    try:
+        frame = canonical_json(value)
+    except (TypeError, ValueError, UnicodeError) as exc:
+        raise WorkerProtocolError(f"worker {role} is not canonical JSON") from exc
+    if len(frame) > maximum:
+        raise WorkerProtocolError(f"worker {role} frame exceeds protocol bounds")
+    return frame
+
+
+def _decode_bounded(frame: bytes, maximum: int, role: str) -> object:
+    if not isinstance(frame, bytes) or len(frame) > maximum:
+        raise WorkerProtocolError(f"worker {role} frame exceeds protocol bounds")
+    try:
+        text = frame.decode("utf-8")
+        value = json.loads(
+            text,
+            object_pairs_hook=_unique_object,
+            parse_constant=_reject_constant,
+            parse_int=_bounded_json_int,
+            parse_float=_finite_json_float,
+        )
+        _validate_json_tree(value)
+        if canonical_json(value) != frame:
+            raise WorkerProtocolError(
+                f"worker {role} frame is not canonical JSON"
+            )
+    except WorkerProtocolError:
+        raise
+    except (UnicodeError, json.JSONDecodeError, TypeError, ValueError) as exc:
+        raise WorkerProtocolError(f"worker {role} frame is not valid JSON") from exc
+    return value
+
+
+def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
+    result: dict[str, object] = {}
+    for key, value in pairs:
+        if key in result:
+            raise WorkerProtocolError("worker JSON contains duplicate object keys")
+        result[key] = value
+    return result
+
+
+def _reject_constant(value: str) -> object:
+    raise WorkerProtocolError(f"worker JSON contains forbidden constant {value}")
+
+
+def _bounded_json_int(value: str) -> int:
+    if len(value.lstrip("-")) > 19:
+        raise WorkerProtocolError("worker JSON integer exceeds protocol bounds")
+    return int(value)
+
+
+def _finite_json_float(value: str) -> float:
+    result = float(value)
+    if not math.isfinite(result):
+        raise WorkerProtocolError("worker JSON number must be finite")
+    return result
+
+
+def _validate_json_tree(value: object, depth: int = 0) -> None:
+    if depth > MAX_JSON_DEPTH:
+        raise WorkerProtocolError("worker JSON nesting exceeds protocol bounds")
+    if isinstance(value, str):
+        if _contains_surrogate(value):
+            raise WorkerProtocolError("worker JSON contains invalid Unicode")
+        return
+    if value is None or isinstance(value, (bool, int)):
+        return
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise WorkerProtocolError("worker JSON number must be finite")
+        return
+    if isinstance(value, list):
+        for item in value:
+            _validate_json_tree(item, depth + 1)
+        return
+    if isinstance(value, dict):
+        for key, item in value.items():
+            if not isinstance(key, str) or _contains_surrogate(key):
+                raise WorkerProtocolError("worker JSON object key is invalid")
+            _validate_json_tree(item, depth + 1)
+        return
+    raise WorkerProtocolError("worker JSON contains unsupported value")
+
+
+def _contains_surrogate(value: str) -> bool:
+    return any(0xD800 <= ord(character) <= 0xDFFF for character in value)
+
+
+def _object(value: object, role: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise WorkerProtocolError(f"{role} must be an object")
+    return value
+
+
+def _optional_object(value: object, role: str) -> dict[str, Any] | None:
+    return None if value is None else _object(value, role)
+
+
+def _list(value: object, role: str) -> list[Any]:
+    if not isinstance(value, list):
+        raise WorkerProtocolError(f"{role} must be an array")
+    return value
+
+
+def _exact_fields(
+    value: dict[str, Any],
+    expected: set[str] | frozenset[str],
+    role: str,
+) -> None:
+    if set(value) != set(expected):
+        raise WorkerProtocolError(f"{role} fields are incomplete or unknown")
+
+
+def _string(value: object, role: str) -> str:
+    if not isinstance(value, str) or _contains_surrogate(value):
+        raise WorkerProtocolError(f"{role} must be valid Unicode text")
+    return value
+
+
+def _nonempty_string(value: object, role: str) -> str:
+    result = _string(value, role)
+    if not result:
+        raise WorkerProtocolError(f"{role} must be non-empty")
+    return result
+
+
+def _string_list(value: object, role: str) -> tuple[str, ...]:
+    return tuple(_string(item, role) for item in _list(value, role))
+
+
+def _nonnegative_integer(value: object, role: str) -> int:
+    if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+        raise WorkerProtocolError(f"{role} must be a non-negative integer")
+    return value
+
+
+def _generation(value: object) -> int:
+    result = _nonnegative_integer(value, "worker generation")
+    if result == 0 or result > 2**63 - 1:
+        raise WorkerProtocolError("worker generation is outside protocol bounds")
+    return result
+
+
+def _sha256(value: object, role: str) -> str:
+    if not isinstance(value, str) or not _SHA256.fullmatch(value):
+        raise WorkerProtocolError(f"{role} must be a lowercase SHA-256 value")
+    return value

pycforge/ide/_worker_protocol_types.py

@@ -0,0 +1,90 @@
+"""Shared immutable types and bounds for the worker protocol."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from pycforge.converter.core.progress import ConversionProgress
+from pycforge.converter.core.request import ConversionRequest, ObservationOptions
+from pycforge.converter.core.result import ConversionResult
+
+
+PROTOCOL_SCHEMA = "pycforge.worker-protocol/0.1"
+MAX_REQUEST_BYTES = 8 * 1024 * 1024
+MAX_CONTROL_BYTES = 4 * 1024
+MAX_EVENT_BYTES = 128 * 1024 * 1024
+MAX_SOURCE_BYTES = 1_000_000
+MAX_SOURCE_DOCUMENTS = 64
+MAX_JSON_DEPTH = 256
+MAX_DIAGNOSTICS = 1_000
+MAX_FAILURE_MESSAGE_CHARS = 1_000
+
+FAILURE_CLASSIFICATIONS = frozenset(
+    {
+        "protocol-error",
+        "request-transport-error",
+        "worker-resource-exhaustion",
+        "worker-internal-error",
+    }
+)
+
+
+class WorkerProtocolError(ValueError):
+    """A frame is malformed, incompatible, oversized, or fails custody checks."""
+
+
+class ByteConnection(Protocol):
+    """The deliberately narrow subset of a multiprocessing connection we use."""
+
+    def send_bytes(self, buffer: bytes) -> None: ...
+
+    def recv_bytes(self, maxlength: int | None = None) -> bytes: ...
+
+    def close(self) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class WorkerRequest:
+    generation: int
+    bundle_fingerprint: str
+    request: ConversionRequest
+    observation: ObservationOptions
+    transport_fingerprint: str
+
+    @classmethod
+    def create(
+        cls,
+        generation: int,
+        bundle_fingerprint: str,
+        request: ConversionRequest,
+        observation: ObservationOptions | None = None,
+    ) -> "WorkerRequest":
+        from ._worker_request_codec import create_worker_request
+
+        return create_worker_request(
+            generation,
+            bundle_fingerprint,
+            request,
+            observation,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class WorkerControl:
+    generation: int
+    bundle_fingerprint: str
+    transport_fingerprint: str
+
+
+@dataclass(frozen=True, slots=True)
+class WorkerEvent:
+    kind: str
+    generation: int | None
+    bundle_fingerprint: str | None
+    transport_fingerprint: str | None
+    progress: ConversionProgress | None = None
+    result: ConversionResult | None = None
+    mappings: tuple[dict[str, Any], ...] = ()
+    failure_classification: str | None = None
+    message: str | None = None