terrine 0.1.0.dev0__py3-none-any.whl

terrine/runtime/changes.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Literal
+
+__all__ = ["Change", "ChangeSet", "affected_by_change", "window_intersects"]
+
+ChangeKind = Literal[
+    "insert", "update", "delete", "restatement", "readiness", "definition", "roster"
+]
+ChangeEvidence = Literal["revision", "change_feed", "checksum", "operator", "conservative_rebuild"]
+TimeRange = tuple[datetime, datetime]
+KeyRange = tuple[str, str]
+
+
+@dataclass(frozen=True, slots=True)
+class Change:
+    kind: ChangeKind
+    source: str
+    before_key_range: KeyRange | None = None
+    after_key_range: KeyRange | None = None
+    before_time_range: TimeRange | None = None
+    after_time_range: TimeRange | None = None
+    revision: str | None = None
+    tombstone: bool = False
+
+    def __post_init__(self) -> None:
+        if not self.source:
+            raise ValueError("change source must not be empty")
+
+
+@dataclass(frozen=True, slots=True)
+class ChangeSet:
+    changes: tuple[Change, ...]
+    evidence: ChangeEvidence
+
+    @classmethod
+    def from_readiness_advance(cls, source: str, before: datetime, after: datetime) -> ChangeSet:
+        if after == before:
+            return cls((), "revision")
+        if after < before:
+            return cls(
+                (Change("readiness", source, before_time_range=(after, before)),), "revision"
+            )
+        # Revalidate completeness-dependent outputs, without asserting row mutation.
+        return cls((Change("readiness", source, after_time_range=(before, after)),), "revision")
+
+
+def window_intersects(support: TimeRange, window: TimeRange) -> bool:
+    left, right = support
+    start, end = window
+    if left.tzinfo is None or right.tzinfo is None or start.tzinfo is None or end.tzinfo is None:
+        raise ValueError("time supports must be timezone-aware")
+    if right < left or end < start:
+        raise ValueError("time ranges must be ordered")
+    return left < end and start < right
+
+
+def affected_by_change(change: Change, window: TimeRange) -> bool:
+    supports = (change.before_time_range, change.after_time_range)
+    if any(support is None for support in supports if change.kind == "update"):
+        return True
+    if change.kind == "delete":
+        if change.before_time_range is None:
+            return True
+        return window_intersects(change.before_time_range, window)
+    exact = tuple(support for support in supports if support is not None)
+    if not exact:
+        return True
+    return any(window_intersects(support, window) for support in exact)

terrine/runtime/cli.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import asdict, is_dataclass
+from pathlib import Path
+from typing import Any
+
+from terrine.errors import TerrineError
+from terrine.runtime.control import ControlClient, ControlEndpoint
+from terrine.runtime.saved_plan import DestinationRef
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="terrine")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    plan = commands.add_parser("plan", help="validate and canonicalize a saved refresh plan")
+    plan.add_argument("plan_file", type=Path)
+
+    run = commands.add_parser("run", help="execute a saved refresh plan through its owner")
+    run.add_argument("plan_file", type=Path, nargs="?")
+    run.add_argument("--request-key")
+    run.add_argument("--lookup-request-key")
+    run.add_argument("--destination-database")
+    run.add_argument("--destination-schema")
+    run.add_argument("--destination-dataset")
+    status = commands.add_parser("status", help="read an owner's durable run status")
+    status.add_argument("run_id")
+
+    events = commands.add_parser("events", help="read an owner's bounded run event page")
+    events.add_argument("run_id")
+    events.add_argument("--after")
+    events.add_argument("--limit", type=int, default=100)
+
+    cancel = commands.add_parser("cancel", help="request cancellation through the owner")
+    cancel.add_argument("run_id")
+    return parser
+
+
+def _json(value: Any) -> str:
+    if is_dataclass(value):
+        value = asdict(value)
+    return json.dumps(value, sort_keys=True, default=str)
+
+
+def _client() -> ControlClient:
+    endpoint = os.environ.get("TERRINE_RUNTIME_SOCKET")
+    namespace = os.environ.get("TERRINE_RUNTIME_NAMESPACE")
+    if not endpoint or not namespace:
+        raise ValueError("TERRINE_RUNTIME_SOCKET and TERRINE_RUNTIME_NAMESPACE are required")
+    return ControlClient(ControlEndpoint(Path(endpoint)), namespace=namespace)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    try:
+        client = _client()
+        if args.command == "plan":
+            serialized = client.plan(args.plan_file.read_bytes())
+            print(serialized)
+        elif args.command == "run":
+            destination_values = (
+                args.destination_database,
+                args.destination_schema,
+                args.destination_dataset,
+            )
+            if args.lookup_request_key and args.request_key:
+                raise ValueError("lookup and execution request keys are mutually exclusive")
+            if (args.request_key or args.lookup_request_key) and not all(destination_values):
+                raise ValueError("a request key requires all destination fields")
+            if any(destination_values) and not all(destination_values):
+                raise ValueError("all destination fields must be supplied together")
+            destination = DestinationRef(*destination_values) if all(destination_values) else None
+            if args.lookup_request_key:
+                if args.plan_file is not None:
+                    raise ValueError("request-key lookup does not accept a plan file")
+                receipt = client.find_run_by_request_key(
+                    args.lookup_request_key, destination=destination
+                )
+                print(_json(receipt))
+            else:
+                if args.plan_file is None:
+                    raise ValueError("run requires a plan file")
+                receipt = client.run(
+                    args.plan_file.read_bytes(),
+                    request_key=args.request_key,
+                    destination=destination,
+                )
+                print(_json(receipt))
+        elif args.command == "status":
+            print(_json(client.get_run(args.run_id)))
+        elif args.command == "events":
+            page = client.read_run_events(args.run_id, after=args.after, limit=args.limit)
+            print(_json(page))
+        else:
+            print(_json(client.cancel_run(args.run_id)))
+    except (OSError, ValueError, TerrineError) as error:
+        code = getattr(error, "code", "terrine.runtime.cli_error")
+        print(json.dumps({"code": code}, sort_keys=True), file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

terrine/runtime/control.py

@@ -0,0 +1,355 @@
+from __future__ import annotations
+
+import errno
+import json
+import os
+import socket
+import socketserver
+import stat
+import threading
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, overload
+
+if TYPE_CHECKING:
+    from terrine.runtime.planning import PlanInvocation, PlanRequest
+
+from terrine.errors import RefusalSpec, TerrineError
+from terrine.runtime.events import (
+    EventPage,
+    RunEvent,
+    read_run_events,
+)
+from terrine.runtime.execute import RunReceipt, _ensure_run_tables
+from terrine.runtime.state import RuntimeStore
+from terrine.runtime.status import (
+    CancelReceipt,
+    RunStatus,
+    _recover_interrupted_runs,
+    cancel_run,
+    get_run,
+)
+
+__all__ = ["ControlEndpoint", "ControlServer", "ControlClient"]
+
+_MAX_CONTROL_MESSAGE_BYTES = 16_777_216
+_MESSAGE_TOO_LARGE = RefusalSpec(
+    "terrine.runtime.control_message_too_large",
+    "Owner control message exceeds the configured limit.",
+)
+_CONTROL_UNAVAILABLE = RefusalSpec(
+    "terrine.runtime.control_unavailable", "Owner control request was refused."
+)
+
+
+@dataclass(frozen=True, slots=True)
+class ControlEndpoint:
+    path: Path
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "path", Path(os.path.abspath(Path(self.path).expanduser())))
+
+
+class ControlServer:
+    """Owner-local Unix socket that serves control requests on the owner's connection."""
+
+    def __init__(
+        self,
+        store: RuntimeStore,
+        *,
+        namespace: str,
+        endpoint: ControlEndpoint,
+        executor: Any | None = None,
+        plan_planner: Any | None = None,
+        resolver: Any | None = None,
+        snapshot_authority: Any | None = None,
+    ) -> None:
+        if store.namespace != namespace:
+            raise ValueError("control namespace does not match runtime store")
+        self.store = store
+        self.namespace = namespace
+        self.endpoint = endpoint
+        self.executor = executor
+        self.plan_planner = plan_planner
+        self.resolver = resolver
+        self.snapshot_authority = snapshot_authority
+        self._server: socketserver.UnixStreamServer | None = None
+        self._thread: threading.Thread | None = None
+        self._endpoint_identity: tuple[int, int] | None = None
+
+    def start(self) -> None:
+        if self._server is not None:
+            return
+        self.endpoint.path.parent.mkdir(parents=True, exist_ok=True)
+        self.store.initialize()
+        _ensure_run_tables(self.store.con, self.store._schema)
+        _recover_interrupted_runs(self.store.con, namespace=self.namespace)
+        server = self
+
+        class Handler(socketserver.StreamRequestHandler):
+            def handle(self) -> None:
+                self.connection.settimeout(2)
+                line = self.rfile.readline(_MAX_CONTROL_MESSAGE_BYTES + 2)
+                if not line:
+                    return
+                if len(line) > _MAX_CONTROL_MESSAGE_BYTES + 1:
+                    response = json.dumps({"ok": False, "code": _MESSAGE_TOO_LARGE.code}).encode()
+                    self.wfile.write(response + b"\n")
+                    return
+                con = server.store.con.cursor()
+                try:
+                    request = json.loads(line)
+                    self.connection.settimeout(
+                        None if request.get("operation") in {"cancel", "run", "plan"} else 2
+                    )
+                    response = server._dispatch(request, con=con)
+                    response_bytes = json.dumps(
+                        {"ok": True, "value": response}, default=str
+                    ).encode()
+                    if len(response_bytes) > _MAX_CONTROL_MESSAGE_BYTES:
+                        response_bytes = json.dumps(
+                            {"ok": False, "code": _MESSAGE_TOO_LARGE.code}
+                        ).encode()
+                    self.wfile.write(response_bytes + b"\n")
+                except Exception as error:
+                    code = getattr(error, "code", _CONTROL_UNAVAILABLE.code)
+                    self.wfile.write(json.dumps({"ok": False, "code": code}).encode() + b"\n")
+                finally:
+                    con.close()
+
+        class OwnerUnixStreamServer(socketserver.ThreadingUnixStreamServer):
+            daemon_threads = True
+
+        try:
+            mode = self.endpoint.path.lstat().st_mode
+        except FileNotFoundError:
+            mode = None
+        if mode is not None:
+            if not stat.S_ISSOCK(mode):
+                raise FileExistsError("control endpoint exists and is not a socket")
+            probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            try:
+                probe.connect(str(self.endpoint.path))
+            except OSError as error:
+                if error.errno != errno.ECONNREFUSED:
+                    raise
+                self.endpoint.path.unlink()
+            else:
+                raise FileExistsError("control endpoint is already active")
+            finally:
+                probe.close()
+        self._server = OwnerUnixStreamServer(str(self.endpoint.path), Handler)
+        os.chmod(self.endpoint.path, 0o600)
+        socket_stat = self.endpoint.path.stat()
+        self._endpoint_identity = (socket_stat.st_dev, socket_stat.st_ino)
+        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+        self._thread.start()
+
+    def close(self) -> None:
+        if self._server is None:
+            return
+        server, self._server = self._server, None
+        server.shutdown()
+        server.server_close()
+        if self._thread is not None:
+            self._thread.join()
+            self._thread = None
+        try:
+            current = self.endpoint.path.lstat()
+        except FileNotFoundError:
+            current = None
+        if (
+            current is not None
+            and self._endpoint_identity == (current.st_dev, current.st_ino)
+            and stat.S_ISSOCK(current.st_mode)
+        ):
+            self.endpoint.path.unlink()
+        self._endpoint_identity = None
+
+    def _dispatch(self, request: dict[str, Any], *, con: Any) -> Any:
+        if request.get("namespace") != self.namespace:
+            raise ValueError("control namespace mismatch")
+        _recover_interrupted_runs(con, namespace=self.namespace)
+        operation = request.get("operation")
+        run_id = request.get("run_id")
+        if operation == "plan":
+            if "plan_request" in request:
+                if self.plan_planner is None:
+                    raise ValueError("owner fresh-planning service is unavailable")
+                from terrine.runtime.planning import PlanRequest
+
+                planning_request = PlanRequest.from_dict(request["plan_request"])
+                invocation = self.plan_planner.plan(
+                    planning_request, con=con, namespace=self.namespace
+                )
+                return invocation.to_dict()
+            from terrine.runtime.planner import RefreshPlan
+
+            plan = RefreshPlan.deserialize(request["serialized"])
+            if plan.namespace != self.namespace:
+                raise ValueError("plan namespace does not match owner")
+            return plan.serialize().decode("utf-8")
+        if operation == "run":
+            from terrine.runtime.api import run_serialized_plan
+            from terrine.runtime.saved_plan import DestinationRef
+
+            destination_data = request.get("destination")
+            destination = (
+                DestinationRef(**destination_data) if destination_data is not None else None
+            )
+            receipt = run_serialized_plan(
+                request["serialized"],
+                con=con,
+                namespace=self.namespace,
+                request_key=request.get("request_key"),
+                destination=destination,
+                executor=self.executor,
+                resolver=self.resolver,
+                snapshot_authority=self.snapshot_authority,
+            )
+            return asdict(receipt)
+
+        if operation == "status":
+            if not isinstance(run_id, str) or not run_id:
+                raise ValueError("run ID is required")
+            return asdict(get_run(run_id, con=con, namespace=self.namespace))
+        if operation == "events":
+            if not isinstance(run_id, str) or not run_id:
+                raise ValueError("run ID is required")
+            page = read_run_events(
+                run_id,
+                con=con,
+                namespace=self.namespace,
+                after=request.get("after"),
+                limit=request.get("limit", 100),
+            )
+            return {
+                "events": [asdict(event) for event in page.events],
+                "next_cursor": page.next_cursor,
+                "has_more": page.has_more,
+                "cursor_gap": page.cursor_gap,
+            }
+        if operation == "cancel":
+            if not isinstance(run_id, str) or not run_id:
+                raise ValueError("run ID is required")
+            result = cancel_run(run_id, con=con, namespace=self.namespace)
+            return {
+                "run_id": result.run_id,
+                "disposition": result.disposition,
+                "status": asdict(result.status),
+            }
+        if operation == "request_key":
+            from terrine.runtime.saved_plan import DestinationRef
+
+            destination = request.get("destination")
+            if not isinstance(destination, dict):
+                raise ValueError("destination reference is required")
+            target = DestinationRef(**destination)
+            from terrine.runtime.api import find_run_by_request_key
+
+            receipt = find_run_by_request_key(
+                request["request_key"],
+                destination=target,
+                con=con,
+                namespace=self.namespace,
+            )
+            return asdict(receipt) if receipt is not None else None
+        raise ValueError("unsupported control operation")
+
+
+class ControlClient:
+    def __init__(self, endpoint: ControlEndpoint, *, namespace: str) -> None:
+        self.endpoint = endpoint
+        self.namespace = namespace
+
+    def _request(self, operation: str, **payload: Any) -> Any:
+        request = json.dumps({"namespace": self.namespace, "operation": operation, **payload})
+        encoded = request.encode()
+        if len(encoded) > _MAX_CONTROL_MESSAGE_BYTES:
+            raise _MESSAGE_TOO_LARGE.error()
+        with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
+            client.settimeout(None if operation in {"cancel", "run", "plan"} else 2)
+            client.connect(str(self.endpoint.path))
+            client.sendall(encoded + b"\n")
+            with client.makefile("rb") as response_stream:
+                line = response_stream.readline(_MAX_CONTROL_MESSAGE_BYTES + 2)
+            if not line.endswith(b"\n") or len(line) > _MAX_CONTROL_MESSAGE_BYTES + 1:
+                raise ConnectionError("owner control response exceeded its limit")
+            response = json.loads(line)
+        if not response.get("ok"):
+            raise TerrineError(
+                _CONTROL_UNAVAILABLE.template,
+                code=response.get("code", _CONTROL_UNAVAILABLE.code),
+                context={},
+            )
+        return response["value"]
+
+    @overload
+    def plan(self, serialized: bytes | str) -> str: ...
+
+    @overload
+    def plan(self, serialized: PlanRequest) -> PlanInvocation: ...
+
+    def plan(self, serialized: bytes | str | PlanRequest) -> str | PlanInvocation:
+        from terrine.runtime.planning import PlanInvocation, PlanRequest
+
+        if isinstance(serialized, PlanRequest):
+            value = self._request("plan", plan_request=serialized.to_dict())
+            return PlanInvocation.from_dict(value)
+        payload = serialized.decode("utf-8") if isinstance(serialized, bytes) else serialized
+        return self._request("plan", serialized=payload)
+
+    def run(
+        self,
+        serialized: bytes | str,
+        *,
+        request_key: str | None = None,
+        destination: Any | None = None,
+    ) -> RunReceipt:
+        payload = serialized.decode("utf-8") if isinstance(serialized, bytes) else serialized
+        value = self._request(
+            "run",
+            serialized=payload,
+            request_key=request_key,
+            destination=asdict(destination) if destination is not None else None,
+        )
+        return RunReceipt(value["run_id"], value["state"], value["published_generation"])
+
+    def get_run(self, run_id: str) -> RunStatus:
+        return _status(self._request("status", run_id=run_id))
+
+    def read_run_events(
+        self, run_id: str, *, after: str | None = None, limit: int = 100
+    ) -> EventPage:
+        value = self._request("events", run_id=run_id, after=after, limit=limit)
+        events = tuple(
+            RunEvent(
+                event["run_id"],
+                event["sequence"],
+                datetime.fromisoformat(event["at"]),
+                event["kind"],
+                event["payload"],
+            )
+            for event in value["events"]
+        )
+        return EventPage(events, value["next_cursor"], value["has_more"], value["cursor_gap"])
+
+    def cancel_run(self, run_id: str) -> CancelReceipt:
+        value = self._request("cancel", run_id=run_id)
+        return CancelReceipt(run_id, value["disposition"], _status(value["status"]))
+
+    def find_run_by_request_key(self, request_key: str, *, destination: Any) -> RunReceipt | None:
+        value = self._request(
+            "request_key", request_key=request_key, destination=asdict(destination)
+        )
+        if value is None:
+            return None
+        return RunReceipt(value["run_id"], value["state"], value["published_generation"])
+
+
+def _status(value: dict[str, Any]) -> RunStatus:
+    for field in ("updated_at", "heartbeat_at", "lease_expires_at"):
+        if value[field] is not None and isinstance(value[field], str):
+            value[field] = datetime.fromisoformat(value[field])
+    return RunStatus(**value)