rocket-engine 0.0.2b1__py3-none-any.whl

rocket_engine/project/ledger/entities.py

@@ -0,0 +1,176 @@
+"""Entity-record projection and backfill reads."""
+
+from ...stores import ecs
+from ..errors import ProjectError
+
+from .shared import _MAX_DOC_BYTES
+
+class _EntityMixin:
+    """Entity projection helpers for ProjectService."""
+
+    def _entity_records(self):
+        """Decode every entity record into world-snapshot entries."""
+        records = []
+        for rel, _size in self._iter_doc_files():
+            entity = ecs.entity_from_doc_path(rel)
+            if entity is None:
+                continue
+            text = self._read_doc_text(rel, rel)
+            if text is None:
+                raise ProjectError(
+                    "invalid_document",
+                    f"Entity record vanished mid-read: {rel}.",
+                )
+            try:
+                decoded = ecs.decode_entity_doc(text, rel)
+            except ecs.EcsError as exc:
+                raise ProjectError(exc.code, str(exc)) from exc
+            records.append(
+                {
+                    "components": decoded["components"],
+                    "id": decoded["entity"],
+                    "kind": decoded["kind"],
+                }
+            )
+        return records
+
+    def _backfill_records(self):
+        """Project every non-entity doc as a virtual entity."""
+        entries = [
+            {"bytes": size, "path": rel}
+            for rel, size in self._iter_doc_files()
+        ]
+        return ecs.backfill_entities(entries)
+
+    def _find_backfill(self, entity):
+        """Return the virtual record for one backfilled id."""
+        for record in self._backfill_records():
+            if record["id"] == entity:
+                return record
+        raise ProjectError("unknown_entity", f"No such entity: {entity}.")
+
+    def _read_backfill_component(self, entity, component):
+        """Serve one projected doc component for a backfilled id."""
+        record = self._find_backfill(entity)
+        if component != ecs.BACKFILL_COMPONENT:
+            raise ProjectError(
+                "unknown_component",
+                f"No component {component!r} on {entity}.",
+            )
+        data = record["components"][ecs.BACKFILL_COMPONENT]
+        return {
+            "address": ecs.format_address(entity, component),
+            "component": component,
+            "current_revision": self._current_revision,
+            "data": dict(data),
+            "entity": entity,
+            "path": data["path"],
+            "project_id": self._project_id,
+        }
+
+    def _read_entity_record(self, entity):
+        """Return (path, decoded) for one live entity id."""
+        path = ecs.entity_doc_path(entity)
+        text = self._read_doc_text(path, path)
+        if text is None:
+            raise ProjectError(
+                "unknown_entity", f"No such entity: {entity}."
+            )
+        try:
+            decoded = ecs.decode_entity_doc(text, path)
+        except ecs.EcsError as exc:
+            raise ProjectError(exc.code, str(exc)) from exc
+        if decoded["entity"] != entity:
+            raise ProjectError(
+                "invalid_document",
+                f"Entity record id drift at {path}.",
+            )
+        return path, decoded
+
+    def _store_entity_record(
+        self, path, entity, kind, components, entity_revision=1
+    ):
+        """Encode and persist one entity record under its path."""
+        text = ecs.encode_entity_doc(
+            entity, kind, components, entity_revision=entity_revision
+        )
+        if len(text.encode("utf-8")) > _MAX_DOC_BYTES:
+            raise ProjectError(
+                "invalid_command",
+                f"Entity record exceeds the size limit: {path}.",
+            )
+        self._write_doc_text(path, text, path)
+
+    @staticmethod
+    def _entity_revision_of(decoded):
+        """Per-entity revision of a decoded record (legacy reads 1)."""
+        return decoded.get("entity_revision", 1)
+
+    def _gate_entity_revision(
+        self, entity, decoded, expected_entity_revision
+    ):
+        """Compare-and-swap one entity write.
+
+        Args:
+            entity: Entity id being written.
+            decoded: Decoded live record.
+            expected_entity_revision: Revision the writer read, or
+                None to skip the entity gate (legacy global gate
+                still applies).
+
+        Returns:
+            The live entity revision the write may build on.
+
+        Raises:
+            ProjectError: revision_conflict naming the entity plus
+                expected vs actual — safe to retry after re-read.
+        """
+        current = self._entity_revision_of(decoded)
+        if (
+            expected_entity_revision is not None
+            and expected_entity_revision != current
+        ):
+            raise ProjectError(
+                "revision_conflict",
+                f"Entity {entity} at revision {current}, "
+                f"writer expected {expected_entity_revision}.",
+            )
+        return current
+
+    def _gate_entity_write(self, expected_revision, expected_entity_revision):
+        """Apply the global head gate unless entity CAS owns the write.
+
+        A mutation naming a current entity revision skips the
+        global head gate even when it also names a stale head —
+        the head moved because of unrelated entities, and the
+        entity revision is the strictly more precise gate. Naming
+        no entity revision keeps the legacy mandatory-head
+        behavior.
+        """
+        if expected_entity_revision is not None:
+            return
+        self._gate_revision(expected_revision)
+
+    def _ecs_response(self, request_id, previous, diff, record, extra):
+        """CRUD-shaped mutation mapping for one entity change."""
+        response = {
+            "project_id": self._project_id,
+            "previous_revision": previous,
+            "current_revision": self._current_revision,
+            "revision_transition": {
+                "from": previous,
+                "to": self._current_revision,
+            },
+            "actor": self._actor,
+            "diff_summary": diff,
+            "rewind_handle": {
+                "project_id": self._project_id,
+                "revision": previous,
+                "request_id": request_id,
+            },
+            "events": [record],
+            "event_cursor": self._cursor(),
+            "request_id": request_id,
+        }
+        response.update(extra)
+        return response

rocket_engine/project/ledger/events.py

@@ -0,0 +1,29 @@
+"""Event cursor, retention, and append machinery."""
+
+
+class _EventMixin:
+    """Event cursor and retention for ProjectService."""
+
+    def _cursor(self):
+        return self._events[-1]["seq"] if self._events else 0
+
+    def _retained_head(self):
+        return self._events[0]["seq"] if self._events else 0
+
+    def _trim(self):
+        over = len(self._events) - self._max_retained_events
+        if over > 0:
+            del self._events[:over]
+
+    def _append_event(self, event, request_id, data):
+        record = {
+            "seq": self._cursor() + 1,
+            "event": event,
+            "request_id": request_id,
+            "project_id": self._project_id,
+            "actor": self._actor,
+            "data": data,
+        }
+        self._events.append(record)
+        self._trim()
+        return record

rocket_engine/project/ledger/runs.py

@@ -0,0 +1,257 @@
+"""Run admission gates and run-event recording."""
+
+from ..errors import ProjectError
+
+from .shared import _REPLAY_EVENTS, _RUN_OUTCOMES
+
+#: Verdict detail kept on the completion event: the ledger answers
+#: findings() and run_logs() after a reboot, when the in-memory
+#: execution record is gone. Lists cap here; the verdict keeps
+#: the full counts and the event keeps the truncated remainder.
+_DURABLE_FINDINGS_MAX = 100
+
+#: Native-log tail kept on the completion event: triage reads the
+#: tail, and the ledger never carries bulk vectors.
+_DURABLE_LOG_CHARS = 4096
+
+
+class _RunMixin:
+    """Run admission helpers for ProjectService."""
+
+    def append_run_event(self, outcome, run_id, operation, verdict,
+                         request_id=None, findings=(), log=""):
+        """Append a run completion event with its verdict link.
+
+        Findings and the native-log tail ride the event so the
+        ledger answers findings() and run_logs() after a reboot.
+        Both cap: the first findings plus the truncated remainder
+        count, and the last log characters plus the full line
+        count.
+
+        Args:
+            outcome: run_completed for a judged run (PASS or FAIL),
+                run_failed when no verdict exists (NOT_RUN).
+            run_id: Run the verdict belongs to; findings read back
+                through it.
+            operation: Operation that ran.
+            verdict: Full verdict mapping stored on the event.
+            request_id: Queued request this run discharges, if any
+                (run_job); direct runs carry None.
+            findings: Finding mappings stored first-N with the
+                truncated remainder count.
+            log: Native log text; the tail is stored with the
+                full line count.
+
+        Returns:
+            The appended event record with its seq.
+
+        Raises:
+            ProjectError: invalid_command for an unknown outcome.
+
+        Caution: Appends to the ordered log (and the cursor) but
+            never advances the project revision — running judges,
+            only submitting mutates.
+        """
+        if outcome not in _RUN_OUTCOMES:
+            raise ProjectError(
+                "invalid_command", f"Unknown run outcome: {outcome}."
+            )
+        kept = list(findings)[:_DURABLE_FINDINGS_MAX]
+        dropped = max(0, len(list(findings)) - len(kept))
+        excerpt = log[-_DURABLE_LOG_CHARS:] if log else ""
+        with self._ledger_lock:
+            self._refresh_durable()
+            record = self._append_event(
+                outcome,
+                request_id,
+                {
+                    "run_id": run_id,
+                    "operation": operation,
+                    "verdict": verdict,
+                    "findings": kept,
+                    "findings_truncated": dropped,
+                    "log_excerpt": excerpt,
+                    "log_lines": len(log.splitlines()) if log else 0,
+                },
+            )
+            self._persist_after_mutation()
+        return record
+
+    def append_replay_event(self, event, request_id, data):
+        """Append a replay-lifecycle event for one run.
+
+        `run_started` marks execution begin (a kill after it reads as
+        an interruption at the next boot); `run_interrupted` records
+        one admitted->interrupted transition for its content key (two
+        quarantines the key); `run_discarded` retires a survivor the
+        user chose not to resume.
+
+        Args:
+            event: Replay event name (run_started, run_interrupted,
+                or run_discarded).
+            request_id: Queued request the run belongs to.
+            data: Event data carrying the run_id (plus content_key
+                for started/interrupted marks).
+
+        Returns:
+            The appended event record with its seq.
+
+        Raises:
+            ProjectError: invalid_command for an unknown replay
+                event or data without a run_id.
+
+        Caution: Appends to the ordered log (and the cursor) but
+            never advances the project revision — replay
+            bookkeeping judges nothing and mutates nothing.
+        """
+        if event not in _REPLAY_EVENTS:
+            raise ProjectError(
+                "invalid_command", f"Unknown replay event: {event}."
+            )
+        if not isinstance(data, dict) or data.get("run_id") is None:
+            raise ProjectError(
+                "invalid_command", "Replay event needs a data.run_id."
+            )
+        with self._ledger_lock:
+            self._refresh_durable()
+            record = self._append_event(event, request_id, dict(data))
+            self._persist_after_mutation()
+        return record
+
+    def log_coalesced_request(self, command_text, request_id, run_id,
+                              snapshot_id):
+        """Log a coalesced envelope against its shared run (COALESCE-I33).
+
+        A duplicate submit that joins a live run records its envelope
+        here instead of admitting: the request_id maps to the shared
+        run_id in the request map (so replays and run_job resolve it)
+        and a `request_coalesced` event lands in the log — but the
+        revision never advances, no snapshot is captured, and the
+        project never mutates. Joining callers are readers of a
+        shared verdict, not writers.
+
+        Args:
+            command_text: YAML command mapping with an operation.
+            request_id: Joining envelope id; resubmitting the
+                identical text returns the identical ack.
+            run_id: Shared live run the envelope joins.
+            snapshot_id: Shared run's snapshot (one revision pins
+                one snapshot).
+
+        Returns:
+            Coalesced ack with the caller's request_id, the shared
+                run_id and snapshot_id, queued state, the current
+                (unadvanced) revision, the event cursor, and a
+                coalesced marker.
+
+        Raises:
+            ProjectError: invalid_yaml for an unparsable command or
+                one without an operation, request_conflict for a
+                reused id with new text.
+
+        Caution: Never advances the revision — a coalesced envelope
+            must not break the pin later duplicates join under.
+        """
+        self._refresh_durable()
+        replay = self._check_replay(request_id, command_text)
+        if replay is not None:
+            return replay
+        try:
+            command = self._yaml.load(command_text)
+        except Exception as exc:
+            raise ProjectError(
+                "invalid_yaml", f"Cannot parse command: {exc}"
+            ) from exc
+        if not isinstance(command, dict) or "operation" not in command:
+            raise ProjectError(
+                "invalid_yaml",
+                "Command must be a mapping with an operation.",
+            )
+        ack = {
+            "request_id": request_id,
+            "project_id": self._project_id,
+            "run_id": run_id,
+            "snapshot_id": snapshot_id,
+            "state": "queued",
+            "current_revision": self._current_revision,
+            "coalesced": True,
+        }
+        self._requests[request_id] = (command_text, ack)
+        self._append_event(
+            "request_coalesced",
+            request_id,
+            {
+                "run_id": run_id,
+                "revision": self._current_revision,
+                "operation": command.get("operation"),
+            },
+        )
+        ack["event_cursor"] = self._cursor()
+        self._persist_after_mutation()
+        return ack
+
+    def command_for(self, request_id):
+        """Return the admitted (command, ack) for a queued request.
+
+        Phase 2 of submit-then-run: unknown ids raise
+        `unknown_request` instead of running a stranger's job.
+
+        Args:
+            request_id: Envelope id admitted by an earlier submit.
+
+        Returns:
+            (command, ack) pair stored at admission time.
+
+        Raises:
+            ProjectError: unknown_request for an id never admitted.
+        """
+        self._refresh_durable()
+        try:
+            command_text, ack = self._requests[str(request_id)]
+        except KeyError:
+            raise ProjectError(
+                "unknown_request", f"No queued request: {request_id}."
+            ) from None
+        return self._yaml.load(command_text), ack
+
+    def _check_replay(self, request_id, command_text):
+        """Return the stored response for an idempotent replay, if any.
+
+        Args:
+            request_id: Client envelope id.
+            command_text: YAML command text for identity comparison.
+
+        Returns:
+            Stored response when the id was seen with identical
+            text, else None.
+
+        Raises:
+            ProjectError: request_conflict for a reused id with
+                new text.
+        """
+        prev = self._requests.get(request_id)
+        if prev is None:
+            return None
+        prev_text, prev_ack = prev
+        if prev_text != command_text:
+            raise ProjectError(
+                "request_conflict",
+                f"Request id reused with different input: {request_id}.",
+            )
+        return prev_ack
+
+    def _gate_revision(self, expected_revision):
+        """Fail stale writers before any effect lands.
+
+        Args:
+            expected_revision: Revision the writer read.
+
+        Raises:
+            ProjectError: revision_conflict for a stale head.
+        """
+        if expected_revision != self._current_revision:
+            raise ProjectError(
+                "revision_conflict",
+                f"Expected revision {self._current_revision}, "
+                f"got {expected_revision}.",
+            )

rocket_engine/project/ledger/shared.py

@@ -0,0 +1,174 @@
+"""Pure ledger primitives shared by the project service."""
+
+import io
+import os
+import re
+import tempfile
+
+from ruamel.yaml import YAML
+
+from ...interfaces.paths import to_unix
+from ..errors import ProjectError
+
+DEFAULT_MAX_RETAINED_EVENTS = 10000
+_MAX_DOC_BYTES = 1_000_000
+_EVENTS_FILENAME = "events.yaml"
+_LEDGER_FILENAME = "ledger.yaml"
+_SNAPSHOTS_DIRNAME = "snapshots"
+_LINKED_CARVEOUT = ".rocket"
+_REVISION_RE = re.compile(r"^(.*?)(\d+)$")
+_PROJECT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]*$")
+_RUN_OUTCOMES = ("run_completed", "run_failed")
+_REPLAY_EVENTS = ("run_started", "run_interrupted", "run_discarded")
+_RESERVED_DOCS = frozenset({_LEDGER_FILENAME, _EVENTS_FILENAME})
+
+# Document prefix owning the five design-truth settings files.
+_SETTINGS_DOC_PREFIX = "settings/"
+
+
+def _advance_revision(revision):
+    """Next revision: trailing digits increment, else `-2` is suffixed."""
+    match = _REVISION_RE.match(revision or "")
+    if match is None:
+        return f"{revision}-2"
+    return f"{match.group(1)}{int(match.group(2)) + 1}"
+
+
+def _dump_yaml(data):
+    buf = io.StringIO()
+    YAML(typ="safe").dump(data, buf)
+    return buf.getvalue()
+
+
+def _atomic_write_text(path, text):
+    """Write through a sibling temp file, then rename into place."""
+    fd, tmp = tempfile.mkstemp(
+        dir=str(path.parent), prefix=f"{path.name}.", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            handle.write(text)
+        os.replace(tmp, path)
+    except BaseException:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+        raise
+
+
+def _atomic_write_bytes(path, data):
+    """Write bytes through a sibling temp file, then rename."""
+    fd, tmp = tempfile.mkstemp(
+        dir=str(path.parent), prefix=f"{path.name}.", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(data)
+        os.replace(tmp, path)
+    except BaseException:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+        raise
+
+
+def _canonical_doc_path(path):
+    """Canonical unix document path; rejects escapes and addresses.
+
+    Args:
+        path: Caller-supplied document path.
+
+    Returns:
+        Canonical slash-joined path with dot and empty segments
+        dropped.
+
+    Raises:
+        ProjectError: invalid_path for a non-text, empty, or
+            directory path; path_escape for an absolute path or
+            one containing `..`. Every message names the supplied
+            path.
+    """
+    if not isinstance(path, str):
+        raise ProjectError(
+            "invalid_path", f"Document path must be text, got {path!r}."
+        )
+    unix = to_unix(path)
+    if unix == "" or unix.endswith("/"):
+        raise ProjectError(
+            "invalid_path", f"Bad document path: {path!r}."
+        )
+    if unix.startswith("/"):
+        raise ProjectError(
+            "path_escape", f"Absolute document path: {path!r}."
+        )
+    parts = [p for p in unix.split("/") if p not in ("", ".")]
+    if ".." in parts:
+        raise ProjectError(
+            "path_escape", f"Document path escapes: {path!r}."
+        )
+    canonical = "/".join(parts)
+    if canonical == "":
+        raise ProjectError(
+            "invalid_path", f"Bad document path: {path!r}."
+        )
+    return canonical
+
+
+def _check_reserved_doc(canonical, original):
+    """Reject engine-owned document paths.
+
+    Args:
+        canonical: Canonical document path.
+        original: Caller-supplied path for error messages.
+
+    Raises:
+        ProjectError: invalid_path when the path names the ledger,
+            the event log, or anything under `runs/` or
+            `snapshots/`.
+    """
+    if (
+        canonical in _RESERVED_DOCS
+        or canonical == "runs"
+        or canonical.startswith("runs/")
+        or canonical == _SNAPSHOTS_DIRNAME
+        or canonical.startswith(_SNAPSHOTS_DIRNAME + "/")
+    ):
+        raise ProjectError(
+            "invalid_path",
+            f"Engine-owned path is not a document: {original!r}.",
+        )
+
+
+def _canonical_prefix(prefix):
+    """Canonical list filter; empty means no filtering.
+
+    Args:
+        prefix: Caller-supplied prefix or None.
+
+    Returns:
+        Canonical prefix string ("" matches everything).
+
+    Raises:
+        ProjectError: invalid_path for a non-text prefix;
+            path_escape for an absolute prefix or one
+            containing `..`.
+    """
+    if prefix is None:
+        return ""
+    if not isinstance(prefix, str):
+        raise ProjectError(
+            "invalid_path", f"Prefix must be text, got {prefix!r}."
+        )
+    unix = to_unix(prefix)
+    if unix.startswith("/"):
+        raise ProjectError(
+            "path_escape", f"Absolute prefix: {prefix!r}."
+        )
+    parts = [p for p in unix.split("/") if p not in ("", ".")]
+    if ".." in parts:
+        raise ProjectError(
+            "path_escape", f"Prefix escapes: {prefix!r}."
+        )
+    return "/".join(parts)