simrun 0.1.0__py3-none-any.whl

simrun/service/edits.py

@@ -0,0 +1,406 @@
+"""Edit sessions: how the browser asks the machine with the code to save and run.
+
+A session lives in the store under `edits/<id>/`: `session.json`, and in
+`requests/` the `<rid>.json` of a request of the browser and the
+`<rid>.result.json` with the answer of the command `simrun edit`. All files are
+written atomically, and the service keeps nothing in memory.
+"""
+
+from __future__ import annotations
+
+import builtins
+import fcntl
+import logging
+import os
+import secrets
+import uuid
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Literal, Self
+
+from pydantic import BaseModel, model_validator
+
+from simrun.service.auth import Identity
+from simrun.snapshot import SourceProvenance
+from simrun.store import RunSummary
+from simrun.workflow import SplitMode, Workflow
+
+logger = logging.getLogger(__name__)
+
+EDIT_CONNECTED_S = 30.0
+# how long a save or a run of the browser waits for the answer of the command; a
+# request older than this is no longer waited for
+EDIT_WAIT_S = 60.0
+SESSION_FILE = "session.json"
+LOCK_FILE = ".lock"
+REQUESTS_DIR = "requests"
+RESULT_SUFFIX = ".result.json"
+
+
+class TreeState(BaseModel):
+    """The live state of the working tree of a source.
+
+    Attributes:
+        name: Name of the source.
+        branch: The branch, none when detached.
+        commit: The commit.
+        changed_files: The files which differ from the commit.
+    """
+
+    name: str
+    branch: str | None
+    commit: str
+    changed_files: list[str]
+
+
+class EditSession(BaseModel):
+    """An edit session.
+
+    Attributes:
+        id: Id of the session.
+        owner: The user of the token which opened it.
+        host: The machine of the command.
+        path: The absolute path of the workflow file there.
+        workflow: The workflow as in the file.
+        file_sha256: Hash of the file content, empty for a file which does not exist.
+        sources: The provenance of the last snapshots.
+        tree: Per source the state of the working tree.
+        created: When the session was opened.
+        seen: The last poll of the command.
+        closed: The command stopped.
+        connected: Not closed and seen lately; computed when read.
+    """
+
+    id: str
+    owner: str
+    host: str
+    path: str
+    workflow: Workflow
+    file_sha256: str
+    sources: list[SourceProvenance]
+    tree: list[TreeState]
+    created: datetime
+    seen: datetime
+    closed: bool = False
+    connected: bool = False
+
+
+class OpenEdit(BaseModel):
+    """The body which opens a session."""
+
+    host: str
+    path: str
+    workflow: Workflow
+    file_sha256: str
+    sources: list[SourceProvenance]
+    tree: list[TreeState]
+
+
+class EditState(BaseModel):
+    """What the command read from disk, sent when it changed."""
+
+    workflow: Workflow
+    file_sha256: str
+    tree: list[TreeState]
+    sources: list[SourceProvenance] | None = None
+
+
+class EditRequest(BaseModel):
+    """A request of the browser to the command.
+
+    Attributes:
+        id: Id of the request.
+        kind: `save` writes the file, `run` submits the workflow.
+        workflow: The workflow of the draft.
+        base: For a save, the hash of the file the draft started from.
+        force: Save although the file changed on disk.
+        group: Group of the run.
+        split: Split mode of the run.
+        reserve: Node reserved for the run.
+        created: When the request was made.
+    """
+
+    id: str
+    kind: Literal["save", "run"]
+    workflow: Workflow
+    base: str
+    force: bool = False
+    group: str | None = None
+    split: SplitMode | None = None
+    reserve: str | None = None
+    created: datetime
+
+
+class EditResult(BaseModel):
+    """The answer of the command to a request.
+
+    Attributes:
+        ok: Whether the request was done.
+        status: The status of a refusal, 4xx or 5xx: 409 conflict, 422 refused.
+        detail: The reason of a refusal.
+        file_sha256: For a save, the hash of the written file.
+        run: For a run, the submitted run.
+    """
+
+    ok: bool
+    status: int = 200
+    detail: str = ""
+    file_sha256: str | None = None
+    run: RunSummary | None = None
+
+    @model_validator(mode="after")
+    def _refusal_has_an_error_status(self) -> Self:
+        if not self.ok and not 400 <= self.status <= 599:
+            raise ValueError(f"a refusal needs a 4xx or 5xx status, not {self.status}")
+        return self
+
+
+class SaveBody(BaseModel):
+    """The body of a save."""
+
+    workflow: Workflow
+    base: str
+    force: bool = False
+
+
+class RunBody(BaseModel):
+    """The body of a run, with the options of `SubmitRequest`."""
+
+    workflow: Workflow
+    group: str | None = None
+    split: SplitMode | None = None
+    reserve: str | None = None
+
+
+def _write(path: Path, text: str) -> None:
+    """Write a file atomically (temporary file in the directory, rename)."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.parent / f".{path.name}.{secrets.token_hex(4)}.tmp"
+    tmp.write_text(text)
+    os.replace(tmp, path)
+
+
+def _now() -> datetime:
+    return datetime.now(UTC)
+
+
+class EditStore:
+    """The edit sessions under `<store>/edits`."""
+
+    def __init__(self, root: Path) -> None:
+        """Create the store.
+
+        Args:
+            root: The store of the service.
+        """
+        self.root = root / "edits"
+
+    def _dir(self, session_id: str) -> Path:
+        return self.root / session_id
+
+    @contextmanager
+    def _locked(self, session_id: str) -> Generator[None]:
+        """Hold the lock of a session while its file is read and written."""
+        directory = self._dir(session_id)
+        if not (directory / SESSION_FILE).is_file():
+            raise KeyError(session_id)
+        with (directory / LOCK_FILE).open("a") as handle:
+            fcntl.flock(handle, fcntl.LOCK_EX)
+            yield
+
+    def _load(self, session_id: str) -> EditSession:
+        try:
+            text = (self._dir(session_id) / SESSION_FILE).read_text()
+        except FileNotFoundError:
+            raise KeyError(session_id) from None
+        return EditSession.model_validate_json(text)
+
+    def _mutate(
+        self, session_id: str, change: Callable[[EditSession], None]
+    ) -> EditSession:
+        with self._locked(session_id):
+            session = self._load(session_id)
+            change(session)
+            _write(
+                self._dir(session_id) / SESSION_FILE,
+                session.model_dump_json(exclude={"connected"}, indent=2),
+            )
+        return self._connected(session)
+
+    @staticmethod
+    def _connected(session: EditSession) -> EditSession:
+        age = _now() - session.seen
+        session.connected = not session.closed and age <= timedelta(
+            seconds=EDIT_CONNECTED_S
+        )
+        return session
+
+    def open(self, identity: Identity, body: OpenEdit) -> EditSession:
+        """Open a session owned by the owner of the identity."""
+        now = _now()
+        session = EditSession(
+            id=uuid.uuid4().hex[:12],
+            owner=identity.owner,
+            created=now,
+            seen=now,
+            **body.model_dump(),
+        )
+        _write(
+            self._dir(session.id) / SESSION_FILE,
+            session.model_dump_json(exclude={"connected"}, indent=2),
+        )
+        return self._connected(session)
+
+    def read(self, session_id: str) -> EditSession:
+        """Read a session, with `connected` computed; `KeyError` when unknown."""
+        return self._connected(self._load(session_id))
+
+    def list(self, owner: str) -> builtins.list[EditSession]:
+        """The sessions of an owner, connected first, newest first."""
+        found: list[EditSession] = []
+        for path in self.root.glob(f"*/{SESSION_FILE}"):
+            try:
+                session = self.read(path.parent.name)
+            except KeyError, ValueError:
+                logger.warning("the edit session %s cannot be read", path.parent.name)
+                continue
+            if session.owner == owner:
+                found.append(session)
+        found.sort(key=lambda s: s.created, reverse=True)
+        found.sort(key=lambda s: not s.connected)
+        return found
+
+    def update(self, session_id: str, state: EditState) -> EditSession:
+        """Store what the command read from disk."""
+
+        def change(session: EditSession) -> None:
+            session.workflow = state.workflow
+            session.file_sha256 = state.file_sha256
+            session.tree = state.tree
+            if state.sources is not None:
+                session.sources = state.sources
+
+        return self._mutate(session_id, change)
+
+    def seen(self, session_id: str) -> EditSession:
+        """Record a poll of the command."""
+
+        def change(session: EditSession) -> None:
+            session.seen = _now()
+
+        return self._mutate(session_id, change)
+
+    def close(self, session_id: str) -> EditSession:
+        """Close a session, the command stopped."""
+
+        def change(session: EditSession) -> None:
+            session.closed = True
+
+        return self._mutate(session_id, change)
+
+    def saved(
+        self, session_id: str, workflow: Workflow, sha256: str | None
+    ) -> EditSession:
+        """Record a save: the workflow, and the hash when the result carried one."""
+
+        def change(session: EditSession) -> None:
+            session.workflow = workflow
+            if sha256 is not None:
+                session.file_sha256 = sha256
+
+        return self._mutate(session_id, change)
+
+    def add_request(
+        self,
+        session_id: str,
+        kind: Literal["save", "run"],
+        workflow: Workflow,
+        base: str = "",
+        force: bool = False,
+        group: str | None = None,
+        split: SplitMode | None = None,
+        reserve: str | None = None,
+    ) -> EditRequest:
+        """Write a request for the command to answer."""
+        self._load(session_id)
+        request = EditRequest(
+            id=uuid.uuid4().hex,
+            kind=kind,
+            workflow=workflow,
+            base=base,
+            force=force,
+            group=group,
+            split=split,
+            reserve=reserve,
+            created=_now(),
+        )
+        _write(
+            self._dir(session_id) / REQUESTS_DIR / f"{request.id}.json",
+            request.model_dump_json(indent=2),
+        )
+        return request
+
+    def requests(self, session_id: str) -> builtins.list[EditRequest]:
+        """The requests without an answer, oldest first.
+
+        A request older than `EDIT_WAIT_S` by the clock of the service is no longer
+        waited for, e.g. one left by a service which stopped while it waited: it is
+        dropped instead of returned, so that the command does not do it late.
+        """
+        directory = self._dir(session_id) / REQUESTS_DIR
+        stale = _now() - timedelta(seconds=EDIT_WAIT_S)
+        found: list[EditRequest] = []
+        for path in directory.glob("*.json"):
+            if path.name.endswith(RESULT_SUFFIX):
+                continue
+            if (directory / f"{path.stem}{RESULT_SUFFIX}").exists():
+                continue
+            try:
+                request = EditRequest.model_validate_json(path.read_text())
+            except FileNotFoundError, ValueError:
+                continue  # dropped or half gone meanwhile
+            if request.created < stale:
+                logger.info("the stale edit request %s is dropped", request.id)
+                self.drop_request(session_id, request.id)
+                continue
+            found.append(request)
+        found.sort(key=lambda r: r.created)
+        return found
+
+    def answer(self, session_id: str, request_id: str, result: EditResult) -> None:
+        """Store the answer of the command; `KeyError` for an unknown request."""
+        directory = self._dir(session_id) / REQUESTS_DIR
+        if not (directory / f"{request_id}.json").is_file():
+            raise KeyError(request_id)
+        _write(directory / f"{request_id}{RESULT_SUFFIX}", result.model_dump_json())
+
+    def result(self, session_id: str, request_id: str) -> EditResult | None:
+        """The answer to a request, none while there is none."""
+        path = self._dir(session_id) / REQUESTS_DIR / f"{request_id}{RESULT_SUFFIX}"
+        try:
+            return EditResult.model_validate_json(path.read_text())
+        except FileNotFoundError:
+            return None
+
+    def drop_request(self, session_id: str, request_id: str) -> None:
+        """Delete a request and its answer."""
+        directory = self._dir(session_id) / REQUESTS_DIR
+        for name in (f"{request_id}.json", f"{request_id}{RESULT_SUFFIX}"):
+            (directory / name).unlink(missing_ok=True)
+
+
+__all__ = [
+    "EDIT_CONNECTED_S",
+    "EDIT_WAIT_S",
+    "EditRequest",
+    "EditResult",
+    "EditSession",
+    "EditState",
+    "EditStore",
+    "OpenEdit",
+    "RunBody",
+    "SaveBody",
+    "TreeState",
+]

simrun/service/index.py

@@ -0,0 +1,292 @@
+"""The index of the runs in SQLite (spec section 4), beside the run store.
+
+The store stays the source of truth: the index is rebuilt from the records at the
+start of the service and updated after every record the service writes
+(`IndexedRunStore.write`), the record first. It holds what the controller and the
+lists need without reading the records over NFS: the id, the owner, the agent, the
+group, the name of the workflow, the state, the times of creation and end, the
+pending release and cancellation, the snapshots and the summary of every run. The
+controller follows only the live runs (not ended, or with a release or a cancellation
+pending); `GET /runs`, a cleanup and the usage filter in the index.
+"""
+
+import json
+import logging
+import sqlite3
+import threading
+from collections.abc import Iterable
+from contextlib import closing
+from dataclasses import dataclass
+from pathlib import Path
+
+from simrun.store import RunRecord, RunState, RunStore, RunSummary
+
+logger = logging.getLogger(__name__)
+
+TERMINAL = (RunState.SUCCEEDED, RunState.FAILED, RunState.CANCELLED)
+COLUMNS = (
+    "id, owner, agent, grp, workflow, state, created, finished, pending_release, "
+    "pending_cancel, pending_logs, snapshots, summary"
+)
+
+
+@dataclass(frozen=True)
+class IndexedRun:
+    """A run as the index has it.
+
+    Attributes:
+        summary: Its summary.
+        owner: Its owner.
+        workflow: The name of its workflow.
+        pending: Whether a release of its node or a cancellation of its workflow is
+            pending.
+        snapshots: The snapshots of its sources, relative to the store.
+    """
+
+    summary: RunSummary
+    owner: str
+    workflow: str
+    pending: bool
+    snapshots: tuple[str, ...]
+
+
+def _row(record: RunRecord) -> tuple[object, ...]:
+    return (
+        record.id,
+        record.owner,
+        record.agent,
+        record.group,
+        record.workflow.name,
+        record.state.value,
+        record.created.isoformat(),
+        None if record.finished is None else record.finished.isoformat(),
+        record.pending_release,
+        int(record.pending_cancel),
+        int(record.pending_logs),
+        json.dumps([s.snapshot for s in record.sources]),
+        RunSummary.of(record).model_dump_json(),
+    )
+
+
+class RunIndex:
+    """The index of the runs in a SQLite file on a local volume.
+
+    A connection is opened per call, so the index is safe to use from several
+    threads of one process.
+    """
+
+    def __init__(self, db: Path) -> None:
+        """Create the table if needed.
+
+        Args:
+            db: Path of the SQLite file.
+        """
+        self.db = db
+        db.parent.mkdir(parents=True, exist_ok=True)
+        with self._connect() as con:
+            con.execute("PRAGMA journal_mode=WAL")
+            columns = {row[1] for row in con.execute("PRAGMA table_info(runs)")}
+            if columns and "pending_logs" not in columns:
+                # an index of an earlier version: a cache, rebuilt at the start
+                con.execute("DROP TABLE runs")
+            con.execute(
+                "CREATE TABLE IF NOT EXISTS runs ("
+                "id TEXT PRIMARY KEY, owner TEXT NOT NULL, agent TEXT, grp TEXT, "
+                "workflow TEXT NOT NULL, state TEXT NOT NULL, created TEXT NOT NULL, "
+                "finished TEXT, pending_release TEXT, pending_cancel INTEGER NOT NULL, "
+                "pending_logs INTEGER NOT NULL DEFAULT 0, "
+                "snapshots TEXT NOT NULL, summary TEXT NOT NULL)"
+            )
+            con.execute("CREATE INDEX IF NOT EXISTS runs_state ON runs (state)")
+            con.commit()
+
+    def _connect(self) -> closing[sqlite3.Connection]:
+        return closing(sqlite3.connect(self.db, timeout=30))
+
+    def rebuild(self, records: Iterable[RunRecord]) -> None:
+        """Replace the index by the runs of `records`."""
+        rows = [_row(r) for r in records]
+        with self._connect() as con:
+            con.execute("DELETE FROM runs")
+            con.executemany(f"INSERT INTO runs ({COLUMNS}) VALUES ({_marks()})", rows)
+            con.commit()
+        logger.info("the index of the runs was rebuilt: %d runs", len(rows))
+
+    def put(self, record: RunRecord) -> None:
+        """Add or update a run."""
+        with self._connect() as con:
+            con.execute(
+                f"INSERT OR REPLACE INTO runs ({COLUMNS}) VALUES ({_marks()})",
+                _row(record),
+            )
+            con.commit()
+
+    def remove(self, run_id: str) -> None:
+        """Remove a run."""
+        with self._connect() as con:
+            con.execute("DELETE FROM runs WHERE id = ?", (run_id,))
+            con.commit()
+
+    def live_ids(self) -> list[str]:
+        """The ids of the runs the controller follows, sorted.
+
+        Runs which have not ended, and ended runs with a release of their node, a
+        cancellation of their workflow or the logs of their pods pending.
+        """
+        terminal = ", ".join("?" for _ in TERMINAL)
+        with self._connect() as con:
+            rows = con.execute(
+                f"SELECT id FROM runs WHERE state NOT IN ({terminal}) "
+                "OR pending_release IS NOT NULL OR pending_cancel OR pending_logs "
+                "ORDER BY id",
+                [s.value for s in TERMINAL],
+            ).fetchall()
+        return [r[0] for r in rows]
+
+    def runs(
+        self,
+        group: str | None = None,
+        owner: str | None = None,
+        agent: str | None = None,
+        state: RunState | None = None,
+    ) -> list[IndexedRun]:
+        """The runs with the given group, owner, agent and state, sorted by id."""
+        filters = {"grp": group, "owner": owner, "agent": agent}
+        clauses = [f"{column} = ?" for column, v in filters.items() if v is not None]
+        values: list[object] = [v for v in filters.values() if v is not None]
+        if state is not None:
+            clauses.append("state = ?")
+            values.append(state.value)
+        where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
+        with self._connect() as con:
+            rows = con.execute(
+                "SELECT summary, owner, workflow, pending_release, pending_cancel, "
+                f"snapshots FROM runs {where} ORDER BY id",
+                values,
+            ).fetchall()
+        return [
+            IndexedRun(
+                summary=RunSummary.model_validate_json(summary),
+                owner=owner_,
+                workflow=workflow,
+                pending=release is not None or bool(cancel),
+                snapshots=tuple(json.loads(snapshots)),
+            )
+            for summary, owner_, workflow, release, cancel, snapshots in rows
+        ]
+
+
+def _marks() -> str:
+    return ", ".join("?" for _ in COLUMNS.split(","))
+
+
+class IndexedRunStore(RunStore):
+    """The run store of the service: every record it writes is indexed too.
+
+    A record whose index row could not be written (the index raised after the
+    record was written) and a record which could not be read when the index was
+    rebuilt are kept aside and indexed again at every `reindex`, which the
+    controller calls at every poll; the ones which still cannot be read are logged
+    then.
+    """
+
+    def __init__(self, root: Path, index: RunIndex) -> None:
+        """Create the store.
+
+        Args:
+            root: Root directory of the store.
+            index: The index of its runs.
+        """
+        super().__init__(root)
+        self.index = index
+        self._lock = threading.Lock()
+        self._unindexed: set[str] = set()
+
+    def write(self, record: RunRecord) -> None:
+        """Write a record, then index it.
+
+        A failure of the index is logged and the run indexed at the next `reindex`:
+        the record is written, so the caller goes on.
+        """
+        super().write(record)
+        try:
+            self.index.put(record)
+        except Exception:
+            logger.exception("indexing run %s failed, retried later", record.id)
+            with self._lock:
+                self._unindexed.add(record.id)
+            return
+        with self._lock:
+            self._unindexed.discard(record.id)
+
+    def forget(self, run_id: str) -> None:
+        """Remove a run from the index, once it was moved out of the runs."""
+        with self._lock:
+            self._unindexed.discard(run_id)
+        self.index.remove(run_id)
+
+    def ids(self) -> list[str]:
+        """The ids of the runs in the store, sorted; hidden directories are no runs."""
+        runs = self.root / "runs"
+        if not runs.is_dir():
+            return []
+        return sorted(p.name for p in runs.iterdir() if not p.name.startswith("."))
+
+    def rebuild(self) -> None:
+        """Rebuild the index from the records.
+
+        A record which cannot be read is logged, left out and tried again at every
+        `reindex`.
+        """
+        records: list[RunRecord] = []
+        unreadable: set[str] = set()
+        for run_id in self.ids():
+            try:
+                records.append(self.read(run_id))
+            except KeyError:
+                continue
+            except (ValueError, OSError) as error:
+                logger.warning("the record of run %s cannot be read: %s", run_id, error)
+                unreadable.add(run_id)
+        self.index.rebuild(records)
+        with self._lock:
+            self._unindexed = unreadable
+
+    def reindex(self) -> list[str]:
+        """Index the runs whose index row is missing, from their records.
+
+        A record which cannot be read is logged at every call, so a run the
+        controller cannot follow is never silent. Only the index row is written,
+        never the record file, so the caller holds the lock of the record writes
+        around the call: a record is read and indexed without a write in between.
+
+        Returns:
+            The runs whose records were read but whose index rows still cannot be
+            written, which the controller follows besides the live runs.
+        """
+        with self._lock:
+            pending = sorted(self._unindexed)
+        unindexed: list[str] = []
+        for run_id in pending:
+            try:
+                record = self.read(run_id)
+            except KeyError:
+                with self._lock:
+                    self._unindexed.discard(run_id)
+                continue
+            except (ValueError, OSError) as error:
+                logger.warning(
+                    "the record of run %s cannot be read, the run is not followed: %s",
+                    run_id,
+                    error,
+                )
+                continue
+            try:
+                self.index.put(record)
+            except Exception:
+                logger.exception("indexing run %s failed, retried later", run_id)
+                unindexed.append(run_id)
+                continue
+            with self._lock:
+                self._unindexed.discard(run_id)
+        return unindexed