simrun 0.1.0__py3-none-any.whl

simrun/service/auth.py

@@ -0,0 +1,93 @@
+"""Identities of the callers of the service and the priorities of their runs."""
+
+import hmac
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, model_validator
+
+from simrun.workflow import Priority
+
+
+class Identity(BaseModel):
+    """Who calls the service: a person or an agent acting for a person.
+
+    Attributes:
+        name: Name of the person or the agent.
+        kind: `person` or `agent`.
+        owner: The person responsible; a person's owner is the person.
+    """
+
+    name: str
+    kind: Literal["person", "agent"]
+    owner: str
+
+    @model_validator(mode="before")
+    @classmethod
+    def _owner(cls, data: Any) -> Any:
+        if isinstance(data, dict) and data.get("kind") == "person":
+            owner = data.get("owner")
+            if owner is not None and owner != data.get("name"):
+                raise ValueError(
+                    f"the owner of the person {data.get('name')!r} is the person, "
+                    f"not {owner!r}"
+                )
+            return {**data, "owner": data.get("name")}
+        if (
+            isinstance(data, dict)
+            and data.get("kind") == "agent"
+            and not data.get("owner")
+        ):
+            raise ValueError(f"the agent {data.get('name')!r} has no owner")
+        return data
+
+
+def load_tokens(path: Path) -> dict[str, Identity]:
+    """Read the tokens of the service from a YAML mapping of token to identity.
+
+    Args:
+        path: The YAML file, e.g. `{<token>: {name: mk, kind: person}}`.
+
+    Returns:
+        The identities by token.
+
+    Raises:
+        ValueError: The file is no mapping, or a token is empty.
+    """
+    data = yaml.safe_load(path.read_text()) or {}
+    if not isinstance(data, dict):
+        raise ValueError(f"{path}: not a mapping of token to identity")
+    tokens: dict[str, Identity] = {}
+    for token, identity in data.items():
+        if not isinstance(token, str) or not token:
+            raise ValueError(f"{path}: a token must be a non-empty string")
+        tokens[token] = Identity.model_validate(identity)
+    return tokens
+
+
+def authenticate(tokens: dict[str, Identity], token: str) -> Identity | None:
+    """The identity of a token, compared in constant time; None if unknown."""
+    found = None
+    for known, identity in tokens.items():
+        if hmac.compare_digest(known.encode(), token.encode()):
+            found = identity
+    return found
+
+
+def priority_of(identity: Identity, owner: str) -> Priority:
+    """The priority of the runs of an identity.
+
+    Args:
+        identity: The submitting identity.
+        owner: The owner of the cluster.
+
+    Returns:
+        `OWNER` for the owner and the owner's agents, `STANDARD` for other people,
+        `AGENT` for their agents.
+    """
+    if identity.owner == owner:
+        return Priority.OWNER
+    if identity.kind == "person":
+        return Priority.STANDARD
+    return Priority.AGENT

simrun/service/backend.py

@@ -0,0 +1,170 @@
+"""The backend the service runs workflows on: Argo Workflows, or a fake in tests."""
+
+from pathlib import Path
+from typing import Any, Protocol
+
+from pydantic import BaseModel, Field
+
+from simrun.planner import Node
+
+# phases of an Argo workflow or node which do not change any more
+FINAL_PHASES = frozenset({"Succeeded", "Failed", "Error"})
+# the phase of a task node whose pod was evicted (or lost) and waits for its retry
+PREEMPTED_PHASE = "Preempted"
+# the phase of a task node whose pod failed of itself (no kill for memory, no
+# eviction) and waits for its retry
+RETRYING_PHASE = "Retrying"
+
+
+class FailedAttempt(BaseModel):
+    """An attempt of a pod which failed.
+
+    Attributes:
+        attempt: The number of the attempt, from 0, as the pod has it in
+            `SIMRUN_ATTEMPT` (Argo's `{{retries}}`) and in the name of its report.
+        exit_code: The exit code of its main container, None without one (e.g. a
+            pod deleted by an eviction of Kueue).
+        oom_killed: Whether the main container was killed by the out of memory
+            killer (the reason `OOMKilled`); an exit 137 alone is no evidence of it.
+    """
+
+    attempt: int
+    exit_code: int | None = None
+    oom_killed: bool = False
+
+
+class PodLog(BaseModel):
+    """The log of the main container of one pod (one attempt) of a task.
+
+    Attributes:
+        node: The node of the pod in the workflow, e.g. `squares-w0(1)`.
+        pod: The name of the pod.
+        text: The log.
+        done: Whether the pod has terminated, so its log is complete.
+    """
+
+    node: str
+    pod: str
+    text: str
+    done: bool = True
+
+
+class Placement(BaseModel):
+    """A pod of a run which has not terminated, with its node and requests.
+
+    Attributes:
+        workflow: The backend name of the run.
+        dag_task: The DAG task of the pod: `<task>-w0`, `<task>-extra`,
+            `<task>-merge` or `<task>`.
+        pod: Name of the pod.
+        node: The node the pod is bound to, `None` while it is not.
+        phase: The phase of the pod.
+        cores: The cores the pod requests.
+        memory_gb: The GiB of memory the pod requests.
+        worker: The worker id of the pod (`SIMRUN_WORKER`, the id of its ledger claims
+            and report names), `None` for a pod without one, e.g. a merge.
+        attempt: The attempt of the pod (`SIMRUN_ATTEMPT`, the number of its
+            retries, the attempt of its report name), `None` when unknown.
+    """
+
+    workflow: str
+    dag_task: str
+    pod: str
+    node: str | None
+    phase: str
+    cores: float
+    memory_gb: float
+    worker: str | None = None
+    attempt: int | None = None
+
+
+class BackendStatus(BaseModel):
+    """The state of a workflow in the backend.
+
+    The service, not the backend, decides the state of a run from it (see
+    `simrun.service.controller`).
+
+    Attributes:
+        phase: Phase of the workflow, e.g. `Pending`, `Running`, `Succeeded`.
+        tasks: Phase of every node of the DAG by its name: `<task>-w0`,
+            `<task>-extra(<i>:<item>)` and `<task>-merge` of a split task, `<task>` of
+            an unsplit one; a node not yet created is missing. A node whose pod was
+            evicted and whose retry waits is `Preempted` (`PREEMPTED_PHASE`), one
+            whose pod failed otherwise `Retrying` (`RETRYING_PHASE`).
+        failed: The attempts of pods which failed, in the order of their number: per
+            worker id (`<task>-w<i>`), per task id of an unsplit task and per
+            `<task>-merge` of a merge.
+    """
+
+    phase: str
+    tasks: dict[str, str]
+    failed: dict[str, list[FailedAttempt]] = Field(default_factory=dict)
+
+
+class Backend(Protocol):
+    """What the service needs of a cluster."""
+
+    def submit(self, manifest: dict[str, Any]) -> str:
+        """Submit a workflow manifest and return the name of the workflow."""
+        ...
+
+    def status(self, name: str) -> BackendStatus:
+        """The status of a workflow.
+
+        Raises:
+            KeyError: The workflow does not exist (any more).
+        """
+        ...
+
+    def cancel(self, name: str) -> None:
+        """Terminate a workflow and its pods."""
+        ...
+
+    def stop(self, name: str) -> None:
+        """Stop a workflow whose run is decided: pods still pending are removed."""
+        ...
+
+    def logs(
+        self, name: str, task_id: str, tail_lines: int | None = None
+    ) -> list[PodLog]:
+        """The logs of the pods of a task of a workflow, oldest first.
+
+        Every attempt whose pod still exists; `tail_lines` limits each log to its
+        last lines.
+        """
+        ...
+
+    def nodes(self) -> list[Node]:
+        """The nodes of the cluster."""
+        ...
+
+    def placements(self) -> list[Placement]:
+        """The pods of the runs which have not terminated, with nodes and requests."""
+        ...
+
+    def reserve(self, node: str, run_id: str) -> None:
+        """Reserve a node for a run.
+
+        The node gets the taint `simrun/reserved=<run id>:NoExecute` and the label
+        `simrun/reserved=<run id>`.
+        """
+        ...
+
+    def unreserve(self, node: str, run_id: str) -> None:
+        """Remove the taint and the label of the reservation of a run.
+
+        A node reserved by another run (or by none) is left as it is, so a late
+        release never removes the reservation of a later run.
+
+        Raises:
+            KeyError: The node does not exist.
+        """
+        ...
+
+    def build_image(self, context: Path, ref: str) -> None:
+        """Build the image of a build context and push it as `ref`."""
+        ...
+
+    def has_image(self, ref: str) -> bool:
+        """Whether the registry has the image `ref`."""
+        ...

simrun/service/compare.py

@@ -0,0 +1,77 @@
+"""The comparison of runs: provenance and the summaries of their tasks side by side."""
+
+import json
+import logging
+from typing import Any
+
+from pydantic import BaseModel
+
+from simrun.store import RunStore
+
+logger = logging.getLogger(__name__)
+
+SUMMARY_FILE = "summary.json"
+
+
+class CompareTable(BaseModel):
+    """A table of runs.
+
+    Attributes:
+        columns: The columns in order: `run`, `group`, `agent`, `state`, then per
+            source `<source>.changed_files` and `<source>.diff_sha256`, then every
+            key of `out/summary.json` of every task as `<task>.<key>`.
+        rows: One row per run, with every column (None where a run has no value).
+    """
+
+    columns: list[str]
+    rows: list[dict[str, Any]]
+
+
+def _summary(store: RunStore, run_id: str, task_id: str) -> dict[str, Any]:
+    path = store.run_dir(run_id) / "tasks" / task_id / "out" / SUMMARY_FILE
+    try:
+        data = json.loads(path.read_text())
+    except FileNotFoundError:
+        return {}
+    except ValueError as error:
+        logger.warning("summary %s is no JSON: %s", path, error)
+        return {}
+    if not isinstance(data, dict):
+        logger.warning("summary %s is no JSON object", path)
+        return {}
+    return data
+
+
+def compare_runs(store: RunStore, run_ids: list[str]) -> CompareTable:
+    """Compare runs by their provenance and the summaries their merges wrote.
+
+    Args:
+        store: The run store.
+        run_ids: The runs, one row each in this order.
+
+    Returns:
+        The table.
+
+    Raises:
+        KeyError: A run does not exist.
+    """
+    rows: list[dict[str, Any]] = []
+    for run_id in run_ids:
+        record = store.read(run_id)
+        row: dict[str, Any] = {
+            "run": record.id,
+            "group": record.group,
+            "agent": record.agent,
+            "state": record.state.value,
+        }
+        for source in record.sources:
+            row[f"{source.name}.changed_files"] = len(source.changed_files)
+            row[f"{source.name}.diff_sha256"] = source.diff_sha256
+        for task in record.workflow.tasks:
+            for key, value in _summary(store, run_id, task.id).items():
+                row[f"{task.id}.{key}"] = value
+        rows.append(row)
+    columns = list(dict.fromkeys(key for row in rows for key in row))
+    return CompareTable(
+        columns=columns, rows=[{c: row.get(c) for c in columns} for row in rows]
+    )