simrun 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
simrun/store.py ADDED
@@ -0,0 +1,190 @@
1
+ """The run store: run records and the layout of a run on the shared drive.
2
+
3
+ Layout: `<store>/runs/<id>/tasks/<task>`, the record in `<store>/runs/<id>/run.json`.
4
+ """
5
+
6
+ import logging
7
+ import os
8
+ import secrets
9
+ from collections.abc import Iterator
10
+ from datetime import UTC, datetime
11
+ from enum import StrEnum
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from pydantic import BaseModel, Field
16
+
17
+ from simrun.plugins import TaskPaths
18
+ from simrun.snapshot import SourceProvenance
19
+ from simrun.workflow import Priority, Workflow
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ RECORD_FILE = "run.json"
24
+
25
+
26
+ class RunState(StrEnum):
27
+ """State of a run."""
28
+
29
+ SUBMITTING = "submitting"
30
+ QUEUED = "queued"
31
+ RUNNING = "running"
32
+ SUCCEEDED = "succeeded"
33
+ FAILED = "failed"
34
+ CANCELLED = "cancelled"
35
+
36
+
37
+ class TaskState(StrEnum):
38
+ """State of a task of a run."""
39
+
40
+ WAITING = "waiting"
41
+ QUEUED = "queued"
42
+ RUNNING = "running"
43
+ PREEMPTED = "preempted"
44
+ RETRYING = "retrying"
45
+ DONE = "done"
46
+ FAILED = "failed"
47
+ SKIPPED = "skipped"
48
+
49
+
50
+ class RunRecord(BaseModel):
51
+ """The record of a run.
52
+
53
+ Attributes:
54
+ id: Run id, `YYYYMMDD-HHMMSS-<6 hex>`.
55
+ submitted_by: Submitting user.
56
+ owner: Owner of the run.
57
+ agent: Agent which submitted the run.
58
+ priority: Priority in the queue.
59
+ group: Group of the run.
60
+ workflow: The workflow.
61
+ sources: Provenance of the snapshots of the sources.
62
+ image: Container image.
63
+ reserve: Reservation.
64
+ state: State of the run.
65
+ tasks: State of every task.
66
+ plans: The task plans as dicts.
67
+ backend_name: Name of the backend workflow.
68
+ created: Time of creation.
69
+ built: Time the image of the run was ready and the submission of its workflow
70
+ began, None while its environment is resolved and built.
71
+ finished: Time of the end.
72
+ message: A message, e.g. the failure.
73
+ oom_kills: The attempts of the pods killed for memory per worker (the worker
74
+ id, the task id of an unsplit task, `<task>-merge` of a merge).
75
+ pending_release: A reserved node whose release failed, retried by the service.
76
+ pending_cancel: Whether the workflow must still be cancelled, e.g. one which
77
+ may have been created by a submission that failed.
78
+ pending_logs: Whether the logs of the pods of an ended run must still be
79
+ kept in the store (`simrun.service.logs`).
80
+ """
81
+
82
+ id: str
83
+ submitted_by: str
84
+ owner: str
85
+ agent: str | None
86
+ priority: Priority
87
+ group: str | None
88
+ workflow: Workflow
89
+ sources: list[SourceProvenance]
90
+ image: str | None
91
+ reserve: str | None
92
+ state: RunState
93
+ tasks: dict[str, TaskState]
94
+ plans: dict[str, Any]
95
+ backend_name: str | None
96
+ created: datetime
97
+ finished: datetime | None
98
+ message: str | None
99
+ built: datetime | None = None
100
+ oom_kills: dict[str, list[int]] = Field(default_factory=dict)
101
+ pending_release: str | None = None
102
+ pending_cancel: bool = False
103
+ pending_logs: bool = False
104
+
105
+
106
+ class RunSummary(BaseModel):
107
+ """The state of a run without its workflow and plans."""
108
+
109
+ id: str
110
+ state: RunState
111
+ group: str | None
112
+ submitted_by: str
113
+ agent: str | None
114
+ priority: Priority
115
+ tasks: dict[str, TaskState]
116
+ created: datetime
117
+ finished: datetime | None
118
+ message: str | None
119
+
120
+ @classmethod
121
+ def of(cls, record: RunRecord) -> RunSummary:
122
+ """The summary of a record."""
123
+ return cls.model_validate(record.model_dump(include=set(cls.model_fields)))
124
+
125
+
126
+ class RunStore:
127
+ """The run store on the shared drive."""
128
+
129
+ def __init__(self, root: Path) -> None:
130
+ """Create a store.
131
+
132
+ Args:
133
+ root: Root directory of the store.
134
+ """
135
+ self.root = root
136
+
137
+ def new_run_id(self) -> str:
138
+ """Return a new run id, `YYYYMMDD-HHMMSS-<6 hex>`."""
139
+ return f"{datetime.now(UTC):%Y%m%d-%H%M%S}-{secrets.token_hex(3)}"
140
+
141
+ def run_dir(self, run_id: str) -> Path:
142
+ """Return the directory of a run."""
143
+ return self.root / "runs" / run_id
144
+
145
+ def task_paths(
146
+ self, run_id: str, task_id: str, inputs: dict[str, Path], scratch: Path
147
+ ) -> TaskPaths:
148
+ """Return the locations of a task of a run."""
149
+ run_dir = self.run_dir(run_id)
150
+ task_dir = run_dir / "tasks" / task_id
151
+ return TaskPaths(
152
+ store=self.root,
153
+ run_dir=run_dir,
154
+ task_dir=task_dir,
155
+ out=task_dir / "out",
156
+ inputs=inputs,
157
+ scratch=scratch,
158
+ )
159
+
160
+ def write(self, record: RunRecord) -> None:
161
+ """Write a record atomically (temporary file and rename)."""
162
+ run_dir = self.run_dir(record.id)
163
+ run_dir.mkdir(parents=True, exist_ok=True)
164
+ tmp = run_dir / f".{RECORD_FILE}.{secrets.token_hex(4)}.tmp"
165
+ tmp.write_text(record.model_dump_json(indent=2))
166
+ os.replace(tmp, run_dir / RECORD_FILE)
167
+
168
+ def read(self, run_id: str) -> RunRecord:
169
+ """Read a record.
170
+
171
+ Raises:
172
+ KeyError: When the run does not exist.
173
+ """
174
+ path = self.run_dir(run_id) / RECORD_FILE
175
+ try:
176
+ text = path.read_text()
177
+ except FileNotFoundError:
178
+ raise KeyError(run_id) from None
179
+ return RunRecord.model_validate_json(text)
180
+
181
+ def iter_records(self) -> Iterator[RunRecord]:
182
+ """Iterate over the records of all runs, ordered by id."""
183
+ runs = self.root / "runs"
184
+ if not runs.is_dir():
185
+ return
186
+ for path in sorted(runs.iterdir()):
187
+ try:
188
+ yield self.read(path.name)
189
+ except KeyError:
190
+ continue
simrun/testing.py ADDED
@@ -0,0 +1,263 @@
1
+ """A backend in memory for tests of the service, advanced by the test."""
2
+
3
+ import dataclasses
4
+ import os
5
+ import threading
6
+ import time
7
+ from collections.abc import Iterable
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from simrun.planner import Node
12
+ from simrun.service.backend import (
13
+ FINAL_PHASES,
14
+ BackendStatus,
15
+ FailedAttempt,
16
+ Placement,
17
+ PodLog,
18
+ )
19
+
20
+ # the tokens of the tests of the service: the owner, the owner's agent, another
21
+ # person and the agent of the other person
22
+ TOKENS: dict[str, dict[str, str]] = {
23
+ "t-mk": {"name": "mk", "kind": "person"},
24
+ "t-mk-bot": {"name": "mk-bot", "kind": "agent", "owner": "mk"},
25
+ "t-alice": {"name": "alice", "kind": "person"},
26
+ "t-alice-bot": {"name": "alice-bot", "kind": "agent", "owner": "alice"},
27
+ }
28
+
29
+ # the nodes of the tests of the planner: one large server and eleven small ones
30
+ TEST_NODES = [Node("srv-01", 104, 1000)] + [
31
+ Node(f"srv-{i:02d}", 40, 188) for i in range(2, 13)
32
+ ]
33
+
34
+
35
+ def dag_nodes(manifest: dict[str, Any]) -> list[str]:
36
+ """The names of the DAG nodes of a workflow manifest, as Argo names them.
37
+
38
+ A DAG task over a sequence is one node per item, `<task>(<index>:<item>)`.
39
+ """
40
+ spec = manifest["spec"]
41
+ dag = next(t for t in spec["templates"] if t["name"] == spec["entrypoint"])["dag"]
42
+ names: list[str] = []
43
+ for task in dag["tasks"]:
44
+ sequence = task.get("withSequence")
45
+ if sequence is None:
46
+ names.append(task["name"])
47
+ continue
48
+ start = int(sequence.get("start", 0))
49
+ for index in range(int(sequence["count"])):
50
+ names.append(f"{task['name']}({index}:{start + index})")
51
+ return names
52
+
53
+
54
+ class FakeBackend:
55
+ """A backend whose workflows change only when the test says so.
56
+
57
+ A submitted workflow is `Pending` without nodes. `start` sets every node
58
+ running, `finish` every node succeeded, `fail` and `set_node` one node,
59
+ `fail_attempt` and `oom_kill` fail attempts of a pod; `finish_after` finishes a
60
+ workflow that many seconds after its submission. The attempts of a pod are
61
+ numbered from 0 in the order they fail.
62
+ The calls which change the cluster are recorded in `calls`.
63
+
64
+ Attributes:
65
+ manifests: The submitted manifests by name.
66
+ calls: `("submit", name)`, `("cancel", name)`, `("stop", name)`,
67
+ `("logs", name, task_id)`, `("reserve", node, run_id)`,
68
+ `("unreserve", node, run_id)` and `("build_image", ref, files of the
69
+ context)`.
70
+ images: The images of the registry.
71
+ pod_logs: The logs of the pods by workflow name and task id, see `add_log`.
72
+ """
73
+
74
+ def __init__(
75
+ self,
76
+ nodes: list[Node] | None = None,
77
+ finish_after: float | None = None,
78
+ images: Iterable[str] = (),
79
+ ) -> None:
80
+ """Create the backend.
81
+
82
+ Args:
83
+ nodes: The nodes of the cluster, `TEST_NODES` by default.
84
+ finish_after: Seconds after which a submitted workflow succeeds.
85
+ images: The images the registry has.
86
+ """
87
+ self._lock = threading.Lock()
88
+ self._nodes = {n.name: n for n in (nodes or TEST_NODES)}
89
+ self.finish_after = finish_after
90
+ self.images = set(images)
91
+ self.manifests: dict[str, dict[str, Any]] = {}
92
+ self.calls: list[tuple[Any, ...]] = []
93
+ self.pod_logs: dict[tuple[str, str], list[PodLog]] = {}
94
+ self._status: dict[str, BackendStatus] = {}
95
+ self._submitted: dict[str, float] = {}
96
+ self._placed: list[Placement] = []
97
+
98
+ def submit(self, manifest: dict[str, Any]) -> str:
99
+ """Accept a workflow, pending without nodes; one which exists is kept."""
100
+ name = manifest["metadata"]["name"]
101
+ with self._lock:
102
+ if name in self.manifests:
103
+ self.calls.append(("submit", name))
104
+ return name
105
+ self.manifests[name] = manifest
106
+ self._status[name] = BackendStatus(phase="Pending", tasks={})
107
+ self._submitted[name] = time.monotonic()
108
+ self.calls.append(("submit", name))
109
+ return name
110
+
111
+ def status(self, name: str) -> BackendStatus:
112
+ """The status of a workflow; KeyError for an unknown one."""
113
+ with self._lock:
114
+ status = self._status[name]
115
+ elapsed = time.monotonic() - self._submitted[name]
116
+ if (
117
+ self.finish_after is not None
118
+ and elapsed >= self.finish_after
119
+ and status.phase not in FINAL_PHASES
120
+ ):
121
+ self._set_all(name, "Succeeded")
122
+ return self._status[name].model_copy(deep=True)
123
+
124
+ def _set_all(self, name: str, phase: str) -> None:
125
+ status = self._status[name]
126
+ status.phase = phase
127
+ status.tasks = dict.fromkeys(dag_nodes(self.manifests[name]), phase)
128
+
129
+ def start(self, name: str) -> None:
130
+ """Set every node of a workflow running."""
131
+ with self._lock:
132
+ self._set_all(name, "Running")
133
+
134
+ def finish(self, name: str) -> None:
135
+ """Let every node of a workflow succeed."""
136
+ with self._lock:
137
+ self._set_all(name, "Succeeded")
138
+
139
+ def set_node(self, name: str, node: str, phase: str) -> None:
140
+ """Set the phase of one node."""
141
+ with self._lock:
142
+ self._status[name].tasks[node] = phase
143
+
144
+ def fail(self, name: str, node: str) -> None:
145
+ """Fail a node, given by its name or by a task id (its coordinator `-w0`)."""
146
+ with self._lock:
147
+ nodes = dag_nodes(self.manifests[name])
148
+ target = node if node in nodes else f"{node}-w0"
149
+ self._status[name].tasks[target] = "Failed"
150
+
151
+ def delete(self, name: str) -> None:
152
+ """Remove a workflow, as its time to live does."""
153
+ with self._lock:
154
+ del self._status[name]
155
+
156
+ def fail_attempt(
157
+ self,
158
+ name: str,
159
+ worker: str,
160
+ exit_code: int | None = None,
161
+ oom_killed: bool = False,
162
+ ) -> None:
163
+ """Fail the next attempt of a pod of a worker, merge or unsplit task."""
164
+ with self._lock:
165
+ attempts = self._status[name].failed.setdefault(worker, [])
166
+ attempts.append(
167
+ FailedAttempt(
168
+ attempt=len(attempts), exit_code=exit_code, oom_killed=oom_killed
169
+ )
170
+ )
171
+
172
+ def oom_kill(self, name: str, worker: str, count: int = 1) -> None:
173
+ """Let the next attempts of a pod be killed by the out of memory killer."""
174
+ for _ in range(count):
175
+ self.fail_attempt(name, worker, exit_code=137, oom_killed=True)
176
+
177
+ def cancel(self, name: str) -> None:
178
+ """Terminate a workflow."""
179
+ with self._lock:
180
+ self.calls.append(("cancel", name))
181
+ self._status[name].phase = "Failed"
182
+
183
+ def stop(self, name: str) -> None:
184
+ """Stop a workflow."""
185
+ with self._lock:
186
+ self.calls.append(("stop", name))
187
+ self._status[name].phase = "Failed"
188
+
189
+ def add_log(
190
+ self, name: str, task_id: str, node: str, text: str, done: bool = True
191
+ ) -> None:
192
+ """Add the log of a pod of a task, the pod named `<workflow>-<node>`."""
193
+ with self._lock:
194
+ pod = PodLog(node=node, pod=f"{name}-{node}", text=text, done=done)
195
+ self.pod_logs.setdefault((name, task_id), []).append(pod)
196
+
197
+ def logs(
198
+ self, name: str, task_id: str, tail_lines: int | None = None
199
+ ) -> list[PodLog]:
200
+ """The logs added with `add_log`, the last `tail_lines` lines of each."""
201
+ with self._lock:
202
+ self.calls.append(("logs", name, task_id))
203
+ logs = list(self.pod_logs.get((name, task_id), []))
204
+ if tail_lines is None:
205
+ return logs
206
+ return [
207
+ log.model_copy(
208
+ update={"text": "".join(log.text.splitlines(True)[-tail_lines:])}
209
+ )
210
+ for log in logs
211
+ ]
212
+
213
+ def nodes(self) -> list[Node]:
214
+ """The nodes, with their reservations."""
215
+ with self._lock:
216
+ return list(self._nodes.values())
217
+
218
+ def place(self, placement: Placement) -> None:
219
+ """Place a pod, listed by `placements` until its workflow has ended.
220
+
221
+ A placement of a pod placed before replaces it, e.g. once the pod is bound.
222
+ """
223
+ with self._lock:
224
+ self._placed = [p for p in self._placed if p.pod != placement.pod]
225
+ self._placed.append(placement)
226
+
227
+ def placements(self) -> list[Placement]:
228
+ """The pods placed for the workflows which have not ended."""
229
+ with self._lock:
230
+ return [
231
+ p
232
+ for p in self._placed
233
+ if p.workflow in self._status
234
+ and self._status[p.workflow].phase not in FINAL_PHASES
235
+ ]
236
+
237
+ def reserve(self, node: str, run_id: str) -> None:
238
+ """Reserve a node; KeyError for an unknown one."""
239
+ with self._lock:
240
+ self._nodes[node] = dataclasses.replace(
241
+ self._nodes[node], reserved_by=run_id
242
+ )
243
+ self.calls.append(("reserve", node, run_id))
244
+
245
+ def unreserve(self, node: str, run_id: str) -> None:
246
+ """Release a node reserved by the run; KeyError for an unknown node."""
247
+ with self._lock:
248
+ self.calls.append(("unreserve", node, run_id))
249
+ if self._nodes[node].reserved_by == run_id:
250
+ self._nodes[node] = dataclasses.replace(
251
+ self._nodes[node], reserved_by=None
252
+ )
253
+
254
+ def build_image(self, context: Path, ref: str) -> None:
255
+ """Record the build and add the image to the registry."""
256
+ with self._lock:
257
+ self.calls.append(("build_image", ref, sorted(os.listdir(context))))
258
+ self.images.add(ref)
259
+
260
+ def has_image(self, ref: str) -> bool:
261
+ """Whether the image was given or built."""
262
+ with self._lock:
263
+ return ref in self.images