autoforge-cli 0.1.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- autoforge/__init__.py +5 -0
- autoforge/agent_worktree.py +653 -0
- autoforge/certificate.py +495 -0
- autoforge/child_env.py +871 -0
- autoforge/claims.py +726 -0
- autoforge/claude_stream.py +501 -0
- autoforge/cli.py +2884 -0
- autoforge/config.py +2178 -0
- autoforge/config_fingerprint.py +374 -0
- autoforge/doctor.py +1551 -0
- autoforge/doctor_sandbox.py +691 -0
- autoforge/effect_ops.py +1272 -0
- autoforge/effects.py +3210 -0
- autoforge/engine.py +16637 -0
- autoforge/epic_commands.py +1486 -0
- autoforge/epic_graph.py +852 -0
- autoforge/errors.py +536 -0
- autoforge/evidence.py +1722 -0
- autoforge/executor.py +1601 -0
- autoforge/executor_duplex.py +985 -0
- autoforge/executor_supervised.py +144 -0
- autoforge/git_transport.py +970 -0
- autoforge/github.py +3402 -0
- autoforge/interaction.py +1027 -0
- autoforge/local_workspace.py +1253 -0
- autoforge/locking.py +1050 -0
- autoforge/loop_detect.py +660 -0
- autoforge/loop_guard.py +404 -0
- autoforge/materialize.py +722 -0
- autoforge/ownership.py +240 -0
- autoforge/pi_rpc.py +713 -0
- autoforge/premerge.py +181 -0
- autoforge/profiles.py +106 -0
- autoforge/progress.py +392 -0
- autoforge/prompts/__init__.py +173 -0
- autoforge/prompts/acceptance_actions.md +39 -0
- autoforge/prompts/analyze_execute.md +128 -0
- autoforge/prompts/common.md +132 -0
- autoforge/prompts/correction.md +12 -0
- autoforge/prompts/fix.md +252 -0
- autoforge/prompts/local_analyze_execute.md +69 -0
- autoforge/prompts/local_common.md +149 -0
- autoforge/prompts/local_fix.md +96 -0
- autoforge/prompts/local_review.md +103 -0
- autoforge/prompts/operator_acceptance.md +61 -0
- autoforge/prompts/replan_reexecute.md +991 -0
- autoforge/prompts/review.md +312 -0
- autoforge/prompts/update_epic.md +120 -0
- autoforge/prompts/update_epic_rerequest.md +85 -0
- autoforge/prompts/validation_evidence.md +26 -0
- autoforge/prompts/validation_failure.md +22 -0
- autoforge/prompts/worktree_state.md +45 -0
- autoforge/providers.py +2893 -0
- autoforge/read_context.py +389 -0
- autoforge/redaction.py +408 -0
- autoforge/replan.py +208 -0
- autoforge/replan_txn.py +1930 -0
- autoforge/result_parser.py +2234 -0
- autoforge/roadmap.py +151 -0
- autoforge/run_contract.py +408 -0
- autoforge/run_namespace.py +839 -0
- autoforge/runlog.py +764 -0
- autoforge/safefs.py +1717 -0
- autoforge/sandbox.py +1313 -0
- autoforge/sandbox_bwrap.py +815 -0
- autoforge/sandbox_pin.py +170 -0
- autoforge/sandbox_policy.py +1107 -0
- autoforge/scheduler.py +1143 -0
- autoforge/scheduler_state.py +2316 -0
- autoforge/scheduler_stop.py +421 -0
- autoforge/sdk_policy.py +988 -0
- autoforge/state.py +2417 -0
- autoforge/transitions.py +388 -0
- autoforge/validation.py +324 -0
- autoforge/worker.py +568 -0
- autoforge_cli-0.1.0a0.dist-info/METADATA +525 -0
- autoforge_cli-0.1.0a0.dist-info/RECORD +81 -0
- autoforge_cli-0.1.0a0.dist-info/WHEEL +5 -0
- autoforge_cli-0.1.0a0.dist-info/entry_points.txt +2 -0
- autoforge_cli-0.1.0a0.dist-info/licenses/LICENSE +21 -0
- autoforge_cli-0.1.0a0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2316 @@
|
|
|
1
|
+
"""Scheduler claims, the per-EPIC scheduler state and restart reconciliation (ADR 0008; #181).
|
|
2
|
+
|
|
3
|
+
The parallel EPIC scheduler's durable, local bookkeeping. None of it is live
|
|
4
|
+
authority: an issue has a live owner exactly when some process holds its run
|
|
5
|
+
lock (L2, :class:`~autoforge.locking.IssueRunLock`). A claim is recovery
|
|
6
|
+
data that says which generation of which invocation was admitted, so that a
|
|
7
|
+
restarted scheduler can tell an interrupted *assignment* from a settled one.
|
|
8
|
+
|
|
9
|
+
* **``claim.json``** (:class:`ClaimFile`, ``autoforge.scheduler-claim/1``,
|
|
10
|
+
D7), one per issue namespace: ``run_key``, ``issue_id``, ``epic`` (copied
|
|
11
|
+
from the namespace's binding), ``next_generation``, ``current`` (the one
|
|
12
|
+
open :class:`Claim`, or none) and ``history`` (at most
|
|
13
|
+
:data:`MAX_HISTORY` :class:`ClosedClaim`, oldest dropped). A claim is
|
|
14
|
+
written before a worker exists and closed only once the worker's outcome
|
|
15
|
+
is durably observable.
|
|
16
|
+
* **One closing rule** (D7.2). ``superseded`` only in the atomic write that
|
|
17
|
+
installs the replacing claim (:func:`open_claim` with ``supersedes``); a
|
|
18
|
+
claim closed without a replacement is ``abandoned`` when no worker was
|
|
19
|
+
recorded on it and ``interrupted`` when one was; ``settled`` when the
|
|
20
|
+
state's settlement is not ``in_progress``; ``archived`` by ``epic
|
|
21
|
+
archive``. The writers enforce it and the loader refuses a file that
|
|
22
|
+
breaks it.
|
|
23
|
+
* **The writers.** The only writer is the L0-holding scheduler. Every
|
|
24
|
+
operation that opens or closes a claim takes the held
|
|
25
|
+
:class:`~autoforge.locking.IssueRunLock` of the issue as a capability
|
|
26
|
+
argument, and every write compares ``(claim_id, generation)`` on disk with
|
|
27
|
+
the caller's record first: a mismatch is :class:`~autoforge.errors.ClaimConflict`,
|
|
28
|
+
never an overwrite, so a stale claimant cannot finalize a newer
|
|
29
|
+
generation. Writes are atomic (fsynced temporary, replace) through
|
|
30
|
+
``safefs`` from the coordination root, every component ``O_NOFOLLOW``,
|
|
31
|
+
and refused while this process holds L1 (D11.2).
|
|
32
|
+
* **``scheduler.json``** (:class:`SchedulerState`, ``autoforge.scheduler/1``,
|
|
33
|
+
D6), one per EPIC under ``scheduler/epics/<n>/``: run-level facts only. It
|
|
34
|
+
holds no claim, so the two files have nothing to disagree about. A corrupt
|
|
35
|
+
one is the scheduler's global uncertainty; ``epic run --force``
|
|
36
|
+
quarantines it (:func:`quarantine_scheduler_state`).
|
|
37
|
+
* **Restart classification** (D14.2). :func:`classify` is pure: given #178's
|
|
38
|
+
strict lookup and the claim file, read under L2, a namespace is
|
|
39
|
+
``uncertain(reason)``, ``settled(settlement)``, ``resumable(reason)`` or
|
|
40
|
+
``fresh``, checked in that order. The scheduler sees a state only through
|
|
41
|
+
:func:`~autoforge.transitions.issue_settlement`.
|
|
42
|
+
* **Reconciliation** (:func:`reconcile_epic`) runs under L0 before any
|
|
43
|
+
admission: it takes each namespace's L2 non-blocking (or reports it
|
|
44
|
+
``observed_live``), classifies, closes an open claim of a settled
|
|
45
|
+
namespace as ``settled`` and keeps every other claim open, and returns the
|
|
46
|
+
local facts #179's ``evaluate`` consumes. :func:`finalize_worker` closes a
|
|
47
|
+
reaped worker's claim by the state it left.
|
|
48
|
+
|
|
49
|
+
Nothing here spawns a process, reads or writes GitHub, writes or quarantines
|
|
50
|
+
a per-issue state, or parses ``CONTROL_RESULT``; timestamps and PIDs are
|
|
51
|
+
diagnostics only. ``detail`` texts are redacted and bounded.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
from __future__ import annotations
|
|
55
|
+
|
|
56
|
+
import json
|
|
57
|
+
import re
|
|
58
|
+
import secrets
|
|
59
|
+
from collections.abc import Collection, Iterable, Mapping
|
|
60
|
+
from dataclasses import dataclass, field, replace
|
|
61
|
+
from datetime import datetime
|
|
62
|
+
from enum import Enum
|
|
63
|
+
from pathlib import Path
|
|
64
|
+
from typing import NoReturn, TypeVar
|
|
65
|
+
|
|
66
|
+
from .config import MAX_PARALLEL_BOUNDS, POLL_INTERVAL_BOUNDS, STOP_GRACE_BOUNDS
|
|
67
|
+
from .epic_graph import LocalFact
|
|
68
|
+
from .errors import (
|
|
69
|
+
ClaimConflict,
|
|
70
|
+
ConfigurationError,
|
|
71
|
+
IssueBusy,
|
|
72
|
+
LockError,
|
|
73
|
+
NamespaceMismatch,
|
|
74
|
+
NetworkFilesystemError,
|
|
75
|
+
SchedulerStateError,
|
|
76
|
+
StateError,
|
|
77
|
+
)
|
|
78
|
+
from .locking import IssueRunLock, assert_shared_git_lock_free
|
|
79
|
+
from .redaction import redact
|
|
80
|
+
from .run_namespace import (
|
|
81
|
+
AUTOFORGE_DIRNAME,
|
|
82
|
+
CLAIM_FILENAME,
|
|
83
|
+
MAX_ID,
|
|
84
|
+
MAX_OWNER_CHARS,
|
|
85
|
+
MAX_REPOSITORY_NAME_CHARS,
|
|
86
|
+
MAX_TIMESTAMP_CHARS,
|
|
87
|
+
MAX_WORKTREE_PATH_CHARS,
|
|
88
|
+
SCHEDULER_DIRNAME,
|
|
89
|
+
EpicRef,
|
|
90
|
+
IssueNamespace,
|
|
91
|
+
NamespaceKind,
|
|
92
|
+
NamespaceLookup,
|
|
93
|
+
RejectedEntry,
|
|
94
|
+
RunBinding,
|
|
95
|
+
RunKey,
|
|
96
|
+
StateStatus,
|
|
97
|
+
check_binding,
|
|
98
|
+
list_issue_namespaces,
|
|
99
|
+
load_issue_namespace,
|
|
100
|
+
)
|
|
101
|
+
from .runlog import RUN_ID_RE
|
|
102
|
+
from .safefs import ReadLimitExceeded, SafeRoot, entry_kind
|
|
103
|
+
from .state import AutoForgeState, archive_state_file, quarantine_state_file, utcnow_iso
|
|
104
|
+
from .transitions import Settlement, WorkflowMode, issue_settlement
|
|
105
|
+
from .validation import parse_issue_url
|
|
106
|
+
|
|
107
|
+
CLAIM_SCHEMA = "autoforge.scheduler-claim/1"
|
|
108
|
+
SCHEDULER_SCHEMA = "autoforge.scheduler/1"
|
|
109
|
+
EPICS_DIRNAME = "epics"
|
|
110
|
+
SCHEDULER_FILENAME = "scheduler.json"
|
|
111
|
+
|
|
112
|
+
#: Closed claims kept in ``history``; the oldest is dropped past it (D7.1).
|
|
113
|
+
MAX_HISTORY = 16
|
|
114
|
+
#: A closed claim's ``detail``, after redaction (D7.1).
|
|
115
|
+
MAX_DETAIL_CHARS = 500
|
|
116
|
+
# A classification's or a report's explanation quotes a loader's message:
|
|
117
|
+
# redacted, then bounded. Never persisted.
|
|
118
|
+
MAX_REASON_CHARS = 1000
|
|
119
|
+
# A claim file holds at most 17 claims of a few kilobytes each, its worst
|
|
120
|
+
# case JSON escaping of a PATH_MAX log path included; a larger file is
|
|
121
|
+
# refused unread, and a writer never produces one.
|
|
122
|
+
MAX_CLAIM_BYTES = 1024 * 1024
|
|
123
|
+
# A scheduler state is under a kilobyte.
|
|
124
|
+
MAX_SCHEDULER_BYTES = 16 * 1024
|
|
125
|
+
# A worker's exit status as Popen reports it: 0-255, or minus a signal number.
|
|
126
|
+
MAX_EXIT_STATUS = 255
|
|
127
|
+
# A last snapshot's counts are bounded by the namespaces one EPIC can have.
|
|
128
|
+
MAX_COUNT = 10_000
|
|
129
|
+
# A refusal quotes at most this many unknown field names, each cut to 80 characters.
|
|
130
|
+
MAX_QUOTED_NAMES = 5
|
|
131
|
+
# A pid fits a 32-bit signed integer on every platform the scheduler runs on.
|
|
132
|
+
MAX_PID = 2**31 - 1
|
|
133
|
+
|
|
134
|
+
# ADR 0008 D18.3, D18.4: the bounds a scheduler's configuration is held to are
|
|
135
|
+
# config.py's (MAX_PARALLEL_BOUNDS, POLL_INTERVAL_BOUNDS, STOP_GRACE_BOUNDS), the
|
|
136
|
+
# ones the ``scheduler.*`` keys are validated against.
|
|
137
|
+
|
|
138
|
+
_HEX32 = re.compile(r"[0-9a-f]{32}")
|
|
139
|
+
_HEX64 = re.compile(r"[0-9a-f]{64}")
|
|
140
|
+
_NAME_RE = re.compile(r"[A-Za-z0-9_.-]+")
|
|
141
|
+
|
|
142
|
+
E = TypeVar("E", bound=Enum)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# -- field checks -----------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _fail(what: str, message: str) -> NoReturn:
|
|
149
|
+
raise SchedulerStateError(f"{what} {message}")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _quoted_names(names: list[str]) -> str:
|
|
153
|
+
shown = ", ".join(f"{name!r:.80}" for name in names[:MAX_QUOTED_NAMES])
|
|
154
|
+
more = len(names) - MAX_QUOTED_NAMES
|
|
155
|
+
return f"[{shown}]" if more <= 0 else f"[{shown}] and {more} more"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _exact(raw: object, keys: Iterable[str], what: str) -> dict:
|
|
159
|
+
if not isinstance(raw, dict):
|
|
160
|
+
_fail(what, f"must be an object, got {type(raw).__name__}")
|
|
161
|
+
wanted = tuple(keys)
|
|
162
|
+
unknown = sorted(str(k) for k in raw if k not in wanted)
|
|
163
|
+
if unknown:
|
|
164
|
+
_fail(what, f"has unknown field(s) {_quoted_names(unknown)}")
|
|
165
|
+
missing = [k for k in wanted if k not in raw]
|
|
166
|
+
if missing:
|
|
167
|
+
_fail(what, f"is missing field(s) {missing}")
|
|
168
|
+
return raw
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _check_int(value: object, what: str, low: int, high: int) -> int:
|
|
172
|
+
if not isinstance(value, int) or isinstance(value, bool):
|
|
173
|
+
_fail(what, f"must be an integer, got {type(value).__name__}")
|
|
174
|
+
if value < low or value > high:
|
|
175
|
+
got = str(value) if len(str(abs(value))) <= 40 else f"{len(str(abs(value)))} digits"
|
|
176
|
+
_fail(what, f"must be an integer from {low} to {high}, got {got}")
|
|
177
|
+
return value
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _check_id(value: object, what: str) -> int:
|
|
181
|
+
return _check_int(value, what, 1, MAX_ID)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _check_bool(value: object, what: str) -> bool:
|
|
185
|
+
if not isinstance(value, bool):
|
|
186
|
+
_fail(what, f"must be a boolean, got {type(value).__name__}")
|
|
187
|
+
return value
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _check_str(value: object, what: str, max_chars: int, *, empty: bool = False) -> str:
|
|
191
|
+
if not isinstance(value, str):
|
|
192
|
+
_fail(what, f"must be a string, got {type(value).__name__}")
|
|
193
|
+
if not value and not empty:
|
|
194
|
+
_fail(what, "must not be empty")
|
|
195
|
+
if len(value) > max_chars:
|
|
196
|
+
_fail(what, f"is {len(value)} characters, over its bound of {max_chars}")
|
|
197
|
+
return value
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _check_hex(value: object, what: str, pattern: re.Pattern[str]) -> str:
|
|
201
|
+
text = _check_str(value, what, 64)
|
|
202
|
+
if not pattern.fullmatch(text):
|
|
203
|
+
digits = 32 if pattern is _HEX32 else 64
|
|
204
|
+
_fail(what, f"must be {digits} lowercase hexadecimal characters, got {text!r:.80}")
|
|
205
|
+
return text
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _check_timestamp(value: object, what: str) -> str:
|
|
209
|
+
text = _check_str(value, what, MAX_TIMESTAMP_CHARS)
|
|
210
|
+
try:
|
|
211
|
+
moment = datetime.fromisoformat(text)
|
|
212
|
+
except ValueError:
|
|
213
|
+
_fail(what, f"is not an ISO 8601 time: {text!r}")
|
|
214
|
+
offset = moment.utcoffset()
|
|
215
|
+
if offset is None or offset.total_seconds() != 0 or "T" not in text:
|
|
216
|
+
_fail(what, f"must be an ISO 8601 UTC time as utcnow_iso writes it, got {text!r}")
|
|
217
|
+
return text
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _check_path(value: object, what: str) -> str:
|
|
221
|
+
text = _check_str(value, what, MAX_WORKTREE_PATH_CHARS)
|
|
222
|
+
if not text.startswith("/") or any(ord(c) < 0x20 or ord(c) == 0x7F for c in text):
|
|
223
|
+
_fail(what, "must be an absolute path without control characters")
|
|
224
|
+
return text
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _check_name(value: object, what: str, max_chars: int) -> str:
|
|
228
|
+
text = _check_str(value, what, max_chars)
|
|
229
|
+
if not _NAME_RE.fullmatch(text) or text in (".", ".."):
|
|
230
|
+
_fail(what, f"is not a GitHub owner or repository name: {text!r}")
|
|
231
|
+
return text
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _check_enum(cls: type[E], value: object, what: str) -> E:
|
|
235
|
+
if isinstance(value, cls):
|
|
236
|
+
return value
|
|
237
|
+
try:
|
|
238
|
+
return cls(value)
|
|
239
|
+
except ValueError:
|
|
240
|
+
allowed = ", ".join(repr(member.value) for member in cls)
|
|
241
|
+
_fail(what, f"must be one of {allowed}, got {value!r:.80}")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _check_type(value: object, cls: type, what: str) -> None:
|
|
245
|
+
if not isinstance(value, cls):
|
|
246
|
+
_fail(what, f"must be a {cls.__name__}, got {type(value).__name__}")
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _bounded(text: str, limit: int = MAX_REASON_CHARS) -> str:
|
|
250
|
+
text = redact(text)
|
|
251
|
+
if len(text) <= limit:
|
|
252
|
+
return text
|
|
253
|
+
marker = " [... truncated]"
|
|
254
|
+
return text[: limit - len(marker)] + marker
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def bounded_detail(text: str) -> str:
|
|
258
|
+
"""``text`` as a closed claim may hold it: redacted, then at most :data:`MAX_DETAIL_CHARS`."""
|
|
259
|
+
return _bounded(text, MAX_DETAIL_CHARS)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def new_invocation_id() -> str:
|
|
263
|
+
"""A fresh ``invocation_id``: 32 lowercase hex characters, random (D6.1)."""
|
|
264
|
+
return secrets.token_hex(16)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _epic_dict(epic: EpicRef) -> dict:
|
|
268
|
+
return {"number": epic.number, "issue_id": epic.issue_id}
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _epic_of(raw: object, what: str) -> EpicRef:
|
|
272
|
+
data = _exact(raw, ("number", "issue_id"), what)
|
|
273
|
+
_check_id(data["number"], f"{what} number")
|
|
274
|
+
_check_id(data["issue_id"], f"{what} issue_id")
|
|
275
|
+
return EpicRef(data["number"], data["issue_id"])
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
# -- claim.json -------------------------------------------------------------------------
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class ClaimKind(Enum):
|
|
282
|
+
"""What an admission launches: a fresh issue, or an ordinary engine resume (D14.4)."""
|
|
283
|
+
|
|
284
|
+
START = "start"
|
|
285
|
+
RESUME = "resume"
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
class ClaimClassification(Enum):
|
|
289
|
+
"""How a claim was closed (D7.1, D7.2)."""
|
|
290
|
+
|
|
291
|
+
SETTLED = "settled"
|
|
292
|
+
INTERRUPTED = "interrupted"
|
|
293
|
+
SUPERSEDED = "superseded"
|
|
294
|
+
ABANDONED = "abandoned"
|
|
295
|
+
ARCHIVED = "archived"
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass(frozen=True)
|
|
299
|
+
class Admission:
|
|
300
|
+
"""The snapshot an admission was decided on (#179's digests): diagnostics."""
|
|
301
|
+
|
|
302
|
+
snapshot_digest: str
|
|
303
|
+
node_digest: str
|
|
304
|
+
observed_at: str
|
|
305
|
+
|
|
306
|
+
def __post_init__(self) -> None:
|
|
307
|
+
_check_hex(self.snapshot_digest, "admission snapshot_digest", _HEX64)
|
|
308
|
+
_check_hex(self.node_digest, "admission node_digest", _HEX64)
|
|
309
|
+
_check_timestamp(self.observed_at, "admission observed_at")
|
|
310
|
+
|
|
311
|
+
def to_dict(self) -> dict:
|
|
312
|
+
return {
|
|
313
|
+
"snapshot_digest": self.snapshot_digest,
|
|
314
|
+
"node_digest": self.node_digest,
|
|
315
|
+
"observed_at": self.observed_at,
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
@classmethod
|
|
319
|
+
def from_dict(cls, raw: object) -> Admission:
|
|
320
|
+
data = _exact(raw, ("snapshot_digest", "node_digest", "observed_at"), "claim admission")
|
|
321
|
+
return cls(data["snapshot_digest"], data["node_digest"], data["observed_at"])
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
@dataclass(frozen=True)
|
|
325
|
+
class StateRef:
|
|
326
|
+
"""The namespace's state as it was at admission (D7.1).
|
|
327
|
+
|
|
328
|
+
``present`` is whether ``state/state.json`` existed; ``run_id`` and
|
|
329
|
+
``settlement`` are its run id and settlement then, and both are null
|
|
330
|
+
when it did not. An open claim whose ``present`` is true while the state
|
|
331
|
+
is absent is ``uncertain`` (D14.2).
|
|
332
|
+
"""
|
|
333
|
+
|
|
334
|
+
present: bool
|
|
335
|
+
run_id: str | None = None
|
|
336
|
+
settlement: Settlement | None = None
|
|
337
|
+
|
|
338
|
+
def __post_init__(self) -> None:
|
|
339
|
+
_check_bool(self.present, "claim state_ref present")
|
|
340
|
+
if self.run_id is not None:
|
|
341
|
+
text = _check_str(self.run_id, "claim state_ref run_id", 128)
|
|
342
|
+
if not RUN_ID_RE.match(text):
|
|
343
|
+
_fail("claim state_ref run_id", f"is not a run id: {text!r:.80}")
|
|
344
|
+
if self.settlement is not None:
|
|
345
|
+
_check_type(self.settlement, Settlement, "claim state_ref settlement")
|
|
346
|
+
if not self.present and (self.run_id is not None or self.settlement is not None):
|
|
347
|
+
_fail("claim state_ref", "names a run_id or a settlement while present is false")
|
|
348
|
+
if self.present and self.settlement is None:
|
|
349
|
+
_fail("claim state_ref", "has no settlement while present is true")
|
|
350
|
+
|
|
351
|
+
@classmethod
|
|
352
|
+
def absent(cls) -> StateRef:
|
|
353
|
+
return cls(present=False)
|
|
354
|
+
|
|
355
|
+
@classmethod
|
|
356
|
+
def of(cls, state: AutoForgeState) -> StateRef:
|
|
357
|
+
"""The reference to a state that was loaded at admission."""
|
|
358
|
+
return cls(True, state.run_id or None, issue_settlement(state))
|
|
359
|
+
|
|
360
|
+
@classmethod
|
|
361
|
+
def from_lookup(cls, lookup: NamespaceLookup) -> StateRef:
|
|
362
|
+
"""The reference to the state #178's lookup found; a corrupt one admits nothing."""
|
|
363
|
+
if lookup.state.status is StateStatus.LOADED:
|
|
364
|
+
assert lookup.state.state is not None
|
|
365
|
+
return cls.of(lookup.state.state)
|
|
366
|
+
if lookup.state.status is StateStatus.ABSENT:
|
|
367
|
+
return cls.absent()
|
|
368
|
+
raise SchedulerStateError(
|
|
369
|
+
f"the state of {lookup.namespace.directory} cannot be read, so nothing is admitted "
|
|
370
|
+
f"for it ({_bounded(lookup.state.reason)})"
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
def to_dict(self) -> dict:
|
|
374
|
+
return {
|
|
375
|
+
"present": self.present,
|
|
376
|
+
"run_id": self.run_id,
|
|
377
|
+
"settlement": None if self.settlement is None else self.settlement.value,
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
@classmethod
|
|
381
|
+
def from_dict(cls, raw: object) -> StateRef:
|
|
382
|
+
data = _exact(raw, ("present", "run_id", "settlement"), "claim state_ref")
|
|
383
|
+
settlement = data["settlement"]
|
|
384
|
+
return cls(
|
|
385
|
+
present=data["present"],
|
|
386
|
+
run_id=data["run_id"],
|
|
387
|
+
settlement=None
|
|
388
|
+
if settlement is None
|
|
389
|
+
else _check_enum(Settlement, settlement, "claim state_ref settlement"),
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@dataclass(frozen=True)
|
|
394
|
+
class WorkerDiagnostics:
|
|
395
|
+
"""A spawned worker, as an operator recognises it (D7.3): never authority.
|
|
396
|
+
|
|
397
|
+
``process_start_time`` is the kernel's start time of ``pid``
|
|
398
|
+
(``/proc/<pid>/stat`` field 22 on Linux), null when it could not be
|
|
399
|
+
read; ``log`` the worker log's absolute path.
|
|
400
|
+
"""
|
|
401
|
+
|
|
402
|
+
pid: int
|
|
403
|
+
process_start_time: int | None
|
|
404
|
+
spawned_at: str
|
|
405
|
+
log: str
|
|
406
|
+
|
|
407
|
+
def __post_init__(self) -> None:
|
|
408
|
+
_check_int(self.pid, "claim worker pid", 1, MAX_PID)
|
|
409
|
+
if self.process_start_time is not None:
|
|
410
|
+
_check_int(self.process_start_time, "claim worker process_start_time", 0, MAX_ID)
|
|
411
|
+
_check_timestamp(self.spawned_at, "claim worker spawned_at")
|
|
412
|
+
_check_path(self.log, "claim worker log")
|
|
413
|
+
|
|
414
|
+
def to_dict(self) -> dict:
|
|
415
|
+
return {
|
|
416
|
+
"pid": self.pid,
|
|
417
|
+
"process_start_time": self.process_start_time,
|
|
418
|
+
"spawned_at": self.spawned_at,
|
|
419
|
+
"log": self.log,
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
@classmethod
|
|
423
|
+
def from_dict(cls, raw: object) -> WorkerDiagnostics:
|
|
424
|
+
data = _exact(raw, ("pid", "process_start_time", "spawned_at", "log"), "claim worker")
|
|
425
|
+
return cls(data["pid"], data["process_start_time"], data["spawned_at"], data["log"])
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
_CLAIM_KEYS = (
|
|
429
|
+
"claim_id",
|
|
430
|
+
"generation",
|
|
431
|
+
"invocation_id",
|
|
432
|
+
"kind",
|
|
433
|
+
"admission",
|
|
434
|
+
"state_ref",
|
|
435
|
+
"created_at",
|
|
436
|
+
"worker",
|
|
437
|
+
)
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
@dataclass(frozen=True)
|
|
441
|
+
class Claim:
|
|
442
|
+
"""One generation of one invocation owning an issue (D7.1); recovery data, never liveness."""
|
|
443
|
+
|
|
444
|
+
claim_id: str
|
|
445
|
+
generation: int
|
|
446
|
+
invocation_id: str
|
|
447
|
+
kind: ClaimKind
|
|
448
|
+
admission: Admission
|
|
449
|
+
state_ref: StateRef
|
|
450
|
+
created_at: str
|
|
451
|
+
worker: WorkerDiagnostics | None = None
|
|
452
|
+
|
|
453
|
+
def __post_init__(self) -> None:
|
|
454
|
+
_check_hex(self.claim_id, "claim claim_id", _HEX32)
|
|
455
|
+
_check_id(self.generation, "claim generation")
|
|
456
|
+
_check_hex(self.invocation_id, "claim invocation_id", _HEX32)
|
|
457
|
+
_check_type(self.kind, ClaimKind, "claim kind")
|
|
458
|
+
_check_type(self.admission, Admission, "claim admission")
|
|
459
|
+
_check_type(self.state_ref, StateRef, "claim state_ref")
|
|
460
|
+
_check_timestamp(self.created_at, "claim created_at")
|
|
461
|
+
if self.worker is not None:
|
|
462
|
+
_check_type(self.worker, WorkerDiagnostics, "claim worker")
|
|
463
|
+
|
|
464
|
+
@property
|
|
465
|
+
def identity(self) -> tuple[str, int]:
|
|
466
|
+
"""``(claim_id, generation)``: what every write compares on disk (D7.2)."""
|
|
467
|
+
return (self.claim_id, self.generation)
|
|
468
|
+
|
|
469
|
+
def to_dict(self) -> dict:
|
|
470
|
+
return {
|
|
471
|
+
"claim_id": self.claim_id,
|
|
472
|
+
"generation": self.generation,
|
|
473
|
+
"invocation_id": self.invocation_id,
|
|
474
|
+
"kind": self.kind.value,
|
|
475
|
+
"admission": self.admission.to_dict(),
|
|
476
|
+
"state_ref": self.state_ref.to_dict(),
|
|
477
|
+
"created_at": self.created_at,
|
|
478
|
+
"worker": None if self.worker is None else self.worker.to_dict(),
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
@classmethod
|
|
482
|
+
def from_dict(cls, raw: object, *, what: str = "claim") -> Claim:
|
|
483
|
+
data = _exact(raw, _CLAIM_KEYS, what)
|
|
484
|
+
return cls._of(data)
|
|
485
|
+
|
|
486
|
+
@classmethod
|
|
487
|
+
def _of(cls, data: dict) -> Claim:
|
|
488
|
+
worker = data["worker"]
|
|
489
|
+
return cls(
|
|
490
|
+
claim_id=data["claim_id"],
|
|
491
|
+
generation=data["generation"],
|
|
492
|
+
invocation_id=data["invocation_id"],
|
|
493
|
+
kind=_check_enum(ClaimKind, data["kind"], "claim kind"),
|
|
494
|
+
admission=Admission.from_dict(data["admission"]),
|
|
495
|
+
state_ref=StateRef.from_dict(data["state_ref"]),
|
|
496
|
+
created_at=data["created_at"],
|
|
497
|
+
worker=None if worker is None else WorkerDiagnostics.from_dict(worker),
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
_CLOSED_KEYS = ("at", "classification", "settlement", "exit_status", "detail")
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
@dataclass(frozen=True)
|
|
505
|
+
class ClosedClaim:
|
|
506
|
+
"""A claim moved to ``history``: the claim plus how and when it was closed (D7.1).
|
|
507
|
+
|
|
508
|
+
Constructing one enforces the closing rule (D7.2): ``abandoned`` only
|
|
509
|
+
for a claim without a recorded worker, ``interrupted`` only for one
|
|
510
|
+
with, and ``settled`` only with a settlement other than ``in_progress``.
|
|
511
|
+
``exit_status`` and ``detail`` are diagnostics; ``detail`` is redacted
|
|
512
|
+
and at most :data:`MAX_DETAIL_CHARS`.
|
|
513
|
+
"""
|
|
514
|
+
|
|
515
|
+
claim: Claim
|
|
516
|
+
classification: ClaimClassification
|
|
517
|
+
at: str
|
|
518
|
+
settlement: Settlement | None = None
|
|
519
|
+
exit_status: int | None = None
|
|
520
|
+
detail: str = ""
|
|
521
|
+
|
|
522
|
+
def __post_init__(self) -> None:
|
|
523
|
+
_check_type(self.claim, Claim, "closed claim")
|
|
524
|
+
_check_type(self.classification, ClaimClassification, "closed claim classification")
|
|
525
|
+
_check_timestamp(self.at, "closed claim at")
|
|
526
|
+
if self.settlement is not None:
|
|
527
|
+
_check_type(self.settlement, Settlement, "closed claim settlement")
|
|
528
|
+
if self.exit_status is not None:
|
|
529
|
+
_check_int(
|
|
530
|
+
self.exit_status, "closed claim exit_status", -MAX_EXIT_STATUS, MAX_EXIT_STATUS
|
|
531
|
+
)
|
|
532
|
+
_check_str(self.detail, "closed claim detail", MAX_DETAIL_CHARS, empty=True)
|
|
533
|
+
problem = closing_rule_problem(
|
|
534
|
+
self.classification, worker_recorded=self.claim.worker is not None
|
|
535
|
+
)
|
|
536
|
+
if problem and self.classification is not ClaimClassification.SUPERSEDED:
|
|
537
|
+
_fail(f"claim generation {self.claim.generation}", problem)
|
|
538
|
+
if self.classification is ClaimClassification.SETTLED and (
|
|
539
|
+
self.settlement is None or self.settlement is Settlement.IN_PROGRESS
|
|
540
|
+
):
|
|
541
|
+
_fail(
|
|
542
|
+
f"claim generation {self.claim.generation}",
|
|
543
|
+
"is closed settled without a settled settlement; settled means the state's "
|
|
544
|
+
"settlement is not in_progress",
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
def to_dict(self) -> dict:
|
|
548
|
+
return {
|
|
549
|
+
**self.claim.to_dict(),
|
|
550
|
+
"closed": {
|
|
551
|
+
"at": self.at,
|
|
552
|
+
"classification": self.classification.value,
|
|
553
|
+
"settlement": None if self.settlement is None else self.settlement.value,
|
|
554
|
+
"exit_status": self.exit_status,
|
|
555
|
+
"detail": self.detail,
|
|
556
|
+
},
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
@classmethod
|
|
560
|
+
def from_dict(cls, raw: object) -> ClosedClaim:
|
|
561
|
+
data = _exact(raw, (*_CLAIM_KEYS, "closed"), "history claim")
|
|
562
|
+
claim = Claim._of(data)
|
|
563
|
+
closed = _exact(data["closed"], _CLOSED_KEYS, "history claim closed")
|
|
564
|
+
settlement = closed["settlement"]
|
|
565
|
+
return cls(
|
|
566
|
+
claim=claim,
|
|
567
|
+
classification=_check_enum(
|
|
568
|
+
ClaimClassification, closed["classification"], "closed claim classification"
|
|
569
|
+
),
|
|
570
|
+
at=closed["at"],
|
|
571
|
+
settlement=None
|
|
572
|
+
if settlement is None
|
|
573
|
+
else _check_enum(Settlement, settlement, "closed claim settlement"),
|
|
574
|
+
exit_status=closed["exit_status"],
|
|
575
|
+
detail=closed["detail"],
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def closing_rule_problem(classification: ClaimClassification, *, worker_recorded: bool) -> str:
|
|
580
|
+
"""Why D7.2's closing rule refuses closing a claim as ``classification``; "" when it allows.
|
|
581
|
+
|
|
582
|
+
``superseded`` is refused here because only :func:`open_claim` closes a
|
|
583
|
+
claim that way, in the write that installs its replacement.
|
|
584
|
+
"""
|
|
585
|
+
if classification is ClaimClassification.SUPERSEDED:
|
|
586
|
+
return (
|
|
587
|
+
"cannot be closed superseded except by the write that installs its replacement "
|
|
588
|
+
"(open_claim with supersedes)"
|
|
589
|
+
)
|
|
590
|
+
if classification is ClaimClassification.ABANDONED and worker_recorded:
|
|
591
|
+
return "has a recorded worker, so it is closed interrupted, never abandoned"
|
|
592
|
+
if classification is ClaimClassification.INTERRUPTED and not worker_recorded:
|
|
593
|
+
return "has no recorded worker, so it is closed abandoned, never interrupted"
|
|
594
|
+
return ""
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
_CLAIM_FILE_KEYS = (
|
|
598
|
+
"schema",
|
|
599
|
+
"run_key",
|
|
600
|
+
"issue_id",
|
|
601
|
+
"epic",
|
|
602
|
+
"next_generation",
|
|
603
|
+
"current",
|
|
604
|
+
"history",
|
|
605
|
+
"updated_at",
|
|
606
|
+
)
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
@dataclass(frozen=True)
|
|
610
|
+
class ClaimFile:
|
|
611
|
+
"""``claim.json``, schema ``autoforge.scheduler-claim/1`` (ADR 0008 D7.1).
|
|
612
|
+
|
|
613
|
+
Validated on construction: ``next_generation`` is above every
|
|
614
|
+
generation in the file, ``current`` (when set) is the last one
|
|
615
|
+
allocated, ``history`` holds at most :data:`MAX_HISTORY` claims in
|
|
616
|
+
ascending generation, and no claim id repeats.
|
|
617
|
+
"""
|
|
618
|
+
|
|
619
|
+
run_key: RunKey
|
|
620
|
+
issue_id: int
|
|
621
|
+
epic: EpicRef
|
|
622
|
+
next_generation: int
|
|
623
|
+
current: Claim | None
|
|
624
|
+
history: tuple[ClosedClaim, ...]
|
|
625
|
+
updated_at: str
|
|
626
|
+
|
|
627
|
+
def __post_init__(self) -> None:
|
|
628
|
+
_check_type(self.run_key, RunKey, "claim file run_key")
|
|
629
|
+
_check_id(self.issue_id, "claim file issue_id")
|
|
630
|
+
_check_type(self.epic, EpicRef, "claim file epic")
|
|
631
|
+
_check_id(self.next_generation, "claim file next_generation")
|
|
632
|
+
if self.current is not None:
|
|
633
|
+
_check_type(self.current, Claim, "claim file current")
|
|
634
|
+
if not isinstance(self.history, tuple):
|
|
635
|
+
_fail("claim file history", f"must be a tuple, got {type(self.history).__name__}")
|
|
636
|
+
if len(self.history) > MAX_HISTORY:
|
|
637
|
+
_fail("claim file history", f"holds {len(self.history)} claims, over {MAX_HISTORY}")
|
|
638
|
+
for closed in self.history:
|
|
639
|
+
_check_type(closed, ClosedClaim, "claim file history entry")
|
|
640
|
+
_check_timestamp(self.updated_at, "claim file updated_at")
|
|
641
|
+
generations = [closed.claim.generation for closed in self.history]
|
|
642
|
+
if self.current is not None:
|
|
643
|
+
generations.append(self.current.generation)
|
|
644
|
+
if self.current.generation != self.next_generation - 1:
|
|
645
|
+
_fail(
|
|
646
|
+
"claim file current",
|
|
647
|
+
f"has generation {self.current.generation}, not the last one allocated "
|
|
648
|
+
f"({self.next_generation - 1})",
|
|
649
|
+
)
|
|
650
|
+
if any(b <= a for a, b in zip(generations, generations[1:], strict=False)):
|
|
651
|
+
_fail("claim file", f"holds generations {generations} out of ascending order")
|
|
652
|
+
if generations and generations[-1] >= self.next_generation:
|
|
653
|
+
_fail(
|
|
654
|
+
"claim file next_generation",
|
|
655
|
+
f"is {self.next_generation}, not above generation {generations[-1]}",
|
|
656
|
+
)
|
|
657
|
+
ids = [closed.claim.claim_id for closed in self.history]
|
|
658
|
+
if self.current is not None:
|
|
659
|
+
ids.append(self.current.claim_id)
|
|
660
|
+
if len(set(ids)) != len(ids):
|
|
661
|
+
_fail("claim file", "holds one claim_id twice")
|
|
662
|
+
|
|
663
|
+
def to_dict(self) -> dict:
|
|
664
|
+
return {
|
|
665
|
+
"schema": CLAIM_SCHEMA,
|
|
666
|
+
"run_key": {
|
|
667
|
+
"repository_id": self.run_key.repository_id,
|
|
668
|
+
"issue_number": self.run_key.issue_number,
|
|
669
|
+
},
|
|
670
|
+
"issue_id": self.issue_id,
|
|
671
|
+
"epic": _epic_dict(self.epic),
|
|
672
|
+
"next_generation": self.next_generation,
|
|
673
|
+
"current": None if self.current is None else self.current.to_dict(),
|
|
674
|
+
"history": [closed.to_dict() for closed in self.history],
|
|
675
|
+
"updated_at": self.updated_at,
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
@classmethod
|
|
679
|
+
def from_dict(cls, raw: object, *, where: str = CLAIM_FILENAME) -> ClaimFile:
|
|
680
|
+
"""Strictly parse a claim file: an unknown version or field is refused, never skipped."""
|
|
681
|
+
try:
|
|
682
|
+
if not isinstance(raw, dict):
|
|
683
|
+
_fail("claim file", f"must be a JSON object, got {type(raw).__name__}")
|
|
684
|
+
schema = raw.get("schema")
|
|
685
|
+
if schema != CLAIM_SCHEMA:
|
|
686
|
+
_fail(
|
|
687
|
+
"claim file",
|
|
688
|
+
f"has schema {schema!r:.80}; this controller reads only {CLAIM_SCHEMA}",
|
|
689
|
+
)
|
|
690
|
+
data = _exact(raw, _CLAIM_FILE_KEYS, "claim file")
|
|
691
|
+
key = _exact(data["run_key"], ("repository_id", "issue_number"), "claim file run_key")
|
|
692
|
+
_check_id(key["repository_id"], "claim file run_key repository_id")
|
|
693
|
+
_check_id(key["issue_number"], "claim file run_key issue_number")
|
|
694
|
+
history = data["history"]
|
|
695
|
+
if not isinstance(history, list):
|
|
696
|
+
_fail("claim file history", f"must be a list, got {type(history).__name__}")
|
|
697
|
+
if len(history) > MAX_HISTORY:
|
|
698
|
+
_fail("claim file history", f"holds {len(history)} claims, over {MAX_HISTORY}")
|
|
699
|
+
current = data["current"]
|
|
700
|
+
return cls(
|
|
701
|
+
run_key=RunKey(key["repository_id"], key["issue_number"]),
|
|
702
|
+
issue_id=data["issue_id"],
|
|
703
|
+
epic=_epic_of(data["epic"], "claim file epic"),
|
|
704
|
+
next_generation=data["next_generation"],
|
|
705
|
+
current=None if current is None else Claim.from_dict(current, what="claim current"),
|
|
706
|
+
history=tuple(ClosedClaim.from_dict(entry) for entry in history),
|
|
707
|
+
updated_at=data["updated_at"],
|
|
708
|
+
)
|
|
709
|
+
except SchedulerStateError as exc:
|
|
710
|
+
raise SchedulerStateError(f"invalid {CLAIM_SCHEMA} document {where}: {exc}") from exc
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def _appended(history: tuple[ClosedClaim, ...], closed: ClosedClaim) -> tuple[ClosedClaim, ...]:
|
|
714
|
+
return (*history, closed)[-MAX_HISTORY:]
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
# -- reading and writing a namespace's claim file -----------------------------------------
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _open_anchor(coordination_root: Path) -> SafeRoot:
|
|
721
|
+
"""A root on the coordination root; it is never created (it is the git common dir)."""
|
|
722
|
+
try:
|
|
723
|
+
return SafeRoot.open(coordination_root)
|
|
724
|
+
except FileNotFoundError:
|
|
725
|
+
raise SchedulerStateError(f"coordination root {coordination_root} does not exist") from None
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def _unique_keys(pairs: list[tuple[str, object]]) -> dict:
|
|
729
|
+
data: dict = {}
|
|
730
|
+
for key, value in pairs:
|
|
731
|
+
if key in data:
|
|
732
|
+
raise SchedulerStateError(f"has the field {key!r:.80} more than once")
|
|
733
|
+
data[key] = value
|
|
734
|
+
return data
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _no_constant(name: str) -> NoReturn:
|
|
738
|
+
raise SchedulerStateError(f"holds the non-JSON number {name}")
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def _read_document(root: SafeRoot, name: str, *, limit: int, where: str) -> object | None:
|
|
742
|
+
"""The JSON document at ``name`` under ``root``; ``None`` only when there is no file.
|
|
743
|
+
|
|
744
|
+
Strict like the binding's read: a link or another non-regular entry is
|
|
745
|
+
refused without being followed, a file over ``limit`` unread, and a
|
|
746
|
+
duplicated key, a non-JSON number and the decoder's own limits are
|
|
747
|
+
refusals (:class:`SchedulerStateError`). So is a file holding JSON
|
|
748
|
+
``null``: it is a corrupt file, kept until explicit recovery, never one
|
|
749
|
+
that is not there.
|
|
750
|
+
"""
|
|
751
|
+
st = root.lstat(name)
|
|
752
|
+
if st is None:
|
|
753
|
+
return None
|
|
754
|
+
kind = entry_kind(st.st_mode)
|
|
755
|
+
if kind is not None:
|
|
756
|
+
raise SchedulerStateError(f"{where} is a {kind}, not a regular file; it is never followed")
|
|
757
|
+
try:
|
|
758
|
+
raw = root.read_bytes(name, limit=limit)
|
|
759
|
+
except ReadLimitExceeded as exc:
|
|
760
|
+
raise SchedulerStateError(
|
|
761
|
+
f"{where} is larger than {limit} bytes, which no scheduler file can be"
|
|
762
|
+
) from exc
|
|
763
|
+
if raw is None:
|
|
764
|
+
return None # removed between the lstat and the open
|
|
765
|
+
try:
|
|
766
|
+
document = json.loads(
|
|
767
|
+
raw.decode("utf-8"), object_pairs_hook=_unique_keys, parse_constant=_no_constant
|
|
768
|
+
)
|
|
769
|
+
except UnicodeDecodeError as exc:
|
|
770
|
+
raise SchedulerStateError(f"{where} is not valid UTF-8 ({exc})") from exc
|
|
771
|
+
except json.JSONDecodeError as exc:
|
|
772
|
+
raise SchedulerStateError(f"{where} is not valid JSON ({exc})") from exc
|
|
773
|
+
except SchedulerStateError as exc:
|
|
774
|
+
raise SchedulerStateError(f"{where} {exc}") from exc
|
|
775
|
+
except (ValueError, RecursionError) as exc:
|
|
776
|
+
raise SchedulerStateError(
|
|
777
|
+
f"{where} exceeds the JSON decoder's integer or nesting limit "
|
|
778
|
+
f"({type(exc).__name__}); no scheduler file comes near it"
|
|
779
|
+
) from exc
|
|
780
|
+
if document is None:
|
|
781
|
+
raise SchedulerStateError(f"{where} holds JSON null, not a JSON object")
|
|
782
|
+
return document
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _payload(document: dict, *, limit: int, where: str) -> bytes:
|
|
786
|
+
payload = (json.dumps(document, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
|
787
|
+
if len(payload) > limit: # pragma: no cover - every field is bounded below it
|
|
788
|
+
raise SchedulerStateError(f"{where} would be {len(payload)} bytes, over {limit}")
|
|
789
|
+
return payload
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def load_claim_file(namespace: IssueNamespace) -> ClaimFile | None:
|
|
793
|
+
"""The namespace's ``claim.json``, strictly; ``None`` when there is none. Read-only.
|
|
794
|
+
|
|
795
|
+
A corrupt file (one holding JSON ``null`` included), one of an unknown
|
|
796
|
+
schema or one that breaks a bound or the closing rule raises
|
|
797
|
+
:class:`SchedulerStateError`; a link at any directory of the path raises
|
|
798
|
+
:class:`~autoforge.safefs.UnsafePathError`. Whether the file names this
|
|
799
|
+
namespace's run is the caller's check (:func:`classify`), as the
|
|
800
|
+
binding's is.
|
|
801
|
+
"""
|
|
802
|
+
with _open_anchor(namespace.coordination_root) as anchor:
|
|
803
|
+
try:
|
|
804
|
+
issue_root = anchor.subroot("/".join(namespace.relative))
|
|
805
|
+
except FileNotFoundError:
|
|
806
|
+
return None
|
|
807
|
+
where = str(namespace.claim_file)
|
|
808
|
+
with issue_root:
|
|
809
|
+
raw = _read_document(issue_root, CLAIM_FILENAME, limit=MAX_CLAIM_BYTES, where=where)
|
|
810
|
+
if raw is None:
|
|
811
|
+
return None
|
|
812
|
+
return ClaimFile.from_dict(raw, where=where)
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _write_claim_file(namespace: IssueNamespace, claim_file: ClaimFile) -> None:
|
|
816
|
+
"""Replace ``claim.json`` atomically: fsynced temporary, replace, fsynced directory."""
|
|
817
|
+
where = str(namespace.claim_file)
|
|
818
|
+
assert_shared_git_lock_free(f"write {where}")
|
|
819
|
+
payload = _payload(claim_file.to_dict(), limit=MAX_CLAIM_BYTES, where=where)
|
|
820
|
+
with _open_anchor(namespace.coordination_root) as anchor:
|
|
821
|
+
anchor.write_bytes("/".join((*namespace.relative, CLAIM_FILENAME)), payload)
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def _require_lock(namespace: IssueNamespace, lock: IssueRunLock, action: str) -> None:
|
|
825
|
+
if not isinstance(lock, IssueRunLock) or not lock.held or lock.namespace != namespace:
|
|
826
|
+
raise SchedulerStateError(
|
|
827
|
+
f"cannot {action} for issue {namespace.issue_number}: this process does not hold "
|
|
828
|
+
"its run lock (L2), which every claim write but the worker's diagnostics holds"
|
|
829
|
+
)
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def _claim_identity_problem(
|
|
833
|
+
claim_file: ClaimFile, binding: RunBinding
|
|
834
|
+
) -> tuple[UncertainReason, str] | None:
|
|
835
|
+
if (claim_file.run_key, claim_file.issue_id) != (binding.run_key, binding.issue_id):
|
|
836
|
+
return (
|
|
837
|
+
UncertainReason.CLAIM_MISMATCH,
|
|
838
|
+
f"the claim file names run {claim_file.run_key} (issue id {claim_file.issue_id}), "
|
|
839
|
+
f"the binding run {binding.run_key} (issue id {binding.issue_id})",
|
|
840
|
+
)
|
|
841
|
+
if claim_file.epic != binding.epic:
|
|
842
|
+
return (
|
|
843
|
+
UncertainReason.CLAIM_EPIC_MISMATCH,
|
|
844
|
+
f"the claim file names EPIC #{claim_file.epic.number} (id {claim_file.epic.issue_id}), "
|
|
845
|
+
f"the binding EPIC #{binding.epic.number} (id {binding.epic.issue_id})",
|
|
846
|
+
)
|
|
847
|
+
return None
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def _load_for_write(namespace: IssueNamespace) -> ClaimFile:
|
|
851
|
+
"""The claim file a writer updates: the existing one, or a new one from the binding.
|
|
852
|
+
|
|
853
|
+
A claim is only ever written into a bound namespace (the binding comes
|
|
854
|
+
first, D2.2 step 5), and only over a file that names the binding's run:
|
|
855
|
+
anything else is refused (:class:`SchedulerStateError`), never rewritten
|
|
856
|
+
to agree.
|
|
857
|
+
"""
|
|
858
|
+
lookup = load_issue_namespace(namespace.coordination_root, namespace.issue_number)
|
|
859
|
+
binding = lookup.binding
|
|
860
|
+
if binding is None:
|
|
861
|
+
raise SchedulerStateError(
|
|
862
|
+
f"the namespace of issue {namespace.issue_number} has no binding "
|
|
863
|
+
f"({namespace.binding_file}); a claim is written only into a bound namespace"
|
|
864
|
+
)
|
|
865
|
+
claim_file = load_claim_file(namespace)
|
|
866
|
+
if claim_file is None:
|
|
867
|
+
return ClaimFile(
|
|
868
|
+
run_key=binding.run_key,
|
|
869
|
+
issue_id=binding.issue_id,
|
|
870
|
+
epic=binding.epic,
|
|
871
|
+
next_generation=1,
|
|
872
|
+
current=None,
|
|
873
|
+
history=(),
|
|
874
|
+
updated_at=utcnow_iso(),
|
|
875
|
+
)
|
|
876
|
+
problem = _claim_identity_problem(claim_file, binding)
|
|
877
|
+
if problem is not None:
|
|
878
|
+
raise SchedulerStateError(f"{namespace.claim_file} is not this namespace's: {problem[1]}")
|
|
879
|
+
return claim_file
|
|
880
|
+
|
|
881
|
+
|
|
882
|
+
def _matching_current(namespace: IssueNamespace, claim_file: ClaimFile, claim: Claim) -> Claim:
|
|
883
|
+
current = claim_file.current
|
|
884
|
+
if current is None or current.identity != claim.identity:
|
|
885
|
+
on_disk = "no open claim" if current is None else f"generation {current.generation}"
|
|
886
|
+
raise ClaimConflict(
|
|
887
|
+
f"{namespace.claim_file} holds {on_disk}, not the caller's generation "
|
|
888
|
+
f"{claim.generation} (claim {claim.claim_id}); a stale claimant never overwrites or "
|
|
889
|
+
"finalizes a newer generation, and the file is left as it is"
|
|
890
|
+
)
|
|
891
|
+
return current
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def _require_resumable(
|
|
895
|
+
namespace: IssueNamespace, refusal: str, expected: ExpectedIdentity | None
|
|
896
|
+
) -> Classification:
|
|
897
|
+
"""The namespace's class, read under the caller's L2; refused unless ``resumable`` (D7.2).
|
|
898
|
+
|
|
899
|
+
Only a ``resumable`` claim is replaced (:func:`open_claim`) or closed
|
|
900
|
+
unready (:func:`close_unready`): a settled or uncertain one is refused
|
|
901
|
+
(:class:`~autoforge.errors.SchedulerStateError`) and the file is left as
|
|
902
|
+
it is. ``expected`` is :func:`classify`'s.
|
|
903
|
+
"""
|
|
904
|
+
_check_optional_expected(expected)
|
|
905
|
+
read = read_namespace(namespace)
|
|
906
|
+
if read.absent:
|
|
907
|
+
raise ClaimConflict(f"{namespace.claim_file} does not exist; there is no open claim")
|
|
908
|
+
classification = classify(read.lookup, read.claim, expected=expected)
|
|
909
|
+
if classification.kind is not ClassificationKind.RESUMABLE:
|
|
910
|
+
raise SchedulerStateError(
|
|
911
|
+
f"{refusal}; the namespace of issue {namespace.issue_number} is {classification}"
|
|
912
|
+
)
|
|
913
|
+
return classification
|
|
914
|
+
|
|
915
|
+
|
|
916
|
+
def _close(
|
|
917
|
+
namespace: IssueNamespace,
|
|
918
|
+
claim_file: ClaimFile,
|
|
919
|
+
current: Claim,
|
|
920
|
+
classification: ClaimClassification,
|
|
921
|
+
*,
|
|
922
|
+
settlement: Settlement | None,
|
|
923
|
+
exit_status: int | None,
|
|
924
|
+
detail: str,
|
|
925
|
+
) -> ClosedClaim:
|
|
926
|
+
closed = ClosedClaim(
|
|
927
|
+
current,
|
|
928
|
+
classification,
|
|
929
|
+
at=utcnow_iso(),
|
|
930
|
+
settlement=settlement,
|
|
931
|
+
exit_status=exit_status,
|
|
932
|
+
detail=bounded_detail(detail),
|
|
933
|
+
)
|
|
934
|
+
_write_claim_file(
|
|
935
|
+
namespace,
|
|
936
|
+
replace(
|
|
937
|
+
claim_file,
|
|
938
|
+
current=None,
|
|
939
|
+
history=_appended(claim_file.history, closed),
|
|
940
|
+
updated_at=closed.at,
|
|
941
|
+
),
|
|
942
|
+
)
|
|
943
|
+
return closed
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
# -- claim operations (ADR 0008 D7.2) ---------------------------------------------------
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def open_claim(
|
|
950
|
+
namespace: IssueNamespace,
|
|
951
|
+
lock: IssueRunLock,
|
|
952
|
+
*,
|
|
953
|
+
kind: ClaimKind,
|
|
954
|
+
invocation_id: str,
|
|
955
|
+
admission: Admission,
|
|
956
|
+
state_ref: StateRef,
|
|
957
|
+
supersedes: Claim | None = None,
|
|
958
|
+
expected: ExpectedIdentity | None = None,
|
|
959
|
+
) -> Claim:
|
|
960
|
+
"""Write a new open claim, durably, before the worker exists; return it (D2.2 step 5).
|
|
961
|
+
|
|
962
|
+
With ``supersedes=None`` it refuses (:class:`~autoforge.errors.ClaimConflict`)
|
|
963
|
+
while ``current`` is set: at most one claim is open per issue. With
|
|
964
|
+
``supersedes`` set to the open claim, it compares it with ``current``,
|
|
965
|
+
classifies the namespace under the held L2 and refuses
|
|
966
|
+
(:class:`~autoforge.errors.SchedulerStateError`, nothing written) unless
|
|
967
|
+
it is ``resumable`` (D7.2, D14.2): a settled or uncertain claim is never
|
|
968
|
+
replaced. Then, in the same atomic write, it closes it as ``superseded``
|
|
969
|
+
and installs the new claim: the only way a claim is ever closed so.
|
|
970
|
+
``expected`` is the classification's (:func:`classify`): the admission's
|
|
971
|
+
repository and issue ids, and the live ``owner/name`` that lets a state
|
|
972
|
+
written after a repository rename be recognised. The generation is
|
|
973
|
+
``next_generation``, which is then incremented. The file is written and
|
|
974
|
+
fsynced before this returns.
|
|
975
|
+
"""
|
|
976
|
+
_require_lock(namespace, lock, "open a claim")
|
|
977
|
+
_check_type(kind, ClaimKind, "claim kind")
|
|
978
|
+
claim_file = _load_for_write(namespace)
|
|
979
|
+
current = claim_file.current
|
|
980
|
+
if supersedes is None:
|
|
981
|
+
if current is not None:
|
|
982
|
+
raise ClaimConflict(
|
|
983
|
+
f"{namespace.claim_file} already holds the open claim of generation "
|
|
984
|
+
f"{current.generation}; only the write that supersedes it may replace it"
|
|
985
|
+
)
|
|
986
|
+
else:
|
|
987
|
+
current = _matching_current(namespace, claim_file, supersedes)
|
|
988
|
+
_require_resumable(
|
|
989
|
+
namespace,
|
|
990
|
+
f"refusing to supersede generation {current.generation}: only a resumable claim "
|
|
991
|
+
"is replaced",
|
|
992
|
+
expected,
|
|
993
|
+
)
|
|
994
|
+
now = utcnow_iso()
|
|
995
|
+
claim = Claim(
|
|
996
|
+
claim_id=secrets.token_hex(16),
|
|
997
|
+
generation=claim_file.next_generation,
|
|
998
|
+
invocation_id=invocation_id,
|
|
999
|
+
kind=kind,
|
|
1000
|
+
admission=admission,
|
|
1001
|
+
state_ref=state_ref,
|
|
1002
|
+
created_at=now,
|
|
1003
|
+
)
|
|
1004
|
+
history = claim_file.history
|
|
1005
|
+
if current is not None:
|
|
1006
|
+
history = _appended(
|
|
1007
|
+
history,
|
|
1008
|
+
ClosedClaim(
|
|
1009
|
+
current,
|
|
1010
|
+
ClaimClassification.SUPERSEDED,
|
|
1011
|
+
at=now,
|
|
1012
|
+
detail=bounded_detail(
|
|
1013
|
+
f"superseded by generation {claim.generation} ({kind.value} admission of "
|
|
1014
|
+
f"invocation {invocation_id})"
|
|
1015
|
+
),
|
|
1016
|
+
),
|
|
1017
|
+
)
|
|
1018
|
+
_write_claim_file(
|
|
1019
|
+
namespace,
|
|
1020
|
+
replace(
|
|
1021
|
+
claim_file,
|
|
1022
|
+
next_generation=claim_file.next_generation + 1,
|
|
1023
|
+
current=claim,
|
|
1024
|
+
history=history,
|
|
1025
|
+
updated_at=now,
|
|
1026
|
+
),
|
|
1027
|
+
)
|
|
1028
|
+
return claim
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def record_worker(namespace: IssueNamespace, claim: Claim, diagnostics: WorkerDiagnostics) -> Claim:
|
|
1032
|
+
"""Record the spawned worker's diagnostics on its open claim; return the updated claim.
|
|
1033
|
+
|
|
1034
|
+
Compare-and-write on ``(claim_id, generation)``
|
|
1035
|
+
(:class:`~autoforge.errors.ClaimConflict` on a mismatch, or when a
|
|
1036
|
+
worker is already recorded: one claim, one worker). The diagnostics are
|
|
1037
|
+
never authority (D7.3). It takes no lock: the worker holds L2 by now.
|
|
1038
|
+
"""
|
|
1039
|
+
_check_type(diagnostics, WorkerDiagnostics, "claim worker")
|
|
1040
|
+
claim_file = _load_for_write(namespace)
|
|
1041
|
+
current = _matching_current(namespace, claim_file, claim)
|
|
1042
|
+
if current.worker is not None:
|
|
1043
|
+
raise ClaimConflict(
|
|
1044
|
+
f"{namespace.claim_file}: generation {current.generation} already records worker "
|
|
1045
|
+
f"pid {current.worker.pid}; a claim has one worker"
|
|
1046
|
+
)
|
|
1047
|
+
updated = replace(current, worker=diagnostics)
|
|
1048
|
+
_write_claim_file(namespace, replace(claim_file, current=updated, updated_at=utcnow_iso()))
|
|
1049
|
+
return updated
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def close_claim(
|
|
1053
|
+
namespace: IssueNamespace,
|
|
1054
|
+
lock: IssueRunLock,
|
|
1055
|
+
claim: Claim,
|
|
1056
|
+
classification: ClaimClassification,
|
|
1057
|
+
*,
|
|
1058
|
+
settlement: Settlement | None = None,
|
|
1059
|
+
exit_status: int | None = None,
|
|
1060
|
+
detail: str = "",
|
|
1061
|
+
) -> ClosedClaim:
|
|
1062
|
+
"""Close the open claim ``claim`` as ``classification`` and move it to ``history``.
|
|
1063
|
+
|
|
1064
|
+
Compare-and-write (:class:`~autoforge.errors.ClaimConflict` when
|
|
1065
|
+
``claim`` is not the open claim on disk). The closing rule decides on the
|
|
1066
|
+
claim as it is on disk: ``superseded`` is refused (only
|
|
1067
|
+
:func:`open_claim` closes a claim so), ``abandoned`` for a claim with a
|
|
1068
|
+
recorded worker and ``interrupted`` for one without are refused
|
|
1069
|
+
(:class:`~autoforge.errors.SchedulerStateError`), and nothing is written.
|
|
1070
|
+
``detail`` is redacted and bounded.
|
|
1071
|
+
"""
|
|
1072
|
+
_require_lock(namespace, lock, f"close a claim as {classification.value}")
|
|
1073
|
+
_check_type(classification, ClaimClassification, "claim classification")
|
|
1074
|
+
claim_file = _load_for_write(namespace)
|
|
1075
|
+
current = _matching_current(namespace, claim_file, claim)
|
|
1076
|
+
problem = closing_rule_problem(classification, worker_recorded=current.worker is not None)
|
|
1077
|
+
if problem:
|
|
1078
|
+
raise SchedulerStateError(
|
|
1079
|
+
f"refusing to close generation {current.generation} of issue "
|
|
1080
|
+
f"{namespace.issue_number} as {classification.value}: it {problem}"
|
|
1081
|
+
)
|
|
1082
|
+
return _close(
|
|
1083
|
+
namespace,
|
|
1084
|
+
claim_file,
|
|
1085
|
+
current,
|
|
1086
|
+
classification,
|
|
1087
|
+
settlement=settlement,
|
|
1088
|
+
exit_status=exit_status,
|
|
1089
|
+
detail=detail,
|
|
1090
|
+
)
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def close_unready(
|
|
1094
|
+
namespace: IssueNamespace,
|
|
1095
|
+
lock: IssueRunLock,
|
|
1096
|
+
claim: Claim,
|
|
1097
|
+
*,
|
|
1098
|
+
detail: str = "",
|
|
1099
|
+
expected: ExpectedIdentity | None = None,
|
|
1100
|
+
) -> ClosedClaim:
|
|
1101
|
+
"""Close an open ``resumable`` claim whose issue a snapshot no longer shows as ready.
|
|
1102
|
+
|
|
1103
|
+
ADR 0008 D14.2: a snapshot gave the issue a verdict other than ``ready``,
|
|
1104
|
+
or no longer lists it, so the earlier invocation's claim cannot be
|
|
1105
|
+
superseded by an admission and must not keep ``epic run`` refused. It
|
|
1106
|
+
is closed ``abandoned`` when no worker was recorded and ``interrupted``
|
|
1107
|
+
when one was. The namespace is classified again first, with
|
|
1108
|
+
``expected`` (:func:`classify`): a claim that is not ``resumable``
|
|
1109
|
+
(settled, uncertain) is refused
|
|
1110
|
+
(:class:`~autoforge.errors.SchedulerStateError`) and left open.
|
|
1111
|
+
"""
|
|
1112
|
+
_require_lock(namespace, lock, "close an unready claim")
|
|
1113
|
+
classification = _require_resumable(
|
|
1114
|
+
namespace, "close_unready closes only a resumable claim", expected
|
|
1115
|
+
)
|
|
1116
|
+
on_disk = classification.open_claim
|
|
1117
|
+
if on_disk is None or on_disk.identity != claim.identity:
|
|
1118
|
+
raise ClaimConflict(
|
|
1119
|
+
f"{namespace.claim_file} does not hold the caller's open claim of generation "
|
|
1120
|
+
f"{claim.generation}; the file is left as it is"
|
|
1121
|
+
)
|
|
1122
|
+
closing = (
|
|
1123
|
+
ClaimClassification.INTERRUPTED
|
|
1124
|
+
if on_disk.worker is not None
|
|
1125
|
+
else ClaimClassification.ABANDONED
|
|
1126
|
+
)
|
|
1127
|
+
return close_claim(
|
|
1128
|
+
namespace,
|
|
1129
|
+
lock,
|
|
1130
|
+
on_disk,
|
|
1131
|
+
closing,
|
|
1132
|
+
settlement=classification.settlement,
|
|
1133
|
+
detail=detail or "the issue is no longer ready for admission",
|
|
1134
|
+
)
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
class ArchiveAction(Enum):
|
|
1138
|
+
NOTHING = "nothing" # no claim file, or no open claim: nothing changed
|
|
1139
|
+
CLOSED = "closed" # the open claim was closed as archived
|
|
1140
|
+
MOVED_ASIDE = "moved-aside" # a claim file the writers refuse was moved aside
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
@dataclass(frozen=True)
|
|
1144
|
+
class ClaimArchive:
|
|
1145
|
+
"""What :func:`archive_claim` did."""
|
|
1146
|
+
|
|
1147
|
+
action: ArchiveAction
|
|
1148
|
+
closed: ClosedClaim | None = None
|
|
1149
|
+
moved_to: Path | None = None
|
|
1150
|
+
reason: str = ""
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
def archive_claim(namespace: IssueNamespace, lock: IssueRunLock) -> ClaimArchive:
|
|
1154
|
+
"""``epic archive``'s claim step (ADR 0008 D17.4): run before the state is moved aside.
|
|
1155
|
+
|
|
1156
|
+
An open claim is closed as ``archived``, whatever the state holds. A
|
|
1157
|
+
claim file the writers refuse to update (corrupt, of an unknown schema,
|
|
1158
|
+
naming another run than the binding, or in a namespace whose binding
|
|
1159
|
+
cannot be read) is moved aside as ``claim.json.archived-<UTC timestamp>``
|
|
1160
|
+
with the link-then-unlink primitive of the state's archive, so nothing
|
|
1161
|
+
is deleted. With no open claim and no such file it changes nothing, so a
|
|
1162
|
+
re-run after a crash completes the archive.
|
|
1163
|
+
"""
|
|
1164
|
+
_require_lock(namespace, lock, "archive a claim")
|
|
1165
|
+
with _open_anchor(namespace.coordination_root) as anchor:
|
|
1166
|
+
try:
|
|
1167
|
+
issue_root = anchor.subroot("/".join(namespace.relative))
|
|
1168
|
+
except FileNotFoundError:
|
|
1169
|
+
return ClaimArchive(ArchiveAction.NOTHING)
|
|
1170
|
+
with issue_root:
|
|
1171
|
+
if issue_root.lstat(CLAIM_FILENAME) is None:
|
|
1172
|
+
return ClaimArchive(ArchiveAction.NOTHING)
|
|
1173
|
+
try:
|
|
1174
|
+
claim_file = _load_for_write(namespace)
|
|
1175
|
+
except StateError as exc:
|
|
1176
|
+
assert_shared_git_lock_free(f"move {namespace.claim_file} aside")
|
|
1177
|
+
moved = archive_state_file(namespace.claim_file, root=issue_root, what="claim file")
|
|
1178
|
+
return ClaimArchive(
|
|
1179
|
+
ArchiveAction.MOVED_ASIDE, moved_to=moved, reason=_bounded(str(exc))
|
|
1180
|
+
)
|
|
1181
|
+
if claim_file.current is None:
|
|
1182
|
+
return ClaimArchive(ArchiveAction.NOTHING)
|
|
1183
|
+
closed = _close(
|
|
1184
|
+
namespace,
|
|
1185
|
+
claim_file,
|
|
1186
|
+
claim_file.current,
|
|
1187
|
+
ClaimClassification.ARCHIVED,
|
|
1188
|
+
settlement=None,
|
|
1189
|
+
exit_status=None,
|
|
1190
|
+
detail="archived by 'epic archive'",
|
|
1191
|
+
)
|
|
1192
|
+
return ClaimArchive(ArchiveAction.CLOSED, closed=closed)
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
# -- restart classification (ADR 0008 D14.2) --------------------------------------------
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
class ClassificationKind(Enum):
|
|
1199
|
+
UNCERTAIN = "uncertain"
|
|
1200
|
+
SETTLED = "settled"
|
|
1201
|
+
RESUMABLE = "resumable"
|
|
1202
|
+
FRESH = "fresh"
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
class UncertainReason(Enum):
|
|
1206
|
+
"""Why a namespace is ``uncertain``: a closed vocabulary (D14.2)."""
|
|
1207
|
+
|
|
1208
|
+
#: The binding, or a directory above it, cannot be read.
|
|
1209
|
+
BINDING_UNREADABLE = "binding-unreadable"
|
|
1210
|
+
#: The binding names another issue or issue id than expected.
|
|
1211
|
+
BINDING_MISMATCH = "binding-mismatch"
|
|
1212
|
+
#: The binding names another repository: the scheduler's global uncertainty (D14.3).
|
|
1213
|
+
REPOSITORY_MISMATCH = "repository-mismatch"
|
|
1214
|
+
#: A claim file or a state exists, and ``binding.json`` does not.
|
|
1215
|
+
UNBOUND = "unbound"
|
|
1216
|
+
#: The state is corrupt or of an unknown protocol.
|
|
1217
|
+
STATE_CORRUPT = "state-corrupt"
|
|
1218
|
+
#: The claim file is corrupt or of an unknown schema.
|
|
1219
|
+
CLAIM_CORRUPT = "claim-corrupt"
|
|
1220
|
+
#: The claim file names another run (RunKey or issue id) than the binding.
|
|
1221
|
+
CLAIM_MISMATCH = "claim-mismatch"
|
|
1222
|
+
#: The claim file names another EPIC than the binding.
|
|
1223
|
+
CLAIM_EPIC_MISMATCH = "claim-epic-mismatch"
|
|
1224
|
+
#: The state names another repository, issue or EPIC than the binding, or is not a
|
|
1225
|
+
#: REMOTE run.
|
|
1226
|
+
STATE_MISMATCH = "state-mismatch"
|
|
1227
|
+
#: An open claim's ``state_ref.present`` is true, and the state is absent.
|
|
1228
|
+
STATE_MISSING = "state-missing"
|
|
1229
|
+
#: The caller's ``(claim_id, generation)`` is not the open claim on disk.
|
|
1230
|
+
CLAIM_CONFLICT = "claim-conflict"
|
|
1231
|
+
#: The run lock cannot be taken for a reason other than a live owner.
|
|
1232
|
+
RUN_LOCK_UNUSABLE = "run-lock-unusable"
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
class ResumableReason(Enum):
|
|
1236
|
+
#: The state loads and is ``in_progress``, whether or not a claim is open.
|
|
1237
|
+
IN_PROGRESS = "in-progress"
|
|
1238
|
+
#: A claim is open, no state exists, and none did at admission (a crash before
|
|
1239
|
+
#: the worker's first save).
|
|
1240
|
+
NOT_STARTED = "not-started"
|
|
1241
|
+
|
|
1242
|
+
|
|
1243
|
+
@dataclass(frozen=True)
|
|
1244
|
+
class Classification:
|
|
1245
|
+
"""One namespace's restart class: total and mutually exclusive (D14.2).
|
|
1246
|
+
|
|
1247
|
+
``uncertain`` is set exactly for ``UNCERTAIN``, ``resumable`` exactly for
|
|
1248
|
+
``RESUMABLE``. ``settlement`` is the state's settlement whenever a state
|
|
1249
|
+
was loaded. ``open_claim`` is the open claim on disk, when one could be
|
|
1250
|
+
read. ``detail`` explains the class, redacted and bounded.
|
|
1251
|
+
"""
|
|
1252
|
+
|
|
1253
|
+
kind: ClassificationKind
|
|
1254
|
+
uncertain: UncertainReason | None = None
|
|
1255
|
+
settlement: Settlement | None = None
|
|
1256
|
+
resumable: ResumableReason | None = None
|
|
1257
|
+
open_claim: Claim | None = None
|
|
1258
|
+
detail: str = ""
|
|
1259
|
+
|
|
1260
|
+
def __post_init__(self) -> None:
|
|
1261
|
+
if (self.kind is ClassificationKind.UNCERTAIN) != (self.uncertain is not None):
|
|
1262
|
+
raise SchedulerStateError(
|
|
1263
|
+
f"classification {self.kind.value} with reason {self.uncertain}"
|
|
1264
|
+
)
|
|
1265
|
+
if (self.kind is ClassificationKind.RESUMABLE) != (self.resumable is not None):
|
|
1266
|
+
raise SchedulerStateError(
|
|
1267
|
+
f"classification {self.kind.value} with reason {self.resumable}"
|
|
1268
|
+
)
|
|
1269
|
+
if self.kind is ClassificationKind.SETTLED and (
|
|
1270
|
+
self.settlement is None or self.settlement is Settlement.IN_PROGRESS
|
|
1271
|
+
):
|
|
1272
|
+
raise SchedulerStateError(f"classification settled with settlement {self.settlement}")
|
|
1273
|
+
|
|
1274
|
+
def __str__(self) -> str:
|
|
1275
|
+
if self.uncertain is not None:
|
|
1276
|
+
return f"uncertain({self.uncertain.value})"
|
|
1277
|
+
if self.resumable is not None:
|
|
1278
|
+
return f"resumable({self.resumable.value})"
|
|
1279
|
+
if self.kind is ClassificationKind.SETTLED:
|
|
1280
|
+
assert self.settlement is not None
|
|
1281
|
+
return f"settled({self.settlement.value})"
|
|
1282
|
+
return self.kind.value
|
|
1283
|
+
|
|
1284
|
+
|
|
1285
|
+
def _uncertain(reason: UncertainReason, detail: str, claim: Claim | None = None) -> Classification:
|
|
1286
|
+
return Classification(
|
|
1287
|
+
ClassificationKind.UNCERTAIN, uncertain=reason, open_claim=claim, detail=_bounded(detail)
|
|
1288
|
+
)
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
class ClaimFileStatus(Enum):
|
|
1292
|
+
ABSENT = "absent"
|
|
1293
|
+
LOADED = "loaded"
|
|
1294
|
+
CORRUPT = "corrupt"
|
|
1295
|
+
|
|
1296
|
+
|
|
1297
|
+
@dataclass(frozen=True)
|
|
1298
|
+
class ClaimFileRead:
|
|
1299
|
+
"""The namespace's ``claim.json`` read strictly, read-only, without raising on corruption.
|
|
1300
|
+
|
|
1301
|
+
``claim_file`` is set when :attr:`status` is ``LOADED``; ``reason``
|
|
1302
|
+
(redacted and bounded) when it is ``CORRUPT``.
|
|
1303
|
+
"""
|
|
1304
|
+
|
|
1305
|
+
status: ClaimFileStatus
|
|
1306
|
+
claim_file: ClaimFile | None = None
|
|
1307
|
+
reason: str = ""
|
|
1308
|
+
|
|
1309
|
+
@property
|
|
1310
|
+
def open_claim(self) -> Claim | None:
|
|
1311
|
+
return None if self.claim_file is None else self.claim_file.current
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
def read_claim_file(namespace: IssueNamespace) -> ClaimFileRead:
|
|
1315
|
+
""":func:`load_claim_file`, with a corrupt or unreachable file reported, not raised."""
|
|
1316
|
+
try:
|
|
1317
|
+
claim_file = load_claim_file(namespace)
|
|
1318
|
+
except (StateError, OSError) as exc:
|
|
1319
|
+
return ClaimFileRead(ClaimFileStatus.CORRUPT, reason=_bounded(str(exc)))
|
|
1320
|
+
if claim_file is None:
|
|
1321
|
+
return ClaimFileRead(ClaimFileStatus.ABSENT)
|
|
1322
|
+
return ClaimFileRead(ClaimFileStatus.LOADED, claim_file)
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
@dataclass(frozen=True)
|
|
1326
|
+
class NamespaceUnreadable:
|
|
1327
|
+
"""#178's lookup refused the namespace: its binding or a directory above it.
|
|
1328
|
+
|
|
1329
|
+
``field`` is the identity a :class:`~autoforge.errors.NamespaceMismatch`
|
|
1330
|
+
named (a binding of another issue than its directory), else "".
|
|
1331
|
+
"""
|
|
1332
|
+
|
|
1333
|
+
namespace: IssueNamespace
|
|
1334
|
+
reason: str
|
|
1335
|
+
field: str = ""
|
|
1336
|
+
|
|
1337
|
+
|
|
1338
|
+
@dataclass(frozen=True)
|
|
1339
|
+
class NamespaceRead:
|
|
1340
|
+
"""One namespace read for classification: #178's lookup and the claim file."""
|
|
1341
|
+
|
|
1342
|
+
namespace: IssueNamespace
|
|
1343
|
+
lookup: NamespaceLookup | NamespaceUnreadable
|
|
1344
|
+
claim: ClaimFileRead
|
|
1345
|
+
|
|
1346
|
+
@property
|
|
1347
|
+
def absent(self) -> bool:
|
|
1348
|
+
"""No binding, no claim file and no state: no namespace, which needs no classification."""
|
|
1349
|
+
return (
|
|
1350
|
+
isinstance(self.lookup, NamespaceLookup)
|
|
1351
|
+
and self.lookup.kind is NamespaceKind.ABSENT
|
|
1352
|
+
and self.claim.status is ClaimFileStatus.ABSENT
|
|
1353
|
+
)
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
def read_namespace(namespace: IssueNamespace) -> NamespaceRead:
|
|
1357
|
+
"""Read a namespace's binding, state and claim file, strictly and read-only (D14.2 step 3).
|
|
1358
|
+
|
|
1359
|
+
The caller holds the issue's L2, so no writer moves anything between
|
|
1360
|
+
the reads. Nothing raises for a corrupt file: each is reported for
|
|
1361
|
+
:func:`classify`.
|
|
1362
|
+
"""
|
|
1363
|
+
lookup: NamespaceLookup | NamespaceUnreadable
|
|
1364
|
+
try:
|
|
1365
|
+
lookup = load_issue_namespace(namespace.coordination_root, namespace.issue_number)
|
|
1366
|
+
except NamespaceMismatch as exc:
|
|
1367
|
+
lookup = NamespaceUnreadable(namespace, _bounded(str(exc)), exc.field)
|
|
1368
|
+
except (StateError, OSError) as exc:
|
|
1369
|
+
lookup = NamespaceUnreadable(namespace, _bounded(str(exc)))
|
|
1370
|
+
return NamespaceRead(namespace, lookup, read_claim_file(namespace))
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
@dataclass(frozen=True)
|
|
1374
|
+
class ExpectedIdentity:
|
|
1375
|
+
"""What the caller knows a namespace must be bound to (D5.1, D14.2).
|
|
1376
|
+
|
|
1377
|
+
``repository_id`` is the live repository's; ``issue_ids`` maps an issue
|
|
1378
|
+
number to its GitHub issue id, as the caller's snapshot lists them. An
|
|
1379
|
+
issue the mapping does not name (one the snapshot no longer lists) has
|
|
1380
|
+
its id unchecked. ``repository`` is the ``owner/name`` the caller
|
|
1381
|
+
resolved together with ``repository_id``, or ``None`` when it has none:
|
|
1382
|
+
a state naming it is of the namespace's repository, also after a
|
|
1383
|
+
rename made the binding's recorded name an old one (D5.1).
|
|
1384
|
+
"""
|
|
1385
|
+
|
|
1386
|
+
repository_id: int
|
|
1387
|
+
issue_ids: Mapping[int, int] = field(default_factory=dict)
|
|
1388
|
+
repository: str | None = None
|
|
1389
|
+
|
|
1390
|
+
def __post_init__(self) -> None:
|
|
1391
|
+
_check_id(self.repository_id, "expected repository_id")
|
|
1392
|
+
issue_ids = dict(self.issue_ids)
|
|
1393
|
+
for number, issue_id in issue_ids.items():
|
|
1394
|
+
_check_id(number, "expected issue number")
|
|
1395
|
+
_check_id(issue_id, f"expected issue id of #{number}")
|
|
1396
|
+
object.__setattr__(self, "issue_ids", issue_ids)
|
|
1397
|
+
if self.repository is not None:
|
|
1398
|
+
text = _check_str(
|
|
1399
|
+
self.repository,
|
|
1400
|
+
"expected repository",
|
|
1401
|
+
MAX_OWNER_CHARS + 1 + MAX_REPOSITORY_NAME_CHARS,
|
|
1402
|
+
)
|
|
1403
|
+
owner, slash, name = text.partition("/")
|
|
1404
|
+
if not slash:
|
|
1405
|
+
_fail("expected repository", f"is not an owner/name pair: {text!r}")
|
|
1406
|
+
_check_name(owner, "expected repository owner", MAX_OWNER_CHARS)
|
|
1407
|
+
_check_name(name, "expected repository name", MAX_REPOSITORY_NAME_CHARS)
|
|
1408
|
+
|
|
1409
|
+
def binding_problem(self, binding: RunBinding) -> tuple[UncertainReason, str] | None:
|
|
1410
|
+
number = binding.run_key.issue_number
|
|
1411
|
+
try:
|
|
1412
|
+
check_binding(
|
|
1413
|
+
binding,
|
|
1414
|
+
run_key=RunKey(self.repository_id, number),
|
|
1415
|
+
issue_id=self.issue_ids.get(number, binding.issue_id),
|
|
1416
|
+
)
|
|
1417
|
+
except NamespaceMismatch as exc:
|
|
1418
|
+
reason = (
|
|
1419
|
+
UncertainReason.REPOSITORY_MISMATCH
|
|
1420
|
+
if exc.field == "repository_id"
|
|
1421
|
+
else UncertainReason.BINDING_MISMATCH
|
|
1422
|
+
)
|
|
1423
|
+
return (reason, str(exc))
|
|
1424
|
+
return None
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def _check_optional_expected(expected: ExpectedIdentity | None) -> None:
|
|
1428
|
+
if expected is not None:
|
|
1429
|
+
_check_type(expected, ExpectedIdentity, "expected identity")
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
def _repository_names(binding: RunBinding, expected: ExpectedIdentity | None) -> tuple[str, ...]:
|
|
1433
|
+
"""The ``owner/name`` pairs a state of this namespace may name (D5.1).
|
|
1434
|
+
|
|
1435
|
+
The binding's, recorded with its ``repository_id`` at first admission,
|
|
1436
|
+
and the caller's, resolved with the live ``repository_id``: a rename
|
|
1437
|
+
since the binding is tolerated, and no other name is this repository's.
|
|
1438
|
+
A name the repository had only in between is not known here, so a state
|
|
1439
|
+
naming it is ``uncertain``.
|
|
1440
|
+
"""
|
|
1441
|
+
names = [binding.repository]
|
|
1442
|
+
if expected is not None and expected.repository is not None:
|
|
1443
|
+
if expected.repository.lower() != binding.repository.lower():
|
|
1444
|
+
names.append(expected.repository)
|
|
1445
|
+
return tuple(names)
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def _state_identity_problem(
|
|
1449
|
+
state: AutoForgeState, binding: RunBinding, expected: ExpectedIdentity | None
|
|
1450
|
+
) -> str:
|
|
1451
|
+
"""Why ``state`` is not the run ``binding`` names, or ``""``.
|
|
1452
|
+
|
|
1453
|
+
A state is this namespace's only when its repository is one of
|
|
1454
|
+
:func:`_repository_names` (compared case-insensitively, as GitHub does),
|
|
1455
|
+
its issue and EPIC are in that repository, and their numbers are the
|
|
1456
|
+
binding's: a number alone names an issue of any repository. A URL the
|
|
1457
|
+
parser refuses (anything malformed, as :class:`ConfigurationError`)
|
|
1458
|
+
names no issue to compare, which is a mismatch too.
|
|
1459
|
+
"""
|
|
1460
|
+
if state.mode is not WorkflowMode.REMOTE:
|
|
1461
|
+
return f"the state is a {state.mode.value} run, which never lives in a namespace"
|
|
1462
|
+
names = _repository_names(binding, expected)
|
|
1463
|
+
if not any(state.repository.lower() == name.lower() for name in names):
|
|
1464
|
+
known = " or ".join(repr(name) for name in names)
|
|
1465
|
+
return f"the state's repository is {state.repository!r:.200}, the namespace's {known}"
|
|
1466
|
+
number = binding.run_key.issue_number
|
|
1467
|
+
for what, url, want in (
|
|
1468
|
+
("issue", state.current_issue_url, number),
|
|
1469
|
+
("EPIC", state.epic_url, binding.epic.number),
|
|
1470
|
+
):
|
|
1471
|
+
try:
|
|
1472
|
+
ref = parse_issue_url(url)
|
|
1473
|
+
except ConfigurationError:
|
|
1474
|
+
return f"the state names no {what} the binding can be compared with ({url!r:.200})"
|
|
1475
|
+
if not ref.same_repository(state.repository):
|
|
1476
|
+
return (
|
|
1477
|
+
f"the state's {what} {ref.canonical!r:.300} is not in its repository "
|
|
1478
|
+
f"{state.repository!r}"
|
|
1479
|
+
)
|
|
1480
|
+
if ref.number != want:
|
|
1481
|
+
return f"the state's {what} is #{ref.number}, the binding's #{want}"
|
|
1482
|
+
return ""
|
|
1483
|
+
|
|
1484
|
+
|
|
1485
|
+
def classify(
|
|
1486
|
+
lookup: NamespaceLookup | NamespaceUnreadable,
|
|
1487
|
+
claim_file: ClaimFileRead,
|
|
1488
|
+
*,
|
|
1489
|
+
expected: ExpectedIdentity | None,
|
|
1490
|
+
) -> Classification:
|
|
1491
|
+
"""Classify one namespace read under L2 (ADR 0008 D14.2 step 4); pure.
|
|
1492
|
+
|
|
1493
|
+
Checked in this order, so the classes are total and mutually exclusive:
|
|
1494
|
+
|
|
1495
|
+
* ``uncertain(reason)`` (:class:`UncertainReason`): the binding cannot be
|
|
1496
|
+
read or does not match ``expected`` (``expected=None`` compares the
|
|
1497
|
+
namespace with its own binding only); a claim file or a state without
|
|
1498
|
+
a binding; a corrupt state or claim file; a claim file or a state
|
|
1499
|
+
naming another run or EPIC than the binding, a state of another
|
|
1500
|
+
repository included (one named neither as the binding recorded it nor
|
|
1501
|
+
as ``expected.repository`` names it); an open claim whose
|
|
1502
|
+
``state_ref.present`` is true while the state is absent;
|
|
1503
|
+
* ``settled(settlement)``: the state's settlement is not ``in_progress``;
|
|
1504
|
+
* ``resumable(reason)``: the state is ``in_progress``, whether or not a
|
|
1505
|
+
claim is open; or a claim is open, the state is absent and its
|
|
1506
|
+
``state_ref.present`` is false;
|
|
1507
|
+
* ``fresh``: the binding is valid, no claim is open and the state is
|
|
1508
|
+
absent.
|
|
1509
|
+
|
|
1510
|
+
The ``state_ref.present`` rule applies only to an open claim: a state
|
|
1511
|
+
absent while no claim is open is never uncertain. A namespace #178
|
|
1512
|
+
reports as absent (no binding, no claim, no state) needs no
|
|
1513
|
+
classification and is refused (:class:`~autoforge.errors.SchedulerStateError`).
|
|
1514
|
+
``observed_live`` comes from the lock layer, before this runs.
|
|
1515
|
+
"""
|
|
1516
|
+
open_claim = claim_file.open_claim
|
|
1517
|
+
if isinstance(lookup, NamespaceUnreadable):
|
|
1518
|
+
reason = (
|
|
1519
|
+
UncertainReason.BINDING_MISMATCH if lookup.field else UncertainReason.BINDING_UNREADABLE
|
|
1520
|
+
)
|
|
1521
|
+
return _uncertain(reason, lookup.reason, open_claim)
|
|
1522
|
+
if lookup.kind is NamespaceKind.ABSENT and claim_file.status is ClaimFileStatus.ABSENT:
|
|
1523
|
+
raise SchedulerStateError(
|
|
1524
|
+
f"{lookup.namespace.directory} holds no binding, claim or state: no namespace, "
|
|
1525
|
+
"which needs no classification"
|
|
1526
|
+
)
|
|
1527
|
+
binding = lookup.binding
|
|
1528
|
+
if binding is None:
|
|
1529
|
+
return _uncertain(
|
|
1530
|
+
UncertainReason.UNBOUND,
|
|
1531
|
+
f"{lookup.namespace.directory} holds a claim file or a state but no binding.json",
|
|
1532
|
+
open_claim,
|
|
1533
|
+
)
|
|
1534
|
+
if expected is not None:
|
|
1535
|
+
mismatch = expected.binding_problem(binding)
|
|
1536
|
+
if mismatch is not None:
|
|
1537
|
+
return _uncertain(mismatch[0], mismatch[1], open_claim)
|
|
1538
|
+
if lookup.state.status is StateStatus.CORRUPT:
|
|
1539
|
+
return _uncertain(UncertainReason.STATE_CORRUPT, lookup.state.reason, open_claim)
|
|
1540
|
+
if claim_file.status is ClaimFileStatus.CORRUPT:
|
|
1541
|
+
return _uncertain(UncertainReason.CLAIM_CORRUPT, claim_file.reason)
|
|
1542
|
+
if claim_file.claim_file is not None:
|
|
1543
|
+
problem = _claim_identity_problem(claim_file.claim_file, binding)
|
|
1544
|
+
if problem is not None:
|
|
1545
|
+
return _uncertain(problem[0], problem[1], open_claim)
|
|
1546
|
+
state = lookup.state.state
|
|
1547
|
+
if state is not None:
|
|
1548
|
+
state_problem = _state_identity_problem(state, binding, expected)
|
|
1549
|
+
if state_problem:
|
|
1550
|
+
return _uncertain(UncertainReason.STATE_MISMATCH, state_problem, open_claim)
|
|
1551
|
+
settlement = issue_settlement(state)
|
|
1552
|
+
if settlement is not Settlement.IN_PROGRESS:
|
|
1553
|
+
return Classification(
|
|
1554
|
+
ClassificationKind.SETTLED,
|
|
1555
|
+
settlement=settlement,
|
|
1556
|
+
open_claim=open_claim,
|
|
1557
|
+
detail=f"the state is {settlement.value} (phase {state.phase.value})",
|
|
1558
|
+
)
|
|
1559
|
+
return Classification(
|
|
1560
|
+
ClassificationKind.RESUMABLE,
|
|
1561
|
+
resumable=ResumableReason.IN_PROGRESS,
|
|
1562
|
+
settlement=settlement,
|
|
1563
|
+
open_claim=open_claim,
|
|
1564
|
+
detail=f"the state is in progress (phase {state.phase.value})",
|
|
1565
|
+
)
|
|
1566
|
+
if open_claim is not None:
|
|
1567
|
+
if open_claim.state_ref.present:
|
|
1568
|
+
return _uncertain(
|
|
1569
|
+
UncertainReason.STATE_MISSING,
|
|
1570
|
+
f"the open claim of generation {open_claim.generation} was admitted with a "
|
|
1571
|
+
"state, and the state is absent",
|
|
1572
|
+
open_claim,
|
|
1573
|
+
)
|
|
1574
|
+
return Classification(
|
|
1575
|
+
ClassificationKind.RESUMABLE,
|
|
1576
|
+
resumable=ResumableReason.NOT_STARTED,
|
|
1577
|
+
open_claim=open_claim,
|
|
1578
|
+
detail=f"the open claim of generation {open_claim.generation} has no state yet",
|
|
1579
|
+
)
|
|
1580
|
+
return Classification(ClassificationKind.FRESH, detail="bound, no open claim and no state")
|
|
1581
|
+
|
|
1582
|
+
|
|
1583
|
+
# -- reconciliation (ADR 0008 D14.1, D14.2) ---------------------------------------------
|
|
1584
|
+
|
|
1585
|
+
|
|
1586
|
+
@dataclass(frozen=True)
|
|
1587
|
+
class IssueReconciliation:
|
|
1588
|
+
"""What reconciliation found for one namespace.
|
|
1589
|
+
|
|
1590
|
+
``fact`` is ``None`` for ``fresh`` and ``resumable``, which are
|
|
1591
|
+
evaluated like an issue without a namespace. ``classification`` is
|
|
1592
|
+
``None`` where none ran: the caller's own child, a live owner, another
|
|
1593
|
+
EPIC's namespace of this repository, or a run lock that cannot be taken.
|
|
1594
|
+
``closed`` is the claim reconciliation closed as ``settled``.
|
|
1595
|
+
"""
|
|
1596
|
+
|
|
1597
|
+
issue_number: int
|
|
1598
|
+
fact: LocalFact | None
|
|
1599
|
+
classification: Classification | None = None
|
|
1600
|
+
observed_live: bool = False
|
|
1601
|
+
own_child: bool = False
|
|
1602
|
+
closed: ClosedClaim | None = None
|
|
1603
|
+
detail: str = ""
|
|
1604
|
+
|
|
1605
|
+
|
|
1606
|
+
@dataclass(frozen=True)
|
|
1607
|
+
class ReconcileReport:
|
|
1608
|
+
"""Every namespace of the scheduler root, as one EPIC's reconciliation saw it.
|
|
1609
|
+
|
|
1610
|
+
``issues`` is in ascending issue number and holds one entry per
|
|
1611
|
+
namespace other than an absent one. ``global_problems`` are D14.3's
|
|
1612
|
+
global uncertainty (a binding of another repository, a refused
|
|
1613
|
+
filesystem, a listing that cannot be read): no admission at all.
|
|
1614
|
+
``rejected`` are the entries of ``issues/`` that are no namespace.
|
|
1615
|
+
"""
|
|
1616
|
+
|
|
1617
|
+
epic: EpicRef
|
|
1618
|
+
issues: tuple[IssueReconciliation, ...]
|
|
1619
|
+
global_problems: tuple[str, ...] = ()
|
|
1620
|
+
rejected: tuple[RejectedEntry, ...] = ()
|
|
1621
|
+
|
|
1622
|
+
@property
|
|
1623
|
+
def admissible(self) -> bool:
|
|
1624
|
+
return not self.global_problems
|
|
1625
|
+
|
|
1626
|
+
def local_facts(self) -> dict[int, LocalFact]:
|
|
1627
|
+
return {item.issue_number: item.fact for item in self.issues if item.fact is not None}
|
|
1628
|
+
|
|
1629
|
+
|
|
1630
|
+
def reconcile_epic(
|
|
1631
|
+
root: str | Path,
|
|
1632
|
+
epic: EpicRef,
|
|
1633
|
+
*,
|
|
1634
|
+
own_children: Collection[int] = (),
|
|
1635
|
+
only: Collection[int] | None = None,
|
|
1636
|
+
expected: ExpectedIdentity,
|
|
1637
|
+
) -> ReconcileReport:
|
|
1638
|
+
"""Restart reconciliation of the namespaces under ``root`` for ``epic`` (D14.1, D14.2).
|
|
1639
|
+
|
|
1640
|
+
The caller holds L0. For each namespace, in ascending issue number: the
|
|
1641
|
+
caller's own unreaped children are skipped as ``owned``; L2 is taken
|
|
1642
|
+
non-blocking, and a held one is ``owned`` (``observed_live``) for this
|
|
1643
|
+
pass; holding it, the namespace is read and classified. A namespace of
|
|
1644
|
+
this repository bound to another EPIC is ``held:other-epic`` and left
|
|
1645
|
+
alone; a binding of another repository is global uncertainty (D14.3),
|
|
1646
|
+
whatever EPIC it names. An open claim of a ``settled`` namespace is
|
|
1647
|
+
closed ``settled`` and the issue is ``held:settled-namespace``;
|
|
1648
|
+
``resumable`` and ``uncertain`` claims are kept open (an admission's
|
|
1649
|
+
supersession, :func:`close_unready` or ``epic archive`` closes them
|
|
1650
|
+
later); ``uncertain`` is ``held:uncertain``; L2 is released by
|
|
1651
|
+
``close``. It never spawns, never writes or quarantines a per-issue
|
|
1652
|
+
state and never contacts GitHub. ``only``, when given, limits the pass
|
|
1653
|
+
to those issues' namespaces: the scheduler's re-check of the namespaces
|
|
1654
|
+
it found live (D14.2 step 2).
|
|
1655
|
+
"""
|
|
1656
|
+
_check_type(epic, EpicRef, "reconciled epic")
|
|
1657
|
+
_check_type(expected, ExpectedIdentity, "reconciliation's expected identity")
|
|
1658
|
+
try:
|
|
1659
|
+
listing = list_issue_namespaces(root)
|
|
1660
|
+
except (StateError, OSError) as exc:
|
|
1661
|
+
return ReconcileReport(
|
|
1662
|
+
epic, (), (f"the scheduler's namespaces cannot be listed ({_bounded(str(exc))})",)
|
|
1663
|
+
)
|
|
1664
|
+
own = frozenset(own_children)
|
|
1665
|
+
wanted = None if only is None else frozenset(only)
|
|
1666
|
+
problems: list[str] = []
|
|
1667
|
+
issues: list[IssueReconciliation] = []
|
|
1668
|
+
for namespace in sorted((*listing.bound, *listing.unbound), key=lambda ns: ns.issue_number):
|
|
1669
|
+
number = namespace.issue_number
|
|
1670
|
+
if wanted is not None and number not in wanted:
|
|
1671
|
+
continue
|
|
1672
|
+
if number in own:
|
|
1673
|
+
issues.append(
|
|
1674
|
+
IssueReconciliation(
|
|
1675
|
+
number, LocalFact.OWNED, own_child=True, detail="this scheduler's worker"
|
|
1676
|
+
)
|
|
1677
|
+
)
|
|
1678
|
+
continue
|
|
1679
|
+
lock = IssueRunLock(namespace)
|
|
1680
|
+
try:
|
|
1681
|
+
lock.acquire()
|
|
1682
|
+
except IssueBusy as exc:
|
|
1683
|
+
issues.append(
|
|
1684
|
+
IssueReconciliation(
|
|
1685
|
+
number, LocalFact.OWNED, observed_live=True, detail=_bounded(str(exc))
|
|
1686
|
+
)
|
|
1687
|
+
)
|
|
1688
|
+
continue
|
|
1689
|
+
except NetworkFilesystemError as exc:
|
|
1690
|
+
problems.append(_bounded(str(exc)))
|
|
1691
|
+
issues.append(
|
|
1692
|
+
IssueReconciliation(number, LocalFact.HELD_UNCERTAIN, detail=_bounded(str(exc)))
|
|
1693
|
+
)
|
|
1694
|
+
continue
|
|
1695
|
+
except (LockError, StateError, OSError) as exc:
|
|
1696
|
+
classification = _uncertain(UncertainReason.RUN_LOCK_UNUSABLE, str(exc))
|
|
1697
|
+
issues.append(
|
|
1698
|
+
IssueReconciliation(
|
|
1699
|
+
number, LocalFact.HELD_UNCERTAIN, classification, detail=classification.detail
|
|
1700
|
+
)
|
|
1701
|
+
)
|
|
1702
|
+
continue
|
|
1703
|
+
try:
|
|
1704
|
+
item = _reconcile_held(namespace, lock, epic, expected, problems)
|
|
1705
|
+
finally:
|
|
1706
|
+
lock.release()
|
|
1707
|
+
if item is not None:
|
|
1708
|
+
issues.append(item)
|
|
1709
|
+
return ReconcileReport(epic, tuple(issues), tuple(problems), listing.rejected)
|
|
1710
|
+
|
|
1711
|
+
|
|
1712
|
+
@dataclass(frozen=True)
|
|
1713
|
+
class EpicClassification:
|
|
1714
|
+
"""One namespace classified for one EPIC's reconciliation (D14.2 step 4); nothing closed.
|
|
1715
|
+
|
|
1716
|
+
``fact`` is what :func:`~autoforge.epic_graph.evaluate` is given: ``None``
|
|
1717
|
+
for ``fresh`` and ``resumable``, ``held:settled-namespace``,
|
|
1718
|
+
``held:uncertain``, and ``held:other-epic`` for a namespace of this
|
|
1719
|
+
repository bound to another EPIC, which has no ``classification``.
|
|
1720
|
+
"""
|
|
1721
|
+
|
|
1722
|
+
fact: LocalFact | None
|
|
1723
|
+
classification: Classification | None
|
|
1724
|
+
detail: str
|
|
1725
|
+
|
|
1726
|
+
@property
|
|
1727
|
+
def repository_mismatch(self) -> bool:
|
|
1728
|
+
"""A binding of another repository: D14.3's global uncertainty."""
|
|
1729
|
+
found = self.classification
|
|
1730
|
+
return found is not None and found.uncertain is UncertainReason.REPOSITORY_MISMATCH
|
|
1731
|
+
|
|
1732
|
+
|
|
1733
|
+
def classify_for_epic(
|
|
1734
|
+
read: NamespaceRead, epic: EpicRef, *, expected: ExpectedIdentity
|
|
1735
|
+
) -> EpicClassification | None:
|
|
1736
|
+
"""Classify one namespace read for ``epic`` (D14.2 step 4); pure, ``None`` when absent.
|
|
1737
|
+
|
|
1738
|
+
A namespace of this repository bound to another EPIC is
|
|
1739
|
+
``held:other-epic`` and is not classified; one bound to this EPIC's
|
|
1740
|
+
number with another issue id is ``uncertain``; a binding of another
|
|
1741
|
+
repository is classified (``uncertain``, a repository mismatch) whatever
|
|
1742
|
+
EPIC it names. Otherwise :func:`classify` decides. Reconciliation runs it
|
|
1743
|
+
under L2 and then closes a settled namespace's open claim; ``epic
|
|
1744
|
+
status`` and ``epic run --dry-run`` (#183) run it on a lock-free read and
|
|
1745
|
+
close nothing.
|
|
1746
|
+
"""
|
|
1747
|
+
_check_type(epic, EpicRef, "classified epic")
|
|
1748
|
+
_check_type(expected, ExpectedIdentity, "classification's expected identity")
|
|
1749
|
+
if read.absent:
|
|
1750
|
+
return None
|
|
1751
|
+
lookup = read.lookup
|
|
1752
|
+
binding = lookup.binding if isinstance(lookup, NamespaceLookup) else None
|
|
1753
|
+
# A binding of another repository is global uncertainty whatever EPIC it
|
|
1754
|
+
# names (D14.3), so only this repository's bindings are compared by EPIC:
|
|
1755
|
+
# classify reports the other repository's.
|
|
1756
|
+
if binding is not None and binding.run_key.repository_id == expected.repository_id:
|
|
1757
|
+
bound = binding.epic
|
|
1758
|
+
if bound.number != epic.number:
|
|
1759
|
+
return EpicClassification(
|
|
1760
|
+
LocalFact.HELD_OTHER_EPIC,
|
|
1761
|
+
None,
|
|
1762
|
+
f"bound under EPIC #{bound.number}, not #{epic.number}",
|
|
1763
|
+
)
|
|
1764
|
+
if bound != epic:
|
|
1765
|
+
mismatch = _uncertain(
|
|
1766
|
+
UncertainReason.BINDING_MISMATCH,
|
|
1767
|
+
f"bound under EPIC #{bound.number} with issue id {bound.issue_id}, not "
|
|
1768
|
+
f"{epic.issue_id}",
|
|
1769
|
+
read.claim.open_claim,
|
|
1770
|
+
)
|
|
1771
|
+
return EpicClassification(LocalFact.HELD_UNCERTAIN, mismatch, mismatch.detail)
|
|
1772
|
+
classification = classify(lookup, read.claim, expected=expected)
|
|
1773
|
+
if classification.kind is ClassificationKind.UNCERTAIN:
|
|
1774
|
+
fact: LocalFact | None = LocalFact.HELD_UNCERTAIN
|
|
1775
|
+
elif classification.kind is ClassificationKind.SETTLED:
|
|
1776
|
+
fact = LocalFact.HELD_SETTLED_NAMESPACE
|
|
1777
|
+
else:
|
|
1778
|
+
fact = None
|
|
1779
|
+
return EpicClassification(fact, classification, classification.detail)
|
|
1780
|
+
|
|
1781
|
+
|
|
1782
|
+
def _reconcile_held(
|
|
1783
|
+
namespace: IssueNamespace,
|
|
1784
|
+
lock: IssueRunLock,
|
|
1785
|
+
epic: EpicRef,
|
|
1786
|
+
expected: ExpectedIdentity,
|
|
1787
|
+
problems: list[str],
|
|
1788
|
+
) -> IssueReconciliation | None:
|
|
1789
|
+
number = namespace.issue_number
|
|
1790
|
+
found = classify_for_epic(read_namespace(namespace), epic, expected=expected)
|
|
1791
|
+
if found is None:
|
|
1792
|
+
return None
|
|
1793
|
+
classification = found.classification
|
|
1794
|
+
if classification is None:
|
|
1795
|
+
return IssueReconciliation(number, found.fact, detail=found.detail)
|
|
1796
|
+
if found.repository_mismatch:
|
|
1797
|
+
problems.append(classification.detail)
|
|
1798
|
+
if classification.kind is not ClassificationKind.SETTLED:
|
|
1799
|
+
return IssueReconciliation(number, found.fact, classification, detail=found.detail)
|
|
1800
|
+
closed = None
|
|
1801
|
+
if classification.open_claim is not None:
|
|
1802
|
+
try:
|
|
1803
|
+
closed = close_claim(
|
|
1804
|
+
namespace,
|
|
1805
|
+
lock,
|
|
1806
|
+
classification.open_claim,
|
|
1807
|
+
ClaimClassification.SETTLED,
|
|
1808
|
+
settlement=classification.settlement,
|
|
1809
|
+
detail=f"closed at restart reconciliation: {classification.detail}",
|
|
1810
|
+
)
|
|
1811
|
+
except StateError as exc:
|
|
1812
|
+
failed = _uncertain(UncertainReason.CLAIM_CONFLICT, str(exc), classification.open_claim)
|
|
1813
|
+
return IssueReconciliation(
|
|
1814
|
+
number, LocalFact.HELD_UNCERTAIN, failed, detail=failed.detail
|
|
1815
|
+
)
|
|
1816
|
+
return IssueReconciliation(
|
|
1817
|
+
number,
|
|
1818
|
+
LocalFact.HELD_SETTLED_NAMESPACE,
|
|
1819
|
+
classification,
|
|
1820
|
+
closed=closed,
|
|
1821
|
+
detail=classification.detail,
|
|
1822
|
+
)
|
|
1823
|
+
|
|
1824
|
+
|
|
1825
|
+
@dataclass(frozen=True)
|
|
1826
|
+
class Finalization:
|
|
1827
|
+
"""What :func:`finalize_worker` decided: the class it read, and the claim it closed."""
|
|
1828
|
+
|
|
1829
|
+
classification: Classification
|
|
1830
|
+
closed: ClosedClaim | None = None
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
def finalize_worker(
|
|
1834
|
+
namespace: IssueNamespace,
|
|
1835
|
+
lock: IssueRunLock,
|
|
1836
|
+
claim: Claim,
|
|
1837
|
+
exit_status: int | None,
|
|
1838
|
+
*,
|
|
1839
|
+
expected: ExpectedIdentity | None = None,
|
|
1840
|
+
) -> Finalization:
|
|
1841
|
+
"""Close a reaped worker's claim by the state it left (ADR 0008 D7.2, D14.1).
|
|
1842
|
+
|
|
1843
|
+
The caller re-acquired the issue's L2 after the worker exited. The
|
|
1844
|
+
namespace is read strictly and classified: ``settled`` closes the claim
|
|
1845
|
+
``settled``; ``resumable`` (an ``in_progress`` state, or an absent one
|
|
1846
|
+
with ``state_ref.present`` false) closes it ``interrupted`` when a worker
|
|
1847
|
+
was recorded and ``abandoned`` when none was. Anything else (a corrupt
|
|
1848
|
+
state, a state missing while ``present`` is true, a mismatch, an open
|
|
1849
|
+
claim on disk that is not ``claim``) leaves it open as ``uncertain``.
|
|
1850
|
+
``expected`` is :func:`classify`'s. ``exit_status`` is recorded as a
|
|
1851
|
+
diagnostic and never decides.
|
|
1852
|
+
"""
|
|
1853
|
+
_require_lock(namespace, lock, "finalize a worker's claim")
|
|
1854
|
+
_check_optional_expected(expected)
|
|
1855
|
+
if exit_status is not None:
|
|
1856
|
+
_check_int(exit_status, "worker exit status", -MAX_EXIT_STATUS, MAX_EXIT_STATUS)
|
|
1857
|
+
read = read_namespace(namespace)
|
|
1858
|
+
if read.absent:
|
|
1859
|
+
return Finalization(
|
|
1860
|
+
_uncertain(UncertainReason.CLAIM_CONFLICT, f"{namespace.claim_file} does not exist")
|
|
1861
|
+
)
|
|
1862
|
+
classification = classify(read.lookup, read.claim, expected=expected)
|
|
1863
|
+
if classification.kind is ClassificationKind.UNCERTAIN:
|
|
1864
|
+
return Finalization(classification)
|
|
1865
|
+
on_disk = classification.open_claim
|
|
1866
|
+
if on_disk is None or on_disk.identity != claim.identity:
|
|
1867
|
+
held = "no open claim" if on_disk is None else f"generation {on_disk.generation}"
|
|
1868
|
+
return Finalization(
|
|
1869
|
+
_uncertain(
|
|
1870
|
+
UncertainReason.CLAIM_CONFLICT,
|
|
1871
|
+
f"{namespace.claim_file} holds {held}, not the reaped worker's generation "
|
|
1872
|
+
f"{claim.generation}; the file is left as it is",
|
|
1873
|
+
on_disk,
|
|
1874
|
+
)
|
|
1875
|
+
)
|
|
1876
|
+
if classification.kind is ClassificationKind.SETTLED:
|
|
1877
|
+
closing = ClaimClassification.SETTLED
|
|
1878
|
+
else: # RESUMABLE: FRESH needs no open claim
|
|
1879
|
+
closing = (
|
|
1880
|
+
ClaimClassification.INTERRUPTED
|
|
1881
|
+
if on_disk.worker is not None
|
|
1882
|
+
else ClaimClassification.ABANDONED
|
|
1883
|
+
)
|
|
1884
|
+
detail = f"worker exited with status {exit_status}: {classification.detail}"
|
|
1885
|
+
try:
|
|
1886
|
+
closed = close_claim(
|
|
1887
|
+
namespace,
|
|
1888
|
+
lock,
|
|
1889
|
+
on_disk,
|
|
1890
|
+
closing,
|
|
1891
|
+
settlement=classification.settlement,
|
|
1892
|
+
exit_status=exit_status,
|
|
1893
|
+
detail=detail,
|
|
1894
|
+
)
|
|
1895
|
+
except ClaimConflict as exc:
|
|
1896
|
+
return Finalization(_uncertain(UncertainReason.CLAIM_CONFLICT, str(exc), on_disk))
|
|
1897
|
+
return Finalization(classification, closed)
|
|
1898
|
+
|
|
1899
|
+
|
|
1900
|
+
# -- open-claim queries (ADR 0008 D14.4, D17.2) -----------------------------------------
|
|
1901
|
+
|
|
1902
|
+
|
|
1903
|
+
@dataclass(frozen=True)
|
|
1904
|
+
class OpenClaim:
|
|
1905
|
+
"""An open claim found by :func:`open_claims_for_epic`.
|
|
1906
|
+
|
|
1907
|
+
``claim`` is ``None`` when the claim file cannot be read: whose it is and
|
|
1908
|
+
whether it is open are unknown, so it counts as open (fail-closed),
|
|
1909
|
+
with ``problem`` saying why.
|
|
1910
|
+
"""
|
|
1911
|
+
|
|
1912
|
+
issue_number: int
|
|
1913
|
+
claim: Claim | None
|
|
1914
|
+
problem: str = ""
|
|
1915
|
+
|
|
1916
|
+
|
|
1917
|
+
def open_claims_for_epic(root: str | Path, epic: EpicRef) -> tuple[OpenClaim, ...]:
|
|
1918
|
+
"""The claims bound to ``epic`` that are open, in ascending issue number (D14.4).
|
|
1919
|
+
|
|
1920
|
+
Backs ``epic run``'s refusal, which points to ``epic resume``. A claim
|
|
1921
|
+
file names the EPIC its namespace was bound under; one that cannot be
|
|
1922
|
+
read is reported too, since it may be this EPIC's. Read-only and
|
|
1923
|
+
lock-free: a reader sees one whole file or the other. A listing that
|
|
1924
|
+
cannot be read raises.
|
|
1925
|
+
"""
|
|
1926
|
+
_check_type(epic, EpicRef, "epic")
|
|
1927
|
+
listing = list_issue_namespaces(root)
|
|
1928
|
+
found: list[OpenClaim] = []
|
|
1929
|
+
for namespace in sorted((*listing.bound, *listing.unbound), key=lambda ns: ns.issue_number):
|
|
1930
|
+
read = read_claim_file(namespace)
|
|
1931
|
+
if read.status is ClaimFileStatus.CORRUPT:
|
|
1932
|
+
found.append(OpenClaim(namespace.issue_number, None, read.reason))
|
|
1933
|
+
continue
|
|
1934
|
+
claim_file = read.claim_file
|
|
1935
|
+
if claim_file is None or claim_file.current is None:
|
|
1936
|
+
continue
|
|
1937
|
+
if epic.number == claim_file.epic.number or epic.issue_id == claim_file.epic.issue_id:
|
|
1938
|
+
found.append(OpenClaim(namespace.issue_number, claim_file.current))
|
|
1939
|
+
return tuple(found)
|
|
1940
|
+
|
|
1941
|
+
|
|
1942
|
+
# -- scheduler.json (ADR 0008 D6) -------------------------------------------------------
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
class EndReason(Enum):
|
|
1946
|
+
NOTHING_ADMISSIBLE = "nothing-admissible"
|
|
1947
|
+
DRAINED = "drained"
|
|
1948
|
+
STOPPED = "stopped"
|
|
1949
|
+
INTERRUPTED = "interrupted"
|
|
1950
|
+
DEGRADED = "degraded"
|
|
1951
|
+
|
|
1952
|
+
|
|
1953
|
+
class AdmissionStatus(Enum):
|
|
1954
|
+
OPEN = "open"
|
|
1955
|
+
CLOSED = "closed"
|
|
1956
|
+
DEGRADED = "degraded"
|
|
1957
|
+
|
|
1958
|
+
|
|
1959
|
+
class AdmissionReason(Enum):
|
|
1960
|
+
STOP_REQUESTED = "stop-requested" # closed
|
|
1961
|
+
INTERRUPTED = "interrupted" # closed
|
|
1962
|
+
SNAPSHOT_REFUSED = "snapshot-refused" # degraded
|
|
1963
|
+
EPIC_NOT_ADMISSIBLE = "epic-not-admissible" # degraded
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
_ADMISSION_REASONS: dict[AdmissionStatus, frozenset[AdmissionReason]] = {
|
|
1967
|
+
AdmissionStatus.OPEN: frozenset(),
|
|
1968
|
+
AdmissionStatus.CLOSED: frozenset(
|
|
1969
|
+
{AdmissionReason.STOP_REQUESTED, AdmissionReason.INTERRUPTED}
|
|
1970
|
+
),
|
|
1971
|
+
AdmissionStatus.DEGRADED: frozenset(
|
|
1972
|
+
{AdmissionReason.SNAPSHOT_REFUSED, AdmissionReason.EPIC_NOT_ADMISSIBLE}
|
|
1973
|
+
),
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
|
|
1977
|
+
@dataclass(frozen=True)
|
|
1978
|
+
class AdmissionState:
|
|
1979
|
+
"""Whether the scheduler admits: ``open`` without a reason, else with one of its own."""
|
|
1980
|
+
|
|
1981
|
+
status: AdmissionStatus
|
|
1982
|
+
reason: AdmissionReason | None = None
|
|
1983
|
+
|
|
1984
|
+
def __post_init__(self) -> None:
|
|
1985
|
+
_check_type(self.status, AdmissionStatus, "scheduler admission status")
|
|
1986
|
+
if self.reason is not None:
|
|
1987
|
+
_check_type(self.reason, AdmissionReason, "scheduler admission reason")
|
|
1988
|
+
allowed = _ADMISSION_REASONS[self.status]
|
|
1989
|
+
if (self.reason is None) != (not allowed) or (
|
|
1990
|
+
self.reason is not None and self.reason not in allowed
|
|
1991
|
+
):
|
|
1992
|
+
names = ", ".join(sorted(reason.value for reason in allowed)) or "none"
|
|
1993
|
+
_fail(
|
|
1994
|
+
"scheduler admission",
|
|
1995
|
+
f"{self.status.value} takes the reason(s) {names}, got "
|
|
1996
|
+
f"{None if self.reason is None else self.reason.value}",
|
|
1997
|
+
)
|
|
1998
|
+
|
|
1999
|
+
def to_dict(self) -> dict:
|
|
2000
|
+
return {
|
|
2001
|
+
"status": self.status.value,
|
|
2002
|
+
"reason": None if self.reason is None else self.reason.value,
|
|
2003
|
+
}
|
|
2004
|
+
|
|
2005
|
+
@classmethod
|
|
2006
|
+
def from_dict(cls, raw: object) -> AdmissionState:
|
|
2007
|
+
data = _exact(raw, ("status", "reason"), "scheduler admission")
|
|
2008
|
+
reason = data["reason"]
|
|
2009
|
+
return cls(
|
|
2010
|
+
_check_enum(AdmissionStatus, data["status"], "scheduler admission status"),
|
|
2011
|
+
None
|
|
2012
|
+
if reason is None
|
|
2013
|
+
else _check_enum(AdmissionReason, reason, "scheduler admission reason"),
|
|
2014
|
+
)
|
|
2015
|
+
|
|
2016
|
+
|
|
2017
|
+
@dataclass(frozen=True)
|
|
2018
|
+
class Invocation:
|
|
2019
|
+
"""The scheduler process the file belongs to: ``ended_at`` and ``end_reason`` go together."""
|
|
2020
|
+
|
|
2021
|
+
invocation_id: str
|
|
2022
|
+
started_at: str
|
|
2023
|
+
ended_at: str | None = None
|
|
2024
|
+
end_reason: EndReason | None = None
|
|
2025
|
+
|
|
2026
|
+
def __post_init__(self) -> None:
|
|
2027
|
+
_check_hex(self.invocation_id, "scheduler invocation_id", _HEX32)
|
|
2028
|
+
_check_timestamp(self.started_at, "scheduler invocation started_at")
|
|
2029
|
+
if self.ended_at is not None:
|
|
2030
|
+
_check_timestamp(self.ended_at, "scheduler invocation ended_at")
|
|
2031
|
+
if self.end_reason is not None:
|
|
2032
|
+
_check_type(self.end_reason, EndReason, "scheduler invocation end_reason")
|
|
2033
|
+
if (self.ended_at is None) != (self.end_reason is None):
|
|
2034
|
+
_fail("scheduler invocation", "has only one of ended_at and end_reason")
|
|
2035
|
+
|
|
2036
|
+
def to_dict(self) -> dict:
|
|
2037
|
+
return {
|
|
2038
|
+
"invocation_id": self.invocation_id,
|
|
2039
|
+
"started_at": self.started_at,
|
|
2040
|
+
"ended_at": self.ended_at,
|
|
2041
|
+
"end_reason": None if self.end_reason is None else self.end_reason.value,
|
|
2042
|
+
}
|
|
2043
|
+
|
|
2044
|
+
@classmethod
|
|
2045
|
+
def from_dict(cls, raw: object) -> Invocation:
|
|
2046
|
+
data = _exact(
|
|
2047
|
+
raw, ("invocation_id", "started_at", "ended_at", "end_reason"), "scheduler invocation"
|
|
2048
|
+
)
|
|
2049
|
+
end_reason = data["end_reason"]
|
|
2050
|
+
return cls(
|
|
2051
|
+
data["invocation_id"],
|
|
2052
|
+
data["started_at"],
|
|
2053
|
+
data["ended_at"],
|
|
2054
|
+
None
|
|
2055
|
+
if end_reason is None
|
|
2056
|
+
else _check_enum(EndReason, end_reason, "scheduler invocation end_reason"),
|
|
2057
|
+
)
|
|
2058
|
+
|
|
2059
|
+
|
|
2060
|
+
@dataclass(frozen=True)
|
|
2061
|
+
class SchedulerConfig:
|
|
2062
|
+
"""The configuration a scheduler runs under (D18.3, D18.4) and its digest (D9.2 step 4)."""
|
|
2063
|
+
|
|
2064
|
+
max_parallel: int
|
|
2065
|
+
poll_interval_seconds: int
|
|
2066
|
+
stop_grace_seconds: int
|
|
2067
|
+
config_digest: str
|
|
2068
|
+
|
|
2069
|
+
def __post_init__(self) -> None:
|
|
2070
|
+
_check_int(self.max_parallel, "scheduler config max_parallel", *MAX_PARALLEL_BOUNDS)
|
|
2071
|
+
_check_int(
|
|
2072
|
+
self.poll_interval_seconds,
|
|
2073
|
+
"scheduler config poll_interval_seconds",
|
|
2074
|
+
*POLL_INTERVAL_BOUNDS,
|
|
2075
|
+
)
|
|
2076
|
+
_check_int(
|
|
2077
|
+
self.stop_grace_seconds, "scheduler config stop_grace_seconds", *STOP_GRACE_BOUNDS
|
|
2078
|
+
)
|
|
2079
|
+
_check_hex(self.config_digest, "scheduler config config_digest", _HEX64)
|
|
2080
|
+
|
|
2081
|
+
def to_dict(self) -> dict:
|
|
2082
|
+
return {
|
|
2083
|
+
"max_parallel": self.max_parallel,
|
|
2084
|
+
"poll_interval_seconds": self.poll_interval_seconds,
|
|
2085
|
+
"stop_grace_seconds": self.stop_grace_seconds,
|
|
2086
|
+
"config_digest": self.config_digest,
|
|
2087
|
+
}
|
|
2088
|
+
|
|
2089
|
+
@classmethod
|
|
2090
|
+
def from_dict(cls, raw: object) -> SchedulerConfig:
|
|
2091
|
+
data = _exact(
|
|
2092
|
+
raw,
|
|
2093
|
+
("max_parallel", "poll_interval_seconds", "stop_grace_seconds", "config_digest"),
|
|
2094
|
+
"scheduler config",
|
|
2095
|
+
)
|
|
2096
|
+
return cls(
|
|
2097
|
+
data["max_parallel"],
|
|
2098
|
+
data["poll_interval_seconds"],
|
|
2099
|
+
data["stop_grace_seconds"],
|
|
2100
|
+
data["config_digest"],
|
|
2101
|
+
)
|
|
2102
|
+
|
|
2103
|
+
|
|
2104
|
+
_COUNT_KEYS = ("children", "ready", "blocked", "closed", "unsupported", "owned", "held")
|
|
2105
|
+
|
|
2106
|
+
|
|
2107
|
+
@dataclass(frozen=True)
|
|
2108
|
+
class SnapshotSummary:
|
|
2109
|
+
"""The last snapshot's digest, time and verdict counts (#179): diagnostics only."""
|
|
2110
|
+
|
|
2111
|
+
digest: str
|
|
2112
|
+
observed_at: str
|
|
2113
|
+
children: int = 0
|
|
2114
|
+
ready: int = 0
|
|
2115
|
+
blocked: int = 0
|
|
2116
|
+
closed: int = 0
|
|
2117
|
+
unsupported: int = 0
|
|
2118
|
+
owned: int = 0
|
|
2119
|
+
held: int = 0
|
|
2120
|
+
|
|
2121
|
+
def __post_init__(self) -> None:
|
|
2122
|
+
_check_hex(self.digest, "scheduler last_snapshot digest", _HEX64)
|
|
2123
|
+
_check_timestamp(self.observed_at, "scheduler last_snapshot observed_at")
|
|
2124
|
+
for name in _COUNT_KEYS:
|
|
2125
|
+
_check_int(getattr(self, name), f"scheduler last_snapshot count {name}", 0, MAX_COUNT)
|
|
2126
|
+
|
|
2127
|
+
def to_dict(self) -> dict:
|
|
2128
|
+
return {
|
|
2129
|
+
"digest": self.digest,
|
|
2130
|
+
"observed_at": self.observed_at,
|
|
2131
|
+
"counts": {name: getattr(self, name) for name in _COUNT_KEYS},
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
@classmethod
|
|
2135
|
+
def from_dict(cls, raw: object) -> SnapshotSummary:
|
|
2136
|
+
data = _exact(raw, ("digest", "observed_at", "counts"), "scheduler last_snapshot")
|
|
2137
|
+
counts = _exact(data["counts"], _COUNT_KEYS, "scheduler last_snapshot counts")
|
|
2138
|
+
return cls(data["digest"], data["observed_at"], **counts)
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
_SCHEDULER_KEYS = (
|
|
2142
|
+
"schema",
|
|
2143
|
+
"repository",
|
|
2144
|
+
"epic",
|
|
2145
|
+
"invocation",
|
|
2146
|
+
"config",
|
|
2147
|
+
"admission",
|
|
2148
|
+
"last_snapshot",
|
|
2149
|
+
"created_at",
|
|
2150
|
+
"updated_at",
|
|
2151
|
+
)
|
|
2152
|
+
|
|
2153
|
+
|
|
2154
|
+
@dataclass(frozen=True)
|
|
2155
|
+
class SchedulerState:
|
|
2156
|
+
"""``scheduler.json``, schema ``autoforge.scheduler/1`` (ADR 0008 D6.1): run-level facts.
|
|
2157
|
+
|
|
2158
|
+
It lists no claim (D6.2): claims live only in each issue's ``claim.json``.
|
|
2159
|
+
"""
|
|
2160
|
+
|
|
2161
|
+
repository_id: int
|
|
2162
|
+
repository_owner: str
|
|
2163
|
+
repository_name: str
|
|
2164
|
+
epic: EpicRef
|
|
2165
|
+
invocation: Invocation
|
|
2166
|
+
config: SchedulerConfig
|
|
2167
|
+
admission: AdmissionState
|
|
2168
|
+
last_snapshot: SnapshotSummary | None = None
|
|
2169
|
+
created_at: str = field(default_factory=utcnow_iso)
|
|
2170
|
+
updated_at: str = field(default_factory=utcnow_iso)
|
|
2171
|
+
|
|
2172
|
+
def __post_init__(self) -> None:
|
|
2173
|
+
_check_id(self.repository_id, "scheduler repository_id")
|
|
2174
|
+
_check_name(self.repository_owner, "scheduler repository owner", MAX_OWNER_CHARS)
|
|
2175
|
+
_check_name(self.repository_name, "scheduler repository name", MAX_REPOSITORY_NAME_CHARS)
|
|
2176
|
+
_check_type(self.epic, EpicRef, "scheduler epic")
|
|
2177
|
+
_check_type(self.invocation, Invocation, "scheduler invocation")
|
|
2178
|
+
_check_type(self.config, SchedulerConfig, "scheduler config")
|
|
2179
|
+
_check_type(self.admission, AdmissionState, "scheduler admission")
|
|
2180
|
+
if self.last_snapshot is not None:
|
|
2181
|
+
_check_type(self.last_snapshot, SnapshotSummary, "scheduler last_snapshot")
|
|
2182
|
+
_check_timestamp(self.created_at, "scheduler created_at")
|
|
2183
|
+
_check_timestamp(self.updated_at, "scheduler updated_at")
|
|
2184
|
+
|
|
2185
|
+
def to_dict(self) -> dict:
|
|
2186
|
+
return {
|
|
2187
|
+
"schema": SCHEDULER_SCHEMA,
|
|
2188
|
+
"repository": {
|
|
2189
|
+
"owner": self.repository_owner,
|
|
2190
|
+
"name": self.repository_name,
|
|
2191
|
+
"repository_id": self.repository_id,
|
|
2192
|
+
},
|
|
2193
|
+
"epic": _epic_dict(self.epic),
|
|
2194
|
+
"invocation": self.invocation.to_dict(),
|
|
2195
|
+
"config": self.config.to_dict(),
|
|
2196
|
+
"admission": self.admission.to_dict(),
|
|
2197
|
+
"last_snapshot": None if self.last_snapshot is None else self.last_snapshot.to_dict(),
|
|
2198
|
+
"created_at": self.created_at,
|
|
2199
|
+
"updated_at": self.updated_at,
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
@classmethod
|
|
2203
|
+
def from_dict(cls, raw: object, *, where: str = SCHEDULER_FILENAME) -> SchedulerState:
|
|
2204
|
+
"""Strictly parse a scheduler state: an unknown version or field is refused."""
|
|
2205
|
+
try:
|
|
2206
|
+
if not isinstance(raw, dict):
|
|
2207
|
+
_fail("scheduler state", f"must be a JSON object, got {type(raw).__name__}")
|
|
2208
|
+
schema = raw.get("schema")
|
|
2209
|
+
if schema != SCHEDULER_SCHEMA:
|
|
2210
|
+
_fail(
|
|
2211
|
+
"scheduler state",
|
|
2212
|
+
f"has schema {schema!r:.80}; this controller reads only {SCHEDULER_SCHEMA}",
|
|
2213
|
+
)
|
|
2214
|
+
data = _exact(raw, _SCHEDULER_KEYS, "scheduler state")
|
|
2215
|
+
repo = _exact(
|
|
2216
|
+
data["repository"], ("owner", "name", "repository_id"), "scheduler repository"
|
|
2217
|
+
)
|
|
2218
|
+
snapshot = data["last_snapshot"]
|
|
2219
|
+
return cls(
|
|
2220
|
+
repository_id=repo["repository_id"],
|
|
2221
|
+
repository_owner=repo["owner"],
|
|
2222
|
+
repository_name=repo["name"],
|
|
2223
|
+
epic=_epic_of(data["epic"], "scheduler epic"),
|
|
2224
|
+
invocation=Invocation.from_dict(data["invocation"]),
|
|
2225
|
+
config=SchedulerConfig.from_dict(data["config"]),
|
|
2226
|
+
admission=AdmissionState.from_dict(data["admission"]),
|
|
2227
|
+
last_snapshot=None if snapshot is None else SnapshotSummary.from_dict(snapshot),
|
|
2228
|
+
created_at=data["created_at"],
|
|
2229
|
+
updated_at=data["updated_at"],
|
|
2230
|
+
)
|
|
2231
|
+
except SchedulerStateError as exc:
|
|
2232
|
+
raise SchedulerStateError(
|
|
2233
|
+
f"invalid {SCHEDULER_SCHEMA} document {where}: {exc}"
|
|
2234
|
+
) from exc
|
|
2235
|
+
|
|
2236
|
+
|
|
2237
|
+
def _epic_relative(epic_number: int) -> tuple[str, ...]:
|
|
2238
|
+
_check_id(epic_number, "epic number")
|
|
2239
|
+
return (AUTOFORGE_DIRNAME, SCHEDULER_DIRNAME, EPICS_DIRNAME, str(epic_number))
|
|
2240
|
+
|
|
2241
|
+
|
|
2242
|
+
def scheduler_state_path(coordination_root: str | Path, epic_number: int) -> Path:
|
|
2243
|
+
"""``<coordination root>/autoforge/scheduler/epics/<n>/scheduler.json``."""
|
|
2244
|
+
return Path(coordination_root).joinpath(*_epic_relative(epic_number), SCHEDULER_FILENAME)
|
|
2245
|
+
|
|
2246
|
+
|
|
2247
|
+
def load_scheduler_state(coordination_root: str | Path, epic_number: int) -> SchedulerState | None:
|
|
2248
|
+
"""EPIC ``epic_number``'s ``scheduler.json``, strictly; ``None`` when there is none.
|
|
2249
|
+
|
|
2250
|
+
A corrupt file (one holding JSON ``null`` included), or one of an
|
|
2251
|
+
unknown version, raises :class:`~autoforge.errors.SchedulerStateError`:
|
|
2252
|
+
the scheduler's global uncertainty (D6.3), recovered only by
|
|
2253
|
+
``epic run --force``. Read-only.
|
|
2254
|
+
"""
|
|
2255
|
+
relative = _epic_relative(epic_number)
|
|
2256
|
+
where = str(scheduler_state_path(coordination_root, epic_number))
|
|
2257
|
+
with _open_anchor(Path(coordination_root)) as anchor:
|
|
2258
|
+
try:
|
|
2259
|
+
epic_root = anchor.subroot("/".join(relative))
|
|
2260
|
+
except FileNotFoundError:
|
|
2261
|
+
return None
|
|
2262
|
+
with epic_root:
|
|
2263
|
+
raw = _read_document(epic_root, SCHEDULER_FILENAME, limit=MAX_SCHEDULER_BYTES, where=where)
|
|
2264
|
+
if raw is None:
|
|
2265
|
+
return None
|
|
2266
|
+
state = SchedulerState.from_dict(raw, where=where)
|
|
2267
|
+
if state.epic.number != epic_number:
|
|
2268
|
+
raise SchedulerStateError(f"{where} names EPIC #{state.epic.number}, not #{epic_number}")
|
|
2269
|
+
return state
|
|
2270
|
+
|
|
2271
|
+
|
|
2272
|
+
def write_scheduler_state(coordination_root: str | Path, state: SchedulerState) -> SchedulerState:
|
|
2273
|
+
"""Write ``scheduler.json`` atomically, ``updated_at`` stamped; return what was written.
|
|
2274
|
+
|
|
2275
|
+
The single writer is the L0 holder (D11.1). The directories from the
|
|
2276
|
+
coordination root down are created ``O_NOFOLLOW``, and the write is an
|
|
2277
|
+
fsynced temporary replaced over the name, refused while this process
|
|
2278
|
+
holds L1 (D11.2).
|
|
2279
|
+
"""
|
|
2280
|
+
_check_type(state, SchedulerState, "scheduler state")
|
|
2281
|
+
written = replace(state, updated_at=utcnow_iso())
|
|
2282
|
+
path = scheduler_state_path(coordination_root, state.epic.number)
|
|
2283
|
+
assert_shared_git_lock_free(f"write {path}")
|
|
2284
|
+
payload = _payload(written.to_dict(), limit=MAX_SCHEDULER_BYTES, where=str(path))
|
|
2285
|
+
with _open_anchor(Path(coordination_root)) as anchor:
|
|
2286
|
+
anchor.write_bytes(
|
|
2287
|
+
"/".join((*_epic_relative(state.epic.number), SCHEDULER_FILENAME)), payload
|
|
2288
|
+
)
|
|
2289
|
+
return written
|
|
2290
|
+
|
|
2291
|
+
|
|
2292
|
+
def quarantine_scheduler_state(coordination_root: str | Path, epic_number: int) -> Path | None:
|
|
2293
|
+
"""Move a corrupt ``scheduler.json`` aside for ``epic run --force`` (D6.3).
|
|
2294
|
+
|
|
2295
|
+
Renamed to ``scheduler.json.corrupt-<UTC timestamp>`` with the
|
|
2296
|
+
link-then-unlink primitive of ``run --force``'s quarantine, so nothing
|
|
2297
|
+
is lost; safe because the file holds no claim. Returns the new path, or
|
|
2298
|
+
``None`` when there is no file. A file that loads is refused
|
|
2299
|
+
(:class:`~autoforge.errors.SchedulerStateError`): ``--force`` recovers
|
|
2300
|
+
from corruption only.
|
|
2301
|
+
"""
|
|
2302
|
+
path = scheduler_state_path(coordination_root, epic_number)
|
|
2303
|
+
try:
|
|
2304
|
+
if load_scheduler_state(coordination_root, epic_number) is None:
|
|
2305
|
+
return None
|
|
2306
|
+
except SchedulerStateError:
|
|
2307
|
+
pass
|
|
2308
|
+
else:
|
|
2309
|
+
raise SchedulerStateError(
|
|
2310
|
+
f"{path} is a valid {SCHEDULER_SCHEMA} document; only a corrupt one is quarantined"
|
|
2311
|
+
)
|
|
2312
|
+
assert_shared_git_lock_free(f"quarantine {path}")
|
|
2313
|
+
with _open_anchor(Path(coordination_root)) as anchor:
|
|
2314
|
+
epic_root = anchor.subroot("/".join(_epic_relative(epic_number)))
|
|
2315
|
+
with epic_root:
|
|
2316
|
+
return quarantine_state_file(path, root=epic_root, what="scheduler state file")
|