worktree-controller 1.30.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. worktree_controller/__init__.py +3 -0
  2. worktree_controller/activation_layout.py +483 -0
  3. worktree_controller/branch_retirement.py +1865 -0
  4. worktree_controller/canonical_refresh.py +150 -0
  5. worktree_controller/canonical_transition.py +324 -0
  6. worktree_controller/cleanup_attempts.py +442 -0
  7. worktree_controller/cleanup_evidence.py +75 -0
  8. worktree_controller/cleanup_recovery.py +912 -0
  9. worktree_controller/cli.py +1907 -0
  10. worktree_controller/controller.py +1974 -0
  11. worktree_controller/convergence.py +893 -0
  12. worktree_controller/delivered_closeout.py +928 -0
  13. worktree_controller/disposable.py +677 -0
  14. worktree_controller/environment.py +1038 -0
  15. worktree_controller/frozen_recovery.py +291 -0
  16. worktree_controller/git_backend.py +682 -0
  17. worktree_controller/governance.py +365 -0
  18. worktree_controller/hardlinks.py +294 -0
  19. worktree_controller/integration_cleanup.py +92 -0
  20. worktree_controller/integration_conflict.py +117 -0
  21. worktree_controller/integration_retry.py +526 -0
  22. worktree_controller/keyed_document_scope.py +162 -0
  23. worktree_controller/lifecycle.py +8992 -0
  24. worktree_controller/lifecycle_history.py +358 -0
  25. worktree_controller/lifecycle_observation.py +234 -0
  26. worktree_controller/metadata_codec.py +1694 -0
  27. worktree_controller/metadata_relocation.py +131 -0
  28. worktree_controller/models.py +546 -0
  29. worktree_controller/mount_diagnostics.py +324 -0
  30. worktree_controller/multiple_cleanup_recovery.py +730 -0
  31. worktree_controller/namespace_migration.py +1412 -0
  32. worktree_controller/native_session_recovery.py +171 -0
  33. worktree_controller/owner_actions.py +34 -0
  34. worktree_controller/policy.py +333 -0
  35. worktree_controller/preservation.py +983 -0
  36. worktree_controller/preserved_cleanup.py +849 -0
  37. worktree_controller/publication_execution.py +361 -0
  38. worktree_controller/reconciler.py +430 -0
  39. worktree_controller/recovery_guidance.py +205 -0
  40. worktree_controller/registration_backfill.py +166 -0
  41. worktree_controller/registration_prune.py +391 -0
  42. worktree_controller/rendering.py +179 -0
  43. worktree_controller/resource_admission.py +1736 -0
  44. worktree_controller/retirement_adapters.py +130 -0
  45. worktree_controller/retirement_execution.py +234 -0
  46. worktree_controller/semantics.py +459 -0
  47. worktree_controller/session.py +505 -0
  48. worktree_controller/session_admission.py +265 -0
  49. worktree_controller/source_cleanup.py +515 -0
  50. worktree_controller/source_preservation.py +692 -0
  51. worktree_controller/source_recovery.py +592 -0
  52. worktree_controller/store.py +1228 -0
  53. worktree_controller/tool_cli_contract.json +1791 -0
  54. worktree_controller/transaction_scope.py +876 -0
  55. worktree_controller/writer_lease.py +662 -0
  56. worktree_controller/writer_session.py +887 -0
  57. worktree_controller-1.30.4.dist-info/METADATA +3149 -0
  58. worktree_controller-1.30.4.dist-info/RECORD +61 -0
  59. worktree_controller-1.30.4.dist-info/WHEEL +4 -0
  60. worktree_controller-1.30.4.dist-info/entry_points.txt +5 -0
  61. worktree_controller-1.30.4.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,1736 @@
1
+ """Host resource reservations and durable checkout requests.
2
+
3
+ The filesystem and kernel report occupancy; this ledger accounts only for growth
4
+ not yet materialized. Repository locks still own allocations and Git mutations.
5
+ Lock order is repository -> host ledger. No host lock is held while calling a
6
+ repository mutation, a process, or a cleanup coordinator.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import fcntl
12
+ import hashlib
13
+ import json
14
+ import os
15
+ import stat
16
+ import time
17
+ from contextlib import contextmanager, nullcontext, suppress
18
+ from pathlib import Path
19
+ from typing import Any
20
+ from uuid import UUID
21
+
22
+ from .git_backend import _git, inventory, public_refs, resolve_commit, resolve_repository
23
+ from .models import (
24
+ ControllerError,
25
+ MetadataSnapshot,
26
+ RepositoryInventory,
27
+ WorktreePolicy,
28
+ WriterHolder,
29
+ )
30
+ from .semantics import integration_intent_abandoned
31
+ from .store import MetadataStore, decode_cache
32
+
33
+ CONTRACT = "host-resource-admission/v1"
34
+ GIB = 1024**3
35
+ # Safety headroom is a resource budget, never a checkout/concurrency count.
36
+ MIN_FREE_BYTES = 16 * GIB
37
+ MIN_FREE_INODES = 100_000
38
+ MIN_AVAILABLE_MEMORY = 4 * GIB
39
+ DEFAULT_EXECUTION_MEMORY = 2 * GIB
40
+ CREATION_BURST = 16
41
+ CREATION_INTERVAL_NS = 1_000_000_000
42
+ PRESSURE_HIGH = {"cpu": 60.0, "memory": 10.0, "io": 20.0}
43
+ PRESSURE_LOW = {"cpu": 30.0, "memory": 2.0, "io": 5.0}
44
+ STATES = {"WAITING", "RESERVED", "ALLOCATED", "RELEASED", "ATTENTION_REQUIRED", "CANCELLED"}
45
+
46
+
47
+ def fingerprint(value: object) -> str:
48
+ return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
49
+
50
+
51
+ def canonical_holder_digest(holder: dict[str, Any]) -> str:
52
+ """SHA-256 over the canonical holder bytes: sorted-key, compact JSON, UTF-8.
53
+
54
+ The holder document is exactly the six WriterHolder fields as persisted in a
55
+ reservation row (client_id, pid, boot_id, start_ticks, session_id, run_id).
56
+ ``host-execution --action inspect`` publishes both the canonical bytes and
57
+ this digest so recovery callers never have to read the ledger layout.
58
+ """
59
+ return hashlib.sha256(
60
+ json.dumps(holder, sort_keys=True, separators=(",", ":")).encode("utf-8")
61
+ ).hexdigest()
62
+
63
+
64
+ def unbounded_host_scope(path: Path) -> bool:
65
+ """True for the home root itself or any ancestor still containing it.
66
+
67
+ Such a target cannot bound a host reservation to one session, so the clear
68
+ side already refused identity-only recovery there; the admit side must not
69
+ mint those reservations either. An unresolvable home is treated as
70
+ unbounded (fail closed).
71
+ """
72
+ home = Path.home()
73
+ if path == home or path in home.parents:
74
+ return True
75
+ try:
76
+ resolved_home = home.resolve(strict=True)
77
+ except OSError:
78
+ return True
79
+ return path == resolved_home or path in resolved_home.parents
80
+
81
+
82
+ def filesystem(path: Path) -> dict[str, int | str]:
83
+ parent = path
84
+ while not parent.exists():
85
+ if parent == parent.parent:
86
+ raise ControllerError("RESOURCE_OBSERVATION_UNAVAILABLE", "target filesystem is unavailable")
87
+ parent = parent.parent
88
+ parent = parent.resolve(strict=True)
89
+ info, capacity = parent.stat(), os.statvfs(parent)
90
+ return {"device": info.st_dev, "observed_path": str(parent),
91
+ "available_bytes": capacity.f_bavail * capacity.f_frsize,
92
+ "available_inodes": capacity.f_favail}
93
+
94
+
95
+ def pressure() -> dict[str, Any]:
96
+ """Only public aggregate kernel counters; never process environment/state."""
97
+ try:
98
+ memory = dict(line.split(":", 1) for line in Path("/proc/meminfo").read_text().splitlines())
99
+ available = int(memory["MemAvailable"].split()[0]) * 1024
100
+ observed = {}
101
+ for kind in PRESSURE_HIGH:
102
+ lines = Path(f"/proc/pressure/{kind}").read_text().splitlines()
103
+ some = next(line for line in lines if line.startswith("some "))
104
+ values = dict(item.split("=", 1) for item in some.split()[1:])
105
+ observed[kind] = float(values["avg10"])
106
+ if available < 0 or any(not 0 <= value <= 100 for value in observed.values()):
107
+ raise ValueError("invalid resource counters")
108
+ return {"available_memory_bytes": available, "psi_avg10": observed,
109
+ "observed_at_ns": time.time_ns()}
110
+ except (OSError, ValueError, KeyError, StopIteration) as exc:
111
+ raise ControllerError("RESOURCE_OBSERVATION_UNAVAILABLE", "kernel resource counters unavailable") from exc
112
+
113
+
114
+ def checkout_estimate(repository: Path, base: str, *, growth_bytes: int, growth_inodes: int) -> dict[str, int]:
115
+ """Estimate checkout + environment/artifact growth + an independent preservation.
116
+
117
+ Git object storage/shared caches already occupy the target filesystem and are
118
+ not billed as additional copies. gitlinks are not recursively initialized.
119
+ """
120
+ total = 0
121
+ directories: set[bytes] = set()
122
+ files = 0
123
+ for row in _git(repository, "ls-tree", "-r", "-l", "-z", base).stdout.split(b"\0"):
124
+ if not row:
125
+ continue
126
+ identity, path = row.split(b"\t", 1)
127
+ fields = identity.split()
128
+ if len(fields) != 4:
129
+ raise ControllerError("RESOURCE_OBSERVATION_UNAVAILABLE", "invalid Git tree estimate")
130
+ if fields[1] == b"blob":
131
+ total += int(fields[3])
132
+ files += 1
133
+ parts = path.split(b"/")
134
+ directories.update(b"/".join(parts[:index]) for index in range(1, len(parts)))
135
+ return {"checkout_bytes": total, "checkout_inodes": files + len(directories) + 2,
136
+ "growth_bytes": growth_bytes, "growth_inodes": growth_inodes,
137
+ "preservation_bytes": total + growth_bytes,
138
+ "preservation_inodes": files + len(directories) + growth_inodes + 2}
139
+
140
+
141
+ def allocation_request(repository: RepositoryInventory, repository_id: str, policy: WorktreePolicy,
142
+ *, target: Path, base_oid: str, arguments: dict[str, Any]) -> dict[str, Any]:
143
+ arguments = {"scope_paths": (), "source_allocations": (), "source_handoffs": (),
144
+ "external_bindings": (), "integration_intent_id": None,
145
+ "lineage_base_commit": None, **arguments}
146
+ return {"repository_id": repository_id, "repository_path": str(repository.canonical_path),
147
+ "common_dir": str(repository.common_dir), "target_path": str(target),
148
+ "filesystem_device": filesystem(target)["device"], "base_oid": base_oid,
149
+ "arguments": {key: str(value) if isinstance(value, Path) else list(value) if isinstance(value, tuple) else value
150
+ for key, value in arguments.items()},
151
+ "estimate": checkout_estimate(repository.canonical_path, base_oid,
152
+ growth_bytes=policy.checkout_growth_bytes,
153
+ growth_inodes=policy.checkout_growth_inodes)}
154
+
155
+
156
+ def prepare_request(repository_path: Path, arguments: dict[str, Any]) -> dict[str, Any]:
157
+ from .policy import load_policy
158
+
159
+ repository = inventory(repository_path)
160
+ snapshot = MetadataStore(repository.common_dir).load()
161
+ policy = load_policy(repository.canonical_path)
162
+ if not policy.resource_admission or snapshot.repository_id is None:
163
+ raise ControllerError("RESOURCE_ADMISSION_UNSUPPORTED", "repository has not adopted resource admission")
164
+ base = resolve_commit(repository.canonical_path, arguments["base_ref"])
165
+ target = Path(arguments.get("target") or Path.home() / "worktrees" / repository.canonical_path.name / arguments["task_key"]).expanduser().absolute()
166
+ return allocation_request(repository, snapshot.repository_id, policy, target=target, base_oid=base,
167
+ arguments={**arguments, "base_ref": base, "target": str(target)})
168
+
169
+
170
+ def materialized(path: Path, device: int) -> tuple[int, int]:
171
+ """Credit actual private allocated blocks only; no-follow and dedup by inode.
172
+
173
+ Shared/hardlinked files need no new storage. Their bytes were never additional
174
+ occupancy and cannot be used to claim that a future private growth budget has
175
+ already been spent. This inspection neither opens nor interprets file content.
176
+ """
177
+ total = 0
178
+ entries = 0
179
+ seen: set[tuple[int, int]] = set()
180
+ if not path.exists():
181
+ return 0, 0
182
+ flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
183
+ descriptor = os.open(path, flags)
184
+
185
+ def visit(directory: int) -> None:
186
+ nonlocal total, entries
187
+ for name in os.listdir(directory):
188
+ before = os.stat(name, dir_fd=directory, follow_symlinks=False)
189
+ if before.st_dev != device:
190
+ raise ControllerError("RESOURCE_OBSERVATION_UNAVAILABLE", "allocation crosses filesystem")
191
+ key = before.st_dev, before.st_ino
192
+ if key not in seen:
193
+ seen.add(key)
194
+ if stat.S_ISDIR(before.st_mode) or before.st_nlink == 1:
195
+ total += before.st_blocks * 512
196
+ entries += 1
197
+ if stat.S_ISDIR(before.st_mode):
198
+ child = os.open(name, flags, dir_fd=directory)
199
+ try:
200
+ current = os.fstat(child)
201
+ if (current.st_dev, current.st_ino) != key:
202
+ raise ControllerError("RESOURCE_OBSERVATION_UNAVAILABLE", "allocation tree changed")
203
+ visit(child)
204
+ finally:
205
+ os.close(child)
206
+ after = os.stat(name, dir_fd=directory, follow_symlinks=False)
207
+ if (before.st_dev, before.st_ino, before.st_mode) != (after.st_dev, after.st_ino, after.st_mode):
208
+ raise ControllerError("RESOURCE_OBSERVATION_UNAVAILABLE", "allocation tree changed")
209
+
210
+ try:
211
+ if os.fstat(descriptor).st_dev != device:
212
+ raise ControllerError("RESOURCE_OBSERVATION_UNAVAILABLE", "allocation filesystem changed")
213
+ visit(descriptor)
214
+ return total, entries
215
+ finally:
216
+ os.close(descriptor)
217
+
218
+
219
+ def execution_memory(row: dict[str, Any]) -> int:
220
+ """Credit observed proportional memory, never shared pages twice.
221
+
222
+ Child memory is already in MemAvailable. Crediting only the exact holder's
223
+ PSS deliberately leaves conservative headroom for children/future growth.
224
+ Unobservable or changed process identity receives no reservation credit.
225
+ """
226
+ from .writer_lease import holder_is_live
227
+
228
+ identity = row.get("holder")
229
+ if not isinstance(identity, dict):
230
+ return 0
231
+ try:
232
+ holder = WriterHolder(**identity)
233
+ if not holder_is_live(holder):
234
+ return 0
235
+ lines = Path(f"/proc/{holder.pid}/smaps_rollup").read_text().splitlines()
236
+ pss = next(int(line.split()[1]) * 1024 for line in lines if line.startswith("Pss:"))
237
+ return max(0, pss) if holder_is_live(holder) else 0
238
+ except (OSError, ValueError, TypeError, StopIteration, ControllerError):
239
+ return 0
240
+
241
+
242
+ def execution_request(repository: RepositoryInventory | None, repository_id: str, allocation_id: str | None,
243
+ target: Path, *, client_id: str, session_id: str | None, run_id: str | None,
244
+ memory_bytes: int = DEFAULT_EXECUTION_MEMORY, process_key: str | None = None) -> dict[str, Any]:
245
+ from .writer_lease import CLIENT_ID_PATTERN
246
+
247
+ identity = session_id or process_key
248
+ if (not CLIENT_ID_PATTERN.fullmatch(client_id) or not identity or len(identity) > 256
249
+ or "\x00" in identity or (run_id is not None and (not run_id or len(run_id) > 256 or "\x00" in run_id))
250
+ or type(memory_bytes) is not int or memory_bytes < 1):
251
+ raise ControllerError("INVALID_REQUEST", "execution resource request identity or budget is invalid")
252
+ operation = f"execution:{client_id}:{identity}"
253
+ return {"kind": "execution", "repository_id": repository_id,
254
+ "repository_path": str(repository.canonical_path) if repository else str(target),
255
+ "common_dir": str(repository.common_dir) if repository else "",
256
+ "target_path": str(target), "allocation_id": allocation_id, "memory_bytes": memory_bytes,
257
+ "client_id": client_id, "session_id": session_id, "run_id": run_id,
258
+ **({"process_key": process_key} if session_id is None else {}),
259
+ "arguments": {"task_key": operation, "authority_key": operation}}
260
+
261
+
262
+ def admit_execution(repository_path: Path, *, target: Path, owner: str, client_id: str,
263
+ session_id: str, run_id: str | None = None,
264
+ memory_bytes: int = DEFAULT_EXECUTION_MEMORY, cancel: bool = False,
265
+ _cache_recovery: bool = False) -> dict[str, Any]:
266
+ """Reserve (or cancel) one execution's budget.
267
+
268
+ Exit recovery of finished executions is paid only when it can change the answer:
269
+ an admission that is immediately available proceeds without it, while a wait or a
270
+ still-recorded earlier execution of the same task first reconciles finished
271
+ executions and then asks once more with the same durable request.
272
+ """
273
+ arguments = {"target": target, "owner": owner, "client_id": client_id, "session_id": session_id,
274
+ "run_id": run_id, "memory_bytes": memory_bytes}
275
+ if cancel:
276
+ return _admit_execution_once(repository_path, cancel=True, **arguments)
277
+ try:
278
+ admitted = _admit_execution_once(repository_path, cancel=False, **arguments)
279
+ except ControllerError as exc:
280
+ if exc.code != "ADMISSION_REQUEST_DRIFT":
281
+ raise
282
+ else:
283
+ if admitted["decision"].get("code") != "RESOURCE_WAIT":
284
+ return admitted
285
+ # CLI recovery can reload several snapshots; the ordinary first attempt stays uncached.
286
+ # Direct library callers keep their existing cache behavior unless already in a cache scope.
287
+ with decode_cache() if _cache_recovery else nullcontext():
288
+ reconcile_finished_executions()
289
+ return _admit_execution_once(repository_path, cancel=False, **arguments)
290
+
291
+
292
+ def _admit_execution_once(repository_path: Path, *, target: Path, owner: str, client_id: str,
293
+ session_id: str, run_id: str | None,
294
+ memory_bytes: int, cancel: bool) -> dict[str, Any]:
295
+ from .controller import envelope
296
+ from .policy import load_policy
297
+ from .semantics import managed_record as _managed_record
298
+
299
+ repository = inventory(repository_path)
300
+ store = MetadataStore(repository.common_dir)
301
+ with store.lock(exclusive=True):
302
+ if not load_policy(repository.canonical_path).resource_admission:
303
+ raise ControllerError("RESOURCE_ADMISSION_UNSUPPORTED", "repository resource policy is not enabled")
304
+ snapshot = store._load_unlocked()
305
+ item, record = _managed_record(repository, snapshot, target)
306
+ if record.owner != owner or record.lifecycle != "ACTIVE" or not snapshot.repository_id:
307
+ raise ControllerError("EXECUTION_AUTHORITY_CHANGED", "allocation is not active under this owner")
308
+ request = execution_request(repository, snapshot.repository_id, record.allocation_id, item.path,
309
+ client_id=client_id, session_id=session_id, run_id=run_id,
310
+ memory_bytes=memory_bytes)
311
+ request["owner"] = owner
312
+ from .writer_lease import holder_is_live, process_group_is_live
313
+
314
+ lease = snapshot.writer_leases.get(record.allocation_id)
315
+ if cancel:
316
+ if lease is not None and (lease.holder.client_id, lease.holder.session_id, lease.holder.run_id) == (client_id, session_id, run_id):
317
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "an acquired writer requires exit reconciliation")
318
+ decision = AdmissionLedger().cancel_execution(request)
319
+ else:
320
+ ledger = AdmissionLedger()
321
+ reserved = any(key != fingerprint(request) and row["state"] in {"RESERVED", "ALLOCATED", "ATTENTION_REQUIRED"}
322
+ and row["request"].get("kind") == "execution"
323
+ and (row["request"]["repository_id"], row["request"].get("allocation_id"))
324
+ == (snapshot.repository_id, record.allocation_id)
325
+ for key, row in ledger.read()["requests"].items())
326
+ # The repository lock serializes this allocation's reservation and
327
+ # writer grant. Two launch barriers must not both become admissible.
328
+ occupied = bool(record.external_bindings) or (lease is not None and
329
+ (holder_is_live(lease.holder) or process_group_is_live(lease.holder))) or reserved
330
+ decision = ledger.reserve([request], blockers=("ALLOCATION_OCCUPIED",) if occupied else ())
331
+ result = envelope("execution-cancel" if cancel else "execution-admit")
332
+ result["decision"] = {**decision, "allocation_id": record.allocation_id,
333
+ "session_id": session_id, "run_id": run_id}
334
+ return result
335
+
336
+
337
+ def bind_execution(repository_id: str, allocation_id: str, holder: WriterHolder, lease_id: str,
338
+ *, required: bool = False) -> None:
339
+ """The repository mutation lock and exact writer check precede this call."""
340
+ from dataclasses import asdict
341
+
342
+ ledger = AdmissionLedger()
343
+ if not ledger.path.exists():
344
+ if required:
345
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "execution reservation is missing")
346
+ return
347
+ with ledger.locked():
348
+ state = ledger.read()
349
+ matched = False
350
+ for row in state["requests"].values():
351
+ request = row["request"]
352
+ if (request.get("kind") != "execution" or request["repository_id"] != repository_id
353
+ or request["allocation_id"] != allocation_id
354
+ or (request["client_id"], request["session_id"], request["run_id"])
355
+ != (holder.client_id, holder.session_id, holder.run_id)):
356
+ continue
357
+ if request.get("process_key") not in {None, f"{holder.boot_id}:{holder.pid}:{holder.start_ticks}"}:
358
+ continue
359
+ matched = True
360
+ if row["state"] not in {"RESERVED", "ALLOCATED"} or row.get("lease_id") not in {None, lease_id}:
361
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "execution reservation cannot bind another attempt")
362
+ if row["state"] == "ALLOCATED" and row.get("holder") == asdict(holder):
363
+ continue
364
+ row.update(state="ALLOCATED", holder=asdict(holder), lease_id=lease_id,
365
+ allocation_id=allocation_id, updated_at_ns=time.time_ns())
366
+ row["history"].append({"state": "ALLOCATED", "at_ns": row["updated_at_ns"]})
367
+ if required and not matched:
368
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "execution reservation does not match this holder")
369
+ if not matched:
370
+ return
371
+ ledger._save(state)
372
+
373
+
374
+ def release_execution(repository_id: str, allocation_id: str, lease_id: str) -> None:
375
+ """Only a runtime-clear proof authorizes releasing the memory reservation."""
376
+ ledger = AdmissionLedger()
377
+ if not ledger.path.exists():
378
+ return
379
+ with ledger.locked():
380
+ state = ledger.read()
381
+ for row in state["requests"].values():
382
+ request = row["request"]
383
+ if (request.get("kind") == "execution" and request["repository_id"] == repository_id
384
+ and request["allocation_id"] == allocation_id and row.get("lease_id") == lease_id
385
+ and row["state"] == "ALLOCATED"):
386
+ row.update(state="RELEASED", updated_at_ns=time.time_ns())
387
+ row["history"].append({"state": "RELEASED", "at_ns": row["updated_at_ns"]})
388
+ ledger._save(state)
389
+
390
+
391
+ class AdmissionLedger:
392
+ def __init__(self, root: Path | None = None):
393
+ state_home = Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local/state")
394
+ if not state_home.is_absolute():
395
+ raise ControllerError("INVALID_RESOURCE_LEDGER", "XDG state directory must be absolute")
396
+ self.root = root or state_home / "worktree-controller/resource-admission/v1"
397
+ self.path = self.root / "state.json"
398
+ # Reuse the Controller's fsync/atomic-replace metadata backend. This is
399
+ # one host ledger in the existing state root, not a parallel database.
400
+ self.backend = MetadataStore(self.root)
401
+
402
+ @contextmanager
403
+ def locked(self):
404
+ self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
405
+ descriptor = os.open(self.root / "ledger.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
406
+ try:
407
+ fcntl.flock(descriptor, fcntl.LOCK_EX)
408
+ yield
409
+ finally:
410
+ fcntl.flock(descriptor, fcntl.LOCK_UN)
411
+ os.close(descriptor)
412
+
413
+ def read(self) -> dict[str, Any]:
414
+ if not self.path.exists():
415
+ return {"contract": CONTRACT, "requests": {}, "pressure_high": False}
416
+ value = dict(self.backend._read_json(self.path))
417
+ if value.get("contract") != CONTRACT or not isinstance(value.get("requests"), dict) or type(value.get("pressure_high")) is not bool:
418
+ raise ControllerError("INVALID_RESOURCE_LEDGER", "unsupported resource ledger")
419
+ for key, row in value["requests"].items():
420
+ if not isinstance(row, dict) or row.get("state") not in STATES or fingerprint(row.get("request")) != key:
421
+ raise ControllerError("INVALID_RESOURCE_LEDGER", "resource request identity changed")
422
+ return value
423
+
424
+ def _save(self, state: dict[str, Any]) -> None:
425
+ self.backend._atomic_json(self.path, state, compact=True)
426
+
427
+ def cancel_execution(self, request: dict[str, Any]) -> dict[str, Any]:
428
+ with self.locked():
429
+ state = self.read()
430
+ cancelled = []
431
+ for key, row in state["requests"].items():
432
+ previous = row["request"]
433
+ if (previous.get("kind") != "execution" or any(previous.get(name) != request.get(name)
434
+ for name in ("repository_id", "allocation_id", "owner", "client_id", "session_id", "run_id", "process_key"))):
435
+ continue
436
+ if row.get("holder") is not None or row["state"] not in {"WAITING", "RESERVED", "CANCELLED"}:
437
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "an attempted execution cannot be cancelled as no-action")
438
+ cancelled.append(key)
439
+ if row["state"] != "CANCELLED":
440
+ row.update(state="CANCELLED", updated_at_ns=time.time_ns())
441
+ row["history"].append({"state": "CANCELLED", "at_ns": row["updated_at_ns"]})
442
+ self._save(state)
443
+ return {"code": "EXECUTION_ADMISSION_CANCELLED", "request_ids": cancelled,
444
+ "process_started": False, "task_completed": False}
445
+
446
+ @staticmethod
447
+ def _outstanding(row: dict[str, Any]) -> tuple[int, int]:
448
+ if row["request"].get("kind") == "execution":
449
+ return 0, 0
450
+ estimate = row["request"]["estimate"]
451
+ # Preservation remains independently reserved until allocation release.
452
+ allocated = row.get("materialized_bytes", 0)
453
+ inodes = row.get("materialized_inodes", 0)
454
+ return (max(0, estimate["checkout_bytes"] + estimate["growth_bytes"] - allocated) + estimate["preservation_bytes"],
455
+ max(0, estimate["checkout_inodes"] + estimate["growth_inodes"] - inodes) + estimate["preservation_inodes"])
456
+
457
+ def _decision(self, state: dict[str, Any], requests: list[dict[str, Any]]) -> dict[str, Any]:
458
+ observed = pressure()
459
+ values = observed["psi_avg10"]
460
+ pressured = (any(values[key] >= PRESSURE_HIGH[key] for key in values)
461
+ or (state["pressure_high"] and not all(values[key] <= PRESSURE_LOW[key] for key in values)))
462
+ reasons = []
463
+ executing = any(request.get("kind") == "execution" for request in requests)
464
+ if executing and pressured:
465
+ reasons.append("HOST_PRESSURE")
466
+ filesystems = {}
467
+ for request in requests:
468
+ if request.get("kind") == "execution":
469
+ continue
470
+ sample = filesystem(Path(request["target_path"]))
471
+ if sample["device"] != request["filesystem_device"]:
472
+ raise ControllerError("RESOURCE_IDENTITY_CHANGED", "target filesystem changed")
473
+ filesystems[sample["device"]] = sample
474
+ candidates = {fingerprint(request): {"request": request} for request in requests}
475
+ held = {key: row for key, row in state["requests"].items()
476
+ if row["state"] in {"RESERVED", "ALLOCATED", "ATTENTION_REQUIRED"}}
477
+ held.update({key: row for key, row in candidates.items() if key not in held})
478
+ outstanding_memory = sum(
479
+ max(0, row["request"]["memory_bytes"] - execution_memory(row))
480
+ for row in held.values() if row["request"].get("kind") == "execution"
481
+ )
482
+ if executing and observed["available_memory_bytes"] - outstanding_memory < MIN_AVAILABLE_MEMORY:
483
+ reasons.append("HOST_MEMORY_HEADROOM")
484
+ rate = state.get("creation_rate", {"tokens": float(CREATION_BURST), "at_ns": time.time_ns()})
485
+ tokens = min(CREATION_BURST, rate["tokens"] + max(0, time.time_ns() - rate["at_ns"]) / CREATION_INTERVAL_NS)
486
+ new_creations = sum(request.get("kind") != "execution"
487
+ and state["requests"].get(fingerprint(request), {}).get("state") not in {"RESERVED", "ALLOCATED"}
488
+ for request in requests)
489
+ if min(new_creations, CREATION_BURST) > tokens:
490
+ reasons.append("CREATION_RATE")
491
+ for device, sample in filesystems.items():
492
+ pending = [self._outstanding(row) for row in held.values()
493
+ if row["request"].get("filesystem_device") == device]
494
+ reserved_bytes, reserved_inodes = map(sum, zip(*pending, strict=True)) if pending else (0, 0)
495
+ sample.update(outstanding_bytes=reserved_bytes, outstanding_inodes=reserved_inodes)
496
+ if sample["available_bytes"] - reserved_bytes < MIN_FREE_BYTES:
497
+ reasons.append("FILESYSTEM_BYTES")
498
+ if sample["available_inodes"] - reserved_inodes < MIN_FREE_INODES:
499
+ reasons.append("FILESYSTEM_INODES")
500
+ return {"code": "RESOURCE_WAIT" if reasons else "RESOURCE_AVAILABLE", "blockers": sorted(set(reasons)),
501
+ "host": observed, "filesystems": list(filesystems.values()), "pressure_high": pressured,
502
+ "minimum_free_bytes": MIN_FREE_BYTES, "minimum_free_inodes": MIN_FREE_INODES,
503
+ "minimum_available_memory_bytes": MIN_AVAILABLE_MEMORY,
504
+ "outstanding_memory_bytes": outstanding_memory,
505
+ "creation_rate": {"tokens": tokens, "requested": new_creations,
506
+ "burst": CREATION_BURST, "per_second": 1}}
507
+
508
+ def plan(self, requests: list[dict[str, Any]]) -> dict[str, Any]:
509
+ # Read-only advisory snapshot. Only reserve() grants an atomic budget.
510
+ return self._decision(self.read(), requests)
511
+
512
+ def reserve(self, requests: list[dict[str, Any]], *, blockers: tuple[str, ...] = (),
513
+ integration_sources: tuple[str, ...] = ()) -> dict[str, Any]:
514
+ keys = [fingerprint(request) for request in requests]
515
+ if len(set(keys)) != len(keys) or not keys:
516
+ raise ControllerError("INVALID_REQUEST", "resource requests must be nonempty and unique")
517
+ with self.locked():
518
+ state = self.read()
519
+ if any(request.get("kind") == "execution" and
520
+ state["requests"].get(fingerprint(request), {}).get("state") == "ALLOCATED" for request in requests):
521
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "execution already bound; query the original Run instead of launching")
522
+ decision = self._decision(state, requests)
523
+ if blockers:
524
+ decision.update(code="RESOURCE_WAIT", blockers=sorted(set(decision["blockers"]) | set(blockers)))
525
+ state["pressure_high"] = decision["pressure_high"]
526
+ now = time.time_ns()
527
+ for request in requests:
528
+ key = fingerprint(request)
529
+ for previous_key, previous in state["requests"].items():
530
+ old = previous["request"]
531
+ if (previous_key != key and previous["state"] not in {"RELEASED", "CANCELLED"}
532
+ and old["repository_id"] == request["repository_id"]
533
+ and old.get("kind", "checkout") == request.get("kind", "checkout")
534
+ and (old["arguments"]["task_key"] == request["arguments"]["task_key"]
535
+ or old["arguments"]["authority_key"] == request["arguments"]["authority_key"])):
536
+ # Only the repository producer may supply this proof, after
537
+ # checking its frozen intent under the repository lock.
538
+ source_lineage = (
539
+ old["arguments"]["task_key"] != request["arguments"]["task_key"]
540
+ and request.get("kind", "checkout") == "checkout"
541
+ and request["arguments"].get("role") == "integration"
542
+ and request["arguments"].get("integration_intent_id")
543
+ and tuple(request["arguments"].get("source_allocations", ())) == integration_sources
544
+ and previous["state"] == "ALLOCATED"
545
+ and previous.get("allocation_id") in integration_sources
546
+ and old["arguments"].get("role") in {"feature", "audit", "hotfix", "emergency"}
547
+ )
548
+ if not source_lineage:
549
+ raise ControllerError(
550
+ "ADMISSION_REQUEST_DRIFT",
551
+ f"existing durable request {previous_key} has different identity; "
552
+ "inspect admission-queue and preserve its task, authority and intent. "
553
+ "For a WAITING request after main advances, replay its recorded "
554
+ "arguments through admission-queue --repo <repository> --apply, "
555
+ "then integration-refresh and revalidate before publication",
556
+ )
557
+ existing = state["requests"].get(key)
558
+ if existing and existing["state"] in {"CANCELLED", "RELEASED", "ATTENTION_REQUIRED"}:
559
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "request requires formal reconciliation")
560
+ if existing and existing["state"] in {"RESERVED", "ALLOCATED"}:
561
+ continue
562
+ row = existing or {"request": request, "created_at_ns": now, "history": []}
563
+ if len(requests) > 1:
564
+ row["group"] = sorted(keys)
565
+ next_state = "WAITING" if decision["blockers"] else "RESERVED"
566
+ if row.get("state") != next_state:
567
+ row["history"].append({"state": next_state, "at_ns": now})
568
+ row.update(state=next_state, updated_at_ns=now,
569
+ decision=decision)
570
+ state["requests"][key] = row
571
+ rate = decision["creation_rate"]
572
+ state["creation_rate"] = {"tokens": rate["tokens"] - (rate["requested"] if not decision["blockers"] else 0),
573
+ "at_ns": now}
574
+ self._save(state)
575
+ return {**decision, "request_ids": [fingerprint(request) for request in requests]}
576
+
577
+ def bind(self, request: dict[str, Any], *, transaction_id: str, allocation_id: str) -> None:
578
+ with self.locked():
579
+ state = self.read()
580
+ row = state["requests"].get(fingerprint(request))
581
+ if row is None or row["state"] != "RESERVED" or row.get("transaction_id") not in {None, transaction_id}:
582
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "reservation already belongs to another attempt")
583
+ row.update(transaction_id=transaction_id, allocation_id=allocation_id)
584
+ self._save(state)
585
+
586
+ def allocated(self, request: dict[str, Any], *, allocation_id: str) -> None:
587
+ warning = None
588
+ try:
589
+ allocated, inodes = materialized(Path(request["target_path"]), request["filesystem_device"])
590
+ except (OSError, ControllerError):
591
+ # Credit is optional; uncertainty retains the entire reservation.
592
+ # It cannot turn a completed Git allocation into a failed operation.
593
+ allocated, inodes = 0, 0
594
+ warning = "MATERIALIZED_CREDIT_UNAVAILABLE"
595
+ with self.locked():
596
+ state = self.read()
597
+ row = state["requests"][fingerprint(request)]
598
+ if row.get("allocation_id") != allocation_id or row["state"] not in {"RESERVED", "ALLOCATED"}:
599
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "allocation differs from resource reservation")
600
+ previous_state = row["state"]
601
+ row.update(state="ALLOCATED", materialized_bytes=allocated, materialized_inodes=inodes,
602
+ updated_at_ns=time.time_ns(), observation_warning=warning)
603
+ if previous_state != "ALLOCATED":
604
+ row["history"].append({"state": "ALLOCATED", "at_ns": row["updated_at_ns"]})
605
+ self._save(state)
606
+
607
+ def reset_no_effect_acquire(self, store: MetadataStore, key: str, *, apply: bool) -> bool:
608
+ """Make a proved no-effect attempt retryable, archiving its journal in history.
609
+
610
+ The caller holds the repository flock. Neither an error code nor a missing
611
+ allocation alone authorizes dropping the attempted reservation identity.
612
+ """
613
+ from .transaction_scope import acquire_evidence_terminal
614
+
615
+ if not store.flock_held:
616
+ raise ControllerError("MISSING_REPOSITORY_LOCK", "acquire recovery requires its flock")
617
+ row = self.read()["requests"].get(key)
618
+ if row is None or row["state"] != "RESERVED" or not row.get("transaction_id"):
619
+ return False
620
+ transaction = row["transaction_id"]
621
+ try:
622
+ if str(UUID(transaction)) != transaction:
623
+ raise ValueError("noncanonical transaction")
624
+ history = row["history"]
625
+ if not isinstance(history, list) or any(
626
+ not isinstance(event, dict) for event in history
627
+ ):
628
+ raise ValueError("invalid request history")
629
+ except (ValueError, TypeError, AttributeError, KeyError) as exc:
630
+ raise ControllerError(
631
+ "ADMISSION_RECOVERY_REQUIRED", "acquire attempt identity is invalid"
632
+ ) from exc
633
+ proof = next(
634
+ (
635
+ event
636
+ for event in reversed(history)
637
+ if event.get("terminal_evidence") == "ACQUIRE_NO_EFFECT_SUPERSEDED"
638
+ and event.get("transaction_id") == transaction
639
+ and event.get("allocation_id") == row.get("allocation_id")
640
+ ),
641
+ None,
642
+ )
643
+ journal_path = store.transactions_dir / f"{transaction}.json"
644
+ if proof is None:
645
+ if acquire_evidence_terminal(store, transaction) != "ACQUIRE_NO_EFFECT_SUPERSEDED":
646
+ return False
647
+ journal = store._read_json(journal_path)
648
+ else:
649
+ journal = proof.get("journal")
650
+ if not isinstance(journal, dict):
651
+ raise ControllerError(
652
+ "ADMISSION_RECOVERY_REQUIRED", "no-effect proof is incomplete"
653
+ )
654
+ if journal_path.exists() and store._read_json(journal_path) != journal:
655
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "no-effect journal changed")
656
+ request = row["request"]
657
+ if (
658
+ request.get("kind", "checkout") != "checkout"
659
+ or request["common_dir"] != str(store.root.parents[1])
660
+ or request["repository_id"] != store._read_json(store.repo_file).get("repository_id")
661
+ or journal.get("transaction_id") != transaction
662
+ or journal.get("command") != "acquire"
663
+ or journal.get("phase") not in {"PREPARED", "WORKTREE_ADDED"}
664
+ or tuple(
665
+ journal.get(field)
666
+ for field in (
667
+ "repository_id",
668
+ "allocation_id",
669
+ "target_path",
670
+ "branch",
671
+ "expected_head",
672
+ )
673
+ )
674
+ != (
675
+ request["repository_id"],
676
+ row.get("allocation_id"),
677
+ request["target_path"],
678
+ request["arguments"].get("branch"),
679
+ request["base_oid"],
680
+ )
681
+ ):
682
+ raise ControllerError(
683
+ "ADMISSION_RECOVERY_REQUIRED", "no-effect journal identity differs"
684
+ )
685
+ if not apply:
686
+ return True
687
+ with self.locked():
688
+ state = self.read()
689
+ if state["requests"].get(key) != row:
690
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "acquire reservation changed")
691
+ if proof is None:
692
+ # Durable proof precedes journal removal. Never hold the host
693
+ # ledger lock while invoking a repository mutation or Git.
694
+ row["history"].append(
695
+ {
696
+ "state": "RESERVED",
697
+ "at_ns": time.time_ns(),
698
+ "terminal_evidence": "ACQUIRE_NO_EFFECT_SUPERSEDED",
699
+ "transaction_id": transaction,
700
+ "allocation_id": row["allocation_id"],
701
+ "journal": journal,
702
+ }
703
+ )
704
+ state["requests"][key] = row
705
+ self._save(state)
706
+ if journal_path.exists():
707
+ store.remove_transaction(transaction)
708
+ with self.locked():
709
+ state = self.read()
710
+ if state["requests"].get(key) != row:
711
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "acquire reservation changed")
712
+ now = time.time_ns()
713
+ row["history"].append(
714
+ {
715
+ "state": "WAITING",
716
+ "at_ns": now,
717
+ "transaction_id": transaction,
718
+ "allocation_id": row["allocation_id"],
719
+ }
720
+ )
721
+ row.update(state="WAITING", updated_at_ns=now)
722
+ row.pop("transaction_id")
723
+ row.pop("allocation_id")
724
+ state["requests"][key] = row
725
+ self._save(state)
726
+ return True
727
+
728
+ def vanished(self, key: str, evidence: dict[str, Any]) -> None:
729
+ """Settle a checkout budget whose repository and checkout no longer exist anywhere."""
730
+ with self.locked():
731
+ state = self.read()
732
+ row = state["requests"][key]
733
+ if row["state"] == "RELEASED":
734
+ return
735
+ if row["request"].get("kind") == "execution" or row["state"] not in {"WAITING", "RESERVED", "ALLOCATED"}:
736
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "only a checkout budget settles as vanished")
737
+ row.update(state="RELEASED", updated_at_ns=time.time_ns(), vanished_evidence=evidence)
738
+ row["history"].append({"state": "RELEASED", "at_ns": row["updated_at_ns"],
739
+ "terminal_evidence": evidence["contract"]})
740
+ self._save(state)
741
+
742
+ def released(self, key: str, allocation_id: str) -> None:
743
+ """Called only after formal release evidence and Git absence verification."""
744
+ with self.locked():
745
+ state = self.read()
746
+ row = state["requests"][key]
747
+ if row.get("allocation_id") != allocation_id:
748
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "released allocation identity differs")
749
+ if row["state"] == "RELEASED":
750
+ return
751
+ row.update(state="RELEASED", updated_at_ns=time.time_ns())
752
+ row["history"].append({"state": "RELEASED", "at_ns": row["updated_at_ns"]})
753
+ self._save(state)
754
+
755
+ def _cancel_abandoned_claim(self, key: str, expected: dict[str, Any], evidence: dict[str, Any]) -> None:
756
+ """The caller proves abandonment and no attempt under the repository flock."""
757
+ with self.locked():
758
+ state = self.read()
759
+ row = state["requests"].get(key)
760
+ if row != expected:
761
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "claim reservation changed")
762
+ row.update(state="CANCELLED", updated_at_ns=time.time_ns(), abandoned_intent_evidence=evidence)
763
+ row["history"].append({"state": "CANCELLED", "at_ns": row["updated_at_ns"],
764
+ "terminal_evidence": "PREPARATION_ABANDONED"})
765
+ self._save(state)
766
+
767
+
768
+ def release_allocation_budget(repository_id: str, allocation_id: str, receipt_id: str) -> None:
769
+ """Settle reservations at the verified formal allocation release boundary.
770
+
771
+ The caller holds the repository lock and has verified Git/path absence and
772
+ persisted the release receipt. Periodic reconciliation covers a crash before
773
+ this settlement. Release cannot pass an active runtime binding; historical
774
+ writer-only budget entries can outlive their already released allocation.
775
+ An execution admitted for this allocation but never bound to a holder can no
776
+ longer start in it, so its reservation settles here too.
777
+ """
778
+ ledger = AdmissionLedger()
779
+ if not ledger.path.exists():
780
+ return
781
+ with ledger.locked():
782
+ state = ledger.read()
783
+ changed = False
784
+ for row in state["requests"].values():
785
+ request = row["request"]
786
+ unbound = _unbound_execution(row) and request.get("allocation_id") == allocation_id
787
+ if (request["repository_id"] != repository_id
788
+ or (row.get("allocation_id") != allocation_id and not unbound)
789
+ or row["state"] in {"RELEASED", "CANCELLED"}
790
+ or (request.get("kind") == "execution" and row["state"] != "ALLOCATED" and not unbound)):
791
+ continue
792
+ now = time.time_ns()
793
+ row.update(state="RELEASED", updated_at_ns=now, release_receipt_id=receipt_id)
794
+ row["history"].append({"state": "RELEASED", "at_ns": now,
795
+ "release_receipt_id": receipt_id})
796
+ changed = True
797
+ if changed:
798
+ ledger._save(state)
799
+
800
+
801
+ def _unbound_execution(row: dict[str, Any]) -> bool:
802
+ """Admitted or waiting, but no holder was ever bound: nothing was started under it."""
803
+ return (row["request"].get("kind") == "execution" and row["state"] in {"WAITING", "RESERVED"}
804
+ and row.get("holder") is None and row["request"].get("allocation_id") is not None)
805
+
806
+
807
+ def _absent(path: Path) -> bool | None:
808
+ """True only when the filesystem reports ENOENT; None when observation itself failed.
809
+
810
+ ``os.path.lexists`` reports permission (EACCES), I/O (EIO) and similar errors
811
+ as absence. Those say nothing about whether the path exists, so they never
812
+ count as absence evidence.
813
+ """
814
+ try:
815
+ os.lstat(path)
816
+ except FileNotFoundError:
817
+ return True
818
+ except OSError:
819
+ return None
820
+ return False
821
+
822
+
823
+ def _absence_anchor(path: Path, device: int) -> Path | None:
824
+ """The nearest existing ancestor, only if it is a real directory on ``device``."""
825
+ anchor = path.parent
826
+ while True:
827
+ try:
828
+ info = os.lstat(anchor)
829
+ except FileNotFoundError:
830
+ if anchor.parent == anchor:
831
+ return None
832
+ anchor = anchor.parent
833
+ continue
834
+ except OSError:
835
+ return None
836
+ return anchor if stat.S_ISDIR(info.st_mode) and info.st_dev == device else None
837
+
838
+
839
+ def _vanished_repository(request: dict[str, Any]) -> dict[str, Any] | None:
840
+ """Positive evidence that nothing this checkout budget reserved can exist.
841
+
842
+ The repository, its common directory and the checkout path all report ENOENT,
843
+ and each one's nearest existing ancestor is a real directory on the very
844
+ filesystem the budget was reserved on, so the absence is observed on a mounted
845
+ filesystem rather than inferred from a missing mount or an unreadable path.
846
+ Anything present, and any failed observation, keeps the reservation.
847
+ """
848
+ paths = [Path(request[name]) for name in ("common_dir", "repository_path", "target_path")]
849
+ if any(_absent(path) is not True for path in paths):
850
+ return None
851
+ anchors = [_absence_anchor(path, request["filesystem_device"]) for path in paths]
852
+ if any(anchor is None for anchor in anchors):
853
+ return None
854
+ return {"contract": "vanished-repository/v1", "observed_at_ns": time.time_ns(),
855
+ "absent_paths": [str(path) for path in paths], "anchor": str(anchors[2]),
856
+ "anchors": [str(anchor) for anchor in anchors],
857
+ "filesystem_device": request["filesystem_device"]}
858
+
859
+
860
+ def _observe_checkout(
861
+ ledger: AdmissionLedger, key: str, row: dict[str, Any], result: dict[str, Any],
862
+ repository: RepositoryInventory, store: MetadataStore, snapshot: MetadataSnapshot,
863
+ *, apply: bool,
864
+ ) -> bool:
865
+ """Settle host accounting under the repository lock; never acquire a checkout.
866
+
867
+ Only RESERVED acquire recovery can change repository evidence and needs an
868
+ exclusive lock. ALLOCATED observations change at most the separate host ledger.
869
+ """
870
+ from . import transaction_scope
871
+ from .policy import load_policy
872
+
873
+ request = row["request"]
874
+ if str(repository.common_dir) != request["common_dir"] or snapshot.repository_id != request["repository_id"]:
875
+ raise ControllerError("RESOURCE_IDENTITY_CHANGED", "repository identity changed")
876
+ if not load_policy(repository.canonical_path).resource_admission:
877
+ raise ControllerError("RESOURCE_ADMISSION_UNSUPPORTED", "resource policy is no longer enabled")
878
+ target = Path(request["target_path"])
879
+ pending = transaction_scope.conflicts(store, allocation=row.get("allocation_id"), target=target)
880
+ record = snapshot.records.get(row.get("allocation_id"))
881
+ item = next((item for item in repository.worktrees if item.path == target), None)
882
+ if (row["state"] == "RESERVED" and row.get("transaction_id")
883
+ and ledger.reset_no_effect_acquire(store, key, apply=apply)):
884
+ result.update(state="WAITING" if apply else row["state"],
885
+ terminal_evidence="ACQUIRE_NO_EFFECT_SUPERSEDED")
886
+ return True
887
+ arguments = request["arguments"]
888
+ intent = snapshot.integration_intents.get(arguments.get("integration_intent_id"))
889
+ if (row["state"] in {"WAITING", "RESERVED"} and not row.get("transaction_id")
890
+ and not row.get("allocation_id") and intent is not None
891
+ and integration_intent_abandoned(intent)):
892
+ if (arguments.get("role") != "integration"
893
+ or any(arguments.get(name) != getattr(intent, name)
894
+ for name in ("task_key", "authority_key", "module", "owner"))
895
+ or tuple(arguments.get("source_allocations", ())) != intent.source_allocations
896
+ or arguments.get("source_handoffs") or len(row.get("group", [key])) != 1):
897
+ raise ControllerError("INTEGRATION_INTENT_MISMATCH", "abandoned claim identity changed")
898
+ if pending:
899
+ raise ControllerError("INTERRUPTED_TRANSACTION", "abandoned claim has unresolved lifecycle effects")
900
+ if (item is not None or _absent(target) is not True
901
+ or f"refs/heads/{arguments['branch']}" in public_refs(repository.canonical_path)
902
+ or any(record.role == "integration" and record.task_key == intent.task_key
903
+ and record.lifecycle != "RELEASED" for record in snapshot.records.values())):
904
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "abandoned claim cannot be proven unattempted")
905
+ if apply:
906
+ if not store.flock_held:
907
+ raise ControllerError("MISSING_REPOSITORY_LOCK", "claim settlement requires its flock")
908
+ ledger._cancel_abandoned_claim(key, row, {"intent_id": intent.intent_id,
909
+ "repository_id": snapshot.repository_id, "terminal_event": dict(intent.history[-1]),
910
+ "failure_reason": intent.blocked_reason})
911
+ result.update(state="CANCELLED", terminal_evidence="PREPARATION_ABANDONED",
912
+ integration_intent_id=intent.intent_id, task_completed=False)
913
+ return True
914
+ if record is not None and record.lifecycle == "RELEASED":
915
+ if item or target.exists() or target.is_symlink() or not any(receipt.allocation_id == record.allocation_id for receipt in snapshot.release_receipts.values()):
916
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "release evidence and Git disagree")
917
+ if apply:
918
+ ledger.released(key, record.allocation_id)
919
+ result["state"] = "RELEASED"
920
+ elif record is None and row.get("allocation_id") and any(receipt.allocation_id == row.get("allocation_id") for receipt in snapshot.release_receipts.values()):
921
+ if item or target.exists() or target.is_symlink():
922
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "release evidence and Git disagree")
923
+ if apply:
924
+ ledger.released(key, row["allocation_id"])
925
+ result["state"] = "RELEASED"
926
+ elif pending:
927
+ raise ControllerError("INTERRUPTED_TRANSACTION", "nonterminal repository transaction requires recovery")
928
+ elif row.get("transaction_id"):
929
+ expected_base = request["arguments"].get("lineage_base_commit") or request["base_oid"]
930
+ if (record is None or item is None or snapshot.bindings.get(item.key) != record.allocation_id
931
+ or record.task_key != request["arguments"]["task_key"]
932
+ or record.authority_key != request["arguments"]["authority_key"]
933
+ or (record.base_commit != expected_base and record.base_commit != request["base_oid"])
934
+ or record.owner != request["arguments"]["owner"]):
935
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "an attempted allocation cannot be proven complete")
936
+ if apply and (row["state"] == "RESERVED" or time.time_ns() - row["updated_at_ns"] >= 300 * 10**9):
937
+ ledger.allocated(request, allocation_id=record.allocation_id)
938
+ result["state"] = "ALLOCATED"
939
+ return False
940
+
941
+
942
+ def _allocated_observations(
943
+ ledger: AdmissionLedger, rows: dict[str, Any], selected: str | None, *, apply: bool,
944
+ ) -> dict[str, dict[str, Any]]:
945
+ """Share one fresh Git/metadata observation only while its shared flock is held.
946
+
947
+ Complete checkout requests cannot replay an acquire or change repository
948
+ evidence. Observe these before any queued mutation; discard every snapshot
949
+ when the lock ends. Recovery and queued acquisitions keep their ordinary path.
950
+ """
951
+ grouped: dict[str, list[tuple[str, dict[str, Any]]]] = {}
952
+ for key, row in rows.items():
953
+ request = row["request"]
954
+ if (row["state"] == "ALLOCATED" and row.get("transaction_id")
955
+ and request.get("kind", "checkout") == "checkout"
956
+ and (selected is None or request["common_dir"] == selected)):
957
+ if selected is None and _vanished_repository(request) is not None:
958
+ continue # Settle absence through the ordinary per-request path.
959
+ grouped.setdefault(request["common_dir"], []).append((key, row))
960
+ observed: dict[str, dict[str, Any]] = {}
961
+ for group in grouped.values():
962
+ results = {key: {"request_id": key, "state": row["state"],
963
+ "task_key": row["request"]["arguments"]["task_key"],
964
+ "repository_id": row["request"]["repository_id"],
965
+ "allocation_id": row.get("allocation_id")} for key, row in group}
966
+ try:
967
+ path = Path(group[0][1]["request"]["repository_path"])
968
+ _, common = resolve_repository(path)
969
+ store = MetadataStore(common)
970
+ with store.lock(exclusive=False):
971
+ if apply and not store.flock_held:
972
+ raise ControllerError("MISSING_REPOSITORY_LOCK", "accounting requires the repository lock")
973
+ repository = inventory(path)
974
+ snapshot = store._load_unlocked()
975
+ for key, row in group:
976
+ try:
977
+ _observe_checkout(ledger, key, row, results[key], repository, store, snapshot, apply=apply)
978
+ except (ControllerError, OSError) as exc:
979
+ results[key]["blocker"] = getattr(exc, "code", "RESOURCE_OBSERVATION_UNAVAILABLE")
980
+ except (ControllerError, OSError) as exc:
981
+ for result in results.values():
982
+ result["blocker"] = getattr(exc, "code", "RESOURCE_OBSERVATION_UNAVAILABLE")
983
+ observed.update(results)
984
+ return observed
985
+
986
+
987
+ def reconcile(*, repository_path: Path | None = None, apply: bool = False) -> dict[str, Any]:
988
+ """Resume persisted requests, using the ordinary acquisition/recovery gates.
989
+
990
+ No-action WAITING requests can run again. A RESERVED request with an attempt
991
+ identity is never replayed: only an already recorded allocation can settle it.
992
+ A partial/missing allocation remains a formal repository recovery blocker.
993
+ """
994
+ from . import lifecycle
995
+ from .policy import load_policy
996
+
997
+ ledger = AdmissionLedger()
998
+ rows = ledger.read()["requests"]
999
+ selected = str(resolve_repository(repository_path)[1]) if repository_path else None
1000
+ outcomes = []
1001
+ maintenance = []
1002
+ if apply:
1003
+ # The existing host ledger identifies enrolled repositories even when an
1004
+ # external task has never been added to AW's workspace navigation list.
1005
+ enrolled = {}
1006
+ absent_checkouts = {(request["repository_id"], request["common_dir"], request["target_path"])
1007
+ for row in rows.values() if selected is None
1008
+ and (request := row["request"]).get("kind", "checkout") == "checkout"
1009
+ and request["common_dir"] and _vanished_repository(request) is not None}
1010
+ for row in rows.values():
1011
+ request = row["request"]
1012
+ if request["common_dir"] and (selected is None or request["common_dir"] == selected):
1013
+ # Historical rows keep live repositories enrolled. Only mounted,
1014
+ # positively absent checkout evidence excludes a vanished one,
1015
+ # including its exact terminal execution (which has no FS budget).
1016
+ if ((request["repository_id"], request["common_dir"], request["target_path"]) in absent_checkouts
1017
+ and (request.get("kind", "checkout") == "checkout"
1018
+ or row["state"] in {"RELEASED", "CANCELLED"})):
1019
+ continue
1020
+ enrolled.setdefault(request["common_dir"], request)
1021
+ for common, request in enrolled.items():
1022
+ observation = {"repository_id": request["repository_id"], "repository_path": request["repository_path"]}
1023
+ try:
1024
+ repository = inventory(Path(request["repository_path"]))
1025
+ snapshot = MetadataStore(repository.common_dir).load()
1026
+ if str(repository.common_dir) != common or snapshot.repository_id != request["repository_id"]:
1027
+ raise ControllerError("RESOURCE_IDENTITY_CHANGED", "enrolled repository identity changed")
1028
+ policy = load_policy(repository.canonical_path)
1029
+ eligible = any(intent.status in {"PENDING", "BLOCKED_RETRYABLE"}
1030
+ or (intent.status == "STALE" and intent.allocation_id in snapshot.records
1031
+ and snapshot.records[intent.allocation_id].lifecycle == "HANDOFF_READY")
1032
+ for intent in snapshot.release_intents.values())
1033
+ if policy.automatic_feature_audit_release and eligible:
1034
+ observation["release"] = lifecycle.reconcile_release_intents(repository.canonical_path,
1035
+ actor=policy.reconciler_actor, trigger="capacity-pressure")["decision"]
1036
+ maintenance.append(observation)
1037
+ except (ControllerError, OSError) as exc:
1038
+ maintenance.append({**observation, "blocker": getattr(exc, "code", "RESOURCE_OBSERVATION_UNAVAILABLE")})
1039
+ rows = ledger.read()["requests"]
1040
+ settled_host_requests: set[str] = set()
1041
+ if apply and selected is None:
1042
+ host_requests = [key for key, row in rows.items()
1043
+ if row["state"] == "ALLOCATED"
1044
+ and row["request"].get("kind") == "execution"
1045
+ and row["request"].get("allocation_id") is None]
1046
+ # Preserve ordinary per-request diagnostics and independent repository recovery.
1047
+ with suppress(ControllerError, OSError):
1048
+ settled_host_requests = _reconcile_finished_host_executions(ledger, host_requests)
1049
+ allocated = _allocated_observations(ledger, rows, selected, apply=apply)
1050
+ handled: set[str] = set()
1051
+ for key, row in sorted(rows.items(), key=lambda item: item[1]["created_at_ns"]):
1052
+ request = row["request"]
1053
+ if key in handled or (selected and request["common_dir"] != selected):
1054
+ continue
1055
+ handled.add(key)
1056
+ result = {"request_id": key, "state": row["state"], "task_key": request["arguments"]["task_key"],
1057
+ "repository_id": request["repository_id"], "allocation_id": row.get("allocation_id")}
1058
+ if row["state"] in {"RELEASED", "CANCELLED"}:
1059
+ continue
1060
+ if key in allocated:
1061
+ outcomes.append(allocated[key])
1062
+ continue
1063
+ if request.get("kind") == "execution":
1064
+ # The owning AW Run resumes a launch that is durably known not to
1065
+ # have started. Controller never executes or replays a command.
1066
+ try:
1067
+ if key in settled_host_requests:
1068
+ settled = {"code": "HOST_EXECUTION_CLEARED", "task_completed": False,
1069
+ "request_ids": [key]}
1070
+ else:
1071
+ settled = (_reconcile_execution(row, apply=apply)
1072
+ if row["state"] == "ALLOCATED" or _unbound_execution(row) else None)
1073
+ if settled is None or settled["code"] == "EXECUTION_NOT_STARTED":
1074
+ settled = ledger.plan([request])
1075
+ result["result"] = settled
1076
+ except (ControllerError, OSError) as exc:
1077
+ result["blocker"] = getattr(exc, "code", "RESOURCE_OBSERVATION_UNAVAILABLE")
1078
+ outcomes.append(result)
1079
+ continue
1080
+ try:
1081
+ if (selected is None and row["state"] in {"WAITING", "RESERVED", "ALLOCATED"}
1082
+ and (evidence := _vanished_repository(request)) is not None):
1083
+ # A repository that no longer exists cannot be selected by path,
1084
+ # so only the all-repositories pass observes it.
1085
+ if apply:
1086
+ ledger.vanished(key, evidence)
1087
+ result.update(state="RELEASED", terminal_evidence=evidence["contract"])
1088
+ outcomes.append(result)
1089
+ continue
1090
+ repository = inventory(Path(request["repository_path"]))
1091
+ store = MetadataStore(repository.common_dir)
1092
+ with store.lock(exclusive=apply and row["state"] == "RESERVED" and bool(row.get("transaction_id"))):
1093
+ if apply and not store.flock_held:
1094
+ raise ControllerError("MISSING_REPOSITORY_LOCK", "accounting requires the repository lock")
1095
+ snapshot = store._load_unlocked()
1096
+ target = Path(request["target_path"])
1097
+ if _observe_checkout(ledger, key, row, result, repository, store, snapshot, apply=apply):
1098
+ outcomes.append(result)
1099
+ continue
1100
+ if apply and result["state"] in {"WAITING", "RESERVED"} and not row.get("transaction_id"):
1101
+ group = row.get("group", [key])
1102
+ if any(member not in rows for member in group):
1103
+ raise ControllerError("INVALID_RESOURCE_LEDGER", "request group is incomplete")
1104
+ if len(group) > 1:
1105
+ requests = []
1106
+ for member in group:
1107
+ member_row = rows[member]
1108
+ if member_row.get("transaction_id") or member_row["state"] not in {"WAITING", "RESERVED"}:
1109
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "partial request group requires reconciliation")
1110
+ member_request = member_row["request"]
1111
+ requests.append({"repository": member_request["repository_path"], **member_request["arguments"]})
1112
+ handled.update(group)
1113
+ result["result"] = lifecycle.acquire_across(requests)["decision"]
1114
+ else:
1115
+ arguments = {**request["arguments"], "target": target}
1116
+ for name in ("scope_paths", "source_allocations", "source_handoffs", "external_bindings"):
1117
+ arguments[name] = tuple(arguments[name])
1118
+ result["result"] = lifecycle.acquire(repository.canonical_path, **arguments)["decision"]
1119
+ except (ControllerError, OSError) as exc:
1120
+ result["blocker"] = getattr(exc, "code", "RESOURCE_OBSERVATION_UNAVAILABLE")
1121
+ outcomes.append(result)
1122
+ return {"schema_version": 1, "command": "admission-queue", "repository": None,
1123
+ "decision": {"code": "ADMISSION_RECONCILED" if apply else "ADMISSION_QUEUE",
1124
+ "dry_run": not apply, "outcomes": outcomes, "maintenance": maintenance}, "errors": [], "warnings": [], "worktrees": []}
1125
+
1126
+
1127
+ def _host_runtime_holders(row: dict[str, Any], *, allow_broad_scope: bool = False) -> tuple[WriterHolder, ...]:
1128
+ """Return the exact runtime chain for one host-scoped reservation.
1129
+
1130
+ ``allow_broad_scope`` is set only by the typed dead-holder recovery, which has
1131
+ already tied the request to the row's own recorded holder and terminal
1132
+ evidence. Unqualified recovery still refuses a home-root or broader target,
1133
+ because there the target path cannot bound a reservation to one session.
1134
+ """
1135
+ from .writer_lease import runtime_binding
1136
+
1137
+ if row.get("lease_id") is not None:
1138
+ raise ControllerError("WRITER_LEASE_HELD", "host execution has a writer lease")
1139
+ request = row.get("request")
1140
+ bindings = row.get("runtime_bindings")
1141
+ recorded = row.get("holder")
1142
+ if (not isinstance(request, dict) or not isinstance(recorded, dict)
1143
+ or not isinstance(bindings, list) or not bindings):
1144
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1145
+ "host execution runtime identity is uncertain")
1146
+ try:
1147
+ raw_target = Path(request["target_path"])
1148
+ except (KeyError, OSError, TypeError, ValueError) as exc:
1149
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1150
+ "host execution recovery target is uncertain") from exc
1151
+ try:
1152
+ target = raw_target.resolve(strict=True)
1153
+ target_present = True
1154
+ except FileNotFoundError:
1155
+ target, target_present = raw_target, False
1156
+ except (OSError, TypeError, ValueError) as exc:
1157
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1158
+ "host execution recovery target is uncertain") from exc
1159
+ if not allow_broad_scope:
1160
+ # Unqualified recovery is identity-only: it needs a present, bounded
1161
+ # target to tie the recorded runtime chain to one session. Typed
1162
+ # recovery carries its own recorded-holder authority and may instead
1163
+ # proceed against a provably absent target (orphan release).
1164
+ if not target_present:
1165
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1166
+ "host execution recovery target is uncertain")
1167
+ if unbounded_host_scope(target):
1168
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1169
+ "host execution target is too broad for identity-only recovery")
1170
+ try:
1171
+ primary = WriterHolder(**recorded)
1172
+ except (TypeError, ValueError) as exc:
1173
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1174
+ "host execution runtime identity is uncertain") from exc
1175
+ holders = []
1176
+ for value in bindings:
1177
+ try:
1178
+ binding = runtime_binding(value) if isinstance(value, str) else None
1179
+ except (ControllerError, TypeError, ValueError) as exc:
1180
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1181
+ "host execution runtime identity is uncertain") from exc
1182
+ if binding is None:
1183
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1184
+ "host execution runtime identity is uncertain")
1185
+ holder, lease_id = binding
1186
+ if lease_id is not None:
1187
+ raise ControllerError("WRITER_LEASE_HELD", "host execution has a writer lease")
1188
+ holders.append(holder)
1189
+ expected = (request.get("client_id"), request.get("session_id"), request.get("run_id"))
1190
+ if holders[0] != primary or any(
1191
+ (holder.client_id, holder.session_id, holder.run_id) != expected for holder in holders):
1192
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1193
+ "host execution runtime identity is uncertain")
1194
+ return tuple(holders)
1195
+
1196
+
1197
+ def _host_runtime_is_live(holders: tuple[WriterHolder, ...]) -> bool:
1198
+ from .writer_lease import holder_is_live, process_group_is_live
1199
+
1200
+ return any(holder_is_live(holder) or process_group_is_live(holder) for holder in holders)
1201
+
1202
+
1203
+ def _check_host_target(request: dict[str, Any], *, allow_missing: bool = False) -> None:
1204
+ try:
1205
+ info = os.lstat(Path(request["target_path"]))
1206
+ except FileNotFoundError as missing:
1207
+ if not allow_missing:
1208
+ raise ControllerError(
1209
+ "RESOURCE_IDENTITY_CHANGED",
1210
+ "host execution target is missing; only typed dead-holder recovery may proceed",
1211
+ ) from missing
1212
+ return
1213
+ if [info.st_dev, info.st_ino] != request["path_identity"]:
1214
+ raise ControllerError("RESOURCE_IDENTITY_CHANGED", "host execution directory changed")
1215
+
1216
+
1217
+ def _check_host_exit(row: dict[str, Any]) -> None:
1218
+ if _host_runtime_is_live(_host_runtime_holders(row)):
1219
+ raise ControllerError("RUNTIME_OCCUPIED", "runtime holder is still alive")
1220
+
1221
+
1222
+ def _reconcile_finished_host_executions(ledger: AdmissionLedger, request_ids: list[str]) -> set[str]:
1223
+ """Settle proven host exits with one atomic write, rechecking every row under lock.
1224
+
1225
+ Only host reservations belong here: repository recovery must never run while
1226
+ holding the host ledger lock. Uncertain rows retain their complete budget.
1227
+ """
1228
+ settled: set[str] = set()
1229
+ if not request_ids:
1230
+ return settled
1231
+ with ledger.locked():
1232
+ state = ledger.read()
1233
+ for request_id in request_ids:
1234
+ row = state["requests"].get(request_id)
1235
+ if (row is None or row["state"] != "ALLOCATED"
1236
+ or row["request"].get("kind") != "execution"
1237
+ or row["request"].get("allocation_id") is not None
1238
+ or row["request"].get("owner") != os.getuid()):
1239
+ continue
1240
+ try:
1241
+ _check_host_target(row["request"])
1242
+ _check_host_exit(row)
1243
+ except (ControllerError, OSError):
1244
+ continue
1245
+ row["state"] = "RELEASED"
1246
+ row["history"].append({"state": "RELEASED", "at_ns": time.time_ns()})
1247
+ row["updated_at_ns"] = time.time_ns()
1248
+ settled.add(request_id)
1249
+ if settled:
1250
+ ledger._save(state)
1251
+ return settled
1252
+
1253
+
1254
+ HOST_RECOVERY_CONTRACT = "host-execution-recovery/v1"
1255
+ HOST_TERMINAL_EVIDENCE_KIND = "aw-terminal-record"
1256
+ HOST_TERMINAL_STATUSES = frozenset(
1257
+ {"EXITED", "CLOSED", "ERROR", "COMPLETED", "FAILED", "CANCELLED", "INTERRUPTED", "SUCCEEDED"}
1258
+ )
1259
+
1260
+
1261
+ def _dead_holder_proof(holder: WriterHolder) -> dict[str, Any]:
1262
+ """Prove the exact recorded holder cannot still be running, or refuse.
1263
+
1264
+ PID reuse and a stale boot are evidence that the original process is gone, not
1265
+ that its task succeeded; they are recorded rather than guessed at.
1266
+ """
1267
+ from .writer_lease import boot_id, holder_is_live, process_group_is_live, start_ticks
1268
+
1269
+ if not holder.boot_id or holder.boot_id == "unknown-boot":
1270
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1271
+ "host execution recorded boot identity is unknown")
1272
+ observed = start_ticks(holder.pid)
1273
+ proof = {
1274
+ "recorded_pid": holder.pid,
1275
+ "recorded_boot_id": holder.boot_id,
1276
+ "recorded_start_ticks": holder.start_ticks,
1277
+ "observed_boot_id": boot_id(),
1278
+ "observed_start_ticks": observed,
1279
+ "pid_present": observed is not None,
1280
+ "pid_reused": observed is not None and observed != holder.start_ticks,
1281
+ "boot_changed": holder.boot_id != boot_id(),
1282
+ }
1283
+ if holder_is_live(holder) or process_group_is_live(holder):
1284
+ raise ControllerError("RUNTIME_OCCUPIED", "typed recovery found the recorded holder alive")
1285
+ if observed is not None and observed == holder.start_ticks:
1286
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1287
+ "host execution holder liveness is uncertain")
1288
+ return proof
1289
+
1290
+
1291
+ def _terminal_evidence(evidence: Any, request: dict[str, Any]) -> dict[str, Any]:
1292
+ """Bind the caller's terminal-state evidence to this row's own identity."""
1293
+ if not isinstance(evidence, dict) or evidence.get("kind") != HOST_TERMINAL_EVIDENCE_KIND:
1294
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1295
+ "typed recovery requires terminal execution evidence")
1296
+ status = evidence.get("status")
1297
+ ended_at = evidence.get("ended_at")
1298
+ if (not isinstance(status, str) or status.upper() not in HOST_TERMINAL_STATUSES
1299
+ or not isinstance(ended_at, str) or not ended_at.strip()):
1300
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1301
+ "terminal execution evidence is not a final state")
1302
+ if evidence.get("session_id") != request.get("session_id"):
1303
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1304
+ "terminal evidence session identity differs from the reservation")
1305
+ if request.get("run_id") is not None and evidence.get("run_id") != request.get("run_id"):
1306
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1307
+ "terminal evidence run identity differs from the reservation")
1308
+ return {"kind": HOST_TERMINAL_EVIDENCE_KIND, "status": status.upper(),
1309
+ "ended_at": ended_at, "session_id": evidence.get("session_id"),
1310
+ "run_id": evidence.get("run_id"), "end_reason": evidence.get("end_reason")}
1311
+
1312
+
1313
+ def _host_recovery_reference(ledger: AdmissionLedger, request_id: str, value: dict[str, Any]) -> dict[str, str]:
1314
+ """Write recovery evidence once; a repeat must present identical bytes."""
1315
+ from . import cleanup_evidence as e
1316
+
1317
+ directory = ledger.root / "host-recoveries"
1318
+ path = directory / f"{request_id}.json"
1319
+ def stable(document: dict[str, Any]) -> dict[str, Any]:
1320
+ return {key: item for key, item in document.items() if key != "recovered_at_ns"}
1321
+
1322
+ if path.exists():
1323
+ recorded, digest = e.read(path)
1324
+ if stable(recorded) != stable(value):
1325
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1326
+ "host recovery evidence already exists and differs")
1327
+ # Identical authority replayed after a crash between writing the
1328
+ # evidence and settling the row: reuse the original immutable record
1329
+ # instead of deadlocking on its timestamp.
1330
+ return {"path": str(path), "sha256": digest}
1331
+ return e.write(directory, path.name, value)
1332
+
1333
+
1334
+ def _typed_host_recovery(ledger: AdmissionLedger, request_id: str, row: dict[str, Any], *,
1335
+ expected_holder_sha256: str | None,
1336
+ terminal_evidence: Any) -> tuple[tuple[WriterHolder, ...], dict[str, str]]:
1337
+ """Release one historical execution whose recorded holder is provably dead.
1338
+
1339
+ Deliberately opt-in: the home-scope refusal stays intact for every unqualified
1340
+ caller. The authority here is the row's own recorded identity, cross-checked
1341
+ against the caller's terminal evidence, so no process identity is invented and
1342
+ a live holder can never be released through this path.
1343
+ """
1344
+
1345
+ request = row["request"]
1346
+ if request.get("kind") != "execution" or request.get("allocation_id") is not None:
1347
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1348
+ "typed recovery applies only to host execution reservations")
1349
+ recorded = row.get("holder")
1350
+ if not isinstance(recorded, dict) or any(recorded.get(key) in (None, "") for key in
1351
+ ("pid", "boot_id", "start_ticks", "client_id", "session_id", "run_id")):
1352
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1353
+ "host execution recorded identity is incomplete")
1354
+ if not expected_holder_sha256 or expected_holder_sha256 != canonical_holder_digest(recorded):
1355
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1356
+ "typed recovery must present the reservation's own holder digest")
1357
+ evidence = _terminal_evidence(terminal_evidence, request)
1358
+ holders = _host_runtime_holders(row, allow_broad_scope=True)
1359
+ proof = _dead_holder_proof(holders[0])
1360
+ try:
1361
+ os.lstat(Path(str(request.get("target_path"))))
1362
+ target_observation = "PRESENT"
1363
+ except FileNotFoundError:
1364
+ target_observation = "ABSENT"
1365
+ except OSError as exc:
1366
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED",
1367
+ "host execution recovery target observation failed") from exc
1368
+ recovery = {
1369
+ "schema_version": 1,
1370
+ "contract": HOST_RECOVERY_CONTRACT,
1371
+ "request_id": request_id,
1372
+ "original_request": {
1373
+ "repository_id": request.get("repository_id"),
1374
+ "client_id": request.get("client_id"),
1375
+ "session_id": request.get("session_id"),
1376
+ "run_id": request.get("run_id"),
1377
+ "target_path": request.get("target_path"),
1378
+ "memory_bytes": request.get("memory_bytes"),
1379
+ },
1380
+ "original_holder_digest": canonical_holder_digest(recorded),
1381
+ "target_observation": target_observation,
1382
+ "dead_holder_proof": proof,
1383
+ "terminal_evidence": evidence,
1384
+ "terminal_evidence_digest": canonical_holder_digest(evidence),
1385
+ "released_state": "RELEASED",
1386
+ "recovered_at_ns": time.time_ns(),
1387
+ }
1388
+ reference = _host_recovery_reference(ledger, request_id, recovery)
1389
+ return holders, reference
1390
+
1391
+
1392
+ def reconcile_finished_executions() -> None:
1393
+ """Use existing guarded exit recovery before admitting another local execution.
1394
+
1395
+ This also supports standalone clients without an external periodic reaper.
1396
+ Live or uncertain bindings remain reserved; checkout requests are never replayed.
1397
+ """
1398
+ ledger = AdmissionLedger()
1399
+ rows = ledger.read()["requests"]
1400
+ host_requests = [key for key, row in rows.items()
1401
+ if row["state"] == "ALLOCATED" and row["request"].get("kind") == "execution"
1402
+ and row["request"].get("allocation_id") is None]
1403
+ # A failed atomic write keeps all durable budgets. Repository recovery
1404
+ # remains independent, without reversing the repository -> host lock order.
1405
+ with suppress(ControllerError, OSError):
1406
+ _reconcile_finished_host_executions(ledger, host_requests)
1407
+ for row in rows.values():
1408
+ if (row["request"].get("allocation_id") is None
1409
+ or row["request"].get("kind") != "execution"
1410
+ or (row["state"] != "ALLOCATED" and not _unbound_execution(row))):
1411
+ continue
1412
+ if _allocation_holder_running(row):
1413
+ # A running holder has nothing to recover yet. Its repository is not
1414
+ # scanned or locked; the reservation is evaluated again once it exits.
1415
+ continue
1416
+ try:
1417
+ _reconcile_execution(row, apply=True)
1418
+ except (ControllerError, OSError):
1419
+ # Recovery failure keeps its budget and ordinary admission accounts
1420
+ # for it. The explicit admission-queue surface reports the blocker.
1421
+ continue
1422
+
1423
+
1424
+ def _allocation_holder_running(row: dict[str, Any]) -> bool:
1425
+ """Exact recorded holder identity is alive; absent or malformed identity is not."""
1426
+ from .writer_lease import holder_is_live
1427
+
1428
+ if row["request"].get("allocation_id") is None or not isinstance(row.get("holder"), dict):
1429
+ return False
1430
+ try:
1431
+ return holder_is_live(WriterHolder(**row["holder"]))
1432
+ except (ControllerError, TypeError, ValueError):
1433
+ return False
1434
+
1435
+
1436
+ def _reconcile_execution(row: dict[str, Any], *, apply: bool) -> dict[str, Any]:
1437
+ """Recover lost exit notifications using existing typed runtime removal gates."""
1438
+ from .lifecycle import update_external_binding
1439
+ from .policy import load_policy
1440
+ from .reconciler import reconcile_writers
1441
+ from .semantics import managed_record as _managed_record
1442
+ from .writer_lease import RUNTIME_BINDING_PREFIX, holder_is_live, runtime_binding
1443
+
1444
+ request = row["request"]
1445
+ if request["allocation_id"] is None:
1446
+ holders = _host_runtime_holders(row)
1447
+ if _host_runtime_is_live(holders):
1448
+ return {"code": "EXECUTION_OCCUPIED", "process_alive": True}
1449
+ if not apply:
1450
+ return {"code": "EXECUTION_RECONCILIATION_PENDING", "runtime_bindings": len(holders)}
1451
+ return host_execution("clear", request_id=fingerprint(request))["decision"]
1452
+ repository = inventory(Path(request["repository_path"]))
1453
+ store = MetadataStore(repository.common_dir)
1454
+ with store.lock(exclusive=False):
1455
+ if apply and not store.flock_held:
1456
+ raise ControllerError("MISSING_REPOSITORY_LOCK", "accounting requires the repository lock")
1457
+ snapshot = store._load_unlocked()
1458
+ if (str(repository.common_dir) != request["common_dir"]
1459
+ or snapshot.repository_id != request["repository_id"]):
1460
+ raise ControllerError("RESOURCE_IDENTITY_CHANGED", "execution repository identity changed")
1461
+ recorded = snapshot.records.get(request["allocation_id"])
1462
+ if recorded is not None and recorded.lifecycle == "RELEASED":
1463
+ from . import transaction_scope
1464
+
1465
+ target = Path(request["target_path"])
1466
+ receipts = [receipt for receipt in snapshot.release_receipts.values()
1467
+ if (receipt.repository_id, receipt.allocation_id, receipt.target_path,
1468
+ receipt.released_record_version, receipt.result)
1469
+ == (snapshot.repository_id, recorded.allocation_id, str(target),
1470
+ recorded.record_version, "RELEASED")]
1471
+ if (len(receipts) != 1 or recorded.external_bindings
1472
+ or recorded.allocation_id in snapshot.writer_leases
1473
+ or any(item.path == target for item in repository.worktrees)
1474
+ or target.exists() or target.is_symlink()
1475
+ or transaction_scope.conflicts(store, allocation=recorded.allocation_id, target=target)):
1476
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "execution release evidence and Git disagree")
1477
+ if apply:
1478
+ release_allocation_budget(snapshot.repository_id, recorded.allocation_id, receipts[0].receipt_id)
1479
+ return {"code": "EXECUTION_RESOURCES_RELEASED" if apply else "EXECUTION_RECONCILIATION_PENDING",
1480
+ "release_receipt_id": receipts[0].receipt_id, "allocation_released": True,
1481
+ "task_completed": False}
1482
+ if row.get("holder") is None:
1483
+ # Admitted but never bound while its allocation is still held: a launch
1484
+ # may be in flight. Only the allocation's formal release settles it.
1485
+ return {"code": "EXECUTION_NOT_STARTED", "allocation_released": False}
1486
+ if holder_is_live(WriterHolder(**row["holder"])):
1487
+ return {"code": "EXECUTION_OCCUPIED", "process_alive": True}
1488
+ item, record = _managed_record(repository, snapshot, Path(request["target_path"]))
1489
+ if (snapshot.repository_id != request["repository_id"] or record.allocation_id != request["allocation_id"]
1490
+ or record.owner != request.get("owner")):
1491
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "execution allocation authority changed")
1492
+ bindings = [value for value in record.external_bindings
1493
+ if (binding := runtime_binding(value)) is not None and binding[1] == row.get("lease_id")]
1494
+ if not bindings:
1495
+ # A crash can occur between recording a reservation's holder and writing
1496
+ # its runtime binding. The same removal path proves absence; no fake past
1497
+ # receipt or binding is inserted into repository history.
1498
+ bindings = [RUNTIME_BINDING_PREFIX + json.dumps({"holder": row["holder"],
1499
+ "lease_id": row["lease_id"], "kind": "command"}, sort_keys=True, separators=(",", ":"))]
1500
+ if not apply:
1501
+ return {"code": "EXECUTION_RECONCILIATION_PENDING", "runtime_bindings": len(bindings)}
1502
+ for value in bindings:
1503
+ current = store.load().records[record.allocation_id]
1504
+ update_external_binding(repository.canonical_path, target=item.path, owner=current.owner,
1505
+ expected_record_version=current.record_version, binding=value, action="remove")
1506
+ policy = load_policy(repository.canonical_path)
1507
+ if policy.reconciler_enabled:
1508
+ reconcile_writers(repository.canonical_path, actor=policy.reconciler_actor,
1509
+ trigger="background", dry_run=False, only=(record.task_key,))
1510
+ return {"code": "EXECUTION_RESOURCES_RELEASED", "task_completed": False}
1511
+
1512
+
1513
+ def _recovery_guidance(row: dict[str, Any], entry: dict[str, Any], *, dead: bool,
1514
+ note: str | None) -> dict[str, Any]:
1515
+ """What a typed caller may do about this reservation, without granting anything."""
1516
+ request = row["request"]
1517
+ if row["state"] in {"RELEASED", "CANCELLED"}:
1518
+ return {"supported_action": None, "reason": "ALREADY_TERMINAL"}
1519
+ if request.get("allocation_id") is not None:
1520
+ return {"supported_action": None, "reason": "USE_ALLOCATION_RECONCILIATION"}
1521
+ if row.get("lease_id") is not None:
1522
+ return {"supported_action": None, "reason": "WRITER_LEASE_HELD"}
1523
+ if row["state"] in {"WAITING", "RESERVED"} and not row.get("holder"):
1524
+ return {"supported_action": "cancel", "reason": "NOT_ATTEMPTED"}
1525
+ if dead:
1526
+ return {"supported_action": "clear --recover-dead-holder",
1527
+ "reason": "HOLDER_PROVABLY_DEAD",
1528
+ "expected_holder_sha256": entry.get("holder_sha256"),
1529
+ "target_observation": entry.get("target_observation"),
1530
+ "terminal_evidence_template": {
1531
+ "kind": HOST_TERMINAL_EVIDENCE_KIND,
1532
+ "status": "EXITED",
1533
+ "ended_at": "<caller-attested RFC3339 end time>",
1534
+ "session_id": request.get("session_id"),
1535
+ "run_id": request.get("run_id"),
1536
+ "end_reason": "<caller-attested reason>"}}
1537
+ return {"supported_action": None, "reason": note or "HOLDER_LIVENESS_UNCERTAIN"}
1538
+
1539
+
1540
+ def _inspect_host_executions(ledger: AdmissionLedger, *, request_id: str | None = None,
1541
+ session_id: str | None = None,
1542
+ run_id: str | None = None) -> dict[str, Any]:
1543
+ """Read-only recovery intelligence for host execution reservations.
1544
+
1545
+ Publishes exactly the identity a typed caller must present (canonical
1546
+ holder bytes and their SHA-256), the runtime observation proving the
1547
+ recorded holder dead, the missing-target fact, and a terminal-evidence
1548
+ template, so recovery never needs to read the private ledger layout.
1549
+ Nothing here mutates ledger state; authority stays with the typed clear
1550
+ verb, which re-verifies every fact at settle time.
1551
+ """
1552
+ from .controller import envelope
1553
+
1554
+ with ledger.locked():
1555
+ state = ledger.read()
1556
+ rows = []
1557
+ for key, row in sorted(state["requests"].items(),
1558
+ key=lambda item: item[1].get("created_at_ns", 0)):
1559
+ request = row["request"]
1560
+ if request.get("kind") != "execution" or request.get("owner") != os.getuid():
1561
+ continue
1562
+ if request_id is not None and key != request_id:
1563
+ continue
1564
+ if session_id is not None and request.get("session_id") != session_id:
1565
+ continue
1566
+ if run_id is not None and request.get("run_id") != run_id:
1567
+ continue
1568
+ entry: dict[str, Any] = {
1569
+ "request_id": key, "state": row["state"],
1570
+ "client_id": request.get("client_id"), "session_id": request.get("session_id"),
1571
+ "run_id": request.get("run_id"), "target_path": request.get("target_path"),
1572
+ "memory_bytes": request.get("memory_bytes"),
1573
+ "accounted_memory_bytes": (max(
1574
+ 0, int(request.get("memory_bytes", 0)) - execution_memory(row))
1575
+ if row["state"] in {"RESERVED", "ALLOCATED", "ATTENTION_REQUIRED"} else 0),
1576
+ "allocation_bound": request.get("allocation_id") is not None,
1577
+ "writer_lease_held": row.get("lease_id") is not None,
1578
+ }
1579
+ try:
1580
+ info = os.lstat(Path(str(request.get("target_path"))))
1581
+ entry["target_observation"] = ("PRESENT" if [info.st_dev, info.st_ino]
1582
+ == request.get("path_identity") else "IDENTITY_CHANGED")
1583
+ except FileNotFoundError:
1584
+ entry["target_observation"] = "ABSENT"
1585
+ except OSError:
1586
+ entry["target_observation"] = "UNCERTAIN"
1587
+ holder = row.get("holder")
1588
+ dead, note = False, None
1589
+ if isinstance(holder, dict):
1590
+ canonical = json.dumps(holder, sort_keys=True, separators=(",", ":"))
1591
+ entry["holder_canonical_json"] = canonical
1592
+ entry["holder_sha256"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
1593
+ try:
1594
+ entry["dead_holder_proof"] = _dead_holder_proof(WriterHolder(**holder))
1595
+ dead = True
1596
+ except ControllerError as exc:
1597
+ note = exc.code
1598
+ else:
1599
+ note = "NO_RECORDED_HOLDER"
1600
+ entry["recovery"] = _recovery_guidance(row, entry, dead=dead, note=note)
1601
+ rows.append(entry)
1602
+ payload = envelope("host-execution")
1603
+ payload["decision"] = {"code": "HOST_EXECUTION_INSPECTION", "dry_run": True,
1604
+ "read_only": True, "rows": rows}
1605
+ return payload
1606
+
1607
+
1608
+ def host_execution(action: str, *, target: Path | None = None, request_id: str | None = None,
1609
+ client_id: str = "agent-workspace", session_id: str | None = None,
1610
+ run_id: str | None = None, holder_pid: int | None = None,
1611
+ memory_bytes: int = DEFAULT_EXECUTION_MEMORY,
1612
+ recover_dead_holder: bool = False,
1613
+ expected_holder_sha256: str | None = None,
1614
+ terminal_evidence: dict[str, Any] | None = None) -> dict[str, Any]:
1615
+ """Budget an AW process without inventing a repository allocation or writer lease.
1616
+
1617
+ Uses the same host request, typed occupancy, exit checks and periodic reconciler
1618
+ as governed executions. This grants resources only, never repository writes.
1619
+ """
1620
+ from dataclasses import asdict
1621
+
1622
+ from .controller import envelope
1623
+ from .environment import verify_runtime_transfer
1624
+ from .writer_lease import RUNTIME_BINDING_PREFIX, describe_holder
1625
+
1626
+ ledger = AdmissionLedger()
1627
+ if action == "inspect":
1628
+ return _inspect_host_executions(ledger, request_id=request_id,
1629
+ session_id=session_id, run_id=run_id)
1630
+ if action == "admit":
1631
+ if target is None or not session_id:
1632
+ raise ControllerError("INVALID_REQUEST", "host admission needs a path and session identity")
1633
+ path = target.resolve(strict=True)
1634
+ info = path.stat()
1635
+ if not path.is_dir():
1636
+ raise ControllerError("INVALID_REQUEST", "host execution path must be a directory")
1637
+ if unbounded_host_scope(path):
1638
+ raise ControllerError(
1639
+ "INVALID_REQUEST",
1640
+ "host execution target must not be the home root or one of its ancestors; "
1641
+ "admit against the session's concrete working directory instead",
1642
+ )
1643
+ request = execution_request(None, "host:" + str(os.getuid()), None, path,
1644
+ client_id=client_id, session_id=session_id, run_id=run_id, memory_bytes=memory_bytes)
1645
+ request.update(owner=os.getuid(), path_identity=[info.st_dev, info.st_ino])
1646
+ reconcile_finished_executions()
1647
+ decision = ledger.reserve([request])
1648
+ else:
1649
+ with ledger.locked():
1650
+ state = ledger.read()
1651
+ if request_id is None and action == "cancel" and session_id and target is not None:
1652
+ matches = [(key, value) for key, value in state["requests"].items()
1653
+ if value["request"].get("kind") == "execution" and value["request"].get("allocation_id") is None
1654
+ and all(value["request"].get(name) == expected for name, expected in
1655
+ (("owner", os.getuid()), ("client_id", client_id), ("session_id", session_id), ("run_id", run_id)))]
1656
+ if not matches:
1657
+ payload = envelope("host-execution")
1658
+ payload["decision"] = {"code": "HOST_EXECUTION_CLEARED", "request_ids": [], "task_completed": False}
1659
+ return payload
1660
+ if len(matches) != 1 or matches[0][1]["request"]["target_path"] != str(target.resolve()):
1661
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "host cancellation identity changed")
1662
+ request_id = matches[0][0]
1663
+ row = state["requests"].get(request_id)
1664
+ if (row is None or row["request"].get("kind") != "execution"
1665
+ or row["request"].get("allocation_id") is not None or row["request"]["owner"] != os.getuid()):
1666
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "host execution identity is unavailable")
1667
+ request = row["request"]
1668
+ unattempted_cancel = (
1669
+ action == "cancel"
1670
+ and row["state"] in {"WAITING", "RESERVED", "CANCELLED"}
1671
+ and not row.get("holder")
1672
+ and not row.get("runtime_bindings")
1673
+ and row.get("lease_id") is None
1674
+ and not unbounded_host_scope(Path(request["target_path"]))
1675
+ )
1676
+ _check_host_target(
1677
+ request,
1678
+ allow_missing=unattempted_cancel or (action == "clear" and recover_dead_holder),
1679
+ )
1680
+ if action in {"bind", "transfer"}:
1681
+ if holder_pid is None:
1682
+ raise ControllerError("INVALID_HOLDER", "host execution requires its exact process")
1683
+ holder = describe_holder(holder_pid, client_id=request["client_id"],
1684
+ session_id=request["session_id"], run_id=request["run_id"])
1685
+ if action == "bind":
1686
+ if row["state"] != "RESERVED":
1687
+ if row["state"] != "ALLOCATED" or row.get("holder") != asdict(holder):
1688
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "host execution was already attempted")
1689
+ else:
1690
+ row.update(state="ALLOCATED", holder=asdict(holder), lease_id=None)
1691
+ row["history"].append({"state": "ALLOCATED", "at_ns": time.time_ns()})
1692
+ else:
1693
+ if row["state"] != "ALLOCATED":
1694
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "host transfer needs an allocated execution")
1695
+ verify_runtime_transfer(WriterHolder(**row["holder"]), holder)
1696
+ binding = RUNTIME_BINDING_PREFIX + json.dumps({"holder": asdict(holder),
1697
+ "lease_id": None, "kind": "background" if action == "transfer" else "command"},
1698
+ sort_keys=True, separators=(",", ":"))
1699
+ bindings = row.setdefault("runtime_bindings", [])
1700
+ if binding not in bindings:
1701
+ bindings.append(binding)
1702
+ decision = {"code": "HOST_EXECUTION_BOUND", "binding": binding}
1703
+ elif action in {"clear", "cancel"}:
1704
+ terminal = "RELEASED" if action == "clear" else "CANCELLED"
1705
+ recovery = None
1706
+ if row["state"] == terminal or (action == "cancel" and row["state"] == "RELEASED"):
1707
+ decision = {"code": "HOST_EXECUTION_CLEARED", "task_completed": False}
1708
+ changed = False
1709
+ else:
1710
+ if action == "clear":
1711
+ if row["state"] != "ALLOCATED":
1712
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "host execution has not been bound")
1713
+ if recover_dead_holder:
1714
+ _, recovery = _typed_host_recovery(
1715
+ ledger, request_id, row,
1716
+ expected_holder_sha256=expected_holder_sha256,
1717
+ terminal_evidence=terminal_evidence)
1718
+ else:
1719
+ _check_host_exit(row)
1720
+ elif row["state"] not in {"WAITING", "RESERVED"} or row.get("holder"):
1721
+ raise ControllerError("ADMISSION_RECOVERY_REQUIRED", "an attempted host execution cannot be cancelled")
1722
+ row["state"] = terminal
1723
+ row["history"].append({"state": terminal, "at_ns": time.time_ns()})
1724
+ decision = {"code": "HOST_EXECUTION_CLEARED", "task_completed": False}
1725
+ if recovery is not None:
1726
+ decision["recovery"] = recovery
1727
+ changed = True
1728
+ else:
1729
+ raise ControllerError("INVALID_REQUEST", "unknown host execution action")
1730
+ if action not in {"clear", "cancel"} or changed:
1731
+ row["updated_at_ns"] = time.time_ns()
1732
+ ledger._save(state)
1733
+ decision["request_ids"] = [request_id]
1734
+ payload = envelope("host-execution")
1735
+ payload["decision"] = decision
1736
+ return payload