polyctl 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. polyctl/__init__.py +7 -0
  2. polyctl/__main__.py +5 -0
  3. polyctl/a2a.py +226 -0
  4. polyctl/a2a_mcp.py +546 -0
  5. polyctl/acp.py +417 -0
  6. polyctl/adapters/README.md +101 -0
  7. polyctl/adapters/aider.json +32 -0
  8. polyctl/adapters/amazon-q.json +15 -0
  9. polyctl/adapters/amp.json +16 -0
  10. polyctl/adapters/antigravity.json +120 -0
  11. polyctl/adapters/claude-code.json +229 -0
  12. polyctl/adapters/cloud-generic.json +57 -0
  13. polyctl/adapters/codex.json +47 -0
  14. polyctl/adapters/command-code.json +45 -0
  15. polyctl/adapters/copilot.json +17 -0
  16. polyctl/adapters/cursor-agent.json +29 -0
  17. polyctl/adapters/deepseek.json +13 -0
  18. polyctl/adapters/droid.json +34 -0
  19. polyctl/adapters/gemini.json +30 -0
  20. polyctl/adapters/grok.json +22 -0
  21. polyctl/adapters/hermes.json +34 -0
  22. polyctl/adapters/kilocode.json +30 -0
  23. polyctl/adapters/kimi.json +23 -0
  24. polyctl/adapters/kiro.json +332 -0
  25. polyctl/adapters/ollama.json +29 -0
  26. polyctl/adapters/omp.json +34 -0
  27. polyctl/adapters/openai-api.json +68 -0
  28. polyctl/adapters/opencode.json +162 -0
  29. polyctl/adapters/pi.json +24 -0
  30. polyctl/adapters/pr-agent.json +54 -0
  31. polyctl/adapters/profiles.json +9 -0
  32. polyctl/adapters/qoder.json +163 -0
  33. polyctl/adapters/qwen.json +27 -0
  34. polyctl/adapters/reasonix.json +25 -0
  35. polyctl/adapters/rewriters/rtk.json +24 -0
  36. polyctl/adapters/trae.json +31 -0
  37. polyctl/adapters/vendors.json +42 -0
  38. polyctl/adapters/warp.json +88 -0
  39. polyctl/allowlist.py +1756 -0
  40. polyctl/api_review.py +920 -0
  41. polyctl/cli.py +835 -0
  42. polyctl/conformance_harness.py +9 -0
  43. polyctl/demo.py +503 -0
  44. polyctl/drivers.py +146 -0
  45. polyctl/email.py +160 -0
  46. polyctl/features.py +98 -0
  47. polyctl/journal.py +144 -0
  48. polyctl/language.py +945 -0
  49. polyctl/lib.py +1296 -0
  50. polyctl/matrix.py +660 -0
  51. polyctl/mcp.py +1362 -0
  52. polyctl/ntfy.py +169 -0
  53. polyctl/outbound.py +39 -0
  54. polyctl/parts/__init__.py +5 -0
  55. polyctl/parts/cli_authority.py +2615 -0
  56. polyctl/parts/cli_channels.py +1325 -0
  57. polyctl/parts/cli_checks.py +2293 -0
  58. polyctl/parts/cli_completion.py +686 -0
  59. polyctl/parts/cli_evidence.py +141 -0
  60. polyctl/parts/cli_gain.py +375 -0
  61. polyctl/parts/cli_harness.py +392 -0
  62. polyctl/parts/cli_hooks.py +1816 -0
  63. polyctl/parts/cli_install.py +638 -0
  64. polyctl/parts/cli_lanes.py +106 -0
  65. polyctl/parts/cli_maintenance.py +2536 -0
  66. polyctl/parts/cli_next.py +248 -0
  67. polyctl/parts/cli_pr.py +53 -0
  68. polyctl/parts/cli_project.py +2857 -0
  69. polyctl/parts/cli_review.py +5158 -0
  70. polyctl/parts/cli_status.py +898 -0
  71. polyctl/parts/lib_adapters.py +2933 -0
  72. polyctl/parts/lib_alarms.py +2106 -0
  73. polyctl/parts/lib_events.py +224 -0
  74. polyctl/parts/lib_evidence.py +412 -0
  75. polyctl/parts/lib_gates.py +2099 -0
  76. polyctl/parts/lib_lanes.py +829 -0
  77. polyctl/parts/lib_mail.py +1541 -0
  78. polyctl/parts/lib_next.py +652 -0
  79. polyctl/parts/lib_pr.py +331 -0
  80. polyctl/parts/lib_slices.py +3337 -0
  81. polyctl/parts/lib_state.py +2415 -0
  82. polyctl/parts/lib_transcript.py +4118 -0
  83. polyctl/pressure.py +325 -0
  84. polyctl/prices.json +47 -0
  85. polyctl/procs.py +1194 -0
  86. polyctl/settings.py +505 -0
  87. polyctl/skill/SKILL.md +365 -0
  88. polyctl/skillkit.py +503 -0
  89. polyctl/storage.py +2055 -0
  90. polyctl/telegram.py +327 -0
  91. polyctl/tiers.py +123 -0
  92. polyctl/verdicts.py +41 -0
  93. polyctl/watchdog.py +3887 -0
  94. polyctl/web.py +337 -0
  95. polyctl-0.7.0.dist-info/METADATA +235 -0
  96. polyctl-0.7.0.dist-info/RECORD +99 -0
  97. polyctl-0.7.0.dist-info/WHEEL +4 -0
  98. polyctl-0.7.0.dist-info/entry_points.txt +5 -0
  99. polyctl-0.7.0.dist-info/licenses/LICENSE +21 -0
polyctl/storage.py ADDED
@@ -0,0 +1,2055 @@
1
+ """Crash-safe local persistence primitives for polyctl's append-only ledgers.
2
+
3
+ The ledgers are authority evidence, not logging. A caller may act on a row as
4
+ soon as this module returns, so returning means all bytes reached ``fsync`` and,
5
+ for a newly-created file, the containing directory entry did too. The API stays
6
+ standard-library-only and uses an OS lock to keep independent polyctl processes from
7
+ interleaving JSON records or selecting competing hash-chain predecessors.
8
+ """
9
+ from contextlib import contextmanager
10
+ import errno
11
+ import hashlib
12
+ import json
13
+ import os
14
+ import re
15
+ import stat
16
+ import time
17
+
18
+ UTF8 = "utf-8"
19
+ CHAIN_PREVIOUS_FIELD = "previous"
20
+
21
+
22
+ class LedgerCorruption(RuntimeError):
23
+ """A committed ledger record is malformed or violates its hash chain."""
24
+
25
+
26
+ class LedgerLockTimeout(TimeoutError):
27
+ """Another process held a ledger lock past the caller's deadline.
28
+
29
+ `path` is the lock and `timeout` the seconds waited for it, so a command can say in one
30
+ line which lock stood in its way (CLI-ROBUST).
31
+ """
32
+
33
+ def __init__(self, message, path=None, timeout=None):
34
+ super().__init__(message)
35
+ self.path, self.timeout = path, timeout
36
+
37
+
38
+ class LockThroughLink(OSError):
39
+ """A lock file that is a link, or not a regular file: polyctl takes no lock through it (LOCK-LINKS)."""
40
+
41
+
42
+ def _call(checkpoint, step):
43
+ if checkpoint is not None:
44
+ checkpoint(step)
45
+
46
+
47
+ def _open_lock_nt(path, refused):
48
+ """A Windows lock file's handle, opened as the link itself where the path is one, never through it (LOCK-LINKS).
49
+
50
+ FILE_FLAG_OPEN_REPARSE_POINT opens a link, not what it names, so OPEN_ALWAYS makes nothing through one; the
51
+ caller refuses the handle when it is a reparse point. A link Windows will not open that way is refused here.
52
+ """
53
+ import ctypes
54
+ import msvcrt
55
+ import stat as _stat
56
+ from ctypes import wintypes
57
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
58
+ create, close = kernel32.CreateFileW, kernel32.CloseHandle
59
+ create.argtypes = (wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD,
60
+ wintypes.DWORD, wintypes.HANDLE)
61
+ create.restype = wintypes.HANDLE
62
+ close.argtypes = (wintypes.HANDLE,)
63
+ generic_read_write, share_read_write, open_always = 0x80000000 | 0x40000000, 0x1 | 0x2, 4
64
+ normal_open_reparse_point = 0x80 | 0x00200000
65
+ raw = create(path, generic_read_write, share_read_write, None, open_always, normal_open_reparse_point, None)
66
+ if raw is None or raw == wintypes.HANDLE(-1).value:
67
+ error = ctypes.WinError(ctypes.get_last_error())
68
+ try:
69
+ linked = _stat.S_ISLNK(os.lstat(path).st_mode)
70
+ except OSError:
71
+ linked = False
72
+ if linked:
73
+ raise LockThroughLink(refused) from error
74
+ raise error
75
+ try:
76
+ fd = msvcrt.open_osfhandle(raw, os.O_APPEND)
77
+ except BaseException:
78
+ close(raw)
79
+ raise
80
+ try:
81
+ return open(fd, "a+b", buffering=0)
82
+ except BaseException:
83
+ os.close(fd)
84
+ raise
85
+
86
+
87
+ def _open_lock(path):
88
+ """A lock file's handle: the file reached through no final link, a regular file of one name; LockThroughLink
89
+ otherwise.
90
+
91
+ A lock was opened "a+b", which follows a final link and creates the file it names: a link put where a lock is
92
+ kept - `.polyctl/` in the working tree holds several - had polyctl create a file wherever the link pointed and write a
93
+ byte into it, outside the tree an agent may write. On POSIX a lock is opened with O_NOFOLLOW, and without
94
+ blocking, so a FIFO put in its place is refused at once rather than waited on; a POSIX platform without
95
+ O_NOFOLLOW takes no lock. Windows has no O_NOFOLLOW, and looking at the path before and after a creating open
96
+ left that open to follow a link put there in between: there the path is opened as the link itself
97
+ (`_open_lock_nt`), so nothing is made through one. A link, anything but a regular file, or a file that has
98
+ another name - a hard link put in the lock's place - is then refused before anything is written (LOCK-LINKS).
99
+
100
+ The way is chosen by what the platform offers - O_NOFOLLOW where it has one - and not by `os.name`, which a
101
+ test of Windows' behaviour sets on another platform: there the lock went the Windows way, to a kernel32 that
102
+ platform has not, and the hold the test ran failed where its lock was taken (LOCK-LINKS-2).
103
+ """
104
+ import stat as _stat
105
+ refused = (f"{path} is a link, not a regular file, or a file with another name: polyctl takes no lock through it; "
106
+ "remove it, and polyctl makes the lock again")
107
+ nofollow = getattr(os, "O_NOFOLLOW", None)
108
+ if nofollow is None and os.name == "nt":
109
+ handle = _open_lock_nt(path, refused)
110
+ else:
111
+ if nofollow is None:
112
+ raise LockThroughLink(f"{path}: this platform opens no file through no link (it has no O_NOFOLLOW), "
113
+ "and polyctl takes no lock it cannot open that way")
114
+ flags = nofollow | getattr(os, "O_NONBLOCK", 0)
115
+ try:
116
+ handle = open(path, "a+b", buffering=0, opener=lambda name, mode: os.open(name, mode | flags, 0o666))
117
+ except OSError as exc:
118
+ if exc.errno in (errno.ELOOP, errno.EMLINK):
119
+ raise LockThroughLink(refused) from exc # what O_NOFOLLOW answers a final link with
120
+ raise
121
+ try:
122
+ opened = os.fstat(handle.fileno())
123
+ reparse = getattr(opened, "st_file_attributes", 0) & getattr(_stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
124
+ if not _stat.S_ISREG(opened.st_mode) or reparse or opened.st_nlink != 1:
125
+ raise LockThroughLink(refused)
126
+ except BaseException:
127
+ handle.close()
128
+ raise
129
+ return handle
130
+
131
+
132
+ def _lock_still_named(path, handle):
133
+ """Whether `path` still names, through no link, the one file `handle` holds (LOCK-LINKS)."""
134
+ import stat as _stat
135
+ try:
136
+ at = os.lstat(path)
137
+ except FileNotFoundError:
138
+ return False
139
+ held = os.fstat(handle.fileno())
140
+ if _stat.S_ISLNK(at.st_mode) or held.st_nlink != 1:
141
+ return False
142
+ if not at.st_ino or not held.st_ino:
143
+ return True # a file system that numbers no file cannot tell, and has no links to swap in
144
+ return (at.st_dev, at.st_ino) == (held.st_dev, held.st_ino)
145
+
146
+
147
+ def _let_go(handle):
148
+ try:
149
+ handle.seek(0)
150
+ if os.name == "nt":
151
+ import msvcrt
152
+ msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
153
+ else:
154
+ import fcntl
155
+ fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
156
+ except OSError:
157
+ pass
158
+ handle.close()
159
+
160
+
161
+ @contextmanager
162
+ def _exclusive_lock(path, timeout=10.0, *, _folder=None):
163
+ """Cross-platform advisory lock on one byte of a sidecar file.
164
+
165
+ The byte is written once, by whichever process finds the file empty. On Windows a lock is mandatory for
166
+ writes, and another process could take the file between this one's look and its write: the write then
167
+ failed with PermissionError, as four processes appending to a new ledger at once showed on the Windows
168
+ lane. The file is no longer empty then, which is all the write was for, and the handle buffers nothing,
169
+ so nothing is written again when it closes (WINDOWS-LOCK-WRITE).
170
+
171
+ A lock is held only while its path still names the file locked: a file whose name was taken away, or given
172
+ to another file, after it was opened - an agent writes `.polyctl/` - is let go and the path's file taken instead,
173
+ where two of polyctl's writers each held a file of the one name. A lock in the tree orders polyctl's own writers; an
174
+ agent that replaces one while it is held could already write the ledger it guards, which the ledger's chain
175
+ and checkpoints show (LOCK-LINKS).
176
+ """
177
+ parent = os.path.dirname(path) or "."
178
+ if _folder is None:
179
+ os.makedirs(parent, exist_ok=True)
180
+ deadline = time.monotonic() + max(0.0, timeout)
181
+ while True:
182
+ if _folder is None:
183
+ handle = _open_lock(path)
184
+ else:
185
+ fd = _tree_open(_folder, os.path.basename(path), os.O_RDWR | os.O_CREAT | os.O_APPEND)
186
+ if os.fstat(fd).st_nlink != 1:
187
+ os.close(fd)
188
+ raise LockThroughLink(f"{path} has another name; remove the link before retrying")
189
+ handle = open(fd, "a+b", buffering=0)
190
+ try:
191
+ handle.seek(0, os.SEEK_END)
192
+ if handle.tell() == 0:
193
+ try:
194
+ handle.write(b"\0")
195
+ except PermissionError:
196
+ pass
197
+ while True:
198
+ try:
199
+ handle.seek(0)
200
+ if os.name == "nt":
201
+ import msvcrt
202
+ msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
203
+ else:
204
+ import fcntl
205
+ fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
206
+ break
207
+ except (OSError, BlockingIOError):
208
+ if time.monotonic() >= deadline:
209
+ raise LedgerLockTimeout(f"timed out locking {path}", path, timeout)
210
+ time.sleep(0.02)
211
+ if _folder is None:
212
+ named = _lock_still_named(path, handle)
213
+ else:
214
+ try:
215
+ at = _tree_stat(_folder, os.path.basename(path))
216
+ held = os.fstat(handle.fileno())
217
+ named = not stat.S_ISLNK(at.st_mode) and held.st_nlink == 1 and \
218
+ (at.st_dev, at.st_ino) == (held.st_dev, held.st_ino)
219
+ except FileNotFoundError:
220
+ named = False
221
+ except BaseException:
222
+ _let_go(handle)
223
+ raise
224
+ if named:
225
+ break
226
+ _let_go(handle)
227
+ if time.monotonic() >= deadline:
228
+ raise LedgerLockTimeout(f"timed out locking {path}: its file was replaced as it was taken", path,
229
+ timeout)
230
+ try:
231
+ yield
232
+ finally:
233
+ _let_go(handle)
234
+
235
+
236
+ def _read_jsonl_unlocked(path, allow_partial_tail=True):
237
+ """Parse a ledger while the caller owns its lock.
238
+
239
+ A row is committed by its newline. A final line without one - even valid
240
+ JSON - is what an interrupted or failed append leaves, and is never a row:
241
+ a grant whose append raised must not be read back as authority (#68).
242
+ """
243
+ try:
244
+ data = open(path, "rb").read()
245
+ except FileNotFoundError:
246
+ return []
247
+ return _jsonl_rows(data, path, allow_partial_tail)
248
+
249
+
250
+ def _jsonl_records(data):
251
+ """([(line number, record bytes)] of each complete, non-empty line, the line number of an unfinished last one or
252
+ None) - the one way polyctl splits JSONL bytes into records, which a read, a count and an append all use (OPTIN-ATTESTED).
253
+
254
+ A line ends at `\n`, `\r` or `\r\n`; an empty one is no record, and one of blanks is a record a parse refuses.
255
+ Counted any other way, a row put after `\r` on polyctl's own line was one line to a count and two records to a read.
256
+ """
257
+ records = []
258
+ for index, raw in enumerate(data.splitlines(keepends=True)):
259
+ body = raw.rstrip(b"\r\n")
260
+ if not body:
261
+ continue
262
+ if not raw.endswith((b"\n", b"\r")):
263
+ return records, index
264
+ records.append((index, body))
265
+ return records, None
266
+
267
+
268
+ def _jsonl_row(body, index, path):
269
+ try:
270
+ return json.loads(body.decode(UTF8))
271
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
272
+ raise LedgerCorruption(f"malformed JSONL record {index + 1} in {path}: {exc}") from exc
273
+
274
+
275
+ def _jsonl_rows(data, path, allow_partial_tail=True):
276
+ """Parse bytes already read through a held file (LEADS-ADAPTERS)."""
277
+ records, unfinished = _jsonl_records(data)
278
+ rows = [_jsonl_row(body, index, path) for index, body in records]
279
+ if unfinished is not None and not allow_partial_tail:
280
+ raise LedgerCorruption(f"uncommitted final record {unfinished + 1} in {path}")
281
+ return rows
282
+
283
+
284
+ def read_jsonl(path, allow_partial_tail=True, timeout=10.0):
285
+ """Read committed JSONL records and fail on corruption before the tail.
286
+
287
+ Readers share the writer's sidecar lock so they cannot observe a complete
288
+ row in the interval after ``write`` but before ``fsync``. A killed process
289
+ may still leave the final line without its newline; readers may ignore that
290
+ one incomplete tail, while malformed complete rows always fail closed.
291
+ """
292
+ if not os.path.exists(path):
293
+ return []
294
+ with _exclusive_lock(path + ".lock", timeout=timeout):
295
+ return _read_jsonl_unlocked(path, allow_partial_tail)
296
+
297
+
298
+ def chained_row_digest(record, chain):
299
+ """Canonical, domain-separated SHA-256 digest of one chained JSON object."""
300
+ if not isinstance(record, dict):
301
+ raise TypeError("a chained ledger row must be a JSON object")
302
+ if not isinstance(chain, str) or not chain:
303
+ raise ValueError("a chained ledger requires a non-empty domain")
304
+ canonical = json.dumps(
305
+ record,
306
+ ensure_ascii=True,
307
+ allow_nan=False,
308
+ sort_keys=True,
309
+ separators=(",", ":"),
310
+ ).encode("ascii")
311
+ domain = chain.encode(UTF8)
312
+ digest = hashlib.sha256()
313
+ digest.update(domain)
314
+ digest.update(b"\0")
315
+ digest.update(canonical)
316
+ return "sha256:" + digest.hexdigest()
317
+
318
+
319
+ def checkpoint_path():
320
+ """Where chained ledgers record their committed length: outside the repository (#62).
321
+
322
+ A hash chain detects a changed or inserted row but not a removed tail, because
323
+ every prefix of a valid chain is itself valid; deleting the newest grants, or
324
+ the whole file, left a ledger that passed every check. The recorded length is
325
+ kept apart from the ledger, so removing rows from the file does not remove the
326
+ record of how many there were.
327
+ """
328
+ return (os.environ.get("POLYCTL_LEDGER_CHECKPOINTS")
329
+ or os.path.join(os.path.expanduser("~"), ".polyctl", "ledger-checkpoints.json"))
330
+
331
+
332
+ def _load_committed_lengths():
333
+ try:
334
+ with open(checkpoint_path(), encoding="utf-8") as handle:
335
+ data = json.load(handle)
336
+ except (OSError, ValueError):
337
+ return {}
338
+ return data if isinstance(data, dict) else {}
339
+
340
+
341
+ # What Windows answers, for a moment, when a rename would replace a file another handle
342
+ # holds open: ERROR_ACCESS_DENIED and ERROR_SHARING_VIOLATION.
343
+ _WINDOWS_IN_USE = (5, 32)
344
+ _REPLACE_ATTEMPTS = 8
345
+ _REPLACE_FIRST_WAIT = 0.01
346
+
347
+
348
+ def _replace(source, target):
349
+ """``os.replace``, retried with backoff for about a second while Windows reports the target in use (#71).
350
+
351
+ A chained read opens the checkpoint store under its own ledger's lock, not the
352
+ store's, and Windows does not rename over a file another handle holds open: an
353
+ append to one ledger failed while another ledger was being read, and the row it had
354
+ written was taken back. A virus scanner holds a new file open the same way. Every
355
+ replace in this module comes here. Only those two Windows errors are retried, a
356
+ bounded number of times, and the last one still reaches the caller; POSIX renames
357
+ over an open file, and a PermissionError there is raised at once, as before.
358
+ """
359
+ for attempt in range(_REPLACE_ATTEMPTS):
360
+ try:
361
+ return os.replace(source, target)
362
+ except PermissionError as exc:
363
+ if getattr(exc, "winerror", None) not in _WINDOWS_IN_USE or attempt + 1 == _REPLACE_ATTEMPTS:
364
+ raise
365
+ time.sleep(_REPLACE_FIRST_WAIT * 2 ** attempt)
366
+
367
+
368
+ def _write_committed_lengths(data):
369
+ """Replace the checkpoint store whole; the caller holds its lock.
370
+
371
+ Through replace_file_durably, under a name no file had and never through a link: `<store>.<pid>.tmp`, opened to
372
+ truncate, wrote the target of a link left at that name, and POLYCTL_LEDGER_CHECKPOINTS can name a store in a folder an
373
+ agent writes (STORAGE-TMP). It syncs the folder after the rename too: renamed into place with no sync, an append
374
+ returned on a record a power cut could still take back (the review of LEDGER-LENGTH, LEDGER-HISTORY).
375
+ """
376
+ replace_file_durably(checkpoint_path(), json.dumps(data, indent=1, sort_keys=True).encode("utf-8"))
377
+
378
+
379
+ def _checkpoint_entry(path, *, identity=None):
380
+ """What the checkpoint store records for one ledger: {} when nothing."""
381
+ entry = _load_committed_lengths().get(identity or os.path.realpath(path))
382
+ return entry if isinstance(entry, dict) else {}
383
+
384
+
385
+ def _update_checkpoint(path, change, *, identity=None):
386
+ """Replace one ledger's record in the checkpoint store with `change(record)`, under the store's lock."""
387
+ store = checkpoint_path()
388
+ os.makedirs(os.path.dirname(store) or ".", exist_ok=True)
389
+ with _exclusive_lock(store + ".lock"):
390
+ data = _load_committed_lengths()
391
+ key = identity or os.path.realpath(path)
392
+ data[key] = change(dict(data[key]) if isinstance(data.get(key), dict) else {})
393
+ _write_committed_lengths(data)
394
+
395
+
396
+ def _record_committed_length(path, count, digest, sealed=None, *, identity=None):
397
+ def change(previous):
398
+ mark = {"count": count, "digest": digest, "at": int(time.time())}
399
+ # A seal is recorded outside the repository too, so one written inside it
400
+ # cannot retire rows this machine never retired (#50).
401
+ if sealed is not None or previous.get("sealed"):
402
+ mark["sealed"] = sealed if sealed is not None else previous["sealed"]
403
+ # A seal under way is settled before any append; kept, one an append met anyway is refused, not lost.
404
+ if previous.get("intent"):
405
+ mark["intent"] = previous["intent"]
406
+ return mark
407
+ _update_checkpoint(path, change, identity=identity)
408
+
409
+
410
+ def _record_append_intent(path, digest, *, identity=None):
411
+ """Record the digest of the row an append is about to write, cleared when its count is recorded (LEDGER-ATTESTED)."""
412
+ _update_checkpoint(path, lambda previous: dict(previous, append=digest), identity=identity)
413
+
414
+
415
+ def _clear_append_intent(path, *, identity=None):
416
+ """Clear a write-ahead append intent once the uncommitted row has been taken back (LEDGER-ATTESTED)."""
417
+ _update_checkpoint(path, lambda previous: {k: v for k, v in previous.items() if k != "append"}, identity=identity)
418
+
419
+
420
+ def _check_committed_length(path, rows, chain, seal=None, *, identity=None):
421
+ """Refuse a ledger shorter than its recorded length, or diverging from it (#62), across a seal (#50)."""
422
+ mark = _load_committed_lengths().get(identity or os.path.realpath(path))
423
+ if not isinstance(mark, dict) or not isinstance(mark.get("count"), int) or mark["count"] <= 0:
424
+ return
425
+ recorded = mark.get("sealed")
426
+ if recorded or seal:
427
+ same = bool(recorded and seal) and recorded.get("retired") == seal.get("retired") \
428
+ and recorded.get("digest") == seal.get("digest")
429
+ if not same:
430
+ raise LedgerCorruption(
431
+ f"{path} carries a seal this machine did not record, or lost the one it did "
432
+ f"(the record is kept in {checkpoint_path()})"
433
+ )
434
+ retired = seal["retired"] if seal else 0
435
+ count = mark["count"]
436
+ if retired + len(rows) < count:
437
+ raise LedgerCorruption(
438
+ f"{path} holds {retired + len(rows)} committed rows but recorded {count}: rows were removed from its end"
439
+ f" (the count is kept in {checkpoint_path()})"
440
+ )
441
+ if count <= retired:
442
+ if count == retired and seal.get("digest") == mark.get("digest"):
443
+ return
444
+ raise LedgerCorruption(f"{path} row {count} is sealed but does not match the digest recorded for it")
445
+ if chained_row_digest(rows[count - retired - 1], chain) != mark.get("digest"):
446
+ raise LedgerCorruption(
447
+ f"{path} row {count} does not match the digest recorded for it in {checkpoint_path()}"
448
+ )
449
+
450
+
451
+ def seal_path(path):
452
+ return path + ".seal.json"
453
+
454
+
455
+ def _check_missing_chained_ledger(path, chain, seal=None, *, identity=None):
456
+ """A file missing where a checkpoint or seal records rows is unreadable, never an empty history."""
457
+ _check_committed_length(path, [], chain, seal=seal, identity=identity)
458
+ entry = _checkpoint_entry(path, identity=identity)
459
+ if seal or entry.get("sealed") or "intent" in entry or \
460
+ (isinstance(entry.get("count"), int) and entry["count"] > 0):
461
+ raise LedgerCorruption(f"{path} is absent but its checkpoint or seal records rows: rows were removed; "
462
+ "restore the ledger before reading its history")
463
+
464
+
465
+ def _load_seal(path, *, _folder=None):
466
+ """The seal that retired a ledger's prefix, or None (#50)."""
467
+ try:
468
+ if _folder is None:
469
+ with open(seal_path(path), encoding="utf-8") as handle:
470
+ seal = json.load(handle)
471
+ else:
472
+ data = _tree_bytes(_folder, os.path.basename(seal_path(path)))
473
+ if data is None:
474
+ return None
475
+ seal = json.loads(data.decode(UTF8))
476
+ except FileNotFoundError:
477
+ return None
478
+ except (OSError, ValueError) as exc:
479
+ raise LedgerCorruption(f"the seal of {path} cannot be read: {exc}") from exc
480
+ if not isinstance(seal, dict) or not isinstance(seal.get("retired"), int) or seal["retired"] < 1 \
481
+ or not isinstance(seal.get("digest"), str):
482
+ raise LedgerCorruption(f"the seal of {path} is malformed")
483
+ return seal
484
+
485
+
486
+ def _seal_intent(path, entry):
487
+ """The seal of `path` under way that this machine recorded in `entry`, its checkpoint record, or None."""
488
+ intent = entry.get("intent")
489
+ if "intent" not in entry:
490
+ return None
491
+ before = intent.get("before") if isinstance(intent, dict) else None
492
+ if not isinstance(intent, dict) or not isinstance(intent.get("retired"), int) or intent["retired"] < 1 \
493
+ or not isinstance(intent.get("digest"), str) or not isinstance(intent.get("archives"), list) \
494
+ or not isinstance(before, dict) or not isinstance(before.get("count"), int) \
495
+ or before["count"] <= intent["retired"] \
496
+ or not isinstance(before.get("digest"), str):
497
+ raise LedgerCorruption(f"the seal of {path} under way, as {checkpoint_path()} records it, is malformed: "
498
+ "nothing is read from the ledger until a person compares the two")
499
+ return intent
500
+
501
+
502
+ def _settle_seal(path, chain, previous_field, legacy_prefix=False, *, _folder=None, identity=None):
503
+ """Finish or undo a seal a crash interrupted, from the intent this machine recorded; the caller holds the ledger lock (#50).
504
+
505
+ The intent was read from `<ledger>.seal.json.pending`, in the tree the implementer writes: a
506
+ ledger cut to its newest rows, beside an intent naming the cut rows as sealed, was finished
507
+ as a seal and the record outside the repository rewritten to match, so the cut, or a whole
508
+ chain of the writer's own with any genesis, read as authority (the review of BOUNDED-STORES).
509
+ A seal now records its intent in the checkpoint store, under that store's lock, before it
510
+ rewrites the ledger, and only that intent is settled: finished when the ledger starts after
511
+ the rows it retires, holds exactly the rows it kept and the archives hold exactly those it
512
+ retired; undone when the ledger is still the one it was taken from; refused otherwise. polyctl
513
+ writes no intent in the tree, so a `.pending` found there is none of its own: it is refused,
514
+ named and left where it is (LEDGER-HISTORY). The opt-in reader uses the same settlement,
515
+ with every tree operation relative to its held folder and its checkpoint identity fixed
516
+ before the folder can move (LEADS-ADAPTERS).
517
+ """
518
+ stray = seal_path(path) + ".pending"
519
+ if _folder is None:
520
+ stray_exists = os.path.lexists(stray)
521
+ else:
522
+ try:
523
+ _tree_stat(_folder, os.path.basename(stray))
524
+ stray_exists = True
525
+ except FileNotFoundError:
526
+ stray_exists = False
527
+ if stray_exists:
528
+ raise LedgerCorruption(
529
+ f"{stray} names a seal of {os.path.basename(path)} that this machine did not begin: polyctl records a seal's "
530
+ f"intent in {checkpoint_path()}, never in the tree. It is left where it is and the ledger is not read "
531
+ "while it stands; read it, then move it out of .polyctl/ledger/"
532
+ )
533
+ entry = _checkpoint_entry(path, identity=identity)
534
+ intent = _seal_intent(path, entry)
535
+ if intent is None:
536
+ return
537
+ before = intent["before"]
538
+ if (entry.get("count"), entry.get("digest")) != (before["count"], before["digest"]):
539
+ raise LedgerCorruption(
540
+ f"the record of {path} in {checkpoint_path()} changed while a seal of it was under way: nothing is read "
541
+ "from the ledger until a person compares the two"
542
+ )
543
+ if intent.get("chain") != chain:
544
+ raise LedgerCorruption(f"the seal of {path} under way was begun for another chain than {chain!r}")
545
+ if _folder is None:
546
+ rows = _read_jsonl_unlocked(path, allow_partial_tail=False) if os.path.exists(path) else []
547
+ else:
548
+ rows = _jsonl_rows(_tree_bytes(_folder, os.path.basename(path)) or b"", path, allow_partial_tail=False)
549
+ if rows and isinstance(rows[0], dict) and rows[0].get(previous_field) == intent["digest"]:
550
+ # Rewritten: what stands must be the newest rows the intent kept, and the archives the rows it retired.
551
+ seal = {key: intent.get(key) for key in ("chain", "retired", "digest", "archives", "at")}
552
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
553
+ if intent["retired"] + len(rows) != before["count"] or chained_row_digest(rows[-1], chain) != before["digest"]:
554
+ raise LedgerCorruption(
555
+ f"{path} was rewritten by a seal and no longer holds the rows it kept, through row {before['count']}: "
556
+ "rows were changed or removed after it"
557
+ )
558
+ _, archived = _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix, _folder=_folder)
559
+ data = (json.dumps(seal, sort_keys=True) + "\n").encode(UTF8)
560
+ if _folder is None:
561
+ replace_file_durably(seal_path(path), data)
562
+ else:
563
+ _tree_replace_bytes(_folder, os.path.basename(seal_path(path)), data)
564
+
565
+ def finished(record):
566
+ record.pop("intent", None)
567
+ record.update(count=before["count"], digest=before["digest"], at=int(time.time()),
568
+ sealed={"retired": intent["retired"], "digest": intent["digest"], "archives": archived})
569
+ return record
570
+ _update_checkpoint(path, finished, identity=identity)
571
+ return
572
+ # Not rewritten: the ledger must still be the one the seal was taken from.
573
+ seal = _load_seal(path, _folder=_folder)
574
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
575
+ held = (seal["retired"] if seal else 0) + len(rows)
576
+ if held != before["count"] or not rows or chained_row_digest(rows[-1], chain) != before["digest"]:
577
+ raise LedgerCorruption(
578
+ f"{path} changed while a seal of it was under way: it holds {held} rows where the seal found "
579
+ f"{before['count']}"
580
+ )
581
+ _check_committed_length(path, rows, chain, seal=seal, identity=identity)
582
+ _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix, _folder=_folder)
583
+
584
+ def undone(record):
585
+ record.pop("intent", None)
586
+ return record
587
+ _update_checkpoint(path, undone, identity=identity)
588
+
589
+
590
+ def seal_chained_jsonl(path, chain, keep, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False,
591
+ _checkpoint=None):
592
+ """Retire all but the newest `keep` rows of a chained ledger into an archive, never deleting one (#50).
593
+
594
+ The retired rows are written whole to `sealed/<ledger>.<first>-<last>.jsonl`
595
+ beside the ledger. The seal names how many rows it retired and the digest of
596
+ the last of them, which becomes the genesis the first kept row links to, and
597
+ the same seal is recorded outside the repository with the committed length.
598
+ A sealed ledger and the unsealed one give every reader the same newest rows.
599
+ Returns the seal, or None when there is nothing to retire.
600
+
601
+ Before the ledger is rewritten, the seal's intent - what it retires, the digest of
602
+ the last of those, its archives, and the length and newest digest the ledger had -
603
+ is recorded in the checkpoint store under that store's lock, and a seal cut short
604
+ is settled from that record alone (LEDGER-HISTORY). `_checkpoint` is the private
605
+ test seam the appends take.
606
+ """
607
+ if keep < 1:
608
+ raise ValueError("a seal keeps at least one row")
609
+ with _exclusive_lock(path + ".lock"):
610
+ _settle_seal(path, chain, previous_field, legacy_prefix)
611
+ if not os.path.exists(path):
612
+ _check_missing_chained_ledger(path, chain, seal=_load_seal(path))
613
+ return None
614
+ rows = _read_jsonl_unlocked(path, allow_partial_tail=False)
615
+ seal = _load_seal(path)
616
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
617
+ _check_committed_length(path, rows, chain, seal=seal)
618
+ if len(rows) <= keep:
619
+ return None
620
+ retire, kept = rows[:-keep], rows[-keep:]
621
+ if any(not isinstance(row, dict) or previous_field not in row for row in kept):
622
+ return None # rows from before the chain cannot start a sealed ledger
623
+ # A seal adds its archive to ones that hold: stacked on a broken one, it would carry the break on.
624
+ _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix)
625
+ base = seal["retired"] if seal else 0
626
+ first, last = base + 1, base + len(retire)
627
+ archive = os.path.join(os.path.dirname(path) or ".", "sealed", f"{os.path.basename(path)}.{first}-{last}.jsonl")
628
+ replace_file_durably(archive, b"".join(_encode_jsonl(row) for row in retire))
629
+ intent = {"chain": chain, "retired": last, "digest": chained_row_digest(retire[-1], chain),
630
+ "archives": list((seal or {}).get("archives") or []) + [os.path.basename(archive)],
631
+ "at": int(time.time()),
632
+ "before": {"count": base + len(rows), "digest": chained_row_digest(rows[-1], chain)}}
633
+
634
+ def begun(record):
635
+ record.update(count=intent["before"]["count"], digest=intent["before"]["digest"], at=intent["at"],
636
+ intent=intent)
637
+ return record
638
+ _update_checkpoint(path, begun)
639
+ _call(_checkpoint, "intent-recorded")
640
+ replace_file_durably(path, b"".join(_encode_jsonl(row) for row in kept))
641
+ _call(_checkpoint, "ledger-rewritten")
642
+ _settle_seal(path, chain, previous_field, legacy_prefix)
643
+ return _load_seal(path)
644
+
645
+
646
+ def _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix, recorded=None, *, _folder=None):
647
+ """(the rows `seal` retired, oldest first, the digest of its archives' bytes), held to the seal (LEDGER-HISTORY).
648
+
649
+ The archives were read as they stood: a row edited in one was returned as it was, and a reader
650
+ took its id for one polyctl had recorded (the review of BOUNDED-STORES). They must now be one chain,
651
+ from the ledger's first row to the seal's digest, and hold exactly the rows it retired, or
652
+ LedgerCorruption names what does not hold. Archives whose bytes are those this machine held to
653
+ the seal when it sealed them - `recorded`, the digest the checkpoint store keeps - are not
654
+ walked link by link again: a reader of a long history pays for parsing it, not for its chain.
655
+ """
656
+ if not seal:
657
+ return [], None
658
+ folder = os.path.join(os.path.dirname(path) or ".", "sealed")
659
+ names = seal.get("archives")
660
+ if not isinstance(names, list) or not names \
661
+ or not all(isinstance(name, str) and name and os.path.basename(name) == name for name in names):
662
+ raise LedgerCorruption(f"the seal of {path} names no archives polyctl can read")
663
+ parts, whole = [], hashlib.sha256()
664
+ for name in names:
665
+ archive = os.path.join(folder, name)
666
+ try:
667
+ if _folder is None:
668
+ with open(archive, "rb") as handle:
669
+ data = handle.read()
670
+ else:
671
+ with _tree_folder(_folder, ["sealed"], make=False) as held:
672
+ data = _tree_bytes(held, name) if held is not None else None
673
+ if data is None:
674
+ raise FileNotFoundError(archive)
675
+ except OSError as exc:
676
+ raise LedgerCorruption(f"{archive}, which the seal of {path} names, cannot be read: {exc}") from exc
677
+ whole.update(data)
678
+ parts.append((archive, data))
679
+ digest = "sha256:" + whole.hexdigest()
680
+ rows, expected = [], None
681
+ for archive, data in parts:
682
+ part = _jsonl_rows(data, archive, allow_partial_tail=False)
683
+ if digest != recorded:
684
+ _validate_chained_rows(archive, part, chain, previous_field, legacy_prefix and not rows,
685
+ seal={"digest": expected, "retired": len(rows)} if rows else None)
686
+ if part:
687
+ expected = chained_row_digest(part[-1], chain)
688
+ rows.extend(part)
689
+ if len(rows) != seal["retired"] or not rows or chained_row_digest(rows[-1], chain) != seal["digest"]:
690
+ raise LedgerCorruption(
691
+ f"the archives of {path} hold {len(rows)} rows ending at "
692
+ f"{chained_row_digest(rows[-1], chain) if rows else 'none'}, where its seal retired {seal['retired']} "
693
+ f"ending at {seal['digest']}"
694
+ )
695
+ return rows, digest
696
+
697
+
698
+ def sealed_rows(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, seal=None):
699
+ """Every row a seal retired from a ledger, oldest first, read from its archives and held to its seal (#50).
700
+
701
+ `seal` is the ledger's own when not given. The caller holds the ledger's lock, or reads a copy
702
+ no one appends to; `read_chained_history` reads the whole history under the lock (LEDGER-HISTORY).
703
+ """
704
+ # Snapshot readers do not settle or write. They must refuse an unfinished seal rather than measure half of it.
705
+ stray = seal_path(path) + ".pending"
706
+ if os.path.lexists(stray):
707
+ raise LedgerCorruption(f"{stray} names a seal this machine did not begin: polyctl records its intent in "
708
+ f"{checkpoint_path()}, never in the tree; the ledger is not read while it stands")
709
+ if "intent" in _checkpoint_entry(path):
710
+ raise LedgerCorruption(f"the seal of {path} awaits settlement; read the ledger under its lock before "
711
+ "reading its archives")
712
+ seal = seal if seal is not None else _load_seal(path)
713
+ if not os.path.exists(path):
714
+ _check_missing_chained_ledger(path, chain, seal=seal)
715
+ return _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix)[0]
716
+
717
+
718
+ def _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix=False, seal=None):
719
+ expected = None
720
+ start = 0
721
+ offset = 0
722
+ if seal is not None:
723
+ # A sealed ledger's first row links to the last row the seal retired (#50).
724
+ expected, offset = seal["digest"], seal["retired"]
725
+ elif legacy_prefix:
726
+ # Rows written before a ledger was chained carry no link. They may only
727
+ # come first, and the first linked row names the digest of the last one.
728
+ while start < len(rows) and isinstance(rows[start], dict) \
729
+ and previous_field not in rows[start]:
730
+ start += 1
731
+ if start:
732
+ try:
733
+ expected = chained_row_digest(rows[start - 1], chain)
734
+ except (TypeError, ValueError) as exc:
735
+ raise LedgerCorruption(
736
+ f"cannot digest chained JSONL record {start} in {path}: {exc}"
737
+ ) from exc
738
+ for index, row in enumerate(rows[start:], start + 1 + offset):
739
+ if not isinstance(row, dict):
740
+ raise LedgerCorruption(
741
+ f"chained JSONL record {index} in {path} is not an object"
742
+ )
743
+ if previous_field not in row:
744
+ raise LedgerCorruption(
745
+ f"chained JSONL record {index} in {path} has no {previous_field!r} field"
746
+ )
747
+ actual = row.get(previous_field)
748
+ if actual != expected:
749
+ raise LedgerCorruption(
750
+ f"broken hash chain at record {index} in {path}: "
751
+ f"expected {expected!r}, got {actual!r}"
752
+ )
753
+ if "ordinal" in row and row.get("ordinal") != index:
754
+ raise LedgerCorruption(
755
+ f"chained JSONL record {index} in {path} carries ordinal {row.get('ordinal')!r}"
756
+ )
757
+ try:
758
+ expected = chained_row_digest(row, chain)
759
+ except (TypeError, ValueError) as exc:
760
+ raise LedgerCorruption(
761
+ f"cannot digest chained JSONL record {index} in {path}: {exc}"
762
+ ) from exc
763
+ return rows
764
+
765
+
766
+ # The gate ledgers: every chained ledger a gate decision reads, defined once for the attested-prefix rule and
767
+ # `polyctl doctor` (LEDGER-ATTESTED).
768
+ GATE_LEDGERS = (
769
+ ("opt-ins", ".polyctl/ledger/opt-ins.jsonl", "polyctl-opt-in-row-v1", False, "no opt-in counts"),
770
+ ("authority", ".polyctl/ledger/authority.jsonl", "polyctl-authority-row-v1", False, "no grant counts"),
771
+ ("verifications", ".polyctl/ledger/verifications.jsonl", "polyctl-verification-row-v1", True, "no verification counts"),
772
+ ("waivers", ".polyctl/ledger/waivers.jsonl", "polyctl-waiver-row-v1", True, "no waiver counts"),
773
+ ("reviews", ".polyctl/ledger/reviews.jsonl", "polyctl-review-row-v1", False, "no review counts"),
774
+ ("decisions", ".polyctl/ledger/decisions.jsonl", "polyctl-decision-row-v1", True, "no decision counts"),
775
+ ("plans", ".polyctl/ledger/plans.jsonl", "polyctl-plan-row-v1", True, "no plan baseline counts"),
776
+ )
777
+ GATE_CHAINS = frozenset(chain for _, _, chain, _, _ in GATE_LEDGERS)
778
+ GATE_LEDGER_FILES = frozenset(os.path.basename(rel) for _, rel, _, _, _ in GATE_LEDGERS)
779
+
780
+
781
+ def _is_gate_ledger(path, chain):
782
+ return chain in GATE_CHAINS or os.path.basename(path) in GATE_LEDGER_FILES
783
+
784
+
785
+ def read_chained_jsonl(path, chain, allow_partial_tail=True, timeout=10.0, *,
786
+ previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, lock=True, attested=False,
787
+ for_append=False):
788
+ """Read and validate every committed row in one predecessor hash chain.
789
+
790
+ The chain proves no row was changed or inserted; the recorded length proves
791
+ none was cut off the end, including all of them (#62), and that for every
792
+ gate ledger (`GATE_LEDGERS`) every row returned is one this machine's
793
+ checkpoint store records polyctl appending (LEDGER-ATTESTED).
794
+ """
795
+ if _is_gate_ledger(path, chain) and _ledger_store_in_tree(path):
796
+ if for_append:
797
+ raise LedgerCorruption(
798
+ f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
799
+ "no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
800
+ return []
801
+ if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
802
+ and "intent" not in _checkpoint_entry(path):
803
+ _check_missing_chained_ledger(path, chain, seal=_load_seal(path))
804
+ return []
805
+ if not lock:
806
+ return _read_chained_snapshot(
807
+ path, chain, allow_partial_tail, previous_field, legacy_prefix, attested=attested
808
+ )[0]
809
+ with _exclusive_lock(path + ".lock", timeout=timeout):
810
+ return _read_chained_unlocked(
811
+ path, chain, allow_partial_tail, previous_field, legacy_prefix, for_append=for_append
812
+ )[0]
813
+
814
+
815
+ def _read_chained_unlocked(path, chain, allow_partial_tail, previous_field, legacy_prefix, *, every=False,
816
+ for_append=False):
817
+ """(the live rows, the seal they follow), validated; the caller holds the lock."""
818
+ gate = _is_gate_ledger(path, chain)
819
+ if gate and _ledger_store_in_tree(path):
820
+ if for_append:
821
+ raise LedgerCorruption(
822
+ f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
823
+ "no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
824
+ return (0 if every else []), None
825
+ _settle_seal(path, chain, previous_field, legacy_prefix)
826
+ seal = _load_seal(path)
827
+ if not os.path.exists(path):
828
+ _check_missing_chained_ledger(path, chain, seal=seal)
829
+ return (0 if every else []), seal
830
+ with open(path, "rb") as handle:
831
+ data = handle.read()
832
+ records, unfinished = _jsonl_records(data)
833
+ if every:
834
+ return len(records), seal
835
+ attested = _attested_length(path, seal) if gate else len(records)
836
+ if unfinished is not None and not allow_partial_tail:
837
+ raise LedgerCorruption(f"uncommitted final record {unfinished + 1} in {path}")
838
+ # For a gate ledger, only the rows polyctl recorded are read and checked: a tail past them - well formed or not - is
839
+ # nobody's record, and never makes polyctl's own unreadable (OPTIN-ATTESTED, LEDGER-ATTESTED).
840
+ rows = [_jsonl_row(body, index, path) for index, body in records[:attested]]
841
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
842
+ _check_committed_length(path, rows, chain, seal=seal)
843
+ if for_append and gate:
844
+ past = len(records) - len(rows)
845
+ if past > 0 and not _is_interrupted_append(records, len(rows), path, chain, unfinished=unfinished):
846
+ raise LedgerCorruption(f"{path}: {_unattested_repair(past, len(rows))}")
847
+ return rows, seal
848
+
849
+
850
+ def _read_chained_snapshot(path, chain, allow_partial_tail, previous_field, legacy_prefix, *, attested=False):
851
+ """(the live rows, the seal they follow, unattested info), validated without locking or settling a seal (EXPORT-OTLP)."""
852
+ stray = seal_path(path) + ".pending"
853
+ if os.path.lexists(stray):
854
+ raise LedgerCorruption(
855
+ f"{stray} names a seal of {os.path.basename(path)} that this machine did not begin: polyctl records a seal's "
856
+ f"intent in {checkpoint_path()}, never in the tree. It is left where it is and the ledger is not read "
857
+ "while it stands; read it, then move it out of .polyctl/ledger/"
858
+ )
859
+ entry = _checkpoint_entry(path)
860
+ if "intent" in entry:
861
+ raise LedgerCorruption(f"the seal of {path} awaits settlement; read the ledger under its lock before "
862
+ "reading a snapshot")
863
+ seal = _load_seal(path)
864
+ if not os.path.exists(path):
865
+ _check_missing_chained_ledger(path, chain, seal=seal)
866
+ return [], seal, None
867
+ rows = _read_jsonl_unlocked(path, allow_partial_tail)
868
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
869
+ _check_committed_length(path, rows, chain, seal=seal)
870
+ retired = seal["retired"] if seal else 0
871
+ total = retired + len(rows)
872
+ count = entry.get("count")
873
+ has_count = isinstance(count, int) and not isinstance(count, bool) and count > 0
874
+ unattested = None
875
+ if has_count:
876
+ if total > count:
877
+ extra = total - count
878
+ msg = (
879
+ f"{os.path.basename(path)} ({path}) holds {total} row(s), {extra} past its recorded count of {count} "
880
+ f"in {checkpoint_path()} (no checkpoint here for the trailing row(s): recorded elsewhere, reset, or "
881
+ "an append still under way); pass --attested to export only the attested part"
882
+ )
883
+ if not attested:
884
+ raise LedgerCorruption(msg)
885
+ rows = rows[:max(0, count - retired)]
886
+ unattested = {"ledger": os.path.basename(path), "reason": "rows_past_checkpoint", "message": msg}
887
+ elif total > 0 or seal is not None:
888
+ msg = (
889
+ f"{os.path.basename(path)} ({path}) holds {total} row(s) with no checkpoint here: moved, reset, "
890
+ f"recorded elsewhere, older than checkpoints (checkpoints are kept in {checkpoint_path()}); "
891
+ "pass --attested to export only the attested part"
892
+ )
893
+ if not attested:
894
+ raise LedgerCorruption(msg)
895
+ rows = []
896
+ seal = None
897
+ unattested = {"ledger": os.path.basename(path), "reason": "no_checkpoint", "message": msg}
898
+ else:
899
+ rows = []
900
+ return rows, seal, unattested
901
+
902
+
903
+ def _read_chained_history_snapshot(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False,
904
+ attested=False):
905
+ """(rows, unattested) for a lockless snapshot of a chained ledger's full history (EXPORT-OTLP)."""
906
+ if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
907
+ and "intent" not in _checkpoint_entry(path):
908
+ _check_missing_chained_ledger(path, chain, seal=_load_seal(path))
909
+ return [], None
910
+ rows, seal, unattested = _read_chained_snapshot(
911
+ path, chain, True, previous_field, legacy_prefix, attested=attested
912
+ )
913
+ recorded = _checkpoint_entry(path).get("sealed")
914
+ retired, _ = _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix,
915
+ recorded=recorded.get("archives") if isinstance(recorded, dict) else None)
916
+ return retired + rows, unattested
917
+
918
+
919
+ def read_chained_history(path, chain, timeout=10.0, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False,
920
+ lock=True, attested=False):
921
+ """Every row a chained ledger ever committed, oldest first: those its seal retired, then the live ones (LEDGER-HISTORY).
922
+
923
+ One chain, read under the ledger's lock: the archives end at the seal's digest with exactly
924
+ the rows it retired, and the live rows go on from it as `read_chained_jsonl` holds them. A
925
+ reader that asks what was ever granted, verified or waived reads this; one that asks for the
926
+ newest decision reads the live rows. Read from the live rows alone, a seal made a spent waiver
927
+ look unused and moved the first grant the landed-tree check starts from (the review of
928
+ BOUNDED-STORES). `lock=False` reads a snapshot without creating a `.lock` file or settling a
929
+ seal (EXPORT-OTLP).
930
+ """
931
+ gate = _is_gate_ledger(path, chain)
932
+ if gate and _ledger_store_in_tree(path):
933
+ return []
934
+ if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
935
+ and "intent" not in _checkpoint_entry(path):
936
+ _check_missing_chained_ledger(path, chain, seal=_load_seal(path))
937
+ return []
938
+ if not lock:
939
+ return _read_chained_history_snapshot(
940
+ path, chain, previous_field=previous_field, legacy_prefix=legacy_prefix, attested=attested
941
+ )[0]
942
+ with _exclusive_lock(path + ".lock", timeout=timeout):
943
+ rows, seal = _read_chained_unlocked(path, chain, True, previous_field, legacy_prefix)
944
+ recorded = _checkpoint_entry(path).get("sealed")
945
+ retired, _ = _sealed_rows_held(path, chain, seal if (not gate or isinstance(recorded, dict)) else None,
946
+ previous_field, legacy_prefix,
947
+ recorded=recorded.get("archives") if isinstance(recorded, dict) else None)
948
+ return retired + rows
949
+
950
+
951
+ def unattested_ledger(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False):
952
+ """(rows this machine's checkpoint store does not record polyctl appending, rows it does) of a chained ledger
953
+ (LEDGER-ATTESTED)."""
954
+ if _ledger_store_in_tree(path):
955
+ if not os.path.exists(path):
956
+ return 0, 0
957
+ with _exclusive_lock(path + ".lock"):
958
+ with open(path, "rb") as handle:
959
+ return len(_jsonl_records(handle.read())[0]), 0
960
+ if not os.path.exists(path) and not os.path.lexists(seal_path(path) + ".pending") \
961
+ and "intent" not in _checkpoint_entry(path):
962
+ _check_missing_chained_ledger(path, chain, seal=_load_seal(path))
963
+ return 0, 0
964
+ with _exclusive_lock(path + ".lock"):
965
+ attested = len(_read_chained_unlocked(path, chain, True, previous_field, legacy_prefix)[0])
966
+ total = _read_chained_unlocked(path, chain, True, previous_field, legacy_prefix, every=True)[0]
967
+ return max(0, total - attested), attested
968
+
969
+
970
+ def attest_restored_ledger(path, chain, *, previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, record=True):
971
+ """Validate a restored chained ledger (and its seal and archives) and, when `record=True`, record this machine's
972
+ checkpoint for it so its rows count; returns `(count, digest, already_attested)` (LEDGER-ATTESTED)."""
973
+ if _ledger_store_in_tree(path):
974
+ raise LedgerCorruption(
975
+ f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
976
+ "no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
977
+ with _exclusive_lock(path + ".lock"):
978
+ stray = seal_path(path) + ".pending"
979
+ if os.path.lexists(stray):
980
+ raise LedgerCorruption(f"{stray} names an unfinished seal in the restored backup")
981
+ seal = _load_seal(path)
982
+ _, archived = _sealed_rows_held(path, chain, seal, previous_field, legacy_prefix)
983
+ rows = _read_jsonl_unlocked(path, allow_partial_tail=False) if os.path.exists(path) else []
984
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
985
+ retired = seal["retired"] if seal else 0
986
+ count = retired + len(rows)
987
+ digest = chained_row_digest(rows[-1], chain) if rows else (seal["digest"] if seal else None)
988
+ sealed = {"retired": seal["retired"], "digest": seal["digest"], "archives": archived} if seal else None
989
+ entry = _checkpoint_entry(path)
990
+ attested = (count == 0 or (entry.get("count") == count and entry.get("digest") == digest
991
+ and entry.get("sealed") == sealed and "intent" not in entry))
992
+ if record and count > 0:
993
+ mark = {"count": count, "digest": digest, "at": int(time.time())}
994
+ if sealed is not None:
995
+ mark["sealed"] = sealed
996
+ _update_checkpoint(path, lambda _: mark)
997
+ attested = True
998
+ return count, digest, attested
999
+
1000
+
1001
+
1002
+ def _sync_directory(directory, fsync):
1003
+ """Persist a newly-created directory entry where the platform supports it."""
1004
+ if os.name == "nt":
1005
+ return False
1006
+ flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
1007
+ fd = os.open(directory or ".", flags)
1008
+ try:
1009
+ return _sync_folder_held(fd, fsync)
1010
+ finally:
1011
+ os.close(fd)
1012
+
1013
+
1014
+ def _sync_folder_held(fd, fsync):
1015
+ """Persist the entries of a directory held open as `fd`; False where its file system cannot."""
1016
+ try:
1017
+ fsync(fd)
1018
+ except OSError as exc:
1019
+ # Some filesystems expose directories but reject fsync. This is a
1020
+ # platform capability limit, not evidence that data was persisted.
1021
+ if exc.errno in (errno.EINVAL, errno.ENOTSUP, getattr(errno, "EOPNOTSUPP", -1)):
1022
+ return False
1023
+ raise
1024
+ return True
1025
+
1026
+
1027
+ class ReplacedNotDurable(OSError):
1028
+ """The replacement happened, but syncing its directory failed (TRIAGE-3-LOW)."""
1029
+
1030
+
1031
+ def _write_all(fd, data):
1032
+ """Write every byte of `data` to `fd`, or raise."""
1033
+ remaining = memoryview(data)
1034
+ while remaining:
1035
+ try:
1036
+ count = os.write(fd, remaining)
1037
+ except InterruptedError:
1038
+ continue
1039
+ if count <= 0:
1040
+ raise OSError(errno.EIO, "write made no forward progress")
1041
+ remaining = remaining[count:]
1042
+
1043
+
1044
+ def replace_file_durably(path, data, *, mode=None, before_replace=None, _checkpoint=None, _fsync=None):
1045
+ """Replace a whole file so it is only ever seen with its old bytes or its new ones.
1046
+
1047
+ ``open(path, "w")`` truncates first. A crash, kill or full disk before the new
1048
+ bytes are down leaves a zero-byte file, and a zero-byte ``.polyctl/config.json`` no
1049
+ longer holds the ``capability_matrix`` that keeps strict authority on (#56).
1050
+ This writes a temporary file beside the target, fsyncs it, renames it over the
1051
+ target and fsyncs the directory. ``_checkpoint`` and ``_fsync`` are the private
1052
+ test seams the ledger appends take.
1053
+
1054
+ ``before_replace`` is called once the new bytes are durable beside the target and
1055
+ right before the rename: the last moment a caller that must not replace what
1056
+ someone changed since its reading can look, and what it raises leaves the target
1057
+ as it was and takes the temporary file away (SLICE-LIFECYCLE-2).
1058
+
1059
+ ``mode`` is for a file that holds a credential. Its temporary file is created
1060
+ with that mode, under a name no file had (``O_EXCL``, and ``O_NOFOLLOW`` where
1061
+ the platform has it), so the bytes are never readable by anyone the mode leaves
1062
+ out - not while they are written, and not when a write is cut short - and a file
1063
+ or a link already at that name is never written through. Written in place and
1064
+ chmod-ed after, ``~/.polyctl/email.json`` was readable by every user of the machine
1065
+ meanwhile, and stayed so when the process stopped between the two
1066
+ (HUNT-LEADS-SEC). Without ``mode`` the file keeps the mode it had, or 0644.
1067
+
1068
+ Every temporary file is made that way, with or without ``mode``. Without one it
1069
+ was ``.<name>.<pid>.tmp``, opened to truncate and through any link: an agent that
1070
+ left a link at that name for a likely pid in a folder polyctl writes - ``.polyctl/`` among
1071
+ them - had polyctl truncate and write the link's target, a file of the person's
1072
+ (STORAGE-TMP).
1073
+ """
1074
+ fsync = _fsync or os.fsync
1075
+ parent = os.path.dirname(path) or "."
1076
+ os.makedirs(parent, exist_ok=True)
1077
+ if mode is None:
1078
+ try:
1079
+ mode = os.stat(path).st_mode & 0o7777
1080
+ except OSError:
1081
+ mode = 0o644
1082
+ temporary = os.path.join(parent, f".{os.path.basename(path)}.{os.getpid()}.{os.urandom(4).hex()}.tmp")
1083
+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_BINARY", 0)
1084
+ opened = False
1085
+ try:
1086
+ fd = os.open(temporary, flags, mode)
1087
+ opened = True
1088
+ try:
1089
+ _call(_checkpoint, "temporary-opened")
1090
+ _write_all(fd, data)
1091
+ _call(_checkpoint, "temporary-written")
1092
+ fsync(fd)
1093
+ _call(_checkpoint, "temporary-fsynced")
1094
+ finally:
1095
+ os.close(fd)
1096
+ if before_replace is not None:
1097
+ before_replace()
1098
+ _replace(temporary, path)
1099
+ except BaseException:
1100
+ # Only a file this call made goes: what stood at an exclusive name before it is not this call's.
1101
+ if opened:
1102
+ try:
1103
+ os.remove(temporary)
1104
+ except OSError:
1105
+ pass
1106
+ raise
1107
+ _call(_checkpoint, "replaced")
1108
+ try:
1109
+ synced = _sync_directory(parent, fsync)
1110
+ except OSError as exc:
1111
+ raise ReplacedNotDurable(f"{path} changed, but durability could not be confirmed ({exc}); "
1112
+ "read the file and retry after fixing the storage error") from exc
1113
+ if synced:
1114
+ _call(_checkpoint, "directory-fsynced")
1115
+
1116
+
1117
+ # ---- a file in a project's tree, written and read through no link (LEADS-ADAPTERS) -------------------------------
1118
+ #
1119
+ # `polyctl init` writes into a tree an agent writes too, and it wrote by path: each folder on the way was whatever stood
1120
+ # there as the file was opened. A `.polyctl` or an `agent-mail` that was a link - or became one between init's look and its
1121
+ # write - took the configuration, the board, the playbook and the manifest out of the project, and a `.polyctl/PLAYBOOK.md`
1122
+ # or an MCP file that was a link had polyctl's text added to the file of the person's it led to. Where the platform opens a
1123
+ # file relative to a folder it holds open, as POSIX does, each folder is opened so from the project's root down, never
1124
+ # through a link, and the file is made, replaced or read in the last of them: a link put on the way meanwhile moves
1125
+ # nothing. Windows holds every folder handle without delete sharing, so no folder can be renamed or replaced while
1126
+ # its path is used; every handle is opened as the reparse point itself and reparse points are refused.
1127
+
1128
+ class InTheWay(OSError):
1129
+ """What stands where a file is written or read in a project's tree and is not gone through: a link or a file where
1130
+ a folder goes, a link at the file's own name, or no regular file. `where` is its path in the tree (LEADS-ADAPTERS).
1131
+ """
1132
+
1133
+ def __init__(self, where, what):
1134
+ super().__init__(f"{where} {what}")
1135
+ self.where = where
1136
+
1137
+
1138
+ _A_FOLDER = "is a link or a file where a folder of the project's own goes"
1139
+ _A_LINK = "is a link, and polyctl reads and writes through none"
1140
+ _NO_FILE = "is no regular file"
1141
+ _WINDOWS_LINK_TAG = 0x20000000 # the name-surrogate bit of a reparse tag: a symbolic link's, a junction's
1142
+
1143
+
1144
+ def folders_held_open():
1145
+ """Whether this platform opens a file relative to a folder it holds open, as POSIX does and Python on Windows does
1146
+ not (LEADS-ADAPTERS)."""
1147
+ return bool(getattr(os, "O_NOFOLLOW", 0) and getattr(os, "O_DIRECTORY", 0)) and \
1148
+ {os.open, os.mkdir, os.rename, os.stat, os.unlink} <= os.supports_dir_fd
1149
+
1150
+
1151
+ def not_a_folder_of_its_own(path):
1152
+ """Whether what stands at `path` is no folder of the project's own: a link, a junction - Windows' link to a folder,
1153
+ which `os.path.islink` does not name before Python 3.12, known by the name-surrogate bit of its reparse tag, which a
1154
+ cloud file's placeholder does not carry - a file, or what cannot be looked at (LEADS-ADAPTERS)."""
1155
+ try:
1156
+ info = os.lstat(path)
1157
+ except FileNotFoundError:
1158
+ return False
1159
+ except OSError:
1160
+ return True
1161
+ return stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) \
1162
+ or bool(getattr(info, "st_reparse_tag", 0) & _WINDOWS_LINK_TAG)
1163
+
1164
+
1165
+ def _tree_parts(rel):
1166
+ """The names of `rel`, a path within a project, folder by folder."""
1167
+ text = str(rel).replace(os.sep, "/")
1168
+ parts = [part for part in text.split("/") if part not in ("", ".")]
1169
+ if not parts or ".." in parts or (os.name == "nt" and any(":" in part for part in parts)):
1170
+ raise InTheWay(text, "names no file within the project")
1171
+ return parts
1172
+
1173
+
1174
+ def _folder_held(root, parts, make):
1175
+ """The descriptor of the folder `parts` names under `root`, each opened relative to the one before it and never
1176
+ through a link, and those missing made where `make`; None where one is missing. InTheWay where one is a link or a
1177
+ file."""
1178
+ flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
1179
+ fd = os.dup(root) if isinstance(root, int) else os.open(root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
1180
+ try:
1181
+ for at, part in enumerate(parts):
1182
+ if make:
1183
+ try:
1184
+ os.mkdir(part, dir_fd=fd)
1185
+ except FileExistsError:
1186
+ pass
1187
+ try:
1188
+ inner = os.open(part, flags, dir_fd=fd)
1189
+ except FileNotFoundError:
1190
+ if make:
1191
+ raise
1192
+ os.close(fd)
1193
+ return None
1194
+ except OSError as exc:
1195
+ if exc.errno in (errno.ELOOP, errno.ENOTDIR, errno.EMLINK):
1196
+ raise InTheWay("/".join(parts[:at + 1]), _A_FOLDER) from None
1197
+ raise
1198
+ held, fd = fd, inner
1199
+ os.close(held)
1200
+ except BaseException:
1201
+ os.close(fd)
1202
+ raise
1203
+ return fd
1204
+
1205
+
1206
+ def _windows_tree_available():
1207
+ """Select the native handle path by its APIs, even when a test changes os.name (LEADS-ADAPTERS)."""
1208
+ import ctypes
1209
+ return callable(getattr(ctypes, "WinDLL", None))
1210
+
1211
+
1212
+ def _require_windows_tree():
1213
+ if not _windows_tree_available():
1214
+ raise InTheWay("this platform", "cannot hold folders through no link; polyctl refuses this file operation")
1215
+
1216
+
1217
+ def open_new_text(path):
1218
+ """Create a UTF-8 file through no final link, or report that its name is taken (WINDOWS-TESTS-7).
1219
+
1220
+ Windows' CRT exclusive open follows a dangling link before creating its target. CREATE_NEW with
1221
+ OPEN_REPARSE_POINT instead refuses the occupied name, including a link planted during the open.
1222
+ Choose by the available APIs: tests can name a POSIX host Windows without giving it WinDLL.
1223
+ """
1224
+ nofollow = getattr(os, "O_NOFOLLOW", None)
1225
+ if nofollow is not None:
1226
+ opener = lambda name, flags: os.open(name, flags | nofollow, 0o666)
1227
+ elif _windows_tree_available():
1228
+ opener = _open_tree_nt
1229
+ else:
1230
+ raise OSError(f"{path}: this platform cannot create a file through no link; "
1231
+ "polyctl refuses the write; use a supported platform")
1232
+ return open(path, "x", encoding=UTF8, opener=opener)
1233
+
1234
+
1235
+ def _open_tree_nt(path, flags=None, mode=0o666):
1236
+ """Open the directory or leaf itself, with no delete sharing; reject every reparse point (LEADS-ADAPTERS)."""
1237
+ import ctypes
1238
+ import msvcrt
1239
+ from ctypes import wintypes
1240
+
1241
+ class Information(ctypes.Structure):
1242
+ _fields_ = [("attributes", wintypes.DWORD), ("created", wintypes.FILETIME),
1243
+ ("accessed", wintypes.FILETIME), ("written", wintypes.FILETIME),
1244
+ ("volume", wintypes.DWORD), ("size_high", wintypes.DWORD), ("size_low", wintypes.DWORD),
1245
+ ("links", wintypes.DWORD), ("index_high", wintypes.DWORD), ("index_low", wintypes.DWORD)]
1246
+
1247
+ kernel = ctypes.WinDLL("kernel32", use_last_error=True)
1248
+ create, close, information = kernel.CreateFileW, kernel.CloseHandle, kernel.GetFileInformationByHandle
1249
+ create.argtypes = (wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD, wintypes.LPVOID,
1250
+ wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE)
1251
+ create.restype = wintypes.HANDLE
1252
+ close.argtypes = (wintypes.HANDLE,)
1253
+ close.restype = wintypes.BOOL
1254
+ information.argtypes = (wintypes.HANDLE, ctypes.POINTER(Information))
1255
+ information.restype = wintypes.BOOL
1256
+ directory = flags is None
1257
+ # A folder is opened to list it, not for its attributes alone: Windows records the share mode of a handle that
1258
+ # asks for data access only, so one that asked for none let another process rename the folder it was holding.
1259
+ access = (0x0001 | 0x0080) if directory else (0xC0000000 if flags & os.O_RDWR else
1260
+ 0x40000000 if flags & os.O_WRONLY else 0x80000000)
1261
+ disposition = 3 if directory or not flags & os.O_CREAT else (1 if flags & os.O_EXCL else 4)
1262
+ raw = create(os.fspath(path), access, 0x1 | 0x2, None, disposition, 0x02000000 | 0x00200000, None)
1263
+ if raw is None or raw == wintypes.HANDLE(-1).value:
1264
+ raise ctypes.WinError(ctypes.get_last_error())
1265
+ try:
1266
+ info = Information()
1267
+ if not information(raw, ctypes.byref(info)):
1268
+ raise ctypes.WinError(ctypes.get_last_error())
1269
+ if info.attributes & 0x400:
1270
+ raise InTheWay(path, _A_FOLDER if directory else _A_LINK)
1271
+ if bool(info.attributes & 0x10) != directory:
1272
+ raise InTheWay(path, _A_FOLDER if directory else _NO_FILE)
1273
+ if directory:
1274
+ return raw, close
1275
+ fd = msvcrt.open_osfhandle(raw, flags & (os.O_APPEND | os.O_RDONLY | os.O_WRONLY | os.O_RDWR)
1276
+ | getattr(os, "O_BINARY", 0))
1277
+ except BaseException:
1278
+ close(raw)
1279
+ raise
1280
+ return fd
1281
+
1282
+
1283
+ @contextmanager
1284
+ def _folders_nt(root, parts, make):
1285
+ """Pin the whole absolute path until the leaf operation ends, including the project's ancestors."""
1286
+ _require_windows_tree()
1287
+ absolute = os.path.abspath(root)
1288
+ drive, tail = os.path.splitdrive(absolute)
1289
+ folder = drive + os.sep
1290
+ ancestors = [part for part in tail.split(os.sep) if part]
1291
+ names = ancestors + list(parts)
1292
+ handles = []
1293
+ try:
1294
+ handles.append(_open_tree_nt(folder))
1295
+ for at, part in enumerate(names):
1296
+ folder = os.path.join(folder, part)
1297
+ # Ancestors and root must already exist; traversal permission need not allow mkdir there
1298
+ # (the round-4 review of LEADS-ADAPTERS).
1299
+ if make and at >= len(ancestors):
1300
+ try:
1301
+ os.mkdir(folder)
1302
+ except FileExistsError:
1303
+ pass
1304
+ try:
1305
+ handles.append(_open_tree_nt(folder))
1306
+ except FileNotFoundError:
1307
+ if make:
1308
+ raise
1309
+ yield None
1310
+ return
1311
+ except OSError as exc:
1312
+ shown = "/".join(parts[:at - len(ancestors) + 1]) if at >= len(ancestors) else folder
1313
+ raise InTheWay(shown, f"{_A_FOLDER}, or cannot be held ({exc})") from exc
1314
+ yield folder
1315
+ finally:
1316
+ for handle, close in reversed(handles):
1317
+ close(handle)
1318
+
1319
+
1320
+ @contextmanager
1321
+ def _tree_folder(root, parts, make):
1322
+ """A dir_fd on POSIX or a path pinned by native handles on Windows (LEADS-ADAPTERS)."""
1323
+ if folders_held_open():
1324
+ fd = _folder_held(root, parts, make)
1325
+ try:
1326
+ yield fd
1327
+ finally:
1328
+ if fd is not None:
1329
+ os.close(fd)
1330
+ else:
1331
+ with _folders_nt(root, parts, make) as folder:
1332
+ yield folder
1333
+
1334
+
1335
+ @contextmanager
1336
+ def _tree_child(folder, name):
1337
+ """Hold a child for reading or removal while its parent stays held, through no link (LEADS-INSTALL)."""
1338
+ if isinstance(folder, int):
1339
+ fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=folder)
1340
+ try:
1341
+ yield fd
1342
+ finally:
1343
+ os.close(fd)
1344
+ else:
1345
+ _require_windows_tree()
1346
+ child = os.path.join(folder, name)
1347
+ handle, close = _open_tree_nt(child)
1348
+ try:
1349
+ yield child
1350
+ finally:
1351
+ close(handle)
1352
+
1353
+
1354
+ def _tree_rmdir(folder, name):
1355
+ if isinstance(folder, int):
1356
+ os.rmdir(name, dir_fd=folder)
1357
+ else:
1358
+ os.rmdir(os.path.join(folder, name))
1359
+
1360
+
1361
+ def _tree_remove_leaf(folder, name):
1362
+ """Unlink a leaf, including a Windows junction itself, never its target (LEADS-INSTALL)."""
1363
+ info = _tree_stat(folder, name)
1364
+ if not isinstance(folder, int) and getattr(info, "st_file_attributes", 0) & 0x10:
1365
+ _tree_rmdir(folder, name)
1366
+ else:
1367
+ _tree_unlink(folder, name)
1368
+
1369
+
1370
+ def _tree_open(folder, name, flags, mode=0o666):
1371
+ if isinstance(folder, int):
1372
+ try:
1373
+ fd = os.open(name, flags | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0), mode, dir_fd=folder)
1374
+ except OSError as exc:
1375
+ if exc.errno in (errno.ELOOP, errno.EMLINK):
1376
+ raise InTheWay(name, _A_LINK) from exc
1377
+ if exc.errno in (errno.ENXIO, errno.ENODEV, getattr(errno, "EOPNOTSUPP", -1)):
1378
+ raise InTheWay(name, _NO_FILE) from exc
1379
+ raise
1380
+ else:
1381
+ try:
1382
+ fd = _open_tree_nt(os.path.join(folder, name), flags, mode)
1383
+ except InTheWay as exc:
1384
+ raise InTheWay(name, str(exc)[len(str(exc.where)) + 1:]) from exc
1385
+ try:
1386
+ if not stat.S_ISREG(os.fstat(fd).st_mode):
1387
+ raise InTheWay(name, _NO_FILE)
1388
+ except BaseException:
1389
+ os.close(fd)
1390
+ raise
1391
+ return fd
1392
+
1393
+
1394
+ def _tree_stat(folder, name):
1395
+ return os.stat(name, dir_fd=folder, follow_symlinks=False) if isinstance(folder, int) else \
1396
+ os.lstat(os.path.join(folder, name))
1397
+
1398
+
1399
+ def _tree_unlink(folder, name):
1400
+ if isinstance(folder, int):
1401
+ os.unlink(name, dir_fd=folder)
1402
+ else:
1403
+ os.unlink(os.path.join(folder, name))
1404
+
1405
+
1406
+ def _tree_sync(folder, fsync):
1407
+ return _sync_folder_held(folder, fsync) if isinstance(folder, int) else False
1408
+
1409
+
1410
+ def create_in_tree(root, rel, data):
1411
+ """Make the file `rel` in the project at `root`, holding `data`, where nothing stands at that name: "wrote", or
1412
+ "kept" where a file, a folder or a link stands - kept as it is and never followed. InTheWay where a folder on the
1413
+ way is a link or a file (LEADS-ADAPTERS)."""
1414
+ parts = _tree_parts(rel)
1415
+ with _tree_folder(root, parts[:-1], make=True) as folder:
1416
+ try:
1417
+ out = _tree_open(folder, parts[-1], os.O_WRONLY | os.O_CREAT | os.O_EXCL)
1418
+ except FileExistsError:
1419
+ return "kept"
1420
+ with os.fdopen(out, "wb") as fh:
1421
+ fh.write(data)
1422
+ fh.flush()
1423
+ os.fsync(fh.fileno())
1424
+ _tree_sync(folder, os.fsync)
1425
+ return "wrote"
1426
+
1427
+
1428
+ def replace_in_tree(root, rel, data, *, mode=None, _checkpoint=None, _fsync=None):
1429
+ """Replace the file `rel` in the project at `root` whole, as replace_file_durably does - a temporary under a name no
1430
+ file had, fsynced, renamed over it, the folder fsynced - in the folder held open from the root down: a link at its
1431
+ name is replaced and nothing is written through one. Without `mode` the file keeps the mode it had, or 0644.
1432
+ `_checkpoint` and `_fsync` are replace_file_durably's test seams, at the same steps. InTheWay where a folder on the
1433
+ way is a link or a file, or a folder stands at its name (LEADS-ADAPTERS)."""
1434
+ fsync = _fsync or os.fsync
1435
+ parts = _tree_parts(rel)
1436
+ shown = "/".join(parts)
1437
+
1438
+ def kept_mode(listed):
1439
+ if stat.S_ISDIR(listed.st_mode):
1440
+ raise InTheWay(shown, "is a folder, where a file of the project's goes")
1441
+ return listed.st_mode & 0o7777 if stat.S_ISREG(listed.st_mode) else 0o644
1442
+
1443
+ if not folders_held_open():
1444
+ with _folders_nt(root, parts[:-1], make=True) as folder:
1445
+ path = os.path.join(folder, parts[-1])
1446
+ try:
1447
+ leaf = _open_tree_nt(path, os.O_RDONLY)
1448
+ except FileNotFoundError:
1449
+ kept = 0o644
1450
+ else:
1451
+ try:
1452
+ kept = kept_mode(os.fstat(leaf))
1453
+ finally:
1454
+ os.close(leaf)
1455
+ temporary = os.path.join(folder, f".{parts[-1]}.{os.getpid()}.{os.urandom(4).hex()}.tmp")
1456
+ out = _open_tree_nt(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
1457
+ try:
1458
+ try:
1459
+ _call(_checkpoint, "temporary-opened")
1460
+ _write_all(out, data)
1461
+ _call(_checkpoint, "temporary-written")
1462
+ fsync(out)
1463
+ _call(_checkpoint, "temporary-fsynced")
1464
+ finally:
1465
+ os.close(out)
1466
+ _replace(temporary, path)
1467
+ except BaseException:
1468
+ try:
1469
+ os.unlink(temporary)
1470
+ except OSError:
1471
+ pass
1472
+ raise
1473
+ _call(_checkpoint, "replaced")
1474
+ return
1475
+ fd = _folder_held(root, parts[:-1], make=True)
1476
+ try:
1477
+ name = parts[-1]
1478
+ try:
1479
+ kept = kept_mode(os.stat(name, dir_fd=fd, follow_symlinks=False))
1480
+ except FileNotFoundError:
1481
+ kept = 0o644
1482
+ temporary = f".{name}.{os.getpid()}.{os.urandom(4).hex()}.tmp"
1483
+ out = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
1484
+ kept if mode is None else mode, dir_fd=fd)
1485
+ try:
1486
+ try:
1487
+ _call(_checkpoint, "temporary-opened")
1488
+ _write_all(out, data)
1489
+ os.fchmod(out, kept if mode is None else mode)
1490
+ _call(_checkpoint, "temporary-written")
1491
+ fsync(out)
1492
+ _call(_checkpoint, "temporary-fsynced")
1493
+ finally:
1494
+ os.close(out)
1495
+ os.replace(temporary, name, src_dir_fd=fd, dst_dir_fd=fd)
1496
+ except BaseException:
1497
+ try:
1498
+ os.unlink(temporary, dir_fd=fd)
1499
+ except OSError:
1500
+ pass
1501
+ raise
1502
+ _call(_checkpoint, "replaced")
1503
+ if _sync_folder_held(fd, fsync):
1504
+ _call(_checkpoint, "directory-fsynced")
1505
+ finally:
1506
+ os.close(fd)
1507
+
1508
+
1509
+ def read_in_tree(root, rel):
1510
+ """The bytes of the regular file `rel` in the project at `root`, or None where nothing is there. InTheWay where it,
1511
+ or a folder on its way, is a link, or it is no regular file - a FIFO is never waited on (LEADS-ADAPTERS)."""
1512
+ parts = _tree_parts(rel)
1513
+ with _tree_folder(root, parts[:-1], make=False) as folder:
1514
+ if folder is None:
1515
+ return None
1516
+ try:
1517
+ leaf = _tree_open(folder, parts[-1], os.O_RDONLY)
1518
+ except FileNotFoundError:
1519
+ return None
1520
+ except InTheWay as exc:
1521
+ raise InTheWay("/".join(parts), str(exc)[len(str(exc.where)) + 1:]) from exc
1522
+ with os.fdopen(leaf, "rb") as fh:
1523
+ return fh.read()
1524
+
1525
+
1526
+ def append_in_tree(root, rel, data):
1527
+ """Append only to a single-name regular file opened through no link (LEADS-ADAPTERS)."""
1528
+ parts = _tree_parts(rel)
1529
+ with _tree_folder(root, parts[:-1], make=True) as folder:
1530
+ fd = _tree_open(folder, parts[-1], os.O_WRONLY | os.O_CREAT | os.O_APPEND)
1531
+ try:
1532
+ if os.fstat(fd).st_nlink != 1:
1533
+ raise InTheWay(rel, "is a file with another name")
1534
+ _write_all(fd, data)
1535
+ os.fsync(fd)
1536
+ finally:
1537
+ os.close(fd)
1538
+ _tree_sync(folder, os.fsync)
1539
+
1540
+
1541
+ def _tree_bytes(folder, name):
1542
+ try:
1543
+ fd = _tree_open(folder, name, os.O_RDONLY)
1544
+ except FileNotFoundError:
1545
+ return None
1546
+ with os.fdopen(fd, "rb") as handle:
1547
+ return handle.read()
1548
+
1549
+
1550
+ def _tree_rename(folder, source, target):
1551
+ if isinstance(folder, int):
1552
+ os.replace(source, target, src_dir_fd=folder, dst_dir_fd=folder)
1553
+ else:
1554
+ _replace(os.path.join(folder, source), os.path.join(folder, target))
1555
+
1556
+
1557
+ def _tree_seal(folder, name, identity):
1558
+ """Read a seal through the held tree, including when its ledger is absent (LEADS-ADAPTERS)."""
1559
+ raw_seal = _tree_bytes(folder, name + ".seal.json")
1560
+ seal = None
1561
+ if raw_seal is not None:
1562
+ try:
1563
+ seal = json.loads(raw_seal.decode(UTF8))
1564
+ except (ValueError, UnicodeError) as exc:
1565
+ raise LedgerCorruption(f"the seal of {identity} cannot be read: {exc}") from exc
1566
+ if not isinstance(seal, dict) or not isinstance(seal.get("retired"), int) or seal["retired"] < 1 \
1567
+ or not isinstance(seal.get("digest"), str):
1568
+ raise LedgerCorruption(f"the seal of {identity} is malformed")
1569
+ return seal
1570
+
1571
+
1572
+ def _tree_replace_bytes(folder, name, data):
1573
+ """Publish a recovered seal in the folder held for its ledger, even if that folder moved (LEDGER-HISTORY)."""
1574
+ temporary = f".{name}.{os.getpid()}.{os.urandom(4).hex()}.tmp"
1575
+ fd = _tree_open(folder, temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
1576
+ try:
1577
+ try:
1578
+ _write_all(fd, data)
1579
+ os.fsync(fd)
1580
+ finally:
1581
+ os.close(fd)
1582
+ _tree_rename(folder, temporary, name)
1583
+ except BaseException:
1584
+ try:
1585
+ _tree_unlink(folder, temporary)
1586
+ except OSError:
1587
+ pass
1588
+ raise
1589
+ _tree_sync(folder, os.fsync)
1590
+
1591
+
1592
+ def _chained_in_tree(root, rel, chain, record=None, every=False):
1593
+ """The opt-in ledger's read or append, including recovery, under its usual sidecar lock (LEADS-ADAPTERS).
1594
+
1595
+ Keep the JSONL, chain, seals and machine checkpoint unchanged. Only this ledger's filesystem operations
1596
+ change: the lock, ledger and seal are opened in the held folder; truncation and rollback use the same
1597
+ descriptor, never a name an agent can redirect. Other ledger callers keep their existing API.
1598
+ """
1599
+ nothing = 0 if every else [] # what an empty ledger is to the caller
1600
+ parts = _tree_parts(rel)
1601
+ name = parts[-1]
1602
+ identity = os.path.join(os.path.realpath(root), *parts)
1603
+ # Before the store is read at all: a store inside the tree attests nothing and is never acted on (OPTIN-ATTESTED).
1604
+ if _store_in_tree(root):
1605
+ if record is None:
1606
+ return nothing
1607
+ raise LedgerCorruption(
1608
+ f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
1609
+ "no opt-in beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
1610
+ recovering = "intent" in _checkpoint_entry(identity, identity=identity)
1611
+ with _tree_folder(root, parts[:-1], make=record is not None or recovering) as folder:
1612
+ if folder is None:
1613
+ _check_missing_chained_ledger(identity, chain, identity=identity)
1614
+ return nothing
1615
+ if record is None and not recovering:
1616
+ # Reading a setting before any opt-in must not create a lock in an otherwise clean project, even
1617
+ # when init refuses during planning. A checkpoint intent needs locked recovery; an in-tree intent
1618
+ # needs locked refusal (LEADS-ADAPTERS, LEDGER-HISTORY).
1619
+ for leaf in (name, name + ".seal.json.pending"):
1620
+ try:
1621
+ _tree_stat(folder, leaf)
1622
+ except FileNotFoundError:
1623
+ continue
1624
+ break
1625
+ else:
1626
+ _check_missing_chained_ledger(identity, chain, seal=_tree_seal(folder, name, identity), identity=identity)
1627
+ return nothing
1628
+ with _exclusive_lock(identity + ".lock", _folder=folder):
1629
+ _settle_seal(identity, chain, CHAIN_PREVIOUS_FIELD, _folder=folder, identity=identity)
1630
+ data = _tree_bytes(folder, name)
1631
+ seal = _load_seal(identity, _folder=folder)
1632
+ if data is None:
1633
+ _check_missing_chained_ledger(identity, chain, seal=seal, identity=identity)
1634
+ data = b""
1635
+ attested = _attested_length(identity, seal, identity=identity)
1636
+ if record is None:
1637
+ records, _ = _jsonl_records(data)
1638
+ if every:
1639
+ return len(records)
1640
+ # Only the rows polyctl recorded are read and checked: a tail past them - well formed or not - is nobody's
1641
+ # record, and never makes polyctl's own unreadable (OPTIN-ATTESTED).
1642
+ rows = [_jsonl_row(body, index, identity) for index, body in records[:attested]]
1643
+ _validate_chained_rows(identity, rows, chain, CHAIN_PREVIOUS_FIELD, seal=seal)
1644
+ _check_committed_length(identity, rows, chain, seal=seal, identity=identity)
1645
+ return rows
1646
+ records, unfinished = _jsonl_records(data)
1647
+ past = len(records) - attested
1648
+ cut_to = None
1649
+ if past > 0:
1650
+ if not _is_interrupted_append(records, attested, identity, chain, identity=identity,
1651
+ unfinished=unfinished):
1652
+ # Appended after them, polyctl's record would vouch for rows it never wrote (OPTIN-ATTESTED). Counted as
1653
+ # a read counts them, before anything parses them.
1654
+ raise LedgerCorruption(f"{rel}: {_unattested_repair(past, attested)}")
1655
+ cut_to = sum(len(line) for line in data.splitlines(keepends=True)[:records[attested][0]])
1656
+ records = records[:attested]
1657
+ rows = [_jsonl_row(body, index, identity) for index, body in records]
1658
+ _validate_chained_rows(identity, rows, chain, CHAIN_PREVIOUS_FIELD, seal=seal)
1659
+ _check_committed_length(identity, rows, chain, seal=seal, identity=identity)
1660
+ retired = seal["retired"] if seal else 0
1661
+ previous = chained_row_digest(rows[-1], chain) if rows else (seal["digest"] if seal else None)
1662
+ chained = {CHAIN_PREVIOUS_FIELD: previous, "ordinal": retired + len(rows) + 1}
1663
+ chained.update(record)
1664
+ payload = _encode_jsonl(chained)
1665
+ digest = chained_row_digest(chained, chain)
1666
+ created = False
1667
+ try:
1668
+ fd = _tree_open(folder, name, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o644)
1669
+ created = True
1670
+ except FileExistsError:
1671
+ fd = _tree_open(folder, name, os.O_RDWR)
1672
+ remove_created = created
1673
+ try:
1674
+ if os.fstat(fd).st_nlink != 1:
1675
+ raise InTheWay(rel, "has another name; polyctl appends only to the project's own file")
1676
+ # Refuse a different leaf swapped in since the read; a hard link is refused before any truncation.
1677
+ with os.fdopen(os.dup(fd), "rb") as handle:
1678
+ current = handle.read()
1679
+ if current != data:
1680
+ raise InTheWay(rel, "changed while polyctl read it; retry the opt-in")
1681
+ if cut_to is not None:
1682
+ boundary = cut_to
1683
+ else:
1684
+ boundary = len(data) if not data or data.endswith((b"\n", b"\r")) else \
1685
+ max(data.rfind(b"\n"), data.rfind(b"\r")) + 1
1686
+ os.ftruncate(fd, boundary)
1687
+ if boundary != len(data):
1688
+ os.fsync(fd)
1689
+ if cut_to is not None:
1690
+ _clear_append_intent(identity, identity=identity)
1691
+ os.lseek(fd, boundary, os.SEEK_SET)
1692
+ _record_append_intent(identity, digest, identity=identity)
1693
+ try:
1694
+ _write_all(fd, payload)
1695
+ os.fsync(fd)
1696
+ if created:
1697
+ _tree_sync(folder, os.fsync)
1698
+ _record_committed_length(identity, retired + len(rows) + 1, digest, identity=identity)
1699
+ except BaseException:
1700
+ os.ftruncate(fd, boundary)
1701
+ os.fsync(fd)
1702
+ raise
1703
+ remove_created = False
1704
+ finally:
1705
+ held = os.fstat(fd)
1706
+ os.close(fd)
1707
+ if remove_created:
1708
+ try:
1709
+ named = _tree_stat(folder, name)
1710
+ if (held.st_dev, held.st_ino) == (named.st_dev, named.st_ino):
1711
+ _tree_unlink(folder, name)
1712
+ except OSError:
1713
+ pass
1714
+ return chained
1715
+
1716
+
1717
+ def append_chained_in_tree(root, rel, record, chain):
1718
+ """Append an opt-in without following a folder or leaf link (LEADS-ADAPTERS)."""
1719
+ return _chained_in_tree(root, rel, chain, record)
1720
+
1721
+
1722
+ def read_chained_in_tree(root, rel, chain):
1723
+ """Read opt-ins under the same held-folder lock as their append (LEADS-ADAPTERS): the rows this machine's checkpoint
1724
+ store records polyctl appending, and no others (OPTIN-ATTESTED).
1725
+
1726
+ The chain proves no committed row was changed and the recorded length that none was cut off; neither says who
1727
+ wrote a row past that length. A person's opt-in weakens a guarantee, and the implementer can write any file in
1728
+ the tree: the first row of a chain needs no digest at all, and one past polyctl's last append needs only the digest of
1729
+ the row before it. A row there - in a ledger this machine never recorded, or added after polyctl's last append - is
1730
+ nobody's record.
1731
+ """
1732
+ return _chained_in_tree(root, rel, chain)
1733
+
1734
+
1735
+ def unattested_in_tree(root, rel, chain):
1736
+ """(rows this machine's checkpoint store does not record polyctl appending, rows it does) of an in-tree ledger
1737
+ (OPTIN-ATTESTED)."""
1738
+ attested = len(_chained_in_tree(root, rel, chain))
1739
+ return max(0, _chained_in_tree(root, rel, chain, every=True) - attested), attested
1740
+
1741
+
1742
+ def _store_in_tree(root):
1743
+ """Whether this machine's checkpoint store lies inside the project's tree (OPTIN-ATTESTED).
1744
+
1745
+ POLYCTL_LEDGER_CHECKPOINTS moves the store, and the implementer can name it on any command it runs: a store inside the
1746
+ tree is a file it writes, so a row it wrote and a count it wrote beside it would vouch for each other. Such a store
1747
+ attests nothing, and polyctl appends no opt-in beside it, as `polyctl doctor` already names the variable (PYTHON-ENV).
1748
+
1749
+ Compared by what the folders are, not how their names are spelled: a case-insensitive file system answers the
1750
+ same folder to `/A/Proj` and `/a/proj`, and the real path keeps whichever spelling it was given. Each folder on
1751
+ the store's path, as named and as resolved, is held to the root's device and inode, so neither case nor a link
1752
+ moves it out.
1753
+ """
1754
+ try:
1755
+ top = os.stat(root)
1756
+ except OSError:
1757
+ return False
1758
+ named = checkpoint_path()
1759
+ if os.pardir in re.split(r"[\\/]+", named):
1760
+ # `a/link/..` is the link target's parent to the file system and `a` to a path read as text: either reading
1761
+ # can hide the other, so a store named with `..` is taken as the tree's.
1762
+ return True
1763
+ given = named if os.path.isabs(named) else os.path.join(os.getcwd(), named)
1764
+ # The path as named and as resolved, both: one through a link the implementer keeps in the tree is the tree's,
1765
+ # wherever the link points as polyctl looks, since the link can point elsewhere by the time the store is read.
1766
+ for path in (given, os.path.realpath(named)):
1767
+ if not top.st_ino:
1768
+ # A file system that numbers no file: the paths, case folded where the platform folds case.
1769
+ store, folder = os.path.normcase(path), os.path.normcase(os.path.realpath(root))
1770
+ if store == folder or store.startswith(folder.rstrip(os.sep) + os.sep):
1771
+ return True
1772
+ continue
1773
+ here = os.path.dirname(path)
1774
+ while True:
1775
+ try:
1776
+ held = os.stat(here)
1777
+ except OSError:
1778
+ held = None
1779
+ if held is not None and (held.st_dev, held.st_ino) == (top.st_dev, top.st_ino):
1780
+ return True
1781
+ parent = os.path.dirname(here)
1782
+ if parent == here:
1783
+ break
1784
+ here = parent
1785
+ return False
1786
+
1787
+
1788
+ def _ledger_store_in_tree(path):
1789
+ """Whether `path` is a project ledger under `.polyctl/ledger/` whose checkpoint store lies inside that project."""
1790
+ ledger_dir = os.path.dirname(os.path.abspath(path))
1791
+ polyctl_dir = os.path.dirname(ledger_dir)
1792
+ if os.path.basename(ledger_dir) == "ledger" and os.path.basename(polyctl_dir) == ".polyctl":
1793
+ return _store_in_tree(os.path.dirname(polyctl_dir))
1794
+ return False
1795
+
1796
+
1797
+ def _unattested_repair(foreign, attested):
1798
+ """What a person does about rows past polyctl's record, keeping the ones polyctl did record (OPTIN-ATTESTED)."""
1799
+ keep = (f"the first {attested} row(s) are polyctl's own and stay; a person removes exactly the last {foreign} line(s) "
1800
+ "of the file") if attested else ("no row of it is polyctl's own; a person moves the file aside, or attests a "
1801
+ "restored backup with polyctl restore <source> --by <name> --ask")
1802
+ return (f"{foreign} row(s) the checkpoint store of this machine ({checkpoint_path()}) does not record polyctl "
1803
+ f"appending - something other than polyctl wrote them, or an append did not finish. They turn nothing off and "
1804
+ f"polyctl appends nothing after them: {keep}, and records again what is missing")
1805
+
1806
+
1807
+ def _attested_length(path, seal, *, identity=None):
1808
+ """How many of a ledger's live rows this machine's checkpoint store records polyctl committing (OPTIN-ATTESTED)."""
1809
+ count = _checkpoint_entry(path, identity=identity).get("count")
1810
+ return max(0, (count if isinstance(count, int) else 0) - (seal["retired"] if seal else 0))
1811
+
1812
+
1813
+ def _is_interrupted_append(records, attested, path, chain, *, identity=None, unfinished=None):
1814
+ """Whether the single record past `attested` is the one this machine recorded an intent to append (LEDGER-ATTESTED)."""
1815
+ if unfinished is not None or len(records) - attested != 1:
1816
+ return False
1817
+ intent = _checkpoint_entry(path, identity=identity).get("append")
1818
+ if not isinstance(intent, str) or not intent:
1819
+ return False
1820
+ index, body = records[attested]
1821
+ try:
1822
+ row = _jsonl_row(body, index, path)
1823
+ return isinstance(row, dict) and chained_row_digest(row, chain) == intent
1824
+ except (LedgerCorruption, TypeError, ValueError):
1825
+ return False
1826
+
1827
+
1828
+ def _repair_partial_tail(path, fsync, checkpoint, *, limit=None):
1829
+ """Cut off the uncommitted final record an interrupted or failed append left.
1830
+
1831
+ It is never completed: its writer did not return, so it was never committed (#68).
1832
+ """
1833
+ if not os.path.exists(path) or os.path.getsize(path) == 0:
1834
+ return "clean"
1835
+
1836
+ # Validate every complete row before touching the tail. Without this pass an
1837
+ # invalid earlier row followed by a partial tail could be retained while a
1838
+ # new record was appended, falsely reporting a successful durable write on
1839
+ # an already-corrupt ledger.
1840
+ data = open(path, "rb").read()
1841
+ if limit is None:
1842
+ _read_jsonl_unlocked(path, allow_partial_tail=True)
1843
+ else:
1844
+ for index, body in _jsonl_records(data)[0][:limit]:
1845
+ _jsonl_row(body, index, path)
1846
+ if data.endswith((b"\n", b"\r")):
1847
+ return "clean"
1848
+
1849
+ boundary = max(data.rfind(b"\n"), data.rfind(b"\r")) + 1
1850
+ with open(path, "r+b", buffering=0) as handle:
1851
+ handle.truncate(boundary)
1852
+ fsync(handle.fileno())
1853
+ _call(checkpoint, "truncated-tail")
1854
+ return "truncated-tail"
1855
+
1856
+
1857
+ def _encode_jsonl(record):
1858
+ # The same inputs the chain digest accepts - ASCII escapes, no NaN - so no row
1859
+ # is written that could never be digested, and a path Git hands over
1860
+ # undecodable is escaped rather than refused (#68).
1861
+ return (json.dumps(
1862
+ record, ensure_ascii=True, allow_nan=False, separators=(",", ":")
1863
+ ) + "\n").encode("ascii")
1864
+
1865
+
1866
+ def _append_payload_unlocked(path, parent, payload, write, fsync, checkpoint):
1867
+ """Append one encoded row while the caller owns the sidecar lock.
1868
+
1869
+ All or nothing (#68): a failure at any step - a short write, the file fsync, the
1870
+ directory fsync - cuts the written bytes off again, and removes a file this
1871
+ append created, before the error goes to the caller. Returns the same undo for
1872
+ a caller whose own next step fails while it still holds the lock.
1873
+ """
1874
+ created = not os.path.exists(path)
1875
+ flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_BINARY", 0)
1876
+ fd = os.open(path, flags, 0o644)
1877
+ size = os.fstat(fd).st_size
1878
+
1879
+ def undo():
1880
+ try:
1881
+ if created:
1882
+ os.remove(path)
1883
+ return
1884
+ os.truncate(path, size)
1885
+ handle = os.open(path, os.O_WRONLY | getattr(os, "O_BINARY", 0))
1886
+ try:
1887
+ os.fsync(handle)
1888
+ finally:
1889
+ os.close(handle)
1890
+ except OSError:
1891
+ pass
1892
+
1893
+ try:
1894
+ _call(checkpoint, "opened")
1895
+ remaining = memoryview(payload)
1896
+ while remaining:
1897
+ try:
1898
+ count = write(fd, remaining)
1899
+ except InterruptedError:
1900
+ continue
1901
+ if not isinstance(count, int) or count <= 0 or count > len(remaining):
1902
+ raise OSError(errno.EIO, "append made no forward progress")
1903
+ remaining = remaining[count:]
1904
+ _call(checkpoint, "written")
1905
+ fsync(fd)
1906
+ _call(checkpoint, "fsynced")
1907
+ except BaseException:
1908
+ os.close(fd)
1909
+ undo()
1910
+ raise
1911
+ os.close(fd)
1912
+
1913
+ if created:
1914
+ try:
1915
+ synced = _sync_directory(parent, fsync)
1916
+ except BaseException:
1917
+ undo()
1918
+ raise
1919
+ if synced:
1920
+ _call(checkpoint, "directory-fsynced")
1921
+ return undo
1922
+
1923
+
1924
+ def append_jsonl(path, record, timeout=10.0, *, _checkpoint=None,
1925
+ _write=None, _fsync=None):
1926
+ """Append one JSON object and durably persist it before returning.
1927
+
1928
+ ``_checkpoint``, ``_write`` and ``_fsync`` are deliberately private test
1929
+ seams. They let the suite stop a real subprocess between storage barriers
1930
+ and inject short writes or I/O failures without a production crash switch.
1931
+ """
1932
+ parent = os.path.dirname(path) or "."
1933
+ os.makedirs(parent, exist_ok=True)
1934
+ payload = _encode_jsonl(record)
1935
+ write = _write or os.write
1936
+ fsync = _fsync or os.fsync
1937
+
1938
+ with _exclusive_lock(path + ".lock", timeout=timeout):
1939
+ _call(_checkpoint, "locked")
1940
+ _repair_partial_tail(path, fsync, _checkpoint)
1941
+ _append_payload_unlocked(
1942
+ path, parent, payload, write, fsync, _checkpoint
1943
+ )
1944
+ return record
1945
+
1946
+
1947
+ def append_chained_jsonl(path, record, chain, timeout=10.0, *,
1948
+ previous_field=CHAIN_PREVIOUS_FIELD, legacy_prefix=False, prepare=None,
1949
+ _checkpoint=None, _write=None, _fsync=None):
1950
+ """Atomically select a predecessor, append, and persist one chained row.
1951
+
1952
+ Existing committed rows are validated before a successor is constructed.
1953
+ Predecessor selection and append share one lock, so concurrent writers form
1954
+ one linear chain instead of siblings that name the same predecessor.
1955
+ `prepare`, given the committed rows, returns fields to set on the record under
1956
+ that lock: what must be unique across writers is decided where they take turns.
1957
+ """
1958
+ if not isinstance(record, dict):
1959
+ raise TypeError("a chained ledger row must be a JSON object")
1960
+ if not isinstance(previous_field, str) or not previous_field:
1961
+ raise ValueError("previous_field must be a non-empty string")
1962
+ if previous_field in record or "ordinal" in record:
1963
+ raise ValueError(f"caller must not set the {previous_field!r} or 'ordinal' chain fields")
1964
+ gate = _is_gate_ledger(path, chain)
1965
+ if gate and _ledger_store_in_tree(path):
1966
+ raise LedgerCorruption(
1967
+ f"the checkpoint store {checkpoint_path()} lies inside the project, where an agent writes: polyctl records "
1968
+ "no row beside it - point POLYCTL_LEDGER_CHECKPOINTS outside the tree, or unset it")
1969
+
1970
+ parent = os.path.dirname(path) or "."
1971
+ os.makedirs(parent, exist_ok=True)
1972
+ write = _write or os.write
1973
+ fsync = _fsync or os.fsync
1974
+
1975
+ with _exclusive_lock(path + ".lock", timeout=timeout):
1976
+ _call(_checkpoint, "locked")
1977
+ _settle_seal(path, chain, previous_field, legacy_prefix)
1978
+ seal = _load_seal(path)
1979
+ if not os.path.exists(path):
1980
+ _check_missing_chained_ledger(path, chain, seal=seal)
1981
+ if gate:
1982
+ attested = _attested_length(path, seal)
1983
+ if os.path.exists(path):
1984
+ with open(path, "rb") as handle:
1985
+ data = handle.read()
1986
+ else:
1987
+ data = b""
1988
+ records, unfinished = _jsonl_records(data)
1989
+ past = len(records) - attested
1990
+ if past > 0:
1991
+ if not _is_interrupted_append(records, attested, path, chain, unfinished=unfinished):
1992
+ # Appended after them, polyctl's record would vouch for rows it never wrote (OPTIN-ATTESTED,
1993
+ # LEDGER-ATTESTED). Counted as a read counts them, before anything parses or truncates them.
1994
+ raise LedgerCorruption(f"{path}: {_unattested_repair(past, attested)}")
1995
+ rows = [_jsonl_row(body, index, path) for index, body in records[:attested]]
1996
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
1997
+ _check_committed_length(path, rows, chain, seal=seal)
1998
+ boundary = sum(len(line) for line in data.splitlines(keepends=True)[:records[attested][0]])
1999
+ with open(path, "r+b", buffering=0) as handle:
2000
+ handle.truncate(boundary)
2001
+ fsync(handle.fileno())
2002
+ _call(_checkpoint, "truncated-tail")
2003
+ _clear_append_intent(path)
2004
+ with open(path, "rb") as handle:
2005
+ data = handle.read()
2006
+ else:
2007
+ rows = [_jsonl_row(body, index, path) for index, body in records[:attested]]
2008
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
2009
+ _check_committed_length(path, rows, chain, seal=seal)
2010
+ _repair_partial_tail(path, fsync, _checkpoint, limit=attested)
2011
+ if os.path.exists(path):
2012
+ with open(path, "rb") as handle:
2013
+ data = handle.read()
2014
+ else:
2015
+ data = b""
2016
+ else:
2017
+ _repair_partial_tail(path, fsync, _checkpoint)
2018
+ if os.path.exists(path):
2019
+ with open(path, "rb") as handle:
2020
+ data = handle.read()
2021
+ else:
2022
+ data = b""
2023
+ rows = _jsonl_rows(data, path, allow_partial_tail=False)
2024
+ _validate_chained_rows(path, rows, chain, previous_field, legacy_prefix, seal=seal)
2025
+ _check_committed_length(path, rows, chain, seal=seal)
2026
+ retired = seal["retired"] if seal else 0
2027
+ previous = chained_row_digest(rows[-1], chain) if rows else (seal["digest"] if seal else None)
2028
+ chained = {previous_field: previous, "ordinal": retired + len(rows) + 1}
2029
+ chained.update(record)
2030
+ if prepare is not None:
2031
+ fields = prepare(rows) or {}
2032
+ if previous_field in fields or "ordinal" in fields:
2033
+ raise ValueError(f"prepare must not set the {previous_field!r} or 'ordinal' chain fields")
2034
+ chained.update(fields)
2035
+ payload = _encode_jsonl(chained)
2036
+ digest = chained_row_digest(chained, chain)
2037
+ _record_append_intent(path, digest)
2038
+ try:
2039
+ undo = _append_payload_unlocked(
2040
+ path, parent, payload, write, fsync, _checkpoint
2041
+ )
2042
+ except BaseException:
2043
+ try:
2044
+ _clear_append_intent(path)
2045
+ except OSError:
2046
+ pass
2047
+ raise
2048
+ try:
2049
+ _record_committed_length(path, retired + len(rows) + 1, digest)
2050
+ except BaseException:
2051
+ # A row whose length could not be recorded is taken back, so the
2052
+ # caller's failure and the file agree (#68).
2053
+ undo()
2054
+ raise
2055
+ return chained