polyctl 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. polyctl/__init__.py +7 -0
  2. polyctl/__main__.py +5 -0
  3. polyctl/a2a.py +226 -0
  4. polyctl/a2a_mcp.py +546 -0
  5. polyctl/acp.py +417 -0
  6. polyctl/adapters/README.md +101 -0
  7. polyctl/adapters/aider.json +32 -0
  8. polyctl/adapters/amazon-q.json +15 -0
  9. polyctl/adapters/amp.json +16 -0
  10. polyctl/adapters/antigravity.json +120 -0
  11. polyctl/adapters/claude-code.json +229 -0
  12. polyctl/adapters/cloud-generic.json +57 -0
  13. polyctl/adapters/codex.json +47 -0
  14. polyctl/adapters/command-code.json +45 -0
  15. polyctl/adapters/copilot.json +17 -0
  16. polyctl/adapters/cursor-agent.json +29 -0
  17. polyctl/adapters/deepseek.json +13 -0
  18. polyctl/adapters/droid.json +34 -0
  19. polyctl/adapters/gemini.json +30 -0
  20. polyctl/adapters/grok.json +22 -0
  21. polyctl/adapters/hermes.json +34 -0
  22. polyctl/adapters/kilocode.json +30 -0
  23. polyctl/adapters/kimi.json +23 -0
  24. polyctl/adapters/kiro.json +332 -0
  25. polyctl/adapters/ollama.json +29 -0
  26. polyctl/adapters/omp.json +34 -0
  27. polyctl/adapters/openai-api.json +68 -0
  28. polyctl/adapters/opencode.json +162 -0
  29. polyctl/adapters/pi.json +24 -0
  30. polyctl/adapters/pr-agent.json +54 -0
  31. polyctl/adapters/profiles.json +9 -0
  32. polyctl/adapters/qoder.json +163 -0
  33. polyctl/adapters/qwen.json +27 -0
  34. polyctl/adapters/reasonix.json +25 -0
  35. polyctl/adapters/rewriters/rtk.json +24 -0
  36. polyctl/adapters/trae.json +31 -0
  37. polyctl/adapters/vendors.json +42 -0
  38. polyctl/adapters/warp.json +88 -0
  39. polyctl/allowlist.py +1756 -0
  40. polyctl/api_review.py +920 -0
  41. polyctl/cli.py +835 -0
  42. polyctl/conformance_harness.py +9 -0
  43. polyctl/demo.py +503 -0
  44. polyctl/drivers.py +146 -0
  45. polyctl/email.py +160 -0
  46. polyctl/features.py +98 -0
  47. polyctl/journal.py +144 -0
  48. polyctl/language.py +945 -0
  49. polyctl/lib.py +1296 -0
  50. polyctl/matrix.py +660 -0
  51. polyctl/mcp.py +1362 -0
  52. polyctl/ntfy.py +169 -0
  53. polyctl/outbound.py +39 -0
  54. polyctl/parts/__init__.py +5 -0
  55. polyctl/parts/cli_authority.py +2615 -0
  56. polyctl/parts/cli_channels.py +1325 -0
  57. polyctl/parts/cli_checks.py +2293 -0
  58. polyctl/parts/cli_completion.py +686 -0
  59. polyctl/parts/cli_evidence.py +141 -0
  60. polyctl/parts/cli_gain.py +375 -0
  61. polyctl/parts/cli_harness.py +392 -0
  62. polyctl/parts/cli_hooks.py +1816 -0
  63. polyctl/parts/cli_install.py +638 -0
  64. polyctl/parts/cli_lanes.py +106 -0
  65. polyctl/parts/cli_maintenance.py +2536 -0
  66. polyctl/parts/cli_next.py +248 -0
  67. polyctl/parts/cli_pr.py +53 -0
  68. polyctl/parts/cli_project.py +2857 -0
  69. polyctl/parts/cli_review.py +5158 -0
  70. polyctl/parts/cli_status.py +898 -0
  71. polyctl/parts/lib_adapters.py +2933 -0
  72. polyctl/parts/lib_alarms.py +2106 -0
  73. polyctl/parts/lib_events.py +224 -0
  74. polyctl/parts/lib_evidence.py +412 -0
  75. polyctl/parts/lib_gates.py +2099 -0
  76. polyctl/parts/lib_lanes.py +829 -0
  77. polyctl/parts/lib_mail.py +1541 -0
  78. polyctl/parts/lib_next.py +652 -0
  79. polyctl/parts/lib_pr.py +331 -0
  80. polyctl/parts/lib_slices.py +3337 -0
  81. polyctl/parts/lib_state.py +2415 -0
  82. polyctl/parts/lib_transcript.py +4118 -0
  83. polyctl/pressure.py +325 -0
  84. polyctl/prices.json +47 -0
  85. polyctl/procs.py +1194 -0
  86. polyctl/settings.py +505 -0
  87. polyctl/skill/SKILL.md +365 -0
  88. polyctl/skillkit.py +503 -0
  89. polyctl/storage.py +2055 -0
  90. polyctl/telegram.py +327 -0
  91. polyctl/tiers.py +123 -0
  92. polyctl/verdicts.py +41 -0
  93. polyctl/watchdog.py +3887 -0
  94. polyctl/web.py +337 -0
  95. polyctl-0.7.0.dist-info/METADATA +235 -0
  96. polyctl-0.7.0.dist-info/RECORD +99 -0
  97. polyctl-0.7.0.dist-info/WHEEL +4 -0
  98. polyctl-0.7.0.dist-info/entry_points.txt +5 -0
  99. polyctl-0.7.0.dist-info/licenses/LICENSE +21 -0
polyctl/watchdog.py ADDED
@@ -0,0 +1,3887 @@
1
+ #!/usr/bin/env python3
2
+ """polyctl-watchdog — restart a stalled implementer, cheaply.
3
+
4
+ Turn-based agents stop when a turn ends, mid-slice or not. Something has to
5
+ notice and nudge. Detection here is free: a file mtime and a few `git` calls.
6
+ Only the nudge itself costs anything, and it costs exactly what the work would
7
+ have cost when a human noticed twenty minutes later.
8
+
9
+ Guards, in order — the point is to spend nothing when spending would not help:
10
+
11
+ -1. Has a human taken the tree? (.polyctl/hold — overrides everything)
12
+ -1b. Has the running slice spent past its budget, or is a reading its budget needs missing?
13
+ (park it; nothing resumes it)
14
+ 0. Is an agent already WORKING here? (cwd match AND a moving transcript; a
15
+ process alive but long silent is hung,
16
+ and is reaped rather than waited on)
17
+ 1. Is the session actually idle? (mtime)
18
+ 1b. Did it stop on its usage limit? (park the running slice until the reset)
19
+ 2. Is there open work to continue? (mailbox / dirty tree / NEEDS_CHANGES)
20
+ 2b. Did the queue run dry rather than end? (wake the architect, not the implementer)
21
+ 3. Is the slice over its round budget? (if so: notify a human, never nudge)
22
+ 4. Is there provider quota left? (keyflip, if installed)
23
+ 4b. Is the provider itself degraded? (5xx at the end of our own nudge log)
24
+ 4c. Are reports sitting unhandled? (wake the architect — its own pid guard)
25
+ 5. Did the last nudge achieve anything? (backoff, then hand over to a human)
26
+ 6. Has a person seen what a harness reads here? (asked before every turn it
27
+ starts: configuration an agent changed
28
+ and nobody saw holds them all)
29
+
30
+ Run it from cron/launchd every couple of minutes, or once by hand.
31
+
32
+ polyctl-watchdog --root ~/work/project # act
33
+ polyctl-watchdog --root ~/work/project --dry-run # decide, change nothing
34
+ """
35
+ import argparse
36
+ from contextvars import ContextVar
37
+ import json
38
+ import math
39
+ import os
40
+ import re
41
+ import subprocess
42
+ import shutil
43
+ import signal
44
+ import sys
45
+ import time
46
+
47
+ from . import journal as J
48
+ from . import language
49
+ from . import lib as A
50
+ from . import settings as S
51
+ from .procs import TableUnreadable
52
+
53
+ STATE_DIR = os.path.join(A.HOME, ".polyctl")
54
+ _DRY_RUN = ContextVar("polyctl_watchdog_dry_run", default=False)
55
+ # A cycle after a long silence gathers what it would ring into one notice (RESUME-QUIET).
56
+ _RESUME = ContextVar("polyctl_watchdog_resume", default=None)
57
+
58
+ # Every decision line this module prints is also kept, so a cycle can be read
59
+ # back as a trace: what was measured, what was decided, in what order. The one
60
+ # question that cost this project the most — "why did it not act?" — used to be
61
+ # answerable only by reading a log of prose. `polyctl watchdog explain` runs a dry
62
+ # cycle and prints the trace; `polyctl watchdog trace` reads the recorded ones.
63
+ import builtins as _builtins
64
+ UTF8 = "utf-8" # every text file polyctl writes or reads; Windows would otherwise use cp1252
65
+ _TRACE = []
66
+ _FACTS = {}
67
+
68
+
69
+ def print(*args, **kw):
70
+ line = " ".join(str(a) for a in args)
71
+ _TRACE.append(line)
72
+ _builtins.print(*args, **kw)
73
+
74
+
75
+ def cycles_path(root):
76
+ key = A.project_key(root)
77
+ return os.path.join(STATE_DIR, A.project_file_name("cycles", key))
78
+
79
+
80
+ def record_cycle(root, args, started=None):
81
+ if getattr(args, "dry_run", False):
82
+ return
83
+ now = time.time()
84
+ rec = {"at": int(now), "verdict": _TRACE[-1] if _TRACE else "",
85
+ "trace": list(_TRACE), "facts": dict(_FACTS)}
86
+ if started is not None:
87
+ # Without a start, a cycle that ran for minutes and a machine that ran
88
+ # none leave the same gap in this log.
89
+ rec.update(started=int(started), seconds=round(now - started, 1))
90
+ try:
91
+ p = cycles_path(root)
92
+ os.makedirs(STATE_DIR, exist_ok=True)
93
+ if os.path.exists(p) and os.path.getsize(p) > 2_000_000:
94
+ tail = open(p, errors="replace", encoding=UTF8).read()[-1_000_000:]
95
+ open(p, "w", encoding=UTF8).write(tail[tail.find("\n") + 1:])
96
+ with open(p, "a", encoding=UTF8) as fh:
97
+ fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
98
+ except OSError:
99
+ pass
100
+
101
+
102
+ def cycles(root, last=20):
103
+ p = cycles_path(root)
104
+ if not os.path.exists(p):
105
+ return []
106
+ rows = []
107
+ for line in open(p, errors="replace", encoding=UTF8):
108
+ try:
109
+ rows.append(json.loads(line))
110
+ except ValueError:
111
+ pass
112
+ return rows[-last:]
113
+
114
+
115
+ # What a nudge, a wake and a refill tell the turn they start is in language.py, in the project's language,
116
+ # read with the config each cycle loads (LANGUAGE-PROMPTS): `prompt.nudge` with what a nudge adds,
117
+ # `prompt.wake` and `prompt.refill`.
118
+
119
+
120
+ def cycle_health(root, last=720):
121
+ """How long recent cycles took, and the longest silence between two of them.
122
+
123
+ A gap in the cycle log has two causes - no cycle ran, or one ran for minutes
124
+ and held the next back - and a row that carries only its end time cannot tell
125
+ them apart. One project's log once showed gaps of 5 to 27 minutes over four
126
+ and a half hours with no sleep recorded, and nothing on disk could say which
127
+ it was. Rows written before `started` existed count by their end times.
128
+ """
129
+ rows = [r for r in cycles(root, last) if isinstance(r.get("at"), (int, float))]
130
+ if not rows:
131
+ return None
132
+ timed = [r for r in rows if isinstance(r.get("seconds"), (int, float))]
133
+ longest = max(timed, key=lambda r: r["seconds"]) if timed else None
134
+ gap, gap_at = 0, None
135
+ for previous, current in zip(rows, rows[1:]):
136
+ silence = current.get("started", current["at"]) - previous["at"]
137
+ if silence > gap:
138
+ gap, gap_at = silence, current["at"]
139
+
140
+ def when(t):
141
+ return time.strftime("%d %b %H:%M", time.localtime(t))
142
+
143
+ return {"count": len(rows),
144
+ "longest_seconds": longest["seconds"] if longest else None,
145
+ "longest_at": when(longest["at"]) if longest else None,
146
+ "gap_minutes": gap / 60, "gap_at": when(gap_at) if gap_at else None}
147
+
148
+
149
+ def child_path():
150
+ """A PATH the nudged agent can actually build with.
151
+
152
+ Two different failures hide here. launchd gives us a minimal PATH, so we
153
+ cannot find the agent CLI. And the agent we spawn inherits whatever PATH we
154
+ hand it — if that lacks the toolchain (node, npm, a version-manager shim),
155
+ the agent starts its turn and then cannot run its own gates. Resolve both by
156
+ building one PATH that covers the user's real toolchain, including any
157
+ version-manager shim directory the current interpreter can see.
158
+ """
159
+ parts = [os.environ.get("PATH", "")]
160
+ # A bundle's CLI may be off PATH: hand children the same declared install directories
161
+ # binary_candidates searches, including those outside the home (WARP-ADAPTER).
162
+ for d in list(S.get(None, "binaries.extra_dirs")) + list(A._BIN_DIRS) + [
163
+ "/usr/bin", "/bin", "/usr/sbin", "/sbin"]:
164
+ parts.append(os.path.expanduser(d))
165
+ # Version-manager installs keep node beside the agent binary; a wake that
166
+ # resolves the newest agent there must find that node too.
167
+ agents = sorted({binary for adapter in A.package_adapters().values()
168
+ if (adapter.get("detect") or {}).get("processes") for binary in A.adapter_binaries(adapter)})
169
+ for cand in [c for binary in agents for c in A.binary_candidates(binary)] + A.binary_candidates("node"):
170
+ parts.append(os.path.dirname(cand))
171
+ # fnm / nvm / asdf style shims: whichever one currently owns `node`
172
+ node = A.program_path("node")
173
+ if node:
174
+ parts.append(os.path.dirname(node))
175
+ for base in ("~/.local/state/fnm_multishells", "~/.nvm/versions/node", "~/.asdf/shims"):
176
+ b = os.path.expanduser(base)
177
+ if os.path.isdir(b):
178
+ try:
179
+ for entry in sorted(os.listdir(b), reverse=True)[:3]:
180
+ cand = os.path.join(b, entry, "bin")
181
+ if os.path.isdir(cand):
182
+ parts.append(cand)
183
+ elif os.path.isdir(os.path.join(b, entry)):
184
+ parts.append(os.path.join(b, entry))
185
+ except OSError:
186
+ pass
187
+ seen, out = set(), []
188
+ for chunk in os.pathsep.join(parts).split(os.pathsep):
189
+ if chunk and chunk not in seen:
190
+ seen.add(chunk)
191
+ out.append(chunk)
192
+ return os.pathsep.join(out)
193
+
194
+
195
+ # Legacy call sites that do not declare an audience are classified by title.
196
+ # Explicit audience is authoritative: wording and localization must never turn a
197
+ # human alarm into an architect-only ledger row (or vice versa).
198
+ HUMAN_AUDIENCE = ("out of quota", "agent stuck", "nudge failed", "watchdog",
199
+ "turns piling up", "needs you", "provider degraded")
200
+
201
+
202
+ def for_human(title):
203
+ # Match the words after the "<project>: " prefix only. Titles name the project,
204
+ # and a project called, say, "watchdog-lab" must not turn every alert it raises
205
+ # into a human one by its name alone.
206
+ subject = title.split(": ", 1)[1] if ": " in title else title
207
+ return any(k in subject.lower() for k in HUMAN_AUDIENCE)
208
+
209
+
210
+ # The text travels in the environment and is never spliced into the script (#9).
211
+ TOAST_SCRIPT = (
212
+ "[Windows.UI.Notifications.ToastNotificationManager, Windows.UI.Notifications, ContentType = WindowsRuntime] > $null; "
213
+ "$t = [Windows.UI.Notifications.ToastNotificationManager]::GetTemplateContent("
214
+ "[Windows.UI.Notifications.ToastTemplateType]::ToastText02); "
215
+ "$x = $t.GetElementsByTagName('text'); "
216
+ "$x.Item(0).AppendChild($t.CreateTextNode($env:POLYCTL_TOAST_TITLE)) > $null; "
217
+ "$x.Item(1).AppendChild($t.CreateTextNode($env:POLYCTL_TOAST_BODY)) > $null; "
218
+ "[Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier("
219
+ "'{1AC14E77-02E7-4E5D-B744-2EB1AE5198B7}\\WindowsPowerShell\\v1.0\\powershell.exe')"
220
+ ".Show([Windows.UI.Notifications.ToastNotification]::new($t))"
221
+ )
222
+
223
+
224
+ def desktop_notify(title, msg, cfg=None):
225
+ """A desktop notification where the platform has one; never an exception (audit).
226
+
227
+ `osascript` exists only on macOS. Elsewhere its FileNotFoundError ended the cycle
228
+ before the notice was recorded or the state saved. On Windows a toast is shown
229
+ through PowerShell when the project's `toast` feature is on (#9); unverified on a
230
+ Windows machine until the Windows lane runs it. Windows' own PowerShell comes from its
231
+ system directory, never one the current directory holds (WINDOWS-NO-CWD-EXEC).
232
+ """
233
+ if sys.platform == "win32":
234
+ from . import features as F
235
+ shell = A.program_path("powershell") or A.program_path("pwsh")
236
+ if not (cfg and F.enabled(cfg, "toast")) or not shell:
237
+ return False
238
+ try:
239
+ done = subprocess.run([shell, "-NoProfile", "-NonInteractive", "-Command", TOAST_SCRIPT],
240
+ capture_output=True, timeout=20,
241
+ env=dict(os.environ, POLYCTL_TOAST_TITLE=str(title), POLYCTL_TOAST_BODY=str(msg)))
242
+ except (OSError, subprocess.SubprocessError):
243
+ return False
244
+ return done.returncode == 0
245
+ osascript = A.program_path("osascript") if sys.platform == "darwin" else None
246
+ if not osascript:
247
+ return False
248
+ title, msg = (str(part).replace('"', "'").replace("\\", "/") for part in (title, msg))
249
+ try:
250
+ subprocess.run([osascript, "-e", f'display notification "{msg}" with title "{title}"'],
251
+ capture_output=True, timeout=10)
252
+ return True
253
+ except (OSError, subprocess.SubprocessError):
254
+ return False
255
+
256
+
257
+ def notify(title, msg, root=None, key=None, window=1800, audience=None, level=None,
258
+ quiet_until=None, evidence=None, what=None, red_after=None):
259
+ """Raise an alert at most once per window, and always record that we did.
260
+
261
+ An explicit audience is a routing decision and is never inferred again from
262
+ the title. Title inference exists only for legacy callers that omit it.
263
+
264
+ The watchdog runs every two minutes. A guard that notifies on each run turns
265
+ a single ongoing condition into thirty alerts an hour, and a human who learns
266
+ to swipe those away has effectively turned the alerting off — which is worse
267
+ than not alerting, because everyone still believes it works.
268
+
269
+ A condition that stands and can say what it is about passes `what`: it rings
270
+ the desktop and the phone once for what it says, then climbs the ladder
271
+ without ringing them again - red mails on its schedule - until it says
272
+ something else. A window alone repeats: on 2026-09-17 "needs you" rang every
273
+ ten minutes about one request nobody could answer (NOTICE-NOISE).
274
+
275
+ A check whose own thresholds say when it is red passes `red_after`, the seconds an
276
+ orange stands before the ladder rings it red; otherwise it is
277
+ `alarms.red_after_minutes`. An unseen request is red at its red threshold, and an
278
+ hour of orange mailed it two hours before that (NOISE-REPEATS).
279
+
280
+ Recording happens either way. A suppressed alert is evidence too: a long run
281
+ of them says the condition has held for a long time.
282
+ """
283
+ key = key or title
284
+ if audience is None:
285
+ audience = "human" if for_human(title) else "architect"
286
+ if audience not in ("architect", "human"):
287
+ raise ValueError("unknown notification audience: %s" % audience)
288
+ dry_run = _DRY_RUN.get()
289
+ resuming = _RESUME.get()
290
+ # Architect-audience alerts are recorded and delivered through the mailbox and
291
+ # the wake; they never use a human channel.
292
+ if audience == "architect":
293
+ if dry_run:
294
+ print(f"DRY RUN: would record architect notice: {title}")
295
+ elif root:
296
+ A.record_notice(root, title, msg, sent=False, key=key, evidence=evidence)
297
+ if resuming is not None:
298
+ resuming["folded"].append({"key": key, "title": title, "msg": msg, "audience": audience})
299
+ return False
300
+ # The ladder: this is an orange (a person must act). Standing an hour, it
301
+ # rings red and goes to mail — the channel people open when they wake up.
302
+ project = (A.project_key(root) if root else "polyctl")
303
+ if red_after is None:
304
+ try:
305
+ red_after = S.get(A.load_config(root) if root else None, "alarms.red_after_minutes") * 60
306
+ except Exception:
307
+ red_after = A.ALARM_RED_AFTER
308
+ # Dry-run uses the same alarm state and calculation as a live cycle, but the
309
+ # preview is deliberately not persisted.
310
+ # A snoozed alarm is recorded, not rung: off the channels and off the ladder, so
311
+ # it neither turns red nor mails until its snooze ends.
312
+ snoozed = A.alarm_snoozed(project, key)
313
+ if snoozed:
314
+ until = time.strftime("%d %b", time.localtime(snoozed["until"]))
315
+ if dry_run:
316
+ print(f"DRY RUN: would hold {title}: snoozed until {until} by {snoozed.get('by')}")
317
+ elif root:
318
+ A.record_notice(root, title, f"{msg} [snoozed until {until}]", sent=False, key=key, evidence=evidence)
319
+ return False
320
+ ring, episode = A.alarm_touch(
321
+ project, key, level or "orange", red_after=red_after, title=title,
322
+ persist=not dry_run, quiet_until=quiet_until, evidence=evidence, what=what,
323
+ )
324
+ # After a silence the ladder still moves, so one condition keeps one alarm (#106), but
325
+ # nothing rings on its own: the cycle's one resume notice names it (RESUME-QUIET).
326
+ if resuming is not None:
327
+ resuming["folded"].append({"key": key, "title": title, "msg": msg, "audience": audience, "ring": ring,
328
+ "what": what})
329
+ if dry_run:
330
+ print(f"DRY RUN: would name in the resume notice: {title}")
331
+ elif root:
332
+ A.record_notice(root, title, f"{msg} [named in the resume notice]", sent=False, key=key,
333
+ evidence=evidence)
334
+ return False
335
+ if ring == "red" and episode.get("red_due"):
336
+ if dry_run:
337
+ print(f"DRY RUN: would send red e-mail: {title}")
338
+ else:
339
+ try:
340
+ from . import email
341
+ since = time.strftime("%d %b %H:%M", time.localtime(episode.get("first", time.time())))
342
+ # The mail is worded in the project's language; its key is the same in any (LANGUAGE-OUTPUT).
343
+ written = _written_in(root)
344
+ body = (f"{msg}\n\n" + language.text(written, "alarm.standing", since=since,
345
+ count=episode.get("count", 1), project=root or "?")
346
+ + ("\n\n" + language.text(written, "alarm.evidence") + "\n"
347
+ + "\n".join(A.evidence_lines(evidence)) if evidence else ""))
348
+ if email.send(f"{title}", body, root):
349
+ A.alarm_mailed(project, key, what=what)
350
+ if root:
351
+ A.record_notice(root, title, msg, sent=True, key="mail:" + key, evidence=evidence)
352
+ except Exception:
353
+ pass
354
+ # Told once, with a known end and nothing new to say before it: record, do not ring.
355
+ # A resume notice that named a red told it as a mail does (RESUME-QUIET).
356
+ told = episode.get("red_sent") if episode.get("red_sent") is not None else episode.get("named_at")
357
+ if told is not None and not episode.get("news") and time.time() < float(episode.get("quiet_until") or 0):
358
+ until = time.strftime("%d %b %H:%M", time.localtime(float(episode["quiet_until"])))
359
+ if dry_run:
360
+ print(f"DRY RUN: would hold {title}: told once, quiet until {until}")
361
+ elif root:
362
+ A.record_notice(root, title, f"{msg} [told once; quiet until {until}]", sent=False, key=key, evidence=evidence)
363
+ return False
364
+ # Rung once for what it says, and saying nothing new: the ladder climbs, the channels stay quiet.
365
+ if what is not None and episode.get("rang_at") is not None and not episode.get("news"):
366
+ if dry_run:
367
+ print(f"DRY RUN: would hold {title}: rung once and nothing new since")
368
+ elif root:
369
+ A.record_notice(root, title, f"{msg} [rung once; nothing new since]", sent=False, key=key,
370
+ evidence=evidence)
371
+ return False
372
+ if root and A.notice_recently_sent(root, key, window):
373
+ if dry_run:
374
+ print(f"DRY RUN: would suppress {ring} desktop/Telegram channels "
375
+ f"(recent notice): {title}")
376
+ else:
377
+ A.record_notice(root, title, msg, sent=False, key=key, evidence=evidence)
378
+ return False
379
+ if root and storm(root):
380
+ if dry_run:
381
+ print(f"DRY RUN: would suppress {ring} desktop/Telegram channels "
382
+ f"(alert storm): {title}")
383
+ else:
384
+ A.record_notice(root, title, msg, sent=False, key=key, evidence=evidence)
385
+ if not A.notice_recently_sent(root, "storm", 3600):
386
+ A.record_notice(root, f"{project}: alert storm", "12+ alerts in an hour; further ones "
387
+ "are recorded only — polyctl notices", sent=True, key="storm")
388
+ desktop_notify(f"{project}: alert storm",
389
+ "12+ alerts in an hour; further ones recorded only (polyctl notices)")
390
+ return False
391
+ if dry_run:
392
+ print(f"DRY RUN: would ring {ring} desktop/Telegram channels: {title}")
393
+ return False
394
+ safe = msg.replace('"', "'")[:200]
395
+ try:
396
+ channel_cfg = A.load_config(root) if root else None
397
+ except Exception:
398
+ channel_cfg = None
399
+ desktop_notify(title, safe, channel_cfg)
400
+ phone_text = f"*{title}*\n{_markdown_plain(msg)}"
401
+ try:
402
+ from . import telegram
403
+ # The message as written, as the resume notice is sent: it can name a report, and one stray
404
+ # underscore in a name makes the phone refuse the whole message as markup (WAITING-ONE-ALARM).
405
+ telegram.send(phone_text, root)
406
+ except Exception:
407
+ pass # a phone being unreachable is not a failure
408
+ try:
409
+ from . import ntfy
410
+ ntfy.send(phone_text, root, level=ring)
411
+ except Exception:
412
+ pass
413
+ A.alarm_rang(project, key, what=what)
414
+ if root:
415
+ A.record_notice(root, title, msg, sent=True, key=key, evidence=evidence)
416
+ return True
417
+
418
+
419
+ def _written_in(root):
420
+ """The project config a text for a person is worded from, for a caller that holds none (LANGUAGE-OUTPUT).
421
+
422
+ Read from the file as it stands, without resolving a session as load_config does: only its
423
+ `language` is wanted. None without a root, which is the machine's choice.
424
+ """
425
+ return A.project_config_document(root)["config"] if root else None
426
+
427
+
428
+ def touch_architect_quota(root, st, cfg=None):
429
+ """Keep a standing architect-quota alarm alive until its reset window.
430
+
431
+ ``notify`` touches the alarm episode before applying its desktop rate limit,
432
+ so calling this on every watchdog cycle rings orange once, advances to red
433
+ after the configured interval, and does not create a notification storm.
434
+ Its words are in the project's language, and its key is not (LANGUAGE-OUTPUT).
435
+
436
+ The one notice says when the wakes start again, and says it again only when that
437
+ time moves (ARCHITECT-WAKE-QUOTA). Held quiet until the end it had named once it was
438
+ mailed, the notice about a five-hour stop left a week's limit read after it untold.
439
+ """
440
+ until = st.get("arch_quota_until", 0)
441
+ if until <= time.time():
442
+ return False
443
+ written = _written_in(root) if cfg is None else cfg
444
+ text = (st.get("wake_error") or {}).get("text") or "architect quota exhausted"
445
+ return notify(
446
+ language.text(written, "alarm.architect-quota-title", project=A.project_key(root)),
447
+ language.text(written, "alarm.architect-quota", error=text[:100], reset=reset_when(until)),
448
+ root,
449
+ key="architect-quota",
450
+ window=6 * 3600,
451
+ audience="human",
452
+ quiet_until=until,
453
+ what=f"until:{int(until)}",
454
+ )
455
+
456
+
457
+ def reset_when(at, now=None):
458
+ """When held wakes start again, as a person reads it: the clock time within a day, the date too beyond one.
459
+
460
+ A weekly limit's end told as "04:00" reads as the coming night, days before the
461
+ week is over (ARCHITECT-WAKE-QUOTA).
462
+ """
463
+ now = time.time() if now is None else now
464
+ return time.strftime("%H:%M", time.localtime(float(at))) if float(at) - now < 86400 else _when(at)
465
+
466
+
467
+ def _announce_resolved(root, e):
468
+ """Record an alarm that stopped being raised; never louder than the raise (#40).
469
+
470
+ An episode ends when nothing has raised it for two hours, which is not proof the
471
+ condition is gone: a check can stop running, or stop repeating itself. On
472
+ 2026-09-15 two episodes about credits that were still exhausted were announced
473
+ "resolved" by e-mail. So the end is recorded as what it is, a red episode's end
474
+ shows once on the desktop, and nothing goes to Telegram or e-mail.
475
+ """
476
+ title = f"{e.get('project', '')}: no longer raised — {e.get('key', '')}"
477
+ msg = (f"stood {e.get('age_s', 0) // 60}m, raised {e.get('count', 1)}×"
478
+ + ("; was red" if e.get("red_sent") else ""))
479
+ was_red = bool(e.get("red_sent")) or e.get("ring") == "red"
480
+ A.record_notice(root, title, msg, sent=was_red, key="resolved:" + e.get("key", ""))
481
+ if not was_red:
482
+ return
483
+ try:
484
+ desktop_notify(title, msg, A.load_config(root) if root else None)
485
+ except Exception:
486
+ pass
487
+
488
+
489
+ def storm(root, limit=12):
490
+ """More than `limit` alerts sent in the last hour is a storm: the person has
491
+ stopped reading them. Record, do not ring — except red, which still mails."""
492
+ cut = time.time() - 3600
493
+ sent = [n for n in A.notices(root, limit=400) if n.get("sent") and n.get("at", 0) > cut
494
+ and not str(n.get("key", "")).startswith(("tg:", "mail:", "resolved:", "storm"))]
495
+ return len(sent) >= limit
496
+
497
+
498
+ def state_path(root, record=True):
499
+ key = A.project_key(root) if record else A.next_project_key(root)
500
+ return os.path.join(STATE_DIR, A.project_file_name("watchdog-state", key))
501
+
502
+
503
+ def load_state(root):
504
+ """The watchdog's state for a project: a fresh one when the file cannot be read or holds no JSON object.
505
+
506
+ Every reader asks it for keys. A file of `null`, a list or a string - it is in ~/.polyctl, which an agent
507
+ can write - was handed on as it was, and `polyctl status`, the MCP status tool and the cycle stopped on
508
+ the first `.get` (STATE-SHAPE).
509
+ """
510
+ try:
511
+ with open(state_path(root, record=False), encoding=UTF8) as fh:
512
+ state = json.load(fh)
513
+ except Exception:
514
+ state = None
515
+ return state if isinstance(state, dict) else {"attempts": 0, "last_nudge": 0, "last_size": 0}
516
+
517
+
518
+ def save_state(root, st):
519
+ """Write the watchdog's state whole, or not at all (JOURNAL).
520
+
521
+ It was written over in place: a kill -9, a power cut or a Ctrl+C mid-write left the file torn,
522
+ a torn file reads as a fresh state, and the markers that a park was resumed, a report handed to
523
+ a wake, a nudge counted were gone - the next cycle could resume or wake a second time. It is now
524
+ replaced whole and synced, so a reader finds the state before the write or after it.
525
+ """
526
+ if _DRY_RUN.get():
527
+ return
528
+ from .storage import replace_file_durably
529
+ os.makedirs(STATE_DIR, exist_ok=True)
530
+ replace_file_durably(state_path(root), json.dumps(st, indent=2).encode(UTF8))
531
+
532
+
533
+ # How long a claimed wake or refill holds when nothing says how it went: the fifteen minutes a failed wake waits.
534
+ WAKE_LEASE = 900
535
+
536
+
537
+ def arch_alive(root, architect):
538
+ """Is any configured architect turn currently alive?
539
+
540
+ A remembered pid or lock can be reused after its process dies. Measure the
541
+ configured binary and cwd again instead; this still prevents two headless
542
+ watchdog helpers while releasing a dead one on the next scan. A wake the
543
+ journal holds counts too: one cut off between its start and its record
544
+ was a headless architect no record named (JOURNAL-2).
545
+ """
546
+ return A.architect_turn_present(root, architect) or _architect_step_under_way(root)
547
+
548
+
549
+ def _architect_step_under_way(root):
550
+ """Whether a wake or refill the journal holds is still under way: claimed and not yet started, or its process
551
+ alive and the one that started (JOURNAL-2)."""
552
+ try:
553
+ steps = J.under_way(A.project_file(root, "journal"), "architect")
554
+ except Exception:
555
+ return False
556
+ for _, pid, start in steps:
557
+ if pid is None:
558
+ return True # started, or cut off starting: the lease says how long to believe it
559
+ if _step_process_alive(pid, start):
560
+ return True
561
+ # Past its lease a step's process still holds while it is provably the one recorded: a long architect turn
562
+ # cut off before it was registered is no record's but the journal's.
563
+ try:
564
+ steps = J.running(A.project_file(root, "journal"), "architect")
565
+ except Exception:
566
+ return False
567
+ return any(_step_process_proven(pid, start) for _, pid, start in steps)
568
+
569
+
570
+ def _step_process_alive(pid, start):
571
+ """Whether the process a step started still runs: its pid alive and its start the one recorded."""
572
+ return bool(pid) and A._pid_alive(pid) and (start is None or A._process_start(pid) in (None, start))
573
+
574
+
575
+ def _step_process_proven(pid, start):
576
+ """Whether the process a step started still runs, its start recorded and read again the same: past a lease
577
+ an unreadable start proves nothing, and a reused pid never holds."""
578
+ return bool(pid) and start is not None and A._pid_alive(pid) and A._process_start(pid) == start
579
+
580
+
581
+ def _close_deferred_done(root, st):
582
+ """Close a deferral the work it waited for has since done: a wake or a nudge started after it (JOURNAL-4).
583
+
584
+ Closed in the cycle after the work, not as the work starts: the notice a silence owes names what was
585
+ deferred through it, and that notice is written after the wake in the same cycle.
586
+ """
587
+ done = {"wake": float(st.get("last_arch_wake") or 0), "nudge": float(st.get("last_nudge") or 0)}
588
+ closed = []
589
+ for row in A.deferred_open(root):
590
+ if done.get(row.get("kind"), 0) >= _deferred_since(row):
591
+ A.deferred_close(root, row["id"], "woken" if row.get("kind") == "wake" else "nudged")
592
+ closed.append(row)
593
+ return closed
594
+
595
+
596
+ def _deferred_since(row):
597
+ """The earliest a start counts as after a deferral. A whole-second `at` was cut down to its second, and the
598
+ deferral was written before the next second began: work from that second on is after it (JOURNAL-4-3). A
599
+ fractional `at` is the moment itself, and only work strictly after it counts."""
600
+ import math
601
+ at = row.get("at") or 0
602
+ return float(at) + 1.0 if isinstance(at, int) else math.nextafter(float(at), math.inf)
603
+
604
+
605
+ def _claim_step(root, step):
606
+ """Whether this cycle may start `step`; where the journal cannot be read, it may, as before it (JOURNAL-2).
607
+
608
+ A step whose process has ended holds nothing: a wake that failed is retried after its fifteen minutes, as
609
+ the wake's own record says, whatever its lease. Only one cut off before its start was written, or one
610
+ still running, holds its key.
611
+ """
612
+ path = A.project_file(root, "journal")
613
+ try:
614
+ current = J.step(path, step)
615
+ if current and current["state"] == "started":
616
+ if _step_process_proven(current["pid"], current["start"]):
617
+ return False # its process still runs: it holds its key past its lease
618
+ if not _step_process_alive(current["pid"], current["start"]):
619
+ J.finished(path, step, "ended")
620
+ return J.claim(path, step, "architect", WAKE_LEASE)
621
+ except Exception as exc:
622
+ print(f"the journal could not be read ({type(exc).__name__}); starting {step} on the process scan alone")
623
+ # Failing open is meant - a broken journal must not stop every wake - and a person is told that two
624
+ # wakes are now kept apart by the process scan alone (JOURNAL-2-2), unless the telling fails too: the step
625
+ # starts whatever the alarm does (JOURNAL-2-3).
626
+ try:
627
+ notify(f"{A.project_key(root)}: the watchdog's journal cannot be written",
628
+ f"{type(exc).__name__}: {exc}. Wakes and refills start on the process scan alone until it can.",
629
+ root, key="journal-unwritable", window=6 * 3600, audience="human")
630
+ except Exception as told:
631
+ print(f"the journal's alarm could not be raised ({type(told).__name__})")
632
+ return True
633
+
634
+
635
+ def _adopt_started_steps(root):
636
+ """Register the architect a cut-off cycle started and never registered: the journal holds its pid and start.
637
+
638
+ Unregistered, it was no helper - the implementer's checks counted it as a turn in this tree and reaped it
639
+ as a hung one - and past its lease a second wake started beside it (JOURNAL-2).
640
+ """
641
+ try:
642
+ steps = J.running(A.project_file(root, "journal"), "architect")
643
+ except Exception:
644
+ return
645
+ helpers = A.helper_pids(root, "architect")
646
+ for _, pid, start in steps:
647
+ if pid not in helpers and _step_process_proven(pid, start):
648
+ A.helper_register(root, pid, "architect")
649
+ print(f"the architect pid {pid} a cut-off cycle started is registered now")
650
+
651
+
652
+ def _record_start(root, step, pid):
653
+ """The pid and start of a step's process, written the moment it has one (JOURNAL-2)."""
654
+ try:
655
+ J.started(A.project_file(root, "journal"), step, pid, A._process_start(pid, refresh=True))
656
+ except Exception as exc:
657
+ print(f"the journal did not take the start of {step} ({type(exc).__name__}); the process scan stands in")
658
+
659
+
660
+ def _logs_written(root):
661
+ """The spawn logs a process an earlier cycle started may still write, each left whole until it ends (JOURNAL-3-2).
662
+
663
+ Every spawn is made inside a cycle, under the cycle's lock, after the logs were bounded: what may write
664
+ one as it is bounded is a turn or a wake an earlier cycle started. Where that cannot be told, none of
665
+ the three is trimmed this cycle. A turn is the one the state names, or any unattended turn in this tree:
666
+ a cycle cut off between starting a turn and recording it left one no state names, which the process
667
+ scan that keeps a second turn from starting beside it still finds (JOURNAL-3-3).
668
+ """
669
+ key = A.project_key(root)
670
+ log = lambda what: os.path.join(STATE_DIR, A.project_file_name(what, key)) # noqa: E731
671
+ try:
672
+ cfg = A.load_config(root)
673
+ impl = cfg.get("implementer") or {}
674
+ turn = child_alive(load_state(root)) or bool(
675
+ impl and A.agent_pids(root, A.load_adapter(impl.get("adapter", ""), root), headless_only=True))
676
+ busy = {log("nudge-log")} if turn else set()
677
+ if arch_alive(root, cfg.get("architect") or {}):
678
+ busy |= {log("escalate-log"), log("refill-log")}
679
+ return busy
680
+ except Exception:
681
+ return {log("nudge-log"), log("escalate-log"), log("refill-log")}
682
+
683
+
684
+ def child_alive(st):
685
+ """Is the turn we last started still running?
686
+
687
+ A transcript mtime cannot answer this. A turn retrying a provider 5xx sits in
688
+ backoff writing nothing, which looks exactly like a turn that ended — and
689
+ nudging then starts a SECOND turn on the same session id. Two turns editing
690
+ one tree produce a half-finished refactor: one renames a field, the other
691
+ still calls the old name, and the build breaks in a way neither writer can
692
+ attribute to itself. Ask the OS instead of inferring from a file.
693
+ """
694
+ pid = st.get("child_pid")
695
+ if not pid:
696
+ return False
697
+ # A bare pid can be reused by an unrelated process, and os.kill(pid, 0) sends
698
+ # CTRL_C_EVENT on Windows. Liveness comes from the portable probe, identity from
699
+ # the start recorded when the turn was spawned.
700
+ if not A._pid_alive(pid):
701
+ return False
702
+ started = st.get("child_start")
703
+ now = A._process_start(pid) if started is not None else None
704
+ if now is not None:
705
+ return now == started
706
+ # No start was read when the turn was spawned, so the pid alone speaks for it, and a pid another process
707
+ # was given since kept the refill waiting for good. It is believed for as long as a turn may run
708
+ # (CYCLE-SAFETY-2). So is a turn whose start cannot be read now: a table the platform could not read
709
+ # holds none, and the turn read as ended while its pid ran (WINDOWS-TABLE-CLOSED).
710
+ return time.time() - float(st.get("last_nudge") or 0) < UNVERIFIED_CHILD_SECONDS
711
+
712
+
713
+ # How long a turn whose start could not be read is taken to be running while its pid is (CYCLE-SAFETY-2).
714
+ UNVERIFIED_CHILD_SECONDS = 6 * 3600
715
+
716
+
717
+ def provider_degraded(root, window=900):
718
+ """A provider-side failure in the *most recent* nudge, recent enough to hold.
719
+
720
+ An exhausted quota and an outage both silence the agent, but only one of them
721
+ is fixed by nudging. Naming it here keeps "the model is down" from being
722
+ reported — and retried — as "the agent is stuck".
723
+
724
+ Scope it to the last nudge attempt, not the whole file. The log is
725
+ append-only, so an hour-old outage stays in it forever, and a check that
726
+ merely greps the tail keeps firing long after the provider recovered — which
727
+ blocks every later nudge and costs exactly the time this guard was added to
728
+ save. Read only what came after the last `=== <time> nudge ===` marker, and
729
+ only if that marker itself is inside the window.
730
+ """
731
+ key = A.project_key(root)
732
+ p = os.path.join(STATE_DIR, A.project_file_name("nudge-log", key))
733
+ if not os.path.exists(p):
734
+ return None
735
+ try:
736
+ size = os.path.getsize(p)
737
+ with open(p, errors="replace", encoding=UTF8) as fh:
738
+ fh.seek(max(0, size - 400_000))
739
+ tail = fh.read()
740
+ except OSError:
741
+ return None
742
+ marks = list(re.finditer(r"=== (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) nudge ===", tail))
743
+ if not marks:
744
+ return None
745
+ last = marks[-1]
746
+ try:
747
+ at = time.mktime(time.strptime(last.group(1), "%Y-%m-%d %H:%M:%S"))
748
+ except ValueError:
749
+ return None
750
+ if time.time() - at > window:
751
+ return None # the last attempt is old news
752
+ # Only the *end* of the segment counts. A 5xx the turn recovered from is
753
+ # history: this log had one at line 130 of 242, with a hundred lines of real
754
+ # work after it. Blocking on that would stand the watchdog down for fifteen
755
+ # minutes because of an error the agent had already handled.
756
+ segment = tail[last.end():].strip().split("\n")[-30:]
757
+ end = "\n".join(segment)
758
+ for marker in ("MODEL_TEMPORARILY_UNAVAILABLE", "ThrottlingException",
759
+ "ServiceUnavailableException", "InternalServerException",
760
+ "experiencing a high volume of traffic"):
761
+ if marker in end:
762
+ return marker
763
+ return None
764
+
765
+
766
+ def architect_command(root, cfg, prompt):
767
+ """Prepare a wake or refill before asking any program its version (ARCH-PROBE-ORDER).
768
+
769
+ The notice holder and doctor checked without the session and prompt values, so they ran version probes and
770
+ held notices for a wake that refused to start. All four callers use this order and these values; a refusal
771
+ returns (None, code, reason), a command (prepared, None, None). Preparation starts no program.
772
+ """
773
+ arch = cfg.get("architect") or {}
774
+ template = arch.get("argv") or []
775
+ if not template:
776
+ return None, "no-architect", "no architect is configured"
777
+ sess, withheld = A.session_to_resume(cfg, "architect")
778
+ if "{session}" in " ".join(template) and not sess:
779
+ return None, "session-unresolved", f"architect session not resolvable ({withheld})"
780
+ # Keep the adapter's pinned mode, rewritten grants and prompt channel in every caller's check
781
+ # (ARCH-PROBE-ORDER).
782
+ template, pinned = A.pinned_argv(A.admit_rewrites(template)[0], "architect")
783
+ plan, refused = A.prompt_plan(template, prompt, A.block_adapter(arch), detached=True)
784
+ if refused:
785
+ return None, "prompt-refused", f"the architect's prompt cannot be handed over: {refused}"
786
+ argv = [x.replace("{prompt}", prompt).replace("{session}", sess or "") for x in plan["argv"]]
787
+ search = child_path()
788
+ held = turn_hold(root, cfg, "architect")
789
+ if held:
790
+ return None, "held", held
791
+ values = {"session": sess, "prompt": arch.get("prompt")}
792
+ refused = A.turn_refusal("architect", A.block_adapter(arch), argv, root=root, search=search, values=values,
793
+ resolved=argv[0] if os.path.isabs(argv[0]) else None, prompt=prompt)
794
+ if refused:
795
+ return None, "refused", refused
796
+ return {"argv": argv, "session": sess, "plan": plan, "pinned": pinned, "values": values,
797
+ "search": search, "path": A.outside_repository(search, root)}, None, None
798
+
799
+
800
+ def architect_hold_reason(root, cfg, adapter, st, found=None, now=None):
801
+ """May a notice be held for the architect? {"holdable", "code", "reason"}.
802
+
803
+ A notice addressed to the architect reaches no person, so holding it is honest
804
+ only when an architect is going to read it: a wake already in flight, or one
805
+ this cycle will start. The rule this replaces held whenever an architect argv
806
+ was configured, and on 2026-09-14 seven notices in a row sat held behind
807
+ "architect will be woken" while the only architect was an interactive session
808
+ that acts when someone prompts it.
809
+
810
+ Built from the predicates the wake in escalate() uses; once its command is
811
+ accepted, resolving it asks candidates their version (ARCH-PROBE-ORDER).
812
+ Blockers are checked before in-flight wakes, so a wake that has already
813
+ failed is never counted as one that is coming.
814
+
815
+ `adapter` is the one the quota is read through, A.metered_adapter's (TRIAGE-HIGHS).
816
+ """
817
+ from . import features as F
818
+ now = now or time.time()
819
+ arch = cfg.get("architect") or {}
820
+ argv = arch.get("argv") or []
821
+
822
+ def no(code, reason):
823
+ return {"holdable": False, "code": code, "reason": reason}
824
+
825
+ if not argv:
826
+ return no("no-architect", "no architect is configured")
827
+ try:
828
+ present = A.architect_present(root, arch)
829
+ except TableUnreadable as exc:
830
+ # No wake starts beside a process nobody could see, so a person is told what waits (WINDOWS-TABLE-CLOSED).
831
+ return no("process-table", f"{exc}, so no architect is woken")
832
+ if present:
833
+ return no("interactive", "the architect session is interactive and acts only when someone prompts it")
834
+ if not F.enabled(cfg, "architect_wake"):
835
+ return no("wake-off", "architect wakes are switched off")
836
+ if not quota_ok(adapter):
837
+ return no("no-quota", "there is no quota headroom to wake the architect")
838
+ if st.get("arch_quota_until", 0) > now:
839
+ return no("quota-block", "the architect is at quota until " + reset_when(st["arch_quota_until"], now))
840
+ try:
841
+ running = arch_alive(root, arch)
842
+ except TableUnreadable as exc:
843
+ return no("process-table", f"{exc}, so no architect is woken")
844
+ if running:
845
+ return {"holdable": True, "code": "wake-running", "reason": "an architect wake is already running"}
846
+ provider = A.provider_of(argv)
847
+ left, reserve = A.window_headroom(provider)
848
+ urgent = any(item.get("kind") == "decision-requested" for item in (found or []))
849
+ if left is not None and left < reserve and not urgent:
850
+ return no("window-reserve", f"the machine's {provider} window has {left}% left, below the {reserve}% reserve")
851
+ command, code, stopped = architect_command(root, cfg, language.text(cfg, "prompt.wake"))
852
+ if stopped:
853
+ return no(code, stopped)
854
+ resolved, ver = A.resolve_binary(command["argv"][0], path=command["path"])
855
+ if not resolved:
856
+ return no("binary-missing", f"{argv[0]} cannot be found")
857
+ key = A.project_key(root)
858
+ err = wake_error(os.path.join(STATE_DIR, A.project_file_name("escalate-log", key)))
859
+ if err and err.get("kind") == "binary" and err.get("binary") == f"{resolved} {ver}" \
860
+ and now - (st.get("wake_error") or {}).get("at", 0) < 6 * 3600:
861
+ return no("binary-failed", f"the last wake failed with this binary: {err.get('text', '')[:80]}")
862
+ if now - st.get("last_arch_wake", 0) < 900:
863
+ return {"holdable": True, "code": "recent-wake",
864
+ "reason": f"the architect was woken {int((now - st.get('last_arch_wake', 0)) / 60)}m ago"}
865
+ return {"holdable": True, "code": "wakeable", "reason": "architect will be woken"}
866
+
867
+
868
+ def escalate(root, cfg, adapter, age, args, st, told=None):
869
+ """Hand every judgement call to the architect, once per condition per hour.
870
+
871
+ The watchdog is good at mechanical questions and bad at everything else. Its
872
+ worst failures came from answering the second kind anyway: a provider outage
873
+ read as a stuck agent, a hung process read as a live writer, a finished slice
874
+ read as work in progress. Each was defensible from the one signal it had and
875
+ wrong given the others.
876
+
877
+ So it reports instead. Facts go to the mailbox where they persist; the
878
+ architect is woken only if one is configured, because a report nobody reads is
879
+ not an escalation.
880
+
881
+ `told`, when given, collects the reports an alarm to a person stands for this cycle, so
882
+ the unseen check does not ring them under a second key (CATCHUP-POLISH).
883
+ """
884
+ # A wake retried after a failed one is told once it is known not to have failed (NOISE-REPEATS).
885
+ tell_retried_wake(root, cfg, st, dry_run=args.dry_run)
886
+ # The architect we spawned resumes with this repo as its cwd, so it reads as a
887
+ # second turn to the process check. We know its pid; exclude it and our own
888
+ # rather than making the detector guess.
889
+ found = A.anomalies(root, cfg, adapter, age, args.idle_minutes * 60,
890
+ exclude_pids={st.get("arch_pid"), os.getpid()})
891
+ # A wake spends the quota the package's adapter reads by its own command; `adapter`, as every layer loads it,
892
+ # says only how the transcript reads (TRIAGE-HIGHS).
893
+ metered = A.metered_adapter(cfg)
894
+
895
+ # A second turn is a hazard only if it persists. A real concurrent writer runs
896
+ # for minutes; a gate helper, a `polyctl` call or a one-shot process blinks in and
897
+ # out and trips a single-scan check for one cycle. Require the same extra roots
898
+ # on two consecutive scans before flagging.
899
+ prev_roots = set(st.get("last_roots") or [])
900
+ seen_several = False
901
+ kept = []
902
+ for a in found:
903
+ if a["kind"] == "several-turns-active":
904
+ seen_several = True
905
+ roots = set(a.get("roots") or [])
906
+ st["last_roots"] = list(roots)
907
+ if len(roots) < 2 or not (roots & prev_roots):
908
+ print(f"several turns seen once ({sorted(roots)}); waiting for a "
909
+ f"second scan before flagging")
910
+ continue
911
+ kept.append(a)
912
+ if not seen_several:
913
+ st["last_roots"] = []
914
+ save_state(root, st)
915
+ found = kept
916
+ if not found:
917
+ return False
918
+
919
+ woke = False
920
+ project = A.project_key(root)
921
+ hold = None
922
+ standing = {}
923
+ asked = {decision.get("id"): decision for decision in A.decisions(root, "open")}
924
+ for a in found:
925
+ key = f"anomaly:{a['kind']}"
926
+ # A handed decision must not suppress the next decision's report (TRIAGE-3-MEDIUM).
927
+ if a.get("key") and a["key"] != "implementer":
928
+ key += ":" + str(a["key"])
929
+ # An open decision reaches a person under its own alarm, `decision-open:<id>`, which names its
930
+ # question and rings whatever the architect does (#20); it is not told as "needs you" as well.
931
+ decision = asked.get(a.get("key")) if a["kind"] == "decision-requested" else None
932
+ alarm = f"decision-open:{decision['id']}" if decision else key
933
+ window = 600 if a["kind"] == "decision-requested" else 3600
934
+ if a["kind"] == "report-waiting" and not open_work(cfg, root):
935
+ continue # nothing is stuck; it can wait for a human
936
+ # Grouped implementer reports share an alarm; individual decisions and reviews each
937
+ # speak under their own key, so an earlier instance cannot hold the next (TRIAGE-3-MEDIUM).
938
+ standing.setdefault(alarm, a)
939
+ # Suppression is read-only and therefore part of explain's decision path.
940
+ # A dry-run must not claim it would write a report that a live cycle would
941
+ # suppress; it merely skips the suppressed-notice ledger write.
942
+ # An architect-audience notice is recorded unsent, so a check on sent rows
943
+ # never held and the same anomaly wrote a row every cycle (#69). Any row
944
+ # inside the window stops the repeat; the report and that row are the record.
945
+ if A.notice_recently_recorded(root, key, window):
946
+ if args.dry_run:
947
+ print(f"DRY RUN: would suppress anomaly {a['kind']} "
948
+ f"(reported within {window}s)")
949
+ continue
950
+ if args.dry_run:
951
+ print(f"DRY RUN: would report anomaly {a['kind']} to the architect")
952
+ woke = True
953
+ continue
954
+ name = A.write_report(root, cfg, a["kind"], a["facts"], key=a.get("key"))
955
+ # Say what actually happened. "reported to the architect" was untrue
956
+ # whenever no architect was configured or startable, and a notification
957
+ # that overstates its own effect is how a gap stays invisible.
958
+ # Hold for the architect only when a wake is actually coming; decided from the
959
+ # wake's own predicates, not from whether an argv is configured.
960
+ hold = hold or architect_hold_reason(root, cfg, metered, st, found)
961
+ if hold["holdable"]:
962
+ notify(f"{project}: anomaly", f"{a['kind']} — {hold['reason']}", root,
963
+ key=key, window=window, audience="architect")
964
+ elif decision:
965
+ # Its own check rings it once it has waited `decisions.human_after_minutes`; before that, and
966
+ # with no architect to act, it rings from here. A raise inside the window is not repeated.
967
+ if not A.notice_recently_recorded(root, alarm, window):
968
+ ring_decision(root, project, decision, why=hold["reason"])
969
+ else:
970
+ # What stands is what waits: rung once, it climbs to red without ringing again
971
+ # until another request waits. A quota block names its own end (NOTICE-NOISE).
972
+ # It names the reports, since when and why nobody is woken: with wakes off it
973
+ # is the only alarm for them (WAITING-ONE-ALARM).
974
+ notify(f"{project}: needs you",
975
+ f"{anomaly_waits(root, cfg, a)} — no architect will act on it: {hold['reason']}", root,
976
+ key=key, window=window, audience="human", what=anomaly_subject(a),
977
+ quiet_until=st.get("arch_quota_until") if hold["code"] == "quota-block" else None)
978
+ print(f"anomaly {a['kind']}: {'reported as ' + name if name else 'already reported'}")
979
+ woke = True
980
+ if not args.dry_run and told is not None and standing:
981
+ # An anomaly no architect will act on reaches a person as "needs you", and stands for the
982
+ # requests it groups whether or not its window let it ring this cycle (CATCHUP-POLISH).
983
+ # A dry cycle delivered no alarm: checking its coverage prepared and probed a wake before
984
+ # building the implementer's nudge (ARCH-PROBE-ORDER-2).
985
+ hold = hold or architect_hold_reason(root, cfg, metered, st, found)
986
+ if not hold["holdable"]:
987
+ told.update(name for anomaly in standing.values() for name in anomaly.get("reports") or [])
988
+ arch = cfg.get("architect") or {}
989
+
990
+ # Wake on durable state, not on the moment of detection. The notification
991
+ # throttle exists so a human is not rung every two minutes; using it to gate
992
+ # the wake as well meant the architect was only ever started on an anomaly's
993
+ # *first* sighting, and a single suppressed cycle lost it for good. What
994
+ # actually matters is whether a report is still sitting unprocessed — the
995
+ # mailbox is the state, the notification is only a bell.
996
+ # Same exclusion as the anomaly scanner: the watchdog's own reports are
997
+ # addressed to the architect but are not themselves reports awaiting the
998
+ # architect, and counting them here re-woke it endlessly on its own output.
999
+ # Addressed by role, never by an actor's name, and the watchdog's own anomaly
1000
+ # reports count: an anomaly nobody is woken for is not escalated (#69). What
1001
+ # keeps them from re-waking the architect on its own output is the handed set
1002
+ # below, not their names. The architect's notes to itself wake nobody (#18).
1003
+ pending = [m for m in A.mailbox(root, cfg["mailbox"])
1004
+ if A.to_architect(m, cfg) and not A.from_architect(m, cfg)]
1005
+ # Closing the loop out loud: an alert that says a thing was detected, with no
1006
+ # later word on whether anything came of it, is what makes someone check by
1007
+ # hand — which is the work the alert was supposed to save.
1008
+ if st.get("arch_pending") and not pending:
1009
+ if args.dry_run:
1010
+ print(f"DRY RUN: would announce {st['arch_pending']} completed architect report(s)")
1011
+ else:
1012
+ _tell_phone(root, language.text(cfg, "watchdog.architect-done", n=st["arch_pending"]))
1013
+ st["arch_pending"] = 0
1014
+ save_state(root, st)
1015
+ elif pending and not args.dry_run:
1016
+ st["arch_pending"] = len(pending)
1017
+ last_wake = st.get("last_arch_wake", 0)
1018
+ # Wake on the durable state (#69). A report is owed a wake until one that did
1019
+ # not fail has been handed it; comparing its time with the last wake dropped it
1020
+ # for good whenever that wake died at once. A wake whose log shows a transport
1021
+ # failure hands nothing back. A model's own prose saying "Error:" does not count.
1022
+ handed = dict(st.get("handed") or {})
1023
+ if handed and wake_failed(wake_error(os.path.join(STATE_DIR, A.project_file_name("escalate-log", project))),
1024
+ last_wake):
1025
+ handed = {}
1026
+ mtimes = {}
1027
+ for m in pending:
1028
+ try:
1029
+ # Whole seconds missed a report rewritten in the same second it was handed (TRIAGE-3-LOW).
1030
+ info = os.stat(os.path.join(root, cfg["mailbox"], m))
1031
+ mtimes[m] = [info.st_mtime_ns, info.st_size]
1032
+ except OSError:
1033
+ continue # acknowledged while this cycle ran
1034
+ stale = [m for m in pending if m in mtimes and handed.get(m) != mtimes[m]]
1035
+ if not args.dry_run and handed != (st.get("handed") or {}):
1036
+ st["handed"] = {m: t for m, t in handed.items() if m in mtimes}
1037
+ save_state(root, st)
1038
+ woke = bool(stale)
1039
+ # What waits, as the notices below name it: the newest report that asks, not the watchdog's
1040
+ # echo of it. A person is told again when it changes (NOTICE-NOISE).
1041
+ waiting = max(stale, key=lambda m: (not A.from_watchdog(m), mtimes.get(m, 0), m)) if stale else None
1042
+ if woke and time.time() - last_wake < 900:
1043
+ print(f"{len(stale)} unhandled report(s), but the architect was woken "
1044
+ f"{int((time.time() - last_wake) / 60)}m ago")
1045
+ woke = False
1046
+
1047
+ # Waking spends a turn on the same provider the implementer uses, so a report
1048
+ # that could have waited must not burn the window the implementer needs.
1049
+ if woke and arch.get("argv") and not quota_ok(metered):
1050
+ print("reports pending, but no quota headroom to wake the architect")
1051
+ # One handoff for what waits: the same note went to the phone again every hour the
1052
+ # architect could not be woken, with nothing new in it (NOTICE-NOISE).
1053
+ if not args.dry_run and st.get("handoff_for") != waiting and time.time() - st.get("last_handoff", 0) > 3600:
1054
+ try:
1055
+ exe = A.program_path("polyctl", path=child_path())
1056
+ if exe:
1057
+ subprocess.run([exe, "-C", root, "handoff", "--reason",
1058
+ language.text(cfg, "watchdog.handoff-no-wake")] + _handoff_quiet(),
1059
+ capture_output=True, timeout=120)
1060
+ st.update(last_handoff=time.time(), handoff_for=waiting)
1061
+ save_state(root, st)
1062
+ except Exception:
1063
+ pass
1064
+ woke = False
1065
+
1066
+ # Wake into absence, never alongside. A live architect does not need a copy of
1067
+ # itself: the copy inherits the conversation in progress and continues that
1068
+ # rather than the triage it was started for.
1069
+ if woke and A.architect_present(root, arch):
1070
+ print("reports pending, but the architect is already at the keyboard")
1071
+ # Suppressing the wake must not silence the reports (#23): the session acts
1072
+ # when someone prompts it, so tell that someone. One condition rings one alarm (#106): a
1073
+ # report an anomaly stands for is that anomaly's, which names it and says the session is
1074
+ # interactive, and telling it here as well mailed one request twice on every red repeat.
1075
+ # This alarm names the rest, as reports-no-wake does with wakes off, and rings again when
1076
+ # another report waits (NOISE-REPEATS).
1077
+ hold = hold or architect_hold_reason(root, cfg, metered, st, found)
1078
+ named = set() if hold["holdable"] else anomaly_reports(standing.values(), stale)
1079
+ rest = [m for m in stale if m not in named]
1080
+ if told is not None:
1081
+ told.update(rest)
1082
+ if named:
1083
+ print(f"{len(named)} report(s) named by the alarm of the anomaly they stand for; "
1084
+ f"{len(rest)} other(s) alarmed here")
1085
+ if rest and not args.dry_run:
1086
+ # The report that asks, not the watchdog's echo of it written this cycle: mtimes are whole
1087
+ # seconds, so the echo won only when the cycle crossed a second.
1088
+ newest = max(rest, key=lambda m: (not A.from_watchdog(m), mtimes.get(m, 0), m))
1089
+ notify(f"{project}: reports wait for the architect",
1090
+ f"{waiting_reports(root, cfg, rest, newest)} — the architect session is interactive and "
1091
+ f"reads {'it' if len(rest) == 1 else 'them'} when prompted",
1092
+ root, key="present-pending", window=3600, audience="human", what=newest)
1093
+ woke = False
1094
+ from . import features as F
1095
+ if woke and not F.enabled(cfg, "architect_wake"):
1096
+ print("reports pending; architect_wake feature is off — recorded and alarmed, not woken")
1097
+ # One condition rings one alarm, whoever sees it (#106). A report an anomaly stands for -
1098
+ # the request it groups, or the watchdog's own report of it - is already a person's
1099
+ # "needs you", which names it and says why nobody is woken; ringing it here too sent a
1100
+ # day's red mail about one request twice. This alarm names the rest: a report that asks
1101
+ # nothing while no work is open, a lead, an anomaly's report whose condition has ended.
1102
+ # With wakes off no anomaly is held for the architect, so every standing one reached a
1103
+ # person (WAITING-ONE-ALARM).
1104
+ hold = hold or architect_hold_reason(root, cfg, metered, st, found)
1105
+ named = set() if hold["holdable"] else anomaly_reports(standing.values(), stale)
1106
+ rest = [m for m in stale if m not in named]
1107
+ if told is not None:
1108
+ told.update(rest)
1109
+ if named:
1110
+ print(f"{len(named)} report(s) named by the alarm of the anomaly they stand for; "
1111
+ f"{len(rest)} other(s) alarmed here")
1112
+ if rest:
1113
+ newest = max(rest, key=lambda m: (not A.from_watchdog(m), mtimes.get(m, 0), m))
1114
+ notify(f"{project}: needs you",
1115
+ f"{waiting_reports(root, cfg, rest, newest)} — no architect will act on "
1116
+ f"{'it' if len(rest) == 1 else 'them'}: {hold['reason']}",
1117
+ root, key="reports-no-wake", window=3600, audience="human", what=newest)
1118
+ woke = False
1119
+ if woke:
1120
+ provider = A.provider_of(arch.get("argv"))
1121
+ left, reserve = A.window_headroom(provider)
1122
+ urgent = any(a.get("kind") == "decision-requested" for a in (found or []))
1123
+ if left is not None and left < reserve and not urgent:
1124
+ print(f"reports pending, but the machine's {provider} window has {left}% left (< reserve {reserve}%); not waking")
1125
+ woke = False
1126
+ if woke and arch_alive(root, arch):
1127
+ print("reports pending, but an architect wake is already running")
1128
+ woke = False
1129
+ if woke and arch.get("argv") and not args.dry_run:
1130
+ prompt = language.text(cfg, "prompt.wake")
1131
+ command, code, stopped = architect_command(root, cfg, prompt)
1132
+ if stopped:
1133
+ print(f"{stopped}; not waking the architect")
1134
+ if code == "refused":
1135
+ tell_turn_refused(root, "architect", stopped)
1136
+ return woke
1137
+ sess, plan, pinned = command["session"], command["plan"], command["pinned"]
1138
+ argv, search = command["argv"], command["search"]
1139
+ resolved, ver = A.resolve_binary(argv[0], path=command["path"])
1140
+ key = A.project_key(root)
1141
+ log_path = os.path.join(STATE_DIR, A.project_file_name("escalate-log", key))
1142
+ # Read what the previous wake said before starting another. Same binary,
1143
+ # same error, less than six hours old: the human has been told, and a
1144
+ # retry is the forty-first identical failure.
1145
+ err = wake_error(log_path)
1146
+ if resolved and err:
1147
+ text, used, when, kind = err["text"], err["binary"], err["when"], err["kind"]
1148
+ # The log merges the model's prose with its errors; mask before it is kept or sent.
1149
+ text = A.redact(text)
1150
+ prev = st.get("wake_error") or {}
1151
+ fresh = prev.get("text") != text or prev.get("binary") != used or prev.get("when") != when
1152
+ if fresh:
1153
+ st["wake_error"] = {"at": time.time(), "text": text, "binary": used,
1154
+ "when": when, "kind": kind}
1155
+ if kind == "quota":
1156
+ # The architect is paused, not broken. Wait for the window, tell
1157
+ # the human once (orange), and remember that the desktop app may
1158
+ # resume the session itself when its auto-continue is on. The window's
1159
+ # end is read as the implementer's quota handling reads it (ARCHITECT-WAKE-QUOTA).
1160
+ until = architect_quota_until(err, argv)
1161
+ if until:
1162
+ st["arch_quota_until"] = until
1163
+ A.deferred_append(root, "wake", reason="architect quota", until=until)
1164
+ save_state(root, st)
1165
+ # notify records the notice; a row written first under the same key
1166
+ # made notify's own rate limit swallow the ring (#69).
1167
+ if kind == "quota":
1168
+ touch_architect_quota(root, st, cfg)
1169
+ else:
1170
+ # Every retry reads a failure with a new time, so a window alone rang the same 529
1171
+ # again every six hours: rung once for what it says, a new failure is told (NOISE-REPEATS).
1172
+ notify(language.text(cfg, "alarm.wake-failed-title", project=key),
1173
+ language.text(cfg, "alarm.wake-failed", kind=kind, error=text[:110], binary=used or "?"),
1174
+ root, key="architect-wake-failed",
1175
+ window=6 * 3600, audience="human", what=wake_failure_told(err))
1176
+ same = used == f"{resolved} {ver}"
1177
+ if kind == "binary" and same and time.time() - (st.get("wake_error") or {}).get("at", 0) < 6 * 3600:
1178
+ print(f"architect wake failed with this same binary ({used}); not retrying: {text[:90]}")
1179
+ resolved = None
1180
+ elif kind == "quota" and st.get("arch_quota_until", 0) > time.time():
1181
+ print(f"architect at quota until {reset_when(st['arch_quota_until'])}; not waking")
1182
+ resolved = None
1183
+ elif kind == "session":
1184
+ # A dead session id: resume a different one or none. Setting the config
1185
+ # back to auto after argv was built resumed the same dead session (#69).
1186
+ dead = st.get("arch_session")
1187
+ if sess and (sess == dead or sess in text):
1188
+ other = A.resolve_session(root, cfg, "architect", avoid={sess})
1189
+ found = other["session"] if other["trusted"] else None
1190
+ if found and found != sess and found not in text:
1191
+ sess = found
1192
+ if other["record"]:
1193
+ A.record_session(root, "architect", other)
1194
+ argv = [x.replace("{prompt}", prompt).replace("{session}", sess)
1195
+ for x in plan["argv"]]
1196
+ print(f"last wake resumed a dead session; resuming {sess[:12]} instead")
1197
+ else:
1198
+ print("last wake resumed a dead session and no other session was found; not waking")
1199
+ resolved = None
1200
+ if resolved and st.get("arch_quota_until", 0) > time.time():
1201
+ print(f"architect at quota until {reset_when(st['arch_quota_until'])}; not waking")
1202
+ resolved = None
1203
+ # The command a wake starts runs the harness of an adapter the package ships, with no grant beyond the one polyctl
1204
+ # composes from it, before anything is spent on it (TURN-ARGV).
1205
+ refused = A.turn_refusal("architect", A.block_adapter(arch), argv, root=root, resolved=resolved,
1206
+ search=search, values=dict(command["values"], session=sess), prompt=prompt) if resolved else None
1207
+ if refused:
1208
+ print(f"{refused}; not waking the architect")
1209
+ tell_turn_refused(root, "architect", refused)
1210
+ resolved = None
1211
+ if resolved and not args.dry_run:
1212
+ # An exhausted window is rotated through keyflip first, when it may be (#32).
1213
+ headroom = A.rotate_if_exhausted(cfg, argv, "architect")
1214
+ if not headroom["ok"]:
1215
+ print(f"{headroom['text']}; not waking the architect")
1216
+ notify(f"{key}: no headroom", headroom["text"] + " — a person decides", root,
1217
+ key=f"no-headroom:{headroom['provider']}", window=6 * 3600, audience="human")
1218
+ resolved = None
1219
+ # Read after the rotation, which can wait minutes on keyflip, as the nudge and the refill read it (JOURNAL-6).
1220
+ held = A.hold_state(root)
1221
+ if resolved and held:
1222
+ print(f"held by {held.get('by')} since this cycle began; not waking the architect")
1223
+ resolved = None
1224
+ # The architect's harness reads its hooks, plugins and instructions from this tree, as late as the hold
1225
+ # is read (HARNESS-CONFIG), and its command is the one a person accepted (TURN-ARGV).
1226
+ unseen = (harness_config_hold(root) or turn_hold(root, cfg, "architect")) if resolved else None
1227
+ if unseen:
1228
+ print(f"{unseen}; not waking the architect")
1229
+ resolved = None
1230
+ if resolved:
1231
+ argv[0] = resolved
1232
+ if not record_pinned(root, pinned):
1233
+ return woke
1234
+ given, refused = A.prompt_input(plan, root, argv)
1235
+ if refused:
1236
+ print(f"the architect's prompt cannot be handed over: {refused}; not waking")
1237
+ return woke
1238
+ # A wake after one that failed in the last day is a retry: it was the same line to the phone
1239
+ # every fifteen minutes while a transport kept failing, and none of them was a woken architect.
1240
+ retried = wake_failed(err, last_wake) and time.time() - float(err.get("at") or 0) < 24 * 3600
1241
+ # Claimed before its process starts, under the session and the reports it is handed (JOURNAL-2).
1242
+ step = "wake:{}:{}".format(sess or "new", A.hashlib.sha256("\n".join(sorted(pending)).encode(UTF8))
1243
+ .hexdigest()[:12])
1244
+ if sess and A.session_in_use(sess, arch):
1245
+ print(f"the architect's session {sess} is resumed by a running process already; not waking")
1246
+ return woke
1247
+ if not _claim_step(root, step):
1248
+ print("a wake for these reports was started and is not known to have ended; not waking again")
1249
+ return woke
1250
+ os.makedirs(STATE_DIR, exist_ok=True)
1251
+ with open(log_path, "a", encoding=UTF8) as log:
1252
+ log.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} escalate {resolved} {ver} ===\n")
1253
+ log.flush()
1254
+ # stdin must be closed, not inherited. A detached child holding a
1255
+ # pipe or tty it will never read can wait forever for an EOF that
1256
+ # never comes: alive, silent, producing nothing. That is the exact
1257
+ # shape of the first architect wake — over a minute, no output —
1258
+ # and very likely of the fifteen agent processes this project
1259
+ # found accumulated in one repository. A prompt handed over on
1260
+ # standard input is a file, which ends.
1261
+ try:
1262
+ proc = subprocess.Popen(given["argv"], cwd=root,
1263
+ env=dict(os.environ, PATH=search, POLYCTL_ROLE="architect"),
1264
+ stdin=subprocess.DEVNULL if given["stdin"] is None else given["stdin"],
1265
+ stdout=log, stderr=subprocess.STDOUT,
1266
+ start_new_session=True)
1267
+ finally:
1268
+ A.release_prompt(given)
1269
+ _record_start(root, step, proc.pid)
1270
+ st["arch_pid"] = proc.pid
1271
+ st["last_arch_wake"] = time.time()
1272
+ st["handed"] = {m: mtimes[m] for m in pending if m in mtimes}
1273
+ st["arch_session"] = sess
1274
+ st.pop("wake_retry", None)
1275
+ if retried:
1276
+ # Told by tell_retried_wake once a later cycle reads that it did not fail (NOISE-REPEATS).
1277
+ st["wake_retry"] = {"at": st["last_arch_wake"], "pid": proc.pid, "reports": len(stale)}
1278
+ A.helper_register(root, proc.pid, "architect") # a judge, not a writer
1279
+ A.acquire_architect(root, proc.pid, "watchdog wake") # one judge at a time
1280
+ save_state(root, st)
1281
+ A.emit_event(root, "wake", {"why": "reports", "reports": len(stale), "pid": proc.pid,
1282
+ "retried": True if retried else None})
1283
+ if retried:
1284
+ print(f"retried the architect's wake (pid {proc.pid}); the phone hears of it once it has not failed")
1285
+ else:
1286
+ _tell_phone(root, language.text(cfg, "watchdog.architect-woken", n=len(stale), pid=proc.pid))
1287
+ print(f"woke the architect (pid {proc.pid}) to judge it")
1288
+ return woke
1289
+
1290
+
1291
+ def anomaly_subject(anomaly):
1292
+ """What an anomaly is about, for the ladder: the report, decision or review it names, never an age (NOTICE-NOISE).
1293
+
1294
+ A fact such as "returned 34m ago" changes every cycle; what waits changes only when
1295
+ something new waits, and that is when a person is told again.
1296
+ """
1297
+ for fact in anomaly.get("facts") or []:
1298
+ wrote = re.search(r"wrote\s+(\S+\.md)", str(fact))
1299
+ if wrote:
1300
+ return f"{anomaly.get('kind')}:{wrote.group(1)}"
1301
+ return f"{anomaly.get('kind')}:{anomaly.get('key') or ''}"
1302
+
1303
+
1304
+ def waiting_reports(root, cfg, names, newest=None):
1305
+ """Which reports wait and since when, as a person reads it (WAITING-ONE-ALARM).
1306
+
1307
+ One report by its name, several by their count and the newest. Since when is the oldest
1308
+ report's own time - the stamp its name carries, else its file's - never its alarm's: an
1309
+ alarm starts again after a resume, and the request under it may have waited for weeks.
1310
+ """
1311
+ box = cfg.get("mailbox", "agent-mail")
1312
+ times = []
1313
+ for name in names:
1314
+ try:
1315
+ times.append(A._name_time(name) or os.path.getmtime(os.path.join(root, box, name)))
1316
+ except OSError:
1317
+ continue # handled while this cycle ran
1318
+ since = f", waiting since {_when(min(times))}" if times else ""
1319
+ if len(names) == 1:
1320
+ return f"{names[0]} in {box}/{since}"
1321
+ return f"{len(names)} reports in {box}/{since}, the newest {newest or names[-1]}"
1322
+
1323
+
1324
+ def anomaly_waits(root, cfg, anomaly):
1325
+ """What an anomaly that reaches a person says of itself: its kind, and what waits (WAITING-ONE-ALARM).
1326
+
1327
+ A request or report the implementer wrote is named with since when; any other anomaly
1328
+ says its first fact, which names what it measured.
1329
+ """
1330
+ kind = anomaly.get("kind")
1331
+ names = [str(name) for name in anomaly.get("reports") or []]
1332
+ if names and kind in ("decision-requested", "report-waiting"):
1333
+ return f"{kind}: {waiting_reports(root, cfg, names)}"
1334
+ facts = anomaly.get("facts")
1335
+ return f"{kind}: {facts[0]}" if isinstance(facts, list) and facts else str(kind)
1336
+
1337
+
1338
+ def anomaly_reports(anomalies, names):
1339
+ """The reports among `names` a standing anomaly stands for: those it groups, and its own (WAITING-ONE-ALARM).
1340
+
1341
+ The watchdog's report of an anomaly is known by the kind its name carries, whatever
1342
+ follows it: one written under an older name is a report of the same kind of condition,
1343
+ and the alarm for that kind stands for it as well. The implementer's reports are known
1344
+ by name, from the anomaly that groups them.
1345
+ """
1346
+ kinds = {str(anomaly.get("kind")) for anomaly in anomalies}
1347
+ grouped = {name for anomaly in anomalies for name in anomaly.get("reports") or []}
1348
+ out = set()
1349
+ for name in names:
1350
+ own = re.search(r"-ANOMALY-(.+)\.md$", name) if A.from_watchdog(name) else None
1351
+ if name in grouped or (own and any(own.group(1) == kind or own.group(1).startswith(kind + "-")
1352
+ for kind in kinds)):
1353
+ out.add(name)
1354
+ return out
1355
+
1356
+
1357
+ def queue_past_a_question(root):
1358
+ """(what waits, the READY item to take) when a question waits and READY work stands (#84).
1359
+
1360
+ The architect is a person's quota and attention, and both run out. An open
1361
+ decision cannot be answered by a nudged turn (#20), but it must not stop the
1362
+ queue either: the slice it blocks is parked and the next READY item is taken.
1363
+ None when nothing waits, or when nothing is READY and waiting is all there is.
1364
+ """
1365
+ ready = A.ready(root)
1366
+ if not ready:
1367
+ return None
1368
+ asked = [d for d in A.decisions(root, "open") if d.get("asked_at")]
1369
+ if asked:
1370
+ return min(asked, key=lambda d: d["asked_at"])["id"], ready[0]["id"]
1371
+ blocked = A.board(root)["blocked"]
1372
+ return (blocked[0]["id"], ready[0]["id"]) if blocked else None
1373
+
1374
+
1375
+ def parked_note(cfg, waits, ready_id):
1376
+ """What a nudge adds when a question waits and READY work stands (#84), in the project's language."""
1377
+ return language.text(cfg, "prompt.nudge-parked", waits=waits, ready=ready_id)
1378
+
1379
+
1380
+ def secondary_note(cfg, found):
1381
+ """What a nudge adds when nothing is READY here and a secondary project has work (#8), in the project's language."""
1382
+ return language.text(cfg, "prompt.nudge-secondary", name=found["name"], root=found["root"], item=found["item"])
1383
+
1384
+
1385
+ def record_turn_flags(root, role, flags, reason):
1386
+ """A turn whose added flags cannot be recorded starts nothing (TRIAGE-3-LOW)."""
1387
+ if not flags:
1388
+ return True
1389
+ try:
1390
+ A.record_actor_flags(root, role, flags, reason)
1391
+ except Exception as exc:
1392
+ refused = (f"could not record the flags added to the {role}: {A.redact(str(exc))}; "
1393
+ "no turn started; restore access to .polyctl/ledger/actor-flags.jsonl and retry")
1394
+ print(refused)
1395
+ tell_turn_refused(root, role, refused)
1396
+ return False
1397
+ return True
1398
+
1399
+
1400
+ def record_pinned(root, pinned):
1401
+ """Record a wake's added flags before allowing its launch (TRIAGE-3-LOW)."""
1402
+ return record_turn_flags(root, "architect", pinned,
1403
+ "the architect's command names no permission mode, so polyctl appends the one its adapter pins")
1404
+
1405
+
1406
+ def _schedule_hunt(root, cfg, st):
1407
+ """Start one bounded, detached bug hunt when it is switched on and due: a schedule, never a loop (#45).
1408
+
1409
+ Nor while harness configuration nobody has seen stands in the tree, as no turn polyctl starts here runs then
1410
+ (HARNESS-CONFIG); a hunt held is due again next cycle.
1411
+ """
1412
+ from . import features as F
1413
+ if not F.enabled(cfg, "hunter") or not S.get(cfg, "hunter.argv") or A.hold_state(root):
1414
+ return False
1415
+ if time.time() - float(st.get("last_hunt") or 0) < S.get(cfg, "hunter.every_hours") * 3600:
1416
+ return False
1417
+ unseen = harness_config_hold(root)
1418
+ if unseen:
1419
+ print(f"{unseen}; not starting a bug hunt")
1420
+ return False
1421
+ log = os.path.join(STATE_DIR, A.project_file_name("hunt-log", A.project_key(root)))
1422
+ os.makedirs(STATE_DIR, exist_ok=True)
1423
+ with open(log, "a", encoding=UTF8) as fh:
1424
+ fh.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} hunt ===\n")
1425
+ fh.flush()
1426
+ subprocess.Popen([sys.executable, "-m", "polyctl", "-C", root, "hunt", "run"], cwd=A.self_child_cwd(),
1427
+ stdin=subprocess.DEVNULL, stdout=fh, stderr=subprocess.STDOUT, start_new_session=True,
1428
+ env=A.self_child_env(dict(os.environ, POLYCTL_ROLE="hunter")))
1429
+ st["last_hunt"] = time.time()
1430
+ save_state(root, st)
1431
+ print("started a bounded bug hunt")
1432
+ return True
1433
+
1434
+
1435
+ def harness_config_hold(root):
1436
+ """Why no turn may start in this tree now, or None: what a harness reads from it changed and no person has seen it,
1437
+ or it cannot be read (HARNESS-CONFIG).
1438
+
1439
+ A hook the implementer writes into .claude/settings.json runs in the next session started here - its own nudge, the
1440
+ architect's wake or refill, a hunt - before any review has seen it. Each asks right before it starts, as each reads
1441
+ the hold, since an agent can write one after the cycle began; one that cannot be read holds them all.
1442
+ """
1443
+ try:
1444
+ unseen = A.unseen_harness_config(root)
1445
+ except Exception as exc:
1446
+ return f"what a harness reads from this tree cannot be read ({' '.join(str(exc).split())[:160]})"
1447
+ if not unseen:
1448
+ return None
1449
+ return (f"harness configuration nobody has seen stands in this tree: {A.harness_config_line(unseen, limit=10)} - "
1450
+ "a person reads it with polyctl harness seen --by <name>")
1451
+
1452
+
1453
+ def turn_hold(root, cfg, role):
1454
+ """Why a turn of `role` may not start with the command composed from `cfg`, or None: its block says otherwise of it
1455
+ than the command last accepted, judged by the configuration the command came from (TURN-ARGV).
1456
+
1457
+ harness_config_hold reads .polyctl/config.json again, and an agent can put it back between the two readings; what
1458
+ starts is what this cycle composed.
1459
+ """
1460
+ try:
1461
+ unseen = A.unseen_turn(root, cfg, role)
1462
+ except Exception as exc:
1463
+ return f"the {role}'s command cannot be compared with the one accepted ({' '.join(str(exc).split())[:160]})"
1464
+ if not unseen:
1465
+ return None
1466
+ return (f"harness configuration nobody has seen stands in this tree: {A.harness_config_line([unseen])} - a person "
1467
+ "reads it with polyctl harness seen --by <name>")
1468
+
1469
+
1470
+ def tell_turn_refused(root, role, refused):
1471
+ """Name a turn the watchdog refuses to a person, once for what refuses it (TURN-ARGV): its configuration must
1472
+ change, and nothing an agent does or a person records lets it start as it stands."""
1473
+ notify(f"{A.project_key(root)}: {'nudge' if role == 'implementer' else 'architect wake'} refused", refused, root,
1474
+ key=f"turn-refused:{role}", window=6 * 3600, audience="human", what=refused)
1475
+
1476
+
1477
+ def tell_harness_config(root, project):
1478
+ """Name to a person, each cycle, every change to what a harness reads here that nobody has seen (HARNESS-CONFIG).
1479
+
1480
+ One alarm for what the changes hold: it rings once for them and climbs the ladder while they stand, and rings again
1481
+ when they change. Seen, or gone from the tree, it is no longer raised and its episode closes.
1482
+ """
1483
+ try:
1484
+ unseen = A.unseen_harness_config(root)
1485
+ except Exception as exc:
1486
+ _FACTS["harness_config"] = "unreadable"
1487
+ notify(f"{project}: harness configuration cannot be read",
1488
+ f"what a harness reads from this tree cannot be read ({' '.join(str(exc).split())[:160]}), so no wake, "
1489
+ "refill, nudge or hunt starts here until it can", root, key="harness-config", window=6 * 3600,
1490
+ audience="human", what="unreadable")
1491
+ return
1492
+ _FACTS["harness_config"] = len(unseen)
1493
+ if unseen:
1494
+ notify(f"{project}: harness configuration nobody has seen",
1495
+ f"{A.harness_config_line(unseen, limit=20)}: what a harness reads from this tree changed, and no wake, "
1496
+ "refill, nudge or hunt starts here until a person reads it - polyctl harness seen --by <name>",
1497
+ root, key="harness-config", window=6 * 3600, audience="human", what=A.harness_config_digest(unseen))
1498
+
1499
+
1500
+ def open_work(cfg, root):
1501
+ """Is there something for the implementer to continue? Cheap signals only.
1502
+
1503
+ Only signals that mean *work*. Mail counts when it is addressed to the
1504
+ implementer — its own outgoing reports and the watchdog's anomaly files once
1505
+ counted as "unread mail", so every report the implementer wrote was a reason
1506
+ to nudge it into writing another. Dirty paths count outside the coordination
1507
+ directories; a review counts only if it is newer than HEAD.
1508
+ """
1509
+ reasons = []
1510
+ if A.implementer_inbox(root, cfg):
1511
+ reasons.append("unread mail")
1512
+ # A slice the board says is running is work, whether or not a file has
1513
+ # changed yet: an implementer that has just started one and ended its turn
1514
+ # on a plan must be nudged back into it.
1515
+ if A.board(root)["running"]:
1516
+ reasons.append("slice running")
1517
+ rs = A.reviewer_state(root)
1518
+ # A reviewer that was unavailable with no known window - an expired login, a
1519
+ # missing binary - is not work a nudge can do; nudging for it never ended (audit).
1520
+ if rs.get("pending_review") and rs.get("until") and rs["until"] <= time.time():
1521
+ reasons.append("reviewer window reopened — re-run the pending review")
1522
+ if A.product_dirty(root, cfg):
1523
+ reasons.append("uncommitted changes")
1524
+ # Findings are the newest review that produced a verdict; an UNAVAILABLE or
1525
+ # INVALID file is not a finding anyone can act on (audit).
1526
+ from .storage import LedgerCorruption, read_chained_jsonl
1527
+ try:
1528
+ rows = read_chained_jsonl(A.review_ledger_path(root), A.REVIEW_CHAIN)
1529
+ except (OSError, ValueError, LedgerCorruption):
1530
+ rows = []
1531
+ latest = next((row for row in reversed(rows) if isinstance(row, dict)
1532
+ and row.get("verdict") in ("APPROVED", "NEEDS_CHANGES")), None)
1533
+ if latest and latest["verdict"] == "NEEDS_CHANGES":
1534
+ try:
1535
+ head_at = int(A._git_text(root, "log", "-1", "--format=%ct") or 0)
1536
+ rev_at = int(latest.get("at") or 0)
1537
+ except (OSError, ValueError):
1538
+ rev_at, head_at = 1, 0
1539
+ if rev_at > head_at:
1540
+ reasons.append("open review findings")
1541
+ return reasons
1542
+
1543
+
1544
+ WAKE_SIGNATURES = (
1545
+ ("quota", r"(hit your (?:session|usage|weekly|monthly) limit[^\n]*|usage limit[^\n]*|"
1546
+ r"rate limit(?:ed)?[^\n]*resets?[^\n]*|out of (?:credits|quota)[^\n]*)"),
1547
+ ("binary", r"(does not support this model[^\n]*|version [^\n]*required[^\n]*|"
1548
+ r"command not found[^\n]*|No such file or directory[^\n]*|env: node: [^\n]*)"),
1549
+ ("session", r"(No conversation found[^\n]*|[Ss]ession[^\n]{0,40}not found[^\n]*|"
1550
+ r"Invalid session[^\n]*)"),
1551
+ ("other", r"(API Error: \d{3}[^\n]*|Error: [^\n]{8,})"),
1552
+ )
1553
+
1554
+
1555
+ # The architect's usage limit comes back within five hours of being hit; a reset
1556
+ # a message names further away than that cannot be the one it meant (#40). That is
1557
+ # the short window: a limit that names its period is as long as that period.
1558
+ ARCHITECT_QUOTA_WINDOW = S.default("architect.quota_window_hours") * 3600
1559
+
1560
+ # The periods a usage-limit message may name, and their lengths (ARCHITECT-WAKE-QUOTA).
1561
+ LIMIT_PERIODS = {"weekly": 7 * 86400, "monthly": 31 * 86400}
1562
+
1563
+
1564
+ def limit_window(text):
1565
+ """How long the limit a usage-limit message names lasts: its period when it names one, else the short window.
1566
+
1567
+ The length bounds how far ahead a clock-only reset may roll (parse_reset). A weekly
1568
+ limit that named only a clock time already past when it was written was read against
1569
+ the five-hour window, as the time that had gone, so its block ended at once and the
1570
+ next cycle could wake the architect again, days before its week was over
1571
+ (ARCHITECT-WAKE-QUOTA).
1572
+ """
1573
+ named = re.search(r"\b(weekly|monthly)\b", str(text or ""), re.I)
1574
+ return LIMIT_PERIODS[named.group(1).lower()] if named else S.get(None, "architect.quota_window_hours") * 3600
1575
+
1576
+
1577
+ RESET_MONTHS = ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec")
1578
+
1579
+
1580
+ def _reset_clock(hour, minute, meridiem):
1581
+ """(hour, minute) on a 24-hour clock from "9", "20", "pm"."""
1582
+ h, mi, ap = int(hour), int(minute or 0), (meridiem or "").lower()
1583
+ if ap == "pm" and h < 12:
1584
+ h += 12
1585
+ if ap == "am" and h == 12:
1586
+ h = 0
1587
+ return h, mi
1588
+
1589
+
1590
+ def parse_reset(text, now=None, window=None):
1591
+ """When does the quota come back? "resets 4:30am" / "resets in 4h 43m" / "resets Sep 14 at 4am" / None.
1592
+
1593
+ `now` is when the message was written, not when it is read: a log line read
1594
+ again hours later keeps the reset it named. A clock time already past rolls to
1595
+ the next day only while that stays inside `window`, the limit's own length;
1596
+ otherwise the message meant the time that has gone. On 2026-09-07 a 17:48
1597
+ "resets 9:20pm" was re-read after 21:20 on every cycle and pushed a day ahead
1598
+ each time, so the block it raised never ended (#40).
1599
+
1600
+ A relative reset may count days as well: keyflip states a weekly window's as
1601
+ "resets in 3d 4h", and read without them it said nothing (ARCHITECT-WAKE-QUOTA).
1602
+ """
1603
+ now = now or time.time()
1604
+ m = re.search(r"resets?\s+in\s+((?:\d+\s*[dhms]\s*)+)", text, re.I)
1605
+ if m:
1606
+ secs = 0
1607
+ for n, u in re.findall(r"(\d+)\s*([dhms])", m.group(1), re.I):
1608
+ secs += int(n) * {"d": 86400, "h": 3600, "m": 60, "s": 1}[u.lower()]
1609
+ return now + secs
1610
+ # A weekly limit names its day: "resets Sep 14 at 4am". It was read as no reset
1611
+ # at all, so a week-long block was retried every few hours (#41).
1612
+ m = re.search(r"resets?\s+(?:on\s+)?(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+"
1613
+ r"(\d{1,2})(?:,?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?)?", text, re.I)
1614
+ if m:
1615
+ month = RESET_MONTHS.index(m.group(1).lower()) + 1
1616
+ h, mi = _reset_clock(m.group(3) or "0", m.group(4), m.group(5))
1617
+ lt = time.localtime(now)
1618
+ cand = time.mktime((lt.tm_year, month, int(m.group(2)), h, mi, 0, 0, 0, -1))
1619
+ if cand < now - 180 * 86400: # December's message read in January
1620
+ cand = time.mktime((lt.tm_year + 1, month, int(m.group(2)), h, mi, 0, 0, 0, -1))
1621
+ return cand
1622
+ m = re.search(r"resets?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?", text, re.I)
1623
+ if not m:
1624
+ return None
1625
+ h, mi = _reset_clock(m.group(1), m.group(2), m.group(3))
1626
+ lt = time.localtime(now)
1627
+ cand = time.mktime((lt.tm_year, lt.tm_mon, lt.tm_mday, h, mi, 0, 0, 0, -1))
1628
+ if cand <= now and (window is None or cand + 24 * 3600 - now <= window):
1629
+ cand += 24 * 3600
1630
+ return cand
1631
+
1632
+
1633
+ def wake_error(log_path):
1634
+ """What the last architect wake said, if it failed.
1635
+
1636
+ {"text", "binary", "when", "kind", "resets_at"} or None. The wake is
1637
+ detached, so its outcome can only be read on the next cycle from the log
1638
+ segment it wrote. Forty identical failures went unread because nothing
1639
+ looked. `kind` is what the caller should do about it: a stale *binary* wants
1640
+ another binary, a *quota* wants a wait until `resets_at`, a dead *session*
1641
+ wants rediscovery.
1642
+ """
1643
+ try:
1644
+ tail = open(log_path, errors="replace", encoding=UTF8).read()[-20000:]
1645
+ except OSError:
1646
+ return None
1647
+ segs = re.split(r"^=== (\S+ \S+) (escalate|refill)(?: (.*?))? ===$", tail, flags=re.M)
1648
+ if len(segs) < 5:
1649
+ return None
1650
+ when, _, binary, body = segs[-4], segs[-3], segs[-2] or "", segs[-1]
1651
+ try:
1652
+ at = time.mktime(time.strptime(when, "%Y-%m-%d %H:%M:%S"))
1653
+ except ValueError:
1654
+ at = None
1655
+ for kind, pat in WAKE_SIGNATURES:
1656
+ m = re.search(pat, body)
1657
+ if m:
1658
+ text = m.group(1).strip()[:300]
1659
+ # A reset is read against when the wake wrote it, not against this cycle, and
1660
+ # within the limit the stop names: its period, or the short window.
1661
+ resets_at = parse_reset(body, now=at, window=limit_window(text)) if kind == "quota" else None
1662
+ streak = 0
1663
+ if kind == "quota" and resets_at is None:
1664
+ # How many wakes in a row stopped on a weekly or monthly limit that named no reset, this
1665
+ # one the last (ARCHITECT-WAKE-QUOTA-2): each waits twice as long as the one before.
1666
+ for index in range(len(segs) - 4, 0, -4):
1667
+ if not _period_without_reset(segs[index], segs[index + 3]):
1668
+ break
1669
+ streak += 1
1670
+ return {"text": text, "binary": binary, "when": when, "at": at, "kind": kind,
1671
+ "resets_at": resets_at, "streak": streak}
1672
+ return None
1673
+
1674
+
1675
+ def _period_without_reset(when, body):
1676
+ """Whether a wake's segment stopped on a weekly or monthly limit whose words name no reset."""
1677
+ m = re.search(dict(WAKE_SIGNATURES)["quota"], body)
1678
+ if not m:
1679
+ return False
1680
+ try:
1681
+ at = time.mktime(time.strptime(when, "%Y-%m-%d %H:%M:%S"))
1682
+ except ValueError:
1683
+ return False
1684
+ text = m.group(1).strip()[:300]
1685
+ return limit_window(text) > S.get(None, "architect.quota_window_hours") * 3600 \
1686
+ and parse_reset(body, now=at, window=limit_window(text)) is None
1687
+
1688
+
1689
+ def wake_failed(failure, since):
1690
+ """Whether the wake started at `since` failed, from what `wake_error` read of the log (#69).
1691
+
1692
+ A failure written before that wake belongs to an earlier one, and a model's own prose
1693
+ saying "Error:" is not a failure.
1694
+ """
1695
+ return bool(failure) and (failure.get("at") or 0) >= int(since or 0) - 2 \
1696
+ and (failure.get("kind") != "other" or "API Error" in failure.get("text", ""))
1697
+
1698
+
1699
+ def wake_failure_told(failure):
1700
+ """What a failed wake says, for the ladder: its kind, its binary and its words (NOISE-REPEATS).
1701
+
1702
+ Not the time it was read at, and not an id that changes on every attempt, such as the
1703
+ request id a provider puts in its error: each retry after a 529 read as another failure.
1704
+ Another kind, another binary or other words is something new, and is told.
1705
+ """
1706
+ failure = failure or {}
1707
+ # A request id is masked by its key first: one of letters alone has no digit for the rule after it to find,
1708
+ # and each retry read as news (NOISE-REPEATS-2).
1709
+ text = re.sub(r'(?i)(request[ _-]?id"?\s*[:=]\s*"?)[^"\s,}]+', "\\1…", A.redact(str(failure.get("text") or "")))
1710
+ words = re.sub(r"(?<![\w-])(?=[\w-]*\d)[\w-]{8,}", "…", text)
1711
+ return f"{failure.get('kind')}:{failure.get('binary') or '?'}:{' '.join(words.split())}"
1712
+
1713
+
1714
+ def tell_retried_wake(root, cfg, st, dry_run=False):
1715
+ """Tell the phone of a wake retried after a failed one, once it is known not to have failed (NOISE-REPEATS).
1716
+
1717
+ The line went out as the wake started, before anything said how it went, and a wake is
1718
+ retried every fifteen minutes while a transport error lasts: rebuilt in a temporary
1719
+ project, a day of 529s sent the phone 95 "architect woken" lines about an architect
1720
+ nobody woke, beside the alarm that said the wake was failing. A retry is told when its own
1721
+ log segment shows no failure and it has ended, or has run for the fifteen minutes the
1722
+ watchdog leaves between wakes; one that failed was no woken architect, and its failure is
1723
+ already on the ladder. Returns whether it was told.
1724
+ """
1725
+ retry = st.get("wake_retry")
1726
+ if not isinstance(retry, dict):
1727
+ return False
1728
+ try:
1729
+ running = time.time() - float(retry.get("at") or 0) < 900 and arch_alive(root, cfg.get("architect") or {})
1730
+ except TableUnreadable:
1731
+ running = True # it may run: it is told once that can be seen (WINDOWS-TABLE-CLOSED)
1732
+ if running:
1733
+ return False # still running, and a failing one can take minutes to say so
1734
+ failure = wake_error(os.path.join(STATE_DIR, A.project_file_name("escalate-log", A.project_key(root))))
1735
+ worked = not wake_failed(failure, retry.get("at") or 0)
1736
+ if dry_run:
1737
+ if worked:
1738
+ print("DRY RUN: would tell the phone the architect was woken after failed attempts")
1739
+ return False
1740
+ st.pop("wake_retry", None)
1741
+ save_state(root, st)
1742
+ if worked:
1743
+ _tell_phone(root, language.text(cfg, "watchdog.architect-woken-retried", n=retry.get("reports"),
1744
+ pid=retry.get("pid")))
1745
+ return worked
1746
+
1747
+
1748
+ def quota_block_until(err, now=None, stated=None):
1749
+ """Until when a quota error read from the wake log blocks a wake; None once it no longer does.
1750
+
1751
+ The block ends at the reset the message named or, when it named none, one
1752
+ limit window after the message was written. An older error describes a
1753
+ window that is over, however many times the log is read again (#40).
1754
+
1755
+ `stated` is the reset the provider's own reading states for a spent window
1756
+ (architect_quota_until). It only moves the end later: a wake started before it
1757
+ stops on that window again, and without it the end is what it always was
1758
+ (ARCHITECT-WAKE-QUOTA).
1759
+ """
1760
+ now = now or time.time()
1761
+ if not err or err.get("kind") != "quota":
1762
+ return None
1763
+ window = S.get(None, "architect.quota_window_hours") * 3600
1764
+ span = window
1765
+ if not err.get("resets_at") and err.get("streak"):
1766
+ # A weekly or monthly limit that named no reset held its wakes one short window each, so the
1767
+ # architect was woken again and again before its period was over. Each wake in a row that stops
1768
+ # on one waits twice as long as the last, and never longer than the period (ARCHITECT-WAKE-QUOTA-2).
1769
+ span = min(window * 2 ** (int(err["streak"]) - 1), limit_window(err.get("text")))
1770
+ until = max(err.get("resets_at") or ((err.get("at") or now) + span), stated or 0)
1771
+ return until if until > now else None
1772
+
1773
+
1774
+ def architect_quota_until(err, argv, now=None):
1775
+ """Until when a usage-limit stop read from an architect wake's log holds its wakes; None once it does not.
1776
+
1777
+ The implementer's quota handling reads the machine's reading of the providers'
1778
+ windows: keyflip's lines, taken by the quota command the adapters declare and kept in
1779
+ ~/.polyctl/quota.json, which the quota gate stops a turn on at `quota.block_percent` and an
1780
+ account rotation reads for the provider an adapter's `quota` block names
1781
+ (A.provider_window). The wake reads the same reading, for the provider `argv` spends.
1782
+ When that window is spent, at the gate's percent, the reset it states is one the wake
1783
+ is not started before. A window with headroom stopped nothing, so its reset says
1784
+ nothing of this stop. With no such reading the block is quota_block_until's alone, as
1785
+ it always was (ARCHITECT-WAKE-QUOTA).
1786
+ """
1787
+ if not err or err.get("kind") != "quota":
1788
+ return None
1789
+ provider = A.provider_of(argv)
1790
+ window = A.provider_window(provider) if provider else None
1791
+ spent = bool(window) and (window.get("pct") or 0) >= S.get(None, "quota.block_percent")
1792
+ return quota_block_until(err, now=now, stated=parse_reset(window.get("raw") or "", now=now) if spent else None)
1793
+
1794
+
1795
+ def quota_ok(adapter):
1796
+ """Is there headroom to spend a turn?
1797
+
1798
+ Prefer an explicit budget over a guessed threshold. keyflip enforces
1799
+ per-account 5h/7d budgets when the user has set them, and a policy someone
1800
+ chose beats a number this script invented — so ask it first and only fall
1801
+ back to reading the raw window when no budget exists. A breached budget stops
1802
+ the turn, whatever the window says; a budget not breached is headroom. Which
1803
+ account's breach counts is quota_budget's to say.
1804
+
1805
+ `adapter` is the package's, A.metered_adapter's: reading the window runs the command it declares, and a
1806
+ layer the implementer writes chose one that ran (TRIAGE-HIGHS).
1807
+ """
1808
+ budget = A.quota_budget(adapter)
1809
+ if budget["breached"]:
1810
+ return False
1811
+ if budget["accounts"]:
1812
+ return True
1813
+ ceiling = S.get(None, "quota.block_percent")
1814
+ for line in A.quota(adapter):
1815
+ for token in line.replace("%", "% ").split():
1816
+ if token.endswith("%"):
1817
+ try:
1818
+ if float(token[:-1]) >= ceiling:
1819
+ return False
1820
+ except ValueError:
1821
+ pass
1822
+ return True
1823
+
1824
+
1825
+ # ---- a slice parked on the implementer's usage limit (QUOTA-PARK) -----------------------------
1826
+ #
1827
+ # A usage limit was read only where an architect's wake had failed on one. An implementer that
1828
+ # stopped on its own limit mid-slice read as an agent ignoring its nudges: each nudge died on the
1829
+ # same limit until the backoff ran out and a person was told it was stuck. Its slice is parked
1830
+ # instead, and the reset resumes it.
1831
+
1832
+ # The `needs:` of a slice parked on quota, and the word the watchdog knows its parks by.
1833
+ QUOTA_NEEDS = "quota"
1834
+ # The note a parked item keeps its own `needs:` in, on the board itself, so a park the watchdog's state lost
1835
+ # gives it back as one that stood does (QUOTA-PARK-2).
1836
+ QUOTA_HELD = "held-needs"
1837
+ # A failed nudge's tail is the log's last lines: what stands before its own header is an earlier turn's.
1838
+ NUDGE_HEADER = re.compile(r"=== \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} nudge.*?===")
1839
+ # While a park holds, keyflip is asked to rotate a spent window at most this often, as wakes are spaced:
1840
+ # a rotation is machine-wide, and one asked every cycle while no account has headroom would move every
1841
+ # session on the provider every few minutes.
1842
+ QUOTA_ROTATION_SPACING = 900
1843
+
1844
+
1845
+ def usage_limit_stop(cfg, st, adapter=None, last=None):
1846
+ """Did the implementer stop on its usage limit? {"at", "text", "resets_at", "hours", "source"} or None (QUOTA-PARK).
1847
+
1848
+ Two readings can say so, and the newer of them decides: the transcript, where its newest words
1849
+ are a reply the harness wrote in the model's place (`last_message`, or `last` when the caller
1850
+ has read it), and the nudge that died within seconds of starting (`last_error`), whose tail is
1851
+ its own process's output. Each is searched for the words the implementer's adapter declares
1852
+ its harness stops on (`quota_stops`), in the order it declares them. An adapter that declares
1853
+ none is never read as stopped: one harness's words are no other's, and while the stop was read
1854
+ by one pattern in one harness's wording, another harness's limit never parked anything. The
1855
+ model's own words are never read either: an implementer at work on quota handling writes about
1856
+ usage limits, and a reply of its own that names one is no stop.
1857
+
1858
+ The reset is the one the harness's reply records beside its words (`quota_reset_path`), else
1859
+ the one its words name - in epoch seconds, where the stop's words hold an `epoch`, or as
1860
+ `parse_reset` reads them against when they were written - and it counts only after the stop
1861
+ and within the limit's own `hours`: a clock time is the next one inside them, and a reset
1862
+ further away is none the words can mean. Read within five hours, a weekly limit that named
1863
+ 4am at 18:17 read as that morning's, already gone, and its slice was parked with no reset.
1864
+ """
1865
+ adapter = A.implementer_adapter(cfg) if adapter is None else adapter
1866
+ stops = A.quota_stops(adapter)
1867
+ if not stops:
1868
+ return None
1869
+ said = []
1870
+ last = A.last_message(cfg) if last is None else last
1871
+ if last and last.get("at"):
1872
+ said.append((last["at"], last["text"] if last["in_place"] else "", "transcript", last.get("record")))
1873
+ failed = st.get("last_error") if isinstance(st.get("last_error"), dict) else {}
1874
+ if failed.get("tail") and isinstance(failed.get("at"), (int, float)):
1875
+ said.append((float(failed["at"]), NUDGE_HEADER.split(str(failed["tail"]))[-1], "nudge", None))
1876
+ if not said:
1877
+ return None
1878
+ at, text, source, record = max(said, key=lambda reading: reading[0])
1879
+ for stop in stops:
1880
+ found = stop["words"].search(text)
1881
+ if not found:
1882
+ continue
1883
+ path, window = A.quota_reset_path(adapter), stop["hours"] * 3600
1884
+ recorded = A._path_value(record, path) if record and path else None
1885
+ epoch = found.groupdict().get("epoch")
1886
+ read = [recorded if isinstance(recorded, (int, float)) and not isinstance(recorded, bool) else None,
1887
+ float(epoch) if epoch else None, parse_reset(text, now=at, window=window)]
1888
+ return {"at": at, "text": A.redact(found.group(0).strip())[:300], "source": source, "hours": stop["hours"],
1889
+ "resets_at": next((reset for reset in read if reset is not None and at < reset <= at + window), None)}
1890
+ return None
1891
+
1892
+
1893
+ def quota_park_for(stop, items, now, retry):
1894
+ """The park a stop makes of `items`: until its reset or, where its words named none, until one nudge tries again.
1895
+
1896
+ The retry is `retry` seconds after the stop, or the limit's own hours where those are fewer: a
1897
+ limit gone within them is gone by then.
1898
+ """
1899
+ named = stop["resets_at"] is not None
1900
+ hours = stop.get("hours")
1901
+ until = stop["resets_at"] if named else stop["at"] + min(retry, hours * 3600 if hours else retry)
1902
+ return {"items": list(items), "at": stop["at"], "text": stop["text"], "until": until, "named": named,
1903
+ "source": stop["source"], "since": now}
1904
+
1905
+
1906
+ def quota_park_step(stop, park, resumed, running, parked, now=None, spoken=None, retry=None):
1907
+ """What a usage-limit stop asks of this cycle: (action, park). It reads and writes nothing (QUOTA-PARK).
1908
+
1909
+ `stop` is what `usage_limit_stop` read this cycle, `park` the park that stands, `resumed` the
1910
+ time of the stop the last park ended for, `running` the implementer's ids under the board's
1911
+ `running`, `parked` the ids blocked on quota, and `spoken` when a person or the model last
1912
+ spoke in the transcript (`last_message`). A stop whose words named no reset is tried again
1913
+ `retry` seconds after it (`watchdog.quota_retry_hours`). The action is None, "park", "hold",
1914
+ "resume" or "clear":
1915
+
1916
+ - a stop parks the running items, unless a park has already ended for it: a stop is parked,
1917
+ and resumed after, once. A stop whose reset has already passed resumes at once;
1918
+ - a park holds until its reset, or until its retry where it named none, and a newer stop
1919
+ gives it that stop's. A park with no end held for as long as its stop stood, and while no
1920
+ turn runs nothing moves a stop on;
1921
+ - it resumes then, or as soon as the transcript went on past its stop - a person resumed the
1922
+ session, the model answered - since that is the limit gone, whatever time it named. A stop
1923
+ that still stands when it is tried again is a newer stop, and parks again: one nudge a
1924
+ retry, never a loop;
1925
+ - a park whose items someone took off `blocked` is over, and nothing is moved back;
1926
+ - items blocked on quota with no park standing are a park the watchdog's state lost, torn or
1927
+ deleted: a stop read now parks them again, and without one nothing says the limit stands,
1928
+ so they resume.
1929
+ """
1930
+ now = time.time() if now is None else now
1931
+ retry = S.get(None, "watchdog.quota_retry_hours") * 3600 if retry is None else retry
1932
+ fresh = stop is not None and stop["at"] > (resumed or 0)
1933
+ if park is None:
1934
+ if parked and not fresh:
1935
+ return "resume", {"items": list(parked), "at": resumed or 0, "until": now, "named": False, "lost": True}
1936
+ if not fresh or not (running or parked):
1937
+ return None, None
1938
+ park = quota_park_for(stop, list(parked) + [item for item in running if item not in parked], now, retry)
1939
+ return ("resume" if park["until"] <= now else "park"), park
1940
+ if not set(park.get("items") or ()) & set(parked):
1941
+ return "clear", park
1942
+ if stop is not None and stop["at"] > (park.get("at") or 0):
1943
+ newer = quota_park_for(stop, park.get("items") or (), now, retry)
1944
+ park = dict(park, **{key: newer[key] for key in ("at", "text", "until", "named", "source")})
1945
+ elif spoken and spoken > (park.get("at") or 0):
1946
+ return "resume", dict(park, went_on=True)
1947
+ return ("resume" if (park.get("until") or 0) <= now else "hold"), park
1948
+
1949
+
1950
+ def quota_needs(park):
1951
+ """The `needs:` a slice parked on quota carries: its reset, or when one nudge tries again where none was named."""
1952
+ return f"{QUOTA_NEEDS} ({'resets' if park.get('named') else 'retry'} {_when(park.get('until'))})"
1953
+
1954
+
1955
+ def parked_on_quota(item):
1956
+ """Is a board item blocked on quota, as a park writes it: `needs: quota`, with or without its time?"""
1957
+ return (item["notes"].get("needs") or "").split(" (")[0].strip() == QUOTA_NEEDS
1958
+
1959
+
1960
+ def implementers_item(item, cfg):
1961
+ """Is a board item the implementer's: tagged for no actor, or for the implementer by its role or its name?
1962
+
1963
+ `role:` tags an item for a particular actor, and an item another actor holds does not stop when
1964
+ the implementer does.
1965
+ """
1966
+ role = (item["notes"].get("role") or "").strip()
1967
+ return not role or A.role_of(role, cfg) == "implementer"
1968
+
1969
+
1970
+ def quota_park(root, cfg, st, dry_run=False, now=None, adapter=None):
1971
+ """Keep the running slice parked while the implementer's usage limit stands; True when the cycle stands down.
1972
+
1973
+ What `quota_park_step` decides, written. A park moves the implementer's running items to
1974
+ `blocked` with `needs: quota (resets <time>)`, or `needs: quota (retry <time>)` where no reset
1975
+ was named, and tells a person; while it holds, no nudge is sent. When it ends the items go back
1976
+ to `running`, with any `needs:` the implementer had written on them, and the cycle goes on to
1977
+ the nudge, which resumes the implementer's session through its adapter's resume command: the
1978
+ session the stop was read from. A park that ends forgets the backoff the limit ran up, since
1979
+ the nudges that died on it were no agent ignoring them.
1980
+
1981
+ With `keyflip.rotation` on, a park that would be made asks keyflip to rotate a spent window,
1982
+ as the nudge it stands in front of would, and one that holds asks again every fifteen minutes:
1983
+ an account with headroom ends it at once. A board line that already says what the park says is
1984
+ left as it is. Dry, it neither rotates nor writes (QUOTA-PARK).
1985
+
1986
+ It decides on one reading of the board and moves on that same reading. Each move read the board
1987
+ again, so a row a hand moved while the park was decided - to done, say, while keyflip was asked -
1988
+ was moved on from wherever it had gone. A board that changed meanwhile moves nothing: the cycle
1989
+ stands down, and the next decides again on the board as it stands (SLICE-LIFECYCLE-2).
1990
+ """
1991
+ now = time.time() if now is None else now
1992
+ adapter = A.implementer_adapter(cfg) if adapter is None else adapter
1993
+ edit = A.BoardEdit(root)
1994
+ board = edit.board
1995
+ parked = [item["id"] for item in board["blocked"] if parked_on_quota(item)]
1996
+ said = {item["id"]: item["notes"].get("needs") for item in board["blocked"]}
1997
+ standing = st.get("quota_park") if isinstance(st.get("quota_park"), dict) else None
1998
+ if standing is None and not parked and not A.quota_stops(adapter):
1999
+ return False # nothing parked, and nothing this harness says is a stop
2000
+ last = A.last_message(cfg)
2001
+ action, park = quota_park_step(usage_limit_stop(cfg, st, adapter, last=last), standing, st.get("quota_resumed"),
2002
+ [item["id"] for item in board["running"] if implementers_item(item, cfg)],
2003
+ parked, now, spoken=(last or {}).get("spoken"),
2004
+ retry=S.get(cfg, "watchdog.quota_retry_hours") * 3600)
2005
+ if action is None:
2006
+ return False
2007
+ if action == "park" or (action == "hold" and now - float(st.get("quota_rotation_asked") or 0)
2008
+ >= QUOTA_ROTATION_SPACING):
2009
+ if not dry_run:
2010
+ st["quota_rotation_asked"] = now
2011
+ headroom = A.rotate_if_exhausted(cfg, (adapter.get("resume") or {}).get("argv") or [], "implementer")
2012
+ if headroom["ok"] and headroom["rotated"]:
2013
+ action, park = "resume", dict(park, rotated=headroom["text"])
2014
+ elif action == "hold" and park == standing:
2015
+ save_state(root, st) # when it was asked, so the next cycle does not ask again
2016
+ items = ", ".join(park.get("items") or ())
2017
+ if action in ("clear", "resume"):
2018
+ back = [item for item in park.get("items") or () if item in parked] if action == "resume" else []
2019
+ if action == "resume" and dry_run:
2020
+ print(f"DRY RUN: would end the park of {items} on quota and go on to resume the implementer's session")
2021
+ return True
2022
+ if not dry_run:
2023
+ # The `needs:` each item had before the park: the park's own record, or the board's where the
2024
+ # watchdog's state lost the park (QUOTA-PARK-2).
2025
+ held = {item["id"]: item["notes"].get(QUOTA_HELD) for item in board["blocked"]}
2026
+ for item in back:
2027
+ edit.move(item, "running",
2028
+ {"needs": (park.get("needs") or {}).get(item) or held.get(item), QUOTA_HELD: None},
2029
+ by="watchdog", why=quota_park_ended(park))
2030
+ if not _park_written(edit):
2031
+ return True
2032
+ if action == "clear":
2033
+ print(f"{items} taken off blocked on quota by someone else; the park is over")
2034
+ else:
2035
+ print(f"{quota_park_ended(park)}: {items} {'running again' if back else 'not parked'}; the cycle goes "
2036
+ "on to resume the implementer's session")
2037
+ if not dry_run:
2038
+ st.pop("quota_park", None)
2039
+ st.pop("quota_rotation_asked", None)
2040
+ st["quota_resumed"] = max(park.get("at") or 0, st.get("quota_resumed") or 0)
2041
+ for key in ("idle_answer", "nudge_size", "nudge_fingerprint", "nudge_inputs"):
2042
+ st.pop(key, None)
2043
+ st["attempts"] = 0
2044
+ save_state(root, st)
2045
+ return False
2046
+ until = park.get("until")
2047
+ _FACTS["quota_park"] = {"items": park.get("items"), "until": until, "named": park.get("named")}
2048
+ wait = f"until {_when(until)}" if park.get("named") \
2049
+ else f"with no reset named; one nudge tries again {_when(until)}"
2050
+ if action == "park":
2051
+ if dry_run:
2052
+ print(f"DRY RUN: would park {items} on quota {wait}; not nudging")
2053
+ return True
2054
+ # A `needs:` the implementer wrote on a running item is kept, and given back when the park ends; an
2055
+ # item a lost park left blocked keeps the one it holds on its line, where it is parked again on a
2056
+ # fresh stop, rather than have it cleared (QUOTA-PARK-2).
2057
+ park["needs"] = {item["id"]: item["notes"]["needs"] for item in board["running"]
2058
+ if item["id"] in park["items"] and item["notes"].get("needs")}
2059
+ park["needs"].update({item["id"]: item["notes"][QUOTA_HELD] for item in board["blocked"]
2060
+ if item["id"] in park["items"] and item["notes"].get(QUOTA_HELD)
2061
+ and item["id"] not in park["needs"]})
2062
+ for item in park["items"]:
2063
+ if said.get(item) != quota_needs(park):
2064
+ edit.move(item, "blocked", {"needs": quota_needs(park), QUOTA_HELD: park["needs"].get(item)},
2065
+ by="watchdog", why="the implementer stopped on its usage limit")
2066
+ if not _park_written(edit):
2067
+ return True
2068
+ st["quota_park"] = park
2069
+ save_state(root, st)
2070
+ touch_quota_park(root, st, cfg)
2071
+ print(f"{items} parked on quota {wait}: the implementer stopped on its usage limit; not nudging")
2072
+ return True
2073
+ if park != standing and not dry_run:
2074
+ # A newer stop named another end: the board says the one that stands, and a person is told.
2075
+ for item in park.get("items") or ():
2076
+ if item in parked and said.get(item) != quota_needs(park):
2077
+ edit.move(item, "blocked", {"needs": quota_needs(park)}, by="watchdog",
2078
+ why="a newer stop named another reset")
2079
+ if not _park_written(edit):
2080
+ return True
2081
+ st["quota_park"] = park
2082
+ save_state(root, st)
2083
+ touch_quota_park(root, st, cfg)
2084
+ print(f"{items} parked on quota {wait}; not nudging")
2085
+ return True
2086
+
2087
+
2088
+ def _park_written(edit):
2089
+ """Write what the park moved on the board it read; False, said, when the board changed meanwhile.
2090
+
2091
+ Nothing moved then, and nothing of the park is saved: the cycle stands down and the next one decides on
2092
+ the board as it stands (SLICE-LIFECYCLE-2).
2093
+ """
2094
+ try:
2095
+ edit.write()
2096
+ except A.BoardChanged:
2097
+ print("the board changed while the watchdog read it: nothing moved on quota this cycle, and the next "
2098
+ "cycle decides again on the board as it stands")
2099
+ return False
2100
+ return True
2101
+
2102
+
2103
+ def quota_park_ended(park):
2104
+ """Why a park ended, in the words a cycle's trace gives it."""
2105
+ if park.get("rotated"):
2106
+ return f"keyflip moved the implementer's window to an account with headroom ({park['rotated']})"
2107
+ if park.get("went_on"):
2108
+ return "the transcript went on past the usage limit"
2109
+ if park.get("lost"):
2110
+ return "blocked on quota with no park standing, and nothing read says the limit stands"
2111
+ if park.get("named"):
2112
+ return f"the usage limit reset {_when(park.get('until'))}"
2113
+ return f"no reset was named, and the retry at {_when(park.get('until'))} is due: one nudge tries again"
2114
+
2115
+
2116
+ def touch_quota_park(root, st, cfg=None):
2117
+ """Keep a standing park's alarm raised while its slice waits, as the architect's quota alarm is kept (QUOTA-PARK).
2118
+
2119
+ Raised every cycle the park stands, whichever guard ends the cycle, it climbs the ladder and
2120
+ never lapses: told once, it went quiet two hours later and nothing told a person again, while
2121
+ a slice whose reset nobody named waited on. It rings once for what it says - the items and the
2122
+ words they stopped on - and again when those change. A park with a named reset turns red only
2123
+ once it outstands that reset, and is then held until the reset that stands; one with no reset
2124
+ named is red after the ladder's hour and mailed on red's schedule until it ends, since a person
2125
+ may be what ends it.
2126
+ """
2127
+ park = st.get("quota_park") if isinstance(st.get("quota_park"), dict) else None
2128
+ if not park or not park.get("items"):
2129
+ return False
2130
+ written = _written_in(root) if cfg is None else cfg
2131
+ items, until, named = ", ".join(park["items"]), float(park.get("until") or 0), bool(park.get("named"))
2132
+ red_after = None # the ladder's own hour
2133
+ if named:
2134
+ # Red an hour past its reset, not an hour after it began: a park that ends at its reset asks nothing of anyone.
2135
+ red_after = max(0.0, until - float(park.get("since") or until)) \
2136
+ + S.get(written, "alarms.red_after_minutes") * 60
2137
+ return notify(language.text(written, "alarm.quota-park-title", project=A.project_key(root), items=items),
2138
+ language.text(written, "alarm.quota-park" if named else "alarm.quota-park-unknown",
2139
+ error=str(park.get("text") or "")[:100], reset=_when(until), items=items),
2140
+ root, key="quota-park", window=6 * 3600, audience="human", quiet_until=until if named else None,
2141
+ red_after=red_after, what=f"{items}: {park.get('text')}")
2142
+
2143
+
2144
+ # ---- a slice past the budget it was given (SLICE-BUDGET) ---------------------------------------
2145
+ #
2146
+ # The watchdog nags a slice that makes no progress and parks one whose window is spent, but nothing
2147
+ # stopped it starting turn after turn on a slice that moved - just slowly, or expensively. What a
2148
+ # slice spent was read only after the fact, by `polyctl gain`. A slice given a budget is parked when its
2149
+ # current run spends past it, and polyctl never gives it more: the architect, or a person, does.
2150
+
2151
+ BUDGET_NEEDS = "budget"
2152
+ BUDGET_WHY = "the slice spent past the budget it was given"
2153
+
2154
+
2155
+ def budget_needs(reading):
2156
+ """The `needs:` a slice past its budget carries: the reading it is past."""
2157
+ return f"{BUDGET_NEEDS} ({reading})"
2158
+
2159
+
2160
+ def _is_budget_park(row):
2161
+ """Whether a slice ledger row is the watchdog parking the slice on its budget."""
2162
+ needs = str(((row.get("notes") if isinstance(row.get("notes"), dict) else None) or {}).get("needs") or "")
2163
+ return (row.get("to") == "blocked" and row.get("by") == "watchdog"
2164
+ and (row.get("why") == BUDGET_WHY or needs == BUDGET_NEEDS or needs.startswith(f"{BUDGET_NEEDS} (")))
2165
+
2166
+
2167
+ def budget_start(item_id, moves):
2168
+ """When the slice's current run started: its recorded budget grant, or None where no line authorizes it.
2169
+
2170
+ A slice's first move to `running` starts its run, and a later move to `running` starts a new
2171
+ run only when a person, the architect, or the watchdog's quota-park resume makes it: the
2172
+ implementer blocking and resuming its own slice keeps the run it already had, so it cannot
2173
+ grant itself a fresh budget. A row moved by hand has no line (docs/slices.md), and a slice that
2174
+ left `running` in the ledger and was later moved back to `running` by hand, or that the
2175
+ watchdog parked on budget without a person or the architect starting it since, has no recorded
2176
+ start for its current run.
2177
+ """
2178
+ start, started, parked, running = None, False, False, False
2179
+ for row in moves:
2180
+ if row.get("slice") != item_id:
2181
+ continue
2182
+ to, by, at = row.get("to"), row.get("by"), row.get("at")
2183
+ stamp = (float(at) if isinstance(at, (int, float)) and not isinstance(at, bool) and math.isfinite(at)
2184
+ else None)
2185
+ if to == "running":
2186
+ if by not in ("implementer", "watchdog", None) or (by == "watchdog" and not parked):
2187
+ start, started, parked, running = stamp, True, False, True
2188
+ elif not started:
2189
+ start, started, parked, running = stamp, True, False, True
2190
+ elif not parked:
2191
+ running = True
2192
+ elif _is_budget_park(row):
2193
+ parked, running = True, False
2194
+ else:
2195
+ running = False
2196
+ return start if running and not parked else None
2197
+
2198
+
2199
+ def budget_runs(board, moves, cfg):
2200
+ """The running intervals [(start, end, closed)] of each implementer slice, from the slice ledger and board.
2201
+
2202
+ A move to `running` opens an interval and a move elsewhere closes it. A slice that left `running`
2203
+ by hand - landing moves a slice to `done` by hand (docs/slices.md) - closes by the next start
2204
+ recorded for another slice, or stays open when none followed. An implementer slice on `running`
2205
+ with no recorded start is open from its last recorded departure from `running` (or the beginning
2206
+ of time), so its share of a turn's spend with another running slice is never guessed away.
2207
+ """
2208
+ items = {item["id"]: item for section in board.values() for item in section}
2209
+ running = {item["id"] for item in board["running"] if implementers_item(item, cfg)}
2210
+ events = [(row.get("slice"), row.get("to"), float(row["at"])) for row in moves
2211
+ if isinstance(row.get("slice"), str) and row.get("slice")
2212
+ and (row["slice"] not in items or implementers_item(items[row["slice"]], cfg))
2213
+ and isinstance(row.get("at"), (int, float)) and not isinstance(row.get("at"), bool)
2214
+ and math.isfinite(row["at"])]
2215
+ runs, opened, closed_at = {}, {}, {}
2216
+ for sid, to, at in events:
2217
+ if to == "running":
2218
+ if sid in opened:
2219
+ runs.setdefault(sid, []).append((opened[sid], at, True))
2220
+ opened[sid] = at
2221
+ elif sid in opened:
2222
+ runs.setdefault(sid, []).append((opened.pop(sid), at, True))
2223
+ closed_at[sid] = at
2224
+ for sid, lo in opened.items():
2225
+ if sid in running:
2226
+ runs.setdefault(sid, []).append((lo, float("inf"), False))
2227
+ else:
2228
+ later = [at for other, to, at in events if to == "running" and other != sid and at >= lo]
2229
+ runs.setdefault(sid, []).append((lo, later[0], True) if later else (lo, float("inf"), False))
2230
+ for sid in running - set(opened):
2231
+ runs.setdefault(sid, []).append((closed_at.get(sid, -float("inf")), float("inf"), False))
2232
+ return runs
2233
+
2234
+
2235
+ def _read_transcript_records(msgs):
2236
+ """The implementer's transcript records; raises when the transcript store is missing or fails to read."""
2237
+ if not msgs or not os.path.exists(msgs):
2238
+ raise OSError("the transcript could not be read")
2239
+ if isinstance(msgs, A.DatabaseTail):
2240
+ declared = msgs.record
2241
+ table, session_key = A._store_name(declared.get("table")), A._store_name(declared.get("session_key"))
2242
+ order = A._store_name(declared.get("order")) or A._store_name(declared.get("time"))
2243
+ if not (msgs.session and table and session_key and order):
2244
+ raise OSError("the transcript could not be read")
2245
+ connection = msgs._connect()
2246
+ if connection is None:
2247
+ raise OSError("the transcript could not be read")
2248
+ try:
2249
+ connection.execute(
2250
+ f"SELECT 1 FROM {table} WHERE {session_key} = ? ORDER BY {order} DESC LIMIT 1",
2251
+ (msgs.session,),
2252
+ ).fetchall()
2253
+ children = A._block(declared, "children")
2254
+ child_table = A._store_name(children.get("table"))
2255
+ child_key = A._store_name(children.get("key"))
2256
+ child_parent = A._store_name(children.get("parent_key"))
2257
+ child_order = A._store_name(children.get("order")) or A._store_name(children.get("time"))
2258
+ child_session = A._store_name(children.get("session_key")) or child_key
2259
+ if child_table and children.get("as") and child_key and child_parent and child_order:
2260
+ connection.execute(
2261
+ f"SELECT 1 FROM {child_table} WHERE {child_session} = ? ORDER BY {child_order} ASC LIMIT 1",
2262
+ (msgs.session,),
2263
+ ).fetchall()
2264
+ finally:
2265
+ connection.close()
2266
+ return A.read_tail(msgs, 400_000_000)
2267
+ with A._open_regular(msgs) as fh:
2268
+ size = os.fstat(fh.fileno()).st_size
2269
+ fh.seek(max(0, size - 400_000_000))
2270
+ blob = fh.read()
2271
+ recs = A.read_tail(msgs, 400_000_000)
2272
+ if not recs and blob.strip():
2273
+ raise OSError("the transcript could not be read")
2274
+ return recs
2275
+
2276
+
2277
+ def _read_subagent_records(path):
2278
+ """A subagent transcript's records; raises when the subagent transcript fails to read."""
2279
+ with A._open_regular(path) as fh:
2280
+ blob = fh.read()
2281
+ recs = A.subagent_records(path)
2282
+ if not recs and blob.strip():
2283
+ raise OSError("the transcript could not be read")
2284
+ return recs
2285
+
2286
+
2287
+ def _usage_events(cfg):
2288
+ """(unit, [(window_start, at, grown)]) from the implementer's transcript, in the unit its adapter declares.
2289
+
2290
+ Each usage record is read in its turn (`next_turn`, `add_usage`), so per-response records and
2291
+ later records of a turn that already wrote usage before a slice started count from the slice's
2292
+ start onwards, and a whole-turn usage summary whose turn opened before the start and closed after
2293
+ it keeps its turn opening time (`window_start < start`) and is charged in full to the slice when no
2294
+ other slice ran in that span. Replies the harness wrote in the model's place (`in_place_reply`)
2295
+ count nothing, and an adapter that declares no usage telemetry or a transcript store that fails to
2296
+ read raises so a spend budget never passes silently.
2297
+ """
2298
+ msgs, _ = A.session_paths(cfg)
2299
+ adapter = A.implementer_adapter(cfg)
2300
+ shape = A.transcript_shape(adapter)
2301
+ usage = shape["usage"]
2302
+ unit = shape["unit"] or "unit"
2303
+ if not usage:
2304
+ name = A.implementer_adapter_id(cfg) or str((cfg.get("implementer") or {}).get("adapter") or "unknown")
2305
+ raise ValueError(f"adapter {name!r} declares no transcript usage telemetry")
2306
+ recs = _read_transcript_records(msgs)
2307
+ turn, counted, events, when = None, {}, [], None
2308
+ for rec, subagent in A.with_subagents(recs, shape, msgs, read=_read_subagent_records):
2309
+ when = A._ledger_time(A.record_time(rec, shape)) or when
2310
+ kind = A.record_kind(rec, shape)
2311
+ turn, opened = A.next_turn(turn, rec, shape, subagent)
2312
+ if opened:
2313
+ turn["at"] = when
2314
+ turn["steps"] = 0
2315
+ body = A.record_body(rec, shape)
2316
+ if body is None or kind != usage["type"] or A.in_place_reply(rec, shape):
2317
+ continue
2318
+ turn = {"at": when, "steps": 0} if turn is None else turn
2319
+ grown = A.add_usage(turn, body, usage, counted, subagent)
2320
+ if when is not None and grown > 0:
2321
+ steps = turn.get("steps", 0)
2322
+ lo = (when if usage["reading"] == "per-response" or steps > 0
2323
+ else (turn.get("at") if turn.get("at") is not None else when))
2324
+ events.append((lo, when, float(grown)))
2325
+ turn["at"] = when
2326
+ turn["steps"] = steps + 1
2327
+ return unit, events
2328
+
2329
+
2330
+ def budget_spend_for(item_id, start, runs, events):
2331
+ """(spent, shared) for `item_id`'s run since `start`: what it spent alone, and which slices shared a turn's spend."""
2332
+ spent, shared = 0.0, []
2333
+ for lo, when, grown in events:
2334
+ if when < start:
2335
+ continue
2336
+ span = max(lo, start)
2337
+ active = sorted(sid for sid, intervals in runs.items()
2338
+ if any(r_lo <= when and (span < r_hi if closed else span <= r_hi)
2339
+ for r_lo, r_hi, closed in intervals))
2340
+ if len(active) > 1 and item_id in active:
2341
+ for sid in active:
2342
+ if sid not in shared:
2343
+ shared.append(sid)
2344
+ elif active == [item_id] or not active:
2345
+ spent += grown
2346
+ return spent, shared
2347
+
2348
+
2349
+ def budget_reading(minutes, minute_budget, spent, unit, spend_budget, spend_missing=None):
2350
+ """The reading a park writes: each budget the run is past or could not be read against."""
2351
+ parts = []
2352
+ if spend_budget:
2353
+ if spend_missing:
2354
+ parts.append(spend_missing)
2355
+ elif spent is not None and spent > spend_budget:
2356
+ parts.append(f"spent {spent:g} {unit} of {spend_budget:g}")
2357
+ if minute_budget and minutes > minute_budget:
2358
+ whole = max(int(minutes), int(minute_budget) + 1)
2359
+ parts.append(f"{whole}m of {minute_budget}m")
2360
+ return "; ".join(parts)
2361
+
2362
+
2363
+ def _committed_project_config(root):
2364
+ """The project `.polyctl/config.json` from `HEAD`, or `{}` when no committed config can be read."""
2365
+ if not root:
2366
+ return {}
2367
+ try:
2368
+ raw = json.loads(A._git_output(root, "show", "HEAD:.polyctl/config.json") or "{}")
2369
+ except Exception:
2370
+ return {}
2371
+ return raw if isinstance(raw, dict) else {}
2372
+
2373
+
2374
+ def _effective_slice_budgets(root, cfg):
2375
+ """(minute_budget, spend_budget) holding the tightest positive caps across config, HEAD, and machine settings.
2376
+
2377
+ A cap the implementer cannot move is the machine settings' cap, outside the tree. A cap
2378
+ committed in `.polyctl/config.json` counts while `HEAD` shows it, and is no guard against an
2379
+ implementer that moves `HEAD` or breaks the git read (which reads as no committed cap); a
2380
+ person widens or switches off a cap with `0` where it is set.
2381
+ """
2382
+ committed = _committed_project_config(root)
2383
+ out = {}
2384
+ for key, kind in (("slice_budget_minutes", int), ("slice_budget_spend", float)):
2385
+ candidates = [S.get(cfg, key), S.get({}, key)]
2386
+ if key in committed and S.usable(key, committed[key]):
2387
+ candidates.append(kind(committed[key]))
2388
+ positive = [kind(val) for val in candidates if val and val > 0]
2389
+ out[key] = min(positive) if positive else kind(0)
2390
+ return out["slice_budget_minutes"], out["slice_budget_spend"]
2391
+
2392
+
2393
+ def slice_budget(root, cfg, dry_run=False, now=None):
2394
+ """Park the implementer's running slices that spent past their budget or lack a reading it needs (SLICE-BUDGET).
2395
+
2396
+ A budget is per run, not per slice: `slice_budget_minutes` of wall time and `slice_budget_spend`
2397
+ of the implementer's own spend - in the unit its adapter's transcript records - since the slice's
2398
+ newest recorded move to running, including the spend of the turn that started it (charged in full
2399
+ when a whole-turn usage summary opened before the start and closed after it), attributed to the
2400
+ slice running then. A slice blocked on quota and resumed starts a new run, and one a person starts
2401
+ again gets its budget again.
2402
+
2403
+ The guard fails closed: every reading a configured budget cannot make - no recorded start, an
2404
+ unreadable slice ledger, a transcript store that fails to read, spend that overlapping slices make
2405
+ unattributable, or usage the adapter does not report for a spend cap - parks the slice with
2406
+ `needs: budget (<reading>)`, by the watchdog, and tells the architect; it never reads a missing
2407
+ reading as zero or as within budget. The implementer may not lift a watchdog's block, and nothing
2408
+ resumes it by itself: only the architect's or a person's `polyctl slice start` does, and the run the
2409
+ budget counts from starts again there. Nothing is read when no budget is set.
2410
+ """
2411
+ now = time.time() if now is None else now
2412
+ minute_budget, spend_budget = _effective_slice_budgets(root, cfg)
2413
+ if not minute_budget and not spend_budget:
2414
+ return False
2415
+ edit = A.BoardEdit(root)
2416
+ running = [item for item in edit.board["running"] if implementers_item(item, cfg)]
2417
+ try:
2418
+ moves = A.slice_moves(root)
2419
+ except Exception as exc:
2420
+ if not running:
2421
+ print(f"the slice ledger could not be read ({exc}); no slice can be judged against its budget, "
2422
+ "so not nudging")
2423
+ return True
2424
+ print(f"the slice ledger could not be read ({exc}); parking {', '.join(item['id'] for item in running)}")
2425
+ overdue = [(item["id"], "the slice ledger could not be read") for item in running]
2426
+ held = {}
2427
+ for item in running:
2428
+ prior_needs = item["notes"].get(A.HELD_NEEDS) or item["notes"].get("needs")
2429
+ held[item["id"]] = None if str(prior_needs or "").startswith(f"{BUDGET_NEEDS} (") else prior_needs
2430
+ moves = None
2431
+ if moves is not None:
2432
+ if not running:
2433
+ return False
2434
+ starts = [(item, budget_start(item["id"], moves)) for item in running]
2435
+ runs = budget_runs(edit.board, moves, cfg) if spend_budget else {}
2436
+ unit, events, spend_err = None, [], None
2437
+ if spend_budget and any(start is not None and start <= now for _, start in starts):
2438
+ try:
2439
+ unit, events = _usage_events(cfg)
2440
+ except Exception as exc:
2441
+ spend_err = exc
2442
+ overdue, held = [], {}
2443
+ for item, start in starts:
2444
+ if start is None or start > now:
2445
+ reading = "no recorded start"
2446
+ else:
2447
+ minutes = max(0.0, (now - start) / 60.0)
2448
+ spent, spend_missing = None, None
2449
+ if spend_budget:
2450
+ if spend_err is not None:
2451
+ if isinstance(spend_err, ValueError) and "declares no transcript usage telemetry" in str(spend_err):
2452
+ spend_missing = str(spend_err)
2453
+ else:
2454
+ spend_missing = "the transcript could not be read"
2455
+ print(f"{item['id']}: its spend could not be read ({spend_err}); parking")
2456
+ else:
2457
+ spent, shared = budget_spend_for(item["id"], start, runs, events)
2458
+ if shared:
2459
+ spent = None
2460
+ spend_missing = f"spend cannot be attributed across {', '.join(shared)}"
2461
+ reading = budget_reading(minutes, minute_budget, spent, unit, spend_budget,
2462
+ spend_missing=spend_missing)
2463
+ if reading:
2464
+ overdue.append((item["id"], reading))
2465
+ prior_needs = item["notes"].get(A.HELD_NEEDS) or item["notes"].get("needs")
2466
+ held[item["id"]] = None if str(prior_needs or "").startswith(f"{BUDGET_NEEDS} (") else prior_needs
2467
+ if not overdue:
2468
+ return False
2469
+ _FACTS["slice_budget"] = {"items": [item for item, _ in overdue],
2470
+ "readings": {item: reading for item, reading in overdue}}
2471
+ readings = "; ".join(f"{item}: {reading}" for item, reading in overdue)
2472
+ if dry_run:
2473
+ print(f"DRY RUN: would park {readings} on the budget it was given; not nudging")
2474
+ return True
2475
+ for item, reading in overdue:
2476
+ edit.move(item, "blocked", {"needs": budget_needs(reading), A.HELD_NEEDS: held.get(item)},
2477
+ by="watchdog", why=BUDGET_WHY)
2478
+ if moves is None:
2479
+ edit.moves.clear()
2480
+ try:
2481
+ edit.write()
2482
+ except A.BoardChanged:
2483
+ print("the board changed while the watchdog read it: nothing moved on budget this cycle, and the next "
2484
+ "cycle decides again on the board as it stands")
2485
+ return True
2486
+ names = ", ".join(item for item, _ in overdue)
2487
+ notify(language.text(cfg, "alarm.slice-budget-title", project=A.project_key(root), items=names),
2488
+ language.text(cfg, "alarm.slice-budget", reading=readings),
2489
+ root, key="slice-budget", window=6 * 3600, audience="architect", what=readings)
2490
+ print(f"{names} parked on the budget it was given ({readings}); not nudging")
2491
+ return True
2492
+
2493
+
2494
+ def main():
2495
+ # What a cycle decided is printed, board titles and mail subjects included, to a terminal, a pipe
2496
+ # or the log a scheduled job keeps: in UTF-8, whatever the platform's code page (#71).
2497
+ A.utf8_streams()
2498
+ p = argparse.ArgumentParser()
2499
+ p.add_argument("--root", required=True)
2500
+ p.add_argument("--idle-minutes", type=float, default=None,
2501
+ help="default: the project's watchdog.idle_minutes setting")
2502
+ p.add_argument("--dry-run", action="store_true")
2503
+ # The default prompt encodes two rules learned the expensive way. Park-and-
2504
+ # continue: a blocked slice must not stop the run, because the architect can
2505
+ # hit a quota limit and the human can be away. And a pointer to one canonical
2506
+ # authority file: when "what may I do" is spread across mail, a prompt and
2507
+ # months-old instructions, an agent facing an apparent conflict refuses and
2508
+ # waits — which is safe, and cost this project four hours with 7,000 lines of
2509
+ # finished work sitting uncommitted.
2510
+ p.add_argument("--prompt", default=None, help="default: the project's nudge, in its language")
2511
+ args = p.parse_args()
2512
+ return run(args)
2513
+
2514
+
2515
+ CYCLE_LOCK = A.PROJECT_FILES["watchdog-cycle-lock"]
2516
+
2517
+
2518
+ def run(args):
2519
+ """One cycle, traced and — unless dry — recorded.
2520
+
2521
+ Live cycles take a per-project lock first. The scheduled job and `polyctl catchup`
2522
+ both ran a cycle, both passed the process scan before either spawned, and both
2523
+ resumed the implementer: the two-writer incident the scan exists to prevent.
2524
+ A cycle that finds the lock held stands down.
2525
+ """
2526
+ _TRACE.clear()
2527
+ _FACTS.clear()
2528
+ root = os.path.abspath(os.path.expanduser(args.root))
2529
+ if getattr(args, "idle_minutes", None) is None:
2530
+ args.idle_minutes = S.get(A.load_config(root), "watchdog.idle_minutes")
2531
+ started = time.time()
2532
+ if args.dry_run:
2533
+ try:
2534
+ return _cycle(args, root)
2535
+ finally:
2536
+ record_cycle(root, args, started)
2537
+ from .storage import LedgerLockTimeout, _exclusive_lock
2538
+ key = A.project_key(root)
2539
+ lock = os.path.join(STATE_DIR, CYCLE_LOCK.format(key=key))
2540
+ try:
2541
+ os.makedirs(STATE_DIR, exist_ok=True)
2542
+ with _exclusive_lock(lock, timeout=0):
2543
+ try:
2544
+ A.bound_observation_logs(root, STATE_DIR, busy=_logs_written(root)) # every store is bounded (#50)
2545
+ except OSError:
2546
+ pass
2547
+ try:
2548
+ code = _cycle(args, root)
2549
+ args.ran = True # this process's cycle ran to its end (catchup reads it)
2550
+ return code
2551
+ finally:
2552
+ record_cycle(root, args, started)
2553
+ except LedgerLockTimeout as exc:
2554
+ if getattr(exc, "path", lock) != lock:
2555
+ # A lock the cycle took inside it was held too long: no other cycle, and standing down hid it (WATCHDOG-LOCK).
2556
+ print(f"the cycle could not take {exc.path} in time; it ends here, and the next cycle tries again")
2557
+ return 1
2558
+ print("another watchdog cycle is running for this project; standing down")
2559
+ return 0
2560
+
2561
+
2562
+ def _cycle(args, root):
2563
+ """Run one cycle with dry-run effects suppressed across nested helpers.
2564
+
2565
+ A resume notice the cycle owes is sent here, once every guard has had its say and
2566
+ whichever of them ended the cycle (RESUME-QUIET). A process table the platform could
2567
+ not read ends the cycle at the guard that asked it (WINDOWS-TABLE-CLOSED).
2568
+ """
2569
+ token = _DRY_RUN.set(bool(args.dry_run))
2570
+ resuming = _RESUME.set(None)
2571
+ try:
2572
+ return _cycle_impl(args, root)
2573
+ except TableUnreadable as exc:
2574
+ return _stand_down_unseen(root, exc)
2575
+ finally:
2576
+ owed = _RESUME.get()
2577
+ _RESUME.reset(resuming)
2578
+ try:
2579
+ if owed is not None:
2580
+ announce_resume(root, owed, dry_run=bool(args.dry_run))
2581
+ except Exception as exc: # a notice that fails must not take the cycle's record
2582
+ _note_above_verdict(f"the resume notice was not sent: {type(exc).__name__}: {str(exc)[:120]}")
2583
+ finally:
2584
+ _DRY_RUN.reset(token)
2585
+
2586
+
2587
+ def _stand_down_unseen(root, exc):
2588
+ """End a cycle that could not read the process table: it starts, wakes and reaps nothing, and a person is told
2589
+ why (WINDOWS-TABLE-CLOSED).
2590
+
2591
+ Every guard that keeps a second turn out of a tree, a second architect out of a session and a
2592
+ person's session out of the reaper asks the table what runs. A table the platform could not read
2593
+ - on Windows, a PowerShell that was missing or did not answer - was an empty one, and each of those
2594
+ guards read it as nothing running. The alarm rings once for the platform it names and climbs as
2595
+ any other while the table stays unread.
2596
+ """
2597
+ try:
2598
+ notify(f"{A.project_key(root)}: the process table cannot be read",
2599
+ f"{exc}. Until it can see what runs in the tree the watchdog starts no turn, wakes nobody and "
2600
+ "reaps nothing.", root, key="process-table-unreadable", window=6 * 3600, audience="human",
2601
+ what=f"unreadable on {exc.platform}")
2602
+ except Exception as told:
2603
+ print(f"the alarm for the process table could not be raised ({type(told).__name__})")
2604
+ print(f"{exc}; this cycle starts no turn, wakes nobody and reaps nothing, and the next cycle looks again")
2605
+ return 1
2606
+
2607
+
2608
+ def _sample_credits(root, st, adapter_id, project, now=None):
2609
+ """Read the implementer's own credit account at most once per half hour and act on it.
2610
+
2611
+ Exhaustion is reported from the reading itself, on the first observation: a
2612
+ projection has nothing to say once the plan has already run out, which is the
2613
+ moment it matters. The burn rate still warns ahead of it. A reading that cannot
2614
+ be taken is recorded as a broken check, which `polyctl doctor` shows, instead of
2615
+ being skipped: a silent check and a passing check must never look the same. A
2616
+ failed attempt waits the same half hour before the next one.
2617
+
2618
+ Either alarm has a known end, the reset the reading names, so it is mailed once
2619
+ and held until then (#40). Raised without it, an exhausted plan whose reset was
2620
+ ten days away mailed every six hours. What it says - run out, or will - is named
2621
+ per account, so the day a projection comes true is still told (NOTICE-NOISE).
2622
+
2623
+ The account is the one the package's adapter of `adapter_id`, the implementer's,
2624
+ declares a lookup for (ACCOUNT-READERS). The first shipped adapter's was read whatever
2625
+ the implementer ran, once a layer the implementer can write said its adapter bills
2626
+ through an API: an implementer on a harness that bills no account polyctl can read was
2627
+ sampled, and alarmed, on another harness's. It has no samples and no credits alarm
2628
+ now. A sample and a broken check name the adapter they were read through, and only
2629
+ that adapter's samples project its burn rate.
2630
+ """
2631
+ now = now or time.time()
2632
+ if now - max(st.get("last_credit_sample", 0), st.get("last_credit_attempt", 0)) <= 1800:
2633
+ return
2634
+ if not A.usage_api(adapter_id):
2635
+ return
2636
+ try:
2637
+ acct = A.account_usage(adapter_id=adapter_id)
2638
+ except Exception as exc:
2639
+ acct = {"error": f"the usage check raised {type(exc).__name__}"}
2640
+ if acct is None:
2641
+ acct = {"error": "the declared usage lookup returned no reading; check the CLI's login and retry"}
2642
+ st["last_credit_attempt"] = now
2643
+ if not acct.get("limit"):
2644
+ reason = acct.get("error") or ("the CLI's token has expired" if acct.get("expired")
2645
+ else "the provider returned no credit limit")
2646
+ st["credit_check_problem"] = {"at": int(now), "reason": reason, "adapter": adapter_id}
2647
+ save_state(root, st)
2648
+ return
2649
+ st.pop("credit_check_problem", None)
2650
+ A.record_credit_sample(root, acct.get("used", 0), acct["limit"], acct.get("reset_at"),
2651
+ account=acct.get("account"), adapter=adapter_id)
2652
+ st["last_credit_sample"] = now
2653
+ save_state(root, st)
2654
+ used, limit = float(acct.get("used") or 0), float(acct["limit"])
2655
+ if used >= limit:
2656
+ reading = A.notice_evidence("credit_reading", [
2657
+ {"value": f"{used:.0f}/{limit:.0f}", "at": int(now),
2658
+ "source": f"GetUsageLimits, account {acct.get('account')}"}])
2659
+ notify(f"{project}: credits exhausted",
2660
+ f"{used:.0f}/{limit:.0f} used; the plan is spent and only overage, if enabled, runs "
2661
+ f"until the reset. New account (keyflip) or `polyctl features off …`", root,
2662
+ key="credits-exhaust", window=6 * 3600, audience="human", level="red", evidence=reading,
2663
+ quiet_until=credit_reset(acct.get("reset_at"), now), what=credits_told("exhausted", acct.get("account")))
2664
+ return
2665
+ br = A.burn_rate(root, adapter_id)
2666
+ if br and br["before_reset"]:
2667
+ evidence = A.notice_evidence("burn_rate", [
2668
+ {"value": f"{sample['used']:.0f}/{sample['limit']:.0f}", "at": sample.get("at"),
2669
+ "source": f"GetUsageLimits, account {sample.get('account')}"} for sample in br["samples"]])
2670
+ notify(f"{project}: credits run out {time.strftime('%d %b', time.localtime(br['exhausts_at']))}",
2671
+ f"{br['used']:.0f}/{br['limit']:.0f} at {br['per_day']:.0f}/day; the reset is later. "
2672
+ f"New account (keyflip) or `polyctl features off …`", root, key="credits-exhaust",
2673
+ window=6 * 3600, audience="human", level="red", evidence=evidence,
2674
+ quiet_until=credit_reset(br.get("reset_at"), now), what=credits_told("projected", br.get("account")))
2675
+
2676
+
2677
+ def credit_reset(value, now=None):
2678
+ """The reset a credit reading names, in epoch seconds; None when it names none it can mean (NOTICE-NOISE).
2679
+
2680
+ The provider gives seconds. A date string or milliseconds is no known end: an alarm
2681
+ held on a misread date would stay silent long past the day it should have spoken,
2682
+ while one without a known end only repeats. A reset already past holds nothing.
2683
+ """
2684
+ now = time.time() if now is None else now
2685
+ try:
2686
+ reset = float(value)
2687
+ except (TypeError, ValueError):
2688
+ return None
2689
+ return reset if 0 < reset <= now + 400 * 86400 else None
2690
+
2691
+
2692
+ def credits_told(kind, account):
2693
+ """What a credits alarm says, for the ladder: run out or projected, and whose (NOTICE-NOISE)."""
2694
+ return f"{kind}:{account or 'unnamed'}"
2695
+
2696
+
2697
+ DECISION_HUMAN_AFTER = S.default("decisions.human_after_minutes") * 60
2698
+
2699
+
2700
+ def escalate_open_decisions(root, project, dry_run=False, now=None, since=0):
2701
+ """A decision nobody has answered reaches a person (#20).
2702
+
2703
+ Fifteen minutes after it was asked (`decisions.human_after_minutes`) it rings
2704
+ orange on the human channel, once for the decision it names, and the alarm ladder
2705
+ turns it red after its hour and mails on red's schedule until it is answered. The wake
2706
+ is not waited for: on 2026-09-07 a decision stood three hours while the only
2707
+ notice about it was held for an architect that never came. An answered
2708
+ decision stops being raised and its alarm ends on its own. Returns the ids
2709
+ that are ringing. One asked before a resume waits from the resume (`since`),
2710
+ not from before the silence (RESUME-QUIET).
2711
+ """
2712
+ now = time.time() if now is None else now
2713
+ ringing = []
2714
+ wait = S.get(A.load_config(root), "decisions.human_after_minutes") * 60
2715
+ for decision in A.decisions(root, "open"):
2716
+ asked = decision.get("asked_at") or 0
2717
+ if not asked or now - max(asked, since) < wait:
2718
+ continue
2719
+ ringing.append(decision.get("id"))
2720
+ if dry_run:
2721
+ print(f"DRY RUN: would ring a person about decision {decision.get('id')}")
2722
+ continue
2723
+ ring_decision(root, project, decision, now=now)
2724
+ return ringing
2725
+
2726
+
2727
+ def decision_told(decision):
2728
+ """What an open decision's alarm says, for the ladder: which decision, and its question (NOISE-REPEATS)."""
2729
+ return f"{decision.get('id')}: {' '.join(str(decision.get('question') or '').split())[:120]}"
2730
+
2731
+
2732
+ def ring_decision(root, project, decision, now=None, why=None):
2733
+ """Raise an open decision's own alarm, `decision-open:<id>`: its question and how to answer it (NOISE-REPEATS).
2734
+
2735
+ It rang every hour, and the anomaly's "needs you" told the same decision beside it: one
2736
+ decision nobody answered reached the desktop and the phone 23 times in a day and was
2737
+ mailed under both keys. Now it rings once for the decision it names and climbs the ladder;
2738
+ another decision is another alarm, and is told. `why` says why no architect acts on it,
2739
+ when that is why it rings before its time.
2740
+ """
2741
+ now = time.time() if now is None else now
2742
+ did = decision.get("id")
2743
+ return notify(f"{project}: decision waiting",
2744
+ f"{did}: {str(decision.get('question') or '')[:120]} — open "
2745
+ f"{int((now - (decision.get('asked_at') or now)) / 60)}m; polyctl answer {did} <key>"
2746
+ + (f" — no architect will act on it: {why}" if why else ""),
2747
+ root, key=f"decision-open:{did}", window=3600, audience="human", what=decision_told(decision))
2748
+
2749
+
2750
+ def report_ungranted_commits(root, project):
2751
+ """Tell a person about commits whose tree no grant bound, every cycle while they stand (#109).
2752
+
2753
+ A harness granted every tool can run `git commit --no-verify`, and nothing in
2754
+ its grant keeps that out. The landed-tree check cannot stop such a commit; run
2755
+ every cycle, it makes one visible within a cycle.
2756
+
2757
+ It was told to the architect, as a notice row nobody sends - recorded, never rung,
2758
+ hidden by `polyctl notices` - about the newest 20 commits, and not in a cycle that ended
2759
+ for want of an implementer or a transcript (the review of ATTACK-SUITE). It is now
2760
+ the alarm the scheduled doctor raises for the same finding, `landed-without-grant`,
2761
+ to a person, saying what `A.landed_without_grant` says: it rings the desktop and
2762
+ the phone once for the commits it names, climbs to red and mails while they stand,
2763
+ and rings again when others land; a check that cannot run is that alarm too, saying
2764
+ why (LEDGER-HISTORY). Returns what it said, or None.
2765
+ """
2766
+ told = A.landed_without_grant(root)
2767
+ if told:
2768
+ notify(f"{project}: landed without a grant", told, root, key="landed-without-grant", window=6 * 3600,
2769
+ audience="human", what=told)
2770
+ return told
2771
+
2772
+
2773
+ # ---- after a silence: one notice, and clocks that restart (RESUME-QUIET) ----------------------
2774
+
2775
+ def _span(seconds):
2776
+ """How long a silence lasted, in the unit it is read in: 14d, 5h, 40m."""
2777
+ seconds = max(0, int(seconds))
2778
+ if seconds >= 2 * 86400:
2779
+ return f"{seconds // 86400}d"
2780
+ if seconds >= 2 * 3600:
2781
+ return f"{seconds // 3600}h"
2782
+ return f"{seconds // 60}m"
2783
+
2784
+
2785
+ def _when(at):
2786
+ return time.strftime("%d %b %H:%M", time.localtime(float(at or 0)))
2787
+
2788
+
2789
+ def _note_above_verdict(line):
2790
+ """A line decided after the verdict was printed, kept above it: a trace ends on its verdict."""
2791
+ _TRACE.insert(max(len(_TRACE) - 1, 0), line)
2792
+ _builtins.print(line)
2793
+
2794
+
2795
+ def _markdown_plain(text):
2796
+ """Text Telegram's Markdown shows as written: one stray underscore in a key loses the whole message."""
2797
+ return re.sub(r"([_*`\[])", r"\\\1", str(text))
2798
+
2799
+
2800
+ def _tell_phone(root, text):
2801
+ """A courtesy line to the phone; after a silence it joins the resume notice instead (RESUME-QUIET)."""
2802
+ resuming = _RESUME.get()
2803
+ if resuming is not None:
2804
+ resuming["folded"].append({"key": "", "title": "", "msg": text.replace("*", ""), "audience": "architect"})
2805
+ return
2806
+ try:
2807
+ from . import telegram
2808
+ telegram.send(text, root)
2809
+ except Exception:
2810
+ pass # a phone being unreachable is not a failure
2811
+ try:
2812
+ from . import ntfy
2813
+ ntfy.send(text, root)
2814
+ except Exception:
2815
+ pass
2816
+
2817
+
2818
+ def _handoff_quiet():
2819
+ """What `polyctl handoff` is also given: after a silence its note is written, not sent (RESUME-QUIET)."""
2820
+ resuming = _RESUME.get()
2821
+ if resuming is None:
2822
+ return []
2823
+ resuming["folded"].append({"key": "", "title": "", "msg": "a handoff note was written to the mailbox, not sent",
2824
+ "audience": "architect"})
2825
+ return ["--no-send"]
2826
+
2827
+
2828
+ def resume_after(root, cfg, silence, dry_run=False, now=None):
2829
+ """The resume this cycle belongs to: {"at", "since", "quiet", "carried", "snoozes", "announced"}, or None.
2830
+
2831
+ On 2026-09-17 an owner switched one project's watchdog and doctor jobs off for two
2832
+ weeks, because they kept ringing about an implementer that had run out of credits.
2833
+ Every record stayed on disk with its date. Rebuilt in a temporary project, the first
2834
+ cycle back mailed an unread request red at once, rang an open decision on the desktop
2835
+ and the phone, announced a red alarm as no longer raised although it still stood,
2836
+ mailed the credits alarm whose snooze had ended while nothing ran, and told the phone
2837
+ it had woken the architect; an hour later the decision mailed too. Each was measured
2838
+ as if the watchdog had been watching all along.
2839
+
2840
+ A cycle that follows more than `watchdog.resume_gap_hours` without one, read from the
2841
+ heartbeat the last cycle left, is a resume. It records when it began, how long the
2842
+ silence was, and what the silence carried: the alarm episodes nothing raised for the
2843
+ gap (or for the quiet that ends an episode, when that is shorter) and the snoozes that
2844
+ ended in it. A heartbeat that cannot be written must not make every cycle a resume, so
2845
+ another needs as long again after this one. A dry run writes nothing.
2846
+ """
2847
+ now = time.time() if now is None else now
2848
+ gap = S.get(cfg, "watchdog.resume_gap_hours") * 3600
2849
+ st = load_state(root)
2850
+ resume = st.get("resume") if isinstance(st.get("resume"), dict) else None
2851
+ if silence is None or silence <= gap or now - float((resume or {}).get("at") or 0) <= gap:
2852
+ return resume
2853
+ project, since, quiet = A.project_key(root), now - silence, min(gap, A._alarm_reset_after())
2854
+ carried = []
2855
+ for name, episode in sorted(A.load_alarms().items()):
2856
+ owner, _, key = name.partition(":")
2857
+ if owner == project and isinstance(episode, dict) and now - float(episode.get("last") or 0) > quiet:
2858
+ carried.append(dict({field: episode.get(field) for field in
2859
+ ("title", "level", "ring", "first", "last", "count", "red_sent",
2860
+ "quiet_until")}, key=key))
2861
+ snoozes = []
2862
+ for name, snooze in sorted(A.load_alarm_snoozes().items()):
2863
+ owner, _, key = name.partition(":")
2864
+ if owner == project and isinstance(snooze, dict) and since < float(snooze.get("until") or 0) <= now:
2865
+ snoozes.append({"key": key, "until": snooze.get("until"), "by": snooze.get("by"),
2866
+ "why": snooze.get("why")})
2867
+ resume = {"at": now, "since": since, "quiet": quiet, "carried": carried, "snoozes": snoozes, "announced": None}
2868
+ if not dry_run:
2869
+ st["resume"] = resume
2870
+ save_state(root, st)
2871
+ return resume
2872
+
2873
+
2874
+ def resume_ended(root, cfg, resume, now=None):
2875
+ """The keys of what the silence carried whose own known end has passed: none of them stands (CATCHUP-POLISH).
2876
+
2877
+ An episode raised with a known end - credits until the reset their reading named, the
2878
+ architect's quota until it comes back - says nothing of what holds after that end. An
2879
+ episode raised before episodes kept their end carries none, so the credits alarm's end is
2880
+ also read from the implementer's last reading before this cycle: a reading whose own reset
2881
+ has passed says nothing of the plan after it, as a spent one is no credits finding
2882
+ (OCT1-FIXES). Rehearsing the catch-up planned for 2026-10-01, with the jobs back six hours
2883
+ after the plan reset and the usage unreadable, the resume notice named the credits as
2884
+ standing, from the episode the silence carried and from the snooze that ended in it. What
2885
+ this cycle raises was measured now, and it stands whatever this says.
2886
+ """
2887
+ now = time.time() if now is None else now
2888
+ ended = {episode["key"] for episode in resume.get("carried") or []
2889
+ if 0 < float(episode.get("quiet_until") or 0) <= now}
2890
+ try:
2891
+ readings = [row for row in A.credit_samples(root, A.implementer_adapter_id(cfg))
2892
+ if float(row.get("at") or 0) < int(resume.get("at") or now)]
2893
+ except Exception:
2894
+ readings = []
2895
+ reset = credit_reset(readings[-1].get("reset_at"), now) if readings else None
2896
+ if reset is not None and reset <= now:
2897
+ ended.add("credits-exhaust")
2898
+ return ended
2899
+
2900
+
2901
+ def resume_items(root, cfg, st, resume, folded, now=None, told=()):
2902
+ """What stands at a resume, one entry per condition: {"key", "audience", "lines", "red"}.
2903
+
2904
+ From the cycle that ran - every notice it would have rung, with the audience its own
2905
+ check declared - and from the records the silence left. A record read here keeps the
2906
+ audience its check gives it elsewhere: an alarm episode, a snooze, a decision, deferred
2907
+ work, a hold and an implementer with nothing to do are a person's; an unseen decision
2908
+ request is first the architect's (#30). An entry without a key is something the cycle
2909
+ did; it is told, never named. A condition a person has snoozed stays off the notice as
2910
+ it stays off every channel (#108). What the silence carried is not named once its own known
2911
+ end has passed, and an unseen request another alarm to a person tells is named under that
2912
+ alarm alone (CATCHUP-POLISH).
2913
+ """
2914
+ now = time.time() if now is None else now
2915
+ project = A.project_key(root)
2916
+ items = {}
2917
+ # What ended in the silence is not named, nor a request an alarm to a person tells (CATCHUP-POLISH).
2918
+ unnamed = resume_ended(root, cfg, resume, now) | {f"unseen:{name}" for name in told}
2919
+
2920
+ def add(key, audience, line, red=False, what=None):
2921
+ if key and A.alarm_snoozed(project, key, now):
2922
+ return
2923
+ item = items.setdefault(key, {"key": key, "audience": audience, "lines": [], "red": False})
2924
+ item["audience"] = "human" if "human" in (item["audience"], audience) else audience
2925
+ item["red"] = item["red"] or red
2926
+ if what is not None:
2927
+ item["what"] = what # what the notice tells of it, for the ladder (NOTICE-NOISE)
2928
+ line = " ".join(str(line).split())[:240]
2929
+ if line and line not in item["lines"]:
2930
+ item["lines"].append(line)
2931
+
2932
+ for note in folded:
2933
+ subject = str(note.get("title") or "").split(": ", 1)[-1]
2934
+ add(note.get("key") or "", note.get("audience") or "human",
2935
+ f"{subject}: {note.get('msg')}" if subject else note.get("msg"), red=note.get("ring") == "red",
2936
+ what=note.get("what"))
2937
+ for episode in (episode for episode in resume.get("carried") or [] if episode["key"] not in unnamed):
2938
+ add(episode["key"], "human", f"stood when the silence began: {episode.get('ring') or episode.get('level')} "
2939
+ f"since {_when(episode.get('first'))}, last raised {_when(episode.get('last'))}"
2940
+ + (f", mailed {_when(episode['red_sent'])}" if episode.get("red_sent") else ""))
2941
+ for snooze in (snooze for snooze in resume.get("snoozes") or [] if snooze["key"] not in unnamed):
2942
+ add(snooze["key"], "human", f"its snooze ended {_when(snooze.get('until'))} "
2943
+ f"({snooze.get('by')}: {snooze.get('why')})")
2944
+ # Each record is read on its own: one that cannot be read must not cost the notice the rest.
2945
+ try:
2946
+ for message in A.unseen_messages(root, cfg):
2947
+ if message["class"] == "needs-decision" and f"unseen:{message['id']}" not in unnamed:
2948
+ add(f"unseen:{message['id']}", "architect",
2949
+ f"a decision request nobody has been shown, written {_when(message['at'])}")
2950
+ except Exception:
2951
+ pass
2952
+ try:
2953
+ for decision in A.decisions(root, "open"):
2954
+ add(f"decision-open:{decision.get('id')}", "human", f"open since {_when(decision.get('asked_at'))}: "
2955
+ f"{str(decision.get('question') or '')[:120]}")
2956
+ except Exception:
2957
+ pass
2958
+ try:
2959
+ for row in A.deferred_open(root):
2960
+ add(f"deferred:{row.get('kind')}", "human", f"deferred since {_when(row.get('at'))}: "
2961
+ f"{row.get('reason') or '?'}; polyctl catchup replays it")
2962
+ except Exception:
2963
+ pass
2964
+ held = A.hold_state(root)
2965
+ if held:
2966
+ add("hold-standing", "human", f"held by {held.get('by')} since {_when(held.get('at') or now)}: "
2967
+ f"{held.get('reason') or ''}")
2968
+ idle = st.get("idle_answer")
2969
+ if isinstance(idle, dict):
2970
+ add("idle-answer", "human", f"no nudges since {_when(idle.get('since') or now)}: the implementer "
2971
+ "answered its last one with nothing")
2972
+ return sorted(items.values(), key=lambda item: (not item["key"], item["audience"] != "human"))
2973
+
2974
+
2975
+ def announce_resume(root, owed, dry_run=False, now=None):
2976
+ """Send the one notice a resume owes, naming once each condition that stands (RESUME-QUIET).
2977
+
2978
+ It goes to the widest audience among what it names: a person when anything it names is
2979
+ a person's, since a person can act on the architect's business while the architect
2980
+ cannot lift a hold, end a snooze or buy credits; a person too when all of it is the
2981
+ architect's but no architect is going to read it (#69); otherwise the architect,
2982
+ through its mailbox. It uses the orange channels and never e-mail, so it is
2983
+ never louder than what it names (#40). It is the ring of each condition it names for
2984
+ that condition's own window, and a red it names counts as told, so the next cycle does
2985
+ not ring or mail again what the notice has just said. The resume counts as announced
2986
+ before anything here can fail and whatever a channel answers: a notice left owed would
2987
+ keep gathering alarms, cycle after cycle, into a notice that never goes.
2988
+ """
2989
+ now = time.time() if now is None else now
2990
+ resume = owed["resume"]
2991
+ cfg = A.load_config(root)
2992
+ st = load_state(root)
2993
+ if not dry_run:
2994
+ st["resume"] = dict(resume, announced=int(now))
2995
+ save_state(root, st)
2996
+ items = resume_items(root, cfg, st, resume, owed["folded"], now, told=owed.get("told") or ())
2997
+ named = [item["key"] for item in items if item["key"]]
2998
+ audience = "human" if any(item["audience"] == "human" for item in items) else "architect"
2999
+ if audience == "architect" and named:
3000
+ if not architect_hold_reason(root, cfg, A.metered_adapter(cfg), st, now=now)["holdable"]:
3001
+ audience = "human"
3002
+ project = A.project_key(root)
3003
+ silence = float(resume["at"]) - float(resume["since"])
3004
+ title = f"{project}: watchdog resumed after {_span(silence)}"
3005
+ lines = [f"{len(named)} stand: {', '.join(named)}" if named else "nothing stands",
3006
+ f"nothing ran from {_when(resume['since'])} to {_when(resume['at'])}; each clock restarts now"]
3007
+ lines += [f"- {item['key'] or 'this cycle'}: {'; '.join(item['lines'])}" for item in items]
3008
+ lines.append("polyctl alarms, polyctl notices and polyctl status show more")
3009
+ msg = "\n".join(lines)
3010
+ _FACTS["resume"] = {"silence_s": int(silence), "audience": audience, "named": named}
3011
+ if dry_run:
3012
+ _note_above_verdict(f"DRY RUN: would send one resume notice to the {audience} naming {len(named)} "
3013
+ f"condition(s){': ' + ', '.join(named) if named else ''}")
3014
+ return None
3015
+ st["resume"].update(audience=audience, named=named)
3016
+ save_state(root, st)
3017
+ if not named:
3018
+ A.record_notice(root, title, msg, sent=False, key="resume")
3019
+ elif audience == "human":
3020
+ desktop_notify(title, lines[0].replace('"', "'")[:200], cfg)
3021
+ phone_text = f"*{_markdown_plain(title)}*\n{_markdown_plain(msg)}"
3022
+ try:
3023
+ from . import telegram
3024
+ telegram.send(phone_text, root)
3025
+ except Exception:
3026
+ pass
3027
+ try:
3028
+ from . import ntfy
3029
+ ntfy.send(phone_text, root)
3030
+ except Exception:
3031
+ pass
3032
+ A.record_notice(root, title, msg, sent=True, key="resume", named=named)
3033
+ for item in items:
3034
+ if item["key"]:
3035
+ A.alarm_rang(project, item["key"], now, what=item.get("what")) # it rang once, for what it said
3036
+ if item["key"] and item["red"]:
3037
+ A.alarm_named(project, item["key"], now)
3038
+ else:
3039
+ A.record_notice(root, title, msg, sent=False, key="resume", named=named)
3040
+ A.write_report(root, cfg, "resumed", lines,
3041
+ key=time.strftime("%Y%m%d-%H%M", time.localtime(float(resume["at"]))))
3042
+ _note_above_verdict(f"resumed after {_span(silence)}: one notice to the {audience} names "
3043
+ f"{len(named)} condition(s)")
3044
+ return audience
3045
+
3046
+
3047
+ def ring_unseen_requests(root, cfg, st, restart, told=()):
3048
+ """Ring each decision request nobody has been shown by its age, unless an alarm to a person tells it (#30).
3049
+
3050
+ The architect is told first, then a person's desktop and phone, then e-mail. One written
3051
+ before a resume waits from the resume: nobody could have acted in the time nothing ran
3052
+ (RESUME-QUIET).
3053
+
3054
+ A request an alarm to a person tells this cycle - a "needs you" no architect will act
3055
+ on, `reports-no-wake` or `present-pending` - is one condition, on that alarm's ladder.
3056
+ Rehearsing the catch-up planned for 2026-10-01, a request nobody had been shown and no
3057
+ architect could act on was mailed eight times in a day, four under its anomaly's key and
3058
+ four under its own. Once no such alarm tells it, as when an architect can act on it again,
3059
+ its own ladder counts from the last cycle one did, as it counts from a resume: a request a
3060
+ person has been told of for hours does not ring red the moment an architect may read it.
3061
+ A resume notice names such a request under that alarm alone (CATCHUP-POLISH).
3062
+ """
3063
+ project = A.project_key(root)
3064
+ now = time.time()
3065
+ was = st.get("unseen_told") if isinstance(st.get("unseen_told"), dict) else {}
3066
+ kept = {}
3067
+ resuming = _RESUME.get()
3068
+ if resuming is not None:
3069
+ resuming["told"] = sorted(told)
3070
+ for message in A.unseen_messages(root, cfg):
3071
+ if message["class"] != "needs-decision":
3072
+ continue
3073
+ if message["id"] in told:
3074
+ kept[message["id"]] = int(now)
3075
+ continue
3076
+ if message["id"] in was:
3077
+ kept[message["id"]] = was[message["id"]]
3078
+ since = max(restart, float(kept.get(message["id"]) or 0))
3079
+ carried = message["at"] < since
3080
+ minutes = (max(0.0, now - since) if carried else message["age"]) / 60
3081
+ level = ("red" if minutes >= S.get(cfg, "mail.unseen_red_minutes")
3082
+ else "orange" if minutes >= S.get(cfg, "mail.unseen_orange_minutes")
3083
+ else "yellow" if minutes >= S.get(cfg, "mail.unseen_yellow_minutes") else None)
3084
+ if level:
3085
+ reason = "the watchdog resumed" if since == restart else "an alarm to a person last named it"
3086
+ text = f"{message['id']} has waited {int(message['age'] / 60)}m and nobody has been shown it" \
3087
+ + (f" ({int(minutes)}m since {reason})" if carried else "")
3088
+ # It rings as it crosses a threshold, and its red is the red threshold: rung every hour between
3089
+ # them, one unseen request reached the desktop and the phone 15 times in a day, and an hour of
3090
+ # orange mailed it two hours before its red (NOISE-REPEATS).
3091
+ notify(f"{project}: unread decision request", text, root, key=f"unseen:{message['id']}", window=3600,
3092
+ audience="architect" if level == "yellow" else "human",
3093
+ level=None if level == "yellow" else level, what=level,
3094
+ red_after=S.get(cfg, "mail.unseen_red_minutes") * 60)
3095
+ if kept != was:
3096
+ st["unseen_told"] = kept
3097
+ save_state(root, st)
3098
+
3099
+
3100
+ def _cycle_impl(args, root):
3101
+ # The silence this cycle ends is read before its own heartbeat overwrites it.
3102
+ silence = A.heartbeat_age(root)
3103
+ # Proof the watchdog ran, before anything that can end the cycle early: a
3104
+ # broken session binding was reported to people as a dead watchdog (audit).
3105
+ if not args.dry_run:
3106
+ A.heartbeat(root)
3107
+ cfg = A.load_config(root)
3108
+ # After a long silence this cycle is a resume: what stands is named once, in one notice
3109
+ # sent as the cycle ends, and what the silence carried ages from now (RESUME-QUIET).
3110
+ resume = resume_after(root, cfg, silence, dry_run=args.dry_run)
3111
+ restart = float((resume or {}).get("at") or 0)
3112
+ if resume and not resume.get("announced") \
3113
+ and time.time() - restart <= S.get(cfg, "watchdog.resume_gap_hours") * 3600:
3114
+ _RESUME.set({"resume": resume, "folded": []})
3115
+ if not args.dry_run:
3116
+ # Nothing watched these episodes: they close unannounced and the notice names them.
3117
+ A.expire_alarms(A.project_key(root), quiet_for=resume.get("quiet") or A._alarm_reset_after())
3118
+ print(f"resume: no watchdog cycle for {_span(restart - float(resume['since']))} before this one; what "
3119
+ "stands is named in one notice when this cycle ends, and its clocks restart")
3120
+ # A person's blocker stands even with no implementer or transcript (TRIAGE-2-MEDIUM).
3121
+ project = A.project_key(root)
3122
+ # What the implementer declares needs a person reaches a person, never held for an agent (#92).
3123
+ for item in A.human_waits(root):
3124
+ needs = item["notes"].get("needs") or item["title"]
3125
+ notify(f"{project}: needs you", f"{item['id']} waits on a person: {needs}", root,
3126
+ key=f"waiting-human:{item['id']}", window=6 * 3600, audience="human", what=needs)
3127
+ # Before anything that can end the cycle: a commit made around the hook was not measured in a cycle that
3128
+ # ended for want of an implementer block or a transcript, and an agent can take either away (the review of
3129
+ # ATTACK-SUITE, LEDGER-HISTORY).
3130
+ if not args.dry_run:
3131
+ landed = report_ungranted_commits(root, A.project_key(root))
3132
+ _FACTS["ungranted_commits"] = landed[:120] if landed else None
3133
+ impl = cfg.get("implementer") or {}
3134
+ if not impl:
3135
+ if not A.hold_state(root) and slice_budget(root, cfg, dry_run=args.dry_run):
3136
+ return 0
3137
+ print("no implementer session; nothing to watch")
3138
+ return 0
3139
+ adapter = A.load_adapter(impl.get("adapter", ""), root)
3140
+ # The quota gates read the quota by the command the package's adapter declares: reading it runs that command,
3141
+ # and the project's layer is a file the implementer writes (TRIAGE-HIGHS).
3142
+ metered = A.metered_adapter(cfg)
3143
+ msgs, _ = A.session_paths(cfg)
3144
+ if not msgs or not os.path.exists(msgs):
3145
+ if not A.hold_state(root) and slice_budget(root, cfg, dry_run=args.dry_run):
3146
+ return 0
3147
+ # An `auto` polyctl could not resolve says why, where it read as a transcript gone missing (SESSION-IDENTITY).
3148
+ state = A.session_state(cfg, "implementer") or {}
3149
+ print(f"the implementer's session is {state['how']}: {state['why']}; nothing to watch"
3150
+ if not state.get("session") and state.get("why") else "no transcript; nothing to watch")
3151
+ # A slice parked on its session's limit waits for that session: with its transcript gone - a harness
3152
+ # deletes old ones - nothing can resume it, and the cycle ended here before the park's alarm, so it
3153
+ # waited on unseen (JOURNAL-6). The alarm stands and says why.
3154
+ st = load_state(root)
3155
+ park = st.get("quota_park") if isinstance(st.get("quota_park"), dict) else None
3156
+ if park and park.get("items") and not args.dry_run:
3157
+ touch_quota_park(root, st, cfg)
3158
+ notify(f"{A.project_key(root)}: a parked slice cannot resume",
3159
+ f"{', '.join(park['items'])} waits for the implementer's session, and its transcript is gone, so "
3160
+ "nothing resumes it when the limit resets - pin a session the store still holds, or move the "
3161
+ "slice on by hand", root, key="park-transcript-gone", window=6 * 3600, audience="human")
3162
+ return 0
3163
+
3164
+ # The implementer's silence ends at its last write, a subagent's included: a session whose transcript is
3165
+ # quiet while its subagent works is not idle, and its lingering runtime is not hung.
3166
+ age = time.time() - A.last_write(msgs, A.transcript_shape(adapter))
3167
+ size = os.path.getsize(msgs)
3168
+ st = load_state(root)
3169
+ if not args.dry_run:
3170
+ _adopt_started_steps(root) # before anything counts writers or decides a wake
3171
+ A.reconcile_mail_ledger(root, cfg) # deleted mail becomes a consumed row
3172
+ A.record_progress(root, cfg) # history of what moved, for the spin check
3173
+ _close_deferred_done(root, st)
3174
+ project = A.project_key(root)
3175
+ try:
3176
+ bd = A.board(root)
3177
+ _FACTS.update(idle_s=int(age), transcript_bytes=size,
3178
+ inbox=len(A.implementer_inbox(root, cfg)),
3179
+ standing_request=bool(A.waiting_on_architect(root, cfg)),
3180
+ queued=len(bd["queued"]), running=len(bd["running"]), blocked=len(bd["blocked"]),
3181
+ hold=bool(A.hold_state(root)),
3182
+ arch_quota_until=st.get("arch_quota_until") or None,
3183
+ arch_present=A.architect_present(root, cfg.get("architect") or {}),
3184
+ last_wake_error=(st.get("wake_error") or {}).get("kind"))
3185
+ except Exception as exc: # facts must never stop a cycle
3186
+ _FACTS["facts_error"] = str(exc)[:120]
3187
+ # Alarm hygiene, every cycle: keep durable conditions raised before closing
3188
+ # episodes that went quiet, then say so once. A quota window lives in state,
3189
+ # so unlike a transient notice it must keep aging toward red while it stands.
3190
+ # In dry-run, notify previews that same touch without persisting it.
3191
+ # A present, working architect is not at quota, whatever the cache says (#40).
3192
+ architect = cfg.get("architect") or {}
3193
+ try:
3194
+ present = st.get("arch_quota_until", 0) > time.time() and architect.get("argv") \
3195
+ and A.architect_present(root, architect)
3196
+ except TableUnreadable as exc:
3197
+ # Presence that was not measured clears nothing; the guards below stand down on the same table
3198
+ # once they ask it, after the alarms and the ping this cycle owes (WINDOWS-TABLE-CLOSED).
3199
+ present = False
3200
+ print(f"{exc}; the architect's cached quota block stands")
3201
+ if present:
3202
+ st.pop("arch_quota_until", None)
3203
+ save_state(root, st)
3204
+ print("the architect is present and working; its cached quota block is cleared")
3205
+ touch_architect_quota(root, st, cfg)
3206
+ touch_quota_park(root, st, cfg)
3207
+ if not args.dry_run:
3208
+ for e in A.expire_alarms(project):
3209
+ _announce_resolved(root, e)
3210
+ # The dead man's switch is a real external ping, not an explain probe.
3211
+ _FACTS["ping"] = A.ping(root)
3212
+ else:
3213
+ _FACTS["ping"] = "dry-run"
3214
+ if not args.dry_run:
3215
+ _sample_credits(root, st, A.implementer_adapter_id(cfg), project)
3216
+ _schedule_hunt(root, cfg, st)
3217
+ # Only meaningful when no implementer turn is running: a sub-agent's writes
3218
+ # do not appear as the parent's tool calls and would read as a stranger's.
3219
+ # Whose they are cannot be told where the process table cannot be read, and
3220
+ # the writer guard below stands down on it (WINDOWS-TABLE-CLOSED).
3221
+ try:
3222
+ fe = [] if A.agent_pids(root, adapter) else A.foreign_edits(root, cfg)
3223
+ except TableUnreadable as exc:
3224
+ fe = []
3225
+ _FACTS["process_table"] = str(exc)
3226
+ _FACTS["foreign_edits"] = fe
3227
+ _FACTS["decisions_ringing"] = escalate_open_decisions(root, project, dry_run=args.dry_run, since=restart)
3228
+ parked = A.reviewer_state(root)
3229
+ if parked.get("pending_review") and not parked.get("until") and not args.dry_run:
3230
+ notify(f"{project}: a review is parked",
3231
+ f"the reviewer was unavailable: {A.redact(parked.get('reason') or '?')[:120]} — a person "
3232
+ "restores it, carries the request with polyctl collect-review, or waives", root,
3233
+ key="review-parked", window=24 * 3600, audience="human")
3234
+ for sib, age_s in A.stale_siblings(root).items():
3235
+ # Right after a resume every sibling looks as silent as this watchdog was, until its
3236
+ # own next cycle: a sibling's silence counts from the resume too (RESUME-QUIET).
3237
+ if time.time() - restart <= 15 * 60:
3238
+ continue
3239
+ notify(f"{sib}: watchdog silent", f"no heartbeat for {age_s // 60}m — its watchdog is not "
3240
+ f"running; {A.scheduler_hint()} / polyctl watchdog status", root, key=f"watchdog-dead:{sib}",
3241
+ window=3600, audience="human")
3242
+ hs = A.hold_state(root)
3243
+ # A hold carried over a silence stands from the resume (RESUME-QUIET).
3244
+ if hs and time.time() - max(int(hs.get("at") or time.time()), restart) > 4 * 3600:
3245
+ notify(f"{project}: hold standing {int((time.time() - hs['at']) / 3600)}h",
3246
+ f"set by {hs.get('by', '?')}: {hs.get('reason', '')} — polyctl hold release when done",
3247
+ root, key="hold-standing", window=6 * 3600, audience="human", level="red")
3248
+
3249
+ # Harness configuration an agent changed here reaches a person, held or not: a person who starts a session in
3250
+ # this tree runs it too (HARNESS-CONFIG).
3251
+ tell_harness_config(root, project)
3252
+ # An agent that is busy and producing nothing never trips the idle guard, so
3253
+ # check it before the guard chain rather than inside it. Notify only; a nudge
3254
+ # would add a turn to a loop that is already spending them.
3255
+ spin = A.spinning(root)
3256
+ if spin and time.time() - st.get("last_spin_notice", 0) > 1800:
3257
+ notify(f"{project}: agent spinning", f"{spin}m busy, nothing committed or changed — needs re-specifying", root, audience="architect")
3258
+ st["last_spin_notice"] = time.time()
3259
+ save_state(root, st)
3260
+ print(f"spinning: {spin}m active with no artifact change")
3261
+
3262
+ # -1 — a human has taken the tree. Nothing else in this chain may override it.
3263
+ held = A.hold_state(root)
3264
+ if held:
3265
+ # Nothing below runs, so no other alarm tells a request nobody has been shown (#30).
3266
+ ring_unseen_requests(root, cfg, st, restart)
3267
+ print(f"held by {held.get('by')} for {held['minutes']}m: {held.get('reason','')}")
3268
+ return 0
3269
+
3270
+ # -1b — a running slice past its budget, or whose budget reading is missing, parks here and
3271
+ # nothing after this runs for it (SLICE-BUDGET).
3272
+ if slice_budget(root, cfg, dry_run=args.dry_run):
3273
+ return 0
3274
+
3275
+ # Report anything needing judgement before the guard chain stands down on it.
3276
+ # Standing down silently is how a condition persists for hours: the watchdog
3277
+ # was right to not act and wrong to be the only one who knew.
3278
+ told = set()
3279
+ escalate(root, cfg, adapter, age, args, st, told=told)
3280
+ # A request nobody has been shown climbs its own ladder only where no alarm to a person
3281
+ # already tells it (#30, CATCHUP-POLISH).
3282
+ ring_unseen_requests(root, cfg, st, restart, told)
3283
+
3284
+ # 0 — is ANY agent already working this tree? Not just the child we started.
3285
+ #
3286
+ # Tracking only our own last child was wrong, and expensively so: every nudge
3287
+ # spawns a detached process, nothing reaps them, and this project accumulated
3288
+ # fifteen live agent processes in one repository with four still burning CPU.
3289
+ # Each was invisible to a guard that remembered a single pid. Ask the OS which
3290
+ # processes have this repo as their cwd, and treat any of them as a writer.
3291
+ # What an ended turn left behind is cleared before anything is counted. A
3292
+ # wrapper that was reaped by pid — or exited on its own — leaves its runtime
3293
+ # and engine children alive with this repo as their cwd; the implementer
3294
+ # counts them as writers and refuses to write, its empty turns trip the
3295
+ # reaper again, and the reaper makes one more. They are identified by shape
3296
+ # (no terminal, dead group leader), never by age, so nothing a person is in
3297
+ # can match.
3298
+ dead = A.orphans(root, adapter)
3299
+ if dead:
3300
+ print(f"{len(dead)} orphaned agent process(es) left by an ended turn; clearing {dead}")
3301
+ if not args.dry_run:
3302
+ A.record_notice(root, "orphans cleared", f"{len(dead)} leftover process(es): {dead}", False, key="orphans")
3303
+ left = A.sweep_orphans(dead)
3304
+ if left:
3305
+ # Said, not assumed cleared: a remnant the sweep could not stop is swept again next cycle
3306
+ # (WINDOWS-STOP-PROVEN).
3307
+ print(f"{len(left)} leftover process(es) still run after the sweep: {left}")
3308
+ running = [p for p in A.agent_pids(root, adapter) if p not in set(dead)]
3309
+ # Windows has no process groups to find an ended turn's leftovers by: none looked for is not none, and what
3310
+ # `running` holds may be such leftovers (WINDOWS-SAY-WHY).
3311
+ _FACTS.update(writers=len(A.process_trees(running)) if running else 0,
3312
+ orphans=None if os.name == "nt" else len(dead))
3313
+ # An agent whose working directory Windows would not let polyctl read cannot be placed (#71,
3314
+ # WINDOWS-CWD-2). One that cannot be placed may be this tree's writer; starting another
3315
+ # beside it is the two-writer incident, so stand down and say why.
3316
+ unplaced = A.unplaced_agent_pids(root, adapter)
3317
+ if unplaced:
3318
+ print(f"{len(unplaced)} agent process(es) cannot be placed in a tree - their working directory "
3319
+ f"could not be read ({unplaced}); not starting another turn")
3320
+ return 0
3321
+ if running:
3322
+ # A process being alive is not a turn being in flight. An agent can finish
3323
+ # its turn and never exit, and the first version of this guard treated that
3324
+ # hung process as a writer — so one zombie blocked every future nudge for
3325
+ # seven hours while the slice it had already finished sat uncommitted.
3326
+ # Fixing an accumulation bug had quietly introduced a deadlock.
3327
+ #
3328
+ # Tell them apart by the transcript: a live turn writes to it. Silence far
3329
+ # past the idle threshold, with a process still up, is hung, not busy.
3330
+ turns = A.process_trees(running)
3331
+ # The transcript closed the turn and the process is still here past the
3332
+ # idle window: a runtime that forgot to exit, not a turn. Reap now.
3333
+ ended = A.turn_ended(cfg) and age >= args.idle_minutes * 60
3334
+ if ended:
3335
+ print(f"turn ended in the transcript {int(age / 60)}m ago but {len(running)} process(es) linger; reaping")
3336
+ if age < args.idle_minutes * 60 * 3 and not ended:
3337
+ print(f"{len(turns)} turn(s) already in this tree "
3338
+ f"(roots {turns}, {len(running)} processes); not starting another")
3339
+ if len(turns) > 2:
3340
+ notify(f"{project}: turns piling up",
3341
+ f"{len(turns)} concurrent turns — `polyctl hold` to clear", root)
3342
+ return 0
3343
+ # Reaping is an action, so it keeps the conservative bound even though
3344
+ # reporting no longer does.
3345
+ print(f"{len(running)} process(es) alive but silent {int(age / 60)}m; reaping")
3346
+ notify(f"{project}: reaping hung turn",
3347
+ f"{len(running)} process(es) silent {int(age / 60)}m — cleaning up", root, audience="architect")
3348
+ if args.dry_run:
3349
+ print("DRY RUN: would reap", running)
3350
+ return 0
3351
+ # Reap only what we could have started. A person's interactive session
3352
+ # in this tree is silent between their keystrokes, not hung.
3353
+ # By process group: the flag is on the wrapper, the children carry none.
3354
+ for pid in A.agent_pids(root, adapter, headless_only=True):
3355
+ A.kill_turn(pid, signal.SIGTERM)
3356
+ for _ in range(20):
3357
+ if not A.agent_pids(root, adapter, headless_only=True):
3358
+ break
3359
+ time.sleep(0.5)
3360
+ for pid in A.agent_pids(root, adapter, headless_only=True):
3361
+ A.kill_turn(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
3362
+ # Only headless turns were reaped. A person's interactive session in this
3363
+ # tree is still here and still a writer; nudging next to it starts a second.
3364
+ # Counted as what the reaper could not have started, from the same scan.
3365
+ headless = set(A.agent_pids(root, adapter, headless_only=True))
3366
+ if headless:
3367
+ # A turn the reaper could not stop - taskkill refused, a child its parent no longer names - is
3368
+ # still this tree's writer: the reaper was taken at its word and a nudge started beside it
3369
+ # (WINDOWS-STOP-PROVEN).
3370
+ print(f"{len(headless)} unattended turn process(es) still run after reaping ({sorted(headless)}); "
3371
+ "not starting another turn")
3372
+ notify(f"{project}: a hung turn could not be stopped",
3373
+ f"{len(headless)} process(es) of an unattended turn still run after the reaper stopped them: "
3374
+ f"{sorted(headless)} - stop them by hand, or `polyctl hold`; no turn starts beside them", root,
3375
+ key="reap-survived", window=6 * 3600, audience="human", what=f"still running: {sorted(headless)}")
3376
+ return 0
3377
+ people = [pid for pid in A.agent_pids(root, adapter) if pid not in headless]
3378
+ if people:
3379
+ print(f"{len(people)} interactive agent process(es) remain in this tree after reaping; "
3380
+ "not starting another turn")
3381
+ return 0
3382
+
3383
+ # 1 — still working
3384
+ #
3385
+ # Progress is not the same as a commit. A slice whose independent review found
3386
+ # real defects is *correct* to withhold the commit while it fixes them, and to
3387
+ # a HEAD-only check that looks exactly like an agent that has stopped: three
3388
+ # nudges, no movement, stand down. It happened — two hours of a live slice sat
3389
+ # idle because the counter could not tell a careful implementer from a dead
3390
+ # one. Fingerprint everything that moves when work is happening.
3391
+ fp = A.work_fingerprint(root)
3392
+ if st.get("attempts") and fp != st.get("last_fingerprint"):
3393
+ st.update(attempts=0, last_fingerprint=fp)
3394
+ save_state(root, st)
3395
+ # An implementer that answered a nudge with nothing is not asked again until
3396
+ # something it is given, or something it produces, moves (#96).
3397
+ idle = st.get("idle_answer")
3398
+ if idle and (fp != idle.get("work") or A.nudge_inputs(root, cfg) != idle.get("inputs")):
3399
+ for key in ("idle_answer", "nudge_size", "nudge_fingerprint", "nudge_inputs"):
3400
+ st.pop(key, None)
3401
+ st["attempts"] = 0
3402
+ save_state(root, st)
3403
+ print("the board, the backlog, a decision, mail or the work moved since the "
3404
+ "implementer had nothing to do; nudges resume")
3405
+ if age < args.idle_minutes * 60:
3406
+ if st.get("attempts") and size != st.get("last_size"):
3407
+ # The transcript moving is not the work moving: a nudged turn that finds
3408
+ # nothing to do writes its answer too. Only the fingerprint above resets
3409
+ # the backoff.
3410
+ st.update(last_size=size)
3411
+ save_state(root, st)
3412
+ print(f"working ({int(age)}s since last write)")
3413
+ return 0
3414
+ # Presence is the agent's, not this tree's: writing in a secondary project is working (#22).
3415
+ elsewhere = A.working_elsewhere(cfg, args.idle_minutes * 60)
3416
+ if elsewhere:
3417
+ print(f"working in {elsewhere['name']} ({int(elsewhere['age'])}s since its last write there); "
3418
+ "not nudging here")
3419
+ return 0
3420
+ # 1b — stopped on its usage limit: the running slice is parked until the limit resets, or one retry
3421
+ # where no reset was named, and no nudge is spent on a window that is spent. When the park ends the
3422
+ # cycle goes on and resumes it (QUOTA-PARK).
3423
+ if quota_park(root, cfg, st, dry_run=args.dry_run, adapter=adapter):
3424
+ return 0
3425
+
3426
+ # 2 — nothing to continue. Before standing down, ask why: an empty board can
3427
+ # mean "all done" or "the queue ran dry". Only the second one is actionable,
3428
+ # and it needs the *architect*, not the implementer — refilling means pulling
3429
+ # from a tracker and deciding what may be worked, and an implementer that
3430
+ # chooses its own scope is the one thing this tool exists to prevent.
3431
+ # An implementer that asked for a decision and got no answer is not idle; it
3432
+ # is waiting, and a nudge cannot answer it. Eighty nudged turns once produced
3433
+ # eighty copies of the same request. The wake path above has already told
3434
+ # the architect; the only useful thing here is to say so and stand down.
3435
+ waiting = A.waiting_on_architect(root, cfg)
3436
+ if waiting:
3437
+ name, at = waiting
3438
+ print(f"implementer is waiting on the architect since "
3439
+ f"{time.strftime('%H:%M', time.localtime(at))} ({name}); not nudging")
3440
+ return 0
3441
+ # An open decision cannot be answered by a nudged turn (#20), and it does not
3442
+ # stop the queue while READY work stands: its slice parks and the next starts (#84).
3443
+ asked = [d for d in A.decisions(root, "open") if d.get("asked_at")]
3444
+ passing = queue_past_a_question(root)
3445
+ if asked and not passing:
3446
+ oldest = min(asked, key=lambda d: d["asked_at"])
3447
+ print(f"decision {oldest['id']} is open since "
3448
+ f"{time.strftime('%H:%M', time.localtime(oldest['asked_at']))}; not nudging")
3449
+ return 0
3450
+ if passing:
3451
+ print(f"{passing[0]} waits, and {passing[1]} is READY: an open question does not stop the queue")
3452
+ reasons = open_work(cfg, root)
3453
+ if not reasons and passing:
3454
+ reasons = [f"READY {passing[1]} while {passing[0]} waits"]
3455
+ # Nothing READY here and a secondary project has work: the nudge names it (#8).
3456
+ elsewhere_ready = None
3457
+ if not reasons and not A.ready(root):
3458
+ elsewhere_ready = A.secondary_ready(cfg)
3459
+ if elsewhere_ready:
3460
+ reasons = [f"READY {elsewhere_ready['item']} in {elsewhere_ready['name']}"]
3461
+ if not reasons:
3462
+ sc = A.sources(root)
3463
+ arch = cfg.get("architect") or {}
3464
+ depth = len(A.board(root)["queued"])
3465
+ # With a source bound, refill below its threshold; without one, an empty
3466
+ # queue is still a refill condition — the architect fills it from the
3467
+ # spec. This project had no source, so the empty queue was never a
3468
+ # refill and the only signal was the implementer's own blocked report.
3469
+ threshold = sc.get("refill_below", 3) if sc else 1
3470
+ from . import features as F
3471
+ if arch.get("argv") and depth < threshold and not F.enabled(cfg, "refill"):
3472
+ print(f"queue low ({depth}); refill feature is off — alarming instead")
3473
+ notify(f"{A.project_key(root)}: needs you", f"queue has {depth} item(s) and refill wakes are off — add slices to .polyctl/backlog.md",
3474
+ root, key="queue-empty-no-refill", window=3600, audience="human", what="refill-off")
3475
+ return 0
3476
+ if arch.get("argv") and depth < threshold and A.architect_present(root, arch):
3477
+ print("queue low, but the architect is already at the keyboard")
3478
+ return 0
3479
+ if arch.get("argv") and depth < threshold:
3480
+ if arch_alive(root, arch):
3481
+ print("queue low, but an architect wake is already running")
3482
+ return 0
3483
+ if args.dry_run:
3484
+ # A dry refill prepares the live command too, without a version probe (ARCH-PROBE-ORDER).
3485
+ _, _, unseen = architect_command(root, cfg, arch.get("prompt", language.text(cfg, "prompt.refill")))
3486
+ unseen = unseen or harness_config_hold(root)
3487
+ print(f"queue low ({depth}); {unseen}; not waking the architect to refill" if unseen
3488
+ else f"queue low ({depth}); would wake the architect to refill")
3489
+ return 0
3490
+ if child_alive(st):
3491
+ print("queue low, but a turn is still running")
3492
+ return 0
3493
+ if time.time() - st.get("last_refill", 0) < 1800:
3494
+ print(f"queue low ({depth}); refill wake sent "
3495
+ f"{int((time.time() - st.get('last_refill', 0)) / 60)}m ago; waiting")
3496
+ return 0
3497
+ # A report wake may have started earlier in this same cycle, now that the
3498
+ # watchdog's own reports wake the architect (#69). The helper scan is the
3499
+ # guard; the wake's own time is the one that cannot miss a young process.
3500
+ if time.time() - st.get("last_arch_wake", 0) < 900:
3501
+ print(f"queue low ({depth}); the architect was woken "
3502
+ f"{int((time.time() - st.get('last_arch_wake', 0)) / 60)}m ago; waiting")
3503
+ return 0
3504
+ if st.get("arch_quota_until", 0) > time.time():
3505
+ print("queue low, but the architect is at quota; waiting")
3506
+ return 0
3507
+ prompt = arch.get("prompt", language.text(cfg, "prompt.refill"))
3508
+ command, code, stopped = architect_command(root, cfg, prompt)
3509
+ if stopped:
3510
+ print(f"queue low ({depth}); {stopped}; not waking the architect to refill")
3511
+ if code == "refused":
3512
+ tell_turn_refused(root, "architect", stopped)
3513
+ return 0
3514
+ sess, plan, pinned = command["session"], command["plan"], command["pinned"]
3515
+ argv, search, values = command["argv"], command["search"], command["values"]
3516
+ resolved, ver = A.resolve_binary(argv[0], path=command["path"])
3517
+ if not resolved:
3518
+ print(f"architect command {argv[0]} not on PATH")
3519
+ return 0
3520
+ # As in escalate(): the shipped adapter's harness, and no grant beyond its own (TURN-ARGV).
3521
+ refused = A.turn_refusal("architect", A.block_adapter(arch), argv, root=root, resolved=resolved,
3522
+ search=search, values=values, prompt=prompt)
3523
+ if refused:
3524
+ print(f"queue low ({depth}); {refused}; not waking the architect to refill")
3525
+ tell_turn_refused(root, "architect", refused)
3526
+ return 0
3527
+ argv[0] = resolved
3528
+ headroom = A.rotate_if_exhausted(cfg, argv, "architect")
3529
+ if not headroom["ok"]:
3530
+ print(f"queue low, but {headroom['text']}")
3531
+ return 0
3532
+ key = A.project_key(root)
3533
+ log_path = os.path.join(STATE_DIR, A.project_file_name("refill-log", key))
3534
+ if A.hold_state(root):
3535
+ print("held since this cycle began; not waking the architect to refill")
3536
+ return 0
3537
+ unseen = harness_config_hold(root) or turn_hold(root, cfg, "architect")
3538
+ if unseen:
3539
+ print(f"queue low ({depth}); {unseen}; not waking the architect to refill")
3540
+ return 0
3541
+ # A refill spends the same window a report wake does, and its failures
3542
+ # were never read (audit): check the quota and the last refill's log.
3543
+ if not quota_ok(metered):
3544
+ print("queue low, but there is no quota headroom to wake the architect")
3545
+ return 0
3546
+ failed = wake_error(log_path)
3547
+ blocked = architect_quota_until(failed, argv)
3548
+ if blocked:
3549
+ print(f"the last refill hit the architect's limit; waiting until {reset_when(blocked)}")
3550
+ return 0
3551
+ if failed and failed.get("kind") != "quota" and time.time() - (failed.get("at") or 0) < 6 * 3600:
3552
+ notify(f"{project}: refill failed", f"{failed.get('kind')}: "
3553
+ f"{A.redact(failed.get('text', ''))[:110]}", root,
3554
+ key="refill-failed", window=6 * 3600, audience="human")
3555
+ # A transport error is retried on the refill spacing; a binary or a
3556
+ # session that failed fails the same way until someone changes it.
3557
+ if failed.get("kind") in ("binary", "session") and failed.get("binary") == f"{resolved} {ver}":
3558
+ print(f"the last refill with this binary failed ({failed.get('kind')}); not retrying")
3559
+ return 0
3560
+ if not record_pinned(root, pinned):
3561
+ return 0
3562
+ given, refused = A.prompt_input(plan, root, argv)
3563
+ if refused:
3564
+ print(f"queue low, but the architect's prompt cannot be handed over: {refused}")
3565
+ return 0
3566
+ # As a wake is: claimed before its process starts (JOURNAL-2).
3567
+ step = "refill:{}:{}".format(sess or "new", int(time.time() // 3600))
3568
+ if (sess and A.session_in_use(sess, arch)) or not _claim_step(root, step):
3569
+ print("queue low, but a refill was started and is not known to have ended; waiting")
3570
+ return 0
3571
+ os.makedirs(STATE_DIR, exist_ok=True)
3572
+ with open(log_path, "a", encoding=UTF8) as log:
3573
+ log.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} refill {resolved} {ver} ===\n")
3574
+ log.flush()
3575
+ try:
3576
+ proc = subprocess.Popen(given["argv"], cwd=root,
3577
+ env=dict(os.environ, PATH=search, POLYCTL_ROLE="architect"),
3578
+ stdin=subprocess.DEVNULL if given["stdin"] is None else given["stdin"],
3579
+ stdout=log, stderr=subprocess.STDOUT, # stdin: see escalate()
3580
+ start_new_session=True)
3581
+ finally:
3582
+ A.release_prompt(given)
3583
+ _record_start(root, step, proc.pid)
3584
+ st.update(arch_pid=proc.pid, last_refill=time.time())
3585
+ A.helper_register(root, proc.pid, "architect") # a judge, not a writer
3586
+ A.acquire_architect(root, proc.pid, "watchdog refill")
3587
+ save_state(root, st)
3588
+ A.emit_event(root, "wake", {"why": "refill", "queue": depth, "pid": proc.pid})
3589
+ print(f"queue low ({depth} < {threshold}); woke the architect")
3590
+ return 0
3591
+ print("idle, but no open work — leaving it alone")
3592
+ return 0
3593
+
3594
+ # 3 — over the round budget: a nudge would only buy another identical round.
3595
+ # Unless the architect has already intervened: a coordination message newer
3596
+ # than the newest review means the slice was re-specified, and the budget
3597
+ # applies to the old specification, not the new one.
3598
+ rn = A.rounds(root, cfg["reviews"])
3599
+ budget = S.get(cfg, "round_budget")
3600
+ intervened = False
3601
+ revs_all = A.reviews(root, cfg["reviews"], limit=1)
3602
+ if revs_all:
3603
+ # Only the architect re-specifies a slice; any other mail, the watchdog's own
3604
+ # included, is not an intervention (audit). Files can be acknowledged while
3605
+ # this reads them.
3606
+ try:
3607
+ rev_mt = os.path.getmtime(os.path.join(root, cfg["reviews"], revs_all[0][0]))
3608
+ except OSError:
3609
+ rev_mt = time.time()
3610
+ for m in A.mailbox(root, cfg["mailbox"]):
3611
+ if not A.from_architect(m, cfg):
3612
+ continue
3613
+ try:
3614
+ newer = os.path.getmtime(os.path.join(root, cfg["mailbox"], m)) > rev_mt
3615
+ except OSError:
3616
+ continue
3617
+ if newer:
3618
+ intervened = True
3619
+ break
3620
+ if rn > budget and not intervened:
3621
+ notify(f"{project}: over budget", f"round {rn}/{budget} — re-specify, split or change actor", root, audience="architect")
3622
+ print(f"over budget ({rn}/{budget}); notified instead of nudging")
3623
+ return 0
3624
+ if rn > budget and intervened:
3625
+ print(f"over budget ({rn}/{budget}) but re-specified since the last review; proceeding")
3626
+
3627
+ # 4 — no quota. Delivery still works: reports are written above this gate and
3628
+ # the transport is HTTP, so a pending question reaches a phone even now. What
3629
+ # stops is deciding — so hand the state to whoever can.
3630
+ if not quota_ok(metered):
3631
+ if not args.dry_run:
3632
+ if not A.recently_deferred(root, "nudge"):
3633
+ A.deferred_append(root, "nudge", reason="implementer quota")
3634
+ if time.time() - st.get("last_handoff", 0) > 3600:
3635
+ try:
3636
+ import subprocess as _sp
3637
+ exe = A.program_path("polyctl", path=child_path())
3638
+ if exe:
3639
+ _sp.run([exe, "-C", root, "handoff", "--reason",
3640
+ language.text(cfg, "watchdog.handoff-provider")] + _handoff_quiet(),
3641
+ capture_output=True, timeout=120)
3642
+ st["last_handoff"] = time.time()
3643
+ save_state(root, st)
3644
+ except Exception:
3645
+ pass
3646
+ notify(f"{project}: out of quota",
3647
+ "provider window exhausted; handoff note sent", root)
3648
+ print("provider out of headroom; not nudging")
3649
+ return 0
3650
+
3651
+ # 4b — the provider stopped, not the agent. Nudging an outage buys nothing
3652
+ # and, while the stalled turn is still retrying, costs a second writer.
3653
+ degraded = provider_degraded(root)
3654
+ if degraded:
3655
+ notify(f"{project}: provider degraded", f"{degraded} — waiting, not nudging", root)
3656
+ print(f"provider degraded ({degraded}); waiting rather than nudging")
3657
+ return 0
3658
+
3659
+ # 4c — the implementer answered the last nudge and changed nothing while nothing
3660
+ # it is given moved: it has no work it can see, whatever the board says. Asking
3661
+ # again every idle window cost 29 turns in one morning (#96). This reads what
3662
+ # the turn did, not the wording of its report.
3663
+ if (st.get("nudge_size") is not None and size > st["nudge_size"] and A.turn_ended(cfg)
3664
+ and fp == st.get("nudge_fingerprint")
3665
+ and A.nudge_inputs(root, cfg) == st.get("nudge_inputs")
3666
+ and (st.get("last_error") or {}).get("at", 0) < st.get("last_nudge", 0)):
3667
+ idle = st.get("idle_answer")
3668
+ if not idle:
3669
+ idle = {"since": st.get("last_nudge") or time.time(), "work": fp,
3670
+ "inputs": st.get("nudge_inputs")}
3671
+ st["idle_answer"] = idle
3672
+ save_state(root, st)
3673
+ notify(f"{project}: implementer has nothing to do",
3674
+ "it answered the last nudge without changing anything; no more nudges until "
3675
+ "the board, the backlog, a decision or its mail changes", root,
3676
+ key="idle-answer", window=12 * 3600, audience="human")
3677
+ since = time.strftime("%H:%M", time.localtime(idle["since"]))
3678
+ print(f"idle since {since}: the implementer answered the last nudge without changing "
3679
+ f"anything; waiting for the board, the backlog, a decision or mail; not nudging")
3680
+ return 0
3681
+
3682
+ # 5 — the previous nudge changed nothing
3683
+ max_attempts = S.get(cfg, "watchdog.max_attempts")
3684
+ if st.get("attempts", 0) >= max_attempts:
3685
+ notify(f"{project}: agent stuck", f"{max_attempts} nudges, no progress — needs a human", root)
3686
+ print("backoff exhausted; notified a human")
3687
+ return 0
3688
+ if st.get("last_nudge") and size == st.get("last_size"):
3689
+ wait = args.idle_minutes * 60 * (2 ** st["attempts"])
3690
+ if time.time() - st["last_nudge"] < wait:
3691
+ print(f"backing off ({st['attempts']} attempts, waiting {int(wait)}s)")
3692
+ return 0
3693
+
3694
+ # The project's own nudge, in its language, unless the cycle was handed another (LANGUAGE-PROMPTS).
3695
+ handed = getattr(args, "prompt", None)
3696
+ prompt = (language.text(cfg, "prompt.nudge") if handed is None else handed) \
3697
+ + (parked_note(cfg, *passing) if passing else "") \
3698
+ + (secondary_note(cfg, elsewhere_ready) if elsewhere_ready else "")
3699
+ if fe:
3700
+ # A person is in these files right now. Say so in the prompt; the
3701
+ # implementer keeps away from them for this turn.
3702
+ prompt += language.text(cfg, "prompt.nudge-editing", paths=", ".join(fe[:8]))
3703
+ # Past what one argument carries, the prompt goes on standard input where the implementer's
3704
+ # adapter declares it may; a detached turn takes no file polyctl would remove after it (PROMPT-CHANNEL).
3705
+ # A batch program on Windows takes it there whatever its size, and one whose adapter declares no
3706
+ # standard input is not started: prompt_input decides once argv[0] is resolved (WINDOWS-CMD-CHANNEL).
3707
+ # A grant the resume carries of its own admits what a command rewriter makes of each command it
3708
+ # names, as the one options.unattended appends does (GRANTS-RTK). The nudge is composed from the
3709
+ # adapter the package ships, never from the user's or the project's layer, which an agent can
3710
+ # write (TURN-ARGV).
3711
+ shipped = A.turn_adapter(impl.get("adapter"))
3712
+ if shipped is None:
3713
+ from . import features as F
3714
+ refused = A.turn_refusal("implementer", impl.get("adapter"), [])
3715
+ print(f"idle {int(age)}s · {', '.join(reasons)} · {refused}; not nudging")
3716
+ if F.enabled(cfg, "nudge"):
3717
+ tell_turn_refused(root, "implementer", refused)
3718
+ return 0
3719
+ plan, refused = A.prompt_plan(A.admit_rewrites((shipped.get("resume") or {}).get("argv") or [])[0], prompt,
3720
+ impl.get("adapter"), detached=True)
3721
+ if refused:
3722
+ print(f"the nudge's prompt cannot be handed over: {refused}; not nudging")
3723
+ return 1
3724
+ # Only a session that is the implementer's is resumed: pinned, recorded, or found where no other
3725
+ # role's sessions are kept beside it. One read beside a role that is not settled is not (SESSION-IDENTITY).
3726
+ sess, withheld = A.session_to_resume(cfg, "implementer")
3727
+ if not sess:
3728
+ print(f"idle {int(age)}s · {', '.join(reasons)} · {withheld}; not nudging")
3729
+ return 0
3730
+ argv = [x.replace("{session}", sess).replace("{prompt}", prompt)
3731
+ for x in plan["argv"]]
3732
+ if not argv:
3733
+ print("adapter has no resume command")
3734
+ return 1
3735
+
3736
+ # Resolve the CLI and hand the child a usable PATH — see child_path(). Never a program the current
3737
+ # directory holds, which Windows looks in before PATH (WINDOWS-NO-CWD-EXEC), nor one inside the tree (ROUTE-ARGV).
3738
+ search = child_path()
3739
+ resolved = A.program_path(argv[0], path=A.outside_repository(search, root))
3740
+ if not resolved:
3741
+ notify(f"{project}: watchdog", f"{argv[0]} not on PATH; cannot nudge", root)
3742
+ print(f"{argv[0]} not found on PATH ({search})")
3743
+ return 1
3744
+ named, argv[0] = argv[0], resolved
3745
+ # What a turn nobody attends starts with: the grant its adapter declares for one, or its
3746
+ # trust_all, and only when the resume carries no allowlist of its own - appending
3747
+ # --dangerously-skip-permissions on top of --allowedTools would silently override the
3748
+ # narrower grant. A flag that turns off the harness's own sandbox waits for a person, and the
3749
+ # permission mode is the one the adapter pins, whatever a person's settings default to (GRANTS-PINNED).
3750
+ added, added_why = A.unattended_flags(shipped, argv)
3751
+ bypass = A.bypass_refusal(shipped, added, cfg)
3752
+ if bypass:
3753
+ notify(f"{project}: nudge refused", bypass, root, key="nudge-bypass-refused", window=12 * 3600,
3754
+ audience="human", what=bypass)
3755
+ print(f"idle {int(age)}s · {', '.join(reasons)} · {bypass}; not nudging")
3756
+ return 0
3757
+ argv += added
3758
+ argv, pinned = A.pinned_argv(argv, "implementer")
3759
+ if pinned:
3760
+ added = added + pinned
3761
+ added_why = "; ".join(filter(None, [added_why, "the implementer's command names no permission mode, "
3762
+ "so polyctl appends the one its adapter pins"]))
3763
+ # The project chooses the implementer's model and effort in its config; the
3764
+ # adapter says how to spell them. Nothing is appended for an adapter that
3765
+ # has no such option, or declares it null: one with no flag for it (TURN-ARGV).
3766
+ opts = shipped.get("options") or {}
3767
+ for key in ("model", "effort"):
3768
+ val = impl.get(key)
3769
+ if val and isinstance(opts.get(key), list) and not any(str(a).startswith(f"--{key}") for a in argv):
3770
+ if key == "effort" and opts.get("effort_values") and val not in opts["effort_values"]:
3771
+ print(f"effort {val!r} not in {opts['effort_values']}; ignored")
3772
+ continue
3773
+ argv += [x.replace("{" + key + "}", str(val)) for x in opts[key]]
3774
+
3775
+ from . import features as F
3776
+ if not F.enabled(cfg, "nudge"):
3777
+ print(f"idle {int(age)}s · {', '.join(reasons)} · nudge feature off; not starting a turn")
3778
+ return 0
3779
+ # The command as it starts runs the shipped adapter's harness, with no value that reads as a flag
3780
+ # and no grant beyond its own, before anything is spent on it (TURN-ARGV).
3781
+ refused = A.turn_refusal("implementer", impl.get("adapter"), [named] + argv[1:], root=root, resolved=resolved,
3782
+ search=search, values={"session": sess, "model": impl.get("model"),
3783
+ "effort": impl.get("effort")}, prompt=prompt)
3784
+ if refused:
3785
+ print(f"idle {int(age)}s · {', '.join(reasons)} · {refused}; not nudging")
3786
+ tell_turn_refused(root, "implementer", refused)
3787
+ return 0
3788
+ headroom = A.rotate_if_exhausted(cfg, argv, "implementer") if not args.dry_run else {"ok": True}
3789
+ if not headroom["ok"]:
3790
+ print(f"idle {int(age)}s · {', '.join(reasons)} · {headroom['text']}; not nudging")
3791
+ return 0
3792
+ # The implementer's harness runs what it wrote into its own configuration: read after the rotation, which can
3793
+ # wait minutes on keyflip, and in a dry cycle as in a live one (HARNESS-CONFIG). Its command is the one a person
3794
+ # accepted (TURN-ARGV).
3795
+ unseen = harness_config_hold(root) or turn_hold(root, cfg, "implementer")
3796
+ if unseen:
3797
+ print(f"idle {int(age)}s · {', '.join(reasons)} · {unseen}; not nudging")
3798
+ return 0
3799
+ print(f"idle {int(age)}s · {', '.join(reasons)} · nudging")
3800
+ if args.dry_run:
3801
+ print("DRY RUN:", " ".join(argv[:4]), "…")
3802
+ return 0
3803
+
3804
+ # Never discard the child's output. A nudge that dies on an expired login or
3805
+ # an exhausted plan looks exactly like an agent that ignored us, and the
3806
+ # difference is the only thing worth knowing at that moment.
3807
+ key = A.project_key(root)
3808
+ log_path = os.path.join(STATE_DIR, A.project_file_name("nudge-log", key))
3809
+ os.makedirs(STATE_DIR, exist_ok=True)
3810
+ env = dict(os.environ, PATH=search, POLYCTL_ROLE="implementer")
3811
+ if not record_turn_flags(root, "implementer", added, added_why):
3812
+ return 0
3813
+ held = A.hold_state(root)
3814
+ if held:
3815
+ print(f"held by {held.get('by')} since this cycle began; not nudging")
3816
+ return 0
3817
+ given, refused = A.prompt_input(plan, root, argv)
3818
+ if refused:
3819
+ print(f"the nudge's prompt cannot be handed over: {refused}; not nudging")
3820
+ return 1
3821
+ # A child can finish its useful work inside the startup wait (TRIAGE-2-MEDIUM).
3822
+ nudged_fp = A.work_fingerprint(root)
3823
+ nudged_inputs = A.nudge_inputs(root, cfg)
3824
+ with open(log_path, "a", encoding=UTF8) as log:
3825
+ log.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} nudge"
3826
+ f"{' (added: ' + ' '.join(added) + ')' if added else ''} ===\n")
3827
+ log.flush()
3828
+ try:
3829
+ proc = subprocess.Popen(given["argv"], cwd=root, env=env,
3830
+ stdin=subprocess.DEVNULL if given["stdin"] is None else given["stdin"],
3831
+ stdout=log, stderr=subprocess.STDOUT,
3832
+ start_new_session=True)
3833
+ finally:
3834
+ A.release_prompt(given)
3835
+ # The turn is on record before its first seconds are waited out: a cycle cut off in them left a turn
3836
+ # running in a session of its own that no state named, and the next cycle cut the log it still wrote
3837
+ # (JOURNAL-3-3).
3838
+ try:
3839
+ child_start = A._process_start(proc.pid, refresh=True)
3840
+ except Exception:
3841
+ child_start = None
3842
+ st.update(child_pid=proc.pid, child_start=child_start, last_nudge=time.time())
3843
+ save_state(root, st)
3844
+
3845
+ # Give it a moment to fail. A healthy turn runs for minutes; anything that
3846
+ # exits within seconds died rather than started.
3847
+ early = None
3848
+ for _ in range(12):
3849
+ time.sleep(1)
3850
+ if proc.poll() is not None:
3851
+ early = proc.returncode
3852
+ break
3853
+
3854
+ st.update(attempts=st.get("attempts", 0) + 1, last_nudge=time.time(),
3855
+ last_size=size, child_pid=proc.pid, child_start=child_start,
3856
+ last_fingerprint=nudged_fp,
3857
+ nudge_size=size, nudge_fingerprint=nudged_fp,
3858
+ nudge_inputs=nudged_inputs)
3859
+ # Told once its first seconds are over, with the exit of a turn that did not outlive them (EVENTS-LOG).
3860
+ A.emit_event(root, "nudge", {"pid": proc.pid, "attempt": st["attempts"], "exit": early})
3861
+ if early not in (None, 0):
3862
+ tail = ""
3863
+ try:
3864
+ with open(log_path, encoding=UTF8) as fh:
3865
+ tail = A.redact(" ".join(fh.read().strip().split("\n")[-3:])[-300:])
3866
+ except OSError:
3867
+ pass
3868
+ st.pop("child_pid", None) # it is gone; do not guard on a dead pid
3869
+ st["last_error"] = {"at": time.time(), "code": early, "tail": tail}
3870
+ # A nudge that died on the usage limit parks its slice now, and the park is what a person
3871
+ # is told: a park tried again at its retry would otherwise ring "nudge failed" each time (QUOTA-PARK).
3872
+ try:
3873
+ parked = quota_park(root, cfg, st, adapter=adapter)
3874
+ except Exception as exc: # a park that cannot be written must not take the nudge's record
3875
+ parked = False
3876
+ print(f"the park could not be written: {type(exc).__name__}: {str(exc)[:120]}")
3877
+ if not parked:
3878
+ notify(f"{project}: nudge failed", f"exit {early}: {tail[-120:] or 'see nudge log'}", root)
3879
+ print(f"nudge failed (exit {early}): {tail}")
3880
+ else:
3881
+ st.pop("last_error", None)
3882
+ save_state(root, st)
3883
+ return 0
3884
+
3885
+
3886
+ if __name__ == "__main__":
3887
+ sys.exit(main())