polyctl 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyctl/__init__.py +7 -0
- polyctl/__main__.py +5 -0
- polyctl/a2a.py +226 -0
- polyctl/a2a_mcp.py +546 -0
- polyctl/acp.py +417 -0
- polyctl/adapters/README.md +101 -0
- polyctl/adapters/aider.json +32 -0
- polyctl/adapters/amazon-q.json +15 -0
- polyctl/adapters/amp.json +16 -0
- polyctl/adapters/antigravity.json +120 -0
- polyctl/adapters/claude-code.json +229 -0
- polyctl/adapters/cloud-generic.json +57 -0
- polyctl/adapters/codex.json +47 -0
- polyctl/adapters/command-code.json +45 -0
- polyctl/adapters/copilot.json +17 -0
- polyctl/adapters/cursor-agent.json +29 -0
- polyctl/adapters/deepseek.json +13 -0
- polyctl/adapters/droid.json +34 -0
- polyctl/adapters/gemini.json +30 -0
- polyctl/adapters/grok.json +22 -0
- polyctl/adapters/hermes.json +34 -0
- polyctl/adapters/kilocode.json +30 -0
- polyctl/adapters/kimi.json +23 -0
- polyctl/adapters/kiro.json +332 -0
- polyctl/adapters/ollama.json +29 -0
- polyctl/adapters/omp.json +34 -0
- polyctl/adapters/openai-api.json +68 -0
- polyctl/adapters/opencode.json +162 -0
- polyctl/adapters/pi.json +24 -0
- polyctl/adapters/pr-agent.json +54 -0
- polyctl/adapters/profiles.json +9 -0
- polyctl/adapters/qoder.json +163 -0
- polyctl/adapters/qwen.json +27 -0
- polyctl/adapters/reasonix.json +25 -0
- polyctl/adapters/rewriters/rtk.json +24 -0
- polyctl/adapters/trae.json +31 -0
- polyctl/adapters/vendors.json +42 -0
- polyctl/adapters/warp.json +88 -0
- polyctl/allowlist.py +1756 -0
- polyctl/api_review.py +920 -0
- polyctl/cli.py +835 -0
- polyctl/conformance_harness.py +9 -0
- polyctl/demo.py +503 -0
- polyctl/drivers.py +146 -0
- polyctl/email.py +160 -0
- polyctl/features.py +98 -0
- polyctl/journal.py +144 -0
- polyctl/language.py +945 -0
- polyctl/lib.py +1296 -0
- polyctl/matrix.py +660 -0
- polyctl/mcp.py +1362 -0
- polyctl/ntfy.py +169 -0
- polyctl/outbound.py +39 -0
- polyctl/parts/__init__.py +5 -0
- polyctl/parts/cli_authority.py +2615 -0
- polyctl/parts/cli_channels.py +1325 -0
- polyctl/parts/cli_checks.py +2293 -0
- polyctl/parts/cli_completion.py +686 -0
- polyctl/parts/cli_evidence.py +141 -0
- polyctl/parts/cli_gain.py +375 -0
- polyctl/parts/cli_harness.py +392 -0
- polyctl/parts/cli_hooks.py +1816 -0
- polyctl/parts/cli_install.py +638 -0
- polyctl/parts/cli_lanes.py +106 -0
- polyctl/parts/cli_maintenance.py +2536 -0
- polyctl/parts/cli_next.py +248 -0
- polyctl/parts/cli_pr.py +53 -0
- polyctl/parts/cli_project.py +2857 -0
- polyctl/parts/cli_review.py +5158 -0
- polyctl/parts/cli_status.py +898 -0
- polyctl/parts/lib_adapters.py +2933 -0
- polyctl/parts/lib_alarms.py +2106 -0
- polyctl/parts/lib_events.py +224 -0
- polyctl/parts/lib_evidence.py +412 -0
- polyctl/parts/lib_gates.py +2099 -0
- polyctl/parts/lib_lanes.py +829 -0
- polyctl/parts/lib_mail.py +1541 -0
- polyctl/parts/lib_next.py +652 -0
- polyctl/parts/lib_pr.py +331 -0
- polyctl/parts/lib_slices.py +3337 -0
- polyctl/parts/lib_state.py +2415 -0
- polyctl/parts/lib_transcript.py +4118 -0
- polyctl/pressure.py +325 -0
- polyctl/prices.json +47 -0
- polyctl/procs.py +1194 -0
- polyctl/settings.py +505 -0
- polyctl/skill/SKILL.md +365 -0
- polyctl/skillkit.py +503 -0
- polyctl/storage.py +2055 -0
- polyctl/telegram.py +327 -0
- polyctl/tiers.py +123 -0
- polyctl/verdicts.py +41 -0
- polyctl/watchdog.py +3887 -0
- polyctl/web.py +337 -0
- polyctl-0.7.0.dist-info/METADATA +235 -0
- polyctl-0.7.0.dist-info/RECORD +99 -0
- polyctl-0.7.0.dist-info/WHEEL +4 -0
- polyctl-0.7.0.dist-info/entry_points.txt +5 -0
- polyctl-0.7.0.dist-info/licenses/LICENSE +21 -0
polyctl/watchdog.py
ADDED
|
@@ -0,0 +1,3887 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""polyctl-watchdog — restart a stalled implementer, cheaply.
|
|
3
|
+
|
|
4
|
+
Turn-based agents stop when a turn ends, mid-slice or not. Something has to
|
|
5
|
+
notice and nudge. Detection here is free: a file mtime and a few `git` calls.
|
|
6
|
+
Only the nudge itself costs anything, and it costs exactly what the work would
|
|
7
|
+
have cost when a human noticed twenty minutes later.
|
|
8
|
+
|
|
9
|
+
Guards, in order — the point is to spend nothing when spending would not help:
|
|
10
|
+
|
|
11
|
+
-1. Has a human taken the tree? (.polyctl/hold — overrides everything)
|
|
12
|
+
-1b. Has the running slice spent past its budget, or is a reading its budget needs missing?
|
|
13
|
+
(park it; nothing resumes it)
|
|
14
|
+
0. Is an agent already WORKING here? (cwd match AND a moving transcript; a
|
|
15
|
+
process alive but long silent is hung,
|
|
16
|
+
and is reaped rather than waited on)
|
|
17
|
+
1. Is the session actually idle? (mtime)
|
|
18
|
+
1b. Did it stop on its usage limit? (park the running slice until the reset)
|
|
19
|
+
2. Is there open work to continue? (mailbox / dirty tree / NEEDS_CHANGES)
|
|
20
|
+
2b. Did the queue run dry rather than end? (wake the architect, not the implementer)
|
|
21
|
+
3. Is the slice over its round budget? (if so: notify a human, never nudge)
|
|
22
|
+
4. Is there provider quota left? (keyflip, if installed)
|
|
23
|
+
4b. Is the provider itself degraded? (5xx at the end of our own nudge log)
|
|
24
|
+
4c. Are reports sitting unhandled? (wake the architect — its own pid guard)
|
|
25
|
+
5. Did the last nudge achieve anything? (backoff, then hand over to a human)
|
|
26
|
+
6. Has a person seen what a harness reads here? (asked before every turn it
|
|
27
|
+
starts: configuration an agent changed
|
|
28
|
+
and nobody saw holds them all)
|
|
29
|
+
|
|
30
|
+
Run it from cron/launchd every couple of minutes, or once by hand.
|
|
31
|
+
|
|
32
|
+
polyctl-watchdog --root ~/work/project # act
|
|
33
|
+
polyctl-watchdog --root ~/work/project --dry-run # decide, change nothing
|
|
34
|
+
"""
|
|
35
|
+
import argparse
|
|
36
|
+
from contextvars import ContextVar
|
|
37
|
+
import json
|
|
38
|
+
import math
|
|
39
|
+
import os
|
|
40
|
+
import re
|
|
41
|
+
import subprocess
|
|
42
|
+
import shutil
|
|
43
|
+
import signal
|
|
44
|
+
import sys
|
|
45
|
+
import time
|
|
46
|
+
|
|
47
|
+
from . import journal as J
|
|
48
|
+
from . import language
|
|
49
|
+
from . import lib as A
|
|
50
|
+
from . import settings as S
|
|
51
|
+
from .procs import TableUnreadable
|
|
52
|
+
|
|
53
|
+
STATE_DIR = os.path.join(A.HOME, ".polyctl")
|
|
54
|
+
_DRY_RUN = ContextVar("polyctl_watchdog_dry_run", default=False)
|
|
55
|
+
# A cycle after a long silence gathers what it would ring into one notice (RESUME-QUIET).
|
|
56
|
+
_RESUME = ContextVar("polyctl_watchdog_resume", default=None)
|
|
57
|
+
|
|
58
|
+
# Every decision line this module prints is also kept, so a cycle can be read
|
|
59
|
+
# back as a trace: what was measured, what was decided, in what order. The one
|
|
60
|
+
# question that cost this project the most — "why did it not act?" — used to be
|
|
61
|
+
# answerable only by reading a log of prose. `polyctl watchdog explain` runs a dry
|
|
62
|
+
# cycle and prints the trace; `polyctl watchdog trace` reads the recorded ones.
|
|
63
|
+
import builtins as _builtins
|
|
64
|
+
UTF8 = "utf-8" # every text file polyctl writes or reads; Windows would otherwise use cp1252
|
|
65
|
+
_TRACE = []
|
|
66
|
+
_FACTS = {}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def print(*args, **kw):
|
|
70
|
+
line = " ".join(str(a) for a in args)
|
|
71
|
+
_TRACE.append(line)
|
|
72
|
+
_builtins.print(*args, **kw)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def cycles_path(root):
|
|
76
|
+
key = A.project_key(root)
|
|
77
|
+
return os.path.join(STATE_DIR, A.project_file_name("cycles", key))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def record_cycle(root, args, started=None):
|
|
81
|
+
if getattr(args, "dry_run", False):
|
|
82
|
+
return
|
|
83
|
+
now = time.time()
|
|
84
|
+
rec = {"at": int(now), "verdict": _TRACE[-1] if _TRACE else "",
|
|
85
|
+
"trace": list(_TRACE), "facts": dict(_FACTS)}
|
|
86
|
+
if started is not None:
|
|
87
|
+
# Without a start, a cycle that ran for minutes and a machine that ran
|
|
88
|
+
# none leave the same gap in this log.
|
|
89
|
+
rec.update(started=int(started), seconds=round(now - started, 1))
|
|
90
|
+
try:
|
|
91
|
+
p = cycles_path(root)
|
|
92
|
+
os.makedirs(STATE_DIR, exist_ok=True)
|
|
93
|
+
if os.path.exists(p) and os.path.getsize(p) > 2_000_000:
|
|
94
|
+
tail = open(p, errors="replace", encoding=UTF8).read()[-1_000_000:]
|
|
95
|
+
open(p, "w", encoding=UTF8).write(tail[tail.find("\n") + 1:])
|
|
96
|
+
with open(p, "a", encoding=UTF8) as fh:
|
|
97
|
+
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
98
|
+
except OSError:
|
|
99
|
+
pass
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def cycles(root, last=20):
|
|
103
|
+
p = cycles_path(root)
|
|
104
|
+
if not os.path.exists(p):
|
|
105
|
+
return []
|
|
106
|
+
rows = []
|
|
107
|
+
for line in open(p, errors="replace", encoding=UTF8):
|
|
108
|
+
try:
|
|
109
|
+
rows.append(json.loads(line))
|
|
110
|
+
except ValueError:
|
|
111
|
+
pass
|
|
112
|
+
return rows[-last:]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# What a nudge, a wake and a refill tell the turn they start is in language.py, in the project's language,
|
|
116
|
+
# read with the config each cycle loads (LANGUAGE-PROMPTS): `prompt.nudge` with what a nudge adds,
|
|
117
|
+
# `prompt.wake` and `prompt.refill`.
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def cycle_health(root, last=720):
|
|
121
|
+
"""How long recent cycles took, and the longest silence between two of them.
|
|
122
|
+
|
|
123
|
+
A gap in the cycle log has two causes - no cycle ran, or one ran for minutes
|
|
124
|
+
and held the next back - and a row that carries only its end time cannot tell
|
|
125
|
+
them apart. One project's log once showed gaps of 5 to 27 minutes over four
|
|
126
|
+
and a half hours with no sleep recorded, and nothing on disk could say which
|
|
127
|
+
it was. Rows written before `started` existed count by their end times.
|
|
128
|
+
"""
|
|
129
|
+
rows = [r for r in cycles(root, last) if isinstance(r.get("at"), (int, float))]
|
|
130
|
+
if not rows:
|
|
131
|
+
return None
|
|
132
|
+
timed = [r for r in rows if isinstance(r.get("seconds"), (int, float))]
|
|
133
|
+
longest = max(timed, key=lambda r: r["seconds"]) if timed else None
|
|
134
|
+
gap, gap_at = 0, None
|
|
135
|
+
for previous, current in zip(rows, rows[1:]):
|
|
136
|
+
silence = current.get("started", current["at"]) - previous["at"]
|
|
137
|
+
if silence > gap:
|
|
138
|
+
gap, gap_at = silence, current["at"]
|
|
139
|
+
|
|
140
|
+
def when(t):
|
|
141
|
+
return time.strftime("%d %b %H:%M", time.localtime(t))
|
|
142
|
+
|
|
143
|
+
return {"count": len(rows),
|
|
144
|
+
"longest_seconds": longest["seconds"] if longest else None,
|
|
145
|
+
"longest_at": when(longest["at"]) if longest else None,
|
|
146
|
+
"gap_minutes": gap / 60, "gap_at": when(gap_at) if gap_at else None}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def child_path():
|
|
150
|
+
"""A PATH the nudged agent can actually build with.
|
|
151
|
+
|
|
152
|
+
Two different failures hide here. launchd gives us a minimal PATH, so we
|
|
153
|
+
cannot find the agent CLI. And the agent we spawn inherits whatever PATH we
|
|
154
|
+
hand it — if that lacks the toolchain (node, npm, a version-manager shim),
|
|
155
|
+
the agent starts its turn and then cannot run its own gates. Resolve both by
|
|
156
|
+
building one PATH that covers the user's real toolchain, including any
|
|
157
|
+
version-manager shim directory the current interpreter can see.
|
|
158
|
+
"""
|
|
159
|
+
parts = [os.environ.get("PATH", "")]
|
|
160
|
+
# A bundle's CLI may be off PATH: hand children the same declared install directories
|
|
161
|
+
# binary_candidates searches, including those outside the home (WARP-ADAPTER).
|
|
162
|
+
for d in list(S.get(None, "binaries.extra_dirs")) + list(A._BIN_DIRS) + [
|
|
163
|
+
"/usr/bin", "/bin", "/usr/sbin", "/sbin"]:
|
|
164
|
+
parts.append(os.path.expanduser(d))
|
|
165
|
+
# Version-manager installs keep node beside the agent binary; a wake that
|
|
166
|
+
# resolves the newest agent there must find that node too.
|
|
167
|
+
agents = sorted({binary for adapter in A.package_adapters().values()
|
|
168
|
+
if (adapter.get("detect") or {}).get("processes") for binary in A.adapter_binaries(adapter)})
|
|
169
|
+
for cand in [c for binary in agents for c in A.binary_candidates(binary)] + A.binary_candidates("node"):
|
|
170
|
+
parts.append(os.path.dirname(cand))
|
|
171
|
+
# fnm / nvm / asdf style shims: whichever one currently owns `node`
|
|
172
|
+
node = A.program_path("node")
|
|
173
|
+
if node:
|
|
174
|
+
parts.append(os.path.dirname(node))
|
|
175
|
+
for base in ("~/.local/state/fnm_multishells", "~/.nvm/versions/node", "~/.asdf/shims"):
|
|
176
|
+
b = os.path.expanduser(base)
|
|
177
|
+
if os.path.isdir(b):
|
|
178
|
+
try:
|
|
179
|
+
for entry in sorted(os.listdir(b), reverse=True)[:3]:
|
|
180
|
+
cand = os.path.join(b, entry, "bin")
|
|
181
|
+
if os.path.isdir(cand):
|
|
182
|
+
parts.append(cand)
|
|
183
|
+
elif os.path.isdir(os.path.join(b, entry)):
|
|
184
|
+
parts.append(os.path.join(b, entry))
|
|
185
|
+
except OSError:
|
|
186
|
+
pass
|
|
187
|
+
seen, out = set(), []
|
|
188
|
+
for chunk in os.pathsep.join(parts).split(os.pathsep):
|
|
189
|
+
if chunk and chunk not in seen:
|
|
190
|
+
seen.add(chunk)
|
|
191
|
+
out.append(chunk)
|
|
192
|
+
return os.pathsep.join(out)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# Legacy call sites that do not declare an audience are classified by title.
|
|
196
|
+
# Explicit audience is authoritative: wording and localization must never turn a
|
|
197
|
+
# human alarm into an architect-only ledger row (or vice versa).
|
|
198
|
+
HUMAN_AUDIENCE = ("out of quota", "agent stuck", "nudge failed", "watchdog",
|
|
199
|
+
"turns piling up", "needs you", "provider degraded")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def for_human(title):
|
|
203
|
+
# Match the words after the "<project>: " prefix only. Titles name the project,
|
|
204
|
+
# and a project called, say, "watchdog-lab" must not turn every alert it raises
|
|
205
|
+
# into a human one by its name alone.
|
|
206
|
+
subject = title.split(": ", 1)[1] if ": " in title else title
|
|
207
|
+
return any(k in subject.lower() for k in HUMAN_AUDIENCE)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# The text travels in the environment and is never spliced into the script (#9).
|
|
211
|
+
TOAST_SCRIPT = (
|
|
212
|
+
"[Windows.UI.Notifications.ToastNotificationManager, Windows.UI.Notifications, ContentType = WindowsRuntime] > $null; "
|
|
213
|
+
"$t = [Windows.UI.Notifications.ToastNotificationManager]::GetTemplateContent("
|
|
214
|
+
"[Windows.UI.Notifications.ToastTemplateType]::ToastText02); "
|
|
215
|
+
"$x = $t.GetElementsByTagName('text'); "
|
|
216
|
+
"$x.Item(0).AppendChild($t.CreateTextNode($env:POLYCTL_TOAST_TITLE)) > $null; "
|
|
217
|
+
"$x.Item(1).AppendChild($t.CreateTextNode($env:POLYCTL_TOAST_BODY)) > $null; "
|
|
218
|
+
"[Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier("
|
|
219
|
+
"'{1AC14E77-02E7-4E5D-B744-2EB1AE5198B7}\\WindowsPowerShell\\v1.0\\powershell.exe')"
|
|
220
|
+
".Show([Windows.UI.Notifications.ToastNotification]::new($t))"
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def desktop_notify(title, msg, cfg=None):
|
|
225
|
+
"""A desktop notification where the platform has one; never an exception (audit).
|
|
226
|
+
|
|
227
|
+
`osascript` exists only on macOS. Elsewhere its FileNotFoundError ended the cycle
|
|
228
|
+
before the notice was recorded or the state saved. On Windows a toast is shown
|
|
229
|
+
through PowerShell when the project's `toast` feature is on (#9); unverified on a
|
|
230
|
+
Windows machine until the Windows lane runs it. Windows' own PowerShell comes from its
|
|
231
|
+
system directory, never one the current directory holds (WINDOWS-NO-CWD-EXEC).
|
|
232
|
+
"""
|
|
233
|
+
if sys.platform == "win32":
|
|
234
|
+
from . import features as F
|
|
235
|
+
shell = A.program_path("powershell") or A.program_path("pwsh")
|
|
236
|
+
if not (cfg and F.enabled(cfg, "toast")) or not shell:
|
|
237
|
+
return False
|
|
238
|
+
try:
|
|
239
|
+
done = subprocess.run([shell, "-NoProfile", "-NonInteractive", "-Command", TOAST_SCRIPT],
|
|
240
|
+
capture_output=True, timeout=20,
|
|
241
|
+
env=dict(os.environ, POLYCTL_TOAST_TITLE=str(title), POLYCTL_TOAST_BODY=str(msg)))
|
|
242
|
+
except (OSError, subprocess.SubprocessError):
|
|
243
|
+
return False
|
|
244
|
+
return done.returncode == 0
|
|
245
|
+
osascript = A.program_path("osascript") if sys.platform == "darwin" else None
|
|
246
|
+
if not osascript:
|
|
247
|
+
return False
|
|
248
|
+
title, msg = (str(part).replace('"', "'").replace("\\", "/") for part in (title, msg))
|
|
249
|
+
try:
|
|
250
|
+
subprocess.run([osascript, "-e", f'display notification "{msg}" with title "{title}"'],
|
|
251
|
+
capture_output=True, timeout=10)
|
|
252
|
+
return True
|
|
253
|
+
except (OSError, subprocess.SubprocessError):
|
|
254
|
+
return False
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def notify(title, msg, root=None, key=None, window=1800, audience=None, level=None,
|
|
258
|
+
quiet_until=None, evidence=None, what=None, red_after=None):
|
|
259
|
+
"""Raise an alert at most once per window, and always record that we did.
|
|
260
|
+
|
|
261
|
+
An explicit audience is a routing decision and is never inferred again from
|
|
262
|
+
the title. Title inference exists only for legacy callers that omit it.
|
|
263
|
+
|
|
264
|
+
The watchdog runs every two minutes. A guard that notifies on each run turns
|
|
265
|
+
a single ongoing condition into thirty alerts an hour, and a human who learns
|
|
266
|
+
to swipe those away has effectively turned the alerting off — which is worse
|
|
267
|
+
than not alerting, because everyone still believes it works.
|
|
268
|
+
|
|
269
|
+
A condition that stands and can say what it is about passes `what`: it rings
|
|
270
|
+
the desktop and the phone once for what it says, then climbs the ladder
|
|
271
|
+
without ringing them again - red mails on its schedule - until it says
|
|
272
|
+
something else. A window alone repeats: on 2026-09-17 "needs you" rang every
|
|
273
|
+
ten minutes about one request nobody could answer (NOTICE-NOISE).
|
|
274
|
+
|
|
275
|
+
A check whose own thresholds say when it is red passes `red_after`, the seconds an
|
|
276
|
+
orange stands before the ladder rings it red; otherwise it is
|
|
277
|
+
`alarms.red_after_minutes`. An unseen request is red at its red threshold, and an
|
|
278
|
+
hour of orange mailed it two hours before that (NOISE-REPEATS).
|
|
279
|
+
|
|
280
|
+
Recording happens either way. A suppressed alert is evidence too: a long run
|
|
281
|
+
of them says the condition has held for a long time.
|
|
282
|
+
"""
|
|
283
|
+
key = key or title
|
|
284
|
+
if audience is None:
|
|
285
|
+
audience = "human" if for_human(title) else "architect"
|
|
286
|
+
if audience not in ("architect", "human"):
|
|
287
|
+
raise ValueError("unknown notification audience: %s" % audience)
|
|
288
|
+
dry_run = _DRY_RUN.get()
|
|
289
|
+
resuming = _RESUME.get()
|
|
290
|
+
# Architect-audience alerts are recorded and delivered through the mailbox and
|
|
291
|
+
# the wake; they never use a human channel.
|
|
292
|
+
if audience == "architect":
|
|
293
|
+
if dry_run:
|
|
294
|
+
print(f"DRY RUN: would record architect notice: {title}")
|
|
295
|
+
elif root:
|
|
296
|
+
A.record_notice(root, title, msg, sent=False, key=key, evidence=evidence)
|
|
297
|
+
if resuming is not None:
|
|
298
|
+
resuming["folded"].append({"key": key, "title": title, "msg": msg, "audience": audience})
|
|
299
|
+
return False
|
|
300
|
+
# The ladder: this is an orange (a person must act). Standing an hour, it
|
|
301
|
+
# rings red and goes to mail — the channel people open when they wake up.
|
|
302
|
+
project = (A.project_key(root) if root else "polyctl")
|
|
303
|
+
if red_after is None:
|
|
304
|
+
try:
|
|
305
|
+
red_after = S.get(A.load_config(root) if root else None, "alarms.red_after_minutes") * 60
|
|
306
|
+
except Exception:
|
|
307
|
+
red_after = A.ALARM_RED_AFTER
|
|
308
|
+
# Dry-run uses the same alarm state and calculation as a live cycle, but the
|
|
309
|
+
# preview is deliberately not persisted.
|
|
310
|
+
# A snoozed alarm is recorded, not rung: off the channels and off the ladder, so
|
|
311
|
+
# it neither turns red nor mails until its snooze ends.
|
|
312
|
+
snoozed = A.alarm_snoozed(project, key)
|
|
313
|
+
if snoozed:
|
|
314
|
+
until = time.strftime("%d %b", time.localtime(snoozed["until"]))
|
|
315
|
+
if dry_run:
|
|
316
|
+
print(f"DRY RUN: would hold {title}: snoozed until {until} by {snoozed.get('by')}")
|
|
317
|
+
elif root:
|
|
318
|
+
A.record_notice(root, title, f"{msg} [snoozed until {until}]", sent=False, key=key, evidence=evidence)
|
|
319
|
+
return False
|
|
320
|
+
ring, episode = A.alarm_touch(
|
|
321
|
+
project, key, level or "orange", red_after=red_after, title=title,
|
|
322
|
+
persist=not dry_run, quiet_until=quiet_until, evidence=evidence, what=what,
|
|
323
|
+
)
|
|
324
|
+
# After a silence the ladder still moves, so one condition keeps one alarm (#106), but
|
|
325
|
+
# nothing rings on its own: the cycle's one resume notice names it (RESUME-QUIET).
|
|
326
|
+
if resuming is not None:
|
|
327
|
+
resuming["folded"].append({"key": key, "title": title, "msg": msg, "audience": audience, "ring": ring,
|
|
328
|
+
"what": what})
|
|
329
|
+
if dry_run:
|
|
330
|
+
print(f"DRY RUN: would name in the resume notice: {title}")
|
|
331
|
+
elif root:
|
|
332
|
+
A.record_notice(root, title, f"{msg} [named in the resume notice]", sent=False, key=key,
|
|
333
|
+
evidence=evidence)
|
|
334
|
+
return False
|
|
335
|
+
if ring == "red" and episode.get("red_due"):
|
|
336
|
+
if dry_run:
|
|
337
|
+
print(f"DRY RUN: would send red e-mail: {title}")
|
|
338
|
+
else:
|
|
339
|
+
try:
|
|
340
|
+
from . import email
|
|
341
|
+
since = time.strftime("%d %b %H:%M", time.localtime(episode.get("first", time.time())))
|
|
342
|
+
# The mail is worded in the project's language; its key is the same in any (LANGUAGE-OUTPUT).
|
|
343
|
+
written = _written_in(root)
|
|
344
|
+
body = (f"{msg}\n\n" + language.text(written, "alarm.standing", since=since,
|
|
345
|
+
count=episode.get("count", 1), project=root or "?")
|
|
346
|
+
+ ("\n\n" + language.text(written, "alarm.evidence") + "\n"
|
|
347
|
+
+ "\n".join(A.evidence_lines(evidence)) if evidence else ""))
|
|
348
|
+
if email.send(f"{title}", body, root):
|
|
349
|
+
A.alarm_mailed(project, key, what=what)
|
|
350
|
+
if root:
|
|
351
|
+
A.record_notice(root, title, msg, sent=True, key="mail:" + key, evidence=evidence)
|
|
352
|
+
except Exception:
|
|
353
|
+
pass
|
|
354
|
+
# Told once, with a known end and nothing new to say before it: record, do not ring.
|
|
355
|
+
# A resume notice that named a red told it as a mail does (RESUME-QUIET).
|
|
356
|
+
told = episode.get("red_sent") if episode.get("red_sent") is not None else episode.get("named_at")
|
|
357
|
+
if told is not None and not episode.get("news") and time.time() < float(episode.get("quiet_until") or 0):
|
|
358
|
+
until = time.strftime("%d %b %H:%M", time.localtime(float(episode["quiet_until"])))
|
|
359
|
+
if dry_run:
|
|
360
|
+
print(f"DRY RUN: would hold {title}: told once, quiet until {until}")
|
|
361
|
+
elif root:
|
|
362
|
+
A.record_notice(root, title, f"{msg} [told once; quiet until {until}]", sent=False, key=key, evidence=evidence)
|
|
363
|
+
return False
|
|
364
|
+
# Rung once for what it says, and saying nothing new: the ladder climbs, the channels stay quiet.
|
|
365
|
+
if what is not None and episode.get("rang_at") is not None and not episode.get("news"):
|
|
366
|
+
if dry_run:
|
|
367
|
+
print(f"DRY RUN: would hold {title}: rung once and nothing new since")
|
|
368
|
+
elif root:
|
|
369
|
+
A.record_notice(root, title, f"{msg} [rung once; nothing new since]", sent=False, key=key,
|
|
370
|
+
evidence=evidence)
|
|
371
|
+
return False
|
|
372
|
+
if root and A.notice_recently_sent(root, key, window):
|
|
373
|
+
if dry_run:
|
|
374
|
+
print(f"DRY RUN: would suppress {ring} desktop/Telegram channels "
|
|
375
|
+
f"(recent notice): {title}")
|
|
376
|
+
else:
|
|
377
|
+
A.record_notice(root, title, msg, sent=False, key=key, evidence=evidence)
|
|
378
|
+
return False
|
|
379
|
+
if root and storm(root):
|
|
380
|
+
if dry_run:
|
|
381
|
+
print(f"DRY RUN: would suppress {ring} desktop/Telegram channels "
|
|
382
|
+
f"(alert storm): {title}")
|
|
383
|
+
else:
|
|
384
|
+
A.record_notice(root, title, msg, sent=False, key=key, evidence=evidence)
|
|
385
|
+
if not A.notice_recently_sent(root, "storm", 3600):
|
|
386
|
+
A.record_notice(root, f"{project}: alert storm", "12+ alerts in an hour; further ones "
|
|
387
|
+
"are recorded only — polyctl notices", sent=True, key="storm")
|
|
388
|
+
desktop_notify(f"{project}: alert storm",
|
|
389
|
+
"12+ alerts in an hour; further ones recorded only (polyctl notices)")
|
|
390
|
+
return False
|
|
391
|
+
if dry_run:
|
|
392
|
+
print(f"DRY RUN: would ring {ring} desktop/Telegram channels: {title}")
|
|
393
|
+
return False
|
|
394
|
+
safe = msg.replace('"', "'")[:200]
|
|
395
|
+
try:
|
|
396
|
+
channel_cfg = A.load_config(root) if root else None
|
|
397
|
+
except Exception:
|
|
398
|
+
channel_cfg = None
|
|
399
|
+
desktop_notify(title, safe, channel_cfg)
|
|
400
|
+
phone_text = f"*{title}*\n{_markdown_plain(msg)}"
|
|
401
|
+
try:
|
|
402
|
+
from . import telegram
|
|
403
|
+
# The message as written, as the resume notice is sent: it can name a report, and one stray
|
|
404
|
+
# underscore in a name makes the phone refuse the whole message as markup (WAITING-ONE-ALARM).
|
|
405
|
+
telegram.send(phone_text, root)
|
|
406
|
+
except Exception:
|
|
407
|
+
pass # a phone being unreachable is not a failure
|
|
408
|
+
try:
|
|
409
|
+
from . import ntfy
|
|
410
|
+
ntfy.send(phone_text, root, level=ring)
|
|
411
|
+
except Exception:
|
|
412
|
+
pass
|
|
413
|
+
A.alarm_rang(project, key, what=what)
|
|
414
|
+
if root:
|
|
415
|
+
A.record_notice(root, title, msg, sent=True, key=key, evidence=evidence)
|
|
416
|
+
return True
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _written_in(root):
|
|
420
|
+
"""The project config a text for a person is worded from, for a caller that holds none (LANGUAGE-OUTPUT).
|
|
421
|
+
|
|
422
|
+
Read from the file as it stands, without resolving a session as load_config does: only its
|
|
423
|
+
`language` is wanted. None without a root, which is the machine's choice.
|
|
424
|
+
"""
|
|
425
|
+
return A.project_config_document(root)["config"] if root else None
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def touch_architect_quota(root, st, cfg=None):
|
|
429
|
+
"""Keep a standing architect-quota alarm alive until its reset window.
|
|
430
|
+
|
|
431
|
+
``notify`` touches the alarm episode before applying its desktop rate limit,
|
|
432
|
+
so calling this on every watchdog cycle rings orange once, advances to red
|
|
433
|
+
after the configured interval, and does not create a notification storm.
|
|
434
|
+
Its words are in the project's language, and its key is not (LANGUAGE-OUTPUT).
|
|
435
|
+
|
|
436
|
+
The one notice says when the wakes start again, and says it again only when that
|
|
437
|
+
time moves (ARCHITECT-WAKE-QUOTA). Held quiet until the end it had named once it was
|
|
438
|
+
mailed, the notice about a five-hour stop left a week's limit read after it untold.
|
|
439
|
+
"""
|
|
440
|
+
until = st.get("arch_quota_until", 0)
|
|
441
|
+
if until <= time.time():
|
|
442
|
+
return False
|
|
443
|
+
written = _written_in(root) if cfg is None else cfg
|
|
444
|
+
text = (st.get("wake_error") or {}).get("text") or "architect quota exhausted"
|
|
445
|
+
return notify(
|
|
446
|
+
language.text(written, "alarm.architect-quota-title", project=A.project_key(root)),
|
|
447
|
+
language.text(written, "alarm.architect-quota", error=text[:100], reset=reset_when(until)),
|
|
448
|
+
root,
|
|
449
|
+
key="architect-quota",
|
|
450
|
+
window=6 * 3600,
|
|
451
|
+
audience="human",
|
|
452
|
+
quiet_until=until,
|
|
453
|
+
what=f"until:{int(until)}",
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def reset_when(at, now=None):
|
|
458
|
+
"""When held wakes start again, as a person reads it: the clock time within a day, the date too beyond one.
|
|
459
|
+
|
|
460
|
+
A weekly limit's end told as "04:00" reads as the coming night, days before the
|
|
461
|
+
week is over (ARCHITECT-WAKE-QUOTA).
|
|
462
|
+
"""
|
|
463
|
+
now = time.time() if now is None else now
|
|
464
|
+
return time.strftime("%H:%M", time.localtime(float(at))) if float(at) - now < 86400 else _when(at)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _announce_resolved(root, e):
|
|
468
|
+
"""Record an alarm that stopped being raised; never louder than the raise (#40).
|
|
469
|
+
|
|
470
|
+
An episode ends when nothing has raised it for two hours, which is not proof the
|
|
471
|
+
condition is gone: a check can stop running, or stop repeating itself. On
|
|
472
|
+
2026-09-15 two episodes about credits that were still exhausted were announced
|
|
473
|
+
"resolved" by e-mail. So the end is recorded as what it is, a red episode's end
|
|
474
|
+
shows once on the desktop, and nothing goes to Telegram or e-mail.
|
|
475
|
+
"""
|
|
476
|
+
title = f"{e.get('project', '')}: no longer raised — {e.get('key', '')}"
|
|
477
|
+
msg = (f"stood {e.get('age_s', 0) // 60}m, raised {e.get('count', 1)}×"
|
|
478
|
+
+ ("; was red" if e.get("red_sent") else ""))
|
|
479
|
+
was_red = bool(e.get("red_sent")) or e.get("ring") == "red"
|
|
480
|
+
A.record_notice(root, title, msg, sent=was_red, key="resolved:" + e.get("key", ""))
|
|
481
|
+
if not was_red:
|
|
482
|
+
return
|
|
483
|
+
try:
|
|
484
|
+
desktop_notify(title, msg, A.load_config(root) if root else None)
|
|
485
|
+
except Exception:
|
|
486
|
+
pass
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def storm(root, limit=12):
|
|
490
|
+
"""More than `limit` alerts sent in the last hour is a storm: the person has
|
|
491
|
+
stopped reading them. Record, do not ring — except red, which still mails."""
|
|
492
|
+
cut = time.time() - 3600
|
|
493
|
+
sent = [n for n in A.notices(root, limit=400) if n.get("sent") and n.get("at", 0) > cut
|
|
494
|
+
and not str(n.get("key", "")).startswith(("tg:", "mail:", "resolved:", "storm"))]
|
|
495
|
+
return len(sent) >= limit
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def state_path(root, record=True):
|
|
499
|
+
key = A.project_key(root) if record else A.next_project_key(root)
|
|
500
|
+
return os.path.join(STATE_DIR, A.project_file_name("watchdog-state", key))
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def load_state(root):
|
|
504
|
+
"""The watchdog's state for a project: a fresh one when the file cannot be read or holds no JSON object.
|
|
505
|
+
|
|
506
|
+
Every reader asks it for keys. A file of `null`, a list or a string - it is in ~/.polyctl, which an agent
|
|
507
|
+
can write - was handed on as it was, and `polyctl status`, the MCP status tool and the cycle stopped on
|
|
508
|
+
the first `.get` (STATE-SHAPE).
|
|
509
|
+
"""
|
|
510
|
+
try:
|
|
511
|
+
with open(state_path(root, record=False), encoding=UTF8) as fh:
|
|
512
|
+
state = json.load(fh)
|
|
513
|
+
except Exception:
|
|
514
|
+
state = None
|
|
515
|
+
return state if isinstance(state, dict) else {"attempts": 0, "last_nudge": 0, "last_size": 0}
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def save_state(root, st):
|
|
519
|
+
"""Write the watchdog's state whole, or not at all (JOURNAL).
|
|
520
|
+
|
|
521
|
+
It was written over in place: a kill -9, a power cut or a Ctrl+C mid-write left the file torn,
|
|
522
|
+
a torn file reads as a fresh state, and the markers that a park was resumed, a report handed to
|
|
523
|
+
a wake, a nudge counted were gone - the next cycle could resume or wake a second time. It is now
|
|
524
|
+
replaced whole and synced, so a reader finds the state before the write or after it.
|
|
525
|
+
"""
|
|
526
|
+
if _DRY_RUN.get():
|
|
527
|
+
return
|
|
528
|
+
from .storage import replace_file_durably
|
|
529
|
+
os.makedirs(STATE_DIR, exist_ok=True)
|
|
530
|
+
replace_file_durably(state_path(root), json.dumps(st, indent=2).encode(UTF8))
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
# How long a claimed wake or refill holds when nothing says how it went: the fifteen minutes a failed wake waits.
|
|
534
|
+
WAKE_LEASE = 900
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def arch_alive(root, architect):
|
|
538
|
+
"""Is any configured architect turn currently alive?
|
|
539
|
+
|
|
540
|
+
A remembered pid or lock can be reused after its process dies. Measure the
|
|
541
|
+
configured binary and cwd again instead; this still prevents two headless
|
|
542
|
+
watchdog helpers while releasing a dead one on the next scan. A wake the
|
|
543
|
+
journal holds counts too: one cut off between its start and its record
|
|
544
|
+
was a headless architect no record named (JOURNAL-2).
|
|
545
|
+
"""
|
|
546
|
+
return A.architect_turn_present(root, architect) or _architect_step_under_way(root)
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _architect_step_under_way(root):
|
|
550
|
+
"""Whether a wake or refill the journal holds is still under way: claimed and not yet started, or its process
|
|
551
|
+
alive and the one that started (JOURNAL-2)."""
|
|
552
|
+
try:
|
|
553
|
+
steps = J.under_way(A.project_file(root, "journal"), "architect")
|
|
554
|
+
except Exception:
|
|
555
|
+
return False
|
|
556
|
+
for _, pid, start in steps:
|
|
557
|
+
if pid is None:
|
|
558
|
+
return True # started, or cut off starting: the lease says how long to believe it
|
|
559
|
+
if _step_process_alive(pid, start):
|
|
560
|
+
return True
|
|
561
|
+
# Past its lease a step's process still holds while it is provably the one recorded: a long architect turn
|
|
562
|
+
# cut off before it was registered is no record's but the journal's.
|
|
563
|
+
try:
|
|
564
|
+
steps = J.running(A.project_file(root, "journal"), "architect")
|
|
565
|
+
except Exception:
|
|
566
|
+
return False
|
|
567
|
+
return any(_step_process_proven(pid, start) for _, pid, start in steps)
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _step_process_alive(pid, start):
|
|
571
|
+
"""Whether the process a step started still runs: its pid alive and its start the one recorded."""
|
|
572
|
+
return bool(pid) and A._pid_alive(pid) and (start is None or A._process_start(pid) in (None, start))
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _step_process_proven(pid, start):
|
|
576
|
+
"""Whether the process a step started still runs, its start recorded and read again the same: past a lease
|
|
577
|
+
an unreadable start proves nothing, and a reused pid never holds."""
|
|
578
|
+
return bool(pid) and start is not None and A._pid_alive(pid) and A._process_start(pid) == start
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _close_deferred_done(root, st):
|
|
582
|
+
"""Close a deferral the work it waited for has since done: a wake or a nudge started after it (JOURNAL-4).
|
|
583
|
+
|
|
584
|
+
Closed in the cycle after the work, not as the work starts: the notice a silence owes names what was
|
|
585
|
+
deferred through it, and that notice is written after the wake in the same cycle.
|
|
586
|
+
"""
|
|
587
|
+
done = {"wake": float(st.get("last_arch_wake") or 0), "nudge": float(st.get("last_nudge") or 0)}
|
|
588
|
+
closed = []
|
|
589
|
+
for row in A.deferred_open(root):
|
|
590
|
+
if done.get(row.get("kind"), 0) >= _deferred_since(row):
|
|
591
|
+
A.deferred_close(root, row["id"], "woken" if row.get("kind") == "wake" else "nudged")
|
|
592
|
+
closed.append(row)
|
|
593
|
+
return closed
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _deferred_since(row):
|
|
597
|
+
"""The earliest a start counts as after a deferral. A whole-second `at` was cut down to its second, and the
|
|
598
|
+
deferral was written before the next second began: work from that second on is after it (JOURNAL-4-3). A
|
|
599
|
+
fractional `at` is the moment itself, and only work strictly after it counts."""
|
|
600
|
+
import math
|
|
601
|
+
at = row.get("at") or 0
|
|
602
|
+
return float(at) + 1.0 if isinstance(at, int) else math.nextafter(float(at), math.inf)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _claim_step(root, step):
|
|
606
|
+
"""Whether this cycle may start `step`; where the journal cannot be read, it may, as before it (JOURNAL-2).
|
|
607
|
+
|
|
608
|
+
A step whose process has ended holds nothing: a wake that failed is retried after its fifteen minutes, as
|
|
609
|
+
the wake's own record says, whatever its lease. Only one cut off before its start was written, or one
|
|
610
|
+
still running, holds its key.
|
|
611
|
+
"""
|
|
612
|
+
path = A.project_file(root, "journal")
|
|
613
|
+
try:
|
|
614
|
+
current = J.step(path, step)
|
|
615
|
+
if current and current["state"] == "started":
|
|
616
|
+
if _step_process_proven(current["pid"], current["start"]):
|
|
617
|
+
return False # its process still runs: it holds its key past its lease
|
|
618
|
+
if not _step_process_alive(current["pid"], current["start"]):
|
|
619
|
+
J.finished(path, step, "ended")
|
|
620
|
+
return J.claim(path, step, "architect", WAKE_LEASE)
|
|
621
|
+
except Exception as exc:
|
|
622
|
+
print(f"the journal could not be read ({type(exc).__name__}); starting {step} on the process scan alone")
|
|
623
|
+
# Failing open is meant - a broken journal must not stop every wake - and a person is told that two
|
|
624
|
+
# wakes are now kept apart by the process scan alone (JOURNAL-2-2), unless the telling fails too: the step
|
|
625
|
+
# starts whatever the alarm does (JOURNAL-2-3).
|
|
626
|
+
try:
|
|
627
|
+
notify(f"{A.project_key(root)}: the watchdog's journal cannot be written",
|
|
628
|
+
f"{type(exc).__name__}: {exc}. Wakes and refills start on the process scan alone until it can.",
|
|
629
|
+
root, key="journal-unwritable", window=6 * 3600, audience="human")
|
|
630
|
+
except Exception as told:
|
|
631
|
+
print(f"the journal's alarm could not be raised ({type(told).__name__})")
|
|
632
|
+
return True
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _adopt_started_steps(root):
|
|
636
|
+
"""Register the architect a cut-off cycle started and never registered: the journal holds its pid and start.
|
|
637
|
+
|
|
638
|
+
Unregistered, it was no helper - the implementer's checks counted it as a turn in this tree and reaped it
|
|
639
|
+
as a hung one - and past its lease a second wake started beside it (JOURNAL-2).
|
|
640
|
+
"""
|
|
641
|
+
try:
|
|
642
|
+
steps = J.running(A.project_file(root, "journal"), "architect")
|
|
643
|
+
except Exception:
|
|
644
|
+
return
|
|
645
|
+
helpers = A.helper_pids(root, "architect")
|
|
646
|
+
for _, pid, start in steps:
|
|
647
|
+
if pid not in helpers and _step_process_proven(pid, start):
|
|
648
|
+
A.helper_register(root, pid, "architect")
|
|
649
|
+
print(f"the architect pid {pid} a cut-off cycle started is registered now")
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
def _record_start(root, step, pid):
|
|
653
|
+
"""The pid and start of a step's process, written the moment it has one (JOURNAL-2)."""
|
|
654
|
+
try:
|
|
655
|
+
J.started(A.project_file(root, "journal"), step, pid, A._process_start(pid, refresh=True))
|
|
656
|
+
except Exception as exc:
|
|
657
|
+
print(f"the journal did not take the start of {step} ({type(exc).__name__}); the process scan stands in")
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def _logs_written(root):
|
|
661
|
+
"""The spawn logs a process an earlier cycle started may still write, each left whole until it ends (JOURNAL-3-2).
|
|
662
|
+
|
|
663
|
+
Every spawn is made inside a cycle, under the cycle's lock, after the logs were bounded: what may write
|
|
664
|
+
one as it is bounded is a turn or a wake an earlier cycle started. Where that cannot be told, none of
|
|
665
|
+
the three is trimmed this cycle. A turn is the one the state names, or any unattended turn in this tree:
|
|
666
|
+
a cycle cut off between starting a turn and recording it left one no state names, which the process
|
|
667
|
+
scan that keeps a second turn from starting beside it still finds (JOURNAL-3-3).
|
|
668
|
+
"""
|
|
669
|
+
key = A.project_key(root)
|
|
670
|
+
log = lambda what: os.path.join(STATE_DIR, A.project_file_name(what, key)) # noqa: E731
|
|
671
|
+
try:
|
|
672
|
+
cfg = A.load_config(root)
|
|
673
|
+
impl = cfg.get("implementer") or {}
|
|
674
|
+
turn = child_alive(load_state(root)) or bool(
|
|
675
|
+
impl and A.agent_pids(root, A.load_adapter(impl.get("adapter", ""), root), headless_only=True))
|
|
676
|
+
busy = {log("nudge-log")} if turn else set()
|
|
677
|
+
if arch_alive(root, cfg.get("architect") or {}):
|
|
678
|
+
busy |= {log("escalate-log"), log("refill-log")}
|
|
679
|
+
return busy
|
|
680
|
+
except Exception:
|
|
681
|
+
return {log("nudge-log"), log("escalate-log"), log("refill-log")}
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def child_alive(st):
|
|
685
|
+
"""Is the turn we last started still running?
|
|
686
|
+
|
|
687
|
+
A transcript mtime cannot answer this. A turn retrying a provider 5xx sits in
|
|
688
|
+
backoff writing nothing, which looks exactly like a turn that ended — and
|
|
689
|
+
nudging then starts a SECOND turn on the same session id. Two turns editing
|
|
690
|
+
one tree produce a half-finished refactor: one renames a field, the other
|
|
691
|
+
still calls the old name, and the build breaks in a way neither writer can
|
|
692
|
+
attribute to itself. Ask the OS instead of inferring from a file.
|
|
693
|
+
"""
|
|
694
|
+
pid = st.get("child_pid")
|
|
695
|
+
if not pid:
|
|
696
|
+
return False
|
|
697
|
+
# A bare pid can be reused by an unrelated process, and os.kill(pid, 0) sends
|
|
698
|
+
# CTRL_C_EVENT on Windows. Liveness comes from the portable probe, identity from
|
|
699
|
+
# the start recorded when the turn was spawned.
|
|
700
|
+
if not A._pid_alive(pid):
|
|
701
|
+
return False
|
|
702
|
+
started = st.get("child_start")
|
|
703
|
+
now = A._process_start(pid) if started is not None else None
|
|
704
|
+
if now is not None:
|
|
705
|
+
return now == started
|
|
706
|
+
# No start was read when the turn was spawned, so the pid alone speaks for it, and a pid another process
|
|
707
|
+
# was given since kept the refill waiting for good. It is believed for as long as a turn may run
|
|
708
|
+
# (CYCLE-SAFETY-2). So is a turn whose start cannot be read now: a table the platform could not read
|
|
709
|
+
# holds none, and the turn read as ended while its pid ran (WINDOWS-TABLE-CLOSED).
|
|
710
|
+
return time.time() - float(st.get("last_nudge") or 0) < UNVERIFIED_CHILD_SECONDS
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
# How long a turn whose start could not be read is taken to be running while its pid is (CYCLE-SAFETY-2).
|
|
714
|
+
UNVERIFIED_CHILD_SECONDS = 6 * 3600
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def provider_degraded(root, window=900):
|
|
718
|
+
"""A provider-side failure in the *most recent* nudge, recent enough to hold.
|
|
719
|
+
|
|
720
|
+
An exhausted quota and an outage both silence the agent, but only one of them
|
|
721
|
+
is fixed by nudging. Naming it here keeps "the model is down" from being
|
|
722
|
+
reported — and retried — as "the agent is stuck".
|
|
723
|
+
|
|
724
|
+
Scope it to the last nudge attempt, not the whole file. The log is
|
|
725
|
+
append-only, so an hour-old outage stays in it forever, and a check that
|
|
726
|
+
merely greps the tail keeps firing long after the provider recovered — which
|
|
727
|
+
blocks every later nudge and costs exactly the time this guard was added to
|
|
728
|
+
save. Read only what came after the last `=== <time> nudge ===` marker, and
|
|
729
|
+
only if that marker itself is inside the window.
|
|
730
|
+
"""
|
|
731
|
+
key = A.project_key(root)
|
|
732
|
+
p = os.path.join(STATE_DIR, A.project_file_name("nudge-log", key))
|
|
733
|
+
if not os.path.exists(p):
|
|
734
|
+
return None
|
|
735
|
+
try:
|
|
736
|
+
size = os.path.getsize(p)
|
|
737
|
+
with open(p, errors="replace", encoding=UTF8) as fh:
|
|
738
|
+
fh.seek(max(0, size - 400_000))
|
|
739
|
+
tail = fh.read()
|
|
740
|
+
except OSError:
|
|
741
|
+
return None
|
|
742
|
+
marks = list(re.finditer(r"=== (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) nudge ===", tail))
|
|
743
|
+
if not marks:
|
|
744
|
+
return None
|
|
745
|
+
last = marks[-1]
|
|
746
|
+
try:
|
|
747
|
+
at = time.mktime(time.strptime(last.group(1), "%Y-%m-%d %H:%M:%S"))
|
|
748
|
+
except ValueError:
|
|
749
|
+
return None
|
|
750
|
+
if time.time() - at > window:
|
|
751
|
+
return None # the last attempt is old news
|
|
752
|
+
# Only the *end* of the segment counts. A 5xx the turn recovered from is
|
|
753
|
+
# history: this log had one at line 130 of 242, with a hundred lines of real
|
|
754
|
+
# work after it. Blocking on that would stand the watchdog down for fifteen
|
|
755
|
+
# minutes because of an error the agent had already handled.
|
|
756
|
+
segment = tail[last.end():].strip().split("\n")[-30:]
|
|
757
|
+
end = "\n".join(segment)
|
|
758
|
+
for marker in ("MODEL_TEMPORARILY_UNAVAILABLE", "ThrottlingException",
|
|
759
|
+
"ServiceUnavailableException", "InternalServerException",
|
|
760
|
+
"experiencing a high volume of traffic"):
|
|
761
|
+
if marker in end:
|
|
762
|
+
return marker
|
|
763
|
+
return None
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def architect_command(root, cfg, prompt):
|
|
767
|
+
"""Prepare a wake or refill before asking any program its version (ARCH-PROBE-ORDER).
|
|
768
|
+
|
|
769
|
+
The notice holder and doctor checked without the session and prompt values, so they ran version probes and
|
|
770
|
+
held notices for a wake that refused to start. All four callers use this order and these values; a refusal
|
|
771
|
+
returns (None, code, reason), a command (prepared, None, None). Preparation starts no program.
|
|
772
|
+
"""
|
|
773
|
+
arch = cfg.get("architect") or {}
|
|
774
|
+
template = arch.get("argv") or []
|
|
775
|
+
if not template:
|
|
776
|
+
return None, "no-architect", "no architect is configured"
|
|
777
|
+
sess, withheld = A.session_to_resume(cfg, "architect")
|
|
778
|
+
if "{session}" in " ".join(template) and not sess:
|
|
779
|
+
return None, "session-unresolved", f"architect session not resolvable ({withheld})"
|
|
780
|
+
# Keep the adapter's pinned mode, rewritten grants and prompt channel in every caller's check
|
|
781
|
+
# (ARCH-PROBE-ORDER).
|
|
782
|
+
template, pinned = A.pinned_argv(A.admit_rewrites(template)[0], "architect")
|
|
783
|
+
plan, refused = A.prompt_plan(template, prompt, A.block_adapter(arch), detached=True)
|
|
784
|
+
if refused:
|
|
785
|
+
return None, "prompt-refused", f"the architect's prompt cannot be handed over: {refused}"
|
|
786
|
+
argv = [x.replace("{prompt}", prompt).replace("{session}", sess or "") for x in plan["argv"]]
|
|
787
|
+
search = child_path()
|
|
788
|
+
held = turn_hold(root, cfg, "architect")
|
|
789
|
+
if held:
|
|
790
|
+
return None, "held", held
|
|
791
|
+
values = {"session": sess, "prompt": arch.get("prompt")}
|
|
792
|
+
refused = A.turn_refusal("architect", A.block_adapter(arch), argv, root=root, search=search, values=values,
|
|
793
|
+
resolved=argv[0] if os.path.isabs(argv[0]) else None, prompt=prompt)
|
|
794
|
+
if refused:
|
|
795
|
+
return None, "refused", refused
|
|
796
|
+
return {"argv": argv, "session": sess, "plan": plan, "pinned": pinned, "values": values,
|
|
797
|
+
"search": search, "path": A.outside_repository(search, root)}, None, None
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def architect_hold_reason(root, cfg, adapter, st, found=None, now=None):
|
|
801
|
+
"""May a notice be held for the architect? {"holdable", "code", "reason"}.
|
|
802
|
+
|
|
803
|
+
A notice addressed to the architect reaches no person, so holding it is honest
|
|
804
|
+
only when an architect is going to read it: a wake already in flight, or one
|
|
805
|
+
this cycle will start. The rule this replaces held whenever an architect argv
|
|
806
|
+
was configured, and on 2026-09-14 seven notices in a row sat held behind
|
|
807
|
+
"architect will be woken" while the only architect was an interactive session
|
|
808
|
+
that acts when someone prompts it.
|
|
809
|
+
|
|
810
|
+
Built from the predicates the wake in escalate() uses; once its command is
|
|
811
|
+
accepted, resolving it asks candidates their version (ARCH-PROBE-ORDER).
|
|
812
|
+
Blockers are checked before in-flight wakes, so a wake that has already
|
|
813
|
+
failed is never counted as one that is coming.
|
|
814
|
+
|
|
815
|
+
`adapter` is the one the quota is read through, A.metered_adapter's (TRIAGE-HIGHS).
|
|
816
|
+
"""
|
|
817
|
+
from . import features as F
|
|
818
|
+
now = now or time.time()
|
|
819
|
+
arch = cfg.get("architect") or {}
|
|
820
|
+
argv = arch.get("argv") or []
|
|
821
|
+
|
|
822
|
+
def no(code, reason):
|
|
823
|
+
return {"holdable": False, "code": code, "reason": reason}
|
|
824
|
+
|
|
825
|
+
if not argv:
|
|
826
|
+
return no("no-architect", "no architect is configured")
|
|
827
|
+
try:
|
|
828
|
+
present = A.architect_present(root, arch)
|
|
829
|
+
except TableUnreadable as exc:
|
|
830
|
+
# No wake starts beside a process nobody could see, so a person is told what waits (WINDOWS-TABLE-CLOSED).
|
|
831
|
+
return no("process-table", f"{exc}, so no architect is woken")
|
|
832
|
+
if present:
|
|
833
|
+
return no("interactive", "the architect session is interactive and acts only when someone prompts it")
|
|
834
|
+
if not F.enabled(cfg, "architect_wake"):
|
|
835
|
+
return no("wake-off", "architect wakes are switched off")
|
|
836
|
+
if not quota_ok(adapter):
|
|
837
|
+
return no("no-quota", "there is no quota headroom to wake the architect")
|
|
838
|
+
if st.get("arch_quota_until", 0) > now:
|
|
839
|
+
return no("quota-block", "the architect is at quota until " + reset_when(st["arch_quota_until"], now))
|
|
840
|
+
try:
|
|
841
|
+
running = arch_alive(root, arch)
|
|
842
|
+
except TableUnreadable as exc:
|
|
843
|
+
return no("process-table", f"{exc}, so no architect is woken")
|
|
844
|
+
if running:
|
|
845
|
+
return {"holdable": True, "code": "wake-running", "reason": "an architect wake is already running"}
|
|
846
|
+
provider = A.provider_of(argv)
|
|
847
|
+
left, reserve = A.window_headroom(provider)
|
|
848
|
+
urgent = any(item.get("kind") == "decision-requested" for item in (found or []))
|
|
849
|
+
if left is not None and left < reserve and not urgent:
|
|
850
|
+
return no("window-reserve", f"the machine's {provider} window has {left}% left, below the {reserve}% reserve")
|
|
851
|
+
command, code, stopped = architect_command(root, cfg, language.text(cfg, "prompt.wake"))
|
|
852
|
+
if stopped:
|
|
853
|
+
return no(code, stopped)
|
|
854
|
+
resolved, ver = A.resolve_binary(command["argv"][0], path=command["path"])
|
|
855
|
+
if not resolved:
|
|
856
|
+
return no("binary-missing", f"{argv[0]} cannot be found")
|
|
857
|
+
key = A.project_key(root)
|
|
858
|
+
err = wake_error(os.path.join(STATE_DIR, A.project_file_name("escalate-log", key)))
|
|
859
|
+
if err and err.get("kind") == "binary" and err.get("binary") == f"{resolved} {ver}" \
|
|
860
|
+
and now - (st.get("wake_error") or {}).get("at", 0) < 6 * 3600:
|
|
861
|
+
return no("binary-failed", f"the last wake failed with this binary: {err.get('text', '')[:80]}")
|
|
862
|
+
if now - st.get("last_arch_wake", 0) < 900:
|
|
863
|
+
return {"holdable": True, "code": "recent-wake",
|
|
864
|
+
"reason": f"the architect was woken {int((now - st.get('last_arch_wake', 0)) / 60)}m ago"}
|
|
865
|
+
return {"holdable": True, "code": "wakeable", "reason": "architect will be woken"}
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def escalate(root, cfg, adapter, age, args, st, told=None):
|
|
869
|
+
"""Hand every judgement call to the architect, once per condition per hour.
|
|
870
|
+
|
|
871
|
+
The watchdog is good at mechanical questions and bad at everything else. Its
|
|
872
|
+
worst failures came from answering the second kind anyway: a provider outage
|
|
873
|
+
read as a stuck agent, a hung process read as a live writer, a finished slice
|
|
874
|
+
read as work in progress. Each was defensible from the one signal it had and
|
|
875
|
+
wrong given the others.
|
|
876
|
+
|
|
877
|
+
So it reports instead. Facts go to the mailbox where they persist; the
|
|
878
|
+
architect is woken only if one is configured, because a report nobody reads is
|
|
879
|
+
not an escalation.
|
|
880
|
+
|
|
881
|
+
`told`, when given, collects the reports an alarm to a person stands for this cycle, so
|
|
882
|
+
the unseen check does not ring them under a second key (CATCHUP-POLISH).
|
|
883
|
+
"""
|
|
884
|
+
# A wake retried after a failed one is told once it is known not to have failed (NOISE-REPEATS).
|
|
885
|
+
tell_retried_wake(root, cfg, st, dry_run=args.dry_run)
|
|
886
|
+
# The architect we spawned resumes with this repo as its cwd, so it reads as a
|
|
887
|
+
# second turn to the process check. We know its pid; exclude it and our own
|
|
888
|
+
# rather than making the detector guess.
|
|
889
|
+
found = A.anomalies(root, cfg, adapter, age, args.idle_minutes * 60,
|
|
890
|
+
exclude_pids={st.get("arch_pid"), os.getpid()})
|
|
891
|
+
# A wake spends the quota the package's adapter reads by its own command; `adapter`, as every layer loads it,
|
|
892
|
+
# says only how the transcript reads (TRIAGE-HIGHS).
|
|
893
|
+
metered = A.metered_adapter(cfg)
|
|
894
|
+
|
|
895
|
+
# A second turn is a hazard only if it persists. A real concurrent writer runs
|
|
896
|
+
# for minutes; a gate helper, a `polyctl` call or a one-shot process blinks in and
|
|
897
|
+
# out and trips a single-scan check for one cycle. Require the same extra roots
|
|
898
|
+
# on two consecutive scans before flagging.
|
|
899
|
+
prev_roots = set(st.get("last_roots") or [])
|
|
900
|
+
seen_several = False
|
|
901
|
+
kept = []
|
|
902
|
+
for a in found:
|
|
903
|
+
if a["kind"] == "several-turns-active":
|
|
904
|
+
seen_several = True
|
|
905
|
+
roots = set(a.get("roots") or [])
|
|
906
|
+
st["last_roots"] = list(roots)
|
|
907
|
+
if len(roots) < 2 or not (roots & prev_roots):
|
|
908
|
+
print(f"several turns seen once ({sorted(roots)}); waiting for a "
|
|
909
|
+
f"second scan before flagging")
|
|
910
|
+
continue
|
|
911
|
+
kept.append(a)
|
|
912
|
+
if not seen_several:
|
|
913
|
+
st["last_roots"] = []
|
|
914
|
+
save_state(root, st)
|
|
915
|
+
found = kept
|
|
916
|
+
if not found:
|
|
917
|
+
return False
|
|
918
|
+
|
|
919
|
+
woke = False
|
|
920
|
+
project = A.project_key(root)
|
|
921
|
+
hold = None
|
|
922
|
+
standing = {}
|
|
923
|
+
asked = {decision.get("id"): decision for decision in A.decisions(root, "open")}
|
|
924
|
+
for a in found:
|
|
925
|
+
key = f"anomaly:{a['kind']}"
|
|
926
|
+
# A handed decision must not suppress the next decision's report (TRIAGE-3-MEDIUM).
|
|
927
|
+
if a.get("key") and a["key"] != "implementer":
|
|
928
|
+
key += ":" + str(a["key"])
|
|
929
|
+
# An open decision reaches a person under its own alarm, `decision-open:<id>`, which names its
|
|
930
|
+
# question and rings whatever the architect does (#20); it is not told as "needs you" as well.
|
|
931
|
+
decision = asked.get(a.get("key")) if a["kind"] == "decision-requested" else None
|
|
932
|
+
alarm = f"decision-open:{decision['id']}" if decision else key
|
|
933
|
+
window = 600 if a["kind"] == "decision-requested" else 3600
|
|
934
|
+
if a["kind"] == "report-waiting" and not open_work(cfg, root):
|
|
935
|
+
continue # nothing is stuck; it can wait for a human
|
|
936
|
+
# Grouped implementer reports share an alarm; individual decisions and reviews each
|
|
937
|
+
# speak under their own key, so an earlier instance cannot hold the next (TRIAGE-3-MEDIUM).
|
|
938
|
+
standing.setdefault(alarm, a)
|
|
939
|
+
# Suppression is read-only and therefore part of explain's decision path.
|
|
940
|
+
# A dry-run must not claim it would write a report that a live cycle would
|
|
941
|
+
# suppress; it merely skips the suppressed-notice ledger write.
|
|
942
|
+
# An architect-audience notice is recorded unsent, so a check on sent rows
|
|
943
|
+
# never held and the same anomaly wrote a row every cycle (#69). Any row
|
|
944
|
+
# inside the window stops the repeat; the report and that row are the record.
|
|
945
|
+
if A.notice_recently_recorded(root, key, window):
|
|
946
|
+
if args.dry_run:
|
|
947
|
+
print(f"DRY RUN: would suppress anomaly {a['kind']} "
|
|
948
|
+
f"(reported within {window}s)")
|
|
949
|
+
continue
|
|
950
|
+
if args.dry_run:
|
|
951
|
+
print(f"DRY RUN: would report anomaly {a['kind']} to the architect")
|
|
952
|
+
woke = True
|
|
953
|
+
continue
|
|
954
|
+
name = A.write_report(root, cfg, a["kind"], a["facts"], key=a.get("key"))
|
|
955
|
+
# Say what actually happened. "reported to the architect" was untrue
|
|
956
|
+
# whenever no architect was configured or startable, and a notification
|
|
957
|
+
# that overstates its own effect is how a gap stays invisible.
|
|
958
|
+
# Hold for the architect only when a wake is actually coming; decided from the
|
|
959
|
+
# wake's own predicates, not from whether an argv is configured.
|
|
960
|
+
hold = hold or architect_hold_reason(root, cfg, metered, st, found)
|
|
961
|
+
if hold["holdable"]:
|
|
962
|
+
notify(f"{project}: anomaly", f"{a['kind']} — {hold['reason']}", root,
|
|
963
|
+
key=key, window=window, audience="architect")
|
|
964
|
+
elif decision:
|
|
965
|
+
# Its own check rings it once it has waited `decisions.human_after_minutes`; before that, and
|
|
966
|
+
# with no architect to act, it rings from here. A raise inside the window is not repeated.
|
|
967
|
+
if not A.notice_recently_recorded(root, alarm, window):
|
|
968
|
+
ring_decision(root, project, decision, why=hold["reason"])
|
|
969
|
+
else:
|
|
970
|
+
# What stands is what waits: rung once, it climbs to red without ringing again
|
|
971
|
+
# until another request waits. A quota block names its own end (NOTICE-NOISE).
|
|
972
|
+
# It names the reports, since when and why nobody is woken: with wakes off it
|
|
973
|
+
# is the only alarm for them (WAITING-ONE-ALARM).
|
|
974
|
+
notify(f"{project}: needs you",
|
|
975
|
+
f"{anomaly_waits(root, cfg, a)} — no architect will act on it: {hold['reason']}", root,
|
|
976
|
+
key=key, window=window, audience="human", what=anomaly_subject(a),
|
|
977
|
+
quiet_until=st.get("arch_quota_until") if hold["code"] == "quota-block" else None)
|
|
978
|
+
print(f"anomaly {a['kind']}: {'reported as ' + name if name else 'already reported'}")
|
|
979
|
+
woke = True
|
|
980
|
+
if not args.dry_run and told is not None and standing:
|
|
981
|
+
# An anomaly no architect will act on reaches a person as "needs you", and stands for the
|
|
982
|
+
# requests it groups whether or not its window let it ring this cycle (CATCHUP-POLISH).
|
|
983
|
+
# A dry cycle delivered no alarm: checking its coverage prepared and probed a wake before
|
|
984
|
+
# building the implementer's nudge (ARCH-PROBE-ORDER-2).
|
|
985
|
+
hold = hold or architect_hold_reason(root, cfg, metered, st, found)
|
|
986
|
+
if not hold["holdable"]:
|
|
987
|
+
told.update(name for anomaly in standing.values() for name in anomaly.get("reports") or [])
|
|
988
|
+
arch = cfg.get("architect") or {}
|
|
989
|
+
|
|
990
|
+
# Wake on durable state, not on the moment of detection. The notification
|
|
991
|
+
# throttle exists so a human is not rung every two minutes; using it to gate
|
|
992
|
+
# the wake as well meant the architect was only ever started on an anomaly's
|
|
993
|
+
# *first* sighting, and a single suppressed cycle lost it for good. What
|
|
994
|
+
# actually matters is whether a report is still sitting unprocessed — the
|
|
995
|
+
# mailbox is the state, the notification is only a bell.
|
|
996
|
+
# Same exclusion as the anomaly scanner: the watchdog's own reports are
|
|
997
|
+
# addressed to the architect but are not themselves reports awaiting the
|
|
998
|
+
# architect, and counting them here re-woke it endlessly on its own output.
|
|
999
|
+
# Addressed by role, never by an actor's name, and the watchdog's own anomaly
|
|
1000
|
+
# reports count: an anomaly nobody is woken for is not escalated (#69). What
|
|
1001
|
+
# keeps them from re-waking the architect on its own output is the handed set
|
|
1002
|
+
# below, not their names. The architect's notes to itself wake nobody (#18).
|
|
1003
|
+
pending = [m for m in A.mailbox(root, cfg["mailbox"])
|
|
1004
|
+
if A.to_architect(m, cfg) and not A.from_architect(m, cfg)]
|
|
1005
|
+
# Closing the loop out loud: an alert that says a thing was detected, with no
|
|
1006
|
+
# later word on whether anything came of it, is what makes someone check by
|
|
1007
|
+
# hand — which is the work the alert was supposed to save.
|
|
1008
|
+
if st.get("arch_pending") and not pending:
|
|
1009
|
+
if args.dry_run:
|
|
1010
|
+
print(f"DRY RUN: would announce {st['arch_pending']} completed architect report(s)")
|
|
1011
|
+
else:
|
|
1012
|
+
_tell_phone(root, language.text(cfg, "watchdog.architect-done", n=st["arch_pending"]))
|
|
1013
|
+
st["arch_pending"] = 0
|
|
1014
|
+
save_state(root, st)
|
|
1015
|
+
elif pending and not args.dry_run:
|
|
1016
|
+
st["arch_pending"] = len(pending)
|
|
1017
|
+
last_wake = st.get("last_arch_wake", 0)
|
|
1018
|
+
# Wake on the durable state (#69). A report is owed a wake until one that did
|
|
1019
|
+
# not fail has been handed it; comparing its time with the last wake dropped it
|
|
1020
|
+
# for good whenever that wake died at once. A wake whose log shows a transport
|
|
1021
|
+
# failure hands nothing back. A model's own prose saying "Error:" does not count.
|
|
1022
|
+
handed = dict(st.get("handed") or {})
|
|
1023
|
+
if handed and wake_failed(wake_error(os.path.join(STATE_DIR, A.project_file_name("escalate-log", project))),
|
|
1024
|
+
last_wake):
|
|
1025
|
+
handed = {}
|
|
1026
|
+
mtimes = {}
|
|
1027
|
+
for m in pending:
|
|
1028
|
+
try:
|
|
1029
|
+
# Whole seconds missed a report rewritten in the same second it was handed (TRIAGE-3-LOW).
|
|
1030
|
+
info = os.stat(os.path.join(root, cfg["mailbox"], m))
|
|
1031
|
+
mtimes[m] = [info.st_mtime_ns, info.st_size]
|
|
1032
|
+
except OSError:
|
|
1033
|
+
continue # acknowledged while this cycle ran
|
|
1034
|
+
stale = [m for m in pending if m in mtimes and handed.get(m) != mtimes[m]]
|
|
1035
|
+
if not args.dry_run and handed != (st.get("handed") or {}):
|
|
1036
|
+
st["handed"] = {m: t for m, t in handed.items() if m in mtimes}
|
|
1037
|
+
save_state(root, st)
|
|
1038
|
+
woke = bool(stale)
|
|
1039
|
+
# What waits, as the notices below name it: the newest report that asks, not the watchdog's
|
|
1040
|
+
# echo of it. A person is told again when it changes (NOTICE-NOISE).
|
|
1041
|
+
waiting = max(stale, key=lambda m: (not A.from_watchdog(m), mtimes.get(m, 0), m)) if stale else None
|
|
1042
|
+
if woke and time.time() - last_wake < 900:
|
|
1043
|
+
print(f"{len(stale)} unhandled report(s), but the architect was woken "
|
|
1044
|
+
f"{int((time.time() - last_wake) / 60)}m ago")
|
|
1045
|
+
woke = False
|
|
1046
|
+
|
|
1047
|
+
# Waking spends a turn on the same provider the implementer uses, so a report
|
|
1048
|
+
# that could have waited must not burn the window the implementer needs.
|
|
1049
|
+
if woke and arch.get("argv") and not quota_ok(metered):
|
|
1050
|
+
print("reports pending, but no quota headroom to wake the architect")
|
|
1051
|
+
# One handoff for what waits: the same note went to the phone again every hour the
|
|
1052
|
+
# architect could not be woken, with nothing new in it (NOTICE-NOISE).
|
|
1053
|
+
if not args.dry_run and st.get("handoff_for") != waiting and time.time() - st.get("last_handoff", 0) > 3600:
|
|
1054
|
+
try:
|
|
1055
|
+
exe = A.program_path("polyctl", path=child_path())
|
|
1056
|
+
if exe:
|
|
1057
|
+
subprocess.run([exe, "-C", root, "handoff", "--reason",
|
|
1058
|
+
language.text(cfg, "watchdog.handoff-no-wake")] + _handoff_quiet(),
|
|
1059
|
+
capture_output=True, timeout=120)
|
|
1060
|
+
st.update(last_handoff=time.time(), handoff_for=waiting)
|
|
1061
|
+
save_state(root, st)
|
|
1062
|
+
except Exception:
|
|
1063
|
+
pass
|
|
1064
|
+
woke = False
|
|
1065
|
+
|
|
1066
|
+
# Wake into absence, never alongside. A live architect does not need a copy of
|
|
1067
|
+
# itself: the copy inherits the conversation in progress and continues that
|
|
1068
|
+
# rather than the triage it was started for.
|
|
1069
|
+
if woke and A.architect_present(root, arch):
|
|
1070
|
+
print("reports pending, but the architect is already at the keyboard")
|
|
1071
|
+
# Suppressing the wake must not silence the reports (#23): the session acts
|
|
1072
|
+
# when someone prompts it, so tell that someone. One condition rings one alarm (#106): a
|
|
1073
|
+
# report an anomaly stands for is that anomaly's, which names it and says the session is
|
|
1074
|
+
# interactive, and telling it here as well mailed one request twice on every red repeat.
|
|
1075
|
+
# This alarm names the rest, as reports-no-wake does with wakes off, and rings again when
|
|
1076
|
+
# another report waits (NOISE-REPEATS).
|
|
1077
|
+
hold = hold or architect_hold_reason(root, cfg, metered, st, found)
|
|
1078
|
+
named = set() if hold["holdable"] else anomaly_reports(standing.values(), stale)
|
|
1079
|
+
rest = [m for m in stale if m not in named]
|
|
1080
|
+
if told is not None:
|
|
1081
|
+
told.update(rest)
|
|
1082
|
+
if named:
|
|
1083
|
+
print(f"{len(named)} report(s) named by the alarm of the anomaly they stand for; "
|
|
1084
|
+
f"{len(rest)} other(s) alarmed here")
|
|
1085
|
+
if rest and not args.dry_run:
|
|
1086
|
+
# The report that asks, not the watchdog's echo of it written this cycle: mtimes are whole
|
|
1087
|
+
# seconds, so the echo won only when the cycle crossed a second.
|
|
1088
|
+
newest = max(rest, key=lambda m: (not A.from_watchdog(m), mtimes.get(m, 0), m))
|
|
1089
|
+
notify(f"{project}: reports wait for the architect",
|
|
1090
|
+
f"{waiting_reports(root, cfg, rest, newest)} — the architect session is interactive and "
|
|
1091
|
+
f"reads {'it' if len(rest) == 1 else 'them'} when prompted",
|
|
1092
|
+
root, key="present-pending", window=3600, audience="human", what=newest)
|
|
1093
|
+
woke = False
|
|
1094
|
+
from . import features as F
|
|
1095
|
+
if woke and not F.enabled(cfg, "architect_wake"):
|
|
1096
|
+
print("reports pending; architect_wake feature is off — recorded and alarmed, not woken")
|
|
1097
|
+
# One condition rings one alarm, whoever sees it (#106). A report an anomaly stands for -
|
|
1098
|
+
# the request it groups, or the watchdog's own report of it - is already a person's
|
|
1099
|
+
# "needs you", which names it and says why nobody is woken; ringing it here too sent a
|
|
1100
|
+
# day's red mail about one request twice. This alarm names the rest: a report that asks
|
|
1101
|
+
# nothing while no work is open, a lead, an anomaly's report whose condition has ended.
|
|
1102
|
+
# With wakes off no anomaly is held for the architect, so every standing one reached a
|
|
1103
|
+
# person (WAITING-ONE-ALARM).
|
|
1104
|
+
hold = hold or architect_hold_reason(root, cfg, metered, st, found)
|
|
1105
|
+
named = set() if hold["holdable"] else anomaly_reports(standing.values(), stale)
|
|
1106
|
+
rest = [m for m in stale if m not in named]
|
|
1107
|
+
if told is not None:
|
|
1108
|
+
told.update(rest)
|
|
1109
|
+
if named:
|
|
1110
|
+
print(f"{len(named)} report(s) named by the alarm of the anomaly they stand for; "
|
|
1111
|
+
f"{len(rest)} other(s) alarmed here")
|
|
1112
|
+
if rest:
|
|
1113
|
+
newest = max(rest, key=lambda m: (not A.from_watchdog(m), mtimes.get(m, 0), m))
|
|
1114
|
+
notify(f"{project}: needs you",
|
|
1115
|
+
f"{waiting_reports(root, cfg, rest, newest)} — no architect will act on "
|
|
1116
|
+
f"{'it' if len(rest) == 1 else 'them'}: {hold['reason']}",
|
|
1117
|
+
root, key="reports-no-wake", window=3600, audience="human", what=newest)
|
|
1118
|
+
woke = False
|
|
1119
|
+
if woke:
|
|
1120
|
+
provider = A.provider_of(arch.get("argv"))
|
|
1121
|
+
left, reserve = A.window_headroom(provider)
|
|
1122
|
+
urgent = any(a.get("kind") == "decision-requested" for a in (found or []))
|
|
1123
|
+
if left is not None and left < reserve and not urgent:
|
|
1124
|
+
print(f"reports pending, but the machine's {provider} window has {left}% left (< reserve {reserve}%); not waking")
|
|
1125
|
+
woke = False
|
|
1126
|
+
if woke and arch_alive(root, arch):
|
|
1127
|
+
print("reports pending, but an architect wake is already running")
|
|
1128
|
+
woke = False
|
|
1129
|
+
if woke and arch.get("argv") and not args.dry_run:
|
|
1130
|
+
prompt = language.text(cfg, "prompt.wake")
|
|
1131
|
+
command, code, stopped = architect_command(root, cfg, prompt)
|
|
1132
|
+
if stopped:
|
|
1133
|
+
print(f"{stopped}; not waking the architect")
|
|
1134
|
+
if code == "refused":
|
|
1135
|
+
tell_turn_refused(root, "architect", stopped)
|
|
1136
|
+
return woke
|
|
1137
|
+
sess, plan, pinned = command["session"], command["plan"], command["pinned"]
|
|
1138
|
+
argv, search = command["argv"], command["search"]
|
|
1139
|
+
resolved, ver = A.resolve_binary(argv[0], path=command["path"])
|
|
1140
|
+
key = A.project_key(root)
|
|
1141
|
+
log_path = os.path.join(STATE_DIR, A.project_file_name("escalate-log", key))
|
|
1142
|
+
# Read what the previous wake said before starting another. Same binary,
|
|
1143
|
+
# same error, less than six hours old: the human has been told, and a
|
|
1144
|
+
# retry is the forty-first identical failure.
|
|
1145
|
+
err = wake_error(log_path)
|
|
1146
|
+
if resolved and err:
|
|
1147
|
+
text, used, when, kind = err["text"], err["binary"], err["when"], err["kind"]
|
|
1148
|
+
# The log merges the model's prose with its errors; mask before it is kept or sent.
|
|
1149
|
+
text = A.redact(text)
|
|
1150
|
+
prev = st.get("wake_error") or {}
|
|
1151
|
+
fresh = prev.get("text") != text or prev.get("binary") != used or prev.get("when") != when
|
|
1152
|
+
if fresh:
|
|
1153
|
+
st["wake_error"] = {"at": time.time(), "text": text, "binary": used,
|
|
1154
|
+
"when": when, "kind": kind}
|
|
1155
|
+
if kind == "quota":
|
|
1156
|
+
# The architect is paused, not broken. Wait for the window, tell
|
|
1157
|
+
# the human once (orange), and remember that the desktop app may
|
|
1158
|
+
# resume the session itself when its auto-continue is on. The window's
|
|
1159
|
+
# end is read as the implementer's quota handling reads it (ARCHITECT-WAKE-QUOTA).
|
|
1160
|
+
until = architect_quota_until(err, argv)
|
|
1161
|
+
if until:
|
|
1162
|
+
st["arch_quota_until"] = until
|
|
1163
|
+
A.deferred_append(root, "wake", reason="architect quota", until=until)
|
|
1164
|
+
save_state(root, st)
|
|
1165
|
+
# notify records the notice; a row written first under the same key
|
|
1166
|
+
# made notify's own rate limit swallow the ring (#69).
|
|
1167
|
+
if kind == "quota":
|
|
1168
|
+
touch_architect_quota(root, st, cfg)
|
|
1169
|
+
else:
|
|
1170
|
+
# Every retry reads a failure with a new time, so a window alone rang the same 529
|
|
1171
|
+
# again every six hours: rung once for what it says, a new failure is told (NOISE-REPEATS).
|
|
1172
|
+
notify(language.text(cfg, "alarm.wake-failed-title", project=key),
|
|
1173
|
+
language.text(cfg, "alarm.wake-failed", kind=kind, error=text[:110], binary=used or "?"),
|
|
1174
|
+
root, key="architect-wake-failed",
|
|
1175
|
+
window=6 * 3600, audience="human", what=wake_failure_told(err))
|
|
1176
|
+
same = used == f"{resolved} {ver}"
|
|
1177
|
+
if kind == "binary" and same and time.time() - (st.get("wake_error") or {}).get("at", 0) < 6 * 3600:
|
|
1178
|
+
print(f"architect wake failed with this same binary ({used}); not retrying: {text[:90]}")
|
|
1179
|
+
resolved = None
|
|
1180
|
+
elif kind == "quota" and st.get("arch_quota_until", 0) > time.time():
|
|
1181
|
+
print(f"architect at quota until {reset_when(st['arch_quota_until'])}; not waking")
|
|
1182
|
+
resolved = None
|
|
1183
|
+
elif kind == "session":
|
|
1184
|
+
# A dead session id: resume a different one or none. Setting the config
|
|
1185
|
+
# back to auto after argv was built resumed the same dead session (#69).
|
|
1186
|
+
dead = st.get("arch_session")
|
|
1187
|
+
if sess and (sess == dead or sess in text):
|
|
1188
|
+
other = A.resolve_session(root, cfg, "architect", avoid={sess})
|
|
1189
|
+
found = other["session"] if other["trusted"] else None
|
|
1190
|
+
if found and found != sess and found not in text:
|
|
1191
|
+
sess = found
|
|
1192
|
+
if other["record"]:
|
|
1193
|
+
A.record_session(root, "architect", other)
|
|
1194
|
+
argv = [x.replace("{prompt}", prompt).replace("{session}", sess)
|
|
1195
|
+
for x in plan["argv"]]
|
|
1196
|
+
print(f"last wake resumed a dead session; resuming {sess[:12]} instead")
|
|
1197
|
+
else:
|
|
1198
|
+
print("last wake resumed a dead session and no other session was found; not waking")
|
|
1199
|
+
resolved = None
|
|
1200
|
+
if resolved and st.get("arch_quota_until", 0) > time.time():
|
|
1201
|
+
print(f"architect at quota until {reset_when(st['arch_quota_until'])}; not waking")
|
|
1202
|
+
resolved = None
|
|
1203
|
+
# The command a wake starts runs the harness of an adapter the package ships, with no grant beyond the one polyctl
|
|
1204
|
+
# composes from it, before anything is spent on it (TURN-ARGV).
|
|
1205
|
+
refused = A.turn_refusal("architect", A.block_adapter(arch), argv, root=root, resolved=resolved,
|
|
1206
|
+
search=search, values=dict(command["values"], session=sess), prompt=prompt) if resolved else None
|
|
1207
|
+
if refused:
|
|
1208
|
+
print(f"{refused}; not waking the architect")
|
|
1209
|
+
tell_turn_refused(root, "architect", refused)
|
|
1210
|
+
resolved = None
|
|
1211
|
+
if resolved and not args.dry_run:
|
|
1212
|
+
# An exhausted window is rotated through keyflip first, when it may be (#32).
|
|
1213
|
+
headroom = A.rotate_if_exhausted(cfg, argv, "architect")
|
|
1214
|
+
if not headroom["ok"]:
|
|
1215
|
+
print(f"{headroom['text']}; not waking the architect")
|
|
1216
|
+
notify(f"{key}: no headroom", headroom["text"] + " — a person decides", root,
|
|
1217
|
+
key=f"no-headroom:{headroom['provider']}", window=6 * 3600, audience="human")
|
|
1218
|
+
resolved = None
|
|
1219
|
+
# Read after the rotation, which can wait minutes on keyflip, as the nudge and the refill read it (JOURNAL-6).
|
|
1220
|
+
held = A.hold_state(root)
|
|
1221
|
+
if resolved and held:
|
|
1222
|
+
print(f"held by {held.get('by')} since this cycle began; not waking the architect")
|
|
1223
|
+
resolved = None
|
|
1224
|
+
# The architect's harness reads its hooks, plugins and instructions from this tree, as late as the hold
|
|
1225
|
+
# is read (HARNESS-CONFIG), and its command is the one a person accepted (TURN-ARGV).
|
|
1226
|
+
unseen = (harness_config_hold(root) or turn_hold(root, cfg, "architect")) if resolved else None
|
|
1227
|
+
if unseen:
|
|
1228
|
+
print(f"{unseen}; not waking the architect")
|
|
1229
|
+
resolved = None
|
|
1230
|
+
if resolved:
|
|
1231
|
+
argv[0] = resolved
|
|
1232
|
+
if not record_pinned(root, pinned):
|
|
1233
|
+
return woke
|
|
1234
|
+
given, refused = A.prompt_input(plan, root, argv)
|
|
1235
|
+
if refused:
|
|
1236
|
+
print(f"the architect's prompt cannot be handed over: {refused}; not waking")
|
|
1237
|
+
return woke
|
|
1238
|
+
# A wake after one that failed in the last day is a retry: it was the same line to the phone
|
|
1239
|
+
# every fifteen minutes while a transport kept failing, and none of them was a woken architect.
|
|
1240
|
+
retried = wake_failed(err, last_wake) and time.time() - float(err.get("at") or 0) < 24 * 3600
|
|
1241
|
+
# Claimed before its process starts, under the session and the reports it is handed (JOURNAL-2).
|
|
1242
|
+
step = "wake:{}:{}".format(sess or "new", A.hashlib.sha256("\n".join(sorted(pending)).encode(UTF8))
|
|
1243
|
+
.hexdigest()[:12])
|
|
1244
|
+
if sess and A.session_in_use(sess, arch):
|
|
1245
|
+
print(f"the architect's session {sess} is resumed by a running process already; not waking")
|
|
1246
|
+
return woke
|
|
1247
|
+
if not _claim_step(root, step):
|
|
1248
|
+
print("a wake for these reports was started and is not known to have ended; not waking again")
|
|
1249
|
+
return woke
|
|
1250
|
+
os.makedirs(STATE_DIR, exist_ok=True)
|
|
1251
|
+
with open(log_path, "a", encoding=UTF8) as log:
|
|
1252
|
+
log.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} escalate {resolved} {ver} ===\n")
|
|
1253
|
+
log.flush()
|
|
1254
|
+
# stdin must be closed, not inherited. A detached child holding a
|
|
1255
|
+
# pipe or tty it will never read can wait forever for an EOF that
|
|
1256
|
+
# never comes: alive, silent, producing nothing. That is the exact
|
|
1257
|
+
# shape of the first architect wake — over a minute, no output —
|
|
1258
|
+
# and very likely of the fifteen agent processes this project
|
|
1259
|
+
# found accumulated in one repository. A prompt handed over on
|
|
1260
|
+
# standard input is a file, which ends.
|
|
1261
|
+
try:
|
|
1262
|
+
proc = subprocess.Popen(given["argv"], cwd=root,
|
|
1263
|
+
env=dict(os.environ, PATH=search, POLYCTL_ROLE="architect"),
|
|
1264
|
+
stdin=subprocess.DEVNULL if given["stdin"] is None else given["stdin"],
|
|
1265
|
+
stdout=log, stderr=subprocess.STDOUT,
|
|
1266
|
+
start_new_session=True)
|
|
1267
|
+
finally:
|
|
1268
|
+
A.release_prompt(given)
|
|
1269
|
+
_record_start(root, step, proc.pid)
|
|
1270
|
+
st["arch_pid"] = proc.pid
|
|
1271
|
+
st["last_arch_wake"] = time.time()
|
|
1272
|
+
st["handed"] = {m: mtimes[m] for m in pending if m in mtimes}
|
|
1273
|
+
st["arch_session"] = sess
|
|
1274
|
+
st.pop("wake_retry", None)
|
|
1275
|
+
if retried:
|
|
1276
|
+
# Told by tell_retried_wake once a later cycle reads that it did not fail (NOISE-REPEATS).
|
|
1277
|
+
st["wake_retry"] = {"at": st["last_arch_wake"], "pid": proc.pid, "reports": len(stale)}
|
|
1278
|
+
A.helper_register(root, proc.pid, "architect") # a judge, not a writer
|
|
1279
|
+
A.acquire_architect(root, proc.pid, "watchdog wake") # one judge at a time
|
|
1280
|
+
save_state(root, st)
|
|
1281
|
+
A.emit_event(root, "wake", {"why": "reports", "reports": len(stale), "pid": proc.pid,
|
|
1282
|
+
"retried": True if retried else None})
|
|
1283
|
+
if retried:
|
|
1284
|
+
print(f"retried the architect's wake (pid {proc.pid}); the phone hears of it once it has not failed")
|
|
1285
|
+
else:
|
|
1286
|
+
_tell_phone(root, language.text(cfg, "watchdog.architect-woken", n=len(stale), pid=proc.pid))
|
|
1287
|
+
print(f"woke the architect (pid {proc.pid}) to judge it")
|
|
1288
|
+
return woke
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
def anomaly_subject(anomaly):
|
|
1292
|
+
"""What an anomaly is about, for the ladder: the report, decision or review it names, never an age (NOTICE-NOISE).
|
|
1293
|
+
|
|
1294
|
+
A fact such as "returned 34m ago" changes every cycle; what waits changes only when
|
|
1295
|
+
something new waits, and that is when a person is told again.
|
|
1296
|
+
"""
|
|
1297
|
+
for fact in anomaly.get("facts") or []:
|
|
1298
|
+
wrote = re.search(r"wrote\s+(\S+\.md)", str(fact))
|
|
1299
|
+
if wrote:
|
|
1300
|
+
return f"{anomaly.get('kind')}:{wrote.group(1)}"
|
|
1301
|
+
return f"{anomaly.get('kind')}:{anomaly.get('key') or ''}"
|
|
1302
|
+
|
|
1303
|
+
|
|
1304
|
+
def waiting_reports(root, cfg, names, newest=None):
|
|
1305
|
+
"""Which reports wait and since when, as a person reads it (WAITING-ONE-ALARM).
|
|
1306
|
+
|
|
1307
|
+
One report by its name, several by their count and the newest. Since when is the oldest
|
|
1308
|
+
report's own time - the stamp its name carries, else its file's - never its alarm's: an
|
|
1309
|
+
alarm starts again after a resume, and the request under it may have waited for weeks.
|
|
1310
|
+
"""
|
|
1311
|
+
box = cfg.get("mailbox", "agent-mail")
|
|
1312
|
+
times = []
|
|
1313
|
+
for name in names:
|
|
1314
|
+
try:
|
|
1315
|
+
times.append(A._name_time(name) or os.path.getmtime(os.path.join(root, box, name)))
|
|
1316
|
+
except OSError:
|
|
1317
|
+
continue # handled while this cycle ran
|
|
1318
|
+
since = f", waiting since {_when(min(times))}" if times else ""
|
|
1319
|
+
if len(names) == 1:
|
|
1320
|
+
return f"{names[0]} in {box}/{since}"
|
|
1321
|
+
return f"{len(names)} reports in {box}/{since}, the newest {newest or names[-1]}"
|
|
1322
|
+
|
|
1323
|
+
|
|
1324
|
+
def anomaly_waits(root, cfg, anomaly):
|
|
1325
|
+
"""What an anomaly that reaches a person says of itself: its kind, and what waits (WAITING-ONE-ALARM).
|
|
1326
|
+
|
|
1327
|
+
A request or report the implementer wrote is named with since when; any other anomaly
|
|
1328
|
+
says its first fact, which names what it measured.
|
|
1329
|
+
"""
|
|
1330
|
+
kind = anomaly.get("kind")
|
|
1331
|
+
names = [str(name) for name in anomaly.get("reports") or []]
|
|
1332
|
+
if names and kind in ("decision-requested", "report-waiting"):
|
|
1333
|
+
return f"{kind}: {waiting_reports(root, cfg, names)}"
|
|
1334
|
+
facts = anomaly.get("facts")
|
|
1335
|
+
return f"{kind}: {facts[0]}" if isinstance(facts, list) and facts else str(kind)
|
|
1336
|
+
|
|
1337
|
+
|
|
1338
|
+
def anomaly_reports(anomalies, names):
|
|
1339
|
+
"""The reports among `names` a standing anomaly stands for: those it groups, and its own (WAITING-ONE-ALARM).
|
|
1340
|
+
|
|
1341
|
+
The watchdog's report of an anomaly is known by the kind its name carries, whatever
|
|
1342
|
+
follows it: one written under an older name is a report of the same kind of condition,
|
|
1343
|
+
and the alarm for that kind stands for it as well. The implementer's reports are known
|
|
1344
|
+
by name, from the anomaly that groups them.
|
|
1345
|
+
"""
|
|
1346
|
+
kinds = {str(anomaly.get("kind")) for anomaly in anomalies}
|
|
1347
|
+
grouped = {name for anomaly in anomalies for name in anomaly.get("reports") or []}
|
|
1348
|
+
out = set()
|
|
1349
|
+
for name in names:
|
|
1350
|
+
own = re.search(r"-ANOMALY-(.+)\.md$", name) if A.from_watchdog(name) else None
|
|
1351
|
+
if name in grouped or (own and any(own.group(1) == kind or own.group(1).startswith(kind + "-")
|
|
1352
|
+
for kind in kinds)):
|
|
1353
|
+
out.add(name)
|
|
1354
|
+
return out
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def queue_past_a_question(root):
|
|
1358
|
+
"""(what waits, the READY item to take) when a question waits and READY work stands (#84).
|
|
1359
|
+
|
|
1360
|
+
The architect is a person's quota and attention, and both run out. An open
|
|
1361
|
+
decision cannot be answered by a nudged turn (#20), but it must not stop the
|
|
1362
|
+
queue either: the slice it blocks is parked and the next READY item is taken.
|
|
1363
|
+
None when nothing waits, or when nothing is READY and waiting is all there is.
|
|
1364
|
+
"""
|
|
1365
|
+
ready = A.ready(root)
|
|
1366
|
+
if not ready:
|
|
1367
|
+
return None
|
|
1368
|
+
asked = [d for d in A.decisions(root, "open") if d.get("asked_at")]
|
|
1369
|
+
if asked:
|
|
1370
|
+
return min(asked, key=lambda d: d["asked_at"])["id"], ready[0]["id"]
|
|
1371
|
+
blocked = A.board(root)["blocked"]
|
|
1372
|
+
return (blocked[0]["id"], ready[0]["id"]) if blocked else None
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def parked_note(cfg, waits, ready_id):
|
|
1376
|
+
"""What a nudge adds when a question waits and READY work stands (#84), in the project's language."""
|
|
1377
|
+
return language.text(cfg, "prompt.nudge-parked", waits=waits, ready=ready_id)
|
|
1378
|
+
|
|
1379
|
+
|
|
1380
|
+
def secondary_note(cfg, found):
|
|
1381
|
+
"""What a nudge adds when nothing is READY here and a secondary project has work (#8), in the project's language."""
|
|
1382
|
+
return language.text(cfg, "prompt.nudge-secondary", name=found["name"], root=found["root"], item=found["item"])
|
|
1383
|
+
|
|
1384
|
+
|
|
1385
|
+
def record_turn_flags(root, role, flags, reason):
|
|
1386
|
+
"""A turn whose added flags cannot be recorded starts nothing (TRIAGE-3-LOW)."""
|
|
1387
|
+
if not flags:
|
|
1388
|
+
return True
|
|
1389
|
+
try:
|
|
1390
|
+
A.record_actor_flags(root, role, flags, reason)
|
|
1391
|
+
except Exception as exc:
|
|
1392
|
+
refused = (f"could not record the flags added to the {role}: {A.redact(str(exc))}; "
|
|
1393
|
+
"no turn started; restore access to .polyctl/ledger/actor-flags.jsonl and retry")
|
|
1394
|
+
print(refused)
|
|
1395
|
+
tell_turn_refused(root, role, refused)
|
|
1396
|
+
return False
|
|
1397
|
+
return True
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
def record_pinned(root, pinned):
|
|
1401
|
+
"""Record a wake's added flags before allowing its launch (TRIAGE-3-LOW)."""
|
|
1402
|
+
return record_turn_flags(root, "architect", pinned,
|
|
1403
|
+
"the architect's command names no permission mode, so polyctl appends the one its adapter pins")
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _schedule_hunt(root, cfg, st):
|
|
1407
|
+
"""Start one bounded, detached bug hunt when it is switched on and due: a schedule, never a loop (#45).
|
|
1408
|
+
|
|
1409
|
+
Nor while harness configuration nobody has seen stands in the tree, as no turn polyctl starts here runs then
|
|
1410
|
+
(HARNESS-CONFIG); a hunt held is due again next cycle.
|
|
1411
|
+
"""
|
|
1412
|
+
from . import features as F
|
|
1413
|
+
if not F.enabled(cfg, "hunter") or not S.get(cfg, "hunter.argv") or A.hold_state(root):
|
|
1414
|
+
return False
|
|
1415
|
+
if time.time() - float(st.get("last_hunt") or 0) < S.get(cfg, "hunter.every_hours") * 3600:
|
|
1416
|
+
return False
|
|
1417
|
+
unseen = harness_config_hold(root)
|
|
1418
|
+
if unseen:
|
|
1419
|
+
print(f"{unseen}; not starting a bug hunt")
|
|
1420
|
+
return False
|
|
1421
|
+
log = os.path.join(STATE_DIR, A.project_file_name("hunt-log", A.project_key(root)))
|
|
1422
|
+
os.makedirs(STATE_DIR, exist_ok=True)
|
|
1423
|
+
with open(log, "a", encoding=UTF8) as fh:
|
|
1424
|
+
fh.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} hunt ===\n")
|
|
1425
|
+
fh.flush()
|
|
1426
|
+
subprocess.Popen([sys.executable, "-m", "polyctl", "-C", root, "hunt", "run"], cwd=A.self_child_cwd(),
|
|
1427
|
+
stdin=subprocess.DEVNULL, stdout=fh, stderr=subprocess.STDOUT, start_new_session=True,
|
|
1428
|
+
env=A.self_child_env(dict(os.environ, POLYCTL_ROLE="hunter")))
|
|
1429
|
+
st["last_hunt"] = time.time()
|
|
1430
|
+
save_state(root, st)
|
|
1431
|
+
print("started a bounded bug hunt")
|
|
1432
|
+
return True
|
|
1433
|
+
|
|
1434
|
+
|
|
1435
|
+
def harness_config_hold(root):
|
|
1436
|
+
"""Why no turn may start in this tree now, or None: what a harness reads from it changed and no person has seen it,
|
|
1437
|
+
or it cannot be read (HARNESS-CONFIG).
|
|
1438
|
+
|
|
1439
|
+
A hook the implementer writes into .claude/settings.json runs in the next session started here - its own nudge, the
|
|
1440
|
+
architect's wake or refill, a hunt - before any review has seen it. Each asks right before it starts, as each reads
|
|
1441
|
+
the hold, since an agent can write one after the cycle began; one that cannot be read holds them all.
|
|
1442
|
+
"""
|
|
1443
|
+
try:
|
|
1444
|
+
unseen = A.unseen_harness_config(root)
|
|
1445
|
+
except Exception as exc:
|
|
1446
|
+
return f"what a harness reads from this tree cannot be read ({' '.join(str(exc).split())[:160]})"
|
|
1447
|
+
if not unseen:
|
|
1448
|
+
return None
|
|
1449
|
+
return (f"harness configuration nobody has seen stands in this tree: {A.harness_config_line(unseen, limit=10)} - "
|
|
1450
|
+
"a person reads it with polyctl harness seen --by <name>")
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def turn_hold(root, cfg, role):
|
|
1454
|
+
"""Why a turn of `role` may not start with the command composed from `cfg`, or None: its block says otherwise of it
|
|
1455
|
+
than the command last accepted, judged by the configuration the command came from (TURN-ARGV).
|
|
1456
|
+
|
|
1457
|
+
harness_config_hold reads .polyctl/config.json again, and an agent can put it back between the two readings; what
|
|
1458
|
+
starts is what this cycle composed.
|
|
1459
|
+
"""
|
|
1460
|
+
try:
|
|
1461
|
+
unseen = A.unseen_turn(root, cfg, role)
|
|
1462
|
+
except Exception as exc:
|
|
1463
|
+
return f"the {role}'s command cannot be compared with the one accepted ({' '.join(str(exc).split())[:160]})"
|
|
1464
|
+
if not unseen:
|
|
1465
|
+
return None
|
|
1466
|
+
return (f"harness configuration nobody has seen stands in this tree: {A.harness_config_line([unseen])} - a person "
|
|
1467
|
+
"reads it with polyctl harness seen --by <name>")
|
|
1468
|
+
|
|
1469
|
+
|
|
1470
|
+
def tell_turn_refused(root, role, refused):
|
|
1471
|
+
"""Name a turn the watchdog refuses to a person, once for what refuses it (TURN-ARGV): its configuration must
|
|
1472
|
+
change, and nothing an agent does or a person records lets it start as it stands."""
|
|
1473
|
+
notify(f"{A.project_key(root)}: {'nudge' if role == 'implementer' else 'architect wake'} refused", refused, root,
|
|
1474
|
+
key=f"turn-refused:{role}", window=6 * 3600, audience="human", what=refused)
|
|
1475
|
+
|
|
1476
|
+
|
|
1477
|
+
def tell_harness_config(root, project):
|
|
1478
|
+
"""Name to a person, each cycle, every change to what a harness reads here that nobody has seen (HARNESS-CONFIG).
|
|
1479
|
+
|
|
1480
|
+
One alarm for what the changes hold: it rings once for them and climbs the ladder while they stand, and rings again
|
|
1481
|
+
when they change. Seen, or gone from the tree, it is no longer raised and its episode closes.
|
|
1482
|
+
"""
|
|
1483
|
+
try:
|
|
1484
|
+
unseen = A.unseen_harness_config(root)
|
|
1485
|
+
except Exception as exc:
|
|
1486
|
+
_FACTS["harness_config"] = "unreadable"
|
|
1487
|
+
notify(f"{project}: harness configuration cannot be read",
|
|
1488
|
+
f"what a harness reads from this tree cannot be read ({' '.join(str(exc).split())[:160]}), so no wake, "
|
|
1489
|
+
"refill, nudge or hunt starts here until it can", root, key="harness-config", window=6 * 3600,
|
|
1490
|
+
audience="human", what="unreadable")
|
|
1491
|
+
return
|
|
1492
|
+
_FACTS["harness_config"] = len(unseen)
|
|
1493
|
+
if unseen:
|
|
1494
|
+
notify(f"{project}: harness configuration nobody has seen",
|
|
1495
|
+
f"{A.harness_config_line(unseen, limit=20)}: what a harness reads from this tree changed, and no wake, "
|
|
1496
|
+
"refill, nudge or hunt starts here until a person reads it - polyctl harness seen --by <name>",
|
|
1497
|
+
root, key="harness-config", window=6 * 3600, audience="human", what=A.harness_config_digest(unseen))
|
|
1498
|
+
|
|
1499
|
+
|
|
1500
|
+
def open_work(cfg, root):
|
|
1501
|
+
"""Is there something for the implementer to continue? Cheap signals only.
|
|
1502
|
+
|
|
1503
|
+
Only signals that mean *work*. Mail counts when it is addressed to the
|
|
1504
|
+
implementer — its own outgoing reports and the watchdog's anomaly files once
|
|
1505
|
+
counted as "unread mail", so every report the implementer wrote was a reason
|
|
1506
|
+
to nudge it into writing another. Dirty paths count outside the coordination
|
|
1507
|
+
directories; a review counts only if it is newer than HEAD.
|
|
1508
|
+
"""
|
|
1509
|
+
reasons = []
|
|
1510
|
+
if A.implementer_inbox(root, cfg):
|
|
1511
|
+
reasons.append("unread mail")
|
|
1512
|
+
# A slice the board says is running is work, whether or not a file has
|
|
1513
|
+
# changed yet: an implementer that has just started one and ended its turn
|
|
1514
|
+
# on a plan must be nudged back into it.
|
|
1515
|
+
if A.board(root)["running"]:
|
|
1516
|
+
reasons.append("slice running")
|
|
1517
|
+
rs = A.reviewer_state(root)
|
|
1518
|
+
# A reviewer that was unavailable with no known window - an expired login, a
|
|
1519
|
+
# missing binary - is not work a nudge can do; nudging for it never ended (audit).
|
|
1520
|
+
if rs.get("pending_review") and rs.get("until") and rs["until"] <= time.time():
|
|
1521
|
+
reasons.append("reviewer window reopened — re-run the pending review")
|
|
1522
|
+
if A.product_dirty(root, cfg):
|
|
1523
|
+
reasons.append("uncommitted changes")
|
|
1524
|
+
# Findings are the newest review that produced a verdict; an UNAVAILABLE or
|
|
1525
|
+
# INVALID file is not a finding anyone can act on (audit).
|
|
1526
|
+
from .storage import LedgerCorruption, read_chained_jsonl
|
|
1527
|
+
try:
|
|
1528
|
+
rows = read_chained_jsonl(A.review_ledger_path(root), A.REVIEW_CHAIN)
|
|
1529
|
+
except (OSError, ValueError, LedgerCorruption):
|
|
1530
|
+
rows = []
|
|
1531
|
+
latest = next((row for row in reversed(rows) if isinstance(row, dict)
|
|
1532
|
+
and row.get("verdict") in ("APPROVED", "NEEDS_CHANGES")), None)
|
|
1533
|
+
if latest and latest["verdict"] == "NEEDS_CHANGES":
|
|
1534
|
+
try:
|
|
1535
|
+
head_at = int(A._git_text(root, "log", "-1", "--format=%ct") or 0)
|
|
1536
|
+
rev_at = int(latest.get("at") or 0)
|
|
1537
|
+
except (OSError, ValueError):
|
|
1538
|
+
rev_at, head_at = 1, 0
|
|
1539
|
+
if rev_at > head_at:
|
|
1540
|
+
reasons.append("open review findings")
|
|
1541
|
+
return reasons
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
WAKE_SIGNATURES = (
|
|
1545
|
+
("quota", r"(hit your (?:session|usage|weekly|monthly) limit[^\n]*|usage limit[^\n]*|"
|
|
1546
|
+
r"rate limit(?:ed)?[^\n]*resets?[^\n]*|out of (?:credits|quota)[^\n]*)"),
|
|
1547
|
+
("binary", r"(does not support this model[^\n]*|version [^\n]*required[^\n]*|"
|
|
1548
|
+
r"command not found[^\n]*|No such file or directory[^\n]*|env: node: [^\n]*)"),
|
|
1549
|
+
("session", r"(No conversation found[^\n]*|[Ss]ession[^\n]{0,40}not found[^\n]*|"
|
|
1550
|
+
r"Invalid session[^\n]*)"),
|
|
1551
|
+
("other", r"(API Error: \d{3}[^\n]*|Error: [^\n]{8,})"),
|
|
1552
|
+
)
|
|
1553
|
+
|
|
1554
|
+
|
|
1555
|
+
# The architect's usage limit comes back within five hours of being hit; a reset
|
|
1556
|
+
# a message names further away than that cannot be the one it meant (#40). That is
|
|
1557
|
+
# the short window: a limit that names its period is as long as that period.
|
|
1558
|
+
ARCHITECT_QUOTA_WINDOW = S.default("architect.quota_window_hours") * 3600
|
|
1559
|
+
|
|
1560
|
+
# The periods a usage-limit message may name, and their lengths (ARCHITECT-WAKE-QUOTA).
|
|
1561
|
+
LIMIT_PERIODS = {"weekly": 7 * 86400, "monthly": 31 * 86400}
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
def limit_window(text):
|
|
1565
|
+
"""How long the limit a usage-limit message names lasts: its period when it names one, else the short window.
|
|
1566
|
+
|
|
1567
|
+
The length bounds how far ahead a clock-only reset may roll (parse_reset). A weekly
|
|
1568
|
+
limit that named only a clock time already past when it was written was read against
|
|
1569
|
+
the five-hour window, as the time that had gone, so its block ended at once and the
|
|
1570
|
+
next cycle could wake the architect again, days before its week was over
|
|
1571
|
+
(ARCHITECT-WAKE-QUOTA).
|
|
1572
|
+
"""
|
|
1573
|
+
named = re.search(r"\b(weekly|monthly)\b", str(text or ""), re.I)
|
|
1574
|
+
return LIMIT_PERIODS[named.group(1).lower()] if named else S.get(None, "architect.quota_window_hours") * 3600
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
RESET_MONTHS = ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec")
|
|
1578
|
+
|
|
1579
|
+
|
|
1580
|
+
def _reset_clock(hour, minute, meridiem):
|
|
1581
|
+
"""(hour, minute) on a 24-hour clock from "9", "20", "pm"."""
|
|
1582
|
+
h, mi, ap = int(hour), int(minute or 0), (meridiem or "").lower()
|
|
1583
|
+
if ap == "pm" and h < 12:
|
|
1584
|
+
h += 12
|
|
1585
|
+
if ap == "am" and h == 12:
|
|
1586
|
+
h = 0
|
|
1587
|
+
return h, mi
|
|
1588
|
+
|
|
1589
|
+
|
|
1590
|
+
def parse_reset(text, now=None, window=None):
|
|
1591
|
+
"""When does the quota come back? "resets 4:30am" / "resets in 4h 43m" / "resets Sep 14 at 4am" / None.
|
|
1592
|
+
|
|
1593
|
+
`now` is when the message was written, not when it is read: a log line read
|
|
1594
|
+
again hours later keeps the reset it named. A clock time already past rolls to
|
|
1595
|
+
the next day only while that stays inside `window`, the limit's own length;
|
|
1596
|
+
otherwise the message meant the time that has gone. On 2026-09-07 a 17:48
|
|
1597
|
+
"resets 9:20pm" was re-read after 21:20 on every cycle and pushed a day ahead
|
|
1598
|
+
each time, so the block it raised never ended (#40).
|
|
1599
|
+
|
|
1600
|
+
A relative reset may count days as well: keyflip states a weekly window's as
|
|
1601
|
+
"resets in 3d 4h", and read without them it said nothing (ARCHITECT-WAKE-QUOTA).
|
|
1602
|
+
"""
|
|
1603
|
+
now = now or time.time()
|
|
1604
|
+
m = re.search(r"resets?\s+in\s+((?:\d+\s*[dhms]\s*)+)", text, re.I)
|
|
1605
|
+
if m:
|
|
1606
|
+
secs = 0
|
|
1607
|
+
for n, u in re.findall(r"(\d+)\s*([dhms])", m.group(1), re.I):
|
|
1608
|
+
secs += int(n) * {"d": 86400, "h": 3600, "m": 60, "s": 1}[u.lower()]
|
|
1609
|
+
return now + secs
|
|
1610
|
+
# A weekly limit names its day: "resets Sep 14 at 4am". It was read as no reset
|
|
1611
|
+
# at all, so a week-long block was retried every few hours (#41).
|
|
1612
|
+
m = re.search(r"resets?\s+(?:on\s+)?(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+"
|
|
1613
|
+
r"(\d{1,2})(?:,?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?)?", text, re.I)
|
|
1614
|
+
if m:
|
|
1615
|
+
month = RESET_MONTHS.index(m.group(1).lower()) + 1
|
|
1616
|
+
h, mi = _reset_clock(m.group(3) or "0", m.group(4), m.group(5))
|
|
1617
|
+
lt = time.localtime(now)
|
|
1618
|
+
cand = time.mktime((lt.tm_year, month, int(m.group(2)), h, mi, 0, 0, 0, -1))
|
|
1619
|
+
if cand < now - 180 * 86400: # December's message read in January
|
|
1620
|
+
cand = time.mktime((lt.tm_year + 1, month, int(m.group(2)), h, mi, 0, 0, 0, -1))
|
|
1621
|
+
return cand
|
|
1622
|
+
m = re.search(r"resets?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?", text, re.I)
|
|
1623
|
+
if not m:
|
|
1624
|
+
return None
|
|
1625
|
+
h, mi = _reset_clock(m.group(1), m.group(2), m.group(3))
|
|
1626
|
+
lt = time.localtime(now)
|
|
1627
|
+
cand = time.mktime((lt.tm_year, lt.tm_mon, lt.tm_mday, h, mi, 0, 0, 0, -1))
|
|
1628
|
+
if cand <= now and (window is None or cand + 24 * 3600 - now <= window):
|
|
1629
|
+
cand += 24 * 3600
|
|
1630
|
+
return cand
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
def wake_error(log_path):
|
|
1634
|
+
"""What the last architect wake said, if it failed.
|
|
1635
|
+
|
|
1636
|
+
{"text", "binary", "when", "kind", "resets_at"} or None. The wake is
|
|
1637
|
+
detached, so its outcome can only be read on the next cycle from the log
|
|
1638
|
+
segment it wrote. Forty identical failures went unread because nothing
|
|
1639
|
+
looked. `kind` is what the caller should do about it: a stale *binary* wants
|
|
1640
|
+
another binary, a *quota* wants a wait until `resets_at`, a dead *session*
|
|
1641
|
+
wants rediscovery.
|
|
1642
|
+
"""
|
|
1643
|
+
try:
|
|
1644
|
+
tail = open(log_path, errors="replace", encoding=UTF8).read()[-20000:]
|
|
1645
|
+
except OSError:
|
|
1646
|
+
return None
|
|
1647
|
+
segs = re.split(r"^=== (\S+ \S+) (escalate|refill)(?: (.*?))? ===$", tail, flags=re.M)
|
|
1648
|
+
if len(segs) < 5:
|
|
1649
|
+
return None
|
|
1650
|
+
when, _, binary, body = segs[-4], segs[-3], segs[-2] or "", segs[-1]
|
|
1651
|
+
try:
|
|
1652
|
+
at = time.mktime(time.strptime(when, "%Y-%m-%d %H:%M:%S"))
|
|
1653
|
+
except ValueError:
|
|
1654
|
+
at = None
|
|
1655
|
+
for kind, pat in WAKE_SIGNATURES:
|
|
1656
|
+
m = re.search(pat, body)
|
|
1657
|
+
if m:
|
|
1658
|
+
text = m.group(1).strip()[:300]
|
|
1659
|
+
# A reset is read against when the wake wrote it, not against this cycle, and
|
|
1660
|
+
# within the limit the stop names: its period, or the short window.
|
|
1661
|
+
resets_at = parse_reset(body, now=at, window=limit_window(text)) if kind == "quota" else None
|
|
1662
|
+
streak = 0
|
|
1663
|
+
if kind == "quota" and resets_at is None:
|
|
1664
|
+
# How many wakes in a row stopped on a weekly or monthly limit that named no reset, this
|
|
1665
|
+
# one the last (ARCHITECT-WAKE-QUOTA-2): each waits twice as long as the one before.
|
|
1666
|
+
for index in range(len(segs) - 4, 0, -4):
|
|
1667
|
+
if not _period_without_reset(segs[index], segs[index + 3]):
|
|
1668
|
+
break
|
|
1669
|
+
streak += 1
|
|
1670
|
+
return {"text": text, "binary": binary, "when": when, "at": at, "kind": kind,
|
|
1671
|
+
"resets_at": resets_at, "streak": streak}
|
|
1672
|
+
return None
|
|
1673
|
+
|
|
1674
|
+
|
|
1675
|
+
def _period_without_reset(when, body):
|
|
1676
|
+
"""Whether a wake's segment stopped on a weekly or monthly limit whose words name no reset."""
|
|
1677
|
+
m = re.search(dict(WAKE_SIGNATURES)["quota"], body)
|
|
1678
|
+
if not m:
|
|
1679
|
+
return False
|
|
1680
|
+
try:
|
|
1681
|
+
at = time.mktime(time.strptime(when, "%Y-%m-%d %H:%M:%S"))
|
|
1682
|
+
except ValueError:
|
|
1683
|
+
return False
|
|
1684
|
+
text = m.group(1).strip()[:300]
|
|
1685
|
+
return limit_window(text) > S.get(None, "architect.quota_window_hours") * 3600 \
|
|
1686
|
+
and parse_reset(body, now=at, window=limit_window(text)) is None
|
|
1687
|
+
|
|
1688
|
+
|
|
1689
|
+
def wake_failed(failure, since):
|
|
1690
|
+
"""Whether the wake started at `since` failed, from what `wake_error` read of the log (#69).
|
|
1691
|
+
|
|
1692
|
+
A failure written before that wake belongs to an earlier one, and a model's own prose
|
|
1693
|
+
saying "Error:" is not a failure.
|
|
1694
|
+
"""
|
|
1695
|
+
return bool(failure) and (failure.get("at") or 0) >= int(since or 0) - 2 \
|
|
1696
|
+
and (failure.get("kind") != "other" or "API Error" in failure.get("text", ""))
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
def wake_failure_told(failure):
|
|
1700
|
+
"""What a failed wake says, for the ladder: its kind, its binary and its words (NOISE-REPEATS).
|
|
1701
|
+
|
|
1702
|
+
Not the time it was read at, and not an id that changes on every attempt, such as the
|
|
1703
|
+
request id a provider puts in its error: each retry after a 529 read as another failure.
|
|
1704
|
+
Another kind, another binary or other words is something new, and is told.
|
|
1705
|
+
"""
|
|
1706
|
+
failure = failure or {}
|
|
1707
|
+
# A request id is masked by its key first: one of letters alone has no digit for the rule after it to find,
|
|
1708
|
+
# and each retry read as news (NOISE-REPEATS-2).
|
|
1709
|
+
text = re.sub(r'(?i)(request[ _-]?id"?\s*[:=]\s*"?)[^"\s,}]+', "\\1…", A.redact(str(failure.get("text") or "")))
|
|
1710
|
+
words = re.sub(r"(?<![\w-])(?=[\w-]*\d)[\w-]{8,}", "…", text)
|
|
1711
|
+
return f"{failure.get('kind')}:{failure.get('binary') or '?'}:{' '.join(words.split())}"
|
|
1712
|
+
|
|
1713
|
+
|
|
1714
|
+
def tell_retried_wake(root, cfg, st, dry_run=False):
|
|
1715
|
+
"""Tell the phone of a wake retried after a failed one, once it is known not to have failed (NOISE-REPEATS).
|
|
1716
|
+
|
|
1717
|
+
The line went out as the wake started, before anything said how it went, and a wake is
|
|
1718
|
+
retried every fifteen minutes while a transport error lasts: rebuilt in a temporary
|
|
1719
|
+
project, a day of 529s sent the phone 95 "architect woken" lines about an architect
|
|
1720
|
+
nobody woke, beside the alarm that said the wake was failing. A retry is told when its own
|
|
1721
|
+
log segment shows no failure and it has ended, or has run for the fifteen minutes the
|
|
1722
|
+
watchdog leaves between wakes; one that failed was no woken architect, and its failure is
|
|
1723
|
+
already on the ladder. Returns whether it was told.
|
|
1724
|
+
"""
|
|
1725
|
+
retry = st.get("wake_retry")
|
|
1726
|
+
if not isinstance(retry, dict):
|
|
1727
|
+
return False
|
|
1728
|
+
try:
|
|
1729
|
+
running = time.time() - float(retry.get("at") or 0) < 900 and arch_alive(root, cfg.get("architect") or {})
|
|
1730
|
+
except TableUnreadable:
|
|
1731
|
+
running = True # it may run: it is told once that can be seen (WINDOWS-TABLE-CLOSED)
|
|
1732
|
+
if running:
|
|
1733
|
+
return False # still running, and a failing one can take minutes to say so
|
|
1734
|
+
failure = wake_error(os.path.join(STATE_DIR, A.project_file_name("escalate-log", A.project_key(root))))
|
|
1735
|
+
worked = not wake_failed(failure, retry.get("at") or 0)
|
|
1736
|
+
if dry_run:
|
|
1737
|
+
if worked:
|
|
1738
|
+
print("DRY RUN: would tell the phone the architect was woken after failed attempts")
|
|
1739
|
+
return False
|
|
1740
|
+
st.pop("wake_retry", None)
|
|
1741
|
+
save_state(root, st)
|
|
1742
|
+
if worked:
|
|
1743
|
+
_tell_phone(root, language.text(cfg, "watchdog.architect-woken-retried", n=retry.get("reports"),
|
|
1744
|
+
pid=retry.get("pid")))
|
|
1745
|
+
return worked
|
|
1746
|
+
|
|
1747
|
+
|
|
1748
|
+
def quota_block_until(err, now=None, stated=None):
|
|
1749
|
+
"""Until when a quota error read from the wake log blocks a wake; None once it no longer does.
|
|
1750
|
+
|
|
1751
|
+
The block ends at the reset the message named or, when it named none, one
|
|
1752
|
+
limit window after the message was written. An older error describes a
|
|
1753
|
+
window that is over, however many times the log is read again (#40).
|
|
1754
|
+
|
|
1755
|
+
`stated` is the reset the provider's own reading states for a spent window
|
|
1756
|
+
(architect_quota_until). It only moves the end later: a wake started before it
|
|
1757
|
+
stops on that window again, and without it the end is what it always was
|
|
1758
|
+
(ARCHITECT-WAKE-QUOTA).
|
|
1759
|
+
"""
|
|
1760
|
+
now = now or time.time()
|
|
1761
|
+
if not err or err.get("kind") != "quota":
|
|
1762
|
+
return None
|
|
1763
|
+
window = S.get(None, "architect.quota_window_hours") * 3600
|
|
1764
|
+
span = window
|
|
1765
|
+
if not err.get("resets_at") and err.get("streak"):
|
|
1766
|
+
# A weekly or monthly limit that named no reset held its wakes one short window each, so the
|
|
1767
|
+
# architect was woken again and again before its period was over. Each wake in a row that stops
|
|
1768
|
+
# on one waits twice as long as the last, and never longer than the period (ARCHITECT-WAKE-QUOTA-2).
|
|
1769
|
+
span = min(window * 2 ** (int(err["streak"]) - 1), limit_window(err.get("text")))
|
|
1770
|
+
until = max(err.get("resets_at") or ((err.get("at") or now) + span), stated or 0)
|
|
1771
|
+
return until if until > now else None
|
|
1772
|
+
|
|
1773
|
+
|
|
1774
|
+
def architect_quota_until(err, argv, now=None):
|
|
1775
|
+
"""Until when a usage-limit stop read from an architect wake's log holds its wakes; None once it does not.
|
|
1776
|
+
|
|
1777
|
+
The implementer's quota handling reads the machine's reading of the providers'
|
|
1778
|
+
windows: keyflip's lines, taken by the quota command the adapters declare and kept in
|
|
1779
|
+
~/.polyctl/quota.json, which the quota gate stops a turn on at `quota.block_percent` and an
|
|
1780
|
+
account rotation reads for the provider an adapter's `quota` block names
|
|
1781
|
+
(A.provider_window). The wake reads the same reading, for the provider `argv` spends.
|
|
1782
|
+
When that window is spent, at the gate's percent, the reset it states is one the wake
|
|
1783
|
+
is not started before. A window with headroom stopped nothing, so its reset says
|
|
1784
|
+
nothing of this stop. With no such reading the block is quota_block_until's alone, as
|
|
1785
|
+
it always was (ARCHITECT-WAKE-QUOTA).
|
|
1786
|
+
"""
|
|
1787
|
+
if not err or err.get("kind") != "quota":
|
|
1788
|
+
return None
|
|
1789
|
+
provider = A.provider_of(argv)
|
|
1790
|
+
window = A.provider_window(provider) if provider else None
|
|
1791
|
+
spent = bool(window) and (window.get("pct") or 0) >= S.get(None, "quota.block_percent")
|
|
1792
|
+
return quota_block_until(err, now=now, stated=parse_reset(window.get("raw") or "", now=now) if spent else None)
|
|
1793
|
+
|
|
1794
|
+
|
|
1795
|
+
def quota_ok(adapter):
|
|
1796
|
+
"""Is there headroom to spend a turn?
|
|
1797
|
+
|
|
1798
|
+
Prefer an explicit budget over a guessed threshold. keyflip enforces
|
|
1799
|
+
per-account 5h/7d budgets when the user has set them, and a policy someone
|
|
1800
|
+
chose beats a number this script invented — so ask it first and only fall
|
|
1801
|
+
back to reading the raw window when no budget exists. A breached budget stops
|
|
1802
|
+
the turn, whatever the window says; a budget not breached is headroom. Which
|
|
1803
|
+
account's breach counts is quota_budget's to say.
|
|
1804
|
+
|
|
1805
|
+
`adapter` is the package's, A.metered_adapter's: reading the window runs the command it declares, and a
|
|
1806
|
+
layer the implementer writes chose one that ran (TRIAGE-HIGHS).
|
|
1807
|
+
"""
|
|
1808
|
+
budget = A.quota_budget(adapter)
|
|
1809
|
+
if budget["breached"]:
|
|
1810
|
+
return False
|
|
1811
|
+
if budget["accounts"]:
|
|
1812
|
+
return True
|
|
1813
|
+
ceiling = S.get(None, "quota.block_percent")
|
|
1814
|
+
for line in A.quota(adapter):
|
|
1815
|
+
for token in line.replace("%", "% ").split():
|
|
1816
|
+
if token.endswith("%"):
|
|
1817
|
+
try:
|
|
1818
|
+
if float(token[:-1]) >= ceiling:
|
|
1819
|
+
return False
|
|
1820
|
+
except ValueError:
|
|
1821
|
+
pass
|
|
1822
|
+
return True
|
|
1823
|
+
|
|
1824
|
+
|
|
1825
|
+
# ---- a slice parked on the implementer's usage limit (QUOTA-PARK) -----------------------------
|
|
1826
|
+
#
|
|
1827
|
+
# A usage limit was read only where an architect's wake had failed on one. An implementer that
|
|
1828
|
+
# stopped on its own limit mid-slice read as an agent ignoring its nudges: each nudge died on the
|
|
1829
|
+
# same limit until the backoff ran out and a person was told it was stuck. Its slice is parked
|
|
1830
|
+
# instead, and the reset resumes it.
|
|
1831
|
+
|
|
1832
|
+
# The `needs:` of a slice parked on quota, and the word the watchdog knows its parks by.
|
|
1833
|
+
QUOTA_NEEDS = "quota"
|
|
1834
|
+
# The note a parked item keeps its own `needs:` in, on the board itself, so a park the watchdog's state lost
|
|
1835
|
+
# gives it back as one that stood does (QUOTA-PARK-2).
|
|
1836
|
+
QUOTA_HELD = "held-needs"
|
|
1837
|
+
# A failed nudge's tail is the log's last lines: what stands before its own header is an earlier turn's.
|
|
1838
|
+
NUDGE_HEADER = re.compile(r"=== \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} nudge.*?===")
|
|
1839
|
+
# While a park holds, keyflip is asked to rotate a spent window at most this often, as wakes are spaced:
|
|
1840
|
+
# a rotation is machine-wide, and one asked every cycle while no account has headroom would move every
|
|
1841
|
+
# session on the provider every few minutes.
|
|
1842
|
+
QUOTA_ROTATION_SPACING = 900
|
|
1843
|
+
|
|
1844
|
+
|
|
1845
|
+
def usage_limit_stop(cfg, st, adapter=None, last=None):
|
|
1846
|
+
"""Did the implementer stop on its usage limit? {"at", "text", "resets_at", "hours", "source"} or None (QUOTA-PARK).
|
|
1847
|
+
|
|
1848
|
+
Two readings can say so, and the newer of them decides: the transcript, where its newest words
|
|
1849
|
+
are a reply the harness wrote in the model's place (`last_message`, or `last` when the caller
|
|
1850
|
+
has read it), and the nudge that died within seconds of starting (`last_error`), whose tail is
|
|
1851
|
+
its own process's output. Each is searched for the words the implementer's adapter declares
|
|
1852
|
+
its harness stops on (`quota_stops`), in the order it declares them. An adapter that declares
|
|
1853
|
+
none is never read as stopped: one harness's words are no other's, and while the stop was read
|
|
1854
|
+
by one pattern in one harness's wording, another harness's limit never parked anything. The
|
|
1855
|
+
model's own words are never read either: an implementer at work on quota handling writes about
|
|
1856
|
+
usage limits, and a reply of its own that names one is no stop.
|
|
1857
|
+
|
|
1858
|
+
The reset is the one the harness's reply records beside its words (`quota_reset_path`), else
|
|
1859
|
+
the one its words name - in epoch seconds, where the stop's words hold an `epoch`, or as
|
|
1860
|
+
`parse_reset` reads them against when they were written - and it counts only after the stop
|
|
1861
|
+
and within the limit's own `hours`: a clock time is the next one inside them, and a reset
|
|
1862
|
+
further away is none the words can mean. Read within five hours, a weekly limit that named
|
|
1863
|
+
4am at 18:17 read as that morning's, already gone, and its slice was parked with no reset.
|
|
1864
|
+
"""
|
|
1865
|
+
adapter = A.implementer_adapter(cfg) if adapter is None else adapter
|
|
1866
|
+
stops = A.quota_stops(adapter)
|
|
1867
|
+
if not stops:
|
|
1868
|
+
return None
|
|
1869
|
+
said = []
|
|
1870
|
+
last = A.last_message(cfg) if last is None else last
|
|
1871
|
+
if last and last.get("at"):
|
|
1872
|
+
said.append((last["at"], last["text"] if last["in_place"] else "", "transcript", last.get("record")))
|
|
1873
|
+
failed = st.get("last_error") if isinstance(st.get("last_error"), dict) else {}
|
|
1874
|
+
if failed.get("tail") and isinstance(failed.get("at"), (int, float)):
|
|
1875
|
+
said.append((float(failed["at"]), NUDGE_HEADER.split(str(failed["tail"]))[-1], "nudge", None))
|
|
1876
|
+
if not said:
|
|
1877
|
+
return None
|
|
1878
|
+
at, text, source, record = max(said, key=lambda reading: reading[0])
|
|
1879
|
+
for stop in stops:
|
|
1880
|
+
found = stop["words"].search(text)
|
|
1881
|
+
if not found:
|
|
1882
|
+
continue
|
|
1883
|
+
path, window = A.quota_reset_path(adapter), stop["hours"] * 3600
|
|
1884
|
+
recorded = A._path_value(record, path) if record and path else None
|
|
1885
|
+
epoch = found.groupdict().get("epoch")
|
|
1886
|
+
read = [recorded if isinstance(recorded, (int, float)) and not isinstance(recorded, bool) else None,
|
|
1887
|
+
float(epoch) if epoch else None, parse_reset(text, now=at, window=window)]
|
|
1888
|
+
return {"at": at, "text": A.redact(found.group(0).strip())[:300], "source": source, "hours": stop["hours"],
|
|
1889
|
+
"resets_at": next((reset for reset in read if reset is not None and at < reset <= at + window), None)}
|
|
1890
|
+
return None
|
|
1891
|
+
|
|
1892
|
+
|
|
1893
|
+
def quota_park_for(stop, items, now, retry):
|
|
1894
|
+
"""The park a stop makes of `items`: until its reset or, where its words named none, until one nudge tries again.
|
|
1895
|
+
|
|
1896
|
+
The retry is `retry` seconds after the stop, or the limit's own hours where those are fewer: a
|
|
1897
|
+
limit gone within them is gone by then.
|
|
1898
|
+
"""
|
|
1899
|
+
named = stop["resets_at"] is not None
|
|
1900
|
+
hours = stop.get("hours")
|
|
1901
|
+
until = stop["resets_at"] if named else stop["at"] + min(retry, hours * 3600 if hours else retry)
|
|
1902
|
+
return {"items": list(items), "at": stop["at"], "text": stop["text"], "until": until, "named": named,
|
|
1903
|
+
"source": stop["source"], "since": now}
|
|
1904
|
+
|
|
1905
|
+
|
|
1906
|
+
def quota_park_step(stop, park, resumed, running, parked, now=None, spoken=None, retry=None):
|
|
1907
|
+
"""What a usage-limit stop asks of this cycle: (action, park). It reads and writes nothing (QUOTA-PARK).
|
|
1908
|
+
|
|
1909
|
+
`stop` is what `usage_limit_stop` read this cycle, `park` the park that stands, `resumed` the
|
|
1910
|
+
time of the stop the last park ended for, `running` the implementer's ids under the board's
|
|
1911
|
+
`running`, `parked` the ids blocked on quota, and `spoken` when a person or the model last
|
|
1912
|
+
spoke in the transcript (`last_message`). A stop whose words named no reset is tried again
|
|
1913
|
+
`retry` seconds after it (`watchdog.quota_retry_hours`). The action is None, "park", "hold",
|
|
1914
|
+
"resume" or "clear":
|
|
1915
|
+
|
|
1916
|
+
- a stop parks the running items, unless a park has already ended for it: a stop is parked,
|
|
1917
|
+
and resumed after, once. A stop whose reset has already passed resumes at once;
|
|
1918
|
+
- a park holds until its reset, or until its retry where it named none, and a newer stop
|
|
1919
|
+
gives it that stop's. A park with no end held for as long as its stop stood, and while no
|
|
1920
|
+
turn runs nothing moves a stop on;
|
|
1921
|
+
- it resumes then, or as soon as the transcript went on past its stop - a person resumed the
|
|
1922
|
+
session, the model answered - since that is the limit gone, whatever time it named. A stop
|
|
1923
|
+
that still stands when it is tried again is a newer stop, and parks again: one nudge a
|
|
1924
|
+
retry, never a loop;
|
|
1925
|
+
- a park whose items someone took off `blocked` is over, and nothing is moved back;
|
|
1926
|
+
- items blocked on quota with no park standing are a park the watchdog's state lost, torn or
|
|
1927
|
+
deleted: a stop read now parks them again, and without one nothing says the limit stands,
|
|
1928
|
+
so they resume.
|
|
1929
|
+
"""
|
|
1930
|
+
now = time.time() if now is None else now
|
|
1931
|
+
retry = S.get(None, "watchdog.quota_retry_hours") * 3600 if retry is None else retry
|
|
1932
|
+
fresh = stop is not None and stop["at"] > (resumed or 0)
|
|
1933
|
+
if park is None:
|
|
1934
|
+
if parked and not fresh:
|
|
1935
|
+
return "resume", {"items": list(parked), "at": resumed or 0, "until": now, "named": False, "lost": True}
|
|
1936
|
+
if not fresh or not (running or parked):
|
|
1937
|
+
return None, None
|
|
1938
|
+
park = quota_park_for(stop, list(parked) + [item for item in running if item not in parked], now, retry)
|
|
1939
|
+
return ("resume" if park["until"] <= now else "park"), park
|
|
1940
|
+
if not set(park.get("items") or ()) & set(parked):
|
|
1941
|
+
return "clear", park
|
|
1942
|
+
if stop is not None and stop["at"] > (park.get("at") or 0):
|
|
1943
|
+
newer = quota_park_for(stop, park.get("items") or (), now, retry)
|
|
1944
|
+
park = dict(park, **{key: newer[key] for key in ("at", "text", "until", "named", "source")})
|
|
1945
|
+
elif spoken and spoken > (park.get("at") or 0):
|
|
1946
|
+
return "resume", dict(park, went_on=True)
|
|
1947
|
+
return ("resume" if (park.get("until") or 0) <= now else "hold"), park
|
|
1948
|
+
|
|
1949
|
+
|
|
1950
|
+
def quota_needs(park):
|
|
1951
|
+
"""The `needs:` a slice parked on quota carries: its reset, or when one nudge tries again where none was named."""
|
|
1952
|
+
return f"{QUOTA_NEEDS} ({'resets' if park.get('named') else 'retry'} {_when(park.get('until'))})"
|
|
1953
|
+
|
|
1954
|
+
|
|
1955
|
+
def parked_on_quota(item):
|
|
1956
|
+
"""Is a board item blocked on quota, as a park writes it: `needs: quota`, with or without its time?"""
|
|
1957
|
+
return (item["notes"].get("needs") or "").split(" (")[0].strip() == QUOTA_NEEDS
|
|
1958
|
+
|
|
1959
|
+
|
|
1960
|
+
def implementers_item(item, cfg):
|
|
1961
|
+
"""Is a board item the implementer's: tagged for no actor, or for the implementer by its role or its name?
|
|
1962
|
+
|
|
1963
|
+
`role:` tags an item for a particular actor, and an item another actor holds does not stop when
|
|
1964
|
+
the implementer does.
|
|
1965
|
+
"""
|
|
1966
|
+
role = (item["notes"].get("role") or "").strip()
|
|
1967
|
+
return not role or A.role_of(role, cfg) == "implementer"
|
|
1968
|
+
|
|
1969
|
+
|
|
1970
|
+
def quota_park(root, cfg, st, dry_run=False, now=None, adapter=None):
|
|
1971
|
+
"""Keep the running slice parked while the implementer's usage limit stands; True when the cycle stands down.
|
|
1972
|
+
|
|
1973
|
+
What `quota_park_step` decides, written. A park moves the implementer's running items to
|
|
1974
|
+
`blocked` with `needs: quota (resets <time>)`, or `needs: quota (retry <time>)` where no reset
|
|
1975
|
+
was named, and tells a person; while it holds, no nudge is sent. When it ends the items go back
|
|
1976
|
+
to `running`, with any `needs:` the implementer had written on them, and the cycle goes on to
|
|
1977
|
+
the nudge, which resumes the implementer's session through its adapter's resume command: the
|
|
1978
|
+
session the stop was read from. A park that ends forgets the backoff the limit ran up, since
|
|
1979
|
+
the nudges that died on it were no agent ignoring them.
|
|
1980
|
+
|
|
1981
|
+
With `keyflip.rotation` on, a park that would be made asks keyflip to rotate a spent window,
|
|
1982
|
+
as the nudge it stands in front of would, and one that holds asks again every fifteen minutes:
|
|
1983
|
+
an account with headroom ends it at once. A board line that already says what the park says is
|
|
1984
|
+
left as it is. Dry, it neither rotates nor writes (QUOTA-PARK).
|
|
1985
|
+
|
|
1986
|
+
It decides on one reading of the board and moves on that same reading. Each move read the board
|
|
1987
|
+
again, so a row a hand moved while the park was decided - to done, say, while keyflip was asked -
|
|
1988
|
+
was moved on from wherever it had gone. A board that changed meanwhile moves nothing: the cycle
|
|
1989
|
+
stands down, and the next decides again on the board as it stands (SLICE-LIFECYCLE-2).
|
|
1990
|
+
"""
|
|
1991
|
+
now = time.time() if now is None else now
|
|
1992
|
+
adapter = A.implementer_adapter(cfg) if adapter is None else adapter
|
|
1993
|
+
edit = A.BoardEdit(root)
|
|
1994
|
+
board = edit.board
|
|
1995
|
+
parked = [item["id"] for item in board["blocked"] if parked_on_quota(item)]
|
|
1996
|
+
said = {item["id"]: item["notes"].get("needs") for item in board["blocked"]}
|
|
1997
|
+
standing = st.get("quota_park") if isinstance(st.get("quota_park"), dict) else None
|
|
1998
|
+
if standing is None and not parked and not A.quota_stops(adapter):
|
|
1999
|
+
return False # nothing parked, and nothing this harness says is a stop
|
|
2000
|
+
last = A.last_message(cfg)
|
|
2001
|
+
action, park = quota_park_step(usage_limit_stop(cfg, st, adapter, last=last), standing, st.get("quota_resumed"),
|
|
2002
|
+
[item["id"] for item in board["running"] if implementers_item(item, cfg)],
|
|
2003
|
+
parked, now, spoken=(last or {}).get("spoken"),
|
|
2004
|
+
retry=S.get(cfg, "watchdog.quota_retry_hours") * 3600)
|
|
2005
|
+
if action is None:
|
|
2006
|
+
return False
|
|
2007
|
+
if action == "park" or (action == "hold" and now - float(st.get("quota_rotation_asked") or 0)
|
|
2008
|
+
>= QUOTA_ROTATION_SPACING):
|
|
2009
|
+
if not dry_run:
|
|
2010
|
+
st["quota_rotation_asked"] = now
|
|
2011
|
+
headroom = A.rotate_if_exhausted(cfg, (adapter.get("resume") or {}).get("argv") or [], "implementer")
|
|
2012
|
+
if headroom["ok"] and headroom["rotated"]:
|
|
2013
|
+
action, park = "resume", dict(park, rotated=headroom["text"])
|
|
2014
|
+
elif action == "hold" and park == standing:
|
|
2015
|
+
save_state(root, st) # when it was asked, so the next cycle does not ask again
|
|
2016
|
+
items = ", ".join(park.get("items") or ())
|
|
2017
|
+
if action in ("clear", "resume"):
|
|
2018
|
+
back = [item for item in park.get("items") or () if item in parked] if action == "resume" else []
|
|
2019
|
+
if action == "resume" and dry_run:
|
|
2020
|
+
print(f"DRY RUN: would end the park of {items} on quota and go on to resume the implementer's session")
|
|
2021
|
+
return True
|
|
2022
|
+
if not dry_run:
|
|
2023
|
+
# The `needs:` each item had before the park: the park's own record, or the board's where the
|
|
2024
|
+
# watchdog's state lost the park (QUOTA-PARK-2).
|
|
2025
|
+
held = {item["id"]: item["notes"].get(QUOTA_HELD) for item in board["blocked"]}
|
|
2026
|
+
for item in back:
|
|
2027
|
+
edit.move(item, "running",
|
|
2028
|
+
{"needs": (park.get("needs") or {}).get(item) or held.get(item), QUOTA_HELD: None},
|
|
2029
|
+
by="watchdog", why=quota_park_ended(park))
|
|
2030
|
+
if not _park_written(edit):
|
|
2031
|
+
return True
|
|
2032
|
+
if action == "clear":
|
|
2033
|
+
print(f"{items} taken off blocked on quota by someone else; the park is over")
|
|
2034
|
+
else:
|
|
2035
|
+
print(f"{quota_park_ended(park)}: {items} {'running again' if back else 'not parked'}; the cycle goes "
|
|
2036
|
+
"on to resume the implementer's session")
|
|
2037
|
+
if not dry_run:
|
|
2038
|
+
st.pop("quota_park", None)
|
|
2039
|
+
st.pop("quota_rotation_asked", None)
|
|
2040
|
+
st["quota_resumed"] = max(park.get("at") or 0, st.get("quota_resumed") or 0)
|
|
2041
|
+
for key in ("idle_answer", "nudge_size", "nudge_fingerprint", "nudge_inputs"):
|
|
2042
|
+
st.pop(key, None)
|
|
2043
|
+
st["attempts"] = 0
|
|
2044
|
+
save_state(root, st)
|
|
2045
|
+
return False
|
|
2046
|
+
until = park.get("until")
|
|
2047
|
+
_FACTS["quota_park"] = {"items": park.get("items"), "until": until, "named": park.get("named")}
|
|
2048
|
+
wait = f"until {_when(until)}" if park.get("named") \
|
|
2049
|
+
else f"with no reset named; one nudge tries again {_when(until)}"
|
|
2050
|
+
if action == "park":
|
|
2051
|
+
if dry_run:
|
|
2052
|
+
print(f"DRY RUN: would park {items} on quota {wait}; not nudging")
|
|
2053
|
+
return True
|
|
2054
|
+
# A `needs:` the implementer wrote on a running item is kept, and given back when the park ends; an
|
|
2055
|
+
# item a lost park left blocked keeps the one it holds on its line, where it is parked again on a
|
|
2056
|
+
# fresh stop, rather than have it cleared (QUOTA-PARK-2).
|
|
2057
|
+
park["needs"] = {item["id"]: item["notes"]["needs"] for item in board["running"]
|
|
2058
|
+
if item["id"] in park["items"] and item["notes"].get("needs")}
|
|
2059
|
+
park["needs"].update({item["id"]: item["notes"][QUOTA_HELD] for item in board["blocked"]
|
|
2060
|
+
if item["id"] in park["items"] and item["notes"].get(QUOTA_HELD)
|
|
2061
|
+
and item["id"] not in park["needs"]})
|
|
2062
|
+
for item in park["items"]:
|
|
2063
|
+
if said.get(item) != quota_needs(park):
|
|
2064
|
+
edit.move(item, "blocked", {"needs": quota_needs(park), QUOTA_HELD: park["needs"].get(item)},
|
|
2065
|
+
by="watchdog", why="the implementer stopped on its usage limit")
|
|
2066
|
+
if not _park_written(edit):
|
|
2067
|
+
return True
|
|
2068
|
+
st["quota_park"] = park
|
|
2069
|
+
save_state(root, st)
|
|
2070
|
+
touch_quota_park(root, st, cfg)
|
|
2071
|
+
print(f"{items} parked on quota {wait}: the implementer stopped on its usage limit; not nudging")
|
|
2072
|
+
return True
|
|
2073
|
+
if park != standing and not dry_run:
|
|
2074
|
+
# A newer stop named another end: the board says the one that stands, and a person is told.
|
|
2075
|
+
for item in park.get("items") or ():
|
|
2076
|
+
if item in parked and said.get(item) != quota_needs(park):
|
|
2077
|
+
edit.move(item, "blocked", {"needs": quota_needs(park)}, by="watchdog",
|
|
2078
|
+
why="a newer stop named another reset")
|
|
2079
|
+
if not _park_written(edit):
|
|
2080
|
+
return True
|
|
2081
|
+
st["quota_park"] = park
|
|
2082
|
+
save_state(root, st)
|
|
2083
|
+
touch_quota_park(root, st, cfg)
|
|
2084
|
+
print(f"{items} parked on quota {wait}; not nudging")
|
|
2085
|
+
return True
|
|
2086
|
+
|
|
2087
|
+
|
|
2088
|
+
def _park_written(edit):
|
|
2089
|
+
"""Write what the park moved on the board it read; False, said, when the board changed meanwhile.
|
|
2090
|
+
|
|
2091
|
+
Nothing moved then, and nothing of the park is saved: the cycle stands down and the next one decides on
|
|
2092
|
+
the board as it stands (SLICE-LIFECYCLE-2).
|
|
2093
|
+
"""
|
|
2094
|
+
try:
|
|
2095
|
+
edit.write()
|
|
2096
|
+
except A.BoardChanged:
|
|
2097
|
+
print("the board changed while the watchdog read it: nothing moved on quota this cycle, and the next "
|
|
2098
|
+
"cycle decides again on the board as it stands")
|
|
2099
|
+
return False
|
|
2100
|
+
return True
|
|
2101
|
+
|
|
2102
|
+
|
|
2103
|
+
def quota_park_ended(park):
|
|
2104
|
+
"""Why a park ended, in the words a cycle's trace gives it."""
|
|
2105
|
+
if park.get("rotated"):
|
|
2106
|
+
return f"keyflip moved the implementer's window to an account with headroom ({park['rotated']})"
|
|
2107
|
+
if park.get("went_on"):
|
|
2108
|
+
return "the transcript went on past the usage limit"
|
|
2109
|
+
if park.get("lost"):
|
|
2110
|
+
return "blocked on quota with no park standing, and nothing read says the limit stands"
|
|
2111
|
+
if park.get("named"):
|
|
2112
|
+
return f"the usage limit reset {_when(park.get('until'))}"
|
|
2113
|
+
return f"no reset was named, and the retry at {_when(park.get('until'))} is due: one nudge tries again"
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def touch_quota_park(root, st, cfg=None):
|
|
2117
|
+
"""Keep a standing park's alarm raised while its slice waits, as the architect's quota alarm is kept (QUOTA-PARK).
|
|
2118
|
+
|
|
2119
|
+
Raised every cycle the park stands, whichever guard ends the cycle, it climbs the ladder and
|
|
2120
|
+
never lapses: told once, it went quiet two hours later and nothing told a person again, while
|
|
2121
|
+
a slice whose reset nobody named waited on. It rings once for what it says - the items and the
|
|
2122
|
+
words they stopped on - and again when those change. A park with a named reset turns red only
|
|
2123
|
+
once it outstands that reset, and is then held until the reset that stands; one with no reset
|
|
2124
|
+
named is red after the ladder's hour and mailed on red's schedule until it ends, since a person
|
|
2125
|
+
may be what ends it.
|
|
2126
|
+
"""
|
|
2127
|
+
park = st.get("quota_park") if isinstance(st.get("quota_park"), dict) else None
|
|
2128
|
+
if not park or not park.get("items"):
|
|
2129
|
+
return False
|
|
2130
|
+
written = _written_in(root) if cfg is None else cfg
|
|
2131
|
+
items, until, named = ", ".join(park["items"]), float(park.get("until") or 0), bool(park.get("named"))
|
|
2132
|
+
red_after = None # the ladder's own hour
|
|
2133
|
+
if named:
|
|
2134
|
+
# Red an hour past its reset, not an hour after it began: a park that ends at its reset asks nothing of anyone.
|
|
2135
|
+
red_after = max(0.0, until - float(park.get("since") or until)) \
|
|
2136
|
+
+ S.get(written, "alarms.red_after_minutes") * 60
|
|
2137
|
+
return notify(language.text(written, "alarm.quota-park-title", project=A.project_key(root), items=items),
|
|
2138
|
+
language.text(written, "alarm.quota-park" if named else "alarm.quota-park-unknown",
|
|
2139
|
+
error=str(park.get("text") or "")[:100], reset=_when(until), items=items),
|
|
2140
|
+
root, key="quota-park", window=6 * 3600, audience="human", quiet_until=until if named else None,
|
|
2141
|
+
red_after=red_after, what=f"{items}: {park.get('text')}")
|
|
2142
|
+
|
|
2143
|
+
|
|
2144
|
+
# ---- a slice past the budget it was given (SLICE-BUDGET) ---------------------------------------
|
|
2145
|
+
#
|
|
2146
|
+
# The watchdog nags a slice that makes no progress and parks one whose window is spent, but nothing
|
|
2147
|
+
# stopped it starting turn after turn on a slice that moved - just slowly, or expensively. What a
|
|
2148
|
+
# slice spent was read only after the fact, by `polyctl gain`. A slice given a budget is parked when its
|
|
2149
|
+
# current run spends past it, and polyctl never gives it more: the architect, or a person, does.
|
|
2150
|
+
|
|
2151
|
+
BUDGET_NEEDS = "budget"
|
|
2152
|
+
BUDGET_WHY = "the slice spent past the budget it was given"
|
|
2153
|
+
|
|
2154
|
+
|
|
2155
|
+
def budget_needs(reading):
|
|
2156
|
+
"""The `needs:` a slice past its budget carries: the reading it is past."""
|
|
2157
|
+
return f"{BUDGET_NEEDS} ({reading})"
|
|
2158
|
+
|
|
2159
|
+
|
|
2160
|
+
def _is_budget_park(row):
|
|
2161
|
+
"""Whether a slice ledger row is the watchdog parking the slice on its budget."""
|
|
2162
|
+
needs = str(((row.get("notes") if isinstance(row.get("notes"), dict) else None) or {}).get("needs") or "")
|
|
2163
|
+
return (row.get("to") == "blocked" and row.get("by") == "watchdog"
|
|
2164
|
+
and (row.get("why") == BUDGET_WHY or needs == BUDGET_NEEDS or needs.startswith(f"{BUDGET_NEEDS} (")))
|
|
2165
|
+
|
|
2166
|
+
|
|
2167
|
+
def budget_start(item_id, moves):
|
|
2168
|
+
"""When the slice's current run started: its recorded budget grant, or None where no line authorizes it.
|
|
2169
|
+
|
|
2170
|
+
A slice's first move to `running` starts its run, and a later move to `running` starts a new
|
|
2171
|
+
run only when a person, the architect, or the watchdog's quota-park resume makes it: the
|
|
2172
|
+
implementer blocking and resuming its own slice keeps the run it already had, so it cannot
|
|
2173
|
+
grant itself a fresh budget. A row moved by hand has no line (docs/slices.md), and a slice that
|
|
2174
|
+
left `running` in the ledger and was later moved back to `running` by hand, or that the
|
|
2175
|
+
watchdog parked on budget without a person or the architect starting it since, has no recorded
|
|
2176
|
+
start for its current run.
|
|
2177
|
+
"""
|
|
2178
|
+
start, started, parked, running = None, False, False, False
|
|
2179
|
+
for row in moves:
|
|
2180
|
+
if row.get("slice") != item_id:
|
|
2181
|
+
continue
|
|
2182
|
+
to, by, at = row.get("to"), row.get("by"), row.get("at")
|
|
2183
|
+
stamp = (float(at) if isinstance(at, (int, float)) and not isinstance(at, bool) and math.isfinite(at)
|
|
2184
|
+
else None)
|
|
2185
|
+
if to == "running":
|
|
2186
|
+
if by not in ("implementer", "watchdog", None) or (by == "watchdog" and not parked):
|
|
2187
|
+
start, started, parked, running = stamp, True, False, True
|
|
2188
|
+
elif not started:
|
|
2189
|
+
start, started, parked, running = stamp, True, False, True
|
|
2190
|
+
elif not parked:
|
|
2191
|
+
running = True
|
|
2192
|
+
elif _is_budget_park(row):
|
|
2193
|
+
parked, running = True, False
|
|
2194
|
+
else:
|
|
2195
|
+
running = False
|
|
2196
|
+
return start if running and not parked else None
|
|
2197
|
+
|
|
2198
|
+
|
|
2199
|
+
def budget_runs(board, moves, cfg):
|
|
2200
|
+
"""The running intervals [(start, end, closed)] of each implementer slice, from the slice ledger and board.
|
|
2201
|
+
|
|
2202
|
+
A move to `running` opens an interval and a move elsewhere closes it. A slice that left `running`
|
|
2203
|
+
by hand - landing moves a slice to `done` by hand (docs/slices.md) - closes by the next start
|
|
2204
|
+
recorded for another slice, or stays open when none followed. An implementer slice on `running`
|
|
2205
|
+
with no recorded start is open from its last recorded departure from `running` (or the beginning
|
|
2206
|
+
of time), so its share of a turn's spend with another running slice is never guessed away.
|
|
2207
|
+
"""
|
|
2208
|
+
items = {item["id"]: item for section in board.values() for item in section}
|
|
2209
|
+
running = {item["id"] for item in board["running"] if implementers_item(item, cfg)}
|
|
2210
|
+
events = [(row.get("slice"), row.get("to"), float(row["at"])) for row in moves
|
|
2211
|
+
if isinstance(row.get("slice"), str) and row.get("slice")
|
|
2212
|
+
and (row["slice"] not in items or implementers_item(items[row["slice"]], cfg))
|
|
2213
|
+
and isinstance(row.get("at"), (int, float)) and not isinstance(row.get("at"), bool)
|
|
2214
|
+
and math.isfinite(row["at"])]
|
|
2215
|
+
runs, opened, closed_at = {}, {}, {}
|
|
2216
|
+
for sid, to, at in events:
|
|
2217
|
+
if to == "running":
|
|
2218
|
+
if sid in opened:
|
|
2219
|
+
runs.setdefault(sid, []).append((opened[sid], at, True))
|
|
2220
|
+
opened[sid] = at
|
|
2221
|
+
elif sid in opened:
|
|
2222
|
+
runs.setdefault(sid, []).append((opened.pop(sid), at, True))
|
|
2223
|
+
closed_at[sid] = at
|
|
2224
|
+
for sid, lo in opened.items():
|
|
2225
|
+
if sid in running:
|
|
2226
|
+
runs.setdefault(sid, []).append((lo, float("inf"), False))
|
|
2227
|
+
else:
|
|
2228
|
+
later = [at for other, to, at in events if to == "running" and other != sid and at >= lo]
|
|
2229
|
+
runs.setdefault(sid, []).append((lo, later[0], True) if later else (lo, float("inf"), False))
|
|
2230
|
+
for sid in running - set(opened):
|
|
2231
|
+
runs.setdefault(sid, []).append((closed_at.get(sid, -float("inf")), float("inf"), False))
|
|
2232
|
+
return runs
|
|
2233
|
+
|
|
2234
|
+
|
|
2235
|
+
def _read_transcript_records(msgs):
|
|
2236
|
+
"""The implementer's transcript records; raises when the transcript store is missing or fails to read."""
|
|
2237
|
+
if not msgs or not os.path.exists(msgs):
|
|
2238
|
+
raise OSError("the transcript could not be read")
|
|
2239
|
+
if isinstance(msgs, A.DatabaseTail):
|
|
2240
|
+
declared = msgs.record
|
|
2241
|
+
table, session_key = A._store_name(declared.get("table")), A._store_name(declared.get("session_key"))
|
|
2242
|
+
order = A._store_name(declared.get("order")) or A._store_name(declared.get("time"))
|
|
2243
|
+
if not (msgs.session and table and session_key and order):
|
|
2244
|
+
raise OSError("the transcript could not be read")
|
|
2245
|
+
connection = msgs._connect()
|
|
2246
|
+
if connection is None:
|
|
2247
|
+
raise OSError("the transcript could not be read")
|
|
2248
|
+
try:
|
|
2249
|
+
connection.execute(
|
|
2250
|
+
f"SELECT 1 FROM {table} WHERE {session_key} = ? ORDER BY {order} DESC LIMIT 1",
|
|
2251
|
+
(msgs.session,),
|
|
2252
|
+
).fetchall()
|
|
2253
|
+
children = A._block(declared, "children")
|
|
2254
|
+
child_table = A._store_name(children.get("table"))
|
|
2255
|
+
child_key = A._store_name(children.get("key"))
|
|
2256
|
+
child_parent = A._store_name(children.get("parent_key"))
|
|
2257
|
+
child_order = A._store_name(children.get("order")) or A._store_name(children.get("time"))
|
|
2258
|
+
child_session = A._store_name(children.get("session_key")) or child_key
|
|
2259
|
+
if child_table and children.get("as") and child_key and child_parent and child_order:
|
|
2260
|
+
connection.execute(
|
|
2261
|
+
f"SELECT 1 FROM {child_table} WHERE {child_session} = ? ORDER BY {child_order} ASC LIMIT 1",
|
|
2262
|
+
(msgs.session,),
|
|
2263
|
+
).fetchall()
|
|
2264
|
+
finally:
|
|
2265
|
+
connection.close()
|
|
2266
|
+
return A.read_tail(msgs, 400_000_000)
|
|
2267
|
+
with A._open_regular(msgs) as fh:
|
|
2268
|
+
size = os.fstat(fh.fileno()).st_size
|
|
2269
|
+
fh.seek(max(0, size - 400_000_000))
|
|
2270
|
+
blob = fh.read()
|
|
2271
|
+
recs = A.read_tail(msgs, 400_000_000)
|
|
2272
|
+
if not recs and blob.strip():
|
|
2273
|
+
raise OSError("the transcript could not be read")
|
|
2274
|
+
return recs
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
def _read_subagent_records(path):
|
|
2278
|
+
"""A subagent transcript's records; raises when the subagent transcript fails to read."""
|
|
2279
|
+
with A._open_regular(path) as fh:
|
|
2280
|
+
blob = fh.read()
|
|
2281
|
+
recs = A.subagent_records(path)
|
|
2282
|
+
if not recs and blob.strip():
|
|
2283
|
+
raise OSError("the transcript could not be read")
|
|
2284
|
+
return recs
|
|
2285
|
+
|
|
2286
|
+
|
|
2287
|
+
def _usage_events(cfg):
|
|
2288
|
+
"""(unit, [(window_start, at, grown)]) from the implementer's transcript, in the unit its adapter declares.
|
|
2289
|
+
|
|
2290
|
+
Each usage record is read in its turn (`next_turn`, `add_usage`), so per-response records and
|
|
2291
|
+
later records of a turn that already wrote usage before a slice started count from the slice's
|
|
2292
|
+
start onwards, and a whole-turn usage summary whose turn opened before the start and closed after
|
|
2293
|
+
it keeps its turn opening time (`window_start < start`) and is charged in full to the slice when no
|
|
2294
|
+
other slice ran in that span. Replies the harness wrote in the model's place (`in_place_reply`)
|
|
2295
|
+
count nothing, and an adapter that declares no usage telemetry or a transcript store that fails to
|
|
2296
|
+
read raises so a spend budget never passes silently.
|
|
2297
|
+
"""
|
|
2298
|
+
msgs, _ = A.session_paths(cfg)
|
|
2299
|
+
adapter = A.implementer_adapter(cfg)
|
|
2300
|
+
shape = A.transcript_shape(adapter)
|
|
2301
|
+
usage = shape["usage"]
|
|
2302
|
+
unit = shape["unit"] or "unit"
|
|
2303
|
+
if not usage:
|
|
2304
|
+
name = A.implementer_adapter_id(cfg) or str((cfg.get("implementer") or {}).get("adapter") or "unknown")
|
|
2305
|
+
raise ValueError(f"adapter {name!r} declares no transcript usage telemetry")
|
|
2306
|
+
recs = _read_transcript_records(msgs)
|
|
2307
|
+
turn, counted, events, when = None, {}, [], None
|
|
2308
|
+
for rec, subagent in A.with_subagents(recs, shape, msgs, read=_read_subagent_records):
|
|
2309
|
+
when = A._ledger_time(A.record_time(rec, shape)) or when
|
|
2310
|
+
kind = A.record_kind(rec, shape)
|
|
2311
|
+
turn, opened = A.next_turn(turn, rec, shape, subagent)
|
|
2312
|
+
if opened:
|
|
2313
|
+
turn["at"] = when
|
|
2314
|
+
turn["steps"] = 0
|
|
2315
|
+
body = A.record_body(rec, shape)
|
|
2316
|
+
if body is None or kind != usage["type"] or A.in_place_reply(rec, shape):
|
|
2317
|
+
continue
|
|
2318
|
+
turn = {"at": when, "steps": 0} if turn is None else turn
|
|
2319
|
+
grown = A.add_usage(turn, body, usage, counted, subagent)
|
|
2320
|
+
if when is not None and grown > 0:
|
|
2321
|
+
steps = turn.get("steps", 0)
|
|
2322
|
+
lo = (when if usage["reading"] == "per-response" or steps > 0
|
|
2323
|
+
else (turn.get("at") if turn.get("at") is not None else when))
|
|
2324
|
+
events.append((lo, when, float(grown)))
|
|
2325
|
+
turn["at"] = when
|
|
2326
|
+
turn["steps"] = steps + 1
|
|
2327
|
+
return unit, events
|
|
2328
|
+
|
|
2329
|
+
|
|
2330
|
+
def budget_spend_for(item_id, start, runs, events):
|
|
2331
|
+
"""(spent, shared) for `item_id`'s run since `start`: what it spent alone, and which slices shared a turn's spend."""
|
|
2332
|
+
spent, shared = 0.0, []
|
|
2333
|
+
for lo, when, grown in events:
|
|
2334
|
+
if when < start:
|
|
2335
|
+
continue
|
|
2336
|
+
span = max(lo, start)
|
|
2337
|
+
active = sorted(sid for sid, intervals in runs.items()
|
|
2338
|
+
if any(r_lo <= when and (span < r_hi if closed else span <= r_hi)
|
|
2339
|
+
for r_lo, r_hi, closed in intervals))
|
|
2340
|
+
if len(active) > 1 and item_id in active:
|
|
2341
|
+
for sid in active:
|
|
2342
|
+
if sid not in shared:
|
|
2343
|
+
shared.append(sid)
|
|
2344
|
+
elif active == [item_id] or not active:
|
|
2345
|
+
spent += grown
|
|
2346
|
+
return spent, shared
|
|
2347
|
+
|
|
2348
|
+
|
|
2349
|
+
def budget_reading(minutes, minute_budget, spent, unit, spend_budget, spend_missing=None):
|
|
2350
|
+
"""The reading a park writes: each budget the run is past or could not be read against."""
|
|
2351
|
+
parts = []
|
|
2352
|
+
if spend_budget:
|
|
2353
|
+
if spend_missing:
|
|
2354
|
+
parts.append(spend_missing)
|
|
2355
|
+
elif spent is not None and spent > spend_budget:
|
|
2356
|
+
parts.append(f"spent {spent:g} {unit} of {spend_budget:g}")
|
|
2357
|
+
if minute_budget and minutes > minute_budget:
|
|
2358
|
+
whole = max(int(minutes), int(minute_budget) + 1)
|
|
2359
|
+
parts.append(f"{whole}m of {minute_budget}m")
|
|
2360
|
+
return "; ".join(parts)
|
|
2361
|
+
|
|
2362
|
+
|
|
2363
|
+
def _committed_project_config(root):
|
|
2364
|
+
"""The project `.polyctl/config.json` from `HEAD`, or `{}` when no committed config can be read."""
|
|
2365
|
+
if not root:
|
|
2366
|
+
return {}
|
|
2367
|
+
try:
|
|
2368
|
+
raw = json.loads(A._git_output(root, "show", "HEAD:.polyctl/config.json") or "{}")
|
|
2369
|
+
except Exception:
|
|
2370
|
+
return {}
|
|
2371
|
+
return raw if isinstance(raw, dict) else {}
|
|
2372
|
+
|
|
2373
|
+
|
|
2374
|
+
def _effective_slice_budgets(root, cfg):
|
|
2375
|
+
"""(minute_budget, spend_budget) holding the tightest positive caps across config, HEAD, and machine settings.
|
|
2376
|
+
|
|
2377
|
+
A cap the implementer cannot move is the machine settings' cap, outside the tree. A cap
|
|
2378
|
+
committed in `.polyctl/config.json` counts while `HEAD` shows it, and is no guard against an
|
|
2379
|
+
implementer that moves `HEAD` or breaks the git read (which reads as no committed cap); a
|
|
2380
|
+
person widens or switches off a cap with `0` where it is set.
|
|
2381
|
+
"""
|
|
2382
|
+
committed = _committed_project_config(root)
|
|
2383
|
+
out = {}
|
|
2384
|
+
for key, kind in (("slice_budget_minutes", int), ("slice_budget_spend", float)):
|
|
2385
|
+
candidates = [S.get(cfg, key), S.get({}, key)]
|
|
2386
|
+
if key in committed and S.usable(key, committed[key]):
|
|
2387
|
+
candidates.append(kind(committed[key]))
|
|
2388
|
+
positive = [kind(val) for val in candidates if val and val > 0]
|
|
2389
|
+
out[key] = min(positive) if positive else kind(0)
|
|
2390
|
+
return out["slice_budget_minutes"], out["slice_budget_spend"]
|
|
2391
|
+
|
|
2392
|
+
|
|
2393
|
+
def slice_budget(root, cfg, dry_run=False, now=None):
|
|
2394
|
+
"""Park the implementer's running slices that spent past their budget or lack a reading it needs (SLICE-BUDGET).
|
|
2395
|
+
|
|
2396
|
+
A budget is per run, not per slice: `slice_budget_minutes` of wall time and `slice_budget_spend`
|
|
2397
|
+
of the implementer's own spend - in the unit its adapter's transcript records - since the slice's
|
|
2398
|
+
newest recorded move to running, including the spend of the turn that started it (charged in full
|
|
2399
|
+
when a whole-turn usage summary opened before the start and closed after it), attributed to the
|
|
2400
|
+
slice running then. A slice blocked on quota and resumed starts a new run, and one a person starts
|
|
2401
|
+
again gets its budget again.
|
|
2402
|
+
|
|
2403
|
+
The guard fails closed: every reading a configured budget cannot make - no recorded start, an
|
|
2404
|
+
unreadable slice ledger, a transcript store that fails to read, spend that overlapping slices make
|
|
2405
|
+
unattributable, or usage the adapter does not report for a spend cap - parks the slice with
|
|
2406
|
+
`needs: budget (<reading>)`, by the watchdog, and tells the architect; it never reads a missing
|
|
2407
|
+
reading as zero or as within budget. The implementer may not lift a watchdog's block, and nothing
|
|
2408
|
+
resumes it by itself: only the architect's or a person's `polyctl slice start` does, and the run the
|
|
2409
|
+
budget counts from starts again there. Nothing is read when no budget is set.
|
|
2410
|
+
"""
|
|
2411
|
+
now = time.time() if now is None else now
|
|
2412
|
+
minute_budget, spend_budget = _effective_slice_budgets(root, cfg)
|
|
2413
|
+
if not minute_budget and not spend_budget:
|
|
2414
|
+
return False
|
|
2415
|
+
edit = A.BoardEdit(root)
|
|
2416
|
+
running = [item for item in edit.board["running"] if implementers_item(item, cfg)]
|
|
2417
|
+
try:
|
|
2418
|
+
moves = A.slice_moves(root)
|
|
2419
|
+
except Exception as exc:
|
|
2420
|
+
if not running:
|
|
2421
|
+
print(f"the slice ledger could not be read ({exc}); no slice can be judged against its budget, "
|
|
2422
|
+
"so not nudging")
|
|
2423
|
+
return True
|
|
2424
|
+
print(f"the slice ledger could not be read ({exc}); parking {', '.join(item['id'] for item in running)}")
|
|
2425
|
+
overdue = [(item["id"], "the slice ledger could not be read") for item in running]
|
|
2426
|
+
held = {}
|
|
2427
|
+
for item in running:
|
|
2428
|
+
prior_needs = item["notes"].get(A.HELD_NEEDS) or item["notes"].get("needs")
|
|
2429
|
+
held[item["id"]] = None if str(prior_needs or "").startswith(f"{BUDGET_NEEDS} (") else prior_needs
|
|
2430
|
+
moves = None
|
|
2431
|
+
if moves is not None:
|
|
2432
|
+
if not running:
|
|
2433
|
+
return False
|
|
2434
|
+
starts = [(item, budget_start(item["id"], moves)) for item in running]
|
|
2435
|
+
runs = budget_runs(edit.board, moves, cfg) if spend_budget else {}
|
|
2436
|
+
unit, events, spend_err = None, [], None
|
|
2437
|
+
if spend_budget and any(start is not None and start <= now for _, start in starts):
|
|
2438
|
+
try:
|
|
2439
|
+
unit, events = _usage_events(cfg)
|
|
2440
|
+
except Exception as exc:
|
|
2441
|
+
spend_err = exc
|
|
2442
|
+
overdue, held = [], {}
|
|
2443
|
+
for item, start in starts:
|
|
2444
|
+
if start is None or start > now:
|
|
2445
|
+
reading = "no recorded start"
|
|
2446
|
+
else:
|
|
2447
|
+
minutes = max(0.0, (now - start) / 60.0)
|
|
2448
|
+
spent, spend_missing = None, None
|
|
2449
|
+
if spend_budget:
|
|
2450
|
+
if spend_err is not None:
|
|
2451
|
+
if isinstance(spend_err, ValueError) and "declares no transcript usage telemetry" in str(spend_err):
|
|
2452
|
+
spend_missing = str(spend_err)
|
|
2453
|
+
else:
|
|
2454
|
+
spend_missing = "the transcript could not be read"
|
|
2455
|
+
print(f"{item['id']}: its spend could not be read ({spend_err}); parking")
|
|
2456
|
+
else:
|
|
2457
|
+
spent, shared = budget_spend_for(item["id"], start, runs, events)
|
|
2458
|
+
if shared:
|
|
2459
|
+
spent = None
|
|
2460
|
+
spend_missing = f"spend cannot be attributed across {', '.join(shared)}"
|
|
2461
|
+
reading = budget_reading(minutes, minute_budget, spent, unit, spend_budget,
|
|
2462
|
+
spend_missing=spend_missing)
|
|
2463
|
+
if reading:
|
|
2464
|
+
overdue.append((item["id"], reading))
|
|
2465
|
+
prior_needs = item["notes"].get(A.HELD_NEEDS) or item["notes"].get("needs")
|
|
2466
|
+
held[item["id"]] = None if str(prior_needs or "").startswith(f"{BUDGET_NEEDS} (") else prior_needs
|
|
2467
|
+
if not overdue:
|
|
2468
|
+
return False
|
|
2469
|
+
_FACTS["slice_budget"] = {"items": [item for item, _ in overdue],
|
|
2470
|
+
"readings": {item: reading for item, reading in overdue}}
|
|
2471
|
+
readings = "; ".join(f"{item}: {reading}" for item, reading in overdue)
|
|
2472
|
+
if dry_run:
|
|
2473
|
+
print(f"DRY RUN: would park {readings} on the budget it was given; not nudging")
|
|
2474
|
+
return True
|
|
2475
|
+
for item, reading in overdue:
|
|
2476
|
+
edit.move(item, "blocked", {"needs": budget_needs(reading), A.HELD_NEEDS: held.get(item)},
|
|
2477
|
+
by="watchdog", why=BUDGET_WHY)
|
|
2478
|
+
if moves is None:
|
|
2479
|
+
edit.moves.clear()
|
|
2480
|
+
try:
|
|
2481
|
+
edit.write()
|
|
2482
|
+
except A.BoardChanged:
|
|
2483
|
+
print("the board changed while the watchdog read it: nothing moved on budget this cycle, and the next "
|
|
2484
|
+
"cycle decides again on the board as it stands")
|
|
2485
|
+
return True
|
|
2486
|
+
names = ", ".join(item for item, _ in overdue)
|
|
2487
|
+
notify(language.text(cfg, "alarm.slice-budget-title", project=A.project_key(root), items=names),
|
|
2488
|
+
language.text(cfg, "alarm.slice-budget", reading=readings),
|
|
2489
|
+
root, key="slice-budget", window=6 * 3600, audience="architect", what=readings)
|
|
2490
|
+
print(f"{names} parked on the budget it was given ({readings}); not nudging")
|
|
2491
|
+
return True
|
|
2492
|
+
|
|
2493
|
+
|
|
2494
|
+
def main():
|
|
2495
|
+
# What a cycle decided is printed, board titles and mail subjects included, to a terminal, a pipe
|
|
2496
|
+
# or the log a scheduled job keeps: in UTF-8, whatever the platform's code page (#71).
|
|
2497
|
+
A.utf8_streams()
|
|
2498
|
+
p = argparse.ArgumentParser()
|
|
2499
|
+
p.add_argument("--root", required=True)
|
|
2500
|
+
p.add_argument("--idle-minutes", type=float, default=None,
|
|
2501
|
+
help="default: the project's watchdog.idle_minutes setting")
|
|
2502
|
+
p.add_argument("--dry-run", action="store_true")
|
|
2503
|
+
# The default prompt encodes two rules learned the expensive way. Park-and-
|
|
2504
|
+
# continue: a blocked slice must not stop the run, because the architect can
|
|
2505
|
+
# hit a quota limit and the human can be away. And a pointer to one canonical
|
|
2506
|
+
# authority file: when "what may I do" is spread across mail, a prompt and
|
|
2507
|
+
# months-old instructions, an agent facing an apparent conflict refuses and
|
|
2508
|
+
# waits — which is safe, and cost this project four hours with 7,000 lines of
|
|
2509
|
+
# finished work sitting uncommitted.
|
|
2510
|
+
p.add_argument("--prompt", default=None, help="default: the project's nudge, in its language")
|
|
2511
|
+
args = p.parse_args()
|
|
2512
|
+
return run(args)
|
|
2513
|
+
|
|
2514
|
+
|
|
2515
|
+
CYCLE_LOCK = A.PROJECT_FILES["watchdog-cycle-lock"]
|
|
2516
|
+
|
|
2517
|
+
|
|
2518
|
+
def run(args):
|
|
2519
|
+
"""One cycle, traced and — unless dry — recorded.
|
|
2520
|
+
|
|
2521
|
+
Live cycles take a per-project lock first. The scheduled job and `polyctl catchup`
|
|
2522
|
+
both ran a cycle, both passed the process scan before either spawned, and both
|
|
2523
|
+
resumed the implementer: the two-writer incident the scan exists to prevent.
|
|
2524
|
+
A cycle that finds the lock held stands down.
|
|
2525
|
+
"""
|
|
2526
|
+
_TRACE.clear()
|
|
2527
|
+
_FACTS.clear()
|
|
2528
|
+
root = os.path.abspath(os.path.expanduser(args.root))
|
|
2529
|
+
if getattr(args, "idle_minutes", None) is None:
|
|
2530
|
+
args.idle_minutes = S.get(A.load_config(root), "watchdog.idle_minutes")
|
|
2531
|
+
started = time.time()
|
|
2532
|
+
if args.dry_run:
|
|
2533
|
+
try:
|
|
2534
|
+
return _cycle(args, root)
|
|
2535
|
+
finally:
|
|
2536
|
+
record_cycle(root, args, started)
|
|
2537
|
+
from .storage import LedgerLockTimeout, _exclusive_lock
|
|
2538
|
+
key = A.project_key(root)
|
|
2539
|
+
lock = os.path.join(STATE_DIR, CYCLE_LOCK.format(key=key))
|
|
2540
|
+
try:
|
|
2541
|
+
os.makedirs(STATE_DIR, exist_ok=True)
|
|
2542
|
+
with _exclusive_lock(lock, timeout=0):
|
|
2543
|
+
try:
|
|
2544
|
+
A.bound_observation_logs(root, STATE_DIR, busy=_logs_written(root)) # every store is bounded (#50)
|
|
2545
|
+
except OSError:
|
|
2546
|
+
pass
|
|
2547
|
+
try:
|
|
2548
|
+
code = _cycle(args, root)
|
|
2549
|
+
args.ran = True # this process's cycle ran to its end (catchup reads it)
|
|
2550
|
+
return code
|
|
2551
|
+
finally:
|
|
2552
|
+
record_cycle(root, args, started)
|
|
2553
|
+
except LedgerLockTimeout as exc:
|
|
2554
|
+
if getattr(exc, "path", lock) != lock:
|
|
2555
|
+
# A lock the cycle took inside it was held too long: no other cycle, and standing down hid it (WATCHDOG-LOCK).
|
|
2556
|
+
print(f"the cycle could not take {exc.path} in time; it ends here, and the next cycle tries again")
|
|
2557
|
+
return 1
|
|
2558
|
+
print("another watchdog cycle is running for this project; standing down")
|
|
2559
|
+
return 0
|
|
2560
|
+
|
|
2561
|
+
|
|
2562
|
+
def _cycle(args, root):
|
|
2563
|
+
"""Run one cycle with dry-run effects suppressed across nested helpers.
|
|
2564
|
+
|
|
2565
|
+
A resume notice the cycle owes is sent here, once every guard has had its say and
|
|
2566
|
+
whichever of them ended the cycle (RESUME-QUIET). A process table the platform could
|
|
2567
|
+
not read ends the cycle at the guard that asked it (WINDOWS-TABLE-CLOSED).
|
|
2568
|
+
"""
|
|
2569
|
+
token = _DRY_RUN.set(bool(args.dry_run))
|
|
2570
|
+
resuming = _RESUME.set(None)
|
|
2571
|
+
try:
|
|
2572
|
+
return _cycle_impl(args, root)
|
|
2573
|
+
except TableUnreadable as exc:
|
|
2574
|
+
return _stand_down_unseen(root, exc)
|
|
2575
|
+
finally:
|
|
2576
|
+
owed = _RESUME.get()
|
|
2577
|
+
_RESUME.reset(resuming)
|
|
2578
|
+
try:
|
|
2579
|
+
if owed is not None:
|
|
2580
|
+
announce_resume(root, owed, dry_run=bool(args.dry_run))
|
|
2581
|
+
except Exception as exc: # a notice that fails must not take the cycle's record
|
|
2582
|
+
_note_above_verdict(f"the resume notice was not sent: {type(exc).__name__}: {str(exc)[:120]}")
|
|
2583
|
+
finally:
|
|
2584
|
+
_DRY_RUN.reset(token)
|
|
2585
|
+
|
|
2586
|
+
|
|
2587
|
+
def _stand_down_unseen(root, exc):
|
|
2588
|
+
"""End a cycle that could not read the process table: it starts, wakes and reaps nothing, and a person is told
|
|
2589
|
+
why (WINDOWS-TABLE-CLOSED).
|
|
2590
|
+
|
|
2591
|
+
Every guard that keeps a second turn out of a tree, a second architect out of a session and a
|
|
2592
|
+
person's session out of the reaper asks the table what runs. A table the platform could not read
|
|
2593
|
+
- on Windows, a PowerShell that was missing or did not answer - was an empty one, and each of those
|
|
2594
|
+
guards read it as nothing running. The alarm rings once for the platform it names and climbs as
|
|
2595
|
+
any other while the table stays unread.
|
|
2596
|
+
"""
|
|
2597
|
+
try:
|
|
2598
|
+
notify(f"{A.project_key(root)}: the process table cannot be read",
|
|
2599
|
+
f"{exc}. Until it can see what runs in the tree the watchdog starts no turn, wakes nobody and "
|
|
2600
|
+
"reaps nothing.", root, key="process-table-unreadable", window=6 * 3600, audience="human",
|
|
2601
|
+
what=f"unreadable on {exc.platform}")
|
|
2602
|
+
except Exception as told:
|
|
2603
|
+
print(f"the alarm for the process table could not be raised ({type(told).__name__})")
|
|
2604
|
+
print(f"{exc}; this cycle starts no turn, wakes nobody and reaps nothing, and the next cycle looks again")
|
|
2605
|
+
return 1
|
|
2606
|
+
|
|
2607
|
+
|
|
2608
|
+
def _sample_credits(root, st, adapter_id, project, now=None):
|
|
2609
|
+
"""Read the implementer's own credit account at most once per half hour and act on it.
|
|
2610
|
+
|
|
2611
|
+
Exhaustion is reported from the reading itself, on the first observation: a
|
|
2612
|
+
projection has nothing to say once the plan has already run out, which is the
|
|
2613
|
+
moment it matters. The burn rate still warns ahead of it. A reading that cannot
|
|
2614
|
+
be taken is recorded as a broken check, which `polyctl doctor` shows, instead of
|
|
2615
|
+
being skipped: a silent check and a passing check must never look the same. A
|
|
2616
|
+
failed attempt waits the same half hour before the next one.
|
|
2617
|
+
|
|
2618
|
+
Either alarm has a known end, the reset the reading names, so it is mailed once
|
|
2619
|
+
and held until then (#40). Raised without it, an exhausted plan whose reset was
|
|
2620
|
+
ten days away mailed every six hours. What it says - run out, or will - is named
|
|
2621
|
+
per account, so the day a projection comes true is still told (NOTICE-NOISE).
|
|
2622
|
+
|
|
2623
|
+
The account is the one the package's adapter of `adapter_id`, the implementer's,
|
|
2624
|
+
declares a lookup for (ACCOUNT-READERS). The first shipped adapter's was read whatever
|
|
2625
|
+
the implementer ran, once a layer the implementer can write said its adapter bills
|
|
2626
|
+
through an API: an implementer on a harness that bills no account polyctl can read was
|
|
2627
|
+
sampled, and alarmed, on another harness's. It has no samples and no credits alarm
|
|
2628
|
+
now. A sample and a broken check name the adapter they were read through, and only
|
|
2629
|
+
that adapter's samples project its burn rate.
|
|
2630
|
+
"""
|
|
2631
|
+
now = now or time.time()
|
|
2632
|
+
if now - max(st.get("last_credit_sample", 0), st.get("last_credit_attempt", 0)) <= 1800:
|
|
2633
|
+
return
|
|
2634
|
+
if not A.usage_api(adapter_id):
|
|
2635
|
+
return
|
|
2636
|
+
try:
|
|
2637
|
+
acct = A.account_usage(adapter_id=adapter_id)
|
|
2638
|
+
except Exception as exc:
|
|
2639
|
+
acct = {"error": f"the usage check raised {type(exc).__name__}"}
|
|
2640
|
+
if acct is None:
|
|
2641
|
+
acct = {"error": "the declared usage lookup returned no reading; check the CLI's login and retry"}
|
|
2642
|
+
st["last_credit_attempt"] = now
|
|
2643
|
+
if not acct.get("limit"):
|
|
2644
|
+
reason = acct.get("error") or ("the CLI's token has expired" if acct.get("expired")
|
|
2645
|
+
else "the provider returned no credit limit")
|
|
2646
|
+
st["credit_check_problem"] = {"at": int(now), "reason": reason, "adapter": adapter_id}
|
|
2647
|
+
save_state(root, st)
|
|
2648
|
+
return
|
|
2649
|
+
st.pop("credit_check_problem", None)
|
|
2650
|
+
A.record_credit_sample(root, acct.get("used", 0), acct["limit"], acct.get("reset_at"),
|
|
2651
|
+
account=acct.get("account"), adapter=adapter_id)
|
|
2652
|
+
st["last_credit_sample"] = now
|
|
2653
|
+
save_state(root, st)
|
|
2654
|
+
used, limit = float(acct.get("used") or 0), float(acct["limit"])
|
|
2655
|
+
if used >= limit:
|
|
2656
|
+
reading = A.notice_evidence("credit_reading", [
|
|
2657
|
+
{"value": f"{used:.0f}/{limit:.0f}", "at": int(now),
|
|
2658
|
+
"source": f"GetUsageLimits, account {acct.get('account')}"}])
|
|
2659
|
+
notify(f"{project}: credits exhausted",
|
|
2660
|
+
f"{used:.0f}/{limit:.0f} used; the plan is spent and only overage, if enabled, runs "
|
|
2661
|
+
f"until the reset. New account (keyflip) or `polyctl features off …`", root,
|
|
2662
|
+
key="credits-exhaust", window=6 * 3600, audience="human", level="red", evidence=reading,
|
|
2663
|
+
quiet_until=credit_reset(acct.get("reset_at"), now), what=credits_told("exhausted", acct.get("account")))
|
|
2664
|
+
return
|
|
2665
|
+
br = A.burn_rate(root, adapter_id)
|
|
2666
|
+
if br and br["before_reset"]:
|
|
2667
|
+
evidence = A.notice_evidence("burn_rate", [
|
|
2668
|
+
{"value": f"{sample['used']:.0f}/{sample['limit']:.0f}", "at": sample.get("at"),
|
|
2669
|
+
"source": f"GetUsageLimits, account {sample.get('account')}"} for sample in br["samples"]])
|
|
2670
|
+
notify(f"{project}: credits run out {time.strftime('%d %b', time.localtime(br['exhausts_at']))}",
|
|
2671
|
+
f"{br['used']:.0f}/{br['limit']:.0f} at {br['per_day']:.0f}/day; the reset is later. "
|
|
2672
|
+
f"New account (keyflip) or `polyctl features off …`", root, key="credits-exhaust",
|
|
2673
|
+
window=6 * 3600, audience="human", level="red", evidence=evidence,
|
|
2674
|
+
quiet_until=credit_reset(br.get("reset_at"), now), what=credits_told("projected", br.get("account")))
|
|
2675
|
+
|
|
2676
|
+
|
|
2677
|
+
def credit_reset(value, now=None):
|
|
2678
|
+
"""The reset a credit reading names, in epoch seconds; None when it names none it can mean (NOTICE-NOISE).
|
|
2679
|
+
|
|
2680
|
+
The provider gives seconds. A date string or milliseconds is no known end: an alarm
|
|
2681
|
+
held on a misread date would stay silent long past the day it should have spoken,
|
|
2682
|
+
while one without a known end only repeats. A reset already past holds nothing.
|
|
2683
|
+
"""
|
|
2684
|
+
now = time.time() if now is None else now
|
|
2685
|
+
try:
|
|
2686
|
+
reset = float(value)
|
|
2687
|
+
except (TypeError, ValueError):
|
|
2688
|
+
return None
|
|
2689
|
+
return reset if 0 < reset <= now + 400 * 86400 else None
|
|
2690
|
+
|
|
2691
|
+
|
|
2692
|
+
def credits_told(kind, account):
|
|
2693
|
+
"""What a credits alarm says, for the ladder: run out or projected, and whose (NOTICE-NOISE)."""
|
|
2694
|
+
return f"{kind}:{account or 'unnamed'}"
|
|
2695
|
+
|
|
2696
|
+
|
|
2697
|
+
DECISION_HUMAN_AFTER = S.default("decisions.human_after_minutes") * 60
|
|
2698
|
+
|
|
2699
|
+
|
|
2700
|
+
def escalate_open_decisions(root, project, dry_run=False, now=None, since=0):
|
|
2701
|
+
"""A decision nobody has answered reaches a person (#20).
|
|
2702
|
+
|
|
2703
|
+
Fifteen minutes after it was asked (`decisions.human_after_minutes`) it rings
|
|
2704
|
+
orange on the human channel, once for the decision it names, and the alarm ladder
|
|
2705
|
+
turns it red after its hour and mails on red's schedule until it is answered. The wake
|
|
2706
|
+
is not waited for: on 2026-09-07 a decision stood three hours while the only
|
|
2707
|
+
notice about it was held for an architect that never came. An answered
|
|
2708
|
+
decision stops being raised and its alarm ends on its own. Returns the ids
|
|
2709
|
+
that are ringing. One asked before a resume waits from the resume (`since`),
|
|
2710
|
+
not from before the silence (RESUME-QUIET).
|
|
2711
|
+
"""
|
|
2712
|
+
now = time.time() if now is None else now
|
|
2713
|
+
ringing = []
|
|
2714
|
+
wait = S.get(A.load_config(root), "decisions.human_after_minutes") * 60
|
|
2715
|
+
for decision in A.decisions(root, "open"):
|
|
2716
|
+
asked = decision.get("asked_at") or 0
|
|
2717
|
+
if not asked or now - max(asked, since) < wait:
|
|
2718
|
+
continue
|
|
2719
|
+
ringing.append(decision.get("id"))
|
|
2720
|
+
if dry_run:
|
|
2721
|
+
print(f"DRY RUN: would ring a person about decision {decision.get('id')}")
|
|
2722
|
+
continue
|
|
2723
|
+
ring_decision(root, project, decision, now=now)
|
|
2724
|
+
return ringing
|
|
2725
|
+
|
|
2726
|
+
|
|
2727
|
+
def decision_told(decision):
|
|
2728
|
+
"""What an open decision's alarm says, for the ladder: which decision, and its question (NOISE-REPEATS)."""
|
|
2729
|
+
return f"{decision.get('id')}: {' '.join(str(decision.get('question') or '').split())[:120]}"
|
|
2730
|
+
|
|
2731
|
+
|
|
2732
|
+
def ring_decision(root, project, decision, now=None, why=None):
|
|
2733
|
+
"""Raise an open decision's own alarm, `decision-open:<id>`: its question and how to answer it (NOISE-REPEATS).
|
|
2734
|
+
|
|
2735
|
+
It rang every hour, and the anomaly's "needs you" told the same decision beside it: one
|
|
2736
|
+
decision nobody answered reached the desktop and the phone 23 times in a day and was
|
|
2737
|
+
mailed under both keys. Now it rings once for the decision it names and climbs the ladder;
|
|
2738
|
+
another decision is another alarm, and is told. `why` says why no architect acts on it,
|
|
2739
|
+
when that is why it rings before its time.
|
|
2740
|
+
"""
|
|
2741
|
+
now = time.time() if now is None else now
|
|
2742
|
+
did = decision.get("id")
|
|
2743
|
+
return notify(f"{project}: decision waiting",
|
|
2744
|
+
f"{did}: {str(decision.get('question') or '')[:120]} — open "
|
|
2745
|
+
f"{int((now - (decision.get('asked_at') or now)) / 60)}m; polyctl answer {did} <key>"
|
|
2746
|
+
+ (f" — no architect will act on it: {why}" if why else ""),
|
|
2747
|
+
root, key=f"decision-open:{did}", window=3600, audience="human", what=decision_told(decision))
|
|
2748
|
+
|
|
2749
|
+
|
|
2750
|
+
def report_ungranted_commits(root, project):
|
|
2751
|
+
"""Tell a person about commits whose tree no grant bound, every cycle while they stand (#109).
|
|
2752
|
+
|
|
2753
|
+
A harness granted every tool can run `git commit --no-verify`, and nothing in
|
|
2754
|
+
its grant keeps that out. The landed-tree check cannot stop such a commit; run
|
|
2755
|
+
every cycle, it makes one visible within a cycle.
|
|
2756
|
+
|
|
2757
|
+
It was told to the architect, as a notice row nobody sends - recorded, never rung,
|
|
2758
|
+
hidden by `polyctl notices` - about the newest 20 commits, and not in a cycle that ended
|
|
2759
|
+
for want of an implementer or a transcript (the review of ATTACK-SUITE). It is now
|
|
2760
|
+
the alarm the scheduled doctor raises for the same finding, `landed-without-grant`,
|
|
2761
|
+
to a person, saying what `A.landed_without_grant` says: it rings the desktop and
|
|
2762
|
+
the phone once for the commits it names, climbs to red and mails while they stand,
|
|
2763
|
+
and rings again when others land; a check that cannot run is that alarm too, saying
|
|
2764
|
+
why (LEDGER-HISTORY). Returns what it said, or None.
|
|
2765
|
+
"""
|
|
2766
|
+
told = A.landed_without_grant(root)
|
|
2767
|
+
if told:
|
|
2768
|
+
notify(f"{project}: landed without a grant", told, root, key="landed-without-grant", window=6 * 3600,
|
|
2769
|
+
audience="human", what=told)
|
|
2770
|
+
return told
|
|
2771
|
+
|
|
2772
|
+
|
|
2773
|
+
# ---- after a silence: one notice, and clocks that restart (RESUME-QUIET) ----------------------
|
|
2774
|
+
|
|
2775
|
+
def _span(seconds):
|
|
2776
|
+
"""How long a silence lasted, in the unit it is read in: 14d, 5h, 40m."""
|
|
2777
|
+
seconds = max(0, int(seconds))
|
|
2778
|
+
if seconds >= 2 * 86400:
|
|
2779
|
+
return f"{seconds // 86400}d"
|
|
2780
|
+
if seconds >= 2 * 3600:
|
|
2781
|
+
return f"{seconds // 3600}h"
|
|
2782
|
+
return f"{seconds // 60}m"
|
|
2783
|
+
|
|
2784
|
+
|
|
2785
|
+
def _when(at):
|
|
2786
|
+
return time.strftime("%d %b %H:%M", time.localtime(float(at or 0)))
|
|
2787
|
+
|
|
2788
|
+
|
|
2789
|
+
def _note_above_verdict(line):
|
|
2790
|
+
"""A line decided after the verdict was printed, kept above it: a trace ends on its verdict."""
|
|
2791
|
+
_TRACE.insert(max(len(_TRACE) - 1, 0), line)
|
|
2792
|
+
_builtins.print(line)
|
|
2793
|
+
|
|
2794
|
+
|
|
2795
|
+
def _markdown_plain(text):
|
|
2796
|
+
"""Text Telegram's Markdown shows as written: one stray underscore in a key loses the whole message."""
|
|
2797
|
+
return re.sub(r"([_*`\[])", r"\\\1", str(text))
|
|
2798
|
+
|
|
2799
|
+
|
|
2800
|
+
def _tell_phone(root, text):
|
|
2801
|
+
"""A courtesy line to the phone; after a silence it joins the resume notice instead (RESUME-QUIET)."""
|
|
2802
|
+
resuming = _RESUME.get()
|
|
2803
|
+
if resuming is not None:
|
|
2804
|
+
resuming["folded"].append({"key": "", "title": "", "msg": text.replace("*", ""), "audience": "architect"})
|
|
2805
|
+
return
|
|
2806
|
+
try:
|
|
2807
|
+
from . import telegram
|
|
2808
|
+
telegram.send(text, root)
|
|
2809
|
+
except Exception:
|
|
2810
|
+
pass # a phone being unreachable is not a failure
|
|
2811
|
+
try:
|
|
2812
|
+
from . import ntfy
|
|
2813
|
+
ntfy.send(text, root)
|
|
2814
|
+
except Exception:
|
|
2815
|
+
pass
|
|
2816
|
+
|
|
2817
|
+
|
|
2818
|
+
def _handoff_quiet():
|
|
2819
|
+
"""What `polyctl handoff` is also given: after a silence its note is written, not sent (RESUME-QUIET)."""
|
|
2820
|
+
resuming = _RESUME.get()
|
|
2821
|
+
if resuming is None:
|
|
2822
|
+
return []
|
|
2823
|
+
resuming["folded"].append({"key": "", "title": "", "msg": "a handoff note was written to the mailbox, not sent",
|
|
2824
|
+
"audience": "architect"})
|
|
2825
|
+
return ["--no-send"]
|
|
2826
|
+
|
|
2827
|
+
|
|
2828
|
+
def resume_after(root, cfg, silence, dry_run=False, now=None):
|
|
2829
|
+
"""The resume this cycle belongs to: {"at", "since", "quiet", "carried", "snoozes", "announced"}, or None.
|
|
2830
|
+
|
|
2831
|
+
On 2026-09-17 an owner switched one project's watchdog and doctor jobs off for two
|
|
2832
|
+
weeks, because they kept ringing about an implementer that had run out of credits.
|
|
2833
|
+
Every record stayed on disk with its date. Rebuilt in a temporary project, the first
|
|
2834
|
+
cycle back mailed an unread request red at once, rang an open decision on the desktop
|
|
2835
|
+
and the phone, announced a red alarm as no longer raised although it still stood,
|
|
2836
|
+
mailed the credits alarm whose snooze had ended while nothing ran, and told the phone
|
|
2837
|
+
it had woken the architect; an hour later the decision mailed too. Each was measured
|
|
2838
|
+
as if the watchdog had been watching all along.
|
|
2839
|
+
|
|
2840
|
+
A cycle that follows more than `watchdog.resume_gap_hours` without one, read from the
|
|
2841
|
+
heartbeat the last cycle left, is a resume. It records when it began, how long the
|
|
2842
|
+
silence was, and what the silence carried: the alarm episodes nothing raised for the
|
|
2843
|
+
gap (or for the quiet that ends an episode, when that is shorter) and the snoozes that
|
|
2844
|
+
ended in it. A heartbeat that cannot be written must not make every cycle a resume, so
|
|
2845
|
+
another needs as long again after this one. A dry run writes nothing.
|
|
2846
|
+
"""
|
|
2847
|
+
now = time.time() if now is None else now
|
|
2848
|
+
gap = S.get(cfg, "watchdog.resume_gap_hours") * 3600
|
|
2849
|
+
st = load_state(root)
|
|
2850
|
+
resume = st.get("resume") if isinstance(st.get("resume"), dict) else None
|
|
2851
|
+
if silence is None or silence <= gap or now - float((resume or {}).get("at") or 0) <= gap:
|
|
2852
|
+
return resume
|
|
2853
|
+
project, since, quiet = A.project_key(root), now - silence, min(gap, A._alarm_reset_after())
|
|
2854
|
+
carried = []
|
|
2855
|
+
for name, episode in sorted(A.load_alarms().items()):
|
|
2856
|
+
owner, _, key = name.partition(":")
|
|
2857
|
+
if owner == project and isinstance(episode, dict) and now - float(episode.get("last") or 0) > quiet:
|
|
2858
|
+
carried.append(dict({field: episode.get(field) for field in
|
|
2859
|
+
("title", "level", "ring", "first", "last", "count", "red_sent",
|
|
2860
|
+
"quiet_until")}, key=key))
|
|
2861
|
+
snoozes = []
|
|
2862
|
+
for name, snooze in sorted(A.load_alarm_snoozes().items()):
|
|
2863
|
+
owner, _, key = name.partition(":")
|
|
2864
|
+
if owner == project and isinstance(snooze, dict) and since < float(snooze.get("until") or 0) <= now:
|
|
2865
|
+
snoozes.append({"key": key, "until": snooze.get("until"), "by": snooze.get("by"),
|
|
2866
|
+
"why": snooze.get("why")})
|
|
2867
|
+
resume = {"at": now, "since": since, "quiet": quiet, "carried": carried, "snoozes": snoozes, "announced": None}
|
|
2868
|
+
if not dry_run:
|
|
2869
|
+
st["resume"] = resume
|
|
2870
|
+
save_state(root, st)
|
|
2871
|
+
return resume
|
|
2872
|
+
|
|
2873
|
+
|
|
2874
|
+
def resume_ended(root, cfg, resume, now=None):
|
|
2875
|
+
"""The keys of what the silence carried whose own known end has passed: none of them stands (CATCHUP-POLISH).
|
|
2876
|
+
|
|
2877
|
+
An episode raised with a known end - credits until the reset their reading named, the
|
|
2878
|
+
architect's quota until it comes back - says nothing of what holds after that end. An
|
|
2879
|
+
episode raised before episodes kept their end carries none, so the credits alarm's end is
|
|
2880
|
+
also read from the implementer's last reading before this cycle: a reading whose own reset
|
|
2881
|
+
has passed says nothing of the plan after it, as a spent one is no credits finding
|
|
2882
|
+
(OCT1-FIXES). Rehearsing the catch-up planned for 2026-10-01, with the jobs back six hours
|
|
2883
|
+
after the plan reset and the usage unreadable, the resume notice named the credits as
|
|
2884
|
+
standing, from the episode the silence carried and from the snooze that ended in it. What
|
|
2885
|
+
this cycle raises was measured now, and it stands whatever this says.
|
|
2886
|
+
"""
|
|
2887
|
+
now = time.time() if now is None else now
|
|
2888
|
+
ended = {episode["key"] for episode in resume.get("carried") or []
|
|
2889
|
+
if 0 < float(episode.get("quiet_until") or 0) <= now}
|
|
2890
|
+
try:
|
|
2891
|
+
readings = [row for row in A.credit_samples(root, A.implementer_adapter_id(cfg))
|
|
2892
|
+
if float(row.get("at") or 0) < int(resume.get("at") or now)]
|
|
2893
|
+
except Exception:
|
|
2894
|
+
readings = []
|
|
2895
|
+
reset = credit_reset(readings[-1].get("reset_at"), now) if readings else None
|
|
2896
|
+
if reset is not None and reset <= now:
|
|
2897
|
+
ended.add("credits-exhaust")
|
|
2898
|
+
return ended
|
|
2899
|
+
|
|
2900
|
+
|
|
2901
|
+
def resume_items(root, cfg, st, resume, folded, now=None, told=()):
|
|
2902
|
+
"""What stands at a resume, one entry per condition: {"key", "audience", "lines", "red"}.
|
|
2903
|
+
|
|
2904
|
+
From the cycle that ran - every notice it would have rung, with the audience its own
|
|
2905
|
+
check declared - and from the records the silence left. A record read here keeps the
|
|
2906
|
+
audience its check gives it elsewhere: an alarm episode, a snooze, a decision, deferred
|
|
2907
|
+
work, a hold and an implementer with nothing to do are a person's; an unseen decision
|
|
2908
|
+
request is first the architect's (#30). An entry without a key is something the cycle
|
|
2909
|
+
did; it is told, never named. A condition a person has snoozed stays off the notice as
|
|
2910
|
+
it stays off every channel (#108). What the silence carried is not named once its own known
|
|
2911
|
+
end has passed, and an unseen request another alarm to a person tells is named under that
|
|
2912
|
+
alarm alone (CATCHUP-POLISH).
|
|
2913
|
+
"""
|
|
2914
|
+
now = time.time() if now is None else now
|
|
2915
|
+
project = A.project_key(root)
|
|
2916
|
+
items = {}
|
|
2917
|
+
# What ended in the silence is not named, nor a request an alarm to a person tells (CATCHUP-POLISH).
|
|
2918
|
+
unnamed = resume_ended(root, cfg, resume, now) | {f"unseen:{name}" for name in told}
|
|
2919
|
+
|
|
2920
|
+
def add(key, audience, line, red=False, what=None):
|
|
2921
|
+
if key and A.alarm_snoozed(project, key, now):
|
|
2922
|
+
return
|
|
2923
|
+
item = items.setdefault(key, {"key": key, "audience": audience, "lines": [], "red": False})
|
|
2924
|
+
item["audience"] = "human" if "human" in (item["audience"], audience) else audience
|
|
2925
|
+
item["red"] = item["red"] or red
|
|
2926
|
+
if what is not None:
|
|
2927
|
+
item["what"] = what # what the notice tells of it, for the ladder (NOTICE-NOISE)
|
|
2928
|
+
line = " ".join(str(line).split())[:240]
|
|
2929
|
+
if line and line not in item["lines"]:
|
|
2930
|
+
item["lines"].append(line)
|
|
2931
|
+
|
|
2932
|
+
for note in folded:
|
|
2933
|
+
subject = str(note.get("title") or "").split(": ", 1)[-1]
|
|
2934
|
+
add(note.get("key") or "", note.get("audience") or "human",
|
|
2935
|
+
f"{subject}: {note.get('msg')}" if subject else note.get("msg"), red=note.get("ring") == "red",
|
|
2936
|
+
what=note.get("what"))
|
|
2937
|
+
for episode in (episode for episode in resume.get("carried") or [] if episode["key"] not in unnamed):
|
|
2938
|
+
add(episode["key"], "human", f"stood when the silence began: {episode.get('ring') or episode.get('level')} "
|
|
2939
|
+
f"since {_when(episode.get('first'))}, last raised {_when(episode.get('last'))}"
|
|
2940
|
+
+ (f", mailed {_when(episode['red_sent'])}" if episode.get("red_sent") else ""))
|
|
2941
|
+
for snooze in (snooze for snooze in resume.get("snoozes") or [] if snooze["key"] not in unnamed):
|
|
2942
|
+
add(snooze["key"], "human", f"its snooze ended {_when(snooze.get('until'))} "
|
|
2943
|
+
f"({snooze.get('by')}: {snooze.get('why')})")
|
|
2944
|
+
# Each record is read on its own: one that cannot be read must not cost the notice the rest.
|
|
2945
|
+
try:
|
|
2946
|
+
for message in A.unseen_messages(root, cfg):
|
|
2947
|
+
if message["class"] == "needs-decision" and f"unseen:{message['id']}" not in unnamed:
|
|
2948
|
+
add(f"unseen:{message['id']}", "architect",
|
|
2949
|
+
f"a decision request nobody has been shown, written {_when(message['at'])}")
|
|
2950
|
+
except Exception:
|
|
2951
|
+
pass
|
|
2952
|
+
try:
|
|
2953
|
+
for decision in A.decisions(root, "open"):
|
|
2954
|
+
add(f"decision-open:{decision.get('id')}", "human", f"open since {_when(decision.get('asked_at'))}: "
|
|
2955
|
+
f"{str(decision.get('question') or '')[:120]}")
|
|
2956
|
+
except Exception:
|
|
2957
|
+
pass
|
|
2958
|
+
try:
|
|
2959
|
+
for row in A.deferred_open(root):
|
|
2960
|
+
add(f"deferred:{row.get('kind')}", "human", f"deferred since {_when(row.get('at'))}: "
|
|
2961
|
+
f"{row.get('reason') or '?'}; polyctl catchup replays it")
|
|
2962
|
+
except Exception:
|
|
2963
|
+
pass
|
|
2964
|
+
held = A.hold_state(root)
|
|
2965
|
+
if held:
|
|
2966
|
+
add("hold-standing", "human", f"held by {held.get('by')} since {_when(held.get('at') or now)}: "
|
|
2967
|
+
f"{held.get('reason') or ''}")
|
|
2968
|
+
idle = st.get("idle_answer")
|
|
2969
|
+
if isinstance(idle, dict):
|
|
2970
|
+
add("idle-answer", "human", f"no nudges since {_when(idle.get('since') or now)}: the implementer "
|
|
2971
|
+
"answered its last one with nothing")
|
|
2972
|
+
return sorted(items.values(), key=lambda item: (not item["key"], item["audience"] != "human"))
|
|
2973
|
+
|
|
2974
|
+
|
|
2975
|
+
def announce_resume(root, owed, dry_run=False, now=None):
|
|
2976
|
+
"""Send the one notice a resume owes, naming once each condition that stands (RESUME-QUIET).
|
|
2977
|
+
|
|
2978
|
+
It goes to the widest audience among what it names: a person when anything it names is
|
|
2979
|
+
a person's, since a person can act on the architect's business while the architect
|
|
2980
|
+
cannot lift a hold, end a snooze or buy credits; a person too when all of it is the
|
|
2981
|
+
architect's but no architect is going to read it (#69); otherwise the architect,
|
|
2982
|
+
through its mailbox. It uses the orange channels and never e-mail, so it is
|
|
2983
|
+
never louder than what it names (#40). It is the ring of each condition it names for
|
|
2984
|
+
that condition's own window, and a red it names counts as told, so the next cycle does
|
|
2985
|
+
not ring or mail again what the notice has just said. The resume counts as announced
|
|
2986
|
+
before anything here can fail and whatever a channel answers: a notice left owed would
|
|
2987
|
+
keep gathering alarms, cycle after cycle, into a notice that never goes.
|
|
2988
|
+
"""
|
|
2989
|
+
now = time.time() if now is None else now
|
|
2990
|
+
resume = owed["resume"]
|
|
2991
|
+
cfg = A.load_config(root)
|
|
2992
|
+
st = load_state(root)
|
|
2993
|
+
if not dry_run:
|
|
2994
|
+
st["resume"] = dict(resume, announced=int(now))
|
|
2995
|
+
save_state(root, st)
|
|
2996
|
+
items = resume_items(root, cfg, st, resume, owed["folded"], now, told=owed.get("told") or ())
|
|
2997
|
+
named = [item["key"] for item in items if item["key"]]
|
|
2998
|
+
audience = "human" if any(item["audience"] == "human" for item in items) else "architect"
|
|
2999
|
+
if audience == "architect" and named:
|
|
3000
|
+
if not architect_hold_reason(root, cfg, A.metered_adapter(cfg), st, now=now)["holdable"]:
|
|
3001
|
+
audience = "human"
|
|
3002
|
+
project = A.project_key(root)
|
|
3003
|
+
silence = float(resume["at"]) - float(resume["since"])
|
|
3004
|
+
title = f"{project}: watchdog resumed after {_span(silence)}"
|
|
3005
|
+
lines = [f"{len(named)} stand: {', '.join(named)}" if named else "nothing stands",
|
|
3006
|
+
f"nothing ran from {_when(resume['since'])} to {_when(resume['at'])}; each clock restarts now"]
|
|
3007
|
+
lines += [f"- {item['key'] or 'this cycle'}: {'; '.join(item['lines'])}" for item in items]
|
|
3008
|
+
lines.append("polyctl alarms, polyctl notices and polyctl status show more")
|
|
3009
|
+
msg = "\n".join(lines)
|
|
3010
|
+
_FACTS["resume"] = {"silence_s": int(silence), "audience": audience, "named": named}
|
|
3011
|
+
if dry_run:
|
|
3012
|
+
_note_above_verdict(f"DRY RUN: would send one resume notice to the {audience} naming {len(named)} "
|
|
3013
|
+
f"condition(s){': ' + ', '.join(named) if named else ''}")
|
|
3014
|
+
return None
|
|
3015
|
+
st["resume"].update(audience=audience, named=named)
|
|
3016
|
+
save_state(root, st)
|
|
3017
|
+
if not named:
|
|
3018
|
+
A.record_notice(root, title, msg, sent=False, key="resume")
|
|
3019
|
+
elif audience == "human":
|
|
3020
|
+
desktop_notify(title, lines[0].replace('"', "'")[:200], cfg)
|
|
3021
|
+
phone_text = f"*{_markdown_plain(title)}*\n{_markdown_plain(msg)}"
|
|
3022
|
+
try:
|
|
3023
|
+
from . import telegram
|
|
3024
|
+
telegram.send(phone_text, root)
|
|
3025
|
+
except Exception:
|
|
3026
|
+
pass
|
|
3027
|
+
try:
|
|
3028
|
+
from . import ntfy
|
|
3029
|
+
ntfy.send(phone_text, root)
|
|
3030
|
+
except Exception:
|
|
3031
|
+
pass
|
|
3032
|
+
A.record_notice(root, title, msg, sent=True, key="resume", named=named)
|
|
3033
|
+
for item in items:
|
|
3034
|
+
if item["key"]:
|
|
3035
|
+
A.alarm_rang(project, item["key"], now, what=item.get("what")) # it rang once, for what it said
|
|
3036
|
+
if item["key"] and item["red"]:
|
|
3037
|
+
A.alarm_named(project, item["key"], now)
|
|
3038
|
+
else:
|
|
3039
|
+
A.record_notice(root, title, msg, sent=False, key="resume", named=named)
|
|
3040
|
+
A.write_report(root, cfg, "resumed", lines,
|
|
3041
|
+
key=time.strftime("%Y%m%d-%H%M", time.localtime(float(resume["at"]))))
|
|
3042
|
+
_note_above_verdict(f"resumed after {_span(silence)}: one notice to the {audience} names "
|
|
3043
|
+
f"{len(named)} condition(s)")
|
|
3044
|
+
return audience
|
|
3045
|
+
|
|
3046
|
+
|
|
3047
|
+
def ring_unseen_requests(root, cfg, st, restart, told=()):
|
|
3048
|
+
"""Ring each decision request nobody has been shown by its age, unless an alarm to a person tells it (#30).
|
|
3049
|
+
|
|
3050
|
+
The architect is told first, then a person's desktop and phone, then e-mail. One written
|
|
3051
|
+
before a resume waits from the resume: nobody could have acted in the time nothing ran
|
|
3052
|
+
(RESUME-QUIET).
|
|
3053
|
+
|
|
3054
|
+
A request an alarm to a person tells this cycle - a "needs you" no architect will act
|
|
3055
|
+
on, `reports-no-wake` or `present-pending` - is one condition, on that alarm's ladder.
|
|
3056
|
+
Rehearsing the catch-up planned for 2026-10-01, a request nobody had been shown and no
|
|
3057
|
+
architect could act on was mailed eight times in a day, four under its anomaly's key and
|
|
3058
|
+
four under its own. Once no such alarm tells it, as when an architect can act on it again,
|
|
3059
|
+
its own ladder counts from the last cycle one did, as it counts from a resume: a request a
|
|
3060
|
+
person has been told of for hours does not ring red the moment an architect may read it.
|
|
3061
|
+
A resume notice names such a request under that alarm alone (CATCHUP-POLISH).
|
|
3062
|
+
"""
|
|
3063
|
+
project = A.project_key(root)
|
|
3064
|
+
now = time.time()
|
|
3065
|
+
was = st.get("unseen_told") if isinstance(st.get("unseen_told"), dict) else {}
|
|
3066
|
+
kept = {}
|
|
3067
|
+
resuming = _RESUME.get()
|
|
3068
|
+
if resuming is not None:
|
|
3069
|
+
resuming["told"] = sorted(told)
|
|
3070
|
+
for message in A.unseen_messages(root, cfg):
|
|
3071
|
+
if message["class"] != "needs-decision":
|
|
3072
|
+
continue
|
|
3073
|
+
if message["id"] in told:
|
|
3074
|
+
kept[message["id"]] = int(now)
|
|
3075
|
+
continue
|
|
3076
|
+
if message["id"] in was:
|
|
3077
|
+
kept[message["id"]] = was[message["id"]]
|
|
3078
|
+
since = max(restart, float(kept.get(message["id"]) or 0))
|
|
3079
|
+
carried = message["at"] < since
|
|
3080
|
+
minutes = (max(0.0, now - since) if carried else message["age"]) / 60
|
|
3081
|
+
level = ("red" if minutes >= S.get(cfg, "mail.unseen_red_minutes")
|
|
3082
|
+
else "orange" if minutes >= S.get(cfg, "mail.unseen_orange_minutes")
|
|
3083
|
+
else "yellow" if minutes >= S.get(cfg, "mail.unseen_yellow_minutes") else None)
|
|
3084
|
+
if level:
|
|
3085
|
+
reason = "the watchdog resumed" if since == restart else "an alarm to a person last named it"
|
|
3086
|
+
text = f"{message['id']} has waited {int(message['age'] / 60)}m and nobody has been shown it" \
|
|
3087
|
+
+ (f" ({int(minutes)}m since {reason})" if carried else "")
|
|
3088
|
+
# It rings as it crosses a threshold, and its red is the red threshold: rung every hour between
|
|
3089
|
+
# them, one unseen request reached the desktop and the phone 15 times in a day, and an hour of
|
|
3090
|
+
# orange mailed it two hours before its red (NOISE-REPEATS).
|
|
3091
|
+
notify(f"{project}: unread decision request", text, root, key=f"unseen:{message['id']}", window=3600,
|
|
3092
|
+
audience="architect" if level == "yellow" else "human",
|
|
3093
|
+
level=None if level == "yellow" else level, what=level,
|
|
3094
|
+
red_after=S.get(cfg, "mail.unseen_red_minutes") * 60)
|
|
3095
|
+
if kept != was:
|
|
3096
|
+
st["unseen_told"] = kept
|
|
3097
|
+
save_state(root, st)
|
|
3098
|
+
|
|
3099
|
+
|
|
3100
|
+
def _cycle_impl(args, root):
|
|
3101
|
+
# The silence this cycle ends is read before its own heartbeat overwrites it.
|
|
3102
|
+
silence = A.heartbeat_age(root)
|
|
3103
|
+
# Proof the watchdog ran, before anything that can end the cycle early: a
|
|
3104
|
+
# broken session binding was reported to people as a dead watchdog (audit).
|
|
3105
|
+
if not args.dry_run:
|
|
3106
|
+
A.heartbeat(root)
|
|
3107
|
+
cfg = A.load_config(root)
|
|
3108
|
+
# After a long silence this cycle is a resume: what stands is named once, in one notice
|
|
3109
|
+
# sent as the cycle ends, and what the silence carried ages from now (RESUME-QUIET).
|
|
3110
|
+
resume = resume_after(root, cfg, silence, dry_run=args.dry_run)
|
|
3111
|
+
restart = float((resume or {}).get("at") or 0)
|
|
3112
|
+
if resume and not resume.get("announced") \
|
|
3113
|
+
and time.time() - restart <= S.get(cfg, "watchdog.resume_gap_hours") * 3600:
|
|
3114
|
+
_RESUME.set({"resume": resume, "folded": []})
|
|
3115
|
+
if not args.dry_run:
|
|
3116
|
+
# Nothing watched these episodes: they close unannounced and the notice names them.
|
|
3117
|
+
A.expire_alarms(A.project_key(root), quiet_for=resume.get("quiet") or A._alarm_reset_after())
|
|
3118
|
+
print(f"resume: no watchdog cycle for {_span(restart - float(resume['since']))} before this one; what "
|
|
3119
|
+
"stands is named in one notice when this cycle ends, and its clocks restart")
|
|
3120
|
+
# A person's blocker stands even with no implementer or transcript (TRIAGE-2-MEDIUM).
|
|
3121
|
+
project = A.project_key(root)
|
|
3122
|
+
# What the implementer declares needs a person reaches a person, never held for an agent (#92).
|
|
3123
|
+
for item in A.human_waits(root):
|
|
3124
|
+
needs = item["notes"].get("needs") or item["title"]
|
|
3125
|
+
notify(f"{project}: needs you", f"{item['id']} waits on a person: {needs}", root,
|
|
3126
|
+
key=f"waiting-human:{item['id']}", window=6 * 3600, audience="human", what=needs)
|
|
3127
|
+
# Before anything that can end the cycle: a commit made around the hook was not measured in a cycle that
|
|
3128
|
+
# ended for want of an implementer block or a transcript, and an agent can take either away (the review of
|
|
3129
|
+
# ATTACK-SUITE, LEDGER-HISTORY).
|
|
3130
|
+
if not args.dry_run:
|
|
3131
|
+
landed = report_ungranted_commits(root, A.project_key(root))
|
|
3132
|
+
_FACTS["ungranted_commits"] = landed[:120] if landed else None
|
|
3133
|
+
impl = cfg.get("implementer") or {}
|
|
3134
|
+
if not impl:
|
|
3135
|
+
if not A.hold_state(root) and slice_budget(root, cfg, dry_run=args.dry_run):
|
|
3136
|
+
return 0
|
|
3137
|
+
print("no implementer session; nothing to watch")
|
|
3138
|
+
return 0
|
|
3139
|
+
adapter = A.load_adapter(impl.get("adapter", ""), root)
|
|
3140
|
+
# The quota gates read the quota by the command the package's adapter declares: reading it runs that command,
|
|
3141
|
+
# and the project's layer is a file the implementer writes (TRIAGE-HIGHS).
|
|
3142
|
+
metered = A.metered_adapter(cfg)
|
|
3143
|
+
msgs, _ = A.session_paths(cfg)
|
|
3144
|
+
if not msgs or not os.path.exists(msgs):
|
|
3145
|
+
if not A.hold_state(root) and slice_budget(root, cfg, dry_run=args.dry_run):
|
|
3146
|
+
return 0
|
|
3147
|
+
# An `auto` polyctl could not resolve says why, where it read as a transcript gone missing (SESSION-IDENTITY).
|
|
3148
|
+
state = A.session_state(cfg, "implementer") or {}
|
|
3149
|
+
print(f"the implementer's session is {state['how']}: {state['why']}; nothing to watch"
|
|
3150
|
+
if not state.get("session") and state.get("why") else "no transcript; nothing to watch")
|
|
3151
|
+
# A slice parked on its session's limit waits for that session: with its transcript gone - a harness
|
|
3152
|
+
# deletes old ones - nothing can resume it, and the cycle ended here before the park's alarm, so it
|
|
3153
|
+
# waited on unseen (JOURNAL-6). The alarm stands and says why.
|
|
3154
|
+
st = load_state(root)
|
|
3155
|
+
park = st.get("quota_park") if isinstance(st.get("quota_park"), dict) else None
|
|
3156
|
+
if park and park.get("items") and not args.dry_run:
|
|
3157
|
+
touch_quota_park(root, st, cfg)
|
|
3158
|
+
notify(f"{A.project_key(root)}: a parked slice cannot resume",
|
|
3159
|
+
f"{', '.join(park['items'])} waits for the implementer's session, and its transcript is gone, so "
|
|
3160
|
+
"nothing resumes it when the limit resets - pin a session the store still holds, or move the "
|
|
3161
|
+
"slice on by hand", root, key="park-transcript-gone", window=6 * 3600, audience="human")
|
|
3162
|
+
return 0
|
|
3163
|
+
|
|
3164
|
+
# The implementer's silence ends at its last write, a subagent's included: a session whose transcript is
|
|
3165
|
+
# quiet while its subagent works is not idle, and its lingering runtime is not hung.
|
|
3166
|
+
age = time.time() - A.last_write(msgs, A.transcript_shape(adapter))
|
|
3167
|
+
size = os.path.getsize(msgs)
|
|
3168
|
+
st = load_state(root)
|
|
3169
|
+
if not args.dry_run:
|
|
3170
|
+
_adopt_started_steps(root) # before anything counts writers or decides a wake
|
|
3171
|
+
A.reconcile_mail_ledger(root, cfg) # deleted mail becomes a consumed row
|
|
3172
|
+
A.record_progress(root, cfg) # history of what moved, for the spin check
|
|
3173
|
+
_close_deferred_done(root, st)
|
|
3174
|
+
project = A.project_key(root)
|
|
3175
|
+
try:
|
|
3176
|
+
bd = A.board(root)
|
|
3177
|
+
_FACTS.update(idle_s=int(age), transcript_bytes=size,
|
|
3178
|
+
inbox=len(A.implementer_inbox(root, cfg)),
|
|
3179
|
+
standing_request=bool(A.waiting_on_architect(root, cfg)),
|
|
3180
|
+
queued=len(bd["queued"]), running=len(bd["running"]), blocked=len(bd["blocked"]),
|
|
3181
|
+
hold=bool(A.hold_state(root)),
|
|
3182
|
+
arch_quota_until=st.get("arch_quota_until") or None,
|
|
3183
|
+
arch_present=A.architect_present(root, cfg.get("architect") or {}),
|
|
3184
|
+
last_wake_error=(st.get("wake_error") or {}).get("kind"))
|
|
3185
|
+
except Exception as exc: # facts must never stop a cycle
|
|
3186
|
+
_FACTS["facts_error"] = str(exc)[:120]
|
|
3187
|
+
# Alarm hygiene, every cycle: keep durable conditions raised before closing
|
|
3188
|
+
# episodes that went quiet, then say so once. A quota window lives in state,
|
|
3189
|
+
# so unlike a transient notice it must keep aging toward red while it stands.
|
|
3190
|
+
# In dry-run, notify previews that same touch without persisting it.
|
|
3191
|
+
# A present, working architect is not at quota, whatever the cache says (#40).
|
|
3192
|
+
architect = cfg.get("architect") or {}
|
|
3193
|
+
try:
|
|
3194
|
+
present = st.get("arch_quota_until", 0) > time.time() and architect.get("argv") \
|
|
3195
|
+
and A.architect_present(root, architect)
|
|
3196
|
+
except TableUnreadable as exc:
|
|
3197
|
+
# Presence that was not measured clears nothing; the guards below stand down on the same table
|
|
3198
|
+
# once they ask it, after the alarms and the ping this cycle owes (WINDOWS-TABLE-CLOSED).
|
|
3199
|
+
present = False
|
|
3200
|
+
print(f"{exc}; the architect's cached quota block stands")
|
|
3201
|
+
if present:
|
|
3202
|
+
st.pop("arch_quota_until", None)
|
|
3203
|
+
save_state(root, st)
|
|
3204
|
+
print("the architect is present and working; its cached quota block is cleared")
|
|
3205
|
+
touch_architect_quota(root, st, cfg)
|
|
3206
|
+
touch_quota_park(root, st, cfg)
|
|
3207
|
+
if not args.dry_run:
|
|
3208
|
+
for e in A.expire_alarms(project):
|
|
3209
|
+
_announce_resolved(root, e)
|
|
3210
|
+
# The dead man's switch is a real external ping, not an explain probe.
|
|
3211
|
+
_FACTS["ping"] = A.ping(root)
|
|
3212
|
+
else:
|
|
3213
|
+
_FACTS["ping"] = "dry-run"
|
|
3214
|
+
if not args.dry_run:
|
|
3215
|
+
_sample_credits(root, st, A.implementer_adapter_id(cfg), project)
|
|
3216
|
+
_schedule_hunt(root, cfg, st)
|
|
3217
|
+
# Only meaningful when no implementer turn is running: a sub-agent's writes
|
|
3218
|
+
# do not appear as the parent's tool calls and would read as a stranger's.
|
|
3219
|
+
# Whose they are cannot be told where the process table cannot be read, and
|
|
3220
|
+
# the writer guard below stands down on it (WINDOWS-TABLE-CLOSED).
|
|
3221
|
+
try:
|
|
3222
|
+
fe = [] if A.agent_pids(root, adapter) else A.foreign_edits(root, cfg)
|
|
3223
|
+
except TableUnreadable as exc:
|
|
3224
|
+
fe = []
|
|
3225
|
+
_FACTS["process_table"] = str(exc)
|
|
3226
|
+
_FACTS["foreign_edits"] = fe
|
|
3227
|
+
_FACTS["decisions_ringing"] = escalate_open_decisions(root, project, dry_run=args.dry_run, since=restart)
|
|
3228
|
+
parked = A.reviewer_state(root)
|
|
3229
|
+
if parked.get("pending_review") and not parked.get("until") and not args.dry_run:
|
|
3230
|
+
notify(f"{project}: a review is parked",
|
|
3231
|
+
f"the reviewer was unavailable: {A.redact(parked.get('reason') or '?')[:120]} — a person "
|
|
3232
|
+
"restores it, carries the request with polyctl collect-review, or waives", root,
|
|
3233
|
+
key="review-parked", window=24 * 3600, audience="human")
|
|
3234
|
+
for sib, age_s in A.stale_siblings(root).items():
|
|
3235
|
+
# Right after a resume every sibling looks as silent as this watchdog was, until its
|
|
3236
|
+
# own next cycle: a sibling's silence counts from the resume too (RESUME-QUIET).
|
|
3237
|
+
if time.time() - restart <= 15 * 60:
|
|
3238
|
+
continue
|
|
3239
|
+
notify(f"{sib}: watchdog silent", f"no heartbeat for {age_s // 60}m — its watchdog is not "
|
|
3240
|
+
f"running; {A.scheduler_hint()} / polyctl watchdog status", root, key=f"watchdog-dead:{sib}",
|
|
3241
|
+
window=3600, audience="human")
|
|
3242
|
+
hs = A.hold_state(root)
|
|
3243
|
+
# A hold carried over a silence stands from the resume (RESUME-QUIET).
|
|
3244
|
+
if hs and time.time() - max(int(hs.get("at") or time.time()), restart) > 4 * 3600:
|
|
3245
|
+
notify(f"{project}: hold standing {int((time.time() - hs['at']) / 3600)}h",
|
|
3246
|
+
f"set by {hs.get('by', '?')}: {hs.get('reason', '')} — polyctl hold release when done",
|
|
3247
|
+
root, key="hold-standing", window=6 * 3600, audience="human", level="red")
|
|
3248
|
+
|
|
3249
|
+
# Harness configuration an agent changed here reaches a person, held or not: a person who starts a session in
|
|
3250
|
+
# this tree runs it too (HARNESS-CONFIG).
|
|
3251
|
+
tell_harness_config(root, project)
|
|
3252
|
+
# An agent that is busy and producing nothing never trips the idle guard, so
|
|
3253
|
+
# check it before the guard chain rather than inside it. Notify only; a nudge
|
|
3254
|
+
# would add a turn to a loop that is already spending them.
|
|
3255
|
+
spin = A.spinning(root)
|
|
3256
|
+
if spin and time.time() - st.get("last_spin_notice", 0) > 1800:
|
|
3257
|
+
notify(f"{project}: agent spinning", f"{spin}m busy, nothing committed or changed — needs re-specifying", root, audience="architect")
|
|
3258
|
+
st["last_spin_notice"] = time.time()
|
|
3259
|
+
save_state(root, st)
|
|
3260
|
+
print(f"spinning: {spin}m active with no artifact change")
|
|
3261
|
+
|
|
3262
|
+
# -1 — a human has taken the tree. Nothing else in this chain may override it.
|
|
3263
|
+
held = A.hold_state(root)
|
|
3264
|
+
if held:
|
|
3265
|
+
# Nothing below runs, so no other alarm tells a request nobody has been shown (#30).
|
|
3266
|
+
ring_unseen_requests(root, cfg, st, restart)
|
|
3267
|
+
print(f"held by {held.get('by')} for {held['minutes']}m: {held.get('reason','')}")
|
|
3268
|
+
return 0
|
|
3269
|
+
|
|
3270
|
+
# -1b — a running slice past its budget, or whose budget reading is missing, parks here and
|
|
3271
|
+
# nothing after this runs for it (SLICE-BUDGET).
|
|
3272
|
+
if slice_budget(root, cfg, dry_run=args.dry_run):
|
|
3273
|
+
return 0
|
|
3274
|
+
|
|
3275
|
+
# Report anything needing judgement before the guard chain stands down on it.
|
|
3276
|
+
# Standing down silently is how a condition persists for hours: the watchdog
|
|
3277
|
+
# was right to not act and wrong to be the only one who knew.
|
|
3278
|
+
told = set()
|
|
3279
|
+
escalate(root, cfg, adapter, age, args, st, told=told)
|
|
3280
|
+
# A request nobody has been shown climbs its own ladder only where no alarm to a person
|
|
3281
|
+
# already tells it (#30, CATCHUP-POLISH).
|
|
3282
|
+
ring_unseen_requests(root, cfg, st, restart, told)
|
|
3283
|
+
|
|
3284
|
+
# 0 — is ANY agent already working this tree? Not just the child we started.
|
|
3285
|
+
#
|
|
3286
|
+
# Tracking only our own last child was wrong, and expensively so: every nudge
|
|
3287
|
+
# spawns a detached process, nothing reaps them, and this project accumulated
|
|
3288
|
+
# fifteen live agent processes in one repository with four still burning CPU.
|
|
3289
|
+
# Each was invisible to a guard that remembered a single pid. Ask the OS which
|
|
3290
|
+
# processes have this repo as their cwd, and treat any of them as a writer.
|
|
3291
|
+
# What an ended turn left behind is cleared before anything is counted. A
|
|
3292
|
+
# wrapper that was reaped by pid — or exited on its own — leaves its runtime
|
|
3293
|
+
# and engine children alive with this repo as their cwd; the implementer
|
|
3294
|
+
# counts them as writers and refuses to write, its empty turns trip the
|
|
3295
|
+
# reaper again, and the reaper makes one more. They are identified by shape
|
|
3296
|
+
# (no terminal, dead group leader), never by age, so nothing a person is in
|
|
3297
|
+
# can match.
|
|
3298
|
+
dead = A.orphans(root, adapter)
|
|
3299
|
+
if dead:
|
|
3300
|
+
print(f"{len(dead)} orphaned agent process(es) left by an ended turn; clearing {dead}")
|
|
3301
|
+
if not args.dry_run:
|
|
3302
|
+
A.record_notice(root, "orphans cleared", f"{len(dead)} leftover process(es): {dead}", False, key="orphans")
|
|
3303
|
+
left = A.sweep_orphans(dead)
|
|
3304
|
+
if left:
|
|
3305
|
+
# Said, not assumed cleared: a remnant the sweep could not stop is swept again next cycle
|
|
3306
|
+
# (WINDOWS-STOP-PROVEN).
|
|
3307
|
+
print(f"{len(left)} leftover process(es) still run after the sweep: {left}")
|
|
3308
|
+
running = [p for p in A.agent_pids(root, adapter) if p not in set(dead)]
|
|
3309
|
+
# Windows has no process groups to find an ended turn's leftovers by: none looked for is not none, and what
|
|
3310
|
+
# `running` holds may be such leftovers (WINDOWS-SAY-WHY).
|
|
3311
|
+
_FACTS.update(writers=len(A.process_trees(running)) if running else 0,
|
|
3312
|
+
orphans=None if os.name == "nt" else len(dead))
|
|
3313
|
+
# An agent whose working directory Windows would not let polyctl read cannot be placed (#71,
|
|
3314
|
+
# WINDOWS-CWD-2). One that cannot be placed may be this tree's writer; starting another
|
|
3315
|
+
# beside it is the two-writer incident, so stand down and say why.
|
|
3316
|
+
unplaced = A.unplaced_agent_pids(root, adapter)
|
|
3317
|
+
if unplaced:
|
|
3318
|
+
print(f"{len(unplaced)} agent process(es) cannot be placed in a tree - their working directory "
|
|
3319
|
+
f"could not be read ({unplaced}); not starting another turn")
|
|
3320
|
+
return 0
|
|
3321
|
+
if running:
|
|
3322
|
+
# A process being alive is not a turn being in flight. An agent can finish
|
|
3323
|
+
# its turn and never exit, and the first version of this guard treated that
|
|
3324
|
+
# hung process as a writer — so one zombie blocked every future nudge for
|
|
3325
|
+
# seven hours while the slice it had already finished sat uncommitted.
|
|
3326
|
+
# Fixing an accumulation bug had quietly introduced a deadlock.
|
|
3327
|
+
#
|
|
3328
|
+
# Tell them apart by the transcript: a live turn writes to it. Silence far
|
|
3329
|
+
# past the idle threshold, with a process still up, is hung, not busy.
|
|
3330
|
+
turns = A.process_trees(running)
|
|
3331
|
+
# The transcript closed the turn and the process is still here past the
|
|
3332
|
+
# idle window: a runtime that forgot to exit, not a turn. Reap now.
|
|
3333
|
+
ended = A.turn_ended(cfg) and age >= args.idle_minutes * 60
|
|
3334
|
+
if ended:
|
|
3335
|
+
print(f"turn ended in the transcript {int(age / 60)}m ago but {len(running)} process(es) linger; reaping")
|
|
3336
|
+
if age < args.idle_minutes * 60 * 3 and not ended:
|
|
3337
|
+
print(f"{len(turns)} turn(s) already in this tree "
|
|
3338
|
+
f"(roots {turns}, {len(running)} processes); not starting another")
|
|
3339
|
+
if len(turns) > 2:
|
|
3340
|
+
notify(f"{project}: turns piling up",
|
|
3341
|
+
f"{len(turns)} concurrent turns — `polyctl hold` to clear", root)
|
|
3342
|
+
return 0
|
|
3343
|
+
# Reaping is an action, so it keeps the conservative bound even though
|
|
3344
|
+
# reporting no longer does.
|
|
3345
|
+
print(f"{len(running)} process(es) alive but silent {int(age / 60)}m; reaping")
|
|
3346
|
+
notify(f"{project}: reaping hung turn",
|
|
3347
|
+
f"{len(running)} process(es) silent {int(age / 60)}m — cleaning up", root, audience="architect")
|
|
3348
|
+
if args.dry_run:
|
|
3349
|
+
print("DRY RUN: would reap", running)
|
|
3350
|
+
return 0
|
|
3351
|
+
# Reap only what we could have started. A person's interactive session
|
|
3352
|
+
# in this tree is silent between their keystrokes, not hung.
|
|
3353
|
+
# By process group: the flag is on the wrapper, the children carry none.
|
|
3354
|
+
for pid in A.agent_pids(root, adapter, headless_only=True):
|
|
3355
|
+
A.kill_turn(pid, signal.SIGTERM)
|
|
3356
|
+
for _ in range(20):
|
|
3357
|
+
if not A.agent_pids(root, adapter, headless_only=True):
|
|
3358
|
+
break
|
|
3359
|
+
time.sleep(0.5)
|
|
3360
|
+
for pid in A.agent_pids(root, adapter, headless_only=True):
|
|
3361
|
+
A.kill_turn(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
|
|
3362
|
+
# Only headless turns were reaped. A person's interactive session in this
|
|
3363
|
+
# tree is still here and still a writer; nudging next to it starts a second.
|
|
3364
|
+
# Counted as what the reaper could not have started, from the same scan.
|
|
3365
|
+
headless = set(A.agent_pids(root, adapter, headless_only=True))
|
|
3366
|
+
if headless:
|
|
3367
|
+
# A turn the reaper could not stop - taskkill refused, a child its parent no longer names - is
|
|
3368
|
+
# still this tree's writer: the reaper was taken at its word and a nudge started beside it
|
|
3369
|
+
# (WINDOWS-STOP-PROVEN).
|
|
3370
|
+
print(f"{len(headless)} unattended turn process(es) still run after reaping ({sorted(headless)}); "
|
|
3371
|
+
"not starting another turn")
|
|
3372
|
+
notify(f"{project}: a hung turn could not be stopped",
|
|
3373
|
+
f"{len(headless)} process(es) of an unattended turn still run after the reaper stopped them: "
|
|
3374
|
+
f"{sorted(headless)} - stop them by hand, or `polyctl hold`; no turn starts beside them", root,
|
|
3375
|
+
key="reap-survived", window=6 * 3600, audience="human", what=f"still running: {sorted(headless)}")
|
|
3376
|
+
return 0
|
|
3377
|
+
people = [pid for pid in A.agent_pids(root, adapter) if pid not in headless]
|
|
3378
|
+
if people:
|
|
3379
|
+
print(f"{len(people)} interactive agent process(es) remain in this tree after reaping; "
|
|
3380
|
+
"not starting another turn")
|
|
3381
|
+
return 0
|
|
3382
|
+
|
|
3383
|
+
# 1 — still working
|
|
3384
|
+
#
|
|
3385
|
+
# Progress is not the same as a commit. A slice whose independent review found
|
|
3386
|
+
# real defects is *correct* to withhold the commit while it fixes them, and to
|
|
3387
|
+
# a HEAD-only check that looks exactly like an agent that has stopped: three
|
|
3388
|
+
# nudges, no movement, stand down. It happened — two hours of a live slice sat
|
|
3389
|
+
# idle because the counter could not tell a careful implementer from a dead
|
|
3390
|
+
# one. Fingerprint everything that moves when work is happening.
|
|
3391
|
+
fp = A.work_fingerprint(root)
|
|
3392
|
+
if st.get("attempts") and fp != st.get("last_fingerprint"):
|
|
3393
|
+
st.update(attempts=0, last_fingerprint=fp)
|
|
3394
|
+
save_state(root, st)
|
|
3395
|
+
# An implementer that answered a nudge with nothing is not asked again until
|
|
3396
|
+
# something it is given, or something it produces, moves (#96).
|
|
3397
|
+
idle = st.get("idle_answer")
|
|
3398
|
+
if idle and (fp != idle.get("work") or A.nudge_inputs(root, cfg) != idle.get("inputs")):
|
|
3399
|
+
for key in ("idle_answer", "nudge_size", "nudge_fingerprint", "nudge_inputs"):
|
|
3400
|
+
st.pop(key, None)
|
|
3401
|
+
st["attempts"] = 0
|
|
3402
|
+
save_state(root, st)
|
|
3403
|
+
print("the board, the backlog, a decision, mail or the work moved since the "
|
|
3404
|
+
"implementer had nothing to do; nudges resume")
|
|
3405
|
+
if age < args.idle_minutes * 60:
|
|
3406
|
+
if st.get("attempts") and size != st.get("last_size"):
|
|
3407
|
+
# The transcript moving is not the work moving: a nudged turn that finds
|
|
3408
|
+
# nothing to do writes its answer too. Only the fingerprint above resets
|
|
3409
|
+
# the backoff.
|
|
3410
|
+
st.update(last_size=size)
|
|
3411
|
+
save_state(root, st)
|
|
3412
|
+
print(f"working ({int(age)}s since last write)")
|
|
3413
|
+
return 0
|
|
3414
|
+
# Presence is the agent's, not this tree's: writing in a secondary project is working (#22).
|
|
3415
|
+
elsewhere = A.working_elsewhere(cfg, args.idle_minutes * 60)
|
|
3416
|
+
if elsewhere:
|
|
3417
|
+
print(f"working in {elsewhere['name']} ({int(elsewhere['age'])}s since its last write there); "
|
|
3418
|
+
"not nudging here")
|
|
3419
|
+
return 0
|
|
3420
|
+
# 1b — stopped on its usage limit: the running slice is parked until the limit resets, or one retry
|
|
3421
|
+
# where no reset was named, and no nudge is spent on a window that is spent. When the park ends the
|
|
3422
|
+
# cycle goes on and resumes it (QUOTA-PARK).
|
|
3423
|
+
if quota_park(root, cfg, st, dry_run=args.dry_run, adapter=adapter):
|
|
3424
|
+
return 0
|
|
3425
|
+
|
|
3426
|
+
# 2 — nothing to continue. Before standing down, ask why: an empty board can
|
|
3427
|
+
# mean "all done" or "the queue ran dry". Only the second one is actionable,
|
|
3428
|
+
# and it needs the *architect*, not the implementer — refilling means pulling
|
|
3429
|
+
# from a tracker and deciding what may be worked, and an implementer that
|
|
3430
|
+
# chooses its own scope is the one thing this tool exists to prevent.
|
|
3431
|
+
# An implementer that asked for a decision and got no answer is not idle; it
|
|
3432
|
+
# is waiting, and a nudge cannot answer it. Eighty nudged turns once produced
|
|
3433
|
+
# eighty copies of the same request. The wake path above has already told
|
|
3434
|
+
# the architect; the only useful thing here is to say so and stand down.
|
|
3435
|
+
waiting = A.waiting_on_architect(root, cfg)
|
|
3436
|
+
if waiting:
|
|
3437
|
+
name, at = waiting
|
|
3438
|
+
print(f"implementer is waiting on the architect since "
|
|
3439
|
+
f"{time.strftime('%H:%M', time.localtime(at))} ({name}); not nudging")
|
|
3440
|
+
return 0
|
|
3441
|
+
# An open decision cannot be answered by a nudged turn (#20), and it does not
|
|
3442
|
+
# stop the queue while READY work stands: its slice parks and the next starts (#84).
|
|
3443
|
+
asked = [d for d in A.decisions(root, "open") if d.get("asked_at")]
|
|
3444
|
+
passing = queue_past_a_question(root)
|
|
3445
|
+
if asked and not passing:
|
|
3446
|
+
oldest = min(asked, key=lambda d: d["asked_at"])
|
|
3447
|
+
print(f"decision {oldest['id']} is open since "
|
|
3448
|
+
f"{time.strftime('%H:%M', time.localtime(oldest['asked_at']))}; not nudging")
|
|
3449
|
+
return 0
|
|
3450
|
+
if passing:
|
|
3451
|
+
print(f"{passing[0]} waits, and {passing[1]} is READY: an open question does not stop the queue")
|
|
3452
|
+
reasons = open_work(cfg, root)
|
|
3453
|
+
if not reasons and passing:
|
|
3454
|
+
reasons = [f"READY {passing[1]} while {passing[0]} waits"]
|
|
3455
|
+
# Nothing READY here and a secondary project has work: the nudge names it (#8).
|
|
3456
|
+
elsewhere_ready = None
|
|
3457
|
+
if not reasons and not A.ready(root):
|
|
3458
|
+
elsewhere_ready = A.secondary_ready(cfg)
|
|
3459
|
+
if elsewhere_ready:
|
|
3460
|
+
reasons = [f"READY {elsewhere_ready['item']} in {elsewhere_ready['name']}"]
|
|
3461
|
+
if not reasons:
|
|
3462
|
+
sc = A.sources(root)
|
|
3463
|
+
arch = cfg.get("architect") or {}
|
|
3464
|
+
depth = len(A.board(root)["queued"])
|
|
3465
|
+
# With a source bound, refill below its threshold; without one, an empty
|
|
3466
|
+
# queue is still a refill condition — the architect fills it from the
|
|
3467
|
+
# spec. This project had no source, so the empty queue was never a
|
|
3468
|
+
# refill and the only signal was the implementer's own blocked report.
|
|
3469
|
+
threshold = sc.get("refill_below", 3) if sc else 1
|
|
3470
|
+
from . import features as F
|
|
3471
|
+
if arch.get("argv") and depth < threshold and not F.enabled(cfg, "refill"):
|
|
3472
|
+
print(f"queue low ({depth}); refill feature is off — alarming instead")
|
|
3473
|
+
notify(f"{A.project_key(root)}: needs you", f"queue has {depth} item(s) and refill wakes are off — add slices to .polyctl/backlog.md",
|
|
3474
|
+
root, key="queue-empty-no-refill", window=3600, audience="human", what="refill-off")
|
|
3475
|
+
return 0
|
|
3476
|
+
if arch.get("argv") and depth < threshold and A.architect_present(root, arch):
|
|
3477
|
+
print("queue low, but the architect is already at the keyboard")
|
|
3478
|
+
return 0
|
|
3479
|
+
if arch.get("argv") and depth < threshold:
|
|
3480
|
+
if arch_alive(root, arch):
|
|
3481
|
+
print("queue low, but an architect wake is already running")
|
|
3482
|
+
return 0
|
|
3483
|
+
if args.dry_run:
|
|
3484
|
+
# A dry refill prepares the live command too, without a version probe (ARCH-PROBE-ORDER).
|
|
3485
|
+
_, _, unseen = architect_command(root, cfg, arch.get("prompt", language.text(cfg, "prompt.refill")))
|
|
3486
|
+
unseen = unseen or harness_config_hold(root)
|
|
3487
|
+
print(f"queue low ({depth}); {unseen}; not waking the architect to refill" if unseen
|
|
3488
|
+
else f"queue low ({depth}); would wake the architect to refill")
|
|
3489
|
+
return 0
|
|
3490
|
+
if child_alive(st):
|
|
3491
|
+
print("queue low, but a turn is still running")
|
|
3492
|
+
return 0
|
|
3493
|
+
if time.time() - st.get("last_refill", 0) < 1800:
|
|
3494
|
+
print(f"queue low ({depth}); refill wake sent "
|
|
3495
|
+
f"{int((time.time() - st.get('last_refill', 0)) / 60)}m ago; waiting")
|
|
3496
|
+
return 0
|
|
3497
|
+
# A report wake may have started earlier in this same cycle, now that the
|
|
3498
|
+
# watchdog's own reports wake the architect (#69). The helper scan is the
|
|
3499
|
+
# guard; the wake's own time is the one that cannot miss a young process.
|
|
3500
|
+
if time.time() - st.get("last_arch_wake", 0) < 900:
|
|
3501
|
+
print(f"queue low ({depth}); the architect was woken "
|
|
3502
|
+
f"{int((time.time() - st.get('last_arch_wake', 0)) / 60)}m ago; waiting")
|
|
3503
|
+
return 0
|
|
3504
|
+
if st.get("arch_quota_until", 0) > time.time():
|
|
3505
|
+
print("queue low, but the architect is at quota; waiting")
|
|
3506
|
+
return 0
|
|
3507
|
+
prompt = arch.get("prompt", language.text(cfg, "prompt.refill"))
|
|
3508
|
+
command, code, stopped = architect_command(root, cfg, prompt)
|
|
3509
|
+
if stopped:
|
|
3510
|
+
print(f"queue low ({depth}); {stopped}; not waking the architect to refill")
|
|
3511
|
+
if code == "refused":
|
|
3512
|
+
tell_turn_refused(root, "architect", stopped)
|
|
3513
|
+
return 0
|
|
3514
|
+
sess, plan, pinned = command["session"], command["plan"], command["pinned"]
|
|
3515
|
+
argv, search, values = command["argv"], command["search"], command["values"]
|
|
3516
|
+
resolved, ver = A.resolve_binary(argv[0], path=command["path"])
|
|
3517
|
+
if not resolved:
|
|
3518
|
+
print(f"architect command {argv[0]} not on PATH")
|
|
3519
|
+
return 0
|
|
3520
|
+
# As in escalate(): the shipped adapter's harness, and no grant beyond its own (TURN-ARGV).
|
|
3521
|
+
refused = A.turn_refusal("architect", A.block_adapter(arch), argv, root=root, resolved=resolved,
|
|
3522
|
+
search=search, values=values, prompt=prompt)
|
|
3523
|
+
if refused:
|
|
3524
|
+
print(f"queue low ({depth}); {refused}; not waking the architect to refill")
|
|
3525
|
+
tell_turn_refused(root, "architect", refused)
|
|
3526
|
+
return 0
|
|
3527
|
+
argv[0] = resolved
|
|
3528
|
+
headroom = A.rotate_if_exhausted(cfg, argv, "architect")
|
|
3529
|
+
if not headroom["ok"]:
|
|
3530
|
+
print(f"queue low, but {headroom['text']}")
|
|
3531
|
+
return 0
|
|
3532
|
+
key = A.project_key(root)
|
|
3533
|
+
log_path = os.path.join(STATE_DIR, A.project_file_name("refill-log", key))
|
|
3534
|
+
if A.hold_state(root):
|
|
3535
|
+
print("held since this cycle began; not waking the architect to refill")
|
|
3536
|
+
return 0
|
|
3537
|
+
unseen = harness_config_hold(root) or turn_hold(root, cfg, "architect")
|
|
3538
|
+
if unseen:
|
|
3539
|
+
print(f"queue low ({depth}); {unseen}; not waking the architect to refill")
|
|
3540
|
+
return 0
|
|
3541
|
+
# A refill spends the same window a report wake does, and its failures
|
|
3542
|
+
# were never read (audit): check the quota and the last refill's log.
|
|
3543
|
+
if not quota_ok(metered):
|
|
3544
|
+
print("queue low, but there is no quota headroom to wake the architect")
|
|
3545
|
+
return 0
|
|
3546
|
+
failed = wake_error(log_path)
|
|
3547
|
+
blocked = architect_quota_until(failed, argv)
|
|
3548
|
+
if blocked:
|
|
3549
|
+
print(f"the last refill hit the architect's limit; waiting until {reset_when(blocked)}")
|
|
3550
|
+
return 0
|
|
3551
|
+
if failed and failed.get("kind") != "quota" and time.time() - (failed.get("at") or 0) < 6 * 3600:
|
|
3552
|
+
notify(f"{project}: refill failed", f"{failed.get('kind')}: "
|
|
3553
|
+
f"{A.redact(failed.get('text', ''))[:110]}", root,
|
|
3554
|
+
key="refill-failed", window=6 * 3600, audience="human")
|
|
3555
|
+
# A transport error is retried on the refill spacing; a binary or a
|
|
3556
|
+
# session that failed fails the same way until someone changes it.
|
|
3557
|
+
if failed.get("kind") in ("binary", "session") and failed.get("binary") == f"{resolved} {ver}":
|
|
3558
|
+
print(f"the last refill with this binary failed ({failed.get('kind')}); not retrying")
|
|
3559
|
+
return 0
|
|
3560
|
+
if not record_pinned(root, pinned):
|
|
3561
|
+
return 0
|
|
3562
|
+
given, refused = A.prompt_input(plan, root, argv)
|
|
3563
|
+
if refused:
|
|
3564
|
+
print(f"queue low, but the architect's prompt cannot be handed over: {refused}")
|
|
3565
|
+
return 0
|
|
3566
|
+
# As a wake is: claimed before its process starts (JOURNAL-2).
|
|
3567
|
+
step = "refill:{}:{}".format(sess or "new", int(time.time() // 3600))
|
|
3568
|
+
if (sess and A.session_in_use(sess, arch)) or not _claim_step(root, step):
|
|
3569
|
+
print("queue low, but a refill was started and is not known to have ended; waiting")
|
|
3570
|
+
return 0
|
|
3571
|
+
os.makedirs(STATE_DIR, exist_ok=True)
|
|
3572
|
+
with open(log_path, "a", encoding=UTF8) as log:
|
|
3573
|
+
log.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} refill {resolved} {ver} ===\n")
|
|
3574
|
+
log.flush()
|
|
3575
|
+
try:
|
|
3576
|
+
proc = subprocess.Popen(given["argv"], cwd=root,
|
|
3577
|
+
env=dict(os.environ, PATH=search, POLYCTL_ROLE="architect"),
|
|
3578
|
+
stdin=subprocess.DEVNULL if given["stdin"] is None else given["stdin"],
|
|
3579
|
+
stdout=log, stderr=subprocess.STDOUT, # stdin: see escalate()
|
|
3580
|
+
start_new_session=True)
|
|
3581
|
+
finally:
|
|
3582
|
+
A.release_prompt(given)
|
|
3583
|
+
_record_start(root, step, proc.pid)
|
|
3584
|
+
st.update(arch_pid=proc.pid, last_refill=time.time())
|
|
3585
|
+
A.helper_register(root, proc.pid, "architect") # a judge, not a writer
|
|
3586
|
+
A.acquire_architect(root, proc.pid, "watchdog refill")
|
|
3587
|
+
save_state(root, st)
|
|
3588
|
+
A.emit_event(root, "wake", {"why": "refill", "queue": depth, "pid": proc.pid})
|
|
3589
|
+
print(f"queue low ({depth} < {threshold}); woke the architect")
|
|
3590
|
+
return 0
|
|
3591
|
+
print("idle, but no open work — leaving it alone")
|
|
3592
|
+
return 0
|
|
3593
|
+
|
|
3594
|
+
# 3 — over the round budget: a nudge would only buy another identical round.
|
|
3595
|
+
# Unless the architect has already intervened: a coordination message newer
|
|
3596
|
+
# than the newest review means the slice was re-specified, and the budget
|
|
3597
|
+
# applies to the old specification, not the new one.
|
|
3598
|
+
rn = A.rounds(root, cfg["reviews"])
|
|
3599
|
+
budget = S.get(cfg, "round_budget")
|
|
3600
|
+
intervened = False
|
|
3601
|
+
revs_all = A.reviews(root, cfg["reviews"], limit=1)
|
|
3602
|
+
if revs_all:
|
|
3603
|
+
# Only the architect re-specifies a slice; any other mail, the watchdog's own
|
|
3604
|
+
# included, is not an intervention (audit). Files can be acknowledged while
|
|
3605
|
+
# this reads them.
|
|
3606
|
+
try:
|
|
3607
|
+
rev_mt = os.path.getmtime(os.path.join(root, cfg["reviews"], revs_all[0][0]))
|
|
3608
|
+
except OSError:
|
|
3609
|
+
rev_mt = time.time()
|
|
3610
|
+
for m in A.mailbox(root, cfg["mailbox"]):
|
|
3611
|
+
if not A.from_architect(m, cfg):
|
|
3612
|
+
continue
|
|
3613
|
+
try:
|
|
3614
|
+
newer = os.path.getmtime(os.path.join(root, cfg["mailbox"], m)) > rev_mt
|
|
3615
|
+
except OSError:
|
|
3616
|
+
continue
|
|
3617
|
+
if newer:
|
|
3618
|
+
intervened = True
|
|
3619
|
+
break
|
|
3620
|
+
if rn > budget and not intervened:
|
|
3621
|
+
notify(f"{project}: over budget", f"round {rn}/{budget} — re-specify, split or change actor", root, audience="architect")
|
|
3622
|
+
print(f"over budget ({rn}/{budget}); notified instead of nudging")
|
|
3623
|
+
return 0
|
|
3624
|
+
if rn > budget and intervened:
|
|
3625
|
+
print(f"over budget ({rn}/{budget}) but re-specified since the last review; proceeding")
|
|
3626
|
+
|
|
3627
|
+
# 4 — no quota. Delivery still works: reports are written above this gate and
|
|
3628
|
+
# the transport is HTTP, so a pending question reaches a phone even now. What
|
|
3629
|
+
# stops is deciding — so hand the state to whoever can.
|
|
3630
|
+
if not quota_ok(metered):
|
|
3631
|
+
if not args.dry_run:
|
|
3632
|
+
if not A.recently_deferred(root, "nudge"):
|
|
3633
|
+
A.deferred_append(root, "nudge", reason="implementer quota")
|
|
3634
|
+
if time.time() - st.get("last_handoff", 0) > 3600:
|
|
3635
|
+
try:
|
|
3636
|
+
import subprocess as _sp
|
|
3637
|
+
exe = A.program_path("polyctl", path=child_path())
|
|
3638
|
+
if exe:
|
|
3639
|
+
_sp.run([exe, "-C", root, "handoff", "--reason",
|
|
3640
|
+
language.text(cfg, "watchdog.handoff-provider")] + _handoff_quiet(),
|
|
3641
|
+
capture_output=True, timeout=120)
|
|
3642
|
+
st["last_handoff"] = time.time()
|
|
3643
|
+
save_state(root, st)
|
|
3644
|
+
except Exception:
|
|
3645
|
+
pass
|
|
3646
|
+
notify(f"{project}: out of quota",
|
|
3647
|
+
"provider window exhausted; handoff note sent", root)
|
|
3648
|
+
print("provider out of headroom; not nudging")
|
|
3649
|
+
return 0
|
|
3650
|
+
|
|
3651
|
+
# 4b — the provider stopped, not the agent. Nudging an outage buys nothing
|
|
3652
|
+
# and, while the stalled turn is still retrying, costs a second writer.
|
|
3653
|
+
degraded = provider_degraded(root)
|
|
3654
|
+
if degraded:
|
|
3655
|
+
notify(f"{project}: provider degraded", f"{degraded} — waiting, not nudging", root)
|
|
3656
|
+
print(f"provider degraded ({degraded}); waiting rather than nudging")
|
|
3657
|
+
return 0
|
|
3658
|
+
|
|
3659
|
+
# 4c — the implementer answered the last nudge and changed nothing while nothing
|
|
3660
|
+
# it is given moved: it has no work it can see, whatever the board says. Asking
|
|
3661
|
+
# again every idle window cost 29 turns in one morning (#96). This reads what
|
|
3662
|
+
# the turn did, not the wording of its report.
|
|
3663
|
+
if (st.get("nudge_size") is not None and size > st["nudge_size"] and A.turn_ended(cfg)
|
|
3664
|
+
and fp == st.get("nudge_fingerprint")
|
|
3665
|
+
and A.nudge_inputs(root, cfg) == st.get("nudge_inputs")
|
|
3666
|
+
and (st.get("last_error") or {}).get("at", 0) < st.get("last_nudge", 0)):
|
|
3667
|
+
idle = st.get("idle_answer")
|
|
3668
|
+
if not idle:
|
|
3669
|
+
idle = {"since": st.get("last_nudge") or time.time(), "work": fp,
|
|
3670
|
+
"inputs": st.get("nudge_inputs")}
|
|
3671
|
+
st["idle_answer"] = idle
|
|
3672
|
+
save_state(root, st)
|
|
3673
|
+
notify(f"{project}: implementer has nothing to do",
|
|
3674
|
+
"it answered the last nudge without changing anything; no more nudges until "
|
|
3675
|
+
"the board, the backlog, a decision or its mail changes", root,
|
|
3676
|
+
key="idle-answer", window=12 * 3600, audience="human")
|
|
3677
|
+
since = time.strftime("%H:%M", time.localtime(idle["since"]))
|
|
3678
|
+
print(f"idle since {since}: the implementer answered the last nudge without changing "
|
|
3679
|
+
f"anything; waiting for the board, the backlog, a decision or mail; not nudging")
|
|
3680
|
+
return 0
|
|
3681
|
+
|
|
3682
|
+
# 5 — the previous nudge changed nothing
|
|
3683
|
+
max_attempts = S.get(cfg, "watchdog.max_attempts")
|
|
3684
|
+
if st.get("attempts", 0) >= max_attempts:
|
|
3685
|
+
notify(f"{project}: agent stuck", f"{max_attempts} nudges, no progress — needs a human", root)
|
|
3686
|
+
print("backoff exhausted; notified a human")
|
|
3687
|
+
return 0
|
|
3688
|
+
if st.get("last_nudge") and size == st.get("last_size"):
|
|
3689
|
+
wait = args.idle_minutes * 60 * (2 ** st["attempts"])
|
|
3690
|
+
if time.time() - st["last_nudge"] < wait:
|
|
3691
|
+
print(f"backing off ({st['attempts']} attempts, waiting {int(wait)}s)")
|
|
3692
|
+
return 0
|
|
3693
|
+
|
|
3694
|
+
# The project's own nudge, in its language, unless the cycle was handed another (LANGUAGE-PROMPTS).
|
|
3695
|
+
handed = getattr(args, "prompt", None)
|
|
3696
|
+
prompt = (language.text(cfg, "prompt.nudge") if handed is None else handed) \
|
|
3697
|
+
+ (parked_note(cfg, *passing) if passing else "") \
|
|
3698
|
+
+ (secondary_note(cfg, elsewhere_ready) if elsewhere_ready else "")
|
|
3699
|
+
if fe:
|
|
3700
|
+
# A person is in these files right now. Say so in the prompt; the
|
|
3701
|
+
# implementer keeps away from them for this turn.
|
|
3702
|
+
prompt += language.text(cfg, "prompt.nudge-editing", paths=", ".join(fe[:8]))
|
|
3703
|
+
# Past what one argument carries, the prompt goes on standard input where the implementer's
|
|
3704
|
+
# adapter declares it may; a detached turn takes no file polyctl would remove after it (PROMPT-CHANNEL).
|
|
3705
|
+
# A batch program on Windows takes it there whatever its size, and one whose adapter declares no
|
|
3706
|
+
# standard input is not started: prompt_input decides once argv[0] is resolved (WINDOWS-CMD-CHANNEL).
|
|
3707
|
+
# A grant the resume carries of its own admits what a command rewriter makes of each command it
|
|
3708
|
+
# names, as the one options.unattended appends does (GRANTS-RTK). The nudge is composed from the
|
|
3709
|
+
# adapter the package ships, never from the user's or the project's layer, which an agent can
|
|
3710
|
+
# write (TURN-ARGV).
|
|
3711
|
+
shipped = A.turn_adapter(impl.get("adapter"))
|
|
3712
|
+
if shipped is None:
|
|
3713
|
+
from . import features as F
|
|
3714
|
+
refused = A.turn_refusal("implementer", impl.get("adapter"), [])
|
|
3715
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · {refused}; not nudging")
|
|
3716
|
+
if F.enabled(cfg, "nudge"):
|
|
3717
|
+
tell_turn_refused(root, "implementer", refused)
|
|
3718
|
+
return 0
|
|
3719
|
+
plan, refused = A.prompt_plan(A.admit_rewrites((shipped.get("resume") or {}).get("argv") or [])[0], prompt,
|
|
3720
|
+
impl.get("adapter"), detached=True)
|
|
3721
|
+
if refused:
|
|
3722
|
+
print(f"the nudge's prompt cannot be handed over: {refused}; not nudging")
|
|
3723
|
+
return 1
|
|
3724
|
+
# Only a session that is the implementer's is resumed: pinned, recorded, or found where no other
|
|
3725
|
+
# role's sessions are kept beside it. One read beside a role that is not settled is not (SESSION-IDENTITY).
|
|
3726
|
+
sess, withheld = A.session_to_resume(cfg, "implementer")
|
|
3727
|
+
if not sess:
|
|
3728
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · {withheld}; not nudging")
|
|
3729
|
+
return 0
|
|
3730
|
+
argv = [x.replace("{session}", sess).replace("{prompt}", prompt)
|
|
3731
|
+
for x in plan["argv"]]
|
|
3732
|
+
if not argv:
|
|
3733
|
+
print("adapter has no resume command")
|
|
3734
|
+
return 1
|
|
3735
|
+
|
|
3736
|
+
# Resolve the CLI and hand the child a usable PATH — see child_path(). Never a program the current
|
|
3737
|
+
# directory holds, which Windows looks in before PATH (WINDOWS-NO-CWD-EXEC), nor one inside the tree (ROUTE-ARGV).
|
|
3738
|
+
search = child_path()
|
|
3739
|
+
resolved = A.program_path(argv[0], path=A.outside_repository(search, root))
|
|
3740
|
+
if not resolved:
|
|
3741
|
+
notify(f"{project}: watchdog", f"{argv[0]} not on PATH; cannot nudge", root)
|
|
3742
|
+
print(f"{argv[0]} not found on PATH ({search})")
|
|
3743
|
+
return 1
|
|
3744
|
+
named, argv[0] = argv[0], resolved
|
|
3745
|
+
# What a turn nobody attends starts with: the grant its adapter declares for one, or its
|
|
3746
|
+
# trust_all, and only when the resume carries no allowlist of its own - appending
|
|
3747
|
+
# --dangerously-skip-permissions on top of --allowedTools would silently override the
|
|
3748
|
+
# narrower grant. A flag that turns off the harness's own sandbox waits for a person, and the
|
|
3749
|
+
# permission mode is the one the adapter pins, whatever a person's settings default to (GRANTS-PINNED).
|
|
3750
|
+
added, added_why = A.unattended_flags(shipped, argv)
|
|
3751
|
+
bypass = A.bypass_refusal(shipped, added, cfg)
|
|
3752
|
+
if bypass:
|
|
3753
|
+
notify(f"{project}: nudge refused", bypass, root, key="nudge-bypass-refused", window=12 * 3600,
|
|
3754
|
+
audience="human", what=bypass)
|
|
3755
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · {bypass}; not nudging")
|
|
3756
|
+
return 0
|
|
3757
|
+
argv += added
|
|
3758
|
+
argv, pinned = A.pinned_argv(argv, "implementer")
|
|
3759
|
+
if pinned:
|
|
3760
|
+
added = added + pinned
|
|
3761
|
+
added_why = "; ".join(filter(None, [added_why, "the implementer's command names no permission mode, "
|
|
3762
|
+
"so polyctl appends the one its adapter pins"]))
|
|
3763
|
+
# The project chooses the implementer's model and effort in its config; the
|
|
3764
|
+
# adapter says how to spell them. Nothing is appended for an adapter that
|
|
3765
|
+
# has no such option, or declares it null: one with no flag for it (TURN-ARGV).
|
|
3766
|
+
opts = shipped.get("options") or {}
|
|
3767
|
+
for key in ("model", "effort"):
|
|
3768
|
+
val = impl.get(key)
|
|
3769
|
+
if val and isinstance(opts.get(key), list) and not any(str(a).startswith(f"--{key}") for a in argv):
|
|
3770
|
+
if key == "effort" and opts.get("effort_values") and val not in opts["effort_values"]:
|
|
3771
|
+
print(f"effort {val!r} not in {opts['effort_values']}; ignored")
|
|
3772
|
+
continue
|
|
3773
|
+
argv += [x.replace("{" + key + "}", str(val)) for x in opts[key]]
|
|
3774
|
+
|
|
3775
|
+
from . import features as F
|
|
3776
|
+
if not F.enabled(cfg, "nudge"):
|
|
3777
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · nudge feature off; not starting a turn")
|
|
3778
|
+
return 0
|
|
3779
|
+
# The command as it starts runs the shipped adapter's harness, with no value that reads as a flag
|
|
3780
|
+
# and no grant beyond its own, before anything is spent on it (TURN-ARGV).
|
|
3781
|
+
refused = A.turn_refusal("implementer", impl.get("adapter"), [named] + argv[1:], root=root, resolved=resolved,
|
|
3782
|
+
search=search, values={"session": sess, "model": impl.get("model"),
|
|
3783
|
+
"effort": impl.get("effort")}, prompt=prompt)
|
|
3784
|
+
if refused:
|
|
3785
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · {refused}; not nudging")
|
|
3786
|
+
tell_turn_refused(root, "implementer", refused)
|
|
3787
|
+
return 0
|
|
3788
|
+
headroom = A.rotate_if_exhausted(cfg, argv, "implementer") if not args.dry_run else {"ok": True}
|
|
3789
|
+
if not headroom["ok"]:
|
|
3790
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · {headroom['text']}; not nudging")
|
|
3791
|
+
return 0
|
|
3792
|
+
# The implementer's harness runs what it wrote into its own configuration: read after the rotation, which can
|
|
3793
|
+
# wait minutes on keyflip, and in a dry cycle as in a live one (HARNESS-CONFIG). Its command is the one a person
|
|
3794
|
+
# accepted (TURN-ARGV).
|
|
3795
|
+
unseen = harness_config_hold(root) or turn_hold(root, cfg, "implementer")
|
|
3796
|
+
if unseen:
|
|
3797
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · {unseen}; not nudging")
|
|
3798
|
+
return 0
|
|
3799
|
+
print(f"idle {int(age)}s · {', '.join(reasons)} · nudging")
|
|
3800
|
+
if args.dry_run:
|
|
3801
|
+
print("DRY RUN:", " ".join(argv[:4]), "…")
|
|
3802
|
+
return 0
|
|
3803
|
+
|
|
3804
|
+
# Never discard the child's output. A nudge that dies on an expired login or
|
|
3805
|
+
# an exhausted plan looks exactly like an agent that ignored us, and the
|
|
3806
|
+
# difference is the only thing worth knowing at that moment.
|
|
3807
|
+
key = A.project_key(root)
|
|
3808
|
+
log_path = os.path.join(STATE_DIR, A.project_file_name("nudge-log", key))
|
|
3809
|
+
os.makedirs(STATE_DIR, exist_ok=True)
|
|
3810
|
+
env = dict(os.environ, PATH=search, POLYCTL_ROLE="implementer")
|
|
3811
|
+
if not record_turn_flags(root, "implementer", added, added_why):
|
|
3812
|
+
return 0
|
|
3813
|
+
held = A.hold_state(root)
|
|
3814
|
+
if held:
|
|
3815
|
+
print(f"held by {held.get('by')} since this cycle began; not nudging")
|
|
3816
|
+
return 0
|
|
3817
|
+
given, refused = A.prompt_input(plan, root, argv)
|
|
3818
|
+
if refused:
|
|
3819
|
+
print(f"the nudge's prompt cannot be handed over: {refused}; not nudging")
|
|
3820
|
+
return 1
|
|
3821
|
+
# A child can finish its useful work inside the startup wait (TRIAGE-2-MEDIUM).
|
|
3822
|
+
nudged_fp = A.work_fingerprint(root)
|
|
3823
|
+
nudged_inputs = A.nudge_inputs(root, cfg)
|
|
3824
|
+
with open(log_path, "a", encoding=UTF8) as log:
|
|
3825
|
+
log.write(f"\n=== {time.strftime('%Y-%m-%d %H:%M:%S')} nudge"
|
|
3826
|
+
f"{' (added: ' + ' '.join(added) + ')' if added else ''} ===\n")
|
|
3827
|
+
log.flush()
|
|
3828
|
+
try:
|
|
3829
|
+
proc = subprocess.Popen(given["argv"], cwd=root, env=env,
|
|
3830
|
+
stdin=subprocess.DEVNULL if given["stdin"] is None else given["stdin"],
|
|
3831
|
+
stdout=log, stderr=subprocess.STDOUT,
|
|
3832
|
+
start_new_session=True)
|
|
3833
|
+
finally:
|
|
3834
|
+
A.release_prompt(given)
|
|
3835
|
+
# The turn is on record before its first seconds are waited out: a cycle cut off in them left a turn
|
|
3836
|
+
# running in a session of its own that no state named, and the next cycle cut the log it still wrote
|
|
3837
|
+
# (JOURNAL-3-3).
|
|
3838
|
+
try:
|
|
3839
|
+
child_start = A._process_start(proc.pid, refresh=True)
|
|
3840
|
+
except Exception:
|
|
3841
|
+
child_start = None
|
|
3842
|
+
st.update(child_pid=proc.pid, child_start=child_start, last_nudge=time.time())
|
|
3843
|
+
save_state(root, st)
|
|
3844
|
+
|
|
3845
|
+
# Give it a moment to fail. A healthy turn runs for minutes; anything that
|
|
3846
|
+
# exits within seconds died rather than started.
|
|
3847
|
+
early = None
|
|
3848
|
+
for _ in range(12):
|
|
3849
|
+
time.sleep(1)
|
|
3850
|
+
if proc.poll() is not None:
|
|
3851
|
+
early = proc.returncode
|
|
3852
|
+
break
|
|
3853
|
+
|
|
3854
|
+
st.update(attempts=st.get("attempts", 0) + 1, last_nudge=time.time(),
|
|
3855
|
+
last_size=size, child_pid=proc.pid, child_start=child_start,
|
|
3856
|
+
last_fingerprint=nudged_fp,
|
|
3857
|
+
nudge_size=size, nudge_fingerprint=nudged_fp,
|
|
3858
|
+
nudge_inputs=nudged_inputs)
|
|
3859
|
+
# Told once its first seconds are over, with the exit of a turn that did not outlive them (EVENTS-LOG).
|
|
3860
|
+
A.emit_event(root, "nudge", {"pid": proc.pid, "attempt": st["attempts"], "exit": early})
|
|
3861
|
+
if early not in (None, 0):
|
|
3862
|
+
tail = ""
|
|
3863
|
+
try:
|
|
3864
|
+
with open(log_path, encoding=UTF8) as fh:
|
|
3865
|
+
tail = A.redact(" ".join(fh.read().strip().split("\n")[-3:])[-300:])
|
|
3866
|
+
except OSError:
|
|
3867
|
+
pass
|
|
3868
|
+
st.pop("child_pid", None) # it is gone; do not guard on a dead pid
|
|
3869
|
+
st["last_error"] = {"at": time.time(), "code": early, "tail": tail}
|
|
3870
|
+
# A nudge that died on the usage limit parks its slice now, and the park is what a person
|
|
3871
|
+
# is told: a park tried again at its retry would otherwise ring "nudge failed" each time (QUOTA-PARK).
|
|
3872
|
+
try:
|
|
3873
|
+
parked = quota_park(root, cfg, st, adapter=adapter)
|
|
3874
|
+
except Exception as exc: # a park that cannot be written must not take the nudge's record
|
|
3875
|
+
parked = False
|
|
3876
|
+
print(f"the park could not be written: {type(exc).__name__}: {str(exc)[:120]}")
|
|
3877
|
+
if not parked:
|
|
3878
|
+
notify(f"{project}: nudge failed", f"exit {early}: {tail[-120:] or 'see nudge log'}", root)
|
|
3879
|
+
print(f"nudge failed (exit {early}): {tail}")
|
|
3880
|
+
else:
|
|
3881
|
+
st.pop("last_error", None)
|
|
3882
|
+
save_state(root, st)
|
|
3883
|
+
return 0
|
|
3884
|
+
|
|
3885
|
+
|
|
3886
|
+
if __name__ == "__main__":
|
|
3887
|
+
sys.exit(main())
|