polyctl 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyctl/__init__.py +7 -0
- polyctl/__main__.py +5 -0
- polyctl/a2a.py +226 -0
- polyctl/a2a_mcp.py +546 -0
- polyctl/acp.py +417 -0
- polyctl/adapters/README.md +101 -0
- polyctl/adapters/aider.json +32 -0
- polyctl/adapters/amazon-q.json +15 -0
- polyctl/adapters/amp.json +16 -0
- polyctl/adapters/antigravity.json +120 -0
- polyctl/adapters/claude-code.json +229 -0
- polyctl/adapters/cloud-generic.json +57 -0
- polyctl/adapters/codex.json +47 -0
- polyctl/adapters/command-code.json +45 -0
- polyctl/adapters/copilot.json +17 -0
- polyctl/adapters/cursor-agent.json +29 -0
- polyctl/adapters/deepseek.json +13 -0
- polyctl/adapters/droid.json +34 -0
- polyctl/adapters/gemini.json +30 -0
- polyctl/adapters/grok.json +22 -0
- polyctl/adapters/hermes.json +34 -0
- polyctl/adapters/kilocode.json +30 -0
- polyctl/adapters/kimi.json +23 -0
- polyctl/adapters/kiro.json +332 -0
- polyctl/adapters/ollama.json +29 -0
- polyctl/adapters/omp.json +34 -0
- polyctl/adapters/openai-api.json +68 -0
- polyctl/adapters/opencode.json +162 -0
- polyctl/adapters/pi.json +24 -0
- polyctl/adapters/pr-agent.json +54 -0
- polyctl/adapters/profiles.json +9 -0
- polyctl/adapters/qoder.json +163 -0
- polyctl/adapters/qwen.json +27 -0
- polyctl/adapters/reasonix.json +25 -0
- polyctl/adapters/rewriters/rtk.json +24 -0
- polyctl/adapters/trae.json +31 -0
- polyctl/adapters/vendors.json +42 -0
- polyctl/adapters/warp.json +88 -0
- polyctl/allowlist.py +1756 -0
- polyctl/api_review.py +920 -0
- polyctl/cli.py +835 -0
- polyctl/conformance_harness.py +9 -0
- polyctl/demo.py +503 -0
- polyctl/drivers.py +146 -0
- polyctl/email.py +160 -0
- polyctl/features.py +98 -0
- polyctl/journal.py +144 -0
- polyctl/language.py +945 -0
- polyctl/lib.py +1296 -0
- polyctl/matrix.py +660 -0
- polyctl/mcp.py +1362 -0
- polyctl/ntfy.py +169 -0
- polyctl/outbound.py +39 -0
- polyctl/parts/__init__.py +5 -0
- polyctl/parts/cli_authority.py +2615 -0
- polyctl/parts/cli_channels.py +1325 -0
- polyctl/parts/cli_checks.py +2293 -0
- polyctl/parts/cli_completion.py +686 -0
- polyctl/parts/cli_evidence.py +141 -0
- polyctl/parts/cli_gain.py +375 -0
- polyctl/parts/cli_harness.py +392 -0
- polyctl/parts/cli_hooks.py +1816 -0
- polyctl/parts/cli_install.py +638 -0
- polyctl/parts/cli_lanes.py +106 -0
- polyctl/parts/cli_maintenance.py +2536 -0
- polyctl/parts/cli_next.py +248 -0
- polyctl/parts/cli_pr.py +53 -0
- polyctl/parts/cli_project.py +2857 -0
- polyctl/parts/cli_review.py +5158 -0
- polyctl/parts/cli_status.py +898 -0
- polyctl/parts/lib_adapters.py +2933 -0
- polyctl/parts/lib_alarms.py +2106 -0
- polyctl/parts/lib_events.py +224 -0
- polyctl/parts/lib_evidence.py +412 -0
- polyctl/parts/lib_gates.py +2099 -0
- polyctl/parts/lib_lanes.py +829 -0
- polyctl/parts/lib_mail.py +1541 -0
- polyctl/parts/lib_next.py +652 -0
- polyctl/parts/lib_pr.py +331 -0
- polyctl/parts/lib_slices.py +3337 -0
- polyctl/parts/lib_state.py +2415 -0
- polyctl/parts/lib_transcript.py +4118 -0
- polyctl/pressure.py +325 -0
- polyctl/prices.json +47 -0
- polyctl/procs.py +1194 -0
- polyctl/settings.py +505 -0
- polyctl/skill/SKILL.md +365 -0
- polyctl/skillkit.py +503 -0
- polyctl/storage.py +2055 -0
- polyctl/telegram.py +327 -0
- polyctl/tiers.py +123 -0
- polyctl/verdicts.py +41 -0
- polyctl/watchdog.py +3887 -0
- polyctl/web.py +337 -0
- polyctl-0.7.0.dist-info/METADATA +235 -0
- polyctl-0.7.0.dist-info/RECORD +99 -0
- polyctl-0.7.0.dist-info/WHEEL +4 -0
- polyctl-0.7.0.dist-info/entry_points.txt +5 -0
- polyctl-0.7.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,4118 @@
|
|
|
1
|
+
"""Sessions and transcripts: discovery, stores, progress and anomalies read from a transcript.
|
|
2
|
+
|
|
3
|
+
A part of src/polyctl/lib.py (#44): moved out byte for byte and run in its namespace by `_part`,
|
|
4
|
+
where it stood; it is not importable on its own.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# ── session discovery ─────────────────────────────────────────────────────────
|
|
9
|
+
|
|
10
|
+
_PACKAGE_ADAPTERS = {}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def package_adapters():
|
|
14
|
+
"""{id: adapter} for the adapters in the package alone (#76).
|
|
15
|
+
|
|
16
|
+
Whatever decides authority - which directories are coordination rather than
|
|
17
|
+
product, which processes are agents - reads only these. A user's or a project's
|
|
18
|
+
adapter layer is writable by the agents it describes, and must not be able to
|
|
19
|
+
move a product path out of review by declaring it a harness's directory. Read
|
|
20
|
+
once per state of the directory: a path check runs this for every changed path.
|
|
21
|
+
"""
|
|
22
|
+
directory = adapters_dir()
|
|
23
|
+
try:
|
|
24
|
+
key = (directory, os.stat(directory).st_mtime_ns)
|
|
25
|
+
names = sorted(os.listdir(directory))
|
|
26
|
+
except OSError:
|
|
27
|
+
return {}
|
|
28
|
+
if key in _PACKAGE_ADAPTERS:
|
|
29
|
+
return _PACKAGE_ADAPTERS[key]
|
|
30
|
+
found = {}
|
|
31
|
+
for name in names:
|
|
32
|
+
if not name.endswith(".json") or name in DATA_FILES:
|
|
33
|
+
continue
|
|
34
|
+
try:
|
|
35
|
+
with open(os.path.join(directory, name), encoding=UTF8) as fh:
|
|
36
|
+
adapter = json.load(fh)
|
|
37
|
+
except (OSError, ValueError):
|
|
38
|
+
continue
|
|
39
|
+
if isinstance(adapter, dict):
|
|
40
|
+
found[str(adapter.get("id") or name[:-5])] = adapter
|
|
41
|
+
_PACKAGE_ADAPTERS.clear()
|
|
42
|
+
_PACKAGE_ADAPTERS[key] = found
|
|
43
|
+
return found
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _home_path(template):
|
|
47
|
+
"""A declared path with a leading ~ resolved against polyctl's HOME, so every reader of it agrees."""
|
|
48
|
+
text = str(template or "")
|
|
49
|
+
if text == "~":
|
|
50
|
+
return HOME
|
|
51
|
+
if text.startswith("~/"):
|
|
52
|
+
return os.path.join(HOME, *text[2:].split("/"))
|
|
53
|
+
return text
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# The kinds of session store this polyctl knows how to reach. An adapter that names none of them has no
|
|
57
|
+
# transcript polyctl can resolve, whatever its records look like once they are in hand, which is why a
|
|
58
|
+
# conformance report says "declared" for one rather than "read".
|
|
59
|
+
SESSION_STORE_KINDS = ("escaped-cwd", "workspace-meta", "sqlite-sessions")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _read_only_uri(path):
|
|
63
|
+
"""A `file:` URI for a SQLite store opened read-only, built in one place because a raw path breaks it.
|
|
64
|
+
|
|
65
|
+
Interpolating a path into `file:{path}?mode=ro` reads a `?`, `#` or `%` in a directory name as URI
|
|
66
|
+
syntax of its own, and the store then opens as something other than the file the adapter named, or
|
|
67
|
+
not at all. `pathname2url` percent-encodes each of those and writes a Windows drive path as the
|
|
68
|
+
`///C:/...` a file URI takes, so the query that follows is the only `?` in the string.
|
|
69
|
+
|
|
70
|
+
A UNC store keeps an empty authority: it converts to `//server/share/...`, and a URI that made the
|
|
71
|
+
server its authority is one SQLite refuses, so the prefix is chosen to leave the authority empty
|
|
72
|
+
and the path whole in both shapes. The adapter's own path is what is opened; a symlink is not
|
|
73
|
+
resolved away, because what a store is named is the store's to say.
|
|
74
|
+
"""
|
|
75
|
+
import urllib.request
|
|
76
|
+
url = urllib.request.pathname2url(os.path.abspath(str(path)))
|
|
77
|
+
return ("file:" if url.startswith("///") else "file://") + url + "?mode=ro"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def session_stores(kind):
|
|
81
|
+
"""[(adapter id, store)] for each adapter whose `sessions` store is of this kind (#76)."""
|
|
82
|
+
return [(ident, adapter["sessions"]) for ident, adapter in sorted(package_adapters().items())
|
|
83
|
+
if isinstance(adapter.get("sessions"), dict) and adapter["sessions"].get("kind") == kind]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _workspace_sessions(store):
|
|
87
|
+
"""Every session a workspace-meta store holds: its id, workspace directory, workspaces and transcript age."""
|
|
88
|
+
base = _home_path(store.get("dir"))
|
|
89
|
+
# A directory gone between the listing and the read, and a metadata file that holds no JSON object, are
|
|
90
|
+
# a session this store does not have; each stopped every reader of the store (REVIEWER-IDENTITY-2).
|
|
91
|
+
try:
|
|
92
|
+
workspaces = os.listdir(base) if os.path.isdir(base) else []
|
|
93
|
+
except OSError:
|
|
94
|
+
workspaces = []
|
|
95
|
+
for ws in workspaces:
|
|
96
|
+
wsd = os.path.join(base, ws)
|
|
97
|
+
try:
|
|
98
|
+
sessions = os.listdir(wsd) if os.path.isdir(wsd) else []
|
|
99
|
+
except OSError:
|
|
100
|
+
continue
|
|
101
|
+
for sess in sessions:
|
|
102
|
+
meta = os.path.join(wsd, sess, store["meta"])
|
|
103
|
+
msgs = os.path.join(wsd, sess, store["transcript"])
|
|
104
|
+
if not (os.path.exists(meta) and os.path.exists(msgs)):
|
|
105
|
+
continue
|
|
106
|
+
try:
|
|
107
|
+
with open(meta, encoding=UTF8) as fh:
|
|
108
|
+
m = json.load(fh)
|
|
109
|
+
mtime = os.path.getmtime(msgs)
|
|
110
|
+
except Exception:
|
|
111
|
+
continue
|
|
112
|
+
if not isinstance(m, dict):
|
|
113
|
+
continue
|
|
114
|
+
yield {"session": sess, "workspace_hash": ws, "paths": m.get(store["workspaces"]) or [], "mtime": mtime,
|
|
115
|
+
"title": m.get(store.get("title") or "title", ""), "status": m.get(store.get("status") or "status", "")}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _declared_store(ident):
|
|
119
|
+
"""The `sessions` store the package adapter `ident` declares, or {}."""
|
|
120
|
+
store = (package_adapters().get(str(ident or "")) or {}).get("sessions")
|
|
121
|
+
return store if isinstance(store, dict) else {}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _sqlite_tail(ident):
|
|
125
|
+
"""How one session of the package adapter `ident` is read: the store and the queries, or None.
|
|
126
|
+
|
|
127
|
+
The store is named by `transcript.path` and how to query it by `transcript.record`, so a
|
|
128
|
+
session found here and a session named in a role's block resolve the same way.
|
|
129
|
+
"""
|
|
130
|
+
declared = _block((package_adapters().get(str(ident or "")) or {}), "transcript")
|
|
131
|
+
db = _home_path(declared.get("path")) if _name(declared.get("kind")) == "sqlite" else ""
|
|
132
|
+
return (db, _block(declared, "record"), _block(declared, "freshness"),
|
|
133
|
+
_block(declared, "subagent_sessions")) if db else None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _sqlite_sessions(ident, store, cwd):
|
|
137
|
+
"""[{session, workspace_hash, title, status, mtime, transcript}]: each session a database store keeps for `cwd`.
|
|
138
|
+
|
|
139
|
+
A database names a session's working directory in a column rather than in the name of the
|
|
140
|
+
directory holding it, so the store declares the query and the columns to read from its rows.
|
|
141
|
+
The working directory is bound as a parameter, never written into the query: it comes from a
|
|
142
|
+
checkout on disk, and a path with a quote in it is a path a person can name.
|
|
143
|
+
"""
|
|
144
|
+
sql, tail = _read_query(store.get("query")), _sqlite_tail(ident)
|
|
145
|
+
if not sql or not tail or not cwd or not os.path.exists(tail[0]):
|
|
146
|
+
return []
|
|
147
|
+
try:
|
|
148
|
+
import sqlite3
|
|
149
|
+
connection = sqlite3.connect(_read_only_uri(tail[0]), uri=True, timeout=2.0)
|
|
150
|
+
except Exception:
|
|
151
|
+
return []
|
|
152
|
+
connection.row_factory = sqlite3.Row
|
|
153
|
+
try:
|
|
154
|
+
rows = [dict(row) for row in connection.execute(sql, (cwd,))]
|
|
155
|
+
except Exception:
|
|
156
|
+
rows = []
|
|
157
|
+
finally:
|
|
158
|
+
connection.close()
|
|
159
|
+
milliseconds = str(store.get("mtime_unit")) in ("ms", "milliseconds")
|
|
160
|
+
found = []
|
|
161
|
+
for row in rows:
|
|
162
|
+
session, stamp = row.get(_store_name(store.get("session"))), row.get(_store_name(store.get("mtime")))
|
|
163
|
+
number = stamp if isinstance(stamp, (int, float)) and not isinstance(stamp, bool) else None
|
|
164
|
+
if not isinstance(session, str) or not session:
|
|
165
|
+
continue
|
|
166
|
+
found.append({"session": session, "workspace_hash": None,
|
|
167
|
+
"title": _name(row.get(_store_name(store.get("title")))), "status": "",
|
|
168
|
+
"mtime": (number / 1000.0 if milliseconds else float(number)) if number is not None else 0.0,
|
|
169
|
+
"transcript": DatabaseTail(tail[0], session, tail[1], tail[2], tail[3]), "meta": None})
|
|
170
|
+
return found
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def store_sessions(ident, cwd):
|
|
174
|
+
"""[{adapter, session, workspace_hash, transcript, meta, mtime}], newest first: each session that the store
|
|
175
|
+
the package adapter `ident` declares keeps for the working directory `cwd` (SESSION-IDENTITY).
|
|
176
|
+
|
|
177
|
+
A workspace-meta store names a session's workspaces in its metadata; an escaped-cwd store
|
|
178
|
+
keeps a directory's sessions in a directory named for it, one transcript each. A helper polyctl
|
|
179
|
+
starts - a reviewer, a hunter - runs in a directory of its own outside the repository, so
|
|
180
|
+
no session of one is ever a workspace's.
|
|
181
|
+
"""
|
|
182
|
+
store = _declared_store(ident)
|
|
183
|
+
rows = []
|
|
184
|
+
if store.get("kind") == "workspace-meta":
|
|
185
|
+
base = _home_path(store.get("dir"))
|
|
186
|
+
for row in _workspace_sessions(store):
|
|
187
|
+
if isinstance(row["paths"], list) and cwd in row["paths"]:
|
|
188
|
+
folder = os.path.join(base, row["workspace_hash"], row["session"])
|
|
189
|
+
rows.append({"adapter": ident, "session": row["session"], "workspace_hash": row["workspace_hash"],
|
|
190
|
+
"transcript": os.path.join(folder, store["transcript"]),
|
|
191
|
+
"meta": os.path.join(folder, store["meta"]), "mtime": row["mtime"]})
|
|
192
|
+
elif store.get("kind") == "escaped-cwd" and cwd:
|
|
193
|
+
rows = [{"adapter": ident, "session": session, "workspace_hash": None, "transcript": path, "meta": None,
|
|
194
|
+
"mtime": mtime} for session, path, mtime in _escaped_transcripts(store, escaped_cwd_dir(store, cwd))]
|
|
195
|
+
elif store.get("kind") == "sqlite-sessions" and cwd:
|
|
196
|
+
rows = [dict(row, adapter=ident) for row in _sqlite_sessions(ident, store, cwd)]
|
|
197
|
+
return sorted(rows, key=lambda row: row["mtime"], reverse=True)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _escaped_transcripts(store, directory):
|
|
201
|
+
"""[(session, transcript, mtime)], newest first: each transcript an escaped-cwd store keeps in one directory."""
|
|
202
|
+
prefix, _, suffix = _name(store.get("transcript")).partition("{session}")
|
|
203
|
+
found = []
|
|
204
|
+
try:
|
|
205
|
+
entries = list(os.scandir(directory))
|
|
206
|
+
except OSError:
|
|
207
|
+
return found
|
|
208
|
+
for entry in entries:
|
|
209
|
+
name = entry.name
|
|
210
|
+
if len(name) <= len(prefix) + len(suffix) or not name.startswith(prefix) or not name.endswith(suffix):
|
|
211
|
+
continue
|
|
212
|
+
try:
|
|
213
|
+
if entry.is_file():
|
|
214
|
+
found.append((name[len(prefix):len(name) - len(suffix)], entry.path, entry.stat().st_mtime))
|
|
215
|
+
except OSError:
|
|
216
|
+
continue
|
|
217
|
+
return sorted(found, key=lambda item: item[2], reverse=True)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# How much of a transcript's end and of its beginning is read to place its directory: a
|
|
221
|
+
# directory is placed from the last or the first records of one of its newest transcripts.
|
|
222
|
+
WORKSPACE_READ_BYTES = 65_536
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _escaped_workspaces(store):
|
|
226
|
+
"""[(workspace, newest session, its mtime)] for each directory of an escaped-cwd store polyctl can place.
|
|
227
|
+
|
|
228
|
+
A directory's name cannot be read back into a path - one dash stood for "/", for "." and
|
|
229
|
+
for every other character that is not a letter or a digit - so the path is one the
|
|
230
|
+
store's own records name at the field it declares as `cwd`, taken only where it escapes to
|
|
231
|
+
that directory's name: a session moved to another directory is kept under the new one, and
|
|
232
|
+
its first records still name the old. The ends of the three newest transcripts are read,
|
|
233
|
+
never a whole one. A store that declares no such field places no directory.
|
|
234
|
+
"""
|
|
235
|
+
field = _name(store.get("cwd"))
|
|
236
|
+
if not field:
|
|
237
|
+
return []
|
|
238
|
+
try:
|
|
239
|
+
directories = list(os.scandir(_home_path(store.get("dir"))))
|
|
240
|
+
except OSError:
|
|
241
|
+
return []
|
|
242
|
+
out = []
|
|
243
|
+
for entry in directories:
|
|
244
|
+
try:
|
|
245
|
+
if not entry.is_dir():
|
|
246
|
+
continue
|
|
247
|
+
except OSError:
|
|
248
|
+
continue
|
|
249
|
+
transcripts = _escaped_transcripts(store, entry.path)
|
|
250
|
+
path = next((value for _, transcript, _ in transcripts[:3] for value in _recorded_values(transcript, field)
|
|
251
|
+
if entry.name in escaped_cwd_names(value)), None)
|
|
252
|
+
if path:
|
|
253
|
+
out.append((path, transcripts[0][0], transcripts[0][2]))
|
|
254
|
+
return out
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _recorded_values(path, field, nbytes=WORKSPACE_READ_BYTES):
|
|
258
|
+
"""The text a transcript's records hold at `field`: its last records newest first, then its first ones."""
|
|
259
|
+
marker = '"' + field.split(".")[0].split("[")[0] + '"'
|
|
260
|
+
try:
|
|
261
|
+
size = os.path.getsize(path)
|
|
262
|
+
with open(path, "rb") as fh:
|
|
263
|
+
fh.seek(max(0, size - nbytes))
|
|
264
|
+
tail = fh.read(nbytes)
|
|
265
|
+
fh.seek(0)
|
|
266
|
+
head = fh.read(nbytes) if size > nbytes else b""
|
|
267
|
+
except OSError:
|
|
268
|
+
return []
|
|
269
|
+
values = []
|
|
270
|
+
for blob, newest_first in ((tail, True), (head, False)):
|
|
271
|
+
lines = blob.decode(UTF8, "replace").split("\n")
|
|
272
|
+
if newest_first and size > nbytes:
|
|
273
|
+
lines = lines[1:] # the tail starts inside a record
|
|
274
|
+
for line in (reversed(lines) if newest_first else lines):
|
|
275
|
+
if marker not in line:
|
|
276
|
+
continue
|
|
277
|
+
try:
|
|
278
|
+
value = _path_value(json.loads(line), field)
|
|
279
|
+
except (ValueError, RecursionError):
|
|
280
|
+
continue
|
|
281
|
+
if isinstance(value, str) and value:
|
|
282
|
+
values.append(value)
|
|
283
|
+
return values
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def discover_session(cwd, exclude=()):
|
|
287
|
+
"""Find the most recently active local agent session whose workspace is cwd.
|
|
288
|
+
|
|
289
|
+
Every store an adapter declares is read (`sessions`): a workspace-meta store's
|
|
290
|
+
metadata resolves the opaque per-workspace directory, and an escaped-cwd store keeps
|
|
291
|
+
the directory's transcripts under its escaped name, without asking the vendor CLI.
|
|
292
|
+
Only one harness's store was read, and a project running the other one had no
|
|
293
|
+
session at all. `exclude` holds the sessions another role pins.
|
|
294
|
+
"""
|
|
295
|
+
best = None
|
|
296
|
+
for ident in sorted(package_adapters()):
|
|
297
|
+
for row in store_sessions(ident, cwd):
|
|
298
|
+
if row["session"] in exclude:
|
|
299
|
+
continue
|
|
300
|
+
if best is None or row["mtime"] > best["_mtime"]:
|
|
301
|
+
best = {"adapter": ident, "session": row["session"], "workspace_hash": row["workspace_hash"],
|
|
302
|
+
"cwd": cwd, "_mtime": row["mtime"]}
|
|
303
|
+
break
|
|
304
|
+
return best
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def all_workspaces():
|
|
308
|
+
"""Every local agent session grouped by the workspace it belongs to, for workspaces that still exist.
|
|
309
|
+
|
|
310
|
+
Lets `polyctl` answer "which projects can I watch?" without any configuration —
|
|
311
|
+
the vendor stores already record their own workspace paths. A workspace-meta store
|
|
312
|
+
names them in each session's metadata; an escaped-cwd store's directories are placed
|
|
313
|
+
by the working directory their records name (`sessions.cwd`). A directory gone from
|
|
314
|
+
disk - a reviewer's temporary one, a deleted checkout - is no project to watch.
|
|
315
|
+
"""
|
|
316
|
+
found = {}
|
|
317
|
+
|
|
318
|
+
def keep(path, row):
|
|
319
|
+
current = found.get(path)
|
|
320
|
+
if current is None or row["mtime"] > current["mtime"]:
|
|
321
|
+
found[path] = dict(row, path=path)
|
|
322
|
+
|
|
323
|
+
for ident, store in session_stores("workspace-meta"):
|
|
324
|
+
for row in _workspace_sessions(store):
|
|
325
|
+
for path in row["paths"] if isinstance(row["paths"], list) else []:
|
|
326
|
+
keep(path, {"mtime": row["mtime"], "session": row["session"], "workspace_hash": row["workspace_hash"],
|
|
327
|
+
"adapter": ident, "title": row["title"], "status": row["status"]})
|
|
328
|
+
for ident, store in session_stores("escaped-cwd"):
|
|
329
|
+
for path, session, mtime in _escaped_workspaces(store):
|
|
330
|
+
keep(path, {"mtime": mtime, "session": session, "workspace_hash": None, "adapter": ident,
|
|
331
|
+
"title": "", "status": ""})
|
|
332
|
+
rows = [row for path, row in found.items() if isinstance(path, str) and os.path.isdir(path)]
|
|
333
|
+
return sorted(rows, key=lambda r: r["mtime"], reverse=True)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def session_paths(cfg):
|
|
337
|
+
"""Transcript and metadata paths for the configured implementer.
|
|
338
|
+
|
|
339
|
+
This was hard-wired to one harness's store, which made the watchdog work for
|
|
340
|
+
that harness only: an implementer on another resolved to no transcript, the
|
|
341
|
+
watchdog said "nothing to watch" and quietly never ran. Every adapter declares
|
|
342
|
+
its store (`sessions`); an implementer whose adapter declares none has no paths,
|
|
343
|
+
and neither has an `auto` polyctl could not resolve: `auto` is no session's id.
|
|
344
|
+
"""
|
|
345
|
+
return role_session_paths(cfg, "implementer")
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def role_session_paths(cfg, role):
|
|
349
|
+
"""(transcript, metadata) of the session a role's block names, or (None, None) (SESSION-IDENTITY).
|
|
350
|
+
|
|
351
|
+
A session pinned by its id alone in a workspace-meta store is found in the workspace
|
|
352
|
+
directory that keeps it. A store that keeps every session in one database has no transcript
|
|
353
|
+
path to name, so what comes back is a `DatabaseTail`: it is the database to a reader that asks
|
|
354
|
+
only whether the transcript exists, and the session to a reader that asks what it says.
|
|
355
|
+
"""
|
|
356
|
+
block = cfg.get(role) if isinstance(cfg.get(role), dict) else {}
|
|
357
|
+
sess = _concrete_session(block.get("session"))
|
|
358
|
+
if not sess:
|
|
359
|
+
return None, None
|
|
360
|
+
adapter = load_adapter(block_adapter(block) or "")
|
|
361
|
+
store = adapter.get("sessions") or {}
|
|
362
|
+
if store.get("kind") not in SESSION_STORE_KINDS:
|
|
363
|
+
return None, None # a shape polyctl has no way to load
|
|
364
|
+
if store.get("kind") == "escaped-cwd":
|
|
365
|
+
cwd = block.get("cwd") or cfg.get("root", "")
|
|
366
|
+
return os.path.join(escaped_cwd_dir(store, cwd), store["transcript"].replace("{session}", sess)), None
|
|
367
|
+
if store.get("kind") == "sqlite-sessions":
|
|
368
|
+
declared = _block(adapter, "transcript")
|
|
369
|
+
db = _home_path(declared.get("path")) if _name(declared.get("kind")) == "sqlite" else ""
|
|
370
|
+
return (DatabaseTail(db, sess, _block(declared, "record"), _block(declared, "freshness"),
|
|
371
|
+
_block(declared, "subagent_sessions")) if db
|
|
372
|
+
else None), None
|
|
373
|
+
ws = block.get("workspace_hash") or _session_workspace(store, sess)
|
|
374
|
+
if not ws:
|
|
375
|
+
return None, None
|
|
376
|
+
d = os.path.join(_home_path(store["dir"]), ws, sess)
|
|
377
|
+
return os.path.join(d, store["transcript"]), os.path.join(d, store["meta"])
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _session_workspace(store, session):
|
|
381
|
+
"""The workspace directory a workspace-meta store keeps a session in, or None."""
|
|
382
|
+
base = _home_path(store.get("dir"))
|
|
383
|
+
steps = _plain_steps(session)
|
|
384
|
+
if not steps or len(steps) != 1:
|
|
385
|
+
return None # an id is one name, never a path
|
|
386
|
+
try:
|
|
387
|
+
names = sorted(os.listdir(base))
|
|
388
|
+
except OSError:
|
|
389
|
+
return None
|
|
390
|
+
return next((ws for ws in names if os.path.exists(os.path.join(base, ws, session, store["meta"]))), None)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
# ── session identity: which session is whose (SESSION-IDENTITY) ───────────────
|
|
394
|
+
#
|
|
395
|
+
# Every `polyctl init --profile` writes `session: auto`, and it was read as a session's id: the
|
|
396
|
+
# implementer's transcript was named auto, the watchdog had nothing to watch, and the nudge
|
|
397
|
+
# would have resumed a session called auto. Where two roles run one harness in one
|
|
398
|
+
# directory, the newest transcript was whichever role wrote last, and a wake resumed the
|
|
399
|
+
# implementer's own session as the architect. `auto` is resolved when the config is loaded,
|
|
400
|
+
# from the store the role's adapter declares, and a role that cannot be resolved says why.
|
|
401
|
+
|
|
402
|
+
SESSION_ROLES = ("implementer", "architect")
|
|
403
|
+
RESOLVED_HOW = ("discovered", "recorded") # how a session polyctl put into a block got there
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _concrete_session(value):
|
|
407
|
+
"""The session id a block names: any text but `auto`; else None."""
|
|
408
|
+
text = value.strip() if isinstance(value, str) else ""
|
|
409
|
+
return text if text and text != "auto" else None
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _pinned_session(block):
|
|
413
|
+
"""The id a block pins itself, never one polyctl resolved into it."""
|
|
414
|
+
resolved = block.get("_session") if isinstance(block, dict) and isinstance(block.get("_session"), dict) else {}
|
|
415
|
+
return None if resolved.get("how") in RESOLVED_HOW else _concrete_session((block or {}).get("session"))
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _resumes_session(role, block):
|
|
419
|
+
"""Whether a role holds a session of its own: every implementer, whose nudge resumes one, and
|
|
420
|
+
an architect whose argv says {session} or whose block names one."""
|
|
421
|
+
if not isinstance(block, dict) or not block:
|
|
422
|
+
return False
|
|
423
|
+
return role == "implementer" or "session" in block or any("{session}" in str(part)
|
|
424
|
+
for part in block.get("argv") or [])
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _role_cwd(root, role, block):
|
|
428
|
+
"""Where a role's sessions are kept: its own `cwd`, else the project root. An architect that
|
|
429
|
+
resumes no session starts a new one in the project root at every wake."""
|
|
430
|
+
if role == "architect" and not _resumes_session(role, block):
|
|
431
|
+
return root
|
|
432
|
+
return (block or {}).get("cwd") or root
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _role_rows(role, ident, cwd):
|
|
436
|
+
"""[{session, workspace_hash}], newest first: the sessions a role could hold in its adapter's store.
|
|
437
|
+
|
|
438
|
+
An architect's are read through discover_architect, where every wake has read them,
|
|
439
|
+
unless its adapter keeps its sessions in another kind of store.
|
|
440
|
+
"""
|
|
441
|
+
kind = _declared_store(ident).get("kind")
|
|
442
|
+
if role == "architect" and (not ident or kind == "escaped-cwd"):
|
|
443
|
+
found = discover_architect(cwd) or {}
|
|
444
|
+
ids = found.get("sessions") if isinstance(found.get("sessions"), list) else [found.get("session")]
|
|
445
|
+
return [{"session": str(sid), "workspace_hash": None} for sid in ids if sid]
|
|
446
|
+
return [{"session": row["session"], "workspace_hash": row["workspace_hash"]} for row in store_sessions(ident, cwd)]
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def sessions_path(root):
|
|
450
|
+
"""Where polyctl keeps the sessions it settled for this checkout's roles; never committed."""
|
|
451
|
+
return os.path.join(root, ".polyctl", "sessions.json")
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def session_records(root):
|
|
455
|
+
"""{role: {adapter, session, cwd, workspace_hash, at}} as `.polyctl/sessions.json` keeps them; {} when it keeps none."""
|
|
456
|
+
try:
|
|
457
|
+
with open(sessions_path(root), encoding=UTF8) as fh:
|
|
458
|
+
document = json.load(fh)
|
|
459
|
+
except (OSError, ValueError):
|
|
460
|
+
return {}
|
|
461
|
+
roles = document.get("roles") if isinstance(document, dict) else None
|
|
462
|
+
if not isinstance(roles, dict):
|
|
463
|
+
return {}
|
|
464
|
+
return {role: dict(entry) for role, entry in roles.items()
|
|
465
|
+
if role in SESSION_ROLES and isinstance(entry, dict) and _concrete_session(entry.get("session"))}
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def record_session(root, role, found):
|
|
469
|
+
"""Keep the session polyctl settled for a role in `.polyctl/sessions.json`, written whole; True when it changed.
|
|
470
|
+
|
|
471
|
+
Nothing is written outside a project, or when the record already says it.
|
|
472
|
+
"""
|
|
473
|
+
if not os.path.isdir(os.path.join(root, ".polyctl")):
|
|
474
|
+
return False
|
|
475
|
+
records = session_records(root)
|
|
476
|
+
kept = {key: found.get(key) for key in ("adapter", "session", "cwd", "workspace_hash")}
|
|
477
|
+
if {key: (records.get(role) or {}).get(key) for key in kept} == kept:
|
|
478
|
+
return False
|
|
479
|
+
records[role] = dict(kept, at=int(time.time()))
|
|
480
|
+
from .storage import replace_file_durably
|
|
481
|
+
try:
|
|
482
|
+
replace_file_durably(sessions_path(root), (json.dumps({"roles": records}, indent=2, sort_keys=True)
|
|
483
|
+
+ "\n").encode(UTF8))
|
|
484
|
+
except OSError:
|
|
485
|
+
return False
|
|
486
|
+
return True
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def resolve_session(root, cfg, role, avoid=()):
|
|
490
|
+
"""The session a role holds, and how polyctl knows: {session, workspace_hash, adapter, cwd, how, why, trusted,
|
|
491
|
+
record, count} (SESSION-IDENTITY).
|
|
492
|
+
|
|
493
|
+
`how` is pinned (the block names an id), recorded (`.polyctl/sessions.json` keeps the one polyctl
|
|
494
|
+
settled on), discovered, ambiguous or unresolved, and `why` is what a person reads beside
|
|
495
|
+
it. A pinned id wins. Otherwise the candidates are the sessions the role's adapter's store
|
|
496
|
+
keeps for its working directory, less `avoid`. Where no other role runs that adapter over
|
|
497
|
+
them, the newest is the role's: a newer one is a conversation a person opened, and `auto`
|
|
498
|
+
follows it. Where the other role's sessions sit among them, the newest is whichever role
|
|
499
|
+
wrote last, so what settles identity is a pin, a record, or the one session the other role
|
|
500
|
+
does not hold. The implementer is resolved first and takes a lone session; the architect
|
|
501
|
+
never takes the implementer's, and beside an implementer that is not settled it cannot
|
|
502
|
+
tell its own. Two sessions neither holds are ambiguous. A session taken beside a role that
|
|
503
|
+
is not settled is read but never resumed (`trusted` false) and never recorded: only what
|
|
504
|
+
settles identity is kept, and `record` says a result is that.
|
|
505
|
+
"""
|
|
506
|
+
block = cfg.get(role) if isinstance(cfg.get(role), dict) else {}
|
|
507
|
+
ident = block_adapter(block) if block else None
|
|
508
|
+
cwd = _role_cwd(root, role, block)
|
|
509
|
+
out = {"role": role, "adapter": ident, "cwd": cwd, "session": None, "workspace_hash": None, "how": "unresolved",
|
|
510
|
+
"why": None, "trusted": False, "record": False, "count": 0}
|
|
511
|
+
if not block:
|
|
512
|
+
return dict(out, why=f"no {role} is configured")
|
|
513
|
+
pinned = _pinned_session(block)
|
|
514
|
+
if pinned:
|
|
515
|
+
return dict(out, session=pinned, workspace_hash=block.get("workspace_hash"), how="pinned", trusted=True)
|
|
516
|
+
if ident and _declared_store(ident).get("kind") not in ("workspace-meta", "escaped-cwd"):
|
|
517
|
+
return dict(out, why=f"the {ident} adapter declares no session store polyctl reads; pin the {role}'s session")
|
|
518
|
+
if not ident and role != "architect":
|
|
519
|
+
return dict(out, why=f"no adapter says where the {role}'s sessions are; pin its session")
|
|
520
|
+
rows = [row for row in _role_rows(role, ident, cwd) if row["session"] not in avoid]
|
|
521
|
+
where = "this workspace" if cwd == root else cwd
|
|
522
|
+
none_yet = f"no {ident or 'agent'} session for {where}" + (" but the one that failed" if avoid else " yet")
|
|
523
|
+
other = SESSION_ROLES[1 - SESSION_ROLES.index(role)]
|
|
524
|
+
oblock = cfg.get(other) if isinstance(cfg.get(other), dict) else {}
|
|
525
|
+
ocwd = _role_cwd(root, other, oblock)
|
|
526
|
+
orows = _role_rows(other, ident, ocwd) if ident and oblock and block_adapter(oblock) == ident else []
|
|
527
|
+
oids = {row["session"] for row in orows}
|
|
528
|
+
if not {row["session"] for row in rows} & oids:
|
|
529
|
+
if not rows:
|
|
530
|
+
return dict(out, why=none_yet)
|
|
531
|
+
return dict(out, session=rows[0]["session"], workspace_hash=rows[0]["workspace_hash"], how="discovered",
|
|
532
|
+
trusted=True, count=len(rows),
|
|
533
|
+
why=f"the newest of {len(rows)} {ident or 'agent'} session(s) for {where}")
|
|
534
|
+
records = session_records(root) if root else {}
|
|
535
|
+
mine, theirs = records.get(role) or {}, records.get(other) or {}
|
|
536
|
+
kept = mine.get("session") if (mine.get("adapter"), mine.get("cwd")) == (ident, cwd) else None
|
|
537
|
+
held = theirs.get("session") if (theirs.get("adapter"), theirs.get("cwd")) == (ident, ocwd) \
|
|
538
|
+
and _resumes_session(other, oblock) and theirs.get("session") in oids \
|
|
539
|
+
and theirs.get("session") != kept else None
|
|
540
|
+
opinned = _pinned_session(oblock) if _resumes_session(other, oblock) else None
|
|
541
|
+
# A pin settles the other role's session only while the store still holds it, as a record does: a pinned
|
|
542
|
+
# transcript the harness cleaned up left the one session in the store taken for this role's, which may
|
|
543
|
+
# be the other's new one (SESSION-IDENTITY-2).
|
|
544
|
+
opinned = opinned if opinned in oids else None
|
|
545
|
+
oresolved = oblock.get("_session") if isinstance(oblock.get("_session"), dict) else {}
|
|
546
|
+
if opinned or held:
|
|
547
|
+
claim, settled = opinned or held, True
|
|
548
|
+
elif role == "architect" and oresolved.get("how") in RESOLVED_HOW and _concrete_session(oblock.get("session")):
|
|
549
|
+
claim, settled = oblock["session"], bool(oresolved.get("trusted"))
|
|
550
|
+
else:
|
|
551
|
+
claim, settled = None, False
|
|
552
|
+
own = [row for row in rows if row["session"] != claim]
|
|
553
|
+
kept_row = next((row for row in own if row["session"] == kept), None)
|
|
554
|
+
if kept_row:
|
|
555
|
+
return dict(out, session=kept, workspace_hash=kept_row["workspace_hash"], how="recorded", trusted=True,
|
|
556
|
+
count=len(own), why="kept in .polyctl/sessions.json since polyctl settled it")
|
|
557
|
+
if not own:
|
|
558
|
+
return dict(out, why=f"the only {ident} session for {where} is the {other}'s" if rows else none_yet)
|
|
559
|
+
if role == "architect" and claim is None:
|
|
560
|
+
return dict(out, how="ambiguous", count=len(own),
|
|
561
|
+
why=f"the {other}'s sessions are kept beside the {role}'s in the {ident} store and the "
|
|
562
|
+
f"{other}'s is not settled, so polyctl cannot tell them apart; pin the {other}'s session "
|
|
563
|
+
f"in .polyctl/config.json")
|
|
564
|
+
if len(own) > 1:
|
|
565
|
+
return dict(out, how="ambiguous", count=len(own),
|
|
566
|
+
why=f"{len(own)} {ident} sessions for {where} that the {other} does not hold, and polyctl cannot "
|
|
567
|
+
f"tell which is the {role}'s; pin it in .polyctl/config.json")
|
|
568
|
+
found = dict(out, session=own[0]["session"], workspace_hash=own[0]["workspace_hash"], how="discovered", count=1)
|
|
569
|
+
if settled:
|
|
570
|
+
return dict(found, trusted=True, record=True,
|
|
571
|
+
why=f"the one {ident} session for {where} that the {other} does not hold")
|
|
572
|
+
return dict(found, why=f"the only {ident} session for {where}, where the {other}'s sessions are kept too and "
|
|
573
|
+
f"the {other}'s is not settled: read, but not resumed until one of the two is pinned "
|
|
574
|
+
f"in .polyctl/config.json")
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def resolve_sessions(root, cfg, record=True):
|
|
578
|
+
"""Resolve each role's `auto` into the block the role is read from, and record what settles it; the config.
|
|
579
|
+
|
|
580
|
+
The implementer is resolved before the architect. A block polyctl resolved keeps `_session`:
|
|
581
|
+
how, why, whether its session may be resumed, and how many sessions it chose among. A
|
|
582
|
+
role whose block names a session, or that holds none, is left as written, and nothing is
|
|
583
|
+
read for a config whose roles all name theirs.
|
|
584
|
+
"""
|
|
585
|
+
for role in SESSION_ROLES:
|
|
586
|
+
block = cfg.get(role)
|
|
587
|
+
if not _resumes_session(role, block) or _concrete_session(block.get("session")):
|
|
588
|
+
continue
|
|
589
|
+
found = resolve_session(root, cfg, role)
|
|
590
|
+
block["_session"] = {key: found[key] for key in ("how", "why", "trusted", "count")}
|
|
591
|
+
if found["session"]:
|
|
592
|
+
block["session"] = found["session"]
|
|
593
|
+
if found["workspace_hash"]:
|
|
594
|
+
block["workspace_hash"] = found["workspace_hash"]
|
|
595
|
+
if record and found["record"]:
|
|
596
|
+
record_session(root, role, found)
|
|
597
|
+
return cfg
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def session_state(cfg, role):
|
|
601
|
+
"""{session, how, why, trusted, count} of a role's session as the loaded config holds it; None for a role
|
|
602
|
+
that holds none. A block that did not come through load_config is resolved here."""
|
|
603
|
+
block = cfg.get(role)
|
|
604
|
+
if not _resumes_session(role, block):
|
|
605
|
+
return None
|
|
606
|
+
if isinstance(block.get("_session"), dict):
|
|
607
|
+
return dict(block["_session"], session=_concrete_session(block.get("session")))
|
|
608
|
+
found = resolve_session(cfg.get("root") or "", cfg, role)
|
|
609
|
+
return {key: found[key] for key in ("session", "how", "why", "trusted", "count")}
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def session_to_resume(cfg, role):
|
|
613
|
+
"""(the session a turn of this role may resume, None), or (None, why none may) (SESSION-IDENTITY).
|
|
614
|
+
|
|
615
|
+
Pinned, recorded, or discovered where nothing else could hold it; (None, None) for a role
|
|
616
|
+
that holds no session of its own.
|
|
617
|
+
"""
|
|
618
|
+
state = session_state(cfg, role)
|
|
619
|
+
if state is None:
|
|
620
|
+
return None, None
|
|
621
|
+
if state["session"] and state["trusted"]:
|
|
622
|
+
return state["session"], None
|
|
623
|
+
return None, state["why"] or f"the {role}'s session is {state['how']}"
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
# ── transcript ────────────────────────────────────────────────────────────────
|
|
627
|
+
|
|
628
|
+
def _open_regular(path):
|
|
629
|
+
"""A binary handle on `path` when, opened, it is a regular file reached through no final link; OSError otherwise.
|
|
630
|
+
|
|
631
|
+
A subagent's file is checked by its real path when the directory is listed and opened later: one swapped in
|
|
632
|
+
between for a link is not followed, and one swapped for a FIFO - or a sidecar that is one - is refused at
|
|
633
|
+
once, opened without blocking, where reading it hung the reader (SUBAGENT-BOUNDS-2).
|
|
634
|
+
|
|
635
|
+
Windows has no O_NOFOLLOW, and the flag that asked for it was 0 there: a final link was followed
|
|
636
|
+
as if the flag were not asked for. Where the platform has none, the path is looked at before it
|
|
637
|
+
is opened - a link is refused - and what was opened must be the file that was looked at, by its
|
|
638
|
+
file id, so a link swapped in between is refused too (WINDOWS-SAY-WHY).
|
|
639
|
+
"""
|
|
640
|
+
import stat as _stat
|
|
641
|
+
nofollow = getattr(os, "O_NOFOLLOW", None)
|
|
642
|
+
before = None
|
|
643
|
+
if nofollow is None:
|
|
644
|
+
before = os.lstat(path)
|
|
645
|
+
if _stat.S_ISLNK(before.st_mode):
|
|
646
|
+
raise OSError(f"{path} is a link")
|
|
647
|
+
flags = (nofollow or 0) | getattr(os, "O_NONBLOCK", 0)
|
|
648
|
+
handle = open(path, "rb", opener=lambda name, mode: os.open(name, mode | flags))
|
|
649
|
+
try:
|
|
650
|
+
opened = os.fstat(handle.fileno())
|
|
651
|
+
if not _stat.S_ISREG(opened.st_mode):
|
|
652
|
+
raise OSError(f"{path} is not a regular file")
|
|
653
|
+
if before is not None and (not before.st_ino or not opened.st_ino or before.st_ino != opened.st_ino
|
|
654
|
+
or before.st_dev != opened.st_dev):
|
|
655
|
+
# A file system that numbers no file - FAT - cannot tell a file swapped in from the one looked at: read as
|
|
656
|
+
# changed, not passed (the review of WINDOWS-SAY-WHY).
|
|
657
|
+
raise OSError(f"{path} changed as it was opened, or its file system cannot say it did not")
|
|
658
|
+
except BaseException:
|
|
659
|
+
handle.close()
|
|
660
|
+
raise
|
|
661
|
+
return handle
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def read_tail(path, nbytes=900_000, regular=False):
|
|
665
|
+
"""Last nbytes of a JSONL transcript, first (partial) line dropped; a database's rows when it keeps them there.
|
|
666
|
+
|
|
667
|
+
With `regular`, the file is read only if it is a regular file reached through no final link (`_open_regular`).
|
|
668
|
+
|
|
669
|
+
A store that holds every session of every project in one database has no file whose tail can
|
|
670
|
+
be read: its transcript is a query bound to one session id. Such a read asks a `DatabaseTail`
|
|
671
|
+
and gets the same kind of answer - a list of record objects - so every reader below sees one
|
|
672
|
+
transcript and names no store's schema (#76).
|
|
673
|
+
"""
|
|
674
|
+
if isinstance(path, DatabaseTail):
|
|
675
|
+
return path.read(nbytes)
|
|
676
|
+
recs = []
|
|
677
|
+
try:
|
|
678
|
+
with (_open_regular(path) if regular else open(path, "rb")) as f:
|
|
679
|
+
size = os.fstat(f.fileno()).st_size
|
|
680
|
+
f.seek(max(0, size - nbytes))
|
|
681
|
+
blob = f.read().decode("utf-8", "ignore")
|
|
682
|
+
except Exception:
|
|
683
|
+
return recs
|
|
684
|
+
lines = blob.split("\n")
|
|
685
|
+
if size > nbytes:
|
|
686
|
+
lines = lines[1:]
|
|
687
|
+
for line in lines:
|
|
688
|
+
line = line.strip()
|
|
689
|
+
if not line:
|
|
690
|
+
continue
|
|
691
|
+
try:
|
|
692
|
+
recs.append(json.loads(line))
|
|
693
|
+
except Exception:
|
|
694
|
+
pass
|
|
695
|
+
return recs
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
# ── a transcript kept as rows ─────────────────────────────────────────────────
|
|
699
|
+
#
|
|
700
|
+
# A JSONL store gives a transcript as a file, so `session_paths` can hand back a path and every
|
|
701
|
+
# reader can ask it for a tail. A database store gives no such thing: one file holds every
|
|
702
|
+
# session, and which records belong to one is a fact inside a column. What makes the two readable
|
|
703
|
+
# by the same readers is that the difference is only how a record is *fetched*: once a row's JSON
|
|
704
|
+
# column is promoted onto the record and its child rows nested beside it, a store that writes a
|
|
705
|
+
# message and its parts as rows of two tables holds exactly what a store that nests blocks in a
|
|
706
|
+
# record holds, and `transcript.messages.blocks` and `transcript.tool_call.blocks` read both.
|
|
707
|
+
#
|
|
708
|
+
# So the reader below is a fetcher and nothing else. It promotes, nests and converts; it renames
|
|
709
|
+
# nothing, because every name it uses - table, session column, order column, JSON column, child
|
|
710
|
+
# table and the key its records are held under - comes from `transcript.record`. A store whose
|
|
711
|
+
# column is called something else says so in its adapter, and a store that declares nothing is
|
|
712
|
+
# read as holding nothing.
|
|
713
|
+
|
|
714
|
+
_STORE_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
|
|
715
|
+
|
|
716
|
+
# How many rows one query of a store is asked for. A session's records are bounded by what an
|
|
717
|
+
# agent writes in a sitting, so this bounds only a store that has outgrown itself: a panel asks
|
|
718
|
+
# these questions every second, and reading a million rows to show eight messages is not a cheap
|
|
719
|
+
# answer but a slow one.
|
|
720
|
+
_STORE_ROW_CAP = 20_000
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _store_name(value):
|
|
724
|
+
"""A declared table or column name, when it is one word a query may hold.
|
|
725
|
+
|
|
726
|
+
These names are written into SQL rather than bound as values, because SQLite binds no name.
|
|
727
|
+
An adapter is package-controlled where authority is decided, but a project's own layer is
|
|
728
|
+
writable by the agent it describes, so a name is taken only in the shape one identifier has
|
|
729
|
+
and anything else reads as no declaration at all.
|
|
730
|
+
"""
|
|
731
|
+
text = _name(value)
|
|
732
|
+
return text if _STORE_NAME.match(text) else ""
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
def _read_query(text):
|
|
736
|
+
"""A declared SELECT, when it reads nothing but one statement.
|
|
737
|
+
|
|
738
|
+
A store's path and session are bound as parameters, never written into the text; the query
|
|
739
|
+
itself has to come from the adapter, since only it knows the schema. That makes it the one
|
|
740
|
+
piece of a declaration that is executed rather than read, so it is taken only when it starts
|
|
741
|
+
as a SELECT and holds no second statement.
|
|
742
|
+
"""
|
|
743
|
+
text = str(text or "").strip()
|
|
744
|
+
return text if text.upper().startswith("SELECT") and ";" not in text else ""
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def _epoch_iso(value, unit):
|
|
748
|
+
"""A number of seconds or milliseconds since the epoch, as the ISO text every time reader expects."""
|
|
749
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
750
|
+
return ""
|
|
751
|
+
millis = int(round(value * (1 if str(unit) in ("ms", "milliseconds") else 1000)))
|
|
752
|
+
return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(millis // 1000)) + ".%03dZ" % (millis % 1000)
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
class DatabaseTail:
|
|
756
|
+
"""A session's transcript where a database holds it: a path to the store, an id, and how to query it.
|
|
757
|
+
|
|
758
|
+
It stands for the database file to a reader that only wants to know whether the transcript is
|
|
759
|
+
there - `os.path.exists` and `os.path.getmtime` reach the file through `__fspath__` - while
|
|
760
|
+
carrying the session the records are read for, which a path alone cannot say.
|
|
761
|
+
"""
|
|
762
|
+
|
|
763
|
+
__slots__ = ("db", "session", "record", "freshness", "descendants")
|
|
764
|
+
|
|
765
|
+
def __init__(self, db, session, record, freshness=None, descendants=None):
|
|
766
|
+
self.db, self.session = _name(db), _name(session)
|
|
767
|
+
self.record = record if isinstance(record, dict) else {}
|
|
768
|
+
self.freshness = freshness if isinstance(freshness, dict) else {}
|
|
769
|
+
self.descendants = descendants if isinstance(descendants, dict) else {}
|
|
770
|
+
|
|
771
|
+
def __fspath__(self):
|
|
772
|
+
return self.db
|
|
773
|
+
|
|
774
|
+
def __str__(self):
|
|
775
|
+
return f"{self.db}#{self.session}"
|
|
776
|
+
|
|
777
|
+
def __repr__(self):
|
|
778
|
+
return f"<DatabaseTail {self.db}#{self.session}>"
|
|
779
|
+
|
|
780
|
+
def __eq__(self, other):
|
|
781
|
+
return isinstance(other, DatabaseTail) and (other.db, other.session) == (self.db, self.session)
|
|
782
|
+
|
|
783
|
+
def __hash__(self):
|
|
784
|
+
return hash((self.db, self.session))
|
|
785
|
+
|
|
786
|
+
def _connect(self):
|
|
787
|
+
"""A read-only connection, or None when the store cannot be opened.
|
|
788
|
+
|
|
789
|
+
Read-only because the agent owns this database and a second writer corrupts its state.
|
|
790
|
+
A short timeout because a panel and a watchdog ask this question repeatedly, and a store
|
|
791
|
+
mid-commit is a store that answers later, not one that answers never.
|
|
792
|
+
"""
|
|
793
|
+
try:
|
|
794
|
+
import sqlite3
|
|
795
|
+
except ImportError:
|
|
796
|
+
return None
|
|
797
|
+
try:
|
|
798
|
+
connection = sqlite3.connect(_read_only_uri(self.db), uri=True, timeout=2.0)
|
|
799
|
+
except Exception:
|
|
800
|
+
return None
|
|
801
|
+
connection.row_factory = sqlite3.Row
|
|
802
|
+
return connection
|
|
803
|
+
|
|
804
|
+
def _fetch(self, connection, sql, arguments, limit):
|
|
805
|
+
"""[dict] for each row a declared query reaches, its columns named, capped at `limit`."""
|
|
806
|
+
try:
|
|
807
|
+
cursor = connection.execute(sql, (*arguments, limit) if limit is not None else arguments)
|
|
808
|
+
return [dict(row) for row in cursor]
|
|
809
|
+
except Exception:
|
|
810
|
+
return []
|
|
811
|
+
|
|
812
|
+
def read(self, nbytes=900_000):
|
|
813
|
+
"""This session's records, oldest first: its newest rows, and no more of them than `nbytes` of stored text.
|
|
814
|
+
|
|
815
|
+
A record is its row's columns with one declared column's JSON promoted onto it, so a path
|
|
816
|
+
the adapter writes reaches either kind of field the same way. A JSON column that does not
|
|
817
|
+
parse promotes nothing, and the record still holds that column as the text it is.
|
|
818
|
+
"""
|
|
819
|
+
declared = self.record
|
|
820
|
+
table, session_key = _store_name(declared.get("table")), _store_name(declared.get("session_key"))
|
|
821
|
+
order = _store_name(declared.get("order")) or _store_name(declared.get("time"))
|
|
822
|
+
if not (self.session and table and session_key and order and os.path.exists(self.db)):
|
|
823
|
+
return []
|
|
824
|
+
connection = self._connect()
|
|
825
|
+
if connection is None:
|
|
826
|
+
return []
|
|
827
|
+
try:
|
|
828
|
+
rows = self._fetch(connection,
|
|
829
|
+
f"SELECT * FROM {table} WHERE {session_key} = ? ORDER BY {order} DESC LIMIT ?",
|
|
830
|
+
(self.session,), min(max(int(nbytes) // 256, 1), _STORE_ROW_CAP))
|
|
831
|
+
finally:
|
|
832
|
+
connection.close()
|
|
833
|
+
rows.reverse()
|
|
834
|
+
return self._nest(self._promote(rows, declared, nbytes))
|
|
835
|
+
|
|
836
|
+
def _promote(self, rows, declared, nbytes=None):
|
|
837
|
+
"""[record] for each row, oldest first: its columns with its declared JSON column promoted onto it.
|
|
838
|
+
|
|
839
|
+
A column that holds no object promotes nothing rather than dropping the row, and a number
|
|
840
|
+
in the declared time column becomes the ISO text every reader of `record.time` expects,
|
|
841
|
+
because a store that counts in milliseconds is the store's own fact to say. With no
|
|
842
|
+
`nbytes` every row is kept; a child table is read whole, since a record's blocks are
|
|
843
|
+
asked for complete and a tail of them is not a tail but a message with parts missing.
|
|
844
|
+
"""
|
|
845
|
+
body, time_column = _store_name(declared.get("body")), _store_name(declared.get("time"))
|
|
846
|
+
kept, spent = [], 0
|
|
847
|
+
for row in reversed(rows) if nbytes else rows:
|
|
848
|
+
record = dict(row)
|
|
849
|
+
blob = _name(row.get(body)) if body else ""
|
|
850
|
+
for key, value in (_json_object(blob) or {}).items():
|
|
851
|
+
record.setdefault(key, value)
|
|
852
|
+
stamp = _epoch_iso(row.get(time_column), declared.get("time_unit")) if time_column else ""
|
|
853
|
+
if stamp:
|
|
854
|
+
record[time_column] = stamp
|
|
855
|
+
kept.append(record)
|
|
856
|
+
if nbytes:
|
|
857
|
+
spent += len(blob) or len(json.dumps(record, default=str))
|
|
858
|
+
if spent >= nbytes:
|
|
859
|
+
break
|
|
860
|
+
return kept[::-1] if nbytes else kept # a budget walks the rows newest first; turn them back
|
|
861
|
+
|
|
862
|
+
def _nest(self, records):
|
|
863
|
+
"""Attach each record's child records under the key its adapter names, when it declares them.
|
|
864
|
+
|
|
865
|
+
A store that writes a message's parts beside it rather than inside it declares the child
|
|
866
|
+
table, the column of the child that names its parent, the column of the parent the child
|
|
867
|
+
names, and the key to hold them under. Children are read for the whole session in one
|
|
868
|
+
query and hung on by that name, because a store's rows are one session's either way, and
|
|
869
|
+
a child naming a parent outside this tail is dropped rather than guessed a home for.
|
|
870
|
+
"""
|
|
871
|
+
declared = _block(self.record, "children")
|
|
872
|
+
table, as_key = _store_name(declared.get("table")), _name(declared.get("as"))
|
|
873
|
+
key, parent_key = _store_name(declared.get("key")), _store_name(declared.get("parent_key"))
|
|
874
|
+
session_key = _store_name(declared.get("session_key")) or key
|
|
875
|
+
order = _store_name(declared.get("order")) or _store_name(declared.get("time"))
|
|
876
|
+
if not (table and as_key and key and parent_key and order and records):
|
|
877
|
+
return records
|
|
878
|
+
connection = self._connect()
|
|
879
|
+
if connection is None:
|
|
880
|
+
return records
|
|
881
|
+
try:
|
|
882
|
+
rows = self._fetch(connection,
|
|
883
|
+
f"SELECT * FROM {table} WHERE {session_key} = ? ORDER BY {order} ASC LIMIT ?",
|
|
884
|
+
(self.session,), _STORE_ROW_CAP)
|
|
885
|
+
finally:
|
|
886
|
+
connection.close()
|
|
887
|
+
grouped = {}
|
|
888
|
+
for child in self._promote(rows, declared):
|
|
889
|
+
grouped.setdefault(child.get(key), []).append(child)
|
|
890
|
+
return [dict(record, **{as_key: grouped.get(record.get(parent_key), [])}) for record in records]
|
|
891
|
+
|
|
892
|
+
def last_activity(self):
|
|
893
|
+
"""Epoch seconds the store says this session last wrote, or None when it declares no reading.
|
|
894
|
+
|
|
895
|
+
A database file is written by every session in it, so its modification time says a machine
|
|
896
|
+
is busy rather than that this agent is. A store that keeps a per-session clock can say
|
|
897
|
+
more, and `transcript.freshness` declares the query that reads it. A session whose rows
|
|
898
|
+
the store updates in place has no row written lately and still working, so this is the
|
|
899
|
+
only reading that sees it.
|
|
900
|
+
"""
|
|
901
|
+
sql = _read_query(self.freshness.get("query"))
|
|
902
|
+
if not sql or not os.path.exists(self.db):
|
|
903
|
+
return None
|
|
904
|
+
connection = self._connect()
|
|
905
|
+
if connection is None:
|
|
906
|
+
return None
|
|
907
|
+
try:
|
|
908
|
+
rows = self._fetch(connection, sql, (self.session,), None)
|
|
909
|
+
finally:
|
|
910
|
+
connection.close()
|
|
911
|
+
value = next(iter(rows[0].values()), None) if rows else None
|
|
912
|
+
if isinstance(value, str) and value:
|
|
913
|
+
try:
|
|
914
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
|
915
|
+
except ValueError:
|
|
916
|
+
return None
|
|
917
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
918
|
+
return None
|
|
919
|
+
return value / 1000.0 if _name(self.freshness.get("time_unit")) in ("ms", "milliseconds") else float(value)
|
|
920
|
+
|
|
921
|
+
def descendant_tails(self):
|
|
922
|
+
"""[DatabaseTail] for each session the store says descends from this one, nearest first.
|
|
923
|
+
|
|
924
|
+
A store that holds a subagent as a session of its own has no file beside the parent for polyctl
|
|
925
|
+
to list: the link is a value in a row, so the store declares the query that names one
|
|
926
|
+
session's children and which column of its rows holds a session id. The query is asked
|
|
927
|
+
again of every session it finds, so a subagent's own subagent is reached by the same one
|
|
928
|
+
declaration, and each id is asked of once - which is what stops a store whose parent links
|
|
929
|
+
closed a loop from being walked for as long as it is read. A tail that declares no query
|
|
930
|
+
has no descendants, and says so without opening the store.
|
|
931
|
+
"""
|
|
932
|
+
sql, column = _read_query(self.descendants.get("query")), _store_name(self.descendants.get("session"))
|
|
933
|
+
if not (sql and column and self.session and os.path.exists(self.db)):
|
|
934
|
+
return []
|
|
935
|
+
connection = self._connect()
|
|
936
|
+
if connection is None:
|
|
937
|
+
return []
|
|
938
|
+
found, seen, queued = [], {self.session}, [self.session]
|
|
939
|
+
try:
|
|
940
|
+
while queued:
|
|
941
|
+
for row in self._fetch(connection, sql, (queued.pop(0),), None):
|
|
942
|
+
child = _name(row.get(column))
|
|
943
|
+
if child and child not in seen:
|
|
944
|
+
seen.add(child)
|
|
945
|
+
queued.append(child)
|
|
946
|
+
found.append(DatabaseTail(self.db, child, self.record, self.freshness,
|
|
947
|
+
self.descendants))
|
|
948
|
+
finally:
|
|
949
|
+
connection.close()
|
|
950
|
+
return found
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def _json_object(text):
|
|
954
|
+
"""The object a stored JSON column holds, or None when it holds nothing readable."""
|
|
955
|
+
try:
|
|
956
|
+
document = json.loads(text)
|
|
957
|
+
except (TypeError, ValueError):
|
|
958
|
+
return None
|
|
959
|
+
return document if isinstance(document, dict) else None
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
# What a record looks like is its adapter's to declare (#76): where the kind and the time
|
|
963
|
+
# are, which kinds open, close and follow a turn, where a tool call keeps its name and
|
|
964
|
+
# arguments and which tools write a file, and where usage, context and failures are.
|
|
965
|
+
# The readers below ask these helpers and name no harness's field.
|
|
966
|
+
|
|
967
|
+
def _path_values(obj, path):
|
|
968
|
+
"""Every value a declared path reaches: `a.b` descends, `a[].b` reads `b` in each element of the list `a`.
|
|
969
|
+
|
|
970
|
+
A `[]` step yields one value per element, None where an element lacks the rest of
|
|
971
|
+
the path, so two paths through one list stay aligned element by element.
|
|
972
|
+
"""
|
|
973
|
+
values = [obj]
|
|
974
|
+
for step in str(path or "").split("."):
|
|
975
|
+
many = step.endswith("[]")
|
|
976
|
+
key = step[:-2] if many else step
|
|
977
|
+
found = []
|
|
978
|
+
for value in values:
|
|
979
|
+
value = value.get(key) if isinstance(value, dict) and key else None
|
|
980
|
+
if many:
|
|
981
|
+
found.extend(value if isinstance(value, list) else [])
|
|
982
|
+
else:
|
|
983
|
+
found.append(value)
|
|
984
|
+
values = found
|
|
985
|
+
return values
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def _path_value(obj, path):
|
|
989
|
+
"""The first value a declared path reaches, or None."""
|
|
990
|
+
values = _path_values(obj, path)
|
|
991
|
+
return values[0] if values else None
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def _name(value):
|
|
995
|
+
"""A declared name, kind or path: the string itself, or "" for anything else."""
|
|
996
|
+
return value if isinstance(value, str) else ""
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _names(value):
|
|
1000
|
+
"""A declared list of names, kinds or paths, as the strings in it; one may be written bare."""
|
|
1001
|
+
return [item for item in (value if isinstance(value, list) else [value]) if _name(item)]
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
def _block(document, key):
|
|
1005
|
+
"""The object a declaration keeps under `key`, or {} when it keeps none there."""
|
|
1006
|
+
value = document.get(key) if isinstance(document, dict) else None
|
|
1007
|
+
return value if isinstance(value, dict) else {}
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def transcript_shape(adapter):
|
|
1011
|
+
"""How an adapter's transcript records are read, from what it declares (#76).
|
|
1012
|
+
|
|
1013
|
+
`transcript.record.kind` is the path to a record's kind, and every other field is
|
|
1014
|
+
read from the object holding it: the payload when the kind is `payload.type`, the
|
|
1015
|
+
record itself when it is `type`. `record.time` is read from the record. A part the
|
|
1016
|
+
adapter does not declare comes back empty or None, and a reader then reads nothing
|
|
1017
|
+
for it rather than another harness's field. Usage carries the reading that adds it
|
|
1018
|
+
up (`billing.fallback.reading`, `sum` when none is declared); under a reading polyctl
|
|
1019
|
+
does not implement there is no usage to read, rather than usage misread.
|
|
1020
|
+
|
|
1021
|
+
`transcript.messages.not_words` lists the values a prompt or a reply holds when it is
|
|
1022
|
+
no one speaking - a harness's own note, a summary it wrote of a compacted
|
|
1023
|
+
conversation - each a `match` of paths and values; a record holding every value of
|
|
1024
|
+
one of them has no words. `transcript.messages.harness_replies` lists, the same way,
|
|
1025
|
+
the values a reply holds when the harness wrote it in the model's place
|
|
1026
|
+
(`in_place_reply`), and `telemetry.failure.ends` the records that tell a session a
|
|
1027
|
+
task it ran in the background ended (`ended_shape`). `telemetry.cost.tokens`, `model`
|
|
1028
|
+
and `priced_when` say how the tokens a usage record holds are priced (`token_shape`).
|
|
1029
|
+
"""
|
|
1030
|
+
transcript, signals = _block(adapter, "transcript"), _block(adapter, "telemetry")
|
|
1031
|
+
record, message_kinds, turn = (_block(transcript, key) for key in ("record", "messages", "turn"))
|
|
1032
|
+
body, _, kind = _name(record.get("kind")).rpartition(".")
|
|
1033
|
+
matches = {key: [match for match in message_kinds[key] if isinstance(match, dict) and match]
|
|
1034
|
+
if isinstance(message_kinds.get(key), list) else [] for key in ("not_words", "harness_replies")}
|
|
1035
|
+
shape = {"time": _name(record.get("time")), "body": body, "kind": kind,
|
|
1036
|
+
"text_keys": [key.rpartition(".")[2] for key in _names(record.get("text_keys"))],
|
|
1037
|
+
"prompt": _names(message_kinds.get("prompt")), "reply": _names(message_kinds.get("reply")),
|
|
1038
|
+
"not_words": matches["not_words"], "harness_replies": matches["harness_replies"],
|
|
1039
|
+
"start": _names(turn.get("start")), "end": _names(turn.get("end")),
|
|
1040
|
+
"bookkeeping": _names(turn.get("bookkeeping")),
|
|
1041
|
+
"tool": None, "usage": None, "context": None, "failure": None}
|
|
1042
|
+
tool = _block(transcript, "tool_call")
|
|
1043
|
+
if _name(tool.get("type")):
|
|
1044
|
+
shape["tool"] = {"type": tool["type"], "name": _name(tool.get("name")), "args": _name(tool.get("args")),
|
|
1045
|
+
"id": _name(tool.get("id")),
|
|
1046
|
+
"path_keys": _names(tool.get("path_keys")), "write_tools": _names(tool.get("write_tools")),
|
|
1047
|
+
"write_words": [word.lower() for word in _names(tool.get("write_words"))]}
|
|
1048
|
+
cost = _block(signals, "cost")
|
|
1049
|
+
shape["unit"] = _name(cost.get("unit")) or "unit"
|
|
1050
|
+
fields = _names(cost.get("fields")) or _names(cost.get("field"))
|
|
1051
|
+
reading = _block(_block(adapter, "billing"), "fallback").get("reading", "sum")
|
|
1052
|
+
if cost.get("from") == "transcript" and _name(cost.get("type")) and fields and reading in USAGE_READINGS:
|
|
1053
|
+
shape["usage"] = {"type": cost["type"], "fields": fields, "tools": _name(cost.get("tools")),
|
|
1054
|
+
"reading": reading}
|
|
1055
|
+
context = _block(signals, "context")
|
|
1056
|
+
if context.get("from") == "transcript" and _name(context.get("type")) and _name(context.get("field")):
|
|
1057
|
+
shape["context"] = {"type": context["type"], "match": _block(context, "match"), "field": context["field"]}
|
|
1058
|
+
failure = _block(signals, "failure")
|
|
1059
|
+
if failure.get("from") == "transcript" and _name(failure.get("type")) and _name(failure.get("field")) \
|
|
1060
|
+
and "failed_when" in failure:
|
|
1061
|
+
shape["failure"] = {"type": failure["type"], "field": failure["field"],
|
|
1062
|
+
"failed_when": failure["failed_when"], "text": _name(failure.get("text"))}
|
|
1063
|
+
shape["ends"] = ended_shape(_block(failure, "ends")) if failure.get("from") == "transcript" else None
|
|
1064
|
+
shape = nested_shape(shape, transcript, signals)
|
|
1065
|
+
shape["tokens"] = token_shape(cost, shape["usage"])
|
|
1066
|
+
return subagent_shape(shape, transcript)
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def ended_shape(declared):
|
|
1070
|
+
"""What the end of a task run in the background looks like, from `telemetry.failure.ends`; None if unusable.
|
|
1071
|
+
|
|
1072
|
+
A harness may tell a session that a task it started in the background ended - a command, a
|
|
1073
|
+
subagent - in a record of its own rather than in a tool's result, the call that started the
|
|
1074
|
+
task having returned when the task began. `records` are the records that tell it, each a kind,
|
|
1075
|
+
the `match` it holds and the `field` holding its text; the text says the end between pairs of
|
|
1076
|
+
markers: `status` the end's status, of which `failed_when` lists the ones that are a failure,
|
|
1077
|
+
`id` the task it ends, and `text` what a person reads of it. Without a status, a failing status
|
|
1078
|
+
or a record to read, nothing is read.
|
|
1079
|
+
"""
|
|
1080
|
+
records = declared.get("records") if isinstance(declared.get("records"), list) else []
|
|
1081
|
+
records = [{"type": record["type"], "match": _block(record, "match"), "field": record["field"]}
|
|
1082
|
+
for record in records if isinstance(record, dict) and _name(record.get("type"))
|
|
1083
|
+
and _name(record.get("field"))]
|
|
1084
|
+
markers = {key: list(declared[key]) for key in ("status", "id", "text") if isinstance(declared.get(key), list)
|
|
1085
|
+
and len(declared[key]) == 2 and all(_name(marker) for marker in declared[key])}
|
|
1086
|
+
failed = _names(declared.get("failed_when"))
|
|
1087
|
+
return dict(markers, records=records, failed_when=failed) if records and failed and "status" in markers else None
|
|
1088
|
+
|
|
1089
|
+
|
|
1090
|
+
def nested_shape(shape, transcript, signals):
|
|
1091
|
+
"""A transcript shape with what a store that nests its records declares beside it.
|
|
1092
|
+
|
|
1093
|
+
One harness writes each tool call, tool result and turn end as a record of its own.
|
|
1094
|
+
Another holds calls and results as blocks of a message, ends a turn in a field of
|
|
1095
|
+
the response that ends it, and writes one response as several records that repeat
|
|
1096
|
+
its usage. That store declares `blocks`, the path to the blocks a record holds, and
|
|
1097
|
+
the `match` a block holds, in `transcript.tool_call` and `telemetry.failure`, and in
|
|
1098
|
+
`transcript.messages` for the blocks that are a message's words; `turn.end_when`,
|
|
1099
|
+
the kind, field and values that close a turn; `turn.conversation`, the kinds a turn
|
|
1100
|
+
is made of, every other kind being bookkeeping; and, for the `per-response` reading,
|
|
1101
|
+
`telemetry.cost.response`, the path to a response's id. Usage read per response
|
|
1102
|
+
without that path is not read, rather than counted once per record. A shape that
|
|
1103
|
+
declares none of these reads as it did.
|
|
1104
|
+
"""
|
|
1105
|
+
turn, words = _block(transcript, "turn"), _block(transcript, "messages")
|
|
1106
|
+
conditions = turn.get("end_when") if isinstance(turn.get("end_when"), list) else [turn.get("end_when")]
|
|
1107
|
+
shape["end_when"] = [{"type": when["type"], "field": when["field"], "values": when["values"]}
|
|
1108
|
+
for when in conditions if isinstance(when, dict) and _name(when.get("type"))
|
|
1109
|
+
and _name(when.get("field")) and isinstance(when.get("values"), list)]
|
|
1110
|
+
shape["conversation"] = _names(turn.get("conversation"))
|
|
1111
|
+
shape["words"] = {"blocks": words["blocks"], "match": _block(words, "match")} if _name(words.get("blocks")) \
|
|
1112
|
+
else None
|
|
1113
|
+
for part, declared in (("tool", _block(transcript, "tool_call")), ("failure", _block(signals, "failure"))):
|
|
1114
|
+
if shape[part] is not None:
|
|
1115
|
+
shape[part].update(blocks=_name(declared.get("blocks")), match=_block(declared, "match"))
|
|
1116
|
+
if shape["usage"] is not None and shape["usage"]["reading"] == "per-response":
|
|
1117
|
+
response = _name(_block(signals, "cost").get("response"))
|
|
1118
|
+
shape["usage"] = dict(shape["usage"], response=response) if response else None
|
|
1119
|
+
return shape
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def token_shape(cost, usage):
|
|
1123
|
+
"""How a usage record's tokens are priced, from what `telemetry.cost` declares: {model, kinds, when}, or None.
|
|
1124
|
+
|
|
1125
|
+
A vendor prices a token by its model and by its kind - read, written to a cache, read from
|
|
1126
|
+
one, written out - and one count of every kind added up cannot be priced (USD-COST).
|
|
1127
|
+
`tokens` maps each kind of token a price table names to the path of its count in a usage
|
|
1128
|
+
record, or to a list of paths of which the first holding a count is read: a store that
|
|
1129
|
+
broke a count down in a later release declares the breakdown first and the whole after it.
|
|
1130
|
+
`model` is the path to the model that answered. `priced_when` lists the fields whose value
|
|
1131
|
+
sets the rate a vendor charged, each `{field, values}`; a record is at a table's prices
|
|
1132
|
+
when every field holds one of its values, null standing for a field the record does not
|
|
1133
|
+
hold. Tokens are read with usage and under its reading, so a shape with no usage to read
|
|
1134
|
+
has no tokens either, and a declaration polyctl cannot read whole prices nothing rather than
|
|
1135
|
+
part of it, or every rate alike (`token_problems` says why).
|
|
1136
|
+
"""
|
|
1137
|
+
if usage is None or "tokens" not in cost or token_problems(cost):
|
|
1138
|
+
return None
|
|
1139
|
+
return {"model": cost["model"], "kinds": {kind: _names(paths) for kind, paths in cost["tokens"].items()},
|
|
1140
|
+
"when": [{"field": item["field"], "values": item["values"]} for item in cost.get("priced_when", [])]}
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
def subagent_shape(shape, transcript):
|
|
1144
|
+
"""A transcript shape with the subagent transcripts a session's records start, when its adapter declares them.
|
|
1145
|
+
|
|
1146
|
+
A harness that delegates to subagents may write each one's records to a transcript of
|
|
1147
|
+
its own beside the session's, and no reading of the session's records then holds that
|
|
1148
|
+
spend or those writes. `transcript.subagents` declares them: `dir`, the directory of a
|
|
1149
|
+
session's subagent transcripts relative to the session transcript's own, `{session}`
|
|
1150
|
+
standing for the session transcript's name without its extension; `transcripts`, each
|
|
1151
|
+
a `path` under that directory, `*` standing for any name and `{id}` for the value a
|
|
1152
|
+
record of the session holds at `named_by` when it starts that transcript; and
|
|
1153
|
+
`sidecar`, a file beside a subagent's transcript (`{name}` its name without its
|
|
1154
|
+
extension) whose `call` field holds the id of the tool call that started it, the id
|
|
1155
|
+
of a call being at `transcript.tool_call.id`. A shape declaring no `dir` or no
|
|
1156
|
+
transcript under it has no subagents.
|
|
1157
|
+
|
|
1158
|
+
Each of these paths is read below the directory it is relative to, and one that is not
|
|
1159
|
+
plain names (`_plain_steps`) - `..`, an absolute path, a drive - is refused where it is
|
|
1160
|
+
declared: a `dir` gives the shape no subagents, a transcript or a sidecar is left out. A
|
|
1161
|
+
`dir` names its session, `{session}` standing in it, so no session reads another's.
|
|
1162
|
+
"""
|
|
1163
|
+
declared = _block(transcript, "subagents")
|
|
1164
|
+
entries = declared.get("transcripts") if isinstance(declared.get("transcripts"), list) else []
|
|
1165
|
+
entries = [{"path": entry["path"], "named_by": _name(entry.get("named_by"))} for entry in entries
|
|
1166
|
+
if isinstance(entry, dict) and _name(entry.get("path")) and entry["path"].count("{id}") <= 1
|
|
1167
|
+
and _plain_steps(entry["path"])]
|
|
1168
|
+
sidecar = _block(declared, "sidecar")
|
|
1169
|
+
shape["subagents"] = {"dir": declared["dir"], "transcripts": entries,
|
|
1170
|
+
"sidecar": {"path": sidecar["path"], "call": sidecar["call"]}
|
|
1171
|
+
if _name(sidecar.get("path")) and _name(sidecar.get("call")) and _plain_steps(sidecar["path"])
|
|
1172
|
+
else None} \
|
|
1173
|
+
if "{session}" in _name(declared.get("dir")) and _plain_steps(declared["dir"]) and entries else None
|
|
1174
|
+
return shape
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
def subagent_problems(adapter):
|
|
1178
|
+
"""Each path `transcript.subagents` declares that polyctl will not read, and why; [] when it reads them all (#77).
|
|
1179
|
+
|
|
1180
|
+
`subagent_shape` refuses such a path without a word, since a reading must not fail on a
|
|
1181
|
+
layer's mistake; `polyctl adapters validate` says why before anyone relies on the declaration.
|
|
1182
|
+
"""
|
|
1183
|
+
declared = _block(_block(adapter, "transcript"), "subagents")
|
|
1184
|
+
if not declared:
|
|
1185
|
+
return []
|
|
1186
|
+
plain = "names joined by `/`, with no empty step, `.`, `..`, absolute path or drive"
|
|
1187
|
+
problems = [] if "{session}" in _name(declared.get("dir")) and _plain_steps(declared["dir"]) else [
|
|
1188
|
+
f"`transcript.subagents.dir` must be {plain}, and hold {{session}}"]
|
|
1189
|
+
entries = declared.get("transcripts") if isinstance(declared.get("transcripts"), list) else []
|
|
1190
|
+
problems += [f"`transcript.subagents.transcripts` path {entry.get('path')!r} must be {plain}"
|
|
1191
|
+
for entry in entries if isinstance(entry, dict) and not _plain_steps(_name(entry.get("path")))]
|
|
1192
|
+
sidecar = _block(declared, "sidecar")
|
|
1193
|
+
if sidecar and not _plain_steps(_name(sidecar.get("path"))):
|
|
1194
|
+
problems.append(f"`transcript.subagents.sidecar.path` must be {plain}")
|
|
1195
|
+
return problems
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
def token_problems(cost):
|
|
1199
|
+
"""What of a `telemetry.cost` block's pricing declaration polyctl will not read, and why; [] when it reads it all.
|
|
1200
|
+
|
|
1201
|
+
`token_shape` prices nothing from a declaration it cannot read whole, without a word, since a
|
|
1202
|
+
reading must not fail on a layer's mistake; `polyctl adapters validate` says why before anyone
|
|
1203
|
+
relies on the estimate (USD-COST).
|
|
1204
|
+
"""
|
|
1205
|
+
if not any(key in cost for key in ("tokens", "model", "priced_when")):
|
|
1206
|
+
return []
|
|
1207
|
+
problems = [] if cost.get("from") == "transcript" else [
|
|
1208
|
+
"`telemetry.cost.from` must be transcript: tokens are read from the transcript's usage records"]
|
|
1209
|
+
declared = cost.get("tokens")
|
|
1210
|
+
if not isinstance(declared, dict) or not declared or not all(
|
|
1211
|
+
_names(paths) and len(_names(paths)) == (len(paths) if isinstance(paths, list) else 1)
|
|
1212
|
+
for paths in declared.values()):
|
|
1213
|
+
problems.append("`telemetry.cost.tokens` must map each kind of token to a path, or to a list of paths")
|
|
1214
|
+
elif not all(isinstance(kind, str) and re.fullmatch(r"[a-z][a-z0-9_]{0,31}", kind) for kind in declared):
|
|
1215
|
+
# A kind's name is printed as it is declared, and a project's own adapter is written by the agents
|
|
1216
|
+
# it describes: a name that is not a plain word could write a line of the estimate's own.
|
|
1217
|
+
problems.append("`telemetry.cost.tokens` names each kind in lowercase letters, digits and `_`")
|
|
1218
|
+
if not _name(cost.get("model")):
|
|
1219
|
+
problems.append("`telemetry.cost.model` must be the path to the model that answered; without it no token "
|
|
1220
|
+
"is priced")
|
|
1221
|
+
when = cost.get("priced_when", [])
|
|
1222
|
+
if not isinstance(when, list) or not all(isinstance(item, dict) and _name(item.get("field"))
|
|
1223
|
+
and isinstance(item.get("values"), list) for item in when):
|
|
1224
|
+
problems.append("`telemetry.cost.priced_when` must be a list of {field, values}, with values a list")
|
|
1225
|
+
return problems
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def _plain_steps(path):
|
|
1229
|
+
"""A declared path's steps when each names an entry below the directory it is read in, else None.
|
|
1230
|
+
|
|
1231
|
+
A declared path is names joined by `/`. An empty step (a path starting or ending with `/`, or
|
|
1232
|
+
holding `//`), `.`, `..`, and a step holding this platform's separator or a drive name no entry
|
|
1233
|
+
below: they are how a path leaves the directory it is read in, so it is refused, never
|
|
1234
|
+
followed.
|
|
1235
|
+
"""
|
|
1236
|
+
steps = str(path).split("/")
|
|
1237
|
+
if any(step in ("", ".", "..") or os.sep in step or (os.altsep and os.altsep in step)
|
|
1238
|
+
or os.path.splitdrive(step)[0] for step in steps):
|
|
1239
|
+
return None
|
|
1240
|
+
return steps
|
|
1241
|
+
|
|
1242
|
+
|
|
1243
|
+
def _inside(path, directory):
|
|
1244
|
+
"""Is a real path inside a real directory, and not the directory itself, as this platform compares names?"""
|
|
1245
|
+
path, directory = os.path.normcase(path), os.path.normcase(directory)
|
|
1246
|
+
return path.startswith(directory.rstrip(os.sep) + os.sep)
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
def _declared_files(directory, steps):
|
|
1250
|
+
"""[(real path, what `{id}` stood for or None)] for each file a declared path's steps reach below a real directory.
|
|
1251
|
+
|
|
1252
|
+
A step names entries of the directory reached so far, `*` standing for any run of characters
|
|
1253
|
+
and `{id}` for one or more, and a name starting with a dot is reached only by a step starting
|
|
1254
|
+
with one, as a shell's glob reads them; a step with neither is the one name it spells. Every
|
|
1255
|
+
step but the last reaches directories, and the last reaches files. An entry that is a symbolic
|
|
1256
|
+
link - or on Windows any directory, since a junction does not say it is a link before Python
|
|
1257
|
+
3.12 - is followed only when its real path is inside `directory`: a path is read inside the
|
|
1258
|
+
directory it is declared in, or not at all.
|
|
1259
|
+
"""
|
|
1260
|
+
reached = [(directory, None)]
|
|
1261
|
+
for depth, step in enumerate(steps):
|
|
1262
|
+
last = depth == len(steps) - 1
|
|
1263
|
+
pattern = re.compile(re.escape(step).replace(re.escape("{id}"), "(?P<id>[^/]+)")
|
|
1264
|
+
.replace(re.escape("*"), "[^/]*"))
|
|
1265
|
+
wild = pattern.pattern != re.escape(step)
|
|
1266
|
+
following = []
|
|
1267
|
+
for parent, ident in reached:
|
|
1268
|
+
try:
|
|
1269
|
+
if wild:
|
|
1270
|
+
with os.scandir(parent) as listing:
|
|
1271
|
+
entries = [(entry.name, entry) for entry in listing]
|
|
1272
|
+
else:
|
|
1273
|
+
entries = [(step, None)]
|
|
1274
|
+
except OSError:
|
|
1275
|
+
continue
|
|
1276
|
+
for name, entry in entries:
|
|
1277
|
+
match = pattern.fullmatch(name)
|
|
1278
|
+
if match is None or (wild and name.startswith(".") and not step.startswith(".")):
|
|
1279
|
+
continue
|
|
1280
|
+
path = os.path.join(parent, name)
|
|
1281
|
+
try:
|
|
1282
|
+
if entry is None or entry.is_symlink() or (os.name == "nt" and not last):
|
|
1283
|
+
path = os.path.realpath(path)
|
|
1284
|
+
if not _inside(path, directory) or not (os.path.isfile(path) if last else os.path.isdir(path)):
|
|
1285
|
+
continue
|
|
1286
|
+
elif not (entry.is_file() if last else entry.is_dir()):
|
|
1287
|
+
continue
|
|
1288
|
+
except OSError:
|
|
1289
|
+
continue
|
|
1290
|
+
following.append((path, match.groupdict().get("id") or ident))
|
|
1291
|
+
reached = following
|
|
1292
|
+
return sorted(reached)
|
|
1293
|
+
|
|
1294
|
+
|
|
1295
|
+
def record_body(rec, shape):
|
|
1296
|
+
"""The object a record's kind and fields are read from, or None when the record has none."""
|
|
1297
|
+
if not shape["kind"] or not isinstance(rec, dict):
|
|
1298
|
+
return None
|
|
1299
|
+
body = _path_value(rec, shape["body"]) if shape["body"] else rec
|
|
1300
|
+
return body if isinstance(body, dict) else None
|
|
1301
|
+
|
|
1302
|
+
|
|
1303
|
+
def record_kind(rec, shape):
|
|
1304
|
+
"""A record's kind where its adapter says it is, or None."""
|
|
1305
|
+
body = record_body(rec, shape)
|
|
1306
|
+
kind = body.get(shape["kind"]) if body is not None else None
|
|
1307
|
+
return kind if isinstance(kind, str) else None
|
|
1308
|
+
|
|
1309
|
+
|
|
1310
|
+
def record_time(rec, shape):
|
|
1311
|
+
"""A record's ISO timestamp where its adapter says it is, or ""."""
|
|
1312
|
+
value = _path_value(rec, shape["time"]) if shape["time"] and isinstance(rec, dict) else None
|
|
1313
|
+
return value if isinstance(value, str) else ""
|
|
1314
|
+
|
|
1315
|
+
|
|
1316
|
+
def usage_entries(body, usage):
|
|
1317
|
+
"""[(values, tools)]: one entry per element a `[]` usage field reaches, else one for the record.
|
|
1318
|
+
|
|
1319
|
+
`values` holds the entry's value at each declared field, in order, and `tools` the
|
|
1320
|
+
value of `tools` beside it; each reader adds the values up as it always has.
|
|
1321
|
+
"""
|
|
1322
|
+
columns = [_path_values(body, field) for field in usage["fields"]]
|
|
1323
|
+
tools = _path_values(body, usage["tools"]) if usage["tools"] else []
|
|
1324
|
+
return [([column[i] if i < len(column) else None for column in columns], tools[i] if i < len(tools) else None)
|
|
1325
|
+
for i in range(max(len(column) for column in columns))]
|
|
1326
|
+
|
|
1327
|
+
|
|
1328
|
+
# How usage records add up to spend is the adapter's to declare too (`billing.fallback.reading`):
|
|
1329
|
+
# `sum` when every record is the whole usage of the turn it reports, `peak-per-turn` when a
|
|
1330
|
+
# record is the running total of the turn in progress, `per-response` when a response is
|
|
1331
|
+
# written as several records that each repeat its usage. The panel, `polyctl cost` and the credit
|
|
1332
|
+
# estimate apply the same reading over the same turns, so no two of them tell a different spend.
|
|
1333
|
+
USAGE_READINGS = ("sum", "peak-per-turn", "per-response")
|
|
1334
|
+
|
|
1335
|
+
|
|
1336
|
+
def record_usage(body, usage):
|
|
1337
|
+
"""The usage one record reports: the numbers at every declared field of every entry, added up."""
|
|
1338
|
+
return sum(value for values, _ in usage_entries(body, usage) for value in values
|
|
1339
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool))
|
|
1340
|
+
|
|
1341
|
+
|
|
1342
|
+
def add_usage(turn, body, usage, counted, subagent=None):
|
|
1343
|
+
"""Charge a usage record to its turn under the declared reading, and return how much the turn's usage grew.
|
|
1344
|
+
|
|
1345
|
+
Under `sum` the record adds on; under `peak-per-turn` the turn costs the highest total its
|
|
1346
|
+
records reached; under `per-response` a response costs the highest total its records reached,
|
|
1347
|
+
once in this reading. `counted` holds what a reading has charged each response, one dict for
|
|
1348
|
+
everything it adds up: every turn of a transcript, every subagent transcript read with it,
|
|
1349
|
+
and every transcript of the credit estimate. What grew is what a reader adds to a total, or
|
|
1350
|
+
to the day the record was written, so a total is always the sum of its turns.
|
|
1351
|
+
|
|
1352
|
+
A subagent's record (`subagent`, the reading's state of its transcript, from
|
|
1353
|
+
`with_subagents`) is read under the reading in the subagent's own turn, and what grew there
|
|
1354
|
+
is charged to `turn`, the session's turn that started the subagent, as delegated too.
|
|
1355
|
+
"""
|
|
1356
|
+
if subagent is not None:
|
|
1357
|
+
subagent["turn"] = {} if subagent["turn"] is None else subagent["turn"]
|
|
1358
|
+
grown = add_usage(subagent["turn"], body, usage, counted)
|
|
1359
|
+
turn["usage"] = (turn.get("usage") or 0.0) + grown
|
|
1360
|
+
turn["delegated"] = (turn.get("delegated") or 0.0) + grown
|
|
1361
|
+
return grown
|
|
1362
|
+
value, before = record_usage(body, usage), turn.get("usage") or 0.0
|
|
1363
|
+
if usage["reading"] == "per-response":
|
|
1364
|
+
value = _response_growth(counted, body, usage, value)
|
|
1365
|
+
if usage["reading"] in ("sum", "per-response"):
|
|
1366
|
+
turn["usage"] = before + value
|
|
1367
|
+
return value
|
|
1368
|
+
turn["usage"] = max(before, value)
|
|
1369
|
+
return turn["usage"] - before
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
def _response_growth(counted, body, usage, value):
|
|
1373
|
+
"""What a record adds to its response, by the id at `response`: what it holds beyond the response's charge so far.
|
|
1374
|
+
|
|
1375
|
+
A session's store repeats a response's whole usage in every record of it, so only the
|
|
1376
|
+
first record the reading meets is charged, whichever record a tail begins at. A subagent's
|
|
1377
|
+
transcript writes a response's records as it streams them, each with the output tokens so
|
|
1378
|
+
far, so the response costs the most its records reached. A store can also write a response
|
|
1379
|
+
again later - the same records, turns after the first, or a session's records copied into
|
|
1380
|
+
another session's transcript - and a copy is not a second spend, so a response is charged
|
|
1381
|
+
once per reading, not once per turn. A record that names no response is its own.
|
|
1382
|
+
"""
|
|
1383
|
+
ident = _path_value(body, usage["response"])
|
|
1384
|
+
if not isinstance(ident, (str, int)) or isinstance(ident, bool):
|
|
1385
|
+
return value
|
|
1386
|
+
charged = counted.get(ident)
|
|
1387
|
+
counted[ident] = value if charged is None else max(charged, value)
|
|
1388
|
+
return value if charged is None else max(0.0, value - charged)
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def add_tokens(turn, body, usage, priced, counted, subagent=None):
|
|
1392
|
+
"""Charge a usage record's tokens of each priced kind to its turn, by model and rate (USD-COST).
|
|
1393
|
+
|
|
1394
|
+
They are read under the reading `add_usage` applies to the same record, one kind at a time:
|
|
1395
|
+
under `sum` the record adds on; under `per-response` a response's tokens of a kind count once
|
|
1396
|
+
in a reading, at the most its records reached (`counted`, by response and kind); under
|
|
1397
|
+
`peak-per-turn` a turn's tokens of a kind are the most its records reached. A subagent's
|
|
1398
|
+
record is read in the subagent's own turn and charged to `turn`, the session's turn that
|
|
1399
|
+
started it, as `add_usage` charges its usage.
|
|
1400
|
+
|
|
1401
|
+
`turn["tokens"]` maps (model, rate, kind) to a count. The model is None when the record names
|
|
1402
|
+
none, and its JSON string when it holds a character that is not printable: polyctl prints it on a
|
|
1403
|
+
line of its own, and a line break or an escape in it stays visible there, as in a review
|
|
1404
|
+
header (`review_header_value`). The rate is None when every `priced_when` field holds a value
|
|
1405
|
+
it lists, and otherwise names the first field that does not, by the last step of its path,
|
|
1406
|
+
and what it holds - such as `speed "fast"` - so a response at a rate no table lists is never
|
|
1407
|
+
priced at the rate one does.
|
|
1408
|
+
"""
|
|
1409
|
+
counts = {kind: _first_count(body, paths) for kind, paths in priced["kinds"].items()}
|
|
1410
|
+
if not any(counts.values()):
|
|
1411
|
+
return
|
|
1412
|
+
model = _path_value(body, priced["model"])
|
|
1413
|
+
model = review_header_value(model) if isinstance(model, str) and model.strip() else None
|
|
1414
|
+
rate = None
|
|
1415
|
+
for when in priced["when"]:
|
|
1416
|
+
value = _path_value(body, when["field"])
|
|
1417
|
+
if not any(_declared_value(value, listed) for listed in when["values"]):
|
|
1418
|
+
rate = f"{when['field'].rpartition('.')[2]} {json.dumps(value, default=str)}"
|
|
1419
|
+
break
|
|
1420
|
+
ident = _path_value(body, usage["response"]) if usage["reading"] == "per-response" else None
|
|
1421
|
+
peaks = None
|
|
1422
|
+
if usage["reading"] == "peak-per-turn":
|
|
1423
|
+
# A subagent's record reaches its peak in the subagent's own turn, as `add_usage` reads its usage.
|
|
1424
|
+
if subagent is not None and subagent["turn"] is None:
|
|
1425
|
+
subagent["turn"] = {}
|
|
1426
|
+
peaks = (turn if subagent is None else subagent["turn"]).setdefault("token_peaks", {})
|
|
1427
|
+
tokens = turn.setdefault("tokens", {})
|
|
1428
|
+
for kind, count in counts.items():
|
|
1429
|
+
# Keyed by model and rate as well as kind, so a turn whose records name two models keeps both.
|
|
1430
|
+
grown = _token_growth(count, usage["reading"], peaks, counted, (model, rate, kind), ident)
|
|
1431
|
+
if grown:
|
|
1432
|
+
tokens[(model, rate, kind)] = tokens.get((model, rate, kind), 0) + grown
|
|
1433
|
+
|
|
1434
|
+
|
|
1435
|
+
def _token_growth(count, reading, peaks, counted, kind, ident):
|
|
1436
|
+
"""What a record's count of one kind adds to its turn under a reading, as `add_usage` adds a record's usage.
|
|
1437
|
+
|
|
1438
|
+
`peaks` holds the most each kind reached in the turn read under `peak-per-turn`, and
|
|
1439
|
+
`counted` what each response was charged of each kind under `per-response`.
|
|
1440
|
+
"""
|
|
1441
|
+
if reading == "per-response" and isinstance(ident, (str, int)) and not isinstance(ident, bool):
|
|
1442
|
+
charged = counted.get((ident, kind))
|
|
1443
|
+
counted[(ident, kind)] = count if charged is None else max(charged, count)
|
|
1444
|
+
return count if charged is None else max(0, count - charged)
|
|
1445
|
+
if reading == "peak-per-turn":
|
|
1446
|
+
before = peaks.get(kind, 0)
|
|
1447
|
+
peaks[kind] = max(before, count)
|
|
1448
|
+
return peaks[kind] - before
|
|
1449
|
+
return count
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _first_count(body, paths):
|
|
1453
|
+
"""The count at the first declared path that reaches one - a finite number of zero or more - else 0.
|
|
1454
|
+
|
|
1455
|
+
A path whose `[]` step reaches several entries counts them all, added up, as `record_usage`
|
|
1456
|
+
adds up every entry of a usage field. A later path is read where an earlier one reaches no count,
|
|
1457
|
+
unless the earlier one reads in an object the later ones do not - a breakdown beside a total -
|
|
1458
|
+
and that object is in the record: the breakdown then says the kind is none, and a total is not
|
|
1459
|
+
counted again as this kind while the breakdown names another (USD-COST). Paths in one object are
|
|
1460
|
+
alternatives, and each is tried.
|
|
1461
|
+
"""
|
|
1462
|
+
for index, path in enumerate(paths):
|
|
1463
|
+
counts = [value for value in _path_values(body, path) if isinstance(value, (int, float))
|
|
1464
|
+
and not isinstance(value, bool) and 0 <= value < float("inf")]
|
|
1465
|
+
if counts:
|
|
1466
|
+
return sum(counts)
|
|
1467
|
+
parent, later = str(path).rpartition(".")[0], paths[index + 1:]
|
|
1468
|
+
if parent and later and not all(str(other).startswith(parent + ".") for other in later) \
|
|
1469
|
+
and any(isinstance(value, dict) for value in _path_values(body, parent)):
|
|
1470
|
+
return 0
|
|
1471
|
+
return 0
|
|
1472
|
+
|
|
1473
|
+
|
|
1474
|
+
def next_turn(turn, rec, shape, subagent=None):
|
|
1475
|
+
"""(the turn a record falls in, whether the record opened it); None until a turn opens.
|
|
1476
|
+
|
|
1477
|
+
A start marker opens a turn, and a prompt opens one when none is open or the open one
|
|
1478
|
+
ended. A harness may write its prompt before the start marker of the same turn, so a
|
|
1479
|
+
start marker opens no second turn while the open one holds only its prompt: opened by a
|
|
1480
|
+
prompt, not started and not ended. A call the harness makes of its own between the two
|
|
1481
|
+
does not split them. An end marker ends the turn it falls in, and so does a record whose
|
|
1482
|
+
field holds a value the adapter declares to end one (`closes_turn`).
|
|
1483
|
+
|
|
1484
|
+
A subagent's record (`subagent`, from `with_subagents`) falls in the session's turn that
|
|
1485
|
+
started the subagent: its prompts and ends move only the subagent's own turn.
|
|
1486
|
+
"""
|
|
1487
|
+
if subagent is not None:
|
|
1488
|
+
subagent["turn"], _ = next_turn(subagent["turn"], rec, shape)
|
|
1489
|
+
return turn, False
|
|
1490
|
+
kind = record_kind(rec, shape)
|
|
1491
|
+
waiting = turn is not None and turn.get("prompted") and not turn.get("started") and not turn.get("closed")
|
|
1492
|
+
ended = turn is None or turn.get("closed")
|
|
1493
|
+
opened = (kind in shape["start"] and not waiting) or (kind in shape["prompt"] and ended)
|
|
1494
|
+
if opened:
|
|
1495
|
+
turn = {"prompted": kind in shape["prompt"]}
|
|
1496
|
+
if turn is not None:
|
|
1497
|
+
turn["started"] = bool(turn.get("started")) or kind in shape["start"]
|
|
1498
|
+
turn["closed"] = bool(turn.get("closed")) or closes_turn(rec, shape)
|
|
1499
|
+
return turn, bool(opened)
|
|
1500
|
+
|
|
1501
|
+
|
|
1502
|
+
def tool_writes_file(name, tool):
|
|
1503
|
+
"""Does a tool of this name write a file: named in `write_tools`, or holding one of `write_words` in any case?"""
|
|
1504
|
+
return name in tool["write_tools"] or any(word in name.lower() for word in tool["write_words"])
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
def tool_path(args, tool):
|
|
1508
|
+
"""The file a tool call's arguments name, under the first of the adapter's `path_keys` that is set, or None."""
|
|
1509
|
+
if not isinstance(args, dict):
|
|
1510
|
+
return None
|
|
1511
|
+
return next((args[key] for key in tool["path_keys"] if args.get(key)), None)
|
|
1512
|
+
|
|
1513
|
+
|
|
1514
|
+
def _declared_value(value, expected):
|
|
1515
|
+
"""A field holds the declared value: by identity for true, false and null, by equality otherwise."""
|
|
1516
|
+
return value is expected if expected is None or isinstance(expected, bool) else value == expected
|
|
1517
|
+
|
|
1518
|
+
|
|
1519
|
+
def _matches(body, matches):
|
|
1520
|
+
"""Does a body hold every value of one of the declared matches, each a mapping of paths to values?"""
|
|
1521
|
+
return any(all(_declared_value(_path_value(body, path), value) for path, value in match.items())
|
|
1522
|
+
for match in matches)
|
|
1523
|
+
|
|
1524
|
+
|
|
1525
|
+
def declared_items(body, declared):
|
|
1526
|
+
"""The objects a tool call's or a tool result's fields are read from, in one record's body.
|
|
1527
|
+
|
|
1528
|
+
The body itself, or each block its `blocks` path reaches when the store nests them
|
|
1529
|
+
in a message; either counts only when it holds every value its `match` declares.
|
|
1530
|
+
"""
|
|
1531
|
+
items = _path_values(body, declared["blocks"]) if declared.get("blocks") else [body]
|
|
1532
|
+
match = declared.get("match") or {}
|
|
1533
|
+
return [item for item in items if isinstance(item, dict)
|
|
1534
|
+
and all(_declared_value(_path_value(item, path), value) for path, value in match.items())]
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
def tool_calls(body, tool):
|
|
1538
|
+
"""[(name, arguments)] for each tool call a record of the declared tool-call kind holds."""
|
|
1539
|
+
return [(str(_path_value(item, tool["name"]) or ""), _path_value(item, tool["args"]))
|
|
1540
|
+
for item in declared_items(body, tool)]
|
|
1541
|
+
|
|
1542
|
+
|
|
1543
|
+
def closes_turn(rec, shape):
|
|
1544
|
+
"""Does a record close a turn: a kind `turn.end` names, or one whose field holds a value `turn.end_when` declares?
|
|
1545
|
+
|
|
1546
|
+
A store that writes no closing record ends a turn in a field of the response that
|
|
1547
|
+
ends it; the response's kind cannot say so, since every response has that kind.
|
|
1548
|
+
"""
|
|
1549
|
+
kind = record_kind(rec, shape)
|
|
1550
|
+
if kind in shape["end"]:
|
|
1551
|
+
return True
|
|
1552
|
+
body = record_body(rec, shape)
|
|
1553
|
+
return any(kind == when["type"] and any(_declared_value(_path_value(body, when["field"]), value)
|
|
1554
|
+
for value in when["values"]) for when in shape.get("end_when") or [])
|
|
1555
|
+
|
|
1556
|
+
|
|
1557
|
+
def is_bookkeeping(kind, shape):
|
|
1558
|
+
"""May a record of this kind follow a turn's end without meaning a turn is running?
|
|
1559
|
+
|
|
1560
|
+
`turn.bookkeeping` names such kinds; `turn.conversation` names the kinds a turn is
|
|
1561
|
+
made of instead, and every other kind is bookkeeping, so a store that adds kinds of
|
|
1562
|
+
its own from release to release still reads as ended when its turn has ended.
|
|
1563
|
+
"""
|
|
1564
|
+
conversation = shape.get("conversation") or []
|
|
1565
|
+
return kind in shape["bookkeeping"] or (bool(conversation) and kind not in conversation)
|
|
1566
|
+
|
|
1567
|
+
|
|
1568
|
+
# A subagent's spend is its session's, charged to the turn that started it (`transcript.subagents`).
|
|
1569
|
+
|
|
1570
|
+
def subagents(transcript, shape, sidecars=True):
|
|
1571
|
+
"""What names each subagent transcript beside a session transcript: {"files", "named", "calls"}; None if undeclared.
|
|
1572
|
+
|
|
1573
|
+
`files` holds every transcript a declared `path` reaches under the session's subagent
|
|
1574
|
+
directory; `named` maps (the declaration's index, the name its `{id}` stands for) to the
|
|
1575
|
+
transcripts of that name, and `calls` a tool call's id to the transcripts whose sidecar
|
|
1576
|
+
names it. One listing of the directory serves a whole reading. A reader that needs only
|
|
1577
|
+
the transcripts asks without `sidecars`, and no sidecar is opened.
|
|
1578
|
+
|
|
1579
|
+
Every path stays inside the session's own subagent directory. `dir` names a directory for the
|
|
1580
|
+
session inside the session transcript's own, and every transcript and sidecar must be inside
|
|
1581
|
+
that one, their real paths compared, so neither `..`, an absolute path, a wildcard nor a
|
|
1582
|
+
symbolic link takes a reading anywhere else the user can read. The paths returned are the
|
|
1583
|
+
real ones. A declaration may come from an adapter layer the agent it describes can write;
|
|
1584
|
+
confined, it chooses only which of the files the store keeps for that session are its
|
|
1585
|
+
subagents.
|
|
1586
|
+
"""
|
|
1587
|
+
declared = shape.get("subagents")
|
|
1588
|
+
if not declared or not transcript:
|
|
1589
|
+
return None
|
|
1590
|
+
found = {"files": [], "named": {}, "calls": {}}
|
|
1591
|
+
session = os.path.splitext(os.path.basename(transcript))[0]
|
|
1592
|
+
steps = _plain_steps(declared["dir"].replace("{session}", session))
|
|
1593
|
+
directory = os.path.realpath(os.path.join(os.path.dirname(transcript), *steps)) if steps else ""
|
|
1594
|
+
if not _inside(directory, os.path.realpath(os.path.dirname(transcript))) or not os.path.isdir(directory):
|
|
1595
|
+
return found
|
|
1596
|
+
listed = set()
|
|
1597
|
+
for number, entry in enumerate(declared["transcripts"]):
|
|
1598
|
+
for path, ident in _declared_files(directory, entry["path"].split("/")):
|
|
1599
|
+
if path not in listed:
|
|
1600
|
+
listed.add(path)
|
|
1601
|
+
found["files"].append(path)
|
|
1602
|
+
if entry["named_by"] and ident:
|
|
1603
|
+
found["named"].setdefault((number, ident), []).append(path)
|
|
1604
|
+
sidecar = declared["sidecar"] if sidecars else None
|
|
1605
|
+
for path in found["files"] if sidecar else []:
|
|
1606
|
+
steps = _plain_steps(sidecar["path"].replace("{name}", os.path.splitext(os.path.basename(path))[0]))
|
|
1607
|
+
meta = os.path.realpath(os.path.join(os.path.dirname(path), *steps)) if steps else ""
|
|
1608
|
+
if not _inside(meta, directory):
|
|
1609
|
+
continue
|
|
1610
|
+
try:
|
|
1611
|
+
with _open_regular(meta) as fh:
|
|
1612
|
+
call = _path_value(json.loads(fh.read().decode(UTF8)), sidecar["call"])
|
|
1613
|
+
except (OSError, ValueError):
|
|
1614
|
+
continue
|
|
1615
|
+
if isinstance(call, str) and call:
|
|
1616
|
+
found["calls"].setdefault(call, []).append(path)
|
|
1617
|
+
return found
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
def started_subagents(rec, shape, found, joined):
|
|
1621
|
+
"""The subagent transcripts a record starts that no record before it in the reading started.
|
|
1622
|
+
|
|
1623
|
+
A record starts the transcripts the value at a declared `named_by` names, and a tool
|
|
1624
|
+
call the transcripts whose sidecar names its id. `joined` holds what the reading has
|
|
1625
|
+
started, so a transcript that two records name - the call that started it and its
|
|
1626
|
+
result - is read once, after the first.
|
|
1627
|
+
"""
|
|
1628
|
+
body = record_body(rec, shape) if found else None
|
|
1629
|
+
if body is None:
|
|
1630
|
+
return []
|
|
1631
|
+
paths = []
|
|
1632
|
+
for number, entry in enumerate(shape["subagents"]["transcripts"]):
|
|
1633
|
+
value = _path_value(body, entry["named_by"]) if entry["named_by"] else None
|
|
1634
|
+
if isinstance(value, str) and value:
|
|
1635
|
+
paths += found["named"].get((number, value), [])
|
|
1636
|
+
tool = shape["tool"]
|
|
1637
|
+
if found["calls"] and tool and tool["id"] and record_kind(rec, shape) == tool["type"]:
|
|
1638
|
+
for item in declared_items(body, tool):
|
|
1639
|
+
value = _path_value(item, tool["id"])
|
|
1640
|
+
if isinstance(value, str) and value:
|
|
1641
|
+
paths += found["calls"].get(value, [])
|
|
1642
|
+
fresh = [path for path in dict.fromkeys(paths) if path not in joined]
|
|
1643
|
+
joined.update(fresh)
|
|
1644
|
+
return fresh
|
|
1645
|
+
|
|
1646
|
+
|
|
1647
|
+
def subagent_records(path):
|
|
1648
|
+
"""A subagent transcript's records: all of them, since its whole spend is charged to the turn that started it."""
|
|
1649
|
+
return read_tail(path, 400_000_000, regular=True)
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def with_subagents(recs, shape, transcript, read=subagent_records):
|
|
1653
|
+
"""(record, subagent) for each record of a session, each followed by the records of the subagents it starts.
|
|
1654
|
+
|
|
1655
|
+
`subagent` is None for a record of the session, and for a subagent's record the reading's
|
|
1656
|
+
state of the transcript it came from, {"path", "turn"}: `next_turn` and `add_usage` charge
|
|
1657
|
+
it to the session's turn it falls in, which is the turn that started it. A subagent a
|
|
1658
|
+
subagent starts follows the record that starts it, charged to the same turn. `read(path)`
|
|
1659
|
+
gives a subagent transcript's records. A transcript no record of `recs` starts is not
|
|
1660
|
+
read: its spend belongs to no turn of the reading - in the panel, a subagent started
|
|
1661
|
+
before the tail it reads - and a total stays the sum of its turns.
|
|
1662
|
+
"""
|
|
1663
|
+
found = subagents(transcript, shape)
|
|
1664
|
+
if not (found or {}).get("files"):
|
|
1665
|
+
return ((rec, None) for rec in recs)
|
|
1666
|
+
joined = set()
|
|
1667
|
+
|
|
1668
|
+
def spliced(records, subagent):
|
|
1669
|
+
for rec in records:
|
|
1670
|
+
yield rec, subagent
|
|
1671
|
+
for path in started_subagents(rec, shape, found, joined):
|
|
1672
|
+
yield from spliced(read(path), {"path": path, "turn": None})
|
|
1673
|
+
return spliced(recs, None)
|
|
1674
|
+
|
|
1675
|
+
|
|
1676
|
+
def subagent_tails(transcript, shape, since, nbytes):
|
|
1677
|
+
"""The records of every subagent transcript beside a session's that was written since `since`, joined or not.
|
|
1678
|
+
|
|
1679
|
+
A subagent's tool calls are the implementer's whether or not a record of the tail read
|
|
1680
|
+
starts it, so what one wrote is never taken for another writer's.
|
|
1681
|
+
"""
|
|
1682
|
+
out = []
|
|
1683
|
+
for path in (subagents(transcript, shape, sidecars=False) or {}).get("files") or []:
|
|
1684
|
+
try:
|
|
1685
|
+
if os.path.getmtime(path) < since:
|
|
1686
|
+
continue
|
|
1687
|
+
except OSError:
|
|
1688
|
+
continue
|
|
1689
|
+
out += read_tail(path, nbytes, regular=True)
|
|
1690
|
+
return out
|
|
1691
|
+
|
|
1692
|
+
|
|
1693
|
+
# A subagent at work is its implementer at work (`transcript.subagents`, or `transcript.subagent_sessions`
|
|
1694
|
+
# where a store keeps one session per subagent): whether the implementer is working, idle or done is read
|
|
1695
|
+
# from the subagents too. Those readers run every watchdog cycle, so they look up what a subagent wrote and
|
|
1696
|
+
# open none of its records.
|
|
1697
|
+
|
|
1698
|
+
def subagent_writes(transcript, shape):
|
|
1699
|
+
"""[(modification time, size)] of each subagent transcript of a session; [] if it declares none.
|
|
1700
|
+
|
|
1701
|
+
Read from one listing and a stat of each transcript: no subagent transcript and no
|
|
1702
|
+
sidecar is opened, however many a session holds or however long they grew.
|
|
1703
|
+
|
|
1704
|
+
A database store joins differently, because a subagent there is a session of its own in the
|
|
1705
|
+
same tables rather than a file beside the parent's: `transcript.subagent_sessions` names the
|
|
1706
|
+
query that finds one session's children, and each child is asked its own clock. No size comes
|
|
1707
|
+
of it, because the parent's size is the whole store and already holds a child's rows: what the
|
|
1708
|
+
join adds is when a child wrote, not how much more of the store there is.
|
|
1709
|
+
"""
|
|
1710
|
+
if isinstance(transcript, DatabaseTail):
|
|
1711
|
+
return [(tail.last_activity() or 0, 0) for tail in transcript.descendant_tails()]
|
|
1712
|
+
out = []
|
|
1713
|
+
for path in (subagents(transcript, shape, sidecars=False) or {}).get("files") or []:
|
|
1714
|
+
try:
|
|
1715
|
+
stat = os.stat(path)
|
|
1716
|
+
except OSError:
|
|
1717
|
+
continue
|
|
1718
|
+
out.append((stat.st_mtime, stat.st_size))
|
|
1719
|
+
return out
|
|
1720
|
+
|
|
1721
|
+
|
|
1722
|
+
def session_write(transcript):
|
|
1723
|
+
"""When a session wrote to its own transcript last, in epoch seconds - and to nothing else.
|
|
1724
|
+
|
|
1725
|
+
A database file is written by every session held in it, so its modification time says a machine
|
|
1726
|
+
is busy rather than that this agent is. A store that keeps a per-session clock can say more, and
|
|
1727
|
+
`transcript.freshness` declares the query that reads it. A session whose rows the store updates
|
|
1728
|
+
in place has no row written lately and is still working, so this is the only reading that sees
|
|
1729
|
+
it. A reader that compares what a session wrote with what its subagent wrote has to take both
|
|
1730
|
+
from one clock, or the machine's answer swamps the agent's.
|
|
1731
|
+
"""
|
|
1732
|
+
return (transcript.last_activity() or os.path.getmtime(transcript)) if isinstance(transcript, DatabaseTail) \
|
|
1733
|
+
else os.path.getmtime(transcript)
|
|
1734
|
+
|
|
1735
|
+
|
|
1736
|
+
def last_write(transcript, shape):
|
|
1737
|
+
"""When the implementer last wrote, in epoch seconds: to its session transcript or to a subagent's.
|
|
1738
|
+
|
|
1739
|
+
A session waits for a subagent with its own transcript quiet, or ends its turn while the
|
|
1740
|
+
subagent works on in the background, and the session transcript alone then reads as an
|
|
1741
|
+
implementer silent for as long as the subagent works: over an hour, measured. A subagent
|
|
1742
|
+
that has finished writes no more, so its last write only ages.
|
|
1743
|
+
"""
|
|
1744
|
+
return max([session_write(transcript)] + [mtime for mtime, _ in subagent_writes(transcript, shape)])
|
|
1745
|
+
|
|
1746
|
+
|
|
1747
|
+
def _strings(o, out, keys, depth=0, shortest=31):
|
|
1748
|
+
if depth > 7:
|
|
1749
|
+
return
|
|
1750
|
+
if isinstance(o, str):
|
|
1751
|
+
if len(o) >= shortest:
|
|
1752
|
+
out.append(o)
|
|
1753
|
+
elif isinstance(o, dict):
|
|
1754
|
+
for k, v in o.items():
|
|
1755
|
+
if k in keys:
|
|
1756
|
+
_strings(v, out, keys, depth + 1, shortest)
|
|
1757
|
+
elif isinstance(v, (dict, list)):
|
|
1758
|
+
_strings(v, out, keys, depth + 1, shortest)
|
|
1759
|
+
elif isinstance(o, list):
|
|
1760
|
+
for x in o:
|
|
1761
|
+
_strings(x, out, keys, depth + 1, shortest)
|
|
1762
|
+
|
|
1763
|
+
|
|
1764
|
+
def local_hhmm(ts):
|
|
1765
|
+
"""Render an ISO timestamp in the reader's own timezone.
|
|
1766
|
+
|
|
1767
|
+
Transcripts store UTC. Slicing "…T00:44:12.525Z" for its characters prints
|
|
1768
|
+
00:44 next to a panel header showing local 03:44, and a message written
|
|
1769
|
+
thirty seconds ago then reads as three hours stale. In an observation tool
|
|
1770
|
+
that is not cosmetic: it is the difference between "the agent just said this"
|
|
1771
|
+
and "the agent stopped saying things", which are opposite conclusions.
|
|
1772
|
+
"""
|
|
1773
|
+
if not ts or len(ts) < 16:
|
|
1774
|
+
return "--:--"
|
|
1775
|
+
try:
|
|
1776
|
+
t = ts.replace("Z", "+00:00")
|
|
1777
|
+
dt = datetime.fromisoformat(t)
|
|
1778
|
+
if dt.tzinfo is None: # naive stamps are already local
|
|
1779
|
+
return dt.strftime("%H:%M")
|
|
1780
|
+
return dt.astimezone().strftime("%H:%M")
|
|
1781
|
+
except ValueError:
|
|
1782
|
+
return ts[11:16]
|
|
1783
|
+
|
|
1784
|
+
|
|
1785
|
+
def message_words(body, shape, shortest=31):
|
|
1786
|
+
"""The text a message record holds, as one line: all of it, or only its word blocks when it holds blocks.
|
|
1787
|
+
|
|
1788
|
+
A store that nests blocks in a message writes a tool's result as a block of the
|
|
1789
|
+
prompt's kind and a tool call as a block of the reply's, and neither is a person or
|
|
1790
|
+
the agent speaking. `transcript.messages.blocks` and `match` declare the blocks that
|
|
1791
|
+
are words; a record holding blocks of which none match holds no words, and a record
|
|
1792
|
+
whose content is not blocks - a prompt typed as one string - is read whole.
|
|
1793
|
+
|
|
1794
|
+
A store also writes notes of its own, and the summary of a compacted conversation, as
|
|
1795
|
+
records of the prompt's kind. They are no one speaking, and a record holding what one of
|
|
1796
|
+
`transcript.messages.not_words` declares holds no words either.
|
|
1797
|
+
|
|
1798
|
+
A string shorter than `shortest` is passed over, as the panel's lines have always passed short
|
|
1799
|
+
strings over; a reader asking whether anyone spoke at all counts a short answer too.
|
|
1800
|
+
"""
|
|
1801
|
+
if _matches(body, shape.get("not_words") or []):
|
|
1802
|
+
return ""
|
|
1803
|
+
words = shape.get("words")
|
|
1804
|
+
parts = declared_items(body, words) if words and _path_values(body, words["blocks"]) else [body]
|
|
1805
|
+
buf = []
|
|
1806
|
+
for part in parts:
|
|
1807
|
+
_strings(part, buf, shape["text_keys"], shortest=shortest)
|
|
1808
|
+
return " ".join(" ".join(buf).split())
|
|
1809
|
+
|
|
1810
|
+
|
|
1811
|
+
def in_place_reply(rec, shape):
|
|
1812
|
+
"""Is a record a reply the harness wrote in the model's place, reporting no usage (`harness_replies`)?
|
|
1813
|
+
|
|
1814
|
+
A store may write a reply of its own where the model gave none - the error its service
|
|
1815
|
+
returned, or an answer to a note that asked the model nothing - holding no usage. It answers
|
|
1816
|
+
nothing and spends nothing: no reading counts it as the implementer's turn, and a turn that
|
|
1817
|
+
holds no other reply was not answered. A reply of those that did report usage is read as any
|
|
1818
|
+
other.
|
|
1819
|
+
"""
|
|
1820
|
+
kind = record_kind(rec, shape) if shape.get("harness_replies") else None
|
|
1821
|
+
if kind not in shape["reply"]:
|
|
1822
|
+
return False
|
|
1823
|
+
body, usage = record_body(rec, shape), shape["usage"]
|
|
1824
|
+
return _matches(body, shape["harness_replies"]) and not (usage and kind == usage["type"]
|
|
1825
|
+
and record_usage(body, usage))
|
|
1826
|
+
|
|
1827
|
+
|
|
1828
|
+
def _between(text, markers):
|
|
1829
|
+
"""Each piece of a text that stands between the two markers of a declared pair, in order."""
|
|
1830
|
+
start, end = markers
|
|
1831
|
+
pieces, at = [], 0
|
|
1832
|
+
while True:
|
|
1833
|
+
first = text.find(start, at)
|
|
1834
|
+
last = text.find(end, first + len(start)) if first >= 0 else -1
|
|
1835
|
+
if last < 0:
|
|
1836
|
+
return pieces
|
|
1837
|
+
pieces.append(text[first + len(start):last])
|
|
1838
|
+
at = last + len(end)
|
|
1839
|
+
|
|
1840
|
+
|
|
1841
|
+
def failed_ends(rec, shape):
|
|
1842
|
+
"""[(task, text)] when a record tells of a background task's end whose status is a failure (`telemetry.failure.ends`).
|
|
1843
|
+
|
|
1844
|
+
`task` names the end: the ids its text holds between the `id` markers, or the whole text when it
|
|
1845
|
+
holds none, so an end told twice - once while a turn ran, once after it - is one end to a reader
|
|
1846
|
+
that keeps the names it has read. `text` is what stands between the `text` markers, else all of it.
|
|
1847
|
+
"""
|
|
1848
|
+
ends = shape.get("ends")
|
|
1849
|
+
body = record_body(rec, shape) if ends else None
|
|
1850
|
+
kind = record_kind(rec, shape) if body is not None else None
|
|
1851
|
+
for record in ends["records"] if kind else []:
|
|
1852
|
+
value = _path_value(body, record["field"]) if kind == record["type"] and _matches(body, [record["match"]]) \
|
|
1853
|
+
else None
|
|
1854
|
+
if not isinstance(value, str):
|
|
1855
|
+
continue
|
|
1856
|
+
status = _between(value, ends["status"])
|
|
1857
|
+
if not status or status[0].strip() not in ends["failed_when"]:
|
|
1858
|
+
return []
|
|
1859
|
+
ids = _between(value, ends["id"]) if "id" in ends else []
|
|
1860
|
+
text = _between(value, ends["text"]) if "text" in ends else []
|
|
1861
|
+
return [(" ".join(ident.strip() for ident in ids) if ids else value, text[0] if text else value)]
|
|
1862
|
+
return []
|
|
1863
|
+
|
|
1864
|
+
|
|
1865
|
+
def messages(recs, limit=8, adapter=None):
|
|
1866
|
+
"""[(HH:MM, "user" | "assistant", text)] oldest→newest.
|
|
1867
|
+
|
|
1868
|
+
A prompt and a reply are the kinds the adapter declares in `transcript.messages`,
|
|
1869
|
+
and their words the text `message_words` reads; asked without an adapter, the
|
|
1870
|
+
implementer's adapter says which they are.
|
|
1871
|
+
"""
|
|
1872
|
+
shape = transcript_shape(implementer_adapter() if adapter is None else adapter)
|
|
1873
|
+
roles = dict.fromkeys(shape["reply"], "assistant")
|
|
1874
|
+
roles.update(dict.fromkeys(shape["prompt"], "user"))
|
|
1875
|
+
out = []
|
|
1876
|
+
for r in reversed(recs):
|
|
1877
|
+
kind = record_kind(r, shape)
|
|
1878
|
+
if kind not in roles:
|
|
1879
|
+
continue
|
|
1880
|
+
text = message_words(record_body(r, shape), shape)
|
|
1881
|
+
if len(text) < 40:
|
|
1882
|
+
continue
|
|
1883
|
+
out.append((local_hhmm(record_time(r, shape)), roles[kind], text))
|
|
1884
|
+
if len(out) >= limit:
|
|
1885
|
+
break
|
|
1886
|
+
return list(reversed(out))
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
def telemetry(recs, adapter, transcript=None):
|
|
1890
|
+
"""Context %, per-turn and session cost, driven by the adapter's block.
|
|
1891
|
+
|
|
1892
|
+
Usage adds up under the adapter's reading, as `polyctl cost` and the credit estimate add it:
|
|
1893
|
+
under `sum` every usage entry is one turn's cost, as written; under `peak-per-turn` a turn
|
|
1894
|
+
- the one already under way where the records begin included - costs the highest total
|
|
1895
|
+
its records reached; under `per-response` a turn costs each response it holds, at its
|
|
1896
|
+
highest record, that no earlier turn of these records held. A turn whose usage names no
|
|
1897
|
+
tools beside it has the tool calls its records hold.
|
|
1898
|
+
|
|
1899
|
+
Given the path of the session `transcript` the records were read from, the subagents a
|
|
1900
|
+
record starts are read after it (`with_subagents`): their spend is in the total, in the
|
|
1901
|
+
turn that started them and in `delegated`, and their tool calls are that turn's. Under
|
|
1902
|
+
`sum`, where a usage record is a turn of its own, a subagent's adds to the total and to
|
|
1903
|
+
`delegated` and is no turn of the session. The context is the session's own.
|
|
1904
|
+
|
|
1905
|
+
A reply the harness wrote in the model's place with no usage (`in_place_reply`) is not read:
|
|
1906
|
+
a turn holding no other reply is no turn of the implementer's, and the average is over the
|
|
1907
|
+
turns the model answered.
|
|
1908
|
+
"""
|
|
1909
|
+
shape = transcript_shape(implementer_adapter() if adapter is None else adapter)
|
|
1910
|
+
ctx_spec, cost_spec = shape["context"], shape["usage"]
|
|
1911
|
+
out = {"ctx": None, "total": 0.0, "turns": 0, "last": None, "unit": shape["unit"], "delegated": 0.0}
|
|
1912
|
+
turn, counted = None, {}
|
|
1913
|
+
for r, subagent in with_subagents(recs, shape, transcript if cost_spec else None):
|
|
1914
|
+
pl = record_body(r, shape)
|
|
1915
|
+
if pl is None:
|
|
1916
|
+
continue
|
|
1917
|
+
t = record_kind(r, shape)
|
|
1918
|
+
if ctx_spec and subagent is None and t == ctx_spec["type"]:
|
|
1919
|
+
if all(pl.get(k) == v for k, v in ctx_spec["match"].items()):
|
|
1920
|
+
val = _path_value(pl, ctx_spec["field"])
|
|
1921
|
+
if isinstance(val, (int, float)):
|
|
1922
|
+
out["ctx"] = val
|
|
1923
|
+
if not cost_spec:
|
|
1924
|
+
continue
|
|
1925
|
+
turn, _ = next_turn(turn, r, shape, subagent)
|
|
1926
|
+
# A turn read from its records, not from a summary, has the tool calls those records hold.
|
|
1927
|
+
calls = len(tool_calls(pl, shape["tool"])) if cost_spec["reading"] != "sum" and shape["tool"] \
|
|
1928
|
+
and t == shape["tool"]["type"] else 0
|
|
1929
|
+
if t != cost_spec["type"]:
|
|
1930
|
+
if calls and turn is not None:
|
|
1931
|
+
turn["calls"] = turn.get("calls", 0) + calls
|
|
1932
|
+
continue
|
|
1933
|
+
if in_place_reply(r, shape):
|
|
1934
|
+
continue
|
|
1935
|
+
if cost_spec["reading"] == "sum":
|
|
1936
|
+
for values, tools in usage_entries(pl, cost_spec):
|
|
1937
|
+
values = [value or 0 for value in values]
|
|
1938
|
+
if all(isinstance(value, (int, float)) for value in values):
|
|
1939
|
+
u = sum(values)
|
|
1940
|
+
out["total"] += u
|
|
1941
|
+
if subagent is not None:
|
|
1942
|
+
out["delegated"] += u
|
|
1943
|
+
continue
|
|
1944
|
+
out["turns"] += 1
|
|
1945
|
+
out["last"] = (u, len(tools or []))
|
|
1946
|
+
continue
|
|
1947
|
+
entries = usage_entries(pl, cost_spec)
|
|
1948
|
+
if not entries:
|
|
1949
|
+
continue
|
|
1950
|
+
turn = {} if turn is None else turn
|
|
1951
|
+
turn["calls"] = turn.get("calls", 0) + calls
|
|
1952
|
+
if "usage" not in turn:
|
|
1953
|
+
out["turns"] += 1
|
|
1954
|
+
grown = add_usage(turn, pl, cost_spec, counted, subagent)
|
|
1955
|
+
out["total"] += grown
|
|
1956
|
+
out["delegated"] += grown if subagent is not None else 0.0
|
|
1957
|
+
out["last"] = (turn["usage"], sum(len(tools or []) for _, tools in entries) if cost_spec["tools"]
|
|
1958
|
+
else turn["calls"])
|
|
1959
|
+
return out
|
|
1960
|
+
|
|
1961
|
+
|
|
1962
|
+
_QUOTA = {} # command -> {"at", "lines"}: the readings this process holds
|
|
1963
|
+
QUOTA_READINGS = 16 # how many commands ~/.polyctl/quota.json keeps a reading for, newest first
|
|
1964
|
+
# A word a POSIX shell passes on as it is written: nothing it would split, quote, expand or glob.
|
|
1965
|
+
_PLAIN_WORD = re.compile(r"[\w./:@%+,=-]+")
|
|
1966
|
+
|
|
1967
|
+
|
|
1968
|
+
def quota_readings_path():
|
|
1969
|
+
return os.path.join(HOME, ".polyctl", "quota.json")
|
|
1970
|
+
|
|
1971
|
+
|
|
1972
|
+
def _quota_readings():
|
|
1973
|
+
"""{command: {"at", "lines"}} as polyctl's processes kept them; a file that cannot be read keeps none."""
|
|
1974
|
+
try:
|
|
1975
|
+
with open(quota_readings_path(), encoding=UTF8) as fh:
|
|
1976
|
+
document = json.load(fh)
|
|
1977
|
+
except (OSError, ValueError):
|
|
1978
|
+
return {}
|
|
1979
|
+
if not isinstance(document, dict):
|
|
1980
|
+
return {}
|
|
1981
|
+
return {command: reading for command, reading in document.items()
|
|
1982
|
+
if isinstance(reading, dict) and isinstance(reading.get("at"), (int, float))
|
|
1983
|
+
and isinstance(reading.get("lines"), list) and all(isinstance(line, str) for line in reading["lines"])}
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
def _quota_output(argv):
|
|
1987
|
+
"""(what `sh(" ".join(argv) + " 2>/dev/null", cwd=HOME)` returned, whether the command exited 0).
|
|
1988
|
+
|
|
1989
|
+
Without a shell when the program is on the binary search path and no word is one a
|
|
1990
|
+
shell would change: run as the shell ran it, from the home directory, its standard
|
|
1991
|
+
error discarded, the program named as written. Anything else - a word the shell would
|
|
1992
|
+
split, quote or expand, a program found nowhere, and every command on Windows, whose
|
|
1993
|
+
cmd.exe has rules of its own - goes through sh()'s shell as before.
|
|
1994
|
+
"""
|
|
1995
|
+
program = argv[0]
|
|
1996
|
+
plain = os.name != "nt" and all(_PLAIN_WORD.fullmatch(word) for word in argv) \
|
|
1997
|
+
and "=" not in program and (os.path.isabs(program) or "/" not in program)
|
|
1998
|
+
found = binary_candidates(program) if plain else []
|
|
1999
|
+
if found:
|
|
2000
|
+
try:
|
|
2001
|
+
done = subprocess.run(list(argv), executable=found[0], cwd=HOME, stdout=subprocess.PIPE,
|
|
2002
|
+
stderr=subprocess.DEVNULL, text=True, encoding=UTF8, errors="replace", timeout=20)
|
|
2003
|
+
return done.stdout.strip(), done.returncode == 0
|
|
2004
|
+
except OSError:
|
|
2005
|
+
pass # not started - a script with no #! line only a shell runs - so the shell runs it
|
|
2006
|
+
except Exception:
|
|
2007
|
+
return "", False
|
|
2008
|
+
out, status = _sh_run(" ".join(argv) + " 2>/dev/null", cwd=HOME)
|
|
2009
|
+
return out, status == 0
|
|
2010
|
+
|
|
2011
|
+
|
|
2012
|
+
def _quota_write(readings):
|
|
2013
|
+
"""Replace ~/.polyctl/quota.json with the newest of these readings, for every other polyctl process."""
|
|
2014
|
+
from .storage import replace_file_durably
|
|
2015
|
+
newest = sorted(readings.items(), key=lambda item: item[1]["at"], reverse=True)[:QUOTA_READINGS]
|
|
2016
|
+
try:
|
|
2017
|
+
replace_file_durably(quota_readings_path(), (json.dumps(dict(newest), indent=1) + "\n").encode(UTF8))
|
|
2018
|
+
except OSError:
|
|
2019
|
+
pass # a reading that cannot be kept is still this call's answer
|
|
2020
|
+
|
|
2021
|
+
|
|
2022
|
+
def _quota_keep(command, facts):
|
|
2023
|
+
"""Keep one reading for this process and, the file replaced whole, for every other polyctl process."""
|
|
2024
|
+
reading = _QUOTA[command] = dict(facts, at=time.time())
|
|
2025
|
+
readings = _quota_readings()
|
|
2026
|
+
readings[command] = reading
|
|
2027
|
+
_quota_write(readings)
|
|
2028
|
+
return reading
|
|
2029
|
+
|
|
2030
|
+
|
|
2031
|
+
def _kept_reading(argv, window, take):
|
|
2032
|
+
"""A command's reading: the one kept under the command while it is younger than `window` seconds, else a new one.
|
|
2033
|
+
|
|
2034
|
+
`take()` runs the command and returns (reading, whether it may be kept): a dict whose
|
|
2035
|
+
"lines" hold what was read, beside any other facts its reader keeps. Only a command
|
|
2036
|
+
that answered is kept: a failure is asked again next time, not served for the whole
|
|
2037
|
+
window.
|
|
2038
|
+
"""
|
|
2039
|
+
command = " ".join(argv)
|
|
2040
|
+
now = time.time()
|
|
2041
|
+
|
|
2042
|
+
def fresh(reading):
|
|
2043
|
+
return reading is not None and 0 <= now - reading["at"] < window
|
|
2044
|
+
|
|
2045
|
+
reading = _QUOTA.get(command)
|
|
2046
|
+
if not fresh(reading):
|
|
2047
|
+
reading = _quota_readings().get(command)
|
|
2048
|
+
if fresh(reading):
|
|
2049
|
+
_QUOTA[command] = reading
|
|
2050
|
+
return reading
|
|
2051
|
+
reading, keep = take()
|
|
2052
|
+
return _quota_keep(command, reading) if keep else reading
|
|
2053
|
+
|
|
2054
|
+
|
|
2055
|
+
def metered_adapter(cfg):
|
|
2056
|
+
"""The adapter the implementer's quota is read through: the one the package ships under the adapter id its block
|
|
2057
|
+
names, else {} (TRIAGE-HIGHS).
|
|
2058
|
+
|
|
2059
|
+
Reading the quota runs the command its adapter declares (`telemetry.quota.argv`), and `polyctl status` and the
|
|
2060
|
+
watchdog's quota gate read it from the adapter as every layer loads it - the project's .polyctl/adapters/ among them,
|
|
2061
|
+
a file the implementer writes, where a planted `touch` ran. A command polyctl runs comes from the package alone, as a
|
|
2062
|
+
turn's, a route's, an MCP registration's and a probe's do (TURN-ARGV, ROUTE-ARGV, HARNESS-SETUP-2, PROBE-SHIPPED);
|
|
2063
|
+
a layer still says how a transcript reads.
|
|
2064
|
+
"""
|
|
2065
|
+
return turn_adapter(((cfg or {}).get("implementer") or {}).get("adapter")) or {}
|
|
2066
|
+
|
|
2067
|
+
|
|
2068
|
+
def quota(adapter, ttl=300):
|
|
2069
|
+
"""Provider quota via the adapter's command, kept for its window across polyctl's processes. Silent if absent.
|
|
2070
|
+
|
|
2071
|
+
`polyctl status` and every watchdog cycle are processes of their own, so a reading held in
|
|
2072
|
+
memory was taken again by each of them, through a shell, although the adapter declares
|
|
2073
|
+
how long one lasts (`cache_seconds`). A reading is kept under its command in
|
|
2074
|
+
~/.polyctl/quota.json and read back while it is younger than that. Only a command that
|
|
2075
|
+
exited 0 is kept: a failure is asked again next time, not served as no lines for the
|
|
2076
|
+
whole window.
|
|
2077
|
+
|
|
2078
|
+
`adapter` is one the package ships - metered_adapter's, or the window's - since the command it names runs
|
|
2079
|
+
(TRIAGE-HIGHS).
|
|
2080
|
+
"""
|
|
2081
|
+
spec = (adapter.get("telemetry") or {}).get("quota") or {}
|
|
2082
|
+
argv = spec.get("argv")
|
|
2083
|
+
if not argv:
|
|
2084
|
+
return []
|
|
2085
|
+
|
|
2086
|
+
def take():
|
|
2087
|
+
out, ok = _quota_output(argv)
|
|
2088
|
+
return {"lines": [l.strip() for l in out.split("\n")[1:] if l.strip() and "unknown" not in l]}, ok
|
|
2089
|
+
|
|
2090
|
+
return _kept_reading(argv, spec.get("cache_seconds", ttl), take)["lines"]
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def forget_quota(adapter):
|
|
2094
|
+
"""Drop the reading kept for this adapter's quota command, here and for every other polyctl process.
|
|
2095
|
+
|
|
2096
|
+
For when what it read has moved - an account rotation - so the next read asks again.
|
|
2097
|
+
"""
|
|
2098
|
+
argv = ((adapter.get("telemetry") or {}).get("quota") or {}).get("argv")
|
|
2099
|
+
if not argv:
|
|
2100
|
+
return
|
|
2101
|
+
command = " ".join(argv)
|
|
2102
|
+
_QUOTA.pop(command, None)
|
|
2103
|
+
readings = _quota_readings()
|
|
2104
|
+
if readings.pop(command, None) is not None:
|
|
2105
|
+
_quota_write(readings)
|
|
2106
|
+
|
|
2107
|
+
|
|
2108
|
+
# keyflip's budget status as one JSON document: {"budget": {"defaults", "accounts", "alerts", "breached"}}.
|
|
2109
|
+
QUOTA_BUDGET_ARGV = ("keyflip", "budget", "status", "--json")
|
|
2110
|
+
|
|
2111
|
+
|
|
2112
|
+
def quota_budget(adapter, ttl=300):
|
|
2113
|
+
"""keyflip's budgets, kept like the quota reading: {"accounts": [name], "breached": [{name, metric, pct, limit}]}.
|
|
2114
|
+
|
|
2115
|
+
The watchdog asked `keyflip budget status` through a shell on every call and looked in
|
|
2116
|
+
its prose for words keyflip does not print, so a budget a person set never stopped a
|
|
2117
|
+
turn. With `--json` keyflip writes the status as one document: every account with a
|
|
2118
|
+
budget, and on each an alert per window at or near its limit, `"breached": true` once
|
|
2119
|
+
usage has reached the limit. The account names and each breach are kept under that
|
|
2120
|
+
command in ~/.polyctl/quota.json for the adapter's `cache_seconds`, when keyflip exited 0
|
|
2121
|
+
and wrote the document.
|
|
2122
|
+
|
|
2123
|
+
Which breach counts: the document names no active account and no provider, so a
|
|
2124
|
+
breach on any account counts for every adapter asked about. keyflip runs without a
|
|
2125
|
+
shell from the home directory, found on the binary search path; absent, failing or
|
|
2126
|
+
unreadable, it reports no account, and the quota window decides.
|
|
2127
|
+
"""
|
|
2128
|
+
spec = (adapter.get("telemetry") or {}).get("quota") or {}
|
|
2129
|
+
|
|
2130
|
+
def take():
|
|
2131
|
+
out, status = _run_program(QUOTA_BUDGET_ARGV, cwd=HOME)
|
|
2132
|
+
try:
|
|
2133
|
+
accounts = json.loads(out)["budget"]["accounts"]
|
|
2134
|
+
names = [str(account["name"]) for account in accounts]
|
|
2135
|
+
breached = [{"name": str(account["name"]), "metric": alert.get("metric"), "pct": alert.get("pct"),
|
|
2136
|
+
"limit": alert.get("limit")}
|
|
2137
|
+
for account in accounts for alert in account.get("alerts") or [] if alert.get("breached") is True]
|
|
2138
|
+
except (ValueError, TypeError, KeyError, AttributeError):
|
|
2139
|
+
return {"lines": [], "breached": []}, False
|
|
2140
|
+
return {"lines": names, "breached": breached}, status == 0
|
|
2141
|
+
|
|
2142
|
+
reading = _kept_reading(QUOTA_BUDGET_ARGV, spec.get("cache_seconds", ttl), take)
|
|
2143
|
+
breached = reading.get("breached")
|
|
2144
|
+
return {"accounts": list(reading["lines"]),
|
|
2145
|
+
"breached": [fact for fact in breached if isinstance(fact, dict)] if isinstance(breached, list) else []}
|
|
2146
|
+
|
|
2147
|
+
|
|
2148
|
+
def busy(cfg, adapter):
|
|
2149
|
+
"""(state, seconds_since_write, description). Conservative: status AND age.
|
|
2150
|
+
|
|
2151
|
+
The age is the implementer's last write, a subagent's included (`last_write`): a session
|
|
2152
|
+
whose transcript is quiet while its subagent writes is working, not idle.
|
|
2153
|
+
"""
|
|
2154
|
+
msgs, meta = session_paths(cfg)
|
|
2155
|
+
if not msgs or not os.path.exists(msgs):
|
|
2156
|
+
return "unknown", None, ""
|
|
2157
|
+
age = int(time.time() - last_write(msgs, transcript_shape(adapter)))
|
|
2158
|
+
status, desc = "", ""
|
|
2159
|
+
if meta and os.path.exists(meta):
|
|
2160
|
+
try:
|
|
2161
|
+
m = json.load(open(meta, encoding=UTF8))
|
|
2162
|
+
status = m.get(
|
|
2163
|
+
(adapter.get("busy") or {}).get("status_field", "status"), "")
|
|
2164
|
+
desc = m.get(
|
|
2165
|
+
(adapter.get("busy") or {}).get("description_field", "description"), "") or ""
|
|
2166
|
+
except Exception:
|
|
2167
|
+
pass
|
|
2168
|
+
idle_s = (adapter.get("busy") or {}).get("idle_seconds", 240)
|
|
2169
|
+
running = status in ((adapter.get("busy") or {}).get("running_values") or [])
|
|
2170
|
+
|
|
2171
|
+
# A freshly-written transcript does not mean a live turn. The file keeps its
|
|
2172
|
+
# mtime after the process exits, so a killed agent reads as WORKING for the
|
|
2173
|
+
# whole idle window — the observation layer asserting the opposite of the
|
|
2174
|
+
# truth, at exactly the moment someone is looking to find out what happened.
|
|
2175
|
+
# Only ask the OS when the mtime would otherwise claim "working"; that is the
|
|
2176
|
+
# only case where the answer changes anything, and it keeps the panel cheap.
|
|
2177
|
+
# Where the OS cannot be asked the state is unknown, and says why, rather than
|
|
2178
|
+
# whatever the mtime claims (WINDOWS-TABLE-CLOSED).
|
|
2179
|
+
from . import procs
|
|
2180
|
+
try:
|
|
2181
|
+
turns = agent_pids(cfg["root"], adapter) if age < idle_s else None
|
|
2182
|
+
except procs.TableUnreadable as exc:
|
|
2183
|
+
return "unknown", age, str(exc)
|
|
2184
|
+
if age < idle_s and not turns:
|
|
2185
|
+
return "stopped", age, desc
|
|
2186
|
+
|
|
2187
|
+
if age < 120:
|
|
2188
|
+
state = "working"
|
|
2189
|
+
elif age < idle_s:
|
|
2190
|
+
state = "slowing"
|
|
2191
|
+
else:
|
|
2192
|
+
state = "idle" if not running else "idle"
|
|
2193
|
+
return state, age, desc
|
|
2194
|
+
|
|
2195
|
+
|
|
2196
|
+
_SURFACES = {"at": 0.0, "rows": {}}
|
|
2197
|
+
|
|
2198
|
+
|
|
2199
|
+
def tool_availability(ttl=600):
|
|
2200
|
+
"""Which agent tools are actually usable on this machine.
|
|
2201
|
+
|
|
2202
|
+
Two independent facts, and both matter: can polyctl find a runnable CLI, and is
|
|
2203
|
+
there an authenticated account for it. keyflip already answers the second for
|
|
2204
|
+
a range of tools and never reads the secret itself, so ask it rather than
|
|
2205
|
+
reinventing credential detection — align, do not depend: if keyflip is absent
|
|
2206
|
+
the CLI check still works on its own. Which tools and which binaries come from
|
|
2207
|
+
the vendor list and the adapters, never from a table here (#89). A CLI is installed where
|
|
2208
|
+
polyctl would start it from, including declared install directories past PATH
|
|
2209
|
+
(WARP-ADAPTER), never the current directory (WINDOWS-NO-CWD-EXEC).
|
|
2210
|
+
"""
|
|
2211
|
+
if time.time() - _SURFACES["at"] < ttl and _SURFACES["rows"]:
|
|
2212
|
+
return _SURFACES["rows"]
|
|
2213
|
+
rows = {}
|
|
2214
|
+
out = _run_program(["keyflip", "surfaces"], cwd=HOME)[0]
|
|
2215
|
+
for line in out.split("\n"):
|
|
2216
|
+
line = line.strip()
|
|
2217
|
+
if not (line.startswith("●") or line.startswith("○")):
|
|
2218
|
+
continue
|
|
2219
|
+
body = line[1:].strip()
|
|
2220
|
+
name = re.split(r"\s{2,}", body)[0].strip().lower()
|
|
2221
|
+
rows[name] = {"account": line.startswith("●")}
|
|
2222
|
+
vendors = [vendor for vendor in vendor_list() if vendor.get("adapter")]
|
|
2223
|
+
alias = {str(surface).lower(): vendor["adapter"] for vendor in vendors for surface in vendor.get("surfaces") or []}
|
|
2224
|
+
named = {}
|
|
2225
|
+
for k, v in rows.items():
|
|
2226
|
+
named[alias.get(k, k)] = v
|
|
2227
|
+
catalog = adapter_catalog()
|
|
2228
|
+
for vendor in vendors:
|
|
2229
|
+
binaries = adapter_binaries((catalog.get(vendor["adapter"]) or {}).get("adapter"))
|
|
2230
|
+
if not binaries:
|
|
2231
|
+
continue
|
|
2232
|
+
found = next((binary for binary in binaries if runnable_binary(binary)), None)
|
|
2233
|
+
entry = named.setdefault(vendor["adapter"], {})
|
|
2234
|
+
entry["installed"] = bool(found)
|
|
2235
|
+
entry["binary"] = found or binaries[0]
|
|
2236
|
+
_SURFACES.update(at=time.time(), rows=named)
|
|
2237
|
+
return named
|
|
2238
|
+
|
|
2239
|
+
|
|
2240
|
+
BOARD_STATES = ("running", "blocked", "queued", "inbox", "verified", "done", "rejected")
|
|
2241
|
+
|
|
2242
|
+
# A2A (Agent2Agent) task states, for the day an A2A endpoint serialises this
|
|
2243
|
+
# board. Kept as a table rather than adopted as the vocabulary: `inbox` and
|
|
2244
|
+
# `queued` are both A2A `submitted`, and `verified` has no A2A equivalent at all
|
|
2245
|
+
# — it is evidence gathered *before* completion, which is the distinction this
|
|
2246
|
+
# whole tool exists to make. Renaming our states to match would delete it.
|
|
2247
|
+
A2A_STATE = {
|
|
2248
|
+
"inbox": "submitted", # pulled from a source, not yet admitted
|
|
2249
|
+
"queued": "submitted", # admitted: has a written acceptance boundary
|
|
2250
|
+
"running": "working",
|
|
2251
|
+
"blocked": "input-required",
|
|
2252
|
+
"verified": "working", # gates passed; authority to land not yet granted
|
|
2253
|
+
"done": "completed",
|
|
2254
|
+
"rejected": "rejected",
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
|
|
2258
|
+
def _board_heading(line):
|
|
2259
|
+
"""Only a state's whole heading line opens its section (SLICE-LIFECYCLE-3)."""
|
|
2260
|
+
match = re.match(r"^##\s+([a-z]+)\s*$", line.strip())
|
|
2261
|
+
return match.group(1) if match and match.group(1) in BOARD_STATES else None
|
|
2262
|
+
|
|
2263
|
+
|
|
2264
|
+
def board(root, data=None):
|
|
2265
|
+
""".polyctl/board.md — where each pre-authorised item currently is.
|
|
2266
|
+
|
|
2267
|
+
A flat backlog answers "what is next" but not "what stopped, and on what".
|
|
2268
|
+
Once an agent is allowed to park a blocked slice and pick up the next item,
|
|
2269
|
+
that second question is the one a human actually needs on returning: the
|
|
2270
|
+
parked item is invisible precisely because work continued without it.
|
|
2271
|
+
|
|
2272
|
+
The file is the single source of truth and the agent edits it directly, the
|
|
2273
|
+
way it edits mail and reviews. Parsing here stays deliberately forgiving —
|
|
2274
|
+
a board a human cannot hand-edit during an incident is a board that goes
|
|
2275
|
+
stale during exactly the incident it was built for.
|
|
2276
|
+
|
|
2277
|
+
`data` is the board's bytes as one reading took them, read as the file is: a
|
|
2278
|
+
decision made on a BoardEdit reads the board its moves are made on
|
|
2279
|
+
(SLICE-LIFECYCLE-2).
|
|
2280
|
+
|
|
2281
|
+
Returns {state: [{"id", "title", "notes": {k: v}}]}.
|
|
2282
|
+
"""
|
|
2283
|
+
p = os.path.join(root, ".polyctl", "board.md")
|
|
2284
|
+
out = {st: [] for st in BOARD_STATES}
|
|
2285
|
+
if data is None and not os.path.exists(p):
|
|
2286
|
+
return out
|
|
2287
|
+
import io
|
|
2288
|
+
state = None
|
|
2289
|
+
for line in (open(p, errors="replace", encoding=UTF8) if data is None
|
|
2290
|
+
else io.TextIOWrapper(io.BytesIO(data), errors="replace", encoding=UTF8)):
|
|
2291
|
+
line = line.rstrip()
|
|
2292
|
+
if line.lstrip().startswith("##"):
|
|
2293
|
+
# Every heading ends a section; only a state's own heading starts one (#33).
|
|
2294
|
+
state = _board_heading(line)
|
|
2295
|
+
continue
|
|
2296
|
+
if not state or not line.lstrip().startswith("- "):
|
|
2297
|
+
continue
|
|
2298
|
+
item = line.lstrip()[2:].strip()
|
|
2299
|
+
m = re.match(r"^\[([^\]]+)\]\s*(.*)$", item)
|
|
2300
|
+
if not m:
|
|
2301
|
+
continue
|
|
2302
|
+
rest = [x.strip() for x in m.group(2).split("·")]
|
|
2303
|
+
notes = {}
|
|
2304
|
+
for chunk in rest[1:]:
|
|
2305
|
+
k, _, v = chunk.partition(":")
|
|
2306
|
+
if v:
|
|
2307
|
+
notes[k.strip()] = v.strip()
|
|
2308
|
+
elif chunk:
|
|
2309
|
+
notes[chunk] = ""
|
|
2310
|
+
out[state].append({"id": m.group(1), "title": rest[0] if rest else "",
|
|
2311
|
+
"notes": notes})
|
|
2312
|
+
return out
|
|
2313
|
+
|
|
2314
|
+
|
|
2315
|
+
def sources(root):
|
|
2316
|
+
""".polyctl/sources.json — external work queues feeding this project's board.
|
|
2317
|
+
|
|
2318
|
+
`polyctl` speaks no tracker API and holds no tracker credential. The MCP client is
|
|
2319
|
+
the *agent*: it pulls from Linear/Jira/GitHub with a server that already
|
|
2320
|
+
exists, normalises the result to a file, and this side admits it. That keeps
|
|
2321
|
+
the zero-install, file-only core intact — a tracker is an addition for people
|
|
2322
|
+
who install one, never a prerequisite.
|
|
2323
|
+
"""
|
|
2324
|
+
p = os.path.join(root, ".polyctl", "sources.json")
|
|
2325
|
+
if not os.path.exists(p):
|
|
2326
|
+
return {}
|
|
2327
|
+
try:
|
|
2328
|
+
cfg = json.load(open(p, encoding=UTF8))
|
|
2329
|
+
except Exception:
|
|
2330
|
+
return {}
|
|
2331
|
+
cfg.setdefault("sources", [])
|
|
2332
|
+
cfg.setdefault("wip_limit", 1)
|
|
2333
|
+
cfg.setdefault("refill_below", 3)
|
|
2334
|
+
return cfg
|
|
2335
|
+
|
|
2336
|
+
|
|
2337
|
+
def inbox_files(root):
|
|
2338
|
+
"""Normalised pulls waiting to be admitted: .polyctl/inbox/<source-id>.json."""
|
|
2339
|
+
d = os.path.join(root, ".polyctl", "inbox")
|
|
2340
|
+
if not os.path.isdir(d):
|
|
2341
|
+
return []
|
|
2342
|
+
return [os.path.join(d, f) for f in sorted(os.listdir(d)) if f.endswith(".json")]
|
|
2343
|
+
|
|
2344
|
+
|
|
2345
|
+
def binding_error(root, declared):
|
|
2346
|
+
"""Refuse work that belongs to another project.
|
|
2347
|
+
|
|
2348
|
+
A source is bound to exactly one repository. Without this check a tracker
|
|
2349
|
+
feeding project A can put an item on project B's board, and an agent that
|
|
2350
|
+
grinds boards without reading URLs will happily implement it there. The
|
|
2351
|
+
damage is silent and lands as a commit in the wrong repository, so the check
|
|
2352
|
+
belongs at the boundary rather than in anyone's memory.
|
|
2353
|
+
"""
|
|
2354
|
+
if not declared:
|
|
2355
|
+
return "item file declares no bound_root"
|
|
2356
|
+
if os.path.realpath(declared) != os.path.realpath(root):
|
|
2357
|
+
return f"bound to {declared}, not {root}"
|
|
2358
|
+
return None
|
|
2359
|
+
|
|
2360
|
+
|
|
2361
|
+
def plan_digest(root, item_id):
|
|
2362
|
+
"""Content hash of the plan an item is worked against, or None."""
|
|
2363
|
+
base = item_id.split("/")[0]
|
|
2364
|
+
p = os.path.join(root, ".polyctl", "plans", f"{base}.md")
|
|
2365
|
+
if not os.path.exists(p):
|
|
2366
|
+
return None
|
|
2367
|
+
h = hashlib.sha256()
|
|
2368
|
+
with open(p, "rb") as fh:
|
|
2369
|
+
for chunk in iter(lambda: fh.read(65536), b""):
|
|
2370
|
+
h.update(chunk)
|
|
2371
|
+
return "sha256:" + h.hexdigest()
|
|
2372
|
+
|
|
2373
|
+
|
|
2374
|
+
PLAN_CHAIN = "polyctl-plan-row-v1"
|
|
2375
|
+
|
|
2376
|
+
|
|
2377
|
+
def _plan_baselines(rows):
|
|
2378
|
+
"""Reject a missing baseline and an unrecorded replacement (TRIAGE-2-MEDIUM)."""
|
|
2379
|
+
from .storage import LedgerCorruption
|
|
2380
|
+
out = {}
|
|
2381
|
+
for rec in rows:
|
|
2382
|
+
if not isinstance(rec, dict) or not all(isinstance(rec.get(key), str) and rec[key]
|
|
2383
|
+
for key in ("item", "digest")):
|
|
2384
|
+
raise LedgerCorruption("plan baseline needs a nonempty item and digest; restore the ledger")
|
|
2385
|
+
item = rec["item"]
|
|
2386
|
+
if item in out and (rec.get("event") != "readmitted" or rec.get("replaces") != out[item]):
|
|
2387
|
+
raise LedgerCorruption(f"plan baseline {item} was replaced without re-admission; restore the ledger")
|
|
2388
|
+
out[item] = rec["digest"]
|
|
2389
|
+
return out
|
|
2390
|
+
|
|
2391
|
+
|
|
2392
|
+
def plan_baseline(root):
|
|
2393
|
+
"""Plan hashes at admission, chained from the legacy prefix (TRIAGE-2-MEDIUM)."""
|
|
2394
|
+
from .storage import read_chained_jsonl
|
|
2395
|
+
return _plan_baselines(read_chained_jsonl(os.path.join(root, ".polyctl", "ledger", "plans.jsonl"),
|
|
2396
|
+
PLAN_CHAIN, legacy_prefix=True))
|
|
2397
|
+
|
|
2398
|
+
|
|
2399
|
+
def plan_drift(root):
|
|
2400
|
+
"""Items whose plan changed after admission.
|
|
2401
|
+
|
|
2402
|
+
The implementer reads its plan; it does not write to it. When the document a
|
|
2403
|
+
slice is measured against can be edited by the thing being measured, every
|
|
2404
|
+
later check is circular. Recording the hash at admission turns that from an
|
|
2405
|
+
invisible failure into a line of output.
|
|
2406
|
+
"""
|
|
2407
|
+
base = plan_baseline(root)
|
|
2408
|
+
drifted = []
|
|
2409
|
+
for item, was in base.items():
|
|
2410
|
+
now = plan_digest(root, item)
|
|
2411
|
+
if now and now != was:
|
|
2412
|
+
drifted.append(item)
|
|
2413
|
+
return drifted
|
|
2414
|
+
|
|
2415
|
+
|
|
2416
|
+
def record_plan(root, item_id, digest):
|
|
2417
|
+
from .storage import append_chained_jsonl
|
|
2418
|
+
|
|
2419
|
+
def admission(rows):
|
|
2420
|
+
base = _plan_baselines(rows)
|
|
2421
|
+
_plan_baselines([{"item": item_id, "digest": digest}])
|
|
2422
|
+
return {"event": "readmitted", "replaces": base[item_id]} if item_id in base else {"event": "admitted"}
|
|
2423
|
+
|
|
2424
|
+
append_chained_jsonl(os.path.join(root, ".polyctl", "ledger", "plans.jsonl"),
|
|
2425
|
+
{"item": item_id, "digest": digest, "at": int(time.time())},
|
|
2426
|
+
PLAN_CHAIN, legacy_prefix=True, prepare=admission)
|
|
2427
|
+
|
|
2428
|
+
|
|
2429
|
+
SLICE_CHAIN = "polyctl-slice-row-v1"
|
|
2430
|
+
# The note in which a blocked item keeps the `needs:` it carried before the block, as the watchdog's quota park keeps
|
|
2431
|
+
# it (QUOTA_HELD there): the block's reason takes `needs:`, and the move that resumes the slice gives the old one back.
|
|
2432
|
+
HELD_NEEDS = "held-needs"
|
|
2433
|
+
|
|
2434
|
+
|
|
2435
|
+
def slice_ledger_path(root):
|
|
2436
|
+
return os.path.join(root, ".polyctl", "ledger", "slices.jsonl")
|
|
2437
|
+
|
|
2438
|
+
|
|
2439
|
+
def slice_moves(root, voided=False):
|
|
2440
|
+
"""Every move between the board's sections polyctl made, oldest first; raises on a broken chain (SLICE-LIFECYCLE).
|
|
2441
|
+
|
|
2442
|
+
A move is recorded before the board is written, and one whose write failed is followed by a line that voids
|
|
2443
|
+
it (`voids`: the ordinal of the line it voids, `why`: what stopped the write). Neither is a move the slice
|
|
2444
|
+
made, so both are left out: a reader that counts moves, takes the newest or says since when reads only what
|
|
2445
|
+
reached the board. `voided` keeps the voided moves in their place, each with the line that voids it as
|
|
2446
|
+
`void`, for a history that shows them (SLICE-LIFECYCLE-2).
|
|
2447
|
+
"""
|
|
2448
|
+
from .storage import read_chained_jsonl
|
|
2449
|
+
rows = [row for row in read_chained_jsonl(slice_ledger_path(root), SLICE_CHAIN) if isinstance(row, dict)]
|
|
2450
|
+
voids = {row["voids"]: row for row in rows if isinstance(row.get("voids"), int)}
|
|
2451
|
+
out = []
|
|
2452
|
+
for row in rows:
|
|
2453
|
+
if "voids" in row:
|
|
2454
|
+
continue
|
|
2455
|
+
void = voids.get(row.get("ordinal"))
|
|
2456
|
+
if void is not None and void.get("slice") == row.get("slice"):
|
|
2457
|
+
if voided:
|
|
2458
|
+
out.append(dict(row, void=void))
|
|
2459
|
+
continue
|
|
2460
|
+
out.append(row)
|
|
2461
|
+
return out
|
|
2462
|
+
|
|
2463
|
+
|
|
2464
|
+
def _line_notes(chunks):
|
|
2465
|
+
"""{key: value} of a board line's notes, as `board` reads them: a chunk with no colon is a key with no value."""
|
|
2466
|
+
out = {}
|
|
2467
|
+
for chunk in chunks:
|
|
2468
|
+
key, _, value = chunk.partition(":")
|
|
2469
|
+
if chunk and key.strip() not in out:
|
|
2470
|
+
out[key.strip()] = value.strip()
|
|
2471
|
+
return out
|
|
2472
|
+
|
|
2473
|
+
|
|
2474
|
+
def record_slice_move(root, item_id, before, after, by=None, why=None, notes=None, replaced=None, title=None):
|
|
2475
|
+
"""Append one move of a board item to the slice ledger: from one section to another, by whom, when and why.
|
|
2476
|
+
|
|
2477
|
+
A slice's state was the section its row sat in and nothing more: nothing said when it moved, who moved it
|
|
2478
|
+
or why, and a slice dropped from the board left no trace at all, although docs/slices.md calls abandoning
|
|
2479
|
+
one an outcome to record. `before` is None for an item that enters the board. `by` is the role of a turn
|
|
2480
|
+
polyctl started, or "person" for a caller polyctl did not start, unless the caller names itself (the watchdog does).
|
|
2481
|
+
`notes` are the notes the move set (None: removed) and `replaced` what they held before, so a boundary or a
|
|
2482
|
+
reason a move overwrote stays on the record; the title says which slice it was once its row is gone.
|
|
2483
|
+
Chained like every ledger that is evidence (SLICE-LIFECYCLE).
|
|
2484
|
+
"""
|
|
2485
|
+
from .storage import append_chained_jsonl
|
|
2486
|
+
row = {"slice": item_id, "from": before, "to": after, "by": by or invoking_role() or "person",
|
|
2487
|
+
"at": int(time.time()), "why": why}
|
|
2488
|
+
if title:
|
|
2489
|
+
row["title"] = title
|
|
2490
|
+
if notes:
|
|
2491
|
+
row["notes"] = dict(notes)
|
|
2492
|
+
if replaced:
|
|
2493
|
+
row["replaced"] = dict(replaced)
|
|
2494
|
+
return append_chained_jsonl(slice_ledger_path(root), scan_record(row), SLICE_CHAIN)
|
|
2495
|
+
|
|
2496
|
+
|
|
2497
|
+
class BoardChanged(RuntimeError):
|
|
2498
|
+
"""The board changed between polyctl's reading it and its writing it, so polyctl moved nothing (SLICE-LIFECYCLE-2)."""
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
def board_lock(root, timeout=10.0):
|
|
2502
|
+
"""The lock polyctl's own writers of the board hold from their reading of it to their write (SLICE-LIFECYCLE-2).
|
|
2503
|
+
|
|
2504
|
+
It sits beside the board, as a ledger's lock sits beside the ledger. The implementer's hand edits take no
|
|
2505
|
+
lock: BoardEdit compares the board for those.
|
|
2506
|
+
"""
|
|
2507
|
+
from .storage import _exclusive_lock
|
|
2508
|
+
return _exclusive_lock(os.path.join(root, ".polyctl", "board.md.lock"), timeout=timeout)
|
|
2509
|
+
|
|
2510
|
+
|
|
2511
|
+
def _board_bytes(path):
|
|
2512
|
+
"""What the board holds as it stands, or None when there is none."""
|
|
2513
|
+
try:
|
|
2514
|
+
with open(path, "rb") as fh:
|
|
2515
|
+
return fh.read()
|
|
2516
|
+
except FileNotFoundError:
|
|
2517
|
+
return None
|
|
2518
|
+
|
|
2519
|
+
|
|
2520
|
+
def _board_changed(recorded):
|
|
2521
|
+
said = ", and the slice ledger voids the move it had recorded" if recorded else " and nothing was recorded"
|
|
2522
|
+
return BoardChanged(f"the board changed while polyctl read it: .polyctl/board.md was edited after polyctl read it and before it "
|
|
2523
|
+
f"wrote it, so nothing moved{said}; run it again, and polyctl decides on the board as it stands")
|
|
2524
|
+
|
|
2525
|
+
|
|
2526
|
+
def _not_written(exc):
|
|
2527
|
+
"""Why a board write did not happen, in words a committed ledger may keep: no path, nothing of the machine."""
|
|
2528
|
+
if isinstance(exc, BoardChanged):
|
|
2529
|
+
return "the board was edited after polyctl read it and before polyctl wrote it; the edit stands"
|
|
2530
|
+
said = exc.strerror if isinstance(exc, OSError) and exc.strerror else ""
|
|
2531
|
+
return f"the board could not be written: {type(exc).__name__}" + (f": {said}" if said else "")
|
|
2532
|
+
|
|
2533
|
+
|
|
2534
|
+
def _void_slice_moves(root, rows, why):
|
|
2535
|
+
"""Append, for each recorded move the board never took, a line that voids it (SLICE-LIFECYCLE-2).
|
|
2536
|
+
|
|
2537
|
+
A move is recorded before the board is written, and a chained ledger is never cut back: a write that failed
|
|
2538
|
+
after the line left a transition that never happened, and a retry recorded a second one. So the line stays
|
|
2539
|
+
and the one after it says the move did not happen and why - `voids` is the ordinal of the line it voids - and
|
|
2540
|
+
`slice_moves` leaves both out of what the slice did.
|
|
2541
|
+
"""
|
|
2542
|
+
from .storage import append_chained_jsonl
|
|
2543
|
+
for row in rows:
|
|
2544
|
+
void = {"slice": row.get("slice"), "from": row.get("from"), "to": row.get("to"), "by": row.get("by"),
|
|
2545
|
+
"at": int(time.time()), "why": why, "voids": row.get("ordinal")}
|
|
2546
|
+
if row.get("title"):
|
|
2547
|
+
void["title"] = row["title"]
|
|
2548
|
+
append_chained_jsonl(slice_ledger_path(root), scan_record(void), SLICE_CHAIN)
|
|
2549
|
+
|
|
2550
|
+
|
|
2551
|
+
class BoardEdit:
|
|
2552
|
+
"""One reading of `.polyctl/board.md`, the moves and entries decided on it, and their one write (SLICE-LIFECYCLE-2).
|
|
2553
|
+
|
|
2554
|
+
`polyctl slice start` judged an item READY on one reading of the board and moved it on a later one, with no lock
|
|
2555
|
+
and no second look: an implementer that wrote `waiting: architect` on the row in between had polyctl record an
|
|
2556
|
+
authorized-looking start of a slice that was not READY. The watchdog's quota park, which decides on a reading
|
|
2557
|
+
and may wait on keyflip before it writes, moved a row from wherever a hand had put it meanwhile. A caller now
|
|
2558
|
+
decides on `board`, what this one reading holds (`data` is its bytes, for whatever else reads the board), and
|
|
2559
|
+
`move` and `append` change that same text: nothing is read again.
|
|
2560
|
+
|
|
2561
|
+
polyctl's own writers hold `board_lock` from their reading to their write (`hold=True`, in a `with`), so two of
|
|
2562
|
+
them take turns instead of each writing over the other's board. The watchdog reads without it, since its
|
|
2563
|
+
decision can wait minutes on keyflip, and its write takes it. The implementer's hand edits take no lock, so
|
|
2564
|
+
`write` looks at the board as it stands twice - before anything is recorded, and once the new board is on
|
|
2565
|
+
disk beside it, right before the rename puts it in place - and anything but the bytes this reading took, an
|
|
2566
|
+
edit or a board made or removed, moves nothing: `BoardChanged` says the board changed while polyctl read it, and
|
|
2567
|
+
the command is run again on the board as it stands.
|
|
2568
|
+
|
|
2569
|
+
What stays open is the instant between that last look and the rename: a read and a rename, measured on a
|
|
2570
|
+
laptop's disk at a third of a millisecond (under one at the 99th percentile; Windows adds the retries of a
|
|
2571
|
+
rename while another handle holds the board open), against the milliseconds a command's decision takes and
|
|
2572
|
+
the minutes the watchdog's can. An edit landing in it is replaced by polyctl's board and lost, though what polyctl
|
|
2573
|
+
decided on is still what stood at its last look. A plain file cannot close it: an editor takes no lock, an
|
|
2574
|
+
advisory lock binds only the processes that take it, and a rename replaces whatever the name holds at that
|
|
2575
|
+
moment - no file system polyctl runs on replaces a file only while it holds given bytes. Only edits made through
|
|
2576
|
+
polyctl could close it, and the board is the file the implementer edits by hand.
|
|
2577
|
+
"""
|
|
2578
|
+
|
|
2579
|
+
def __init__(self, root, hold=False):
|
|
2580
|
+
import contextlib
|
|
2581
|
+
import io
|
|
2582
|
+
self.root, self.path = root, os.path.join(root, ".polyctl", "board.md")
|
|
2583
|
+
# A directory with no .polyctl/ has no board to hold, and its lock is not made there: a write takes it.
|
|
2584
|
+
self.holding = hold and os.path.isdir(os.path.dirname(self.path))
|
|
2585
|
+
self.held, self.moves, self.changed = contextlib.ExitStack(), [], False
|
|
2586
|
+
try:
|
|
2587
|
+
if self.holding:
|
|
2588
|
+
self.held.enter_context(board_lock(root))
|
|
2589
|
+
self.data = _board_bytes(self.path)
|
|
2590
|
+
except BaseException:
|
|
2591
|
+
self.held.close()
|
|
2592
|
+
raise
|
|
2593
|
+
# Read as a move always read it: a byte that is not UTF-8 is written back as it was.
|
|
2594
|
+
self.lines = ["# Board", ""] if self.data is None else io.TextIOWrapper(
|
|
2595
|
+
io.BytesIO(self.data), errors="surrogateescape", encoding=UTF8).read().split("\n")
|
|
2596
|
+
self.board = board(root, self.data) if self.data is not None else {state: [] for state in BOARD_STATES}
|
|
2597
|
+
|
|
2598
|
+
def __enter__(self):
|
|
2599
|
+
return self
|
|
2600
|
+
|
|
2601
|
+
def __exit__(self, *raised):
|
|
2602
|
+
self.held.close()
|
|
2603
|
+
return False
|
|
2604
|
+
|
|
2605
|
+
def _bytes(self):
|
|
2606
|
+
return "\n".join(self.lines).encode(UTF8, errors="surrogateescape")
|
|
2607
|
+
|
|
2608
|
+
def _changed_to(self, lines):
|
|
2609
|
+
self.lines, self.changed = lines, True
|
|
2610
|
+
self.board = board(self.root, self._bytes())
|
|
2611
|
+
|
|
2612
|
+
def move(self, item_id, state, notes=None, by=None, why=None):
|
|
2613
|
+
"""Move one item's line under `## <state>`, its notes changed as `notes` says; False when it is not here.
|
|
2614
|
+
|
|
2615
|
+
Like an append, a move changes one line of what was read and leaves every other line as the implementer
|
|
2616
|
+
wrote it. It also takes that line out of its section, so the file is replaced whole: a reader finds the
|
|
2617
|
+
board before the move or after it, never half of each. A note given None is removed and any other is set;
|
|
2618
|
+
the title and the other notes stay as they were. An item moved to the section it is already in keeps its
|
|
2619
|
+
place there. The item is the first line with its id that `board` reads under a state's heading
|
|
2620
|
+
(QUOTA-PARK).
|
|
2621
|
+
|
|
2622
|
+
A byte that is not UTF-8 - an editor that saved the board in another code page - is read as `board`
|
|
2623
|
+
reads past it, and written back as it was: moving one line rewrites no other.
|
|
2624
|
+
|
|
2625
|
+
A move to another section is recorded in the slice ledger, by `by` and for `why`, when the board is
|
|
2626
|
+
written; a move within a section changes notes, and is no transition (SLICE-LIFECYCLE).
|
|
2627
|
+
"""
|
|
2628
|
+
lines, section, found = list(self.lines), None, None
|
|
2629
|
+
for index, line in enumerate(lines):
|
|
2630
|
+
if line.lstrip().startswith("##"):
|
|
2631
|
+
section = _board_heading(line)
|
|
2632
|
+
continue
|
|
2633
|
+
m = re.match(r"^\s*- \s*\[([^\]]+)\]\s*(.*)$", line)
|
|
2634
|
+
if section and m and m.group(1) == item_id:
|
|
2635
|
+
found = (index, section, m.group(2))
|
|
2636
|
+
break
|
|
2637
|
+
if found is None:
|
|
2638
|
+
return False
|
|
2639
|
+
index, section, rest = found
|
|
2640
|
+
chunks = [chunk.strip() for chunk in rest.split("·")]
|
|
2641
|
+
changed = dict(notes or {})
|
|
2642
|
+
kept = [chunk for chunk in chunks[1:] if chunk and chunk.partition(":")[0].strip() not in changed]
|
|
2643
|
+
line = " · ".join([f"- [{item_id}] {chunks[0]}".rstrip()] + kept
|
|
2644
|
+
+ [f"{key}: {value}" for key, value in changed.items() if value is not None])
|
|
2645
|
+
if section == state:
|
|
2646
|
+
lines[index] = line
|
|
2647
|
+
else:
|
|
2648
|
+
held = _line_notes(chunks[1:])
|
|
2649
|
+
self.moves.append({"item_id": item_id, "before": section, "after": state, "by": by, "why": why,
|
|
2650
|
+
"notes": changed, "title": chunks[0],
|
|
2651
|
+
"replaced": {key: held[key] for key in changed
|
|
2652
|
+
if held.get(key) and held[key] != changed[key]}})
|
|
2653
|
+
del lines[index]
|
|
2654
|
+
head = next((i for i, text in enumerate(lines) if _board_heading(text) == state), None)
|
|
2655
|
+
if head is None:
|
|
2656
|
+
while lines and not lines[-1].strip():
|
|
2657
|
+
lines.pop()
|
|
2658
|
+
lines += ["", f"## {state}", line, ""]
|
|
2659
|
+
else:
|
|
2660
|
+
lines.insert(head + 1, line)
|
|
2661
|
+
self._changed_to(lines)
|
|
2662
|
+
return True
|
|
2663
|
+
|
|
2664
|
+
def append(self, state, line, by=None, why=None):
|
|
2665
|
+
"""Put one item line first under the state's heading, refusing a board without it.
|
|
2666
|
+
|
|
2667
|
+
The rest of the board stays as the implementer wrote it. The item's entry is recorded in the slice
|
|
2668
|
+
ledger, with the title and the notes it entered with, when the board is written (SLICE-LIFECYCLE).
|
|
2669
|
+
A title, note or another heading containing `## <state>` is no section (SLICE-LIFECYCLE-3).
|
|
2670
|
+
"""
|
|
2671
|
+
lines = list(self.lines)
|
|
2672
|
+
head = next((i for i, text in enumerate(lines) if _board_heading(text) == state), None)
|
|
2673
|
+
if head is None:
|
|
2674
|
+
raise ValueError(f"the board has no ## {shown_text(state)} heading; nothing was appended: "
|
|
2675
|
+
"add that heading on its own line in .polyctl/board.md and run it again")
|
|
2676
|
+
if head == len(lines) - 1:
|
|
2677
|
+
lines.append("")
|
|
2678
|
+
lines.insert(head + 1, line.rstrip())
|
|
2679
|
+
entered = re.match(r"^\s*- \s*\[([^\]]+)\]\s*(.*)$", line.rstrip())
|
|
2680
|
+
if entered:
|
|
2681
|
+
chunks = [chunk.strip() for chunk in entered.group(2).split("·")]
|
|
2682
|
+
self.moves.append({"item_id": entered.group(1), "before": None, "after": state, "by": by, "why": why,
|
|
2683
|
+
"notes": _line_notes(chunks[1:]), "title": chunks[0]})
|
|
2684
|
+
self._changed_to(lines)
|
|
2685
|
+
|
|
2686
|
+
def write(self):
|
|
2687
|
+
"""Record what this edit moved and write the board once, if it still holds what was read; False: nothing to.
|
|
2688
|
+
|
|
2689
|
+
The moves are recorded first, as before: a ledger that cannot take a line leaves the board as it was. A
|
|
2690
|
+
board that cannot be written after them - a full disk, a permission, a rename refused, or a hand edit that
|
|
2691
|
+
landed meanwhile - leaves a line voiding each move it recorded, saying why, and the error goes to the
|
|
2692
|
+
caller: the ledger never holds a move the board did not make, and a retry records the move once
|
|
2693
|
+
(SLICE-LIFECYCLE-2). Only a directory that could not be synced after the rename leaves the moves standing,
|
|
2694
|
+
since the board took them.
|
|
2695
|
+
"""
|
|
2696
|
+
import contextlib
|
|
2697
|
+
from .storage import replace_file_durably
|
|
2698
|
+
data = self._bytes()
|
|
2699
|
+
if not self.moves and (not self.changed or data == self.data):
|
|
2700
|
+
return False
|
|
2701
|
+
with contextlib.ExitStack() as held:
|
|
2702
|
+
if not self.holding:
|
|
2703
|
+
held.enter_context(board_lock(self.root))
|
|
2704
|
+
if _board_bytes(self.path) != self.data:
|
|
2705
|
+
raise _board_changed(())
|
|
2706
|
+
recorded = []
|
|
2707
|
+
try:
|
|
2708
|
+
for move in self.moves:
|
|
2709
|
+
recorded.append(record_slice_move(self.root, **move))
|
|
2710
|
+
except BaseException as exc:
|
|
2711
|
+
_void_slice_moves(self.root, recorded, "the board was not written: the slice ledger refused another "
|
|
2712
|
+
f"move of the same write ({type(exc).__name__})")
|
|
2713
|
+
raise
|
|
2714
|
+
looked = []
|
|
2715
|
+
|
|
2716
|
+
def as_read():
|
|
2717
|
+
if _board_bytes(self.path) != self.data:
|
|
2718
|
+
raise _board_changed(recorded)
|
|
2719
|
+
looked.append(True)
|
|
2720
|
+
|
|
2721
|
+
try:
|
|
2722
|
+
replace_file_durably(self.path, data, before_replace=as_read)
|
|
2723
|
+
except BaseException as exc:
|
|
2724
|
+
if not (looked and _board_bytes(self.path) == data):
|
|
2725
|
+
_void_slice_moves(self.root, recorded, _not_written(exc))
|
|
2726
|
+
raise
|
|
2727
|
+
self.data, self.moves, self.changed = data, [], False
|
|
2728
|
+
return True
|
|
2729
|
+
|
|
2730
|
+
|
|
2731
|
+
def board_append(root, state, line, by=None, why=None):
|
|
2732
|
+
"""Append one item line under an existing `## <state>` heading: one BoardEdit, held and written.
|
|
2733
|
+
|
|
2734
|
+
Append rather than rewrite: the implementing agent edits this same file, and a write that replaced what it
|
|
2735
|
+
wrote between our read and our write would silently drop it. The edit's write refuses that instead
|
|
2736
|
+
(SLICE-LIFECYCLE-2), and records the item's entry in the slice ledger (SLICE-LIFECYCLE).
|
|
2737
|
+
"""
|
|
2738
|
+
with BoardEdit(root, hold=True) as edit:
|
|
2739
|
+
edit.append(state, line, by=by, why=why)
|
|
2740
|
+
edit.write()
|
|
2741
|
+
|
|
2742
|
+
|
|
2743
|
+
def board_move(root, item_id, state, notes=None, by=None, why=None):
|
|
2744
|
+
"""Move one item's line under `## <state>` as BoardEdit.move does; False when it is not on the board.
|
|
2745
|
+
|
|
2746
|
+
One BoardEdit, held and written (QUOTA-PARK, SLICE-LIFECYCLE): a caller that decides on what it read moves on
|
|
2747
|
+
that reading instead, so the move is made on the board the decision was made on (SLICE-LIFECYCLE-2).
|
|
2748
|
+
"""
|
|
2749
|
+
with BoardEdit(root, hold=True) as edit:
|
|
2750
|
+
moved = edit.move(item_id, state, notes, by=by, why=why)
|
|
2751
|
+
edit.write()
|
|
2752
|
+
return moved
|
|
2753
|
+
|
|
2754
|
+
|
|
2755
|
+
def record_progress(root, cfg):
|
|
2756
|
+
"""One line per watchdog check: what actually moved.
|
|
2757
|
+
|
|
2758
|
+
Cheap by construction — the watchdog already runs, and this is a git call
|
|
2759
|
+
plus an append. The point is to have *history* of the artifacts, because a
|
|
2760
|
+
single snapshot cannot tell activity from progress.
|
|
2761
|
+
"""
|
|
2762
|
+
msgs, _ = session_paths(cfg)
|
|
2763
|
+
# Coordination state is left out: the watchdog appends to this very ledger and
|
|
2764
|
+
# to its notices every cycle, and while those writes counted as editing the
|
|
2765
|
+
# spin check could never see a frozen run (#96).
|
|
2766
|
+
porcelain, churn = _product_changes(root, cfg)
|
|
2767
|
+
size = 0
|
|
2768
|
+
if msgs and os.path.exists(msgs):
|
|
2769
|
+
# A subagent that writes keeps its implementer working (`last_write`), so what it writes is the
|
|
2770
|
+
# transcript growing too: a subagent looping on nothing is busy without progress, not busy.
|
|
2771
|
+
shape = transcript_shape(implementer_adapter(cfg))
|
|
2772
|
+
size = os.path.getsize(msgs) + sum(grown for _, grown in subagent_writes(msgs, shape))
|
|
2773
|
+
# Content churn, not file count. A slice deep in editing its established file
|
|
2774
|
+
# set holds the dirty *count* stable for many minutes — same eight files, new
|
|
2775
|
+
# content each cycle — and a count-only check reads that as frozen and cries
|
|
2776
|
+
# spin. The newest mtime across the changed set advances on every edit, so it
|
|
2777
|
+
# tells editing apart from a genuine stall. A long gate run writes nothing, so
|
|
2778
|
+
# it correctly stays frozen, and the minute threshold covers that case.
|
|
2779
|
+
rec = {"at": int(time.time()),
|
|
2780
|
+
"head": _git_text(root, "rev-parse", "--short", "HEAD"),
|
|
2781
|
+
"dirty": len(porcelain), "churn": churn, "size": size}
|
|
2782
|
+
d = os.path.join(root, ".polyctl", "ledger")
|
|
2783
|
+
os.makedirs(d, exist_ok=True)
|
|
2784
|
+
p = os.path.join(d, "progress.jsonl")
|
|
2785
|
+
try:
|
|
2786
|
+
with open(p, encoding=UTF8) as fh:
|
|
2787
|
+
last = fh.readlines()[-1:]
|
|
2788
|
+
if last:
|
|
2789
|
+
prev = json.loads(last[0])
|
|
2790
|
+
if (prev.get("head"), prev.get("dirty"), prev.get("churn"), prev.get("size")) == \
|
|
2791
|
+
(rec["head"], rec["dirty"], rec["churn"], rec["size"]):
|
|
2792
|
+
return # nothing changed; do not log noise
|
|
2793
|
+
except Exception:
|
|
2794
|
+
pass
|
|
2795
|
+
with open(p, "a", encoding=UTF8) as fh:
|
|
2796
|
+
fh.write(json.dumps(rec) + "\n")
|
|
2797
|
+
|
|
2798
|
+
|
|
2799
|
+
def spinning(root, min_minutes=6, min_samples=3):
|
|
2800
|
+
"""Is the agent busy without producing anything?
|
|
2801
|
+
|
|
2802
|
+
An agent stuck in an observe-and-wait loop is the hardest failure to see,
|
|
2803
|
+
because every health signal is green: the transcript grows, tool calls fire,
|
|
2804
|
+
cost accrues. The watchdog never questions "working". What separates five
|
|
2805
|
+
productive turns from five turns of re-checking whether the tree is stable is
|
|
2806
|
+
not activity — it is whether any artifact moved.
|
|
2807
|
+
|
|
2808
|
+
So compare the two directly: transcript growing (busy) while HEAD and the
|
|
2809
|
+
dirty-file count hold still (nothing produced), sustained long enough that a
|
|
2810
|
+
slow gate cannot explain it. Returns minutes spent spinning, or None.
|
|
2811
|
+
|
|
2812
|
+
This exact failure cost roughly forty minutes in this project's own run,
|
|
2813
|
+
while the panel showed WORKING in green the entire time.
|
|
2814
|
+
|
|
2815
|
+
The defaults are deliberately low because of what this is used for. Reporting
|
|
2816
|
+
a false positive costs a line of output; acting on one costs a turn. So the
|
|
2817
|
+
*report* threshold is a few watchdog cycles, while anything that spends money
|
|
2818
|
+
or kills a process keeps its own, far more conservative bound. Set one
|
|
2819
|
+
threshold for both and you get the worst of each: too slow to be useful, and
|
|
2820
|
+
still not safe enough to act on.
|
|
2821
|
+
"""
|
|
2822
|
+
p = os.path.join(root, ".polyctl", "ledger", "progress.jsonl")
|
|
2823
|
+
if not os.path.exists(p):
|
|
2824
|
+
return None
|
|
2825
|
+
recs = []
|
|
2826
|
+
for line in open(p, errors="replace", encoding=UTF8):
|
|
2827
|
+
try:
|
|
2828
|
+
recs.append(json.loads(line))
|
|
2829
|
+
except Exception:
|
|
2830
|
+
continue
|
|
2831
|
+
recs = recs[-40:]
|
|
2832
|
+
if len(recs) < min_samples:
|
|
2833
|
+
return None
|
|
2834
|
+
# Frozen means nothing was produced AND nothing was edited: same HEAD, same
|
|
2835
|
+
# dirty count, and no newer mtime in the changed set. Editing the same files
|
|
2836
|
+
# advances churn, so it breaks the run and is not spin.
|
|
2837
|
+
head = recs[-1].get("head")
|
|
2838
|
+
dirty = recs[-1].get("dirty")
|
|
2839
|
+
churn = recs[-1].get("churn")
|
|
2840
|
+
run = [r for r in reversed(recs)
|
|
2841
|
+
if r.get("head") == head and r.get("dirty") == dirty and r.get("churn") == churn]
|
|
2842
|
+
if len(run) < min_samples:
|
|
2843
|
+
return None
|
|
2844
|
+
grew = run[0].get("size", 0) > run[-1].get("size", 0) # transcript still moving
|
|
2845
|
+
span = (run[0]["at"] - run[-1]["at"]) / 60
|
|
2846
|
+
return int(span) if grew and span >= min_minutes else None
|
|
2847
|
+
|
|
2848
|
+
|
|
2849
|
+
HOLD_FILE = ".polyctl/hold"
|
|
2850
|
+
|
|
2851
|
+
|
|
2852
|
+
def hold_state(root):
|
|
2853
|
+
"""Who has stopped this project's agent, and why. None when running free."""
|
|
2854
|
+
p = os.path.join(root, HOLD_FILE)
|
|
2855
|
+
if not os.path.exists(p):
|
|
2856
|
+
return None
|
|
2857
|
+
try:
|
|
2858
|
+
st = json.load(open(p, encoding=UTF8))
|
|
2859
|
+
except Exception:
|
|
2860
|
+
st = {"by": "unknown", "reason": "unreadable hold file"}
|
|
2861
|
+
st["minutes"] = int((time.time() - st.get("at", time.time())) / 60)
|
|
2862
|
+
return st
|
|
2863
|
+
|
|
2864
|
+
|
|
2865
|
+
def agent_pids(root, adapter, headless_only=False):
|
|
2866
|
+
"""Agent processes whose working directory is this repository.
|
|
2867
|
+
|
|
2868
|
+
Match on the process's cwd rather than its command line. The command line is
|
|
2869
|
+
unreliable — a long resume prompt gets truncated by `ps`, and the binary may
|
|
2870
|
+
be a bare `node` under a version manager — whereas the cwd is exactly the
|
|
2871
|
+
question being asked: is something editing *this* tree?
|
|
2872
|
+
|
|
2873
|
+
One pass over the process table through the platform API (procs.py): exact
|
|
2874
|
+
argument vectors, cwd and parentage in milliseconds. The shell path this
|
|
2875
|
+
replaced — `pgrep -f`, one `lsof` per candidate, `ps` output split on
|
|
2876
|
+
whitespace — took seconds and mistook a shell that mentioned the agent, and
|
|
2877
|
+
a runtime under "Application Support", for turns.
|
|
2878
|
+
|
|
2879
|
+
procs.TableUnreadable where the platform could not read its process table: the
|
|
2880
|
+
empty list that stood for it read as no turn in this tree, so `polyctl hold` stopped
|
|
2881
|
+
nothing and the watchdog could start a second turn (WINDOWS-TABLE-CLOSED).
|
|
2882
|
+
"""
|
|
2883
|
+
from . import procs
|
|
2884
|
+
names = set()
|
|
2885
|
+
for key in ("send", "resume"):
|
|
2886
|
+
argv = (adapter.get(key) or {}).get("argv") or []
|
|
2887
|
+
if argv:
|
|
2888
|
+
names.add(os.path.basename(argv[0]))
|
|
2889
|
+
names.update(agent_process_names())
|
|
2890
|
+
want = os.path.realpath(root)
|
|
2891
|
+
me = os.getpid()
|
|
2892
|
+
out = []
|
|
2893
|
+
for pid in procs.all_pids():
|
|
2894
|
+
if pid == me:
|
|
2895
|
+
continue
|
|
2896
|
+
av = procs.argv(pid)
|
|
2897
|
+
if not av or not _is_agent_process(pid, names, av):
|
|
2898
|
+
continue
|
|
2899
|
+
cw = procs.cwd(pid)
|
|
2900
|
+
if cw is None:
|
|
2901
|
+
# No cwd on this platform (Windows): the repository path on the command
|
|
2902
|
+
# line is the next best evidence that this turn works in this tree.
|
|
2903
|
+
if any(os.path.realpath(a.rstrip("/\\")) == want for a in av if os.path.isabs(a)):
|
|
2904
|
+
out.append(pid)
|
|
2905
|
+
elif os.path.realpath(cw) == want:
|
|
2906
|
+
out.append(pid)
|
|
2907
|
+
# Processes polyctl itself started inside the repo — the reviewer above all — are
|
|
2908
|
+
# agents by every other test and writers by none. Exclude them and their
|
|
2909
|
+
# descendants: a reviewer's runtime child would otherwise surface as a root.
|
|
2910
|
+
helpers = helper_pids(root)
|
|
2911
|
+
if helpers and out:
|
|
2912
|
+
table = _proc_table()
|
|
2913
|
+
|
|
2914
|
+
def under_helper(pid):
|
|
2915
|
+
seen = set()
|
|
2916
|
+
current = pid
|
|
2917
|
+
while current > 1 and current not in seen:
|
|
2918
|
+
if current in helpers:
|
|
2919
|
+
return True
|
|
2920
|
+
seen.add(current)
|
|
2921
|
+
current = table.get(current, (0, 0, ""))[0]
|
|
2922
|
+
return False
|
|
2923
|
+
out = [p for p in out if not under_helper(p)]
|
|
2924
|
+
out = sorted(set(out))
|
|
2925
|
+
if headless_only:
|
|
2926
|
+
# Never a human's interactive session. `polyctl hold` once stopped seven
|
|
2927
|
+
# processes in a repository; two were the orchestrator's own turn and
|
|
2928
|
+
# five were the owner's live Claude sessions, cut mid-work. A hold
|
|
2929
|
+
# exists to stop unattended turns — the ones started with an argument
|
|
2930
|
+
# their harness declares for that — and an interactive session, by
|
|
2931
|
+
# definition, has a person in it who did not ask to be stopped.
|
|
2932
|
+
out = [p for p in out if _is_headless(p)]
|
|
2933
|
+
return out
|
|
2934
|
+
|
|
2935
|
+
|
|
2936
|
+
def agent_process_names():
|
|
2937
|
+
"""Program names that are agent turns, as the shipped adapters declare them (`detect.processes`) (#76).
|
|
2938
|
+
|
|
2939
|
+
The package's adapters only: whether a process is a writer decides holds and
|
|
2940
|
+
nudges, and a layer an agent can write must not be able to hide one.
|
|
2941
|
+
"""
|
|
2942
|
+
return {str(name) for adapter in package_adapters().values()
|
|
2943
|
+
for name in (adapter.get("detect") or {}).get("processes") or [] if name}
|
|
2944
|
+
|
|
2945
|
+
|
|
2946
|
+
_WINDOWS_PROGRAM_SUFFIXES = (".exe", ".cmd", ".bat", ".com")
|
|
2947
|
+
|
|
2948
|
+
|
|
2949
|
+
def _executable(t):
|
|
2950
|
+
"""A real executable file at this path (a hook, so scenarios can fabricate a world).
|
|
2951
|
+
|
|
2952
|
+
Windows names the path with backslashes and has no execute bit - os.access(X_OK)
|
|
2953
|
+
holds for any file there - so a sibling agent binary such as C:\\...\\agent-chat.exe
|
|
2954
|
+
was never one, and its process was not taken for the agent's (#71). There a path
|
|
2955
|
+
takes either separator and a program is a file with a suffix Windows runs.
|
|
2956
|
+
"""
|
|
2957
|
+
t = str(t)
|
|
2958
|
+
if os.name == "nt":
|
|
2959
|
+
return ("\\" in t or "/" in t) and t.lower().endswith(_WINDOWS_PROGRAM_SUFFIXES) and os.path.isfile(t)
|
|
2960
|
+
return "/" in t and os.path.isfile(t) and os.access(t, os.X_OK)
|
|
2961
|
+
|
|
2962
|
+
|
|
2963
|
+
def _program_name(token):
|
|
2964
|
+
"""Case-folded command basename without a Windows PATHEXT suffix."""
|
|
2965
|
+
base = os.path.basename(str(token).replace("\\", "/")).lower()
|
|
2966
|
+
for suffix in _WINDOWS_PROGRAM_SUFFIXES:
|
|
2967
|
+
if base.endswith(suffix):
|
|
2968
|
+
return base[:-len(suffix)]
|
|
2969
|
+
return base
|
|
2970
|
+
|
|
2971
|
+
|
|
2972
|
+
def _is_agent_process(pid, names, argv=None):
|
|
2973
|
+
"""Is an agent binary actually on this command line?
|
|
2974
|
+
|
|
2975
|
+
Works on the argument *vector*: a path with a space is one argument. The
|
|
2976
|
+
program itself, a sibling executable (kiro-cli-chat), or a runtime (node…)
|
|
2977
|
+
running something under the agent's install directory count; a shell whose
|
|
2978
|
+
command text mentions the agent, or a file named after it, does not. Windows
|
|
2979
|
+
PATHEXT suffixes and separators are normalized before matching.
|
|
2980
|
+
"""
|
|
2981
|
+
from . import procs
|
|
2982
|
+
toks = argv if argv is not None else (procs.argv(pid) or [])
|
|
2983
|
+
if not toks:
|
|
2984
|
+
return False
|
|
2985
|
+
runtimes = ("node", "bun", "deno", "python", "python3")
|
|
2986
|
+
runtime = _program_name(toks[0]) in runtimes
|
|
2987
|
+
executable = _executable
|
|
2988
|
+
normalized_names = {_program_name(name) for name in names if name}
|
|
2989
|
+
for name in normalized_names:
|
|
2990
|
+
for i, token in enumerate(toks):
|
|
2991
|
+
base = _program_name(token)
|
|
2992
|
+
path = str(token).replace("\\", "/").lower()
|
|
2993
|
+
if base == name and (i == 0 or executable(token)):
|
|
2994
|
+
return True # the program itself
|
|
2995
|
+
if base.startswith(name + "-") and executable(token):
|
|
2996
|
+
return True # a sibling binary: kiro-cli-chat — a real executable
|
|
2997
|
+
if runtime and i <= 2 and f"/{name}/" in path:
|
|
2998
|
+
return True # a runtime under the agent's install dir
|
|
2999
|
+
return False
|
|
3000
|
+
|
|
3001
|
+
|
|
3002
|
+
# Programs that run another program an agent is: its identity is the program they run (JOURNAL-2-3).
|
|
3003
|
+
AGENT_RUNTIMES = frozenset({"node", "bun", "deno", "python", "python3"})
|
|
3004
|
+
|
|
3005
|
+
|
|
3006
|
+
def runtime_family(name):
|
|
3007
|
+
"""The runtime a program name is - python for `python3.12` too - or None (JOURNAL-2-5)."""
|
|
3008
|
+
name = _program_name(name)
|
|
3009
|
+
if re.fullmatch(r"python(\d+(\.\d+)*t?)?", name):
|
|
3010
|
+
return "python"
|
|
3011
|
+
return name if name in AGENT_RUNTIMES else None
|
|
3012
|
+
# What each runtime takes before the program it runs: options whose value is the next word, subcommands that name no
|
|
3013
|
+
# program, and options that run code given inline, which name none (JOURNAL-2-4).
|
|
3014
|
+
_RUNTIME_VALUE_OPTIONS = {
|
|
3015
|
+
"python": {"-W", "-X", "--check-hash-based-pycs"},
|
|
3016
|
+
"node": {"-r", "--require", "--import", "--loader", "--experimental-loader", "-C", "--conditions",
|
|
3017
|
+
"--env-file", "--title", "--input-type", "--inspect-port", "--stack-size"},
|
|
3018
|
+
"bun": {"-r", "--preload", "--cwd", "-c", "--config", "--env-file", "--tsconfig-override"},
|
|
3019
|
+
"deno": {"-c", "--config", "--import-map", "--lock", "--cert", "--location", "--seed", "--v8-flags",
|
|
3020
|
+
"--env-file", "--ext"},
|
|
3021
|
+
}
|
|
3022
|
+
_RUNTIME_SUBCOMMANDS = {"bun": {"run", "x", "exec"}, "deno": {"run", "x"}}
|
|
3023
|
+
_RUNTIME_INLINE = {"python": {"-c"}, "node": {"-e", "--eval", "-p", "--print"}, "bun": {"-e", "--eval", "-p", "--print"},
|
|
3024
|
+
"deno": {"eval"}}
|
|
3025
|
+
|
|
3026
|
+
|
|
3027
|
+
def runtime_program_word(argv):
|
|
3028
|
+
"""The word naming the program a runtime's command line runs, or None (JOURNAL-2-4).
|
|
3029
|
+
|
|
3030
|
+
`python -m claude` runs claude and `node --require x /opt/claude-code/cli.js` cli.js: the program is the module
|
|
3031
|
+
after `-m`, or the first word that is no option, no option's value and no subcommand. The second word was taken
|
|
3032
|
+
for it, so `python -m claude` named `-m`, and any `python -m` process carrying the session held every wake. Code
|
|
3033
|
+
given inline names no program.
|
|
3034
|
+
"""
|
|
3035
|
+
if not argv:
|
|
3036
|
+
return None
|
|
3037
|
+
family = runtime_family(argv[0])
|
|
3038
|
+
if family not in _RUNTIME_VALUE_OPTIONS:
|
|
3039
|
+
return None
|
|
3040
|
+
words, at = [str(word) for word in argv[1:]], 0
|
|
3041
|
+
while at < len(words):
|
|
3042
|
+
word = words[at]
|
|
3043
|
+
if family == "python" and word.startswith("-m"):
|
|
3044
|
+
return (word[2:] or (words[at + 1] if at + 1 < len(words) else "")) or None
|
|
3045
|
+
if word in _RUNTIME_INLINE[family] or family == "python" and word.startswith("-c"):
|
|
3046
|
+
return None
|
|
3047
|
+
if word in _RUNTIME_VALUE_OPTIONS[family]:
|
|
3048
|
+
at += 2
|
|
3049
|
+
elif word.startswith("-") or word in _RUNTIME_SUBCOMMANDS.get(family, ()):
|
|
3050
|
+
at += 1
|
|
3051
|
+
else:
|
|
3052
|
+
return word
|
|
3053
|
+
return None
|
|
3054
|
+
|
|
3055
|
+
|
|
3056
|
+
def _is_configured_agent_process(names, argv):
|
|
3057
|
+
"""Exact configured launcher or runtime-package identity for one agent role.
|
|
3058
|
+
|
|
3059
|
+
Writer discovery intentionally accepts sibling tools such as
|
|
3060
|
+
``kiro-cli-chat``. Architect presence must not: a reviewer or monitor that
|
|
3061
|
+
shares a prefix is a different role and cannot suppress a wake.
|
|
3062
|
+
"""
|
|
3063
|
+
if not argv:
|
|
3064
|
+
return False
|
|
3065
|
+
wanted = {_program_name(name) for name in names if name}
|
|
3066
|
+
if _program_name(argv[0]) in wanted:
|
|
3067
|
+
return True
|
|
3068
|
+
if runtime_family(argv[0]) is None:
|
|
3069
|
+
return False
|
|
3070
|
+
# A runtime is the program it runs, named by that word alone: an option before it, `-m` among them, named
|
|
3071
|
+
# no program, and matched one (JOURNAL-2-4).
|
|
3072
|
+
word = runtime_program_word(argv)
|
|
3073
|
+
components = {_program_name(part) for part in str(word or "").replace("\\", "/").split("/") if part}
|
|
3074
|
+
return bool(components & wanted)
|
|
3075
|
+
|
|
3076
|
+
|
|
3077
|
+
def _is_headless(pid):
|
|
3078
|
+
"""A turn started without a person in it, by what its own harness declares (`_headless_argv`)."""
|
|
3079
|
+
from . import procs
|
|
3080
|
+
return _headless_argv(procs.argv(pid) or [])
|
|
3081
|
+
|
|
3082
|
+
|
|
3083
|
+
def _headless_argv(argv):
|
|
3084
|
+
"""Does a command line hold an argument its own harness declares for a turn started without a person?
|
|
3085
|
+
|
|
3086
|
+
Each shipped adapter declares those arguments (`detect.headless`, #76): a flag, alone or
|
|
3087
|
+
with its value attached after `=`, or a subcommand. They were one list written here, and
|
|
3088
|
+
any of them made any agent's process unattended, but a flag belongs to the grammar of its
|
|
3089
|
+
own command - one harness prints a single answer with -p where another selects a profile
|
|
3090
|
+
with it. An argument counts only for the harness whose names, its adapter's id, binaries
|
|
3091
|
+
and processes, the command line runs as, and a command line no shipped adapter answers to
|
|
3092
|
+
is not one. The package's adapters only: `polyctl hold` and the reaper stop what this calls
|
|
3093
|
+
unattended, and a layer an agent can write must not make a person's session one.
|
|
3094
|
+
"""
|
|
3095
|
+
args = [str(arg) for arg in argv or []]
|
|
3096
|
+
for ident, adapter in sorted(package_adapters().items()):
|
|
3097
|
+
detect = _block(adapter, "detect")
|
|
3098
|
+
words = _names(detect.get("headless"))
|
|
3099
|
+
if not any(arg == word or arg.startswith(word + "=") for arg in args for word in words):
|
|
3100
|
+
continue
|
|
3101
|
+
if _is_agent_process(None, {ident, *adapter_binaries(adapter), *_names(detect.get("processes"))}, args):
|
|
3102
|
+
return True
|
|
3103
|
+
return False
|
|
3104
|
+
|
|
3105
|
+
|
|
3106
|
+
def _proc_table():
|
|
3107
|
+
"""pid -> (ppid, pgid, tty) for every process, from the platform API (see procs.py); procs.TableUnreadable
|
|
3108
|
+
where the platform could not read it, never an empty table (WINDOWS-TABLE-CLOSED)."""
|
|
3109
|
+
from . import procs
|
|
3110
|
+
return procs.table()
|
|
3111
|
+
|
|
3112
|
+
|
|
3113
|
+
def orphans(root, adapter, table=None):
|
|
3114
|
+
"""Agent processes left behind by a turn that already ended.
|
|
3115
|
+
|
|
3116
|
+
A turn is spawned in its own session (`start_new_session=True`), so the wrapper
|
|
3117
|
+
leads the process group and every child it starts — runtime, engine — inherits
|
|
3118
|
+
that group. When the wrapper dies and a child does not, the child is
|
|
3119
|
+
re-parented to init but keeps the dead leader's group id. That is the whole
|
|
3120
|
+
signature: no controlling terminal, and a process-group leader that no longer
|
|
3121
|
+
exists. Nothing a person is sitting in looks like that — a terminal session has
|
|
3122
|
+
a tty, a desktop-app session has the app as its live parent and leader.
|
|
3123
|
+
|
|
3124
|
+
These matter because they are invisible to the reaper (the headless flag is on
|
|
3125
|
+
the wrapper, not on them) and visible to every writer count. Three of them sat
|
|
3126
|
+
in one repository for hours at 0% CPU, and the implementer — correctly applying
|
|
3127
|
+
its single-writer rule to what the process table showed — refused to write for
|
|
3128
|
+
the entire time, while each of its empty turns tripped the reaper again and
|
|
3129
|
+
made one more.
|
|
3130
|
+
"""
|
|
3131
|
+
table = table or _proc_table()
|
|
3132
|
+
out = []
|
|
3133
|
+
for pid in agent_pids(root, adapter):
|
|
3134
|
+
ppid, pgid, tty = table.get(pid, (None, None, None))
|
|
3135
|
+
if pgid and tty == "??" and pgid != pid and pgid not in table:
|
|
3136
|
+
out.append(pid)
|
|
3137
|
+
return out
|
|
3138
|
+
|
|
3139
|
+
|
|
3140
|
+
def writers(root, adapter):
|
|
3141
|
+
"""Live turn roots in this tree, with orphans set aside.
|
|
3142
|
+
|
|
3143
|
+
Returns (roots, orphans). The number of roots is what a single-writer rule
|
|
3144
|
+
should count: one root per turn, however many processes the turn is made of,
|
|
3145
|
+
and none for what a finished turn left behind.
|
|
3146
|
+
"""
|
|
3147
|
+
table = _proc_table()
|
|
3148
|
+
dead = set(orphans(root, adapter, table))
|
|
3149
|
+
live = [p for p in agent_pids(root, adapter) if p not in dead]
|
|
3150
|
+
return process_trees(live, {pid: t[0] for pid, t in table.items()}), sorted(dead)
|
|
3151
|
+
|
|
3152
|
+
|
|
3153
|
+
def _tree_stopped():
|
|
3154
|
+
"""Forget the process snapshot taken before a tree was stopped on Windows (WINDOWS-LANE-5).
|
|
3155
|
+
|
|
3156
|
+
It lists the stopped tree for two seconds more, and a stopped process's directory can no longer be
|
|
3157
|
+
read: `polyctl hold` then named the turn it had just stopped as one it could not place, and exited 1.
|
|
3158
|
+
"""
|
|
3159
|
+
from . import procs
|
|
3160
|
+
procs.refresh()
|
|
3161
|
+
|
|
3162
|
+
|
|
3163
|
+
# How long a stopped process tree is given to be gone before what still runs of it is named (WINDOWS-STOP-PROVEN).
|
|
3164
|
+
STOP_PROOF_SECONDS = 5.0
|
|
3165
|
+
|
|
3166
|
+
|
|
3167
|
+
def process_tree(pid, table=None):
|
|
3168
|
+
"""`pid` and every process started under it, parent link by parent link, as the process table reads now: what a
|
|
3169
|
+
stop of the tree is proven against (WINDOWS-STOP-PROVEN).
|
|
3170
|
+
|
|
3171
|
+
`[pid]` alone where the table cannot be read. This process is never one of them: a hold run
|
|
3172
|
+
from inside a turn would otherwise name itself among what it could not stop. A process that
|
|
3173
|
+
started before the parent it names is not that parent's (_older_than).
|
|
3174
|
+
"""
|
|
3175
|
+
from . import procs
|
|
3176
|
+
if table is None:
|
|
3177
|
+
try:
|
|
3178
|
+
procs.refresh()
|
|
3179
|
+
table = _proc_table()
|
|
3180
|
+
except procs.TableUnreadable:
|
|
3181
|
+
table = {}
|
|
3182
|
+
children = {}
|
|
3183
|
+
for child, row in table.items():
|
|
3184
|
+
children.setdefault(row[0], []).append(child)
|
|
3185
|
+
tree, waiting = [], [pid]
|
|
3186
|
+
while waiting:
|
|
3187
|
+
current = waiting.pop(0)
|
|
3188
|
+
if current in tree or current == os.getpid():
|
|
3189
|
+
continue
|
|
3190
|
+
tree.append(current)
|
|
3191
|
+
waiting.extend(child for child in sorted(children.get(current, [])) if not _older_than(child, current))
|
|
3192
|
+
return tree
|
|
3193
|
+
|
|
3194
|
+
|
|
3195
|
+
def _older_than(child, parent):
|
|
3196
|
+
"""Whether `child` started more than a second before `parent`, so that the parent it names was an earlier process
|
|
3197
|
+
with the same pid (WINDOWS-STOP-PROVEN).
|
|
3198
|
+
|
|
3199
|
+
Windows keeps the pid of a process's parent after that parent exits, and hands the pid to a
|
|
3200
|
+
process started later: a turn given it would have counted an unrelated process of the person's
|
|
3201
|
+
among its own, and a hold would have stopped it, or told them to. False where either age cannot
|
|
3202
|
+
be read, and off Windows, where an orphan is handed to a parent that still runs.
|
|
3203
|
+
"""
|
|
3204
|
+
if os.name != "nt":
|
|
3205
|
+
return False
|
|
3206
|
+
from . import procs
|
|
3207
|
+
child_age, parent_age = procs.elapsed(child), procs.elapsed(parent)
|
|
3208
|
+
return child_age is not None and parent_age is not None and child_age > parent_age + 1
|
|
3209
|
+
|
|
3210
|
+
|
|
3211
|
+
def _still_running(pids, seconds=None):
|
|
3212
|
+
"""Those of `pids` that still run, each asked of the process itself until none does or `seconds` have passed,
|
|
3213
|
+
STOP_PROOF_SECONDS unless given (WINDOWS-STOP-PROVEN). A process neither Windows nor the table answers for is
|
|
3214
|
+
taken as running."""
|
|
3215
|
+
deadline = time.time() + (STOP_PROOF_SECONDS if seconds is None else seconds)
|
|
3216
|
+
while True:
|
|
3217
|
+
running = sorted(pid for pid in set(pids) if _pid_alive(pid))
|
|
3218
|
+
if not running or time.time() >= deadline:
|
|
3219
|
+
return running
|
|
3220
|
+
time.sleep(0.1)
|
|
3221
|
+
|
|
3222
|
+
|
|
3223
|
+
def stop_windows_tree(pid, table=None):
|
|
3224
|
+
"""Stop a process and every process it started, with Windows' own taskkill, and return what of that tree still
|
|
3225
|
+
runs after it (WINDOWS-NO-CWD-EXEC, WINDOWS-STOP-PROVEN).
|
|
3226
|
+
|
|
3227
|
+
Started by its bare name, taskkill was looked for in the current directory before Windows'
|
|
3228
|
+
own, and polyctl stops turns from inside a tree an agent writes: a taskkill.exe left there would
|
|
3229
|
+
have run as the person in its place. taskkill /T finds a process's children by the parent
|
|
3230
|
+
each names, and is refused a process another user or an elevated prompt started; whatever
|
|
3231
|
+
it answered, `polyctl hold` said the tree was stopped. Each process of the tree as the table read
|
|
3232
|
+
it before the stop is now named to taskkill, not its first alone - a child whose parent had
|
|
3233
|
+
exited by then, a turn's wrapper ending as it was stopped, was found by no parent and left
|
|
3234
|
+
running - and each is asked whether it runs, until none does or STOP_PROOF_SECONDS have
|
|
3235
|
+
passed, and what still runs is returned. FileNotFoundError where Windows' system directory
|
|
3236
|
+
holds no taskkill, as a taskkill found nowhere raised before.
|
|
3237
|
+
"""
|
|
3238
|
+
taskkill = program_path("taskkill")
|
|
3239
|
+
if taskkill is None:
|
|
3240
|
+
raise FileNotFoundError(f"taskkill is not in Windows' system directory {windows_system_directory()}")
|
|
3241
|
+
tree = process_tree(pid, table)
|
|
3242
|
+
named = [pid] + [other for other in tree if other != pid]
|
|
3243
|
+
subprocess.run([taskkill, "/T", "/F"] + [word for each in named for word in ("/PID", str(each))],
|
|
3244
|
+
capture_output=True)
|
|
3245
|
+
return _still_running(tree)
|
|
3246
|
+
|
|
3247
|
+
|
|
3248
|
+
def kill_turn(pid, sig):
|
|
3249
|
+
"""Signal a turn — the whole process group when this pid leads one.
|
|
3250
|
+
|
|
3251
|
+
Signalling only the wrapper is how orphans are made: it exits, its runtime and
|
|
3252
|
+
engine children do not, and they keep the repository as their cwd. A turn we
|
|
3253
|
+
started is its own session, so its pid is its group id and one killpg reaches
|
|
3254
|
+
everything it spawned. A pid that leads no group is signalled alone. On
|
|
3255
|
+
Windows there are no groups to signal; `taskkill /T` kills the tree.
|
|
3256
|
+
|
|
3257
|
+
Returns what of the turn still runs after a forced stop - taskkill on Windows,
|
|
3258
|
+
SIGKILL elsewhere - each process of it asked: a stop that left something running
|
|
3259
|
+
was reported as done (WINDOWS-STOP-PROVEN). A turn only signalled, and given time
|
|
3260
|
+
to answer, returns []: its caller waits, and looks.
|
|
3261
|
+
"""
|
|
3262
|
+
import signal as _sig
|
|
3263
|
+
if os.name == "nt":
|
|
3264
|
+
survivors = stop_windows_tree(pid)
|
|
3265
|
+
_tree_stopped()
|
|
3266
|
+
return survivors
|
|
3267
|
+
forced = sig == getattr(_sig, "SIGKILL", None)
|
|
3268
|
+
tree = process_tree(pid) if forced else []
|
|
3269
|
+
try:
|
|
3270
|
+
pgid = os.getpgid(pid)
|
|
3271
|
+
except OSError:
|
|
3272
|
+
return []
|
|
3273
|
+
try:
|
|
3274
|
+
if pgid == pid:
|
|
3275
|
+
os.killpg(pgid, sig)
|
|
3276
|
+
else:
|
|
3277
|
+
os.kill(pid, sig)
|
|
3278
|
+
except OSError:
|
|
3279
|
+
pass
|
|
3280
|
+
if forced:
|
|
3281
|
+
# A process of the tree that moved into a group of its own is not reached by the group's signal: it ran on,
|
|
3282
|
+
# named among what was left but never stopped. Each is signalled by its pid (LEADS-GATE-RUNS).
|
|
3283
|
+
for member in tree:
|
|
3284
|
+
try:
|
|
3285
|
+
if member != pid and os.getpgid(member) != pgid:
|
|
3286
|
+
os.kill(member, sig)
|
|
3287
|
+
except OSError:
|
|
3288
|
+
pass
|
|
3289
|
+
return _still_running(tree) if forced else []
|
|
3290
|
+
|
|
3291
|
+
|
|
3292
|
+
def run_as_group(argv, timeout, **popen):
|
|
3293
|
+
"""subprocess.run(argv, timeout=timeout, **popen), the program leading a session and a process group of its own,
|
|
3294
|
+
so that a timeout or an interrupt stops it with what can be found of it: its process tree, its group and, on
|
|
3295
|
+
POSIX, its session, read again until nothing of it is found - a worker that moved into a group of its own is
|
|
3296
|
+
reached by its session, its parent gone or not (LEADS-GATE-RUNS). A cleanup, not a boundary: a process that
|
|
3297
|
+
started a session of its own, or one forking faster than the table is read, outlives it, and a gate is code the
|
|
3298
|
+
implementer wrote, run as theirs by design (docs/threat-model.md).
|
|
3299
|
+
|
|
3300
|
+
subprocess.run kills the program alone. A gate's shell was killed on its timeout and the test
|
|
3301
|
+
workers it had started ran on, after `polyctl verify` had let go of the machine lock they ran under;
|
|
3302
|
+
the hook Git ran for the execution proof outlived Git the same way, holding the pipes polyctl read.
|
|
3303
|
+
On Windows subprocess.run then reads those pipes until their last holder closes them, so the
|
|
3304
|
+
timeout bounded nothing. The tree is stopped with kill_turn, and on POSIX its group by its id
|
|
3305
|
+
as well: macOS names no group for a leader that has exited while what it started runs on, and
|
|
3306
|
+
the id is the group's until the leader is reaped, which is done here after the stop. What the
|
|
3307
|
+
program wrote is then read for STOP_PROOF_SECONDS at most. TimeoutExpired carries it, `left`,
|
|
3308
|
+
what of the tree still runs after the stop (WINDOWS-STOP-PROVEN), and `unasked`, why the tree could
|
|
3309
|
+
be neither stopped nor asked after where that is so; an interrupt is raised again once the tree is
|
|
3310
|
+
stopped. `capture_output` and `input` are taken as subprocess.run takes them.
|
|
3311
|
+
"""
|
|
3312
|
+
import signal
|
|
3313
|
+
if popen.pop("capture_output", False):
|
|
3314
|
+
popen.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
3315
|
+
given = popen.pop("input", None)
|
|
3316
|
+
if given is not None:
|
|
3317
|
+
popen["stdin"] = subprocess.PIPE
|
|
3318
|
+
group = ({"creationflags": getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)} if os.name == "nt"
|
|
3319
|
+
else {"start_new_session": True})
|
|
3320
|
+
proc = subprocess.Popen(argv, **popen, **group)
|
|
3321
|
+
try:
|
|
3322
|
+
out, err = proc.communicate(given, timeout=timeout)
|
|
3323
|
+
except (subprocess.TimeoutExpired, KeyboardInterrupt, SystemExit) as stopped:
|
|
3324
|
+
unasked = None
|
|
3325
|
+
try:
|
|
3326
|
+
left = kill_turn(proc.pid, getattr(signal, "SIGKILL", signal.SIGTERM)) or []
|
|
3327
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
3328
|
+
# No taskkill, or one that could not run: the program alone is ended, below, and what it started is
|
|
3329
|
+
# neither stopped nor asked after. That is said, never read as nothing left (LEADS-GATE).
|
|
3330
|
+
left, unasked = [], f"what it started could not be stopped or asked after ({' '.join(str(exc).split())})"
|
|
3331
|
+
if os.name != "nt":
|
|
3332
|
+
try:
|
|
3333
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
3334
|
+
except OSError:
|
|
3335
|
+
pass
|
|
3336
|
+
# The program leads a session of its own: a worker that moved into a group of its own and lost its parent
|
|
3337
|
+
# is in neither the tree nor the group, and ran on, named nowhere. The session is read and stopped again
|
|
3338
|
+
# until a reading finds nobody or STOP_PROOF_SECONDS have passed, while the program's pid is still its,
|
|
3339
|
+
# unreaped, and what it leaves is named; a table that cannot be read is said (LEADS-GATE-RUNS). It is a
|
|
3340
|
+
# cleanup, not a boundary: a gate is the implementer's code, and one written to outlive it can
|
|
3341
|
+
# (docs/threat-model.md).
|
|
3342
|
+
strays, deadline = set(), time.time() + STOP_PROOF_SECONDS
|
|
3343
|
+
while True:
|
|
3344
|
+
found = _session_members(proc.pid)
|
|
3345
|
+
if found is None:
|
|
3346
|
+
unasked = "; ".join(filter(None, [unasked, "the process table could not be read for the rest of "
|
|
3347
|
+
"its session, which may still run"]))
|
|
3348
|
+
break
|
|
3349
|
+
for pid in found:
|
|
3350
|
+
try:
|
|
3351
|
+
os.kill(pid, signal.SIGKILL)
|
|
3352
|
+
except OSError:
|
|
3353
|
+
pass
|
|
3354
|
+
strays.update(found)
|
|
3355
|
+
if not found or time.time() >= deadline:
|
|
3356
|
+
break
|
|
3357
|
+
time.sleep(0.05)
|
|
3358
|
+
left = sorted(set(left) | set(_still_running(strays)))
|
|
3359
|
+
try:
|
|
3360
|
+
proc.kill()
|
|
3361
|
+
except OSError:
|
|
3362
|
+
pass
|
|
3363
|
+
out = err = None
|
|
3364
|
+
if isinstance(stopped, subprocess.TimeoutExpired):
|
|
3365
|
+
try:
|
|
3366
|
+
out, err = proc.communicate(timeout=STOP_PROOF_SECONDS)
|
|
3367
|
+
except (OSError, ValueError, subprocess.TimeoutExpired):
|
|
3368
|
+
pass # a holder outside the group keeps the pipes: closed below
|
|
3369
|
+
for stream in (proc.stdin, proc.stdout, proc.stderr):
|
|
3370
|
+
if stream is not None:
|
|
3371
|
+
try:
|
|
3372
|
+
stream.close()
|
|
3373
|
+
except OSError:
|
|
3374
|
+
pass
|
|
3375
|
+
try:
|
|
3376
|
+
proc.wait(timeout=STOP_PROOF_SECONDS)
|
|
3377
|
+
except subprocess.TimeoutExpired:
|
|
3378
|
+
pass
|
|
3379
|
+
if not isinstance(stopped, subprocess.TimeoutExpired):
|
|
3380
|
+
raise
|
|
3381
|
+
expired = subprocess.TimeoutExpired(argv, timeout, output=out, stderr=err)
|
|
3382
|
+
expired.left = left
|
|
3383
|
+
expired.unasked = unasked
|
|
3384
|
+
raise expired from None
|
|
3385
|
+
except BaseException:
|
|
3386
|
+
# Anything else that broke the reading ends the program alone, as subprocess.run ends it: the tree is stopped
|
|
3387
|
+
# for a timeout or an interrupt, what this is for (LEADS-GATE-RUNS).
|
|
3388
|
+
try:
|
|
3389
|
+
proc.kill()
|
|
3390
|
+
except Exception:
|
|
3391
|
+
pass
|
|
3392
|
+
raise
|
|
3393
|
+
return subprocess.CompletedProcess(argv, proc.returncode, out, err)
|
|
3394
|
+
|
|
3395
|
+
|
|
3396
|
+
def _session_members(sid):
|
|
3397
|
+
"""Every process of session `sid` the process table names, but this one; None where the table cannot be read
|
|
3398
|
+
(LEADS-GATE-RUNS)."""
|
|
3399
|
+
from . import procs
|
|
3400
|
+
try:
|
|
3401
|
+
procs.refresh()
|
|
3402
|
+
pids = procs.all_pids()
|
|
3403
|
+
except procs.TableUnreadable:
|
|
3404
|
+
return None
|
|
3405
|
+
members = []
|
|
3406
|
+
for pid in pids:
|
|
3407
|
+
if pid == os.getpid():
|
|
3408
|
+
continue
|
|
3409
|
+
try:
|
|
3410
|
+
if os.getsid(pid) == sid:
|
|
3411
|
+
members.append(pid)
|
|
3412
|
+
except OSError:
|
|
3413
|
+
pass
|
|
3414
|
+
return members
|
|
3415
|
+
|
|
3416
|
+
|
|
3417
|
+
def sweep_orphans(pids, grace=3.0):
|
|
3418
|
+
"""Stop orphaned agent processes by their (leaderless) groups.
|
|
3419
|
+
|
|
3420
|
+
Each orphan still carries the group id of the turn that made it, and its own
|
|
3421
|
+
children carry the same one, so signalling the group clears the whole remnant
|
|
3422
|
+
at once — a child re-parented to an orphan would otherwise be orphaned a second
|
|
3423
|
+
time by the very cleanup. Returns what of the remnant still runs after it, each
|
|
3424
|
+
process asked, where it returned the pids it was handed whatever became of them
|
|
3425
|
+
(WINDOWS-STOP-PROVEN).
|
|
3426
|
+
"""
|
|
3427
|
+
import signal as _sig
|
|
3428
|
+
from . import procs
|
|
3429
|
+
try:
|
|
3430
|
+
procs.refresh()
|
|
3431
|
+
table = _proc_table()
|
|
3432
|
+
except procs.TableUnreadable:
|
|
3433
|
+
table = {}
|
|
3434
|
+
if os.name == "nt":
|
|
3435
|
+
survivors = []
|
|
3436
|
+
for pid in pids:
|
|
3437
|
+
survivors += stop_windows_tree(pid, table)
|
|
3438
|
+
_tree_stopped()
|
|
3439
|
+
return sorted(set(survivors))
|
|
3440
|
+
groups = set()
|
|
3441
|
+
for pid in pids:
|
|
3442
|
+
try:
|
|
3443
|
+
groups.add(os.getpgid(pid))
|
|
3444
|
+
except OSError:
|
|
3445
|
+
pass
|
|
3446
|
+
for g in groups:
|
|
3447
|
+
try:
|
|
3448
|
+
os.killpg(g, _sig.SIGTERM)
|
|
3449
|
+
except OSError:
|
|
3450
|
+
pass
|
|
3451
|
+
deadline = time.time() + grace
|
|
3452
|
+
while time.time() < deadline and any(_pid_alive(p) for p in pids):
|
|
3453
|
+
time.sleep(0.25)
|
|
3454
|
+
for g in groups:
|
|
3455
|
+
try:
|
|
3456
|
+
os.killpg(g, _sig.SIGKILL)
|
|
3457
|
+
except OSError:
|
|
3458
|
+
pass
|
|
3459
|
+
return _still_running(set(pids) | {pid for pid, row in table.items() if row[1] in groups})
|
|
3460
|
+
|
|
3461
|
+
|
|
3462
|
+
def _pid_alive(pid):
|
|
3463
|
+
try:
|
|
3464
|
+
pid = int(pid)
|
|
3465
|
+
except (TypeError, ValueError):
|
|
3466
|
+
return False
|
|
3467
|
+
if os.name == "nt":
|
|
3468
|
+
# On Windows signal 0 is CTRL_C_EVENT, not a harmless existence probe.
|
|
3469
|
+
# Sending it to the lock owner interrupts the very gate run whose
|
|
3470
|
+
# liveness we are checking. The process itself is asked instead: the
|
|
3471
|
+
# snapshot is kept two seconds, and a gate lock taken by a run started
|
|
3472
|
+
# inside them read as a dead run's and was cleared (WINDOWS-PID-ALIVE).
|
|
3473
|
+
# Where Windows does not answer, a fresh snapshot does; where neither
|
|
3474
|
+
# answers, the process is taken as running. A table PowerShell could not
|
|
3475
|
+
# read listed no process, and a lock its holder kept would read as a dead
|
|
3476
|
+
# run's: what waits on a process waits on, and what holds it holds, until
|
|
3477
|
+
# one of them can say (WINDOWS-TABLE-CLOSED).
|
|
3478
|
+
from . import procs
|
|
3479
|
+
answer = procs.alive(pid)
|
|
3480
|
+
if answer is not None:
|
|
3481
|
+
return answer
|
|
3482
|
+
try:
|
|
3483
|
+
procs.refresh()
|
|
3484
|
+
return pid in set(procs.all_pids())
|
|
3485
|
+
except (procs.TableUnreadable, OSError, subprocess.SubprocessError, ValueError):
|
|
3486
|
+
return True
|
|
3487
|
+
try:
|
|
3488
|
+
os.kill(pid, 0)
|
|
3489
|
+
except OSError:
|
|
3490
|
+
return False
|
|
3491
|
+
# A zombie answers the probe and does nothing: it has exited, only its pid is left.
|
|
3492
|
+
from . import procs
|
|
3493
|
+
return not procs.zombie(pid)
|
|
3494
|
+
|
|
3495
|
+
|
|
3496
|
+
def _process_start(pid, refresh=False, strict=False):
|
|
3497
|
+
"""Stable process-start identity, optionally forcing a fresh backend scan; None where it cannot be read.
|
|
3498
|
+
|
|
3499
|
+
A process table the platform could not read holds no start either, and reads as None, which
|
|
3500
|
+
every caller takes for a start it cannot prove: a lane's command is unknown and not stopped, a
|
|
3501
|
+
helper is no helper, a step's process is unproven, a turn is believed only for a bounded time.
|
|
3502
|
+
With `strict` it raises procs.TableUnreadable instead, for a caller that must not take a
|
|
3503
|
+
process it cannot see for one that has gone (WINDOWS-TABLE-CLOSED).
|
|
3504
|
+
"""
|
|
3505
|
+
from . import procs
|
|
3506
|
+
# A fresh Windows snapshot lists every live process, so a pid it lacks has exited, and
|
|
3507
|
+
# each retry there is one more PowerShell query of a second or more (#71).
|
|
3508
|
+
attempts = 3 if refresh and os.name != "nt" else 1
|
|
3509
|
+
for attempt in range(attempts):
|
|
3510
|
+
try:
|
|
3511
|
+
if refresh:
|
|
3512
|
+
procs.refresh()
|
|
3513
|
+
start = (procs.info(int(pid)) or {}).get("start")
|
|
3514
|
+
except procs.TableUnreadable:
|
|
3515
|
+
if strict:
|
|
3516
|
+
raise
|
|
3517
|
+
start = None
|
|
3518
|
+
except (OSError, TypeError, ValueError):
|
|
3519
|
+
start = None
|
|
3520
|
+
if start not in (None, 0, ""):
|
|
3521
|
+
return start
|
|
3522
|
+
if refresh and attempt + 1 < attempts:
|
|
3523
|
+
time.sleep(0.02)
|
|
3524
|
+
return None
|
|
3525
|
+
|
|
3526
|
+
|
|
3527
|
+
PROJECTION_CHECKS = ("burn_rate",)
|
|
3528
|
+
|
|
3529
|
+
|
|
3530
|
+
def notice_evidence(check, samples, source=None):
|
|
3531
|
+
"""What a notice was raised on: the check, and each sample's value, source and time (#37).
|
|
3532
|
+
|
|
3533
|
+
The owner received a quota-looking alert for a limit that did not exist and had no
|
|
3534
|
+
way to ask why. A projection is only as good as the readings it projected from, so
|
|
3535
|
+
one cannot be recorded without them.
|
|
3536
|
+
"""
|
|
3537
|
+
rows = [{"value": sample.get("value"), "source": sample.get("source") or source,
|
|
3538
|
+
"at": sample.get("at")} for sample in samples or []]
|
|
3539
|
+
if check in PROJECTION_CHECKS and not rows:
|
|
3540
|
+
raise ValueError(f"a {check} notice needs the samples it projected from")
|
|
3541
|
+
return {"check": check, "samples": rows}
|
|
3542
|
+
|
|
3543
|
+
|
|
3544
|
+
def record_notice(root, title, msg, sent, key=None, evidence=None, named=None):
|
|
3545
|
+
"""Every notification we raise, kept where the architect can read it.
|
|
3546
|
+
|
|
3547
|
+
A desktop notification is fire-and-forget: it reaches the human and vanishes,
|
|
3548
|
+
so the one participant who could act on a pattern of alerts — the architect
|
|
3549
|
+
reading the panel — is the only one who never sees them. Recording them puts
|
|
3550
|
+
both sides on the same evidence.
|
|
3551
|
+
|
|
3552
|
+
`sent=False` rows matter as much as sent ones: they are the alerts a human
|
|
3553
|
+
would have received without the rate limit, and a long run of them is itself
|
|
3554
|
+
the signal that something has been wrong for a while.
|
|
3555
|
+
"""
|
|
3556
|
+
d = os.path.join(root, ".polyctl", "ledger")
|
|
3557
|
+
try:
|
|
3558
|
+
os.makedirs(d, exist_ok=True)
|
|
3559
|
+
with open(os.path.join(d, "notices.jsonl"), "a", encoding=UTF8) as fh:
|
|
3560
|
+
row = {"id": f"N-{unique_ms()}", "at": int(time.time()), "title": title,
|
|
3561
|
+
"msg": msg, "sent": bool(sent), "key": key or title}
|
|
3562
|
+
if evidence:
|
|
3563
|
+
row["evidence"] = evidence
|
|
3564
|
+
if named:
|
|
3565
|
+
row["named"] = list(named) # the conditions one resume notice told (RESUME-QUIET)
|
|
3566
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
3567
|
+
# Folded before the bound can trim it, so a window still counts it (NOTICE-WINDOW), and observation is
|
|
3568
|
+
# held to its bound as it is written (#50); a fold that could not be written trims nothing (NOTICE-WINDOW-2).
|
|
3569
|
+
if fold_notice_times(root):
|
|
3570
|
+
bound_store(os.path.join(d, "notices.jsonl"), settings.get(load_config(root), "retention.observation_kb"))
|
|
3571
|
+
except OSError:
|
|
3572
|
+
pass
|
|
3573
|
+
|
|
3574
|
+
|
|
3575
|
+
def evidence_lines(evidence, now=None):
|
|
3576
|
+
"""A notice's evidence as lines a person reads: the check, then each sample and its age."""
|
|
3577
|
+
if not evidence:
|
|
3578
|
+
return ["no evidence recorded (raised before notices kept it, or by a check that has none)"]
|
|
3579
|
+
now = now or time.time()
|
|
3580
|
+
lines = [f"check: {evidence.get('check')}"]
|
|
3581
|
+
for sample in evidence.get("samples") or []:
|
|
3582
|
+
at = sample.get("at")
|
|
3583
|
+
age = f"{int((now - at) / 60)}m ago" if isinstance(at, (int, float)) else "time unknown"
|
|
3584
|
+
lines.append(f" {sample.get('value')} from {sample.get('source') or '?'}, {age}")
|
|
3585
|
+
return lines
|
|
3586
|
+
|
|
3587
|
+
|
|
3588
|
+
def notices(root, limit=10, include_suppressed=False):
|
|
3589
|
+
"""Recent notifications, newest first."""
|
|
3590
|
+
p = os.path.join(root, ".polyctl", "ledger", "notices.jsonl")
|
|
3591
|
+
if not os.path.exists(p):
|
|
3592
|
+
return []
|
|
3593
|
+
out = []
|
|
3594
|
+
for line in open(p, errors="replace", encoding=UTF8):
|
|
3595
|
+
try:
|
|
3596
|
+
rec = json.loads(line)
|
|
3597
|
+
except Exception:
|
|
3598
|
+
continue
|
|
3599
|
+
if rec.get("sent") or include_suppressed:
|
|
3600
|
+
out.append(rec)
|
|
3601
|
+
return list(reversed(out))[:limit]
|
|
3602
|
+
|
|
3603
|
+
|
|
3604
|
+
# ---- a window read whole: when each notice key was last recorded and sent (NOTICE-WINDOW) --------
|
|
3605
|
+
|
|
3606
|
+
# A key not recorded for a week is forgotten, and no more than 500 keys are kept: the longest
|
|
3607
|
+
# window a check asks is a day, and a week of conditions names far fewer keys. A key younger
|
|
3608
|
+
# than that day is kept past the 500: the ledger the check would read for it in its place
|
|
3609
|
+
# may already be trimmed of it (NOTICE-WINDOW-2).
|
|
3610
|
+
NOTICE_TIMES_DAYS = 7
|
|
3611
|
+
NOTICE_TIMES_KEYS = 500
|
|
3612
|
+
NOTICE_TIMES_HOLD = 86400
|
|
3613
|
+
|
|
3614
|
+
|
|
3615
|
+
def notice_times_path(root):
|
|
3616
|
+
"""When each notice key was last recorded and last sent, beside the ledger it is folded from."""
|
|
3617
|
+
return os.path.join(root, ".polyctl", "ledger", "notice-times.json")
|
|
3618
|
+
|
|
3619
|
+
|
|
3620
|
+
def _notice_rows_back(path, chunk=65536):
|
|
3621
|
+
"""(digest, row) for each line of the notices ledger, newest first, read a chunk at a time.
|
|
3622
|
+
|
|
3623
|
+
The digest names a line whatever its line ending; a line that is not a JSON object - a
|
|
3624
|
+
write still under way, a foreign line - is skipped.
|
|
3625
|
+
"""
|
|
3626
|
+
try:
|
|
3627
|
+
fh = open(path, "rb")
|
|
3628
|
+
except OSError:
|
|
3629
|
+
return
|
|
3630
|
+
with fh:
|
|
3631
|
+
end, carry = fh.seek(0, os.SEEK_END), b""
|
|
3632
|
+
while end > 0:
|
|
3633
|
+
start = max(0, end - chunk)
|
|
3634
|
+
fh.seek(start)
|
|
3635
|
+
lines = (fh.read(end - start) + carry).split(b"\n")
|
|
3636
|
+
carry, end = (lines.pop(0) if start else b""), start
|
|
3637
|
+
for line in reversed(lines):
|
|
3638
|
+
line = line.rstrip(b"\r")
|
|
3639
|
+
try:
|
|
3640
|
+
row = json.loads(line) if line.strip() else None
|
|
3641
|
+
except (ValueError, RecursionError):
|
|
3642
|
+
continue
|
|
3643
|
+
if isinstance(row, dict):
|
|
3644
|
+
yield hashlib.sha256(line).hexdigest()[:16], row
|
|
3645
|
+
|
|
3646
|
+
|
|
3647
|
+
def _notice_times(root):
|
|
3648
|
+
"""The folded notice times: {"last": digest, "forgotten_through": at, "keys": {key: {"recorded", "sent"}}}."""
|
|
3649
|
+
try:
|
|
3650
|
+
with open(notice_times_path(root), encoding=UTF8) as fh:
|
|
3651
|
+
index = json.load(fh)
|
|
3652
|
+
except (OSError, ValueError, RecursionError):
|
|
3653
|
+
return {"keys": {}}
|
|
3654
|
+
if not isinstance(index, dict) or not isinstance(index.get("keys"), dict):
|
|
3655
|
+
return {"keys": {}}
|
|
3656
|
+
index["keys"] = {name: {field: entry[field] for field in ("recorded", "sent")
|
|
3657
|
+
if isinstance(entry.get(field), (int, float))}
|
|
3658
|
+
for name, entry in index["keys"].items()
|
|
3659
|
+
if isinstance(entry, dict) and isinstance(entry.get("recorded"), (int, float))}
|
|
3660
|
+
return index
|
|
3661
|
+
|
|
3662
|
+
|
|
3663
|
+
def fold_notice_times(root):
|
|
3664
|
+
"""Fold the notices written since the last fold into when each key was last recorded and sent.
|
|
3665
|
+
|
|
3666
|
+
A window check read the last 100 KB of the ledger. Rebuilt after two weeks off, a day of
|
|
3667
|
+
notices was more than that, and a review parked through the day rang the desktop and the
|
|
3668
|
+
phone each time its last ring left those 100 KB; the ledger's own bound (#50) can keep less
|
|
3669
|
+
than a window. So every notice is folded as it is written, and before anything trims the
|
|
3670
|
+
ledger: a trimmed row has already been counted (NOTICE-WINDOW).
|
|
3671
|
+
|
|
3672
|
+
Each process folds from the newest line the file names to the end of the ledger, so a
|
|
3673
|
+
fold another process overtakes, or one that could not be written, leaves only more lines
|
|
3674
|
+
for the next fold and for a check to read. It is rebuilt from the ledger when it cannot be
|
|
3675
|
+
read, and replaced whole. A key not recorded for
|
|
3676
|
+
NOTICE_TIMES_DAYS goes, and past NOTICE_TIMES_KEYS the oldest go once NOTICE_TIMES_HOLD has
|
|
3677
|
+
passed since they were recorded; `forgotten_through` says when the newest of them was
|
|
3678
|
+
recorded, and a window reaching back to it reads the ledger.
|
|
3679
|
+
|
|
3680
|
+
Returns whether the fold stands written to the ledger's end: True when nothing was new,
|
|
3681
|
+
False only when it could not be written, and then nothing may trim the ledger
|
|
3682
|
+
(NOTICE-WINDOW-2).
|
|
3683
|
+
"""
|
|
3684
|
+
ledger = os.path.join(root, ".polyctl", "ledger", "notices.jsonl")
|
|
3685
|
+
index = _notice_times(root)
|
|
3686
|
+
fresh = []
|
|
3687
|
+
for digest, row in _notice_rows_back(ledger):
|
|
3688
|
+
if digest == index.get("last"):
|
|
3689
|
+
break
|
|
3690
|
+
fresh.append((digest, row))
|
|
3691
|
+
if not fresh:
|
|
3692
|
+
return True
|
|
3693
|
+
keys = index["keys"]
|
|
3694
|
+
for _, row in reversed(fresh):
|
|
3695
|
+
at, named = row.get("at"), row.get("named")
|
|
3696
|
+
if not isinstance(at, (int, float)):
|
|
3697
|
+
continue
|
|
3698
|
+
# A resume notice is the ring of every condition it named (RESUME-QUIET).
|
|
3699
|
+
for name in [row.get("key")] + (list(named) if isinstance(named, list) else []):
|
|
3700
|
+
if not isinstance(name, str):
|
|
3701
|
+
continue
|
|
3702
|
+
entry = keys.setdefault(name, {})
|
|
3703
|
+
entry["recorded"] = max(entry.get("recorded", at), at)
|
|
3704
|
+
if row.get("sent"):
|
|
3705
|
+
entry["sent"] = max(entry.get("sent", at), at)
|
|
3706
|
+
forgotten = index.get("forgotten_through") if isinstance(index.get("forgotten_through"), (int, float)) else 0
|
|
3707
|
+
old = time.time() - NOTICE_TIMES_DAYS * 86400
|
|
3708
|
+
newest_first = sorted(keys, key=lambda name: keys[name]["recorded"], reverse=True)
|
|
3709
|
+
held = time.time() - NOTICE_TIMES_HOLD
|
|
3710
|
+
past = [name for name in newest_first[NOTICE_TIMES_KEYS:] if keys[name]["recorded"] < held]
|
|
3711
|
+
for name in [name for name in newest_first if keys[name]["recorded"] < old] + past:
|
|
3712
|
+
if name in keys:
|
|
3713
|
+
forgotten = max(forgotten, keys.pop(name)["recorded"])
|
|
3714
|
+
# Under a name no file had, never through a link: `.notice-times.json.<pid>.tmp` in `.polyctl/ledger/`, a folder the
|
|
3715
|
+
# implementer writes, opened to truncate, wrote the target of a link left there for a likely pid (STORAGE-TMP).
|
|
3716
|
+
from .storage import replace_file_durably
|
|
3717
|
+
try:
|
|
3718
|
+
replace_file_durably(notice_times_path(root), json.dumps(
|
|
3719
|
+
{"last": fresh[0][0], "forgotten_through": forgotten, "keys": keys},
|
|
3720
|
+
ensure_ascii=False, sort_keys=True).encode(UTF8))
|
|
3721
|
+
except OSError:
|
|
3722
|
+
return False
|
|
3723
|
+
return True
|
|
3724
|
+
|
|
3725
|
+
|
|
3726
|
+
def _notice_within(root, key, window, sent):
|
|
3727
|
+
"""Whether `key` was recorded - or, with `sent`, delivered - inside the whole window (NOTICE-WINDOW).
|
|
3728
|
+
|
|
3729
|
+
The folded times answer for every line up to the newest one they name, whatever the ledger
|
|
3730
|
+
still holds; the lines after it are read from the ledger. Where the fold has forgotten a key
|
|
3731
|
+
the window may still need, the ledger is read back to the window's start. A check writes
|
|
3732
|
+
nothing: a dry run and a read-only tool call it. A resume notice is the ring of every
|
|
3733
|
+
condition it named (RESUME-QUIET).
|
|
3734
|
+
"""
|
|
3735
|
+
cutoff = time.time() - window
|
|
3736
|
+
index = _notice_times(root)
|
|
3737
|
+
if index["keys"].get(key, {}).get("sent" if sent else "recorded", cutoff - 1) >= cutoff:
|
|
3738
|
+
return True
|
|
3739
|
+
forgotten = index.get("forgotten_through")
|
|
3740
|
+
last = index.get("last") if not isinstance(forgotten, (int, float)) or cutoff > forgotten else None
|
|
3741
|
+
for digest, row in _notice_rows_back(os.path.join(root, ".polyctl", "ledger", "notices.jsonl")):
|
|
3742
|
+
at = row.get("at", 0)
|
|
3743
|
+
if digest == last or not isinstance(at, (int, float)) or at < cutoff:
|
|
3744
|
+
return False
|
|
3745
|
+
if (row.get("sent") or not sent) and (row.get("key") == key or key in (row.get("named") or ())):
|
|
3746
|
+
return True
|
|
3747
|
+
return False
|
|
3748
|
+
|
|
3749
|
+
|
|
3750
|
+
def notice_recently_sent(root, key, window):
|
|
3751
|
+
"""Was this same alert already delivered inside the window? All of the window counts (NOTICE-WINDOW)."""
|
|
3752
|
+
return _notice_within(root, key, window, sent=True)
|
|
3753
|
+
|
|
3754
|
+
|
|
3755
|
+
def _ledger_time(value):
|
|
3756
|
+
"""Epoch seconds from a ledger `at`: verifications write ISO text, the rest epochs."""
|
|
3757
|
+
if isinstance(value, (int, float)):
|
|
3758
|
+
return float(value)
|
|
3759
|
+
try:
|
|
3760
|
+
return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()
|
|
3761
|
+
except ValueError:
|
|
3762
|
+
return 0.0
|
|
3763
|
+
|
|
3764
|
+
|
|
3765
|
+
def _stall_reason(root, cfg, candidate, waiting, slice_id=None):
|
|
3766
|
+
"""Why the newest staged candidate has not landed, from what was recorded."""
|
|
3767
|
+
try:
|
|
3768
|
+
for row in reversed(authority_rows(root)):
|
|
3769
|
+
if isinstance(row, dict) and row.get("granted") is False \
|
|
3770
|
+
and (row.get("candidate") or {}).get("digest") == candidate.get("digest"):
|
|
3771
|
+
return "commit refused: " + "; ".join(str(r) for r in (row.get("reasons") or [])[:2])
|
|
3772
|
+
except Exception:
|
|
3773
|
+
pass
|
|
3774
|
+
reviews_dir = cfg.get("reviews") or "semantic-review"
|
|
3775
|
+
for name, verdict in reviews(root, reviews_dir, limit=20):
|
|
3776
|
+
try:
|
|
3777
|
+
with open(os.path.join(root, reviews_dir, name), encoding=UTF8, errors="replace") as fh:
|
|
3778
|
+
head = fh.read(4000)
|
|
3779
|
+
except OSError:
|
|
3780
|
+
continue
|
|
3781
|
+
if candidate.get("digest", "-") in head:
|
|
3782
|
+
if "APPROVED" in (verdict or "").upper():
|
|
3783
|
+
return f"review approved ({name}) but nothing committed"
|
|
3784
|
+
return f"review {verdict} ({name})"
|
|
3785
|
+
waiting = [d for d in waiting if slice_id and d.get("slice") == slice_id]
|
|
3786
|
+
if waiting:
|
|
3787
|
+
oldest = min(waiting, key=lambda d: d.get("asked_at") or 0)
|
|
3788
|
+
return f"waiting on decision {oldest.get('id')}: {str(oldest.get('question') or '')[:80]}"
|
|
3789
|
+
return "verified, then neither committed, refused nor reviewed"
|
|
3790
|
+
|
|
3791
|
+
|
|
3792
|
+
def throughput(root, cfg, hours=24.0, now=None):
|
|
3793
|
+
"""Candidates staged and landed, decisions asked and waiting, and what that makes the implementer (#91).
|
|
3794
|
+
|
|
3795
|
+
Busy-detection asks whether a process exists, so an implementer that is alive
|
|
3796
|
+
and lands nothing reads as busy forever. On 2026-09-14 one ran full work
|
|
3797
|
+
cycles for an hour - a candidate reached 255 passed and was staged - and
|
|
3798
|
+
produced four blocked reports, three decision requests and one landing while
|
|
3799
|
+
every surface read healthy. The ratio that would have shown it, candidates
|
|
3800
|
+
staged against candidates landed, was computed nowhere.
|
|
3801
|
+
|
|
3802
|
+
A candidate is staged when `polyctl verify` recorded it ready, and landed when a
|
|
3803
|
+
commit's tree is its index tree. The state is read in order: stalled (the
|
|
3804
|
+
newest staged candidate has not landed for the `stall_minutes` setting),
|
|
3805
|
+
landing (a candidate landed in the window), staging (staged, none landed yet),
|
|
3806
|
+
busy (the transcript moved, nothing staged) and idle (nothing moved). None of
|
|
3807
|
+
it reads the mailbox.
|
|
3808
|
+
"""
|
|
3809
|
+
from .storage import read_jsonl
|
|
3810
|
+
now = now or time.time()
|
|
3811
|
+
cut = now - hours * 3600
|
|
3812
|
+
try:
|
|
3813
|
+
rows = read_jsonl(os.path.join(root, ".polyctl", "ledger", "verifications.jsonl"))
|
|
3814
|
+
except Exception:
|
|
3815
|
+
rows = []
|
|
3816
|
+
ready = []
|
|
3817
|
+
for row in rows:
|
|
3818
|
+
candidate = row.get("candidate") if isinstance(row, dict) else None
|
|
3819
|
+
if isinstance(candidate, dict) and row.get("passed") is True and row.get("candidate_ready") \
|
|
3820
|
+
and candidate.get("index_tree"):
|
|
3821
|
+
ready.append((_ledger_time(row.get("at")), candidate, row.get("slice")))
|
|
3822
|
+
trees = {}
|
|
3823
|
+
try:
|
|
3824
|
+
log = _git_output(root, "log", "-n", "400", "--format=%T %ct").decode(UTF8, "replace")
|
|
3825
|
+
except RuntimeError:
|
|
3826
|
+
log = ""
|
|
3827
|
+
for line in log.split("\n"):
|
|
3828
|
+
parts = line.split()
|
|
3829
|
+
if len(parts) == 2 and parts[1].isdigit():
|
|
3830
|
+
trees.setdefault(parts[0], int(parts[1]))
|
|
3831
|
+
staged = {candidate["digest"] for at, candidate, _ in ready if at >= cut}
|
|
3832
|
+
landed = {candidate["index_tree"] for _, candidate, _ in ready
|
|
3833
|
+
if trees.get(candidate["index_tree"], 0) >= cut}
|
|
3834
|
+
every = decisions(root)
|
|
3835
|
+
asked = [d for d in every if (d.get("asked_at") or 0) >= cut]
|
|
3836
|
+
waiting = [d for d in every if d.get("state") == "open"]
|
|
3837
|
+
oldest = min((d.get("asked_at") or now for d in waiting), default=None)
|
|
3838
|
+
stall = None
|
|
3839
|
+
if ready:
|
|
3840
|
+
at, newest, slice_id = max(ready, key=lambda item: item[0])
|
|
3841
|
+
minutes = (now - at) / 60
|
|
3842
|
+
if newest["index_tree"] not in trees and minutes >= settings.get(cfg, "stall_minutes"):
|
|
3843
|
+
stall = {"minutes": int(minutes), "candidate": newest.get("digest"),
|
|
3844
|
+
"paths": list(newest.get("changed_paths") or []),
|
|
3845
|
+
"reason": _stall_reason(root, cfg, newest, waiting, slice_id)}
|
|
3846
|
+
msgs, _ = session_paths(cfg)
|
|
3847
|
+
try:
|
|
3848
|
+
moved = bool(msgs) and os.path.getmtime(msgs) >= cut
|
|
3849
|
+
except OSError:
|
|
3850
|
+
moved = False
|
|
3851
|
+
state = ("stalled" if stall else "landing" if landed else "staging" if staged
|
|
3852
|
+
else "busy" if moved else "idle")
|
|
3853
|
+
return {"hours": hours, "staged": len(staged), "landed": len(landed),
|
|
3854
|
+
"decisions_asked": len(asked), "decisions_open": len(waiting),
|
|
3855
|
+
"oldest_open_minutes": int((now - oldest) / 60) if oldest is not None else None,
|
|
3856
|
+
"state": state, "stall": stall}
|
|
3857
|
+
|
|
3858
|
+
|
|
3859
|
+
def digest(root, cfg, since_days=1.0):
|
|
3860
|
+
"""What actually happened in a window, from the ledgers rather than memory.
|
|
3861
|
+
|
|
3862
|
+
Event alerts answer "did something just occur". They cannot answer "is this
|
|
3863
|
+
week going well", and asking a human to reconstruct that from thirty
|
|
3864
|
+
notifications is asking them to do the tool's job. Everything here is already
|
|
3865
|
+
on disk — commits, verifications, authority grants, decisions, notices — so
|
|
3866
|
+
the summary is read, never estimated.
|
|
3867
|
+
"""
|
|
3868
|
+
cut = time.time() - since_days * 86400
|
|
3869
|
+
out = {"since_days": since_days, "at": int(time.time())}
|
|
3870
|
+
|
|
3871
|
+
# git's approxidate wants "N hours ago"; a bare "24.hours" parses to nothing
|
|
3872
|
+
# and silently reports zero commits on a day that landed two.
|
|
3873
|
+
# No shell: through one, a bare "|" in --pretty=%h|%ct|%s is a pipe, and on
|
|
3874
|
+
# Windows a quoted one is too, so the commit list quietly came back empty (#71).
|
|
3875
|
+
log = git_text(root, "log", f"--since={int(since_days * 24)} hours ago", "--pretty=%h|%ct|%s")
|
|
3876
|
+
out["commits"] = [dict(zip(("sha", "at", "subject"), l.split("|", 2)))
|
|
3877
|
+
for l in log.split("\n") if l.count("|") >= 2]
|
|
3878
|
+
out["unpushed"] = int(_git_text(root, "rev-list", "--count", "@{u}..HEAD") or 0) \
|
|
3879
|
+
if _git_text(root, "rev-parse", "--abbrev-ref", "@{u}") else \
|
|
3880
|
+
len([l for l in _git_text(root, "log", "--branches", "--not", "--remotes", "--pretty=%h").split("\n") if l])
|
|
3881
|
+
|
|
3882
|
+
def _window(rows):
|
|
3883
|
+
fresh = []
|
|
3884
|
+
for r in rows:
|
|
3885
|
+
if not isinstance(r, dict):
|
|
3886
|
+
continue
|
|
3887
|
+
# The ledgers disagree on the type of `at`: verifications write an
|
|
3888
|
+
# ISO string, everything else an epoch. Read both.
|
|
3889
|
+
at = r.get("at", 0)
|
|
3890
|
+
if isinstance(at, str):
|
|
3891
|
+
try:
|
|
3892
|
+
at = datetime.fromisoformat(at.replace("Z", "+00:00")).timestamp()
|
|
3893
|
+
except ValueError:
|
|
3894
|
+
at = 0
|
|
3895
|
+
if at >= cut:
|
|
3896
|
+
fresh.append(r)
|
|
3897
|
+
return fresh
|
|
3898
|
+
|
|
3899
|
+
def _jsonl(rel):
|
|
3900
|
+
p = os.path.join(root, rel)
|
|
3901
|
+
rows = []
|
|
3902
|
+
if os.path.exists(p):
|
|
3903
|
+
for line in open(p, errors="replace", encoding=UTF8):
|
|
3904
|
+
try:
|
|
3905
|
+
rows.append(json.loads(line))
|
|
3906
|
+
except Exception:
|
|
3907
|
+
continue
|
|
3908
|
+
return _window(rows)
|
|
3909
|
+
|
|
3910
|
+
ver = _jsonl(".polyctl/ledger/verifications.jsonl")
|
|
3911
|
+
out["verifications"] = {"total": len(ver),
|
|
3912
|
+
"passed": sum(1 for v in ver if v.get("passed")),
|
|
3913
|
+
"failed": sum(1 for v in ver if not v.get("passed"))}
|
|
3914
|
+
try:
|
|
3915
|
+
auth = _window(authority_rows(root))
|
|
3916
|
+
except Exception as exc:
|
|
3917
|
+
# This view cannot authorize, but it must not summarize manipulated
|
|
3918
|
+
# authority as trustworthy counts. Preserve the rest of the digest and
|
|
3919
|
+
# make the integrity failure explicit instead of skipping bad rows.
|
|
3920
|
+
auth = []
|
|
3921
|
+
out["authority"] = {
|
|
3922
|
+
"granted": 0, "refused": 0,
|
|
3923
|
+
"integrity": "broken", "error": str(exc),
|
|
3924
|
+
}
|
|
3925
|
+
else:
|
|
3926
|
+
out["authority"] = {
|
|
3927
|
+
"granted": sum(1 for a in auth if a.get("granted")),
|
|
3928
|
+
"refused": sum(1 for a in auth if not a.get("granted")),
|
|
3929
|
+
"integrity": "valid",
|
|
3930
|
+
}
|
|
3931
|
+
# Why authority was withheld is the actionable half — a refusal repeated all
|
|
3932
|
+
# week is a process problem, not an incident.
|
|
3933
|
+
reasons = {}
|
|
3934
|
+
for a in auth:
|
|
3935
|
+
for r in a.get("reasons") or []:
|
|
3936
|
+
key = re.sub(r"[0-9a-f]{7,}|V-\d+|D-\d+|\d{4}-\d\d-\d\d\S*", "…", r)[:70]
|
|
3937
|
+
reasons[key] = reasons.get(key, 0) + 1
|
|
3938
|
+
out["refusal_reasons"] = sorted(reasons.items(), key=lambda x: -x[1])[:5]
|
|
3939
|
+
|
|
3940
|
+
notices = _jsonl(".polyctl/ledger/notices.jsonl")
|
|
3941
|
+
out["alerts"] = {"sent": sum(1 for n in notices if n.get("sent")),
|
|
3942
|
+
"held": sum(1 for n in notices if not n.get("sent"))}
|
|
3943
|
+
|
|
3944
|
+
decs = [d for d in decisions(root) if d.get("asked_at", 0) >= cut]
|
|
3945
|
+
answered = [d for d in decs if d.get("state") == "answered"]
|
|
3946
|
+
out["decisions"] = {
|
|
3947
|
+
"asked": len(decs), "answered": len(answered),
|
|
3948
|
+
"open": len([d for d in decisions(root, "open")]),
|
|
3949
|
+
"median_minutes": (sorted((d["answered_at"] - d["asked_at"]) // 60
|
|
3950
|
+
for d in answered)[len(answered) // 2]
|
|
3951
|
+
if answered else None)}
|
|
3952
|
+
|
|
3953
|
+
b = board(root)
|
|
3954
|
+
out["board"] = {k: len(v) for k, v in b.items() if v}
|
|
3955
|
+
out["blocked"] = [{"id": i["id"], "title": i["title"],
|
|
3956
|
+
"needs": i["notes"].get("needs", "")} for i in b["blocked"]]
|
|
3957
|
+
|
|
3958
|
+
revs = reviews(root, cfg.get("reviews", "semantic-review"), limit=40)
|
|
3959
|
+
fresh = []
|
|
3960
|
+
for f, v in revs:
|
|
3961
|
+
try:
|
|
3962
|
+
if os.path.getmtime(os.path.join(root, cfg.get("reviews", "semantic-review"), f)) >= cut:
|
|
3963
|
+
fresh.append(v)
|
|
3964
|
+
except OSError:
|
|
3965
|
+
pass
|
|
3966
|
+
# Only a review that produced a verdict is a review (audit); the rest are counted apart.
|
|
3967
|
+
out["reviews"] = {"total": sum(1 for v in fresh if v in ("APPROVED", "NEEDS_CHANGES")),
|
|
3968
|
+
"approved": sum(1 for v in fresh if v == "APPROVED"),
|
|
3969
|
+
"changes": sum(1 for v in fresh if v == "NEEDS_CHANGES"),
|
|
3970
|
+
"not_reviewed": sum(1 for v in fresh if v not in ("APPROVED", "NEEDS_CHANGES"))}
|
|
3971
|
+
|
|
3972
|
+
# The implementer's own account and estimate, never the first shipped adapter's: an
|
|
3973
|
+
# implementer whose adapter declares no account polyctl can read is said to have none
|
|
3974
|
+
# (ACCOUNT-READERS). An expired token or a reading without a limit is no figure.
|
|
3975
|
+
ident = implementer_adapter_id(cfg)
|
|
3976
|
+
out["account"] = {"adapter": ident, "readable": bool(usage_api(ident))}
|
|
3977
|
+
acct = account_usage(adapter_id=ident) if out["account"]["readable"] else None
|
|
3978
|
+
if acct and not acct.get("error") and not acct.get("expired") and acct.get("limit"):
|
|
3979
|
+
out["credits"] = {"used": acct["used"], "limit": acct["limit"],
|
|
3980
|
+
"remaining": acct["limit"] - acct["used"]}
|
|
3981
|
+
local = credit_usage(adapter_id=ident)
|
|
3982
|
+
days = sorted(local.get("days", {}).items())
|
|
3983
|
+
out["credit_days"] = [(d, v) for d, v in days
|
|
3984
|
+
if d >= time.strftime("%Y-%m-%d", time.localtime(cut))]
|
|
3985
|
+
return out
|
|
3986
|
+
|
|
3987
|
+
|
|
3988
|
+
AGENT_LEDGERS = ("authority.jsonl", "decisions.jsonl", "fanouts.jsonl", "verifications.jsonl",
|
|
3989
|
+
"waivers.jsonl")
|
|
3990
|
+
|
|
3991
|
+
|
|
3992
|
+
def _product_changes(root, cfg):
|
|
3993
|
+
"""(porcelain lines, newest mtime) for uncommitted paths outside coordination state."""
|
|
3994
|
+
lines = product_dirty(root, cfg)
|
|
3995
|
+
churn = 0
|
|
3996
|
+
for line in lines:
|
|
3997
|
+
name = line[3:].split(" -> ")[-1].strip().strip('"')
|
|
3998
|
+
try:
|
|
3999
|
+
churn = max(churn, int(os.path.getmtime(os.path.join(root, name))))
|
|
4000
|
+
except OSError:
|
|
4001
|
+
pass
|
|
4002
|
+
return lines, churn
|
|
4003
|
+
|
|
4004
|
+
|
|
4005
|
+
def work_fingerprint(root, cfg=None):
|
|
4006
|
+
"""Everything that moves when work is happening, in one short string.
|
|
4007
|
+
|
|
4008
|
+
A commit is one shape of progress, not the only one. A slice whose review
|
|
4009
|
+
found real defects withholds its commit *because it is behaving correctly*,
|
|
4010
|
+
and a HEAD-only progress check cannot tell that apart from an agent that
|
|
4011
|
+
died — so the backoff exhausted itself on a live slice and stood down for two
|
|
4012
|
+
hours. Count the product tree and its content, the reviews and the decisions
|
|
4013
|
+
too: if any of them moved, something is being done.
|
|
4014
|
+
|
|
4015
|
+
Only what an agent produces counts. The watchdog appends to its own ledgers
|
|
4016
|
+
every cycle - progress, notices, credits, the mail ledger - and while those
|
|
4017
|
+
counted, the fingerprint changed on every cycle and a nudge's backoff reset
|
|
4018
|
+
before it could grow: between 07:00 and 12:00 on 2026-09-15 the watchdog
|
|
4019
|
+
nudged 29 times and never once backed off (#96).
|
|
4020
|
+
"""
|
|
4021
|
+
cfg = cfg if cfg is not None else load_config(root)
|
|
4022
|
+
lines, churn = _product_changes(root, cfg)
|
|
4023
|
+
parts = [_git_text(root, "rev-parse", "--short", "HEAD"), "\n".join(lines)]
|
|
4024
|
+
for line in lines:
|
|
4025
|
+
name = line[3:].split(" -> ")[-1].strip().strip('"')
|
|
4026
|
+
try:
|
|
4027
|
+
stat = os.stat(os.path.join(root, name))
|
|
4028
|
+
parts.append(f"{name}:{stat.st_mtime_ns}:{stat.st_size}")
|
|
4029
|
+
except OSError:
|
|
4030
|
+
pass
|
|
4031
|
+
# A review that did not take place is a file, not progress: an unavailable
|
|
4032
|
+
# reviewer written every turn reset the nudge backoff every turn (audit).
|
|
4033
|
+
not_reviews = set()
|
|
4034
|
+
try:
|
|
4035
|
+
from .storage import read_chained_jsonl
|
|
4036
|
+
not_reviews = {row.get("artefact") for row in read_chained_jsonl(review_ledger_path(root), REVIEW_CHAIN)
|
|
4037
|
+
if isinstance(row, dict) and row.get("verdict") not in ("APPROVED", "NEEDS_CHANGES")}
|
|
4038
|
+
except Exception:
|
|
4039
|
+
pass
|
|
4040
|
+
for sub in (cfg.get("reviews") or "semantic-review", DECISION_DIR):
|
|
4041
|
+
d = os.path.join(root, sub)
|
|
4042
|
+
if os.path.isdir(d):
|
|
4043
|
+
try:
|
|
4044
|
+
parts.append("|".join(sorted(
|
|
4045
|
+
f"{f}:{os.stat(os.path.join(d, f)).st_mtime_ns}:{os.path.getsize(os.path.join(d, f))}"
|
|
4046
|
+
for f in os.listdir(d) if f not in not_reviews)))
|
|
4047
|
+
except OSError:
|
|
4048
|
+
pass
|
|
4049
|
+
ledger = os.path.join(root, ".polyctl", "ledger")
|
|
4050
|
+
for name in AGENT_LEDGERS:
|
|
4051
|
+
try:
|
|
4052
|
+
stat = os.stat(os.path.join(ledger, name))
|
|
4053
|
+
parts.append(f"{name}:{stat.st_mtime_ns}:{stat.st_size}")
|
|
4054
|
+
except OSError:
|
|
4055
|
+
pass
|
|
4056
|
+
return hashlib.sha256("\n".join(parts).encode()).hexdigest()[:16]
|
|
4057
|
+
|
|
4058
|
+
|
|
4059
|
+
def nudge_inputs(root, cfg):
|
|
4060
|
+
"""What an idle implementer could act on, in one short string (#96).
|
|
4061
|
+
|
|
4062
|
+
work_fingerprint measures what an agent produces; this measures what it is
|
|
4063
|
+
given: the board, the backlog, the decisions and mail addressed to it. An
|
|
4064
|
+
implementer that answered a nudge by changing nothing is not asked again
|
|
4065
|
+
until one of these, or its work, moves.
|
|
4066
|
+
"""
|
|
4067
|
+
parts = []
|
|
4068
|
+
for rel in (".polyctl/board.md", ".polyctl/backlog.md"):
|
|
4069
|
+
try:
|
|
4070
|
+
with open(os.path.join(root, rel), "rb") as fh:
|
|
4071
|
+
parts.append(hashlib.sha256(fh.read()).hexdigest())
|
|
4072
|
+
except OSError:
|
|
4073
|
+
parts.append("-")
|
|
4074
|
+
d = os.path.join(root, DECISION_DIR)
|
|
4075
|
+
try:
|
|
4076
|
+
parts.append("|".join(sorted(f"{f}:{int(os.path.getmtime(os.path.join(d, f)))}"
|
|
4077
|
+
for f in os.listdir(d))))
|
|
4078
|
+
except OSError:
|
|
4079
|
+
parts.append("-")
|
|
4080
|
+
parts.append("|".join(sorted(implementer_inbox(root, cfg))))
|
|
4081
|
+
return hashlib.sha256("\n".join(parts).encode()).hexdigest()[:16]
|
|
4082
|
+
|
|
4083
|
+
|
|
4084
|
+
def _coordination_dirs(cfg):
|
|
4085
|
+
"""Repository-relative directories generated or consumed by orchestration.
|
|
4086
|
+
|
|
4087
|
+
These files are evidence and coordination state, not the product tree being
|
|
4088
|
+
measured. Keep one definition for review scope, dirty-product detection and
|
|
4089
|
+
commit authority so one polyctl command cannot invalidate another command's
|
|
4090
|
+
evidence merely by recording its result.
|
|
4091
|
+
"""
|
|
4092
|
+
values = list(COORDINATION_DIRS) + list(harness_dirs()) + [
|
|
4093
|
+
(cfg or {}).get("reviews", "semantic-review"),
|
|
4094
|
+
(cfg or {}).get("mailbox", "agent-mail"),
|
|
4095
|
+
]
|
|
4096
|
+
out = []
|
|
4097
|
+
for value in values:
|
|
4098
|
+
raw = str(value or "").replace("\\", "/")
|
|
4099
|
+
while raw.startswith("./"):
|
|
4100
|
+
raw = raw[2:]
|
|
4101
|
+
normal = os.path.normpath(raw).replace("\\", "/")
|
|
4102
|
+
# A coordination setting must never be able to exclude the repository
|
|
4103
|
+
# root, escape it, or hide an absolute product path from authority.
|
|
4104
|
+
if not normal or normal in (".", "..") or normal.startswith("../") \
|
|
4105
|
+
or os.path.isabs(raw):
|
|
4106
|
+
continue
|
|
4107
|
+
prefix = normal.rstrip("/") + "/"
|
|
4108
|
+
if prefix not in out:
|
|
4109
|
+
out.append(prefix)
|
|
4110
|
+
return tuple(out)
|
|
4111
|
+
|
|
4112
|
+
|
|
4113
|
+
def _is_coordination_path(path, cfg):
|
|
4114
|
+
normal = str(path or "").replace("\\", "/")
|
|
4115
|
+
while normal.startswith("./"):
|
|
4116
|
+
normal = normal[2:]
|
|
4117
|
+
return any(normal == prefix.rstrip("/") or normal.startswith(prefix)
|
|
4118
|
+
for prefix in _coordination_dirs(cfg))
|