polyctl 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polyctl/__init__.py +7 -0
- polyctl/__main__.py +5 -0
- polyctl/a2a.py +226 -0
- polyctl/a2a_mcp.py +546 -0
- polyctl/acp.py +417 -0
- polyctl/adapters/README.md +101 -0
- polyctl/adapters/aider.json +32 -0
- polyctl/adapters/amazon-q.json +15 -0
- polyctl/adapters/amp.json +16 -0
- polyctl/adapters/antigravity.json +120 -0
- polyctl/adapters/claude-code.json +229 -0
- polyctl/adapters/cloud-generic.json +57 -0
- polyctl/adapters/codex.json +47 -0
- polyctl/adapters/command-code.json +45 -0
- polyctl/adapters/copilot.json +17 -0
- polyctl/adapters/cursor-agent.json +29 -0
- polyctl/adapters/deepseek.json +13 -0
- polyctl/adapters/droid.json +34 -0
- polyctl/adapters/gemini.json +30 -0
- polyctl/adapters/grok.json +22 -0
- polyctl/adapters/hermes.json +34 -0
- polyctl/adapters/kilocode.json +30 -0
- polyctl/adapters/kimi.json +23 -0
- polyctl/adapters/kiro.json +332 -0
- polyctl/adapters/ollama.json +29 -0
- polyctl/adapters/omp.json +34 -0
- polyctl/adapters/openai-api.json +68 -0
- polyctl/adapters/opencode.json +162 -0
- polyctl/adapters/pi.json +24 -0
- polyctl/adapters/pr-agent.json +54 -0
- polyctl/adapters/profiles.json +9 -0
- polyctl/adapters/qoder.json +163 -0
- polyctl/adapters/qwen.json +27 -0
- polyctl/adapters/reasonix.json +25 -0
- polyctl/adapters/rewriters/rtk.json +24 -0
- polyctl/adapters/trae.json +31 -0
- polyctl/adapters/vendors.json +42 -0
- polyctl/adapters/warp.json +88 -0
- polyctl/allowlist.py +1756 -0
- polyctl/api_review.py +920 -0
- polyctl/cli.py +835 -0
- polyctl/conformance_harness.py +9 -0
- polyctl/demo.py +503 -0
- polyctl/drivers.py +146 -0
- polyctl/email.py +160 -0
- polyctl/features.py +98 -0
- polyctl/journal.py +144 -0
- polyctl/language.py +945 -0
- polyctl/lib.py +1296 -0
- polyctl/matrix.py +660 -0
- polyctl/mcp.py +1362 -0
- polyctl/ntfy.py +169 -0
- polyctl/outbound.py +39 -0
- polyctl/parts/__init__.py +5 -0
- polyctl/parts/cli_authority.py +2615 -0
- polyctl/parts/cli_channels.py +1325 -0
- polyctl/parts/cli_checks.py +2293 -0
- polyctl/parts/cli_completion.py +686 -0
- polyctl/parts/cli_evidence.py +141 -0
- polyctl/parts/cli_gain.py +375 -0
- polyctl/parts/cli_harness.py +392 -0
- polyctl/parts/cli_hooks.py +1816 -0
- polyctl/parts/cli_install.py +638 -0
- polyctl/parts/cli_lanes.py +106 -0
- polyctl/parts/cli_maintenance.py +2536 -0
- polyctl/parts/cli_next.py +248 -0
- polyctl/parts/cli_pr.py +53 -0
- polyctl/parts/cli_project.py +2857 -0
- polyctl/parts/cli_review.py +5158 -0
- polyctl/parts/cli_status.py +898 -0
- polyctl/parts/lib_adapters.py +2933 -0
- polyctl/parts/lib_alarms.py +2106 -0
- polyctl/parts/lib_events.py +224 -0
- polyctl/parts/lib_evidence.py +412 -0
- polyctl/parts/lib_gates.py +2099 -0
- polyctl/parts/lib_lanes.py +829 -0
- polyctl/parts/lib_mail.py +1541 -0
- polyctl/parts/lib_next.py +652 -0
- polyctl/parts/lib_pr.py +331 -0
- polyctl/parts/lib_slices.py +3337 -0
- polyctl/parts/lib_state.py +2415 -0
- polyctl/parts/lib_transcript.py +4118 -0
- polyctl/pressure.py +325 -0
- polyctl/prices.json +47 -0
- polyctl/procs.py +1194 -0
- polyctl/settings.py +505 -0
- polyctl/skill/SKILL.md +365 -0
- polyctl/skillkit.py +503 -0
- polyctl/storage.py +2055 -0
- polyctl/telegram.py +327 -0
- polyctl/tiers.py +123 -0
- polyctl/verdicts.py +41 -0
- polyctl/watchdog.py +3887 -0
- polyctl/web.py +337 -0
- polyctl-0.7.0.dist-info/METADATA +235 -0
- polyctl-0.7.0.dist-info/RECORD +99 -0
- polyctl-0.7.0.dist-info/WHEEL +4 -0
- polyctl-0.7.0.dist-info/entry_points.txt +5 -0
- polyctl-0.7.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1541 @@
|
|
|
1
|
+
"""Quota and mail: fan-out, rotation, binaries, reports, the mail envelope, unread age, the message
|
|
2
|
+
store and its sync.
|
|
3
|
+
|
|
4
|
+
A part of src/polyctl/lib.py (#44): moved out byte for byte and run in its namespace by `_part`,
|
|
5
|
+
where it stood; it is not importable on its own.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# ---- fan-out budget --------------------------------------------------------
|
|
10
|
+
#
|
|
11
|
+
# A coordinator that fans out to sub-agents has no gate of its own. One ran 47
|
|
12
|
+
# verification agents at once: 11 finished, 36 died with "session limit", two
|
|
13
|
+
# million tokens were spent and the answer was mostly missing. The watchdog's
|
|
14
|
+
# quota guard could not have helped — it watches the implementer's pool, and the
|
|
15
|
+
# fan-out was the coordinator's. This is the gate that was missing: a hard cap,
|
|
16
|
+
# the provider window as keyflip reports it, and the project's own record of
|
|
17
|
+
# what fan-outs cost, so the estimate is empirical after the first one.
|
|
18
|
+
|
|
19
|
+
def fanout_config(cfg):
|
|
20
|
+
"""The fan-out limits in force for this project (#74)."""
|
|
21
|
+
return {name: settings.get(cfg, f"fanout.{name}")
|
|
22
|
+
for name in ("max_agents", "per_agent_tokens", "window_reserve_pct")}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _window_adapter():
|
|
26
|
+
"""The adapter whose quota command provider_window reads: the first, by id, that declares one."""
|
|
27
|
+
return next((adapter for _, adapter in sorted(package_adapters().items())
|
|
28
|
+
if ((adapter.get("telemetry") or {}).get("quota") or {}).get("argv")), None)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def provider_window(name):
|
|
32
|
+
"""The machine-wide usage window keyflip reports for a provider, parsed.
|
|
33
|
+
|
|
34
|
+
Read from the quota reading the status panel shows, kept for the adapter's window
|
|
35
|
+
across polyctl's processes, so the panel and the verdict never disagree; this started the
|
|
36
|
+
command again, through a shell, on every call. Returns {pct, window, resets_in,
|
|
37
|
+
resets_s, raw} or None when no readable line exists — and None is reported as
|
|
38
|
+
"unreadable", not as headroom.
|
|
39
|
+
"""
|
|
40
|
+
adapter = _window_adapter()
|
|
41
|
+
if adapter is None:
|
|
42
|
+
return None
|
|
43
|
+
for line in quota(adapter):
|
|
44
|
+
if name.lower() not in line.lower():
|
|
45
|
+
continue
|
|
46
|
+
m = re.search(r"(\d+)\s*%", line)
|
|
47
|
+
if not m:
|
|
48
|
+
continue
|
|
49
|
+
w = re.search(r"\b(\d+[hdw])\b", line)
|
|
50
|
+
r = re.search(r"resets?\s+(?:in\s+)?([0-9hms ]+)", line)
|
|
51
|
+
secs = 0
|
|
52
|
+
if r:
|
|
53
|
+
for n, u in re.findall(r"(\d+)\s*([hms])", r.group(1)):
|
|
54
|
+
secs += int(n) * {"h": 3600, "m": 60, "s": 1}[u]
|
|
55
|
+
return {"pct": int(m.group(1)), "window": w.group(1) if w else "?",
|
|
56
|
+
"resets_in": r.group(1).strip() if r else "?", "resets_s": secs,
|
|
57
|
+
"raw": line.strip()}
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# ---- quota rotation is asked of keyflip, never written into config (#32) -----------------
|
|
62
|
+
|
|
63
|
+
def provider_of(argv):
|
|
64
|
+
"""The keyflip provider an actor's argv spends, as its adapter declares it (`quota.provider`), or None."""
|
|
65
|
+
if not argv or not isinstance(argv[0], str):
|
|
66
|
+
return None
|
|
67
|
+
program = _program_name(argv[0])
|
|
68
|
+
for ident, adapter in sorted(package_adapters().items()):
|
|
69
|
+
provider = (adapter.get("quota") or {}).get("provider")
|
|
70
|
+
if provider and program in {_program_name(name) for name in [ident, *adapter_binaries(adapter)]}:
|
|
71
|
+
return provider
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def rotate_if_exhausted(cfg, argv, who, on_wait=None):
|
|
76
|
+
"""Before an actor starts: rotate its provider's account through keyflip when the window is spent (#32).
|
|
77
|
+
|
|
78
|
+
2026-09-07: kiro-cli hit its overage limit and every nudge failed silently for
|
|
79
|
+
an hour, and the reviewer shared one Claude window with the architect. Account
|
|
80
|
+
routing is keyflip's; polyctl only asks. keyflip.rotation is off unless a person
|
|
81
|
+
turns it on, because a rotation is machine-wide and in place: it moves every
|
|
82
|
+
session on that provider, so rotations are serialised under one machine lock
|
|
83
|
+
and a window another actor already rotated is not rotated again. Returns
|
|
84
|
+
{"ok", "provider", "rotated", "text"}; not ok means no account has headroom,
|
|
85
|
+
and the caller surfaces that instead of spending the attempt. `on_wait` is called once the
|
|
86
|
+
window is read spent, before the lock is waited on, so a caller that says it waits says so only
|
|
87
|
+
then (REVIEW-START-DELAY).
|
|
88
|
+
"""
|
|
89
|
+
provider = provider_of(argv)
|
|
90
|
+
if settings.get(cfg, "keyflip.rotation") != "on" or not provider:
|
|
91
|
+
return {"ok": True, "provider": provider, "rotated": False, "text": "rotation off"}
|
|
92
|
+
ceiling = settings.get(None, "quota.block_percent")
|
|
93
|
+
window = provider_window(provider)
|
|
94
|
+
if not window or window["pct"] < ceiling:
|
|
95
|
+
return {"ok": True, "provider": provider, "rotated": False, "text": "headroom"}
|
|
96
|
+
from .storage import _exclusive_lock
|
|
97
|
+
os.makedirs(os.path.join(HOME, ".polyctl"), exist_ok=True)
|
|
98
|
+
if on_wait is not None:
|
|
99
|
+
on_wait()
|
|
100
|
+
with _exclusive_lock(os.path.join(HOME, ".polyctl", "keyflip-rotation.lock"), timeout=180):
|
|
101
|
+
window = provider_window(provider)
|
|
102
|
+
if window and window["pct"] < ceiling:
|
|
103
|
+
return {"ok": True, "provider": provider, "rotated": False, "text": "another actor rotated first"}
|
|
104
|
+
try:
|
|
105
|
+
keyflip = program_path("keyflip") # never one the current directory holds (WINDOWS-NO-CWD-EXEC)
|
|
106
|
+
if keyflip is None:
|
|
107
|
+
raise FileNotFoundError("keyflip is not on PATH")
|
|
108
|
+
subprocess.run([keyflip, "next", "--strategy", "best"], stdin=subprocess.DEVNULL,
|
|
109
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=120)
|
|
110
|
+
except Exception as exc: # a rotation that cannot run is not a headroom
|
|
111
|
+
return {"ok": False, "provider": provider, "rotated": False,
|
|
112
|
+
"text": f"{provider} window {window['pct']}% used and keyflip could not rotate: {exc}"}
|
|
113
|
+
# The kept reading is the window before the rotation: ask again, and an actor waiting on
|
|
114
|
+
# this lock reads the rotated window, not the spent one it would rotate a second time.
|
|
115
|
+
adapter = _window_adapter()
|
|
116
|
+
if adapter is not None:
|
|
117
|
+
forget_quota(adapter)
|
|
118
|
+
after = provider_window(provider)
|
|
119
|
+
if after and after["pct"] < ceiling:
|
|
120
|
+
return {"ok": True, "provider": provider, "rotated": True,
|
|
121
|
+
"text": f"rotated through keyflip for the {who}: {provider} now {after['pct']}% used"}
|
|
122
|
+
return {"ok": False, "provider": provider, "rotated": True,
|
|
123
|
+
"text": f"no {provider} account has headroom after rotating for the {who}"}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ---- a usage limit's words are declared, not coded (QUOTA-PARK) --------------------------------
|
|
127
|
+
#
|
|
128
|
+
# A harness that hits its usage limit ends the turn with words of its own: a reply it writes in the
|
|
129
|
+
# model's place, a line its process prints as it exits. Each harness words them its own way, and the
|
|
130
|
+
# one pattern polyctl read them by was one harness's wording: another's limit was never read as one.
|
|
131
|
+
|
|
132
|
+
def quota_stops(adapter):
|
|
133
|
+
"""The usage-limit stops an adapter declares (`quota` → `stops`): [{"words", "hours"}]; [] for none.
|
|
134
|
+
|
|
135
|
+
`words` is a regular expression that finds the words the harness stops on, and what it matches is
|
|
136
|
+
the stop a person is told; a group named `epoch` in it holds the reset in epoch seconds, where the
|
|
137
|
+
harness writes it so. `hours` is the longest the limit stands once it is hit: a reset its words name
|
|
138
|
+
by the clock alone is the next such time within it, and one named further away than it is not the
|
|
139
|
+
one meant. A stop without both, or whose words do not compile, is not read, and
|
|
140
|
+
`quota_stop_problems` says why.
|
|
141
|
+
"""
|
|
142
|
+
declared = (adapter or {}).get("quota")
|
|
143
|
+
stops = declared.get("stops") if isinstance(declared, dict) else None
|
|
144
|
+
found = []
|
|
145
|
+
for stop in stops if isinstance(stops, list) else []:
|
|
146
|
+
words, hours = (stop.get("words"), stop.get("hours")) if isinstance(stop, dict) else (None, None)
|
|
147
|
+
if not isinstance(words, str) or not words or isinstance(hours, bool) \
|
|
148
|
+
or not isinstance(hours, (int, float)) or hours <= 0:
|
|
149
|
+
continue
|
|
150
|
+
try:
|
|
151
|
+
found.append({"words": re.compile(words), "hours": float(hours)})
|
|
152
|
+
except re.error:
|
|
153
|
+
continue
|
|
154
|
+
return found
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def quota_reset_path(adapter):
|
|
158
|
+
"""Where a reply the harness wrote in the model's place keeps the reset, in epoch seconds; "" where it keeps none.
|
|
159
|
+
|
|
160
|
+
`quota` → `resets_at`: a harness that records the reset beside its words is read by it, and its
|
|
161
|
+
words are read only where the record holds none.
|
|
162
|
+
"""
|
|
163
|
+
declared = (adapter or {}).get("quota")
|
|
164
|
+
path = declared.get("resets_at") if isinstance(declared, dict) else None
|
|
165
|
+
return path if isinstance(path, str) else ""
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def quota_stop_problems(adapter):
|
|
169
|
+
"""Each usage-limit stop an adapter declares that polyctl will not read, and why; [] when it reads them all (QUOTA-PARK).
|
|
170
|
+
|
|
171
|
+
`quota_stops` passes such a stop over without a word, since a reading must not fail on a layer's
|
|
172
|
+
mistake; `polyctl adapters validate` says why before anyone relies on the declaration.
|
|
173
|
+
"""
|
|
174
|
+
declared = (adapter or {}).get("quota")
|
|
175
|
+
if not isinstance(declared, dict):
|
|
176
|
+
return []
|
|
177
|
+
problems = []
|
|
178
|
+
stops = declared.get("stops", [])
|
|
179
|
+
if not isinstance(stops, list):
|
|
180
|
+
return ["`quota.stops` must be a list of {words, hours}"]
|
|
181
|
+
for number, stop in enumerate(stops, 1):
|
|
182
|
+
words, hours = (stop.get("words"), stop.get("hours")) if isinstance(stop, dict) else (None, None)
|
|
183
|
+
if not isinstance(words, str) or not words:
|
|
184
|
+
problems.append(f"`quota.stops` #{number} must name its `words`, a regular expression")
|
|
185
|
+
else:
|
|
186
|
+
try:
|
|
187
|
+
re.compile(words)
|
|
188
|
+
except re.error as exc:
|
|
189
|
+
problems.append(f"`quota.stops` #{number} words {words!r} do not compile: {exc}")
|
|
190
|
+
if isinstance(hours, bool) or not isinstance(hours, (int, float)) or hours <= 0:
|
|
191
|
+
problems.append(f"`quota.stops` #{number} must say in `hours` how long the limit stands, above 0")
|
|
192
|
+
if "resets_at" in declared and not (isinstance(declared["resets_at"], str) and declared["resets_at"]):
|
|
193
|
+
problems.append("`quota.resets_at` must be the path to the reset in the harness's own reply")
|
|
194
|
+
return problems
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def fanout_history(root, limit=20):
|
|
198
|
+
p = os.path.join(root, ".polyctl", "ledger", "fanouts.jsonl")
|
|
199
|
+
if not os.path.exists(p):
|
|
200
|
+
return []
|
|
201
|
+
rows = []
|
|
202
|
+
for line in open(p, errors="replace", encoding=UTF8):
|
|
203
|
+
try:
|
|
204
|
+
rows.append(json.loads(line))
|
|
205
|
+
except ValueError:
|
|
206
|
+
pass
|
|
207
|
+
return rows[-limit:]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def record_fanout(root, agents, done=None, errors=None, tokens=None, note=None):
|
|
211
|
+
"""What a fan-out actually cost. The next verdict is estimated from this."""
|
|
212
|
+
d = os.path.join(root, ".polyctl", "ledger")
|
|
213
|
+
os.makedirs(d, exist_ok=True)
|
|
214
|
+
rec = {"at": int(time.time()), "agents": int(agents)}
|
|
215
|
+
if done is not None:
|
|
216
|
+
rec["done"] = int(done)
|
|
217
|
+
if errors is not None:
|
|
218
|
+
rec["errors"] = int(errors)
|
|
219
|
+
if tokens is not None:
|
|
220
|
+
rec["tokens"] = int(tokens)
|
|
221
|
+
if note:
|
|
222
|
+
rec["note"] = note
|
|
223
|
+
rec["limit_hit"] = bool(rec.get("errors")) and bool(
|
|
224
|
+
re.search(r"limit|quota|429|rate", note or "", re.I))
|
|
225
|
+
with open(os.path.join(d, "fanouts.jsonl"), "a", encoding=UTF8) as fh:
|
|
226
|
+
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
227
|
+
return rec
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def observed_per_agent_tokens(root):
|
|
231
|
+
"""Average tokens per agent over recorded fan-outs (errored agents spent too)."""
|
|
232
|
+
tok = n = 0
|
|
233
|
+
for r in fanout_history(root, 50):
|
|
234
|
+
if r.get("tokens") and (r.get("done") or r.get("errors")):
|
|
235
|
+
tok += r["tokens"]
|
|
236
|
+
n += (r.get("done") or 0) + (r.get("errors") or 0)
|
|
237
|
+
return int(tok / n) if n else None
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def fanout_verdict(root, cfg, agents, per_agent_tokens=None, provider=None):
|
|
241
|
+
"""May a fan-out of this size start now?
|
|
242
|
+
|
|
243
|
+
Three checks, each a fact the caller can see: the project's hard cap, whether
|
|
244
|
+
a fan-out already hit this provider's limit inside the current window, and
|
|
245
|
+
how much of the window keyflip says is left. The token estimate is shown, not
|
|
246
|
+
gated on — a percentage window cannot be converted to tokens honestly — but
|
|
247
|
+
it becomes empirical after the first recorded run.
|
|
248
|
+
"""
|
|
249
|
+
fc = fanout_config(cfg)
|
|
250
|
+
observed = observed_per_agent_tokens(root)
|
|
251
|
+
per = per_agent_tokens or observed or fc["per_agent_tokens"]
|
|
252
|
+
provider = provider or provider_of((cfg.get("architect") or {}).get("argv"))
|
|
253
|
+
win = provider_window(provider) if provider else None
|
|
254
|
+
reasons, verdict = [], "ok"
|
|
255
|
+
if agents > fc["max_agents"]:
|
|
256
|
+
verdict = "too-many"
|
|
257
|
+
reasons.append(f"{agents} agents > max_agents {fc['max_agents']} — run in batches of "
|
|
258
|
+
f"{fc['max_agents']} and record each")
|
|
259
|
+
now = time.time()
|
|
260
|
+
span = 5 * 3600
|
|
261
|
+
if win and win.get("window", "").endswith("h"):
|
|
262
|
+
try:
|
|
263
|
+
span = int(win["window"][:-1]) * 3600
|
|
264
|
+
except ValueError:
|
|
265
|
+
pass
|
|
266
|
+
window_start = now - (span - win["resets_s"]) if win and win.get("resets_s") else now - span
|
|
267
|
+
for r in reversed(fanout_history(root, 10)):
|
|
268
|
+
if r.get("limit_hit") and r["at"] >= window_start:
|
|
269
|
+
if verdict == "ok":
|
|
270
|
+
verdict = "limit-hit-recently"
|
|
271
|
+
reasons.append(f"a fan-out of {r['agents']} hit the provider limit "
|
|
272
|
+
f"{int((now - r['at']) / 60)}m ago ({r.get('errors')} errors); "
|
|
273
|
+
f"wait for the window to reset"
|
|
274
|
+
+ (f" (in {win['resets_in']})" if win else ""))
|
|
275
|
+
break
|
|
276
|
+
if win:
|
|
277
|
+
left = 100 - win["pct"]
|
|
278
|
+
if left < fc["window_reserve_pct"]:
|
|
279
|
+
if verdict == "ok":
|
|
280
|
+
verdict = "window-low"
|
|
281
|
+
reasons.append(f"{provider} window {win['pct']}% used, {left}% left < reserve "
|
|
282
|
+
f"{fc['window_reserve_pct']}%; resets in {win['resets_in']}")
|
|
283
|
+
else:
|
|
284
|
+
reasons.append((f"{provider} window unreadable (keyflip absent or no line)" if provider
|
|
285
|
+
else "no keyflip provider runs the architect's command, so no window to read")
|
|
286
|
+
+ " — hard cap and history only")
|
|
287
|
+
spent = sum(r.get("tokens") or 0 for r in fanout_history(root, 50) if r["at"] >= window_start)
|
|
288
|
+
return {"verdict": verdict, "ok": verdict == "ok", "agents": agents,
|
|
289
|
+
"per_agent_tokens": per,
|
|
290
|
+
"per_agent_source": "arg" if per_agent_tokens else ("observed" if observed else "default"),
|
|
291
|
+
"estimated_tokens": agents * per, "spent_this_window": spent,
|
|
292
|
+
"window": win, "provider": provider, "max_agents": fc["max_agents"], "reasons": reasons}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
# ---- binaries ----------------------------------------------------------------
|
|
297
|
+
#
|
|
298
|
+
# The architect was woken forty times in eleven hours and every wake died with
|
|
299
|
+
# "Claude Code 2.1.185 does not support this model". The binary was real, on
|
|
300
|
+
# PATH, and two hundred versions stale — an npm-global leftover in /usr/local
|
|
301
|
+
# whose node had long since moved under a version manager, where a current
|
|
302
|
+
# copy sat unused. `which` answers "the first one", and the first one is the
|
|
303
|
+
# wrong question. Ask "the newest one" and remember what it said.
|
|
304
|
+
|
|
305
|
+
# Where a harness installs itself outside the usual directories is its adapter's to declare
|
|
306
|
+
# (`detect.install_dirs`, #76), placed where such a directory always stood in the search.
|
|
307
|
+
class _SearchDirs:
|
|
308
|
+
"""_BIN_DIRS: the directories searched after PATH, the adapter-declared ones in place, read when searched.
|
|
309
|
+
|
|
310
|
+
Built as a tuple at import, the list read every package adapter before any `polyctl`
|
|
311
|
+
run did anything, though most runs never look for a binary. It iterates, indexes
|
|
312
|
+
and answers `in` as that tuple did, and a test may still put a tuple in its place.
|
|
313
|
+
"""
|
|
314
|
+
|
|
315
|
+
def _dirs(self):
|
|
316
|
+
return (("~/.local/bin", "~/bin", "/usr/local/bin", "/opt/homebrew/bin")
|
|
317
|
+
+ tuple(sorted({d for adapter in package_adapters().values()
|
|
318
|
+
for d in (adapter.get("detect") or {}).get("install_dirs") or []}))
|
|
319
|
+
+ ("~/.npm-global/bin", "~/.volta/bin", "~/.asdf/shims"))
|
|
320
|
+
|
|
321
|
+
def __iter__(self):
|
|
322
|
+
return iter(self._dirs())
|
|
323
|
+
|
|
324
|
+
def __getitem__(self, index):
|
|
325
|
+
return self._dirs()[index]
|
|
326
|
+
|
|
327
|
+
def __len__(self):
|
|
328
|
+
return len(self._dirs())
|
|
329
|
+
|
|
330
|
+
def __contains__(self, item):
|
|
331
|
+
return item in self._dirs()
|
|
332
|
+
|
|
333
|
+
def __repr__(self):
|
|
334
|
+
return repr(self._dirs())
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
_BIN_DIRS = _SearchDirs()
|
|
338
|
+
_BIN_GLOBS = ("~/.local/share/fnm/node-versions/*/installation/bin",
|
|
339
|
+
"~/.fnm/node-versions/*/installation/bin",
|
|
340
|
+
"~/.nvm/versions/node/*/bin", "~/.local/share/mise/installs/node/*/bin")
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def binary_candidates(name, path=None):
|
|
344
|
+
"""Every executable called `name` this machine has, PATH first, deduplicated.
|
|
345
|
+
|
|
346
|
+
PATH is read as search_directories reads it, and every other directory searched is taken only
|
|
347
|
+
when absolute: one relative to the current directory names a directory an agent writes
|
|
348
|
+
(WINDOWS-NO-CWD-EXEC).
|
|
349
|
+
"""
|
|
350
|
+
import glob as _glob
|
|
351
|
+
dirs = search_directories(path or None)
|
|
352
|
+
dirs += [os.path.expanduser(d) for d in settings.get(None, "binaries.extra_dirs")]
|
|
353
|
+
dirs += [os.path.expanduser(d) for d in _BIN_DIRS]
|
|
354
|
+
for g in _BIN_GLOBS:
|
|
355
|
+
dirs += sorted(_glob.glob(os.path.expanduser(g)), reverse=True)
|
|
356
|
+
dirs = [d for d in dirs if os.path.isabs(d)]
|
|
357
|
+
exts = [""] + (os.environ.get("PATHEXT", ".EXE;.CMD;.BAT").split(";") if os.name == "nt" else [])
|
|
358
|
+
seen, out = set(), []
|
|
359
|
+
for d in dirs:
|
|
360
|
+
for ext in exts:
|
|
361
|
+
cand = os.path.join(d, name + ext.lower()) if ext else os.path.join(d, name)
|
|
362
|
+
if not (os.path.isfile(cand) and os.access(cand, os.X_OK)):
|
|
363
|
+
continue
|
|
364
|
+
real = os.path.realpath(cand)
|
|
365
|
+
if real in seen:
|
|
366
|
+
continue
|
|
367
|
+
seen.add(real)
|
|
368
|
+
out.append(cand)
|
|
369
|
+
return out
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def runnable_binary(name):
|
|
373
|
+
"""The file polyctl starts for `name` without a shell: the first binary_candidates match this platform runs, or None.
|
|
374
|
+
|
|
375
|
+
Windows starts a program by its extension, and an npm install puts an extensionless
|
|
376
|
+
POSIX script beside the .cmd that cmd.exe found; there only a match with an extension
|
|
377
|
+
is a program. Windows' own programs - PowerShell for the process table above all - come
|
|
378
|
+
from its system directory alone (WINDOWS-NO-CWD-EXEC).
|
|
379
|
+
"""
|
|
380
|
+
if os.name == "nt" and _program_name(name) in WINDOWS_SYSTEM_PROGRAMS:
|
|
381
|
+
return program_path(name)
|
|
382
|
+
return next((path for path in binary_candidates(name) if os.name != "nt" or os.path.splitext(path)[1]), None)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
# Windows' own programs polyctl starts, by the name it starts each by, and where each lies in Windows' system
|
|
386
|
+
# directory: they are taken from there and from nowhere else (WINDOWS-NO-CWD-EXEC).
|
|
387
|
+
WINDOWS_SYSTEM_PROGRAMS = {"cmd": ("cmd.exe",), "where": ("where.exe",), "taskkill": ("taskkill.exe",),
|
|
388
|
+
"tasklist": ("tasklist.exe",), "schtasks": ("schtasks.exe",),
|
|
389
|
+
"powershell": ("WindowsPowerShell", "v1.0", "powershell.exe")}
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def windows_system_directory():
|
|
393
|
+
"""Windows' system directory, %SystemRoot%\\System32 (WINDOWS-NO-CWD-EXEC).
|
|
394
|
+
|
|
395
|
+
SystemRoot is set for every Windows process; where it names no absolute directory
|
|
396
|
+
Windows itself is asked (procs.system_directory), and where it cannot be asked the
|
|
397
|
+
directory is Windows' default one.
|
|
398
|
+
"""
|
|
399
|
+
from . import procs
|
|
400
|
+
for variable in ("SystemRoot", "SYSTEMROOT", "windir", "WINDIR"):
|
|
401
|
+
system_root = os.environ.get(variable) or ""
|
|
402
|
+
if os.path.isabs(system_root):
|
|
403
|
+
return os.path.join(system_root, "System32")
|
|
404
|
+
return procs.system_directory() or "C:\\Windows\\System32"
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def search_directories(path=None):
|
|
408
|
+
"""The directories a program is looked for in: PATH's entries in its order, each absolute and none the current
|
|
409
|
+
directory (WINDOWS-NO-CWD-EXEC).
|
|
410
|
+
|
|
411
|
+
Windows looks for a program named without a directory in the current directory before PATH,
|
|
412
|
+
an empty PATH entry is the current directory to shutil.which and to a POSIX shell alike, and
|
|
413
|
+
a relative entry is a directory below it. polyctl runs in a tree an agent writes, so a program it
|
|
414
|
+
left there under the name of one polyctl starts - a taskkill.exe, a git.bat - would run as the
|
|
415
|
+
person. None of the three is searched, on any platform.
|
|
416
|
+
"""
|
|
417
|
+
try:
|
|
418
|
+
here = os.path.normcase(os.path.normpath(os.getcwd()))
|
|
419
|
+
except OSError:
|
|
420
|
+
here = None
|
|
421
|
+
entries = (os.environ.get("PATH", os.defpath) if path is None else path).split(os.pathsep)
|
|
422
|
+
# Each is absolute already, so it is only normalised: no entry is read against the current directory.
|
|
423
|
+
return [entry for entry in entries
|
|
424
|
+
if entry and os.path.isabs(entry) and os.path.normcase(os.path.normpath(entry)) != here]
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def unsafe_path_entries(path=None):
|
|
428
|
+
"""The entries of PATH that are empty or relative: what search_directories passes over, as PATH writes them."""
|
|
429
|
+
entries = (os.environ.get("PATH", "") if path is None else path).split(os.pathsep)
|
|
430
|
+
return [entry for entry in entries if not entry or not os.path.isabs(entry)]
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def polyctl_program():
|
|
434
|
+
"""The program that starts polyctl, by its absolute path: the polyctl on PATH, else the file this polyctl was started as when
|
|
435
|
+
that is an absolute, executable one, else the clone's own bin/polyctl this polyctl runs from; None when none is. Never a
|
|
436
|
+
bare `polyctl`, which a process started in a tree looks for there, nor one made absolute against the current directory
|
|
437
|
+
(the review of WINDOWS-NO-CWD-EXEC). `polyctl mcp config` and the registration `polyctl init` writes both ask here, so they
|
|
438
|
+
name the same polyctl (WINDOWS-NO-CWD-EXEC-4)."""
|
|
439
|
+
found = program_path("polyctl")
|
|
440
|
+
if found:
|
|
441
|
+
return found
|
|
442
|
+
first = sys.argv[0] if sys.argv else ""
|
|
443
|
+
# Then the file it was started as, and a clone's own bin/polyctl, run as a module there (WINDOWS-NO-CWD-EXEC-4). A
|
|
444
|
+
# Python source is no program a client starts: `python -m polyctl` was started as polyctl/__main__.py, which Windows calls
|
|
445
|
+
# executable as it calls every file (the review of WINDOWS-NO-CWD-EXEC-4).
|
|
446
|
+
for candidate in (first, os.path.join(REPO, "bin", "polyctl")):
|
|
447
|
+
if os.path.isabs(candidate) and os.path.isfile(candidate) and os.access(candidate, os.X_OK) \
|
|
448
|
+
and not candidate.lower().endswith((".py", ".pyw", ".pyc")):
|
|
449
|
+
return candidate
|
|
450
|
+
return None
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def program_path(name, path=None):
|
|
454
|
+
"""The file polyctl starts for program `name`, or None: never one the current directory holds (WINDOWS-NO-CWD-EXEC).
|
|
455
|
+
|
|
456
|
+
A name with a directory in it is taken as it is when absolute and never when relative,
|
|
457
|
+
since it names a file in the current directory. On Windows taskkill, schtasks, tasklist,
|
|
458
|
+
cmd, where and powershell are Windows' own, from its system directory and nowhere else.
|
|
459
|
+
Every other program is looked for as shutil.which looks, honouring PATHEXT, in
|
|
460
|
+
search_directories alone. Before Python 3.12 shutil.which on Windows looked in the
|
|
461
|
+
current directory first whatever directories it was given, and from 3.12 it still does
|
|
462
|
+
unless NoDefaultCurrentDirectoryInExePath is set, so an answer from there is set aside
|
|
463
|
+
for the first one PATH's own directories hold.
|
|
464
|
+
"""
|
|
465
|
+
import shutil
|
|
466
|
+
name = str(name)
|
|
467
|
+
if os.path.isabs(name):
|
|
468
|
+
return shutil.which(name)
|
|
469
|
+
if os.path.dirname(name):
|
|
470
|
+
return None
|
|
471
|
+
if os.name == "nt" and _program_name(name) in WINDOWS_SYSTEM_PROGRAMS \
|
|
472
|
+
and os.path.splitext(name)[1].lower() in ("", ".exe"):
|
|
473
|
+
program = os.path.join(windows_system_directory(), *WINDOWS_SYSTEM_PROGRAMS[_program_name(name)])
|
|
474
|
+
return program if os.path.isfile(program) else None
|
|
475
|
+
directories = search_directories(path)
|
|
476
|
+
found = shutil.which(name, path=os.pathsep.join(directories))
|
|
477
|
+
if found and os.name == "nt" and _in_current_directory(found):
|
|
478
|
+
found = _first_program_in(name, directories)
|
|
479
|
+
return found
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _in_current_directory(found):
|
|
483
|
+
"""Whether a program found is a file in the current directory itself."""
|
|
484
|
+
try:
|
|
485
|
+
here = os.path.normcase(os.path.abspath(os.getcwd()))
|
|
486
|
+
except OSError:
|
|
487
|
+
return False
|
|
488
|
+
return os.path.isfile(found) and os.path.normcase(os.path.dirname(os.path.abspath(found))) == here
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _first_program_in(name, directories):
|
|
492
|
+
"""The first file in these directories that Windows starts for `name`: the name with each PATHEXT extension
|
|
493
|
+
in turn, or the name alone when it ends in one."""
|
|
494
|
+
extensions = [extension.lower() for extension in os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(";")
|
|
495
|
+
if extension]
|
|
496
|
+
names = [name] if os.path.splitext(name)[1].lower() in extensions else [name + ext for ext in extensions]
|
|
497
|
+
for directory in directories:
|
|
498
|
+
for candidate in names:
|
|
499
|
+
program = os.path.join(directory, candidate)
|
|
500
|
+
if os.path.isfile(program):
|
|
501
|
+
return program
|
|
502
|
+
return None
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def console_text(data):
|
|
506
|
+
"""What a Windows console program wrote to a pipe, as text: UTF-8 where it is that, else the OEM code page
|
|
507
|
+
Windows' own programs write in (WINDOWS-SAY-WHY).
|
|
508
|
+
|
|
509
|
+
schtasks wrote why it refused a task in the OEM code page - cp857 on a Turkish Windows - and polyctl
|
|
510
|
+
read it as UTF-8: every letter outside ASCII in the line a person was shown came out as a
|
|
511
|
+
replacement character. A console set to UTF-8 (chcp 65001) writes UTF-8, which is read first;
|
|
512
|
+
OEM bytes are almost never valid UTF-8. Where no OEM code page can be asked, UTF-8 with
|
|
513
|
+
replacements, as before.
|
|
514
|
+
"""
|
|
515
|
+
try:
|
|
516
|
+
return data.decode(UTF8)
|
|
517
|
+
except UnicodeDecodeError:
|
|
518
|
+
pass
|
|
519
|
+
from . import procs
|
|
520
|
+
return data.decode(procs.oem_encoding() or UTF8, "replace")
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _run_program(argv, cwd=None, timeout=20, stderr=subprocess.DEVNULL):
|
|
524
|
+
"""What sh() returned for these words, and the exit status, from the program started without a shell.
|
|
525
|
+
|
|
526
|
+
("", None) when the program is found nowhere, cannot be started or runs past the
|
|
527
|
+
timeout: there the shell printed nothing. The program is looked for on the binary
|
|
528
|
+
search path, so a launchd job whose PATH is minimal finds what a terminal finds; it
|
|
529
|
+
reads nothing from standard input, and its standard error is discarded, as sh()
|
|
530
|
+
discarded it, unless the caller keeps it in the answer (`subprocess.STDOUT`, `2>&1`).
|
|
531
|
+
"""
|
|
532
|
+
program = runnable_binary(argv[0])
|
|
533
|
+
if program is None:
|
|
534
|
+
return "", None
|
|
535
|
+
try:
|
|
536
|
+
done = subprocess.run([program, *argv[1:]], cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
537
|
+
stderr=stderr, text=True, encoding=UTF8, errors="replace", timeout=timeout)
|
|
538
|
+
except Exception:
|
|
539
|
+
return "", None
|
|
540
|
+
return done.stdout.strip(), done.returncode
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def binary_version(path):
|
|
544
|
+
"""`path --version`, cached by (path, mtime) so a wake does not pay for it twice."""
|
|
545
|
+
cache_p = os.path.join(HOME, ".polyctl", "binaries.json")
|
|
546
|
+
try:
|
|
547
|
+
cache = json.load(open(cache_p, encoding=UTF8))
|
|
548
|
+
except (OSError, ValueError):
|
|
549
|
+
cache = {}
|
|
550
|
+
try:
|
|
551
|
+
mtime = os.path.getmtime(os.path.realpath(path))
|
|
552
|
+
except OSError:
|
|
553
|
+
return ""
|
|
554
|
+
ent = cache.get(path)
|
|
555
|
+
if ent and ent.get("mtime") == mtime:
|
|
556
|
+
return ent.get("version", "")
|
|
557
|
+
out = ""
|
|
558
|
+
try:
|
|
559
|
+
# cwd=HOME: a version probe started from inside a repository would inherit
|
|
560
|
+
# that cwd and read, for one cycle, as a turn running in it.
|
|
561
|
+
r = subprocess.run([path, "--version"], capture_output=True, text=True, encoding=UTF8, errors="replace", timeout=25, cwd=HOME,
|
|
562
|
+
# The platform's separator: a colon ran Windows' last PATH entry into these (#71).
|
|
563
|
+
env=dict(os.environ, PATH=os.pathsep.join((os.environ.get("PATH", ""),
|
|
564
|
+
os.path.dirname(os.path.realpath(path)),
|
|
565
|
+
os.path.dirname(path)))))
|
|
566
|
+
m = re.search(r"(\d+\.\d+\.\d+)", (r.stdout or "") + (r.stderr or ""))
|
|
567
|
+
out = m.group(1) if m else ""
|
|
568
|
+
except Exception:
|
|
569
|
+
out = ""
|
|
570
|
+
cache[path] = {"mtime": mtime, "version": out, "at": int(time.time())}
|
|
571
|
+
try:
|
|
572
|
+
os.makedirs(os.path.dirname(cache_p), exist_ok=True)
|
|
573
|
+
json.dump(cache, open(cache_p, "w", encoding=UTF8))
|
|
574
|
+
except OSError:
|
|
575
|
+
pass
|
|
576
|
+
return out
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _vtuple(v):
|
|
580
|
+
return tuple(int(x) for x in v.split(".")) if v else (0,)
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def resolve_binary(name, path=None):
|
|
584
|
+
"""(path, version) of the newest `name` on this machine; (None, "") if none.
|
|
585
|
+
|
|
586
|
+
An absolute path is returned as-is with its version. Ties keep PATH order.
|
|
587
|
+
"""
|
|
588
|
+
if os.path.isabs(name):
|
|
589
|
+
return (name if os.access(name, os.X_OK) else None), binary_version(name) if os.path.exists(name) else ""
|
|
590
|
+
best = (None, "")
|
|
591
|
+
for cand in binary_candidates(name, path):
|
|
592
|
+
v = binary_version(cand)
|
|
593
|
+
if best[0] is None or _vtuple(v) > _vtuple(best[1]):
|
|
594
|
+
best = (cand, v)
|
|
595
|
+
return best
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
# ---- implementer reports ------------------------------------------------------
|
|
599
|
+
|
|
600
|
+
def _report_summary(path):
|
|
601
|
+
try:
|
|
602
|
+
for line in open(path, errors="replace", encoding=UTF8):
|
|
603
|
+
if line.startswith("#"):
|
|
604
|
+
return line.lstrip("# ").strip()
|
|
605
|
+
except OSError:
|
|
606
|
+
pass
|
|
607
|
+
return ""
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def bump_repeat(path, cfg=None):
|
|
611
|
+
"""Fold a repeated report into the standing one; keep its mtime (its age is the fact).
|
|
612
|
+
|
|
613
|
+
The count is read from its line in either language and written back in the project's, so a
|
|
614
|
+
report that stood before the project changed its language keeps counting (LANGUAGE-FILES).
|
|
615
|
+
"""
|
|
616
|
+
try:
|
|
617
|
+
st = os.stat(path)
|
|
618
|
+
body = open(path, errors="replace", encoding=UTF8).read()
|
|
619
|
+
except OSError:
|
|
620
|
+
return 0
|
|
621
|
+
label = "(?:" + "|".join(re.escape(form) for form in language.forms("repeat")) + ")"
|
|
622
|
+
m = re.search(rf"^{label}: (\d+)", body, re.M)
|
|
623
|
+
n = int(m.group(1)) + 1 if m else 2
|
|
624
|
+
line = language.text(cfg, "mail.repeated", repeat=language.marker(cfg, "repeat"), n=n,
|
|
625
|
+
at=time.strftime('%Y-%m-%d %H:%M'))
|
|
626
|
+
body = re.sub(rf"^{label}: \d+.*$", line, body, flags=re.M) if m else body.rstrip("\n") + "\n\n" + line + "\n"
|
|
627
|
+
try:
|
|
628
|
+
open(path, "w", encoding=UTF8).write(body)
|
|
629
|
+
os.utime(path, (st.st_atime, st.st_mtime))
|
|
630
|
+
except OSError:
|
|
631
|
+
pass
|
|
632
|
+
return n
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _name_time(name):
|
|
636
|
+
m = re.match(r"(\d{8})-(\d{4})", name)
|
|
637
|
+
if not m:
|
|
638
|
+
return None
|
|
639
|
+
try:
|
|
640
|
+
return time.mktime(time.strptime(m.group(1) + m.group(2), "%Y%m%d%H%M"))
|
|
641
|
+
except ValueError:
|
|
642
|
+
return None
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def implementer_inbox(root, cfg):
|
|
646
|
+
"""Mail addressed to the implementer: not its own reports, not the watchdog's."""
|
|
647
|
+
box = cfg.get("mailbox", "agent-mail")
|
|
648
|
+
return [m for m in mailbox(root, box) if addressed_to(m, cfg, "implementer")]
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def product_dirty(root, cfg, strict=False):
|
|
652
|
+
"""Uncommitted paths outside the coordination directories."""
|
|
653
|
+
out = []
|
|
654
|
+
# Not sh(): it strips its output, and the first line's leading status column
|
|
655
|
+
# went with it - " M .polyctl/ledger/notices.jsonl" became "M .polyctl/...", the path
|
|
656
|
+
# lost its first character, and a coordination file read as a product change.
|
|
657
|
+
# Retirement needs a successful status read; advisory callers keep their fallback (TRIAGE-3-MEDIUM).
|
|
658
|
+
try:
|
|
659
|
+
status = _git_output(root, "status", "--porcelain").decode(UTF8, "replace")
|
|
660
|
+
except Exception:
|
|
661
|
+
if strict:
|
|
662
|
+
raise RuntimeError("status unreadable; inspect the worktree and retry pruning")
|
|
663
|
+
status = ""
|
|
664
|
+
for line in status.split("\n"):
|
|
665
|
+
if not line.strip():
|
|
666
|
+
continue
|
|
667
|
+
raw = line[3:].strip().strip('"')
|
|
668
|
+
paths = [part.strip().strip('"') for part in raw.split(" -> ")]
|
|
669
|
+
# A rename crossing the boundary is a product change; ignore it only
|
|
670
|
+
# when both its old and new names are coordination state.
|
|
671
|
+
if paths and all(_is_coordination_path(path, cfg) for path in paths):
|
|
672
|
+
continue
|
|
673
|
+
out.append(line)
|
|
674
|
+
return out
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def waiting_on_architect(root, cfg):
|
|
678
|
+
"""The implementer's standing request the architect has not answered.
|
|
679
|
+
|
|
680
|
+
Returns (report, asked_at) when the newest implementer report asks for a
|
|
681
|
+
decision, nothing addressed to the implementer arrived after it, and the
|
|
682
|
+
board has nothing queued. Nudging in that state produces reports, not
|
|
683
|
+
progress: eighty of them, once, at eight-minute intervals.
|
|
684
|
+
"""
|
|
685
|
+
box = cfg.get("mailbox", "agent-mail")
|
|
686
|
+
files = mailbox(root, box)
|
|
687
|
+
reports = [m for m in files if to_architect(m, cfg) and not from_watchdog(m)
|
|
688
|
+
and not from_architect(m, cfg)]
|
|
689
|
+
if not reports:
|
|
690
|
+
return None
|
|
691
|
+
latest = reports[-1]
|
|
692
|
+
p = os.path.join(root, box, latest)
|
|
693
|
+
try:
|
|
694
|
+
low = language.folded(open(p, errors="replace", encoding=UTF8).read(4000))
|
|
695
|
+
except OSError:
|
|
696
|
+
return None
|
|
697
|
+
# In every language a project may choose, whichever this one writes (LANGUAGE-FILES).
|
|
698
|
+
if not any(h in low for h in language.lowered("decision", "urgent") + ("## blocked",)):
|
|
699
|
+
return None
|
|
700
|
+
# Delivery is by deletion, so any file addressed to the implementer is
|
|
701
|
+
# unread — whatever its timestamp. An answer written while the implementer
|
|
702
|
+
# was repeating its request is older than the repeat and still the answer.
|
|
703
|
+
if implementer_inbox(root, cfg):
|
|
704
|
+
return None
|
|
705
|
+
if board(root)["queued"]:
|
|
706
|
+
return None
|
|
707
|
+
return latest, _name_time(latest) or os.path.getmtime(p)
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
# ---- mail envelope, names, ledger --------------------------------------------
|
|
711
|
+
|
|
712
|
+
def mail_names(cfg):
|
|
713
|
+
"""(implementer, architect) mail names; defaults keep the historical files valid."""
|
|
714
|
+
impl = (settings.get(cfg, "implementer.name") or implementer_actor_name(cfg)).strip()
|
|
715
|
+
arch = settings.get(cfg, "architect.name").strip()
|
|
716
|
+
return safe_slug(impl, "implementer"), safe_slug(arch, "architect")
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
# The architect's name in mail was a model's, "fable", by default (#72). A project that never named its architect
|
|
720
|
+
# wrote its mail to that name, so there it is still the architect's.
|
|
721
|
+
LEGACY_ARCHITECT_NAMES = ("fable",)
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def architect_mail_names(cfg):
|
|
725
|
+
"""Every name a message may address the architect by: its own, the role's, and - where no project or machine
|
|
726
|
+
names one - the name the default used to be (#72)."""
|
|
727
|
+
impl, arch = mail_names(cfg)
|
|
728
|
+
names = {arch, "architect"}
|
|
729
|
+
if settings.resolve(cfg, "architect.name")[1] == "default":
|
|
730
|
+
# never the implementer's own name: its mail stays its own
|
|
731
|
+
names.update(legacy for legacy in LEGACY_ARCHITECT_NAMES if legacy != impl.lower())
|
|
732
|
+
return names
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
def reviewer_mail_names(cfg):
|
|
736
|
+
"""Raw and note-normalized reviewer addresses: model ids may contain slashes or exceed 40 chars (TRIAGE-3-LOW)."""
|
|
737
|
+
reviewer = cfg.get("reviewer") or {}
|
|
738
|
+
names = {str(reviewer[key]).strip() for key in ("actor", "name", "id") if reviewer.get(key)}
|
|
739
|
+
return {"reviewer"} | names | {safe_slug(name) for name in names}
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def to_reviewer(name, cfg):
|
|
743
|
+
"""Reviewer mail must not become implementer work (TRIAGE-3-LOW)."""
|
|
744
|
+
return any(f"-to-{reviewer.lower()}-" in name.lower() for reviewer in reviewer_mail_names(cfg))
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def to_architect(name, cfg):
|
|
748
|
+
return any(f"-to-{arch}-" in name for arch in architect_mail_names(cfg))
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def from_architect(name, cfg):
|
|
752
|
+
"""Whether a mail was written by the architect, read from its sender field (#18)."""
|
|
753
|
+
found = re.match(r"^\d{8}-\d{4}-(.+?)-to-", name)
|
|
754
|
+
return bool(found) and found.group(1).lower() in {arch.lower() for arch in architect_mail_names(cfg)}
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def notice_recently_recorded(root, key, window):
|
|
758
|
+
"""Was this key recorded inside the window at all, delivered or held (#69)?
|
|
759
|
+
|
|
760
|
+
An architect-audience notice is recorded unsent by design, so a check on sent
|
|
761
|
+
rows never held for one, and the same anomaly wrote a row every cycle. The whole
|
|
762
|
+
window counts, whatever the ledger still holds (NOTICE-WINDOW).
|
|
763
|
+
"""
|
|
764
|
+
return _notice_within(root, key, window, sent=False)
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
SECRET_PATTERNS = (
|
|
768
|
+
r"sk-[A-Za-z0-9_-]{16,}",
|
|
769
|
+
r"gh[pousr]_[A-Za-z0-9]{20,}",
|
|
770
|
+
r"(?:xox[abprs]|xapp)-[A-Za-z0-9-]{10,}",
|
|
771
|
+
r"AKIA[0-9A-Z]{16}",
|
|
772
|
+
r"(?i)bearer\s+[A-Za-z0-9._~+/=-]{16,}",
|
|
773
|
+
r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}",
|
|
774
|
+
r"[A-Za-z0-9+_=-]{40,}",
|
|
775
|
+
)
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
# What polyctl writes into a repository is scanned for credentials first (#48). Named rules,
|
|
779
|
+
# so a redaction says what it removed; hex digests (commit ids, sha256 values) match
|
|
780
|
+
# none of them, which is why the generic long-token rule `redact` uses is not here.
|
|
781
|
+
EVIDENCE_RULES = (
|
|
782
|
+
("private-key", r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----"),
|
|
783
|
+
("anthropic-key", r"sk-ant-[A-Za-z0-9_-]{16,}"),
|
|
784
|
+
("openai-key", r"sk-(?:proj-)?[A-Za-z0-9_-]{20,}"),
|
|
785
|
+
("github-token", r"(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})"),
|
|
786
|
+
# Slack's app-level tokens begin `xapp-` (EVIDENCE-SCAN-2).
|
|
787
|
+
("slack-token", r"(?:xox[abprs]|xapp)-[A-Za-z0-9-]{10,}"),
|
|
788
|
+
("aws-access-key", r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"),
|
|
789
|
+
("jwt", r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"),
|
|
790
|
+
("bearer-token", r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{16,}"),
|
|
791
|
+
# The name is read with what an environment puts before it - DB_PASSWORD, CLIENT_SECRET, NPM_TOKEN,
|
|
792
|
+
# AWS_SECRET_ACCESS_KEY - where `\b` is no boundary after an underscore and none of them matched
|
|
793
|
+
# (EVIDENCE-SCAN-2). A bare `token` is polyctl's own grant id, and a name that goes on past the secret word
|
|
794
|
+
# (`password_env`, `secret_file`) names where one is kept, not the secret; neither is read.
|
|
795
|
+
("assigned-secret", r"(?i)(?<![A-Za-z0-9])(?:[A-Za-z0-9]+_)*(?:api[_-]?key|secret(?:[_-]access)?(?:[_-]key)?|"
|
|
796
|
+
r"password|passwd|(?:access|auth)[_-]?token|[A-Za-z0-9]+_token|private[_-]?key)"
|
|
797
|
+
r"(?![A-Za-z0-9_])[\"']?\s*[:=]\s*[\"']?[A-Za-z0-9+/_.=-]{12,}"),
|
|
798
|
+
)
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def scan_evidence(text):
|
|
802
|
+
"""(text with every credential-shaped string replaced by `[redacted:<rule>]`, the rules that hit)."""
|
|
803
|
+
out = str(text if text is not None else "")
|
|
804
|
+
hits = []
|
|
805
|
+
for rule, pattern in EVIDENCE_RULES:
|
|
806
|
+
out, count = re.subn(pattern, f"[redacted:{rule}]", out)
|
|
807
|
+
if count:
|
|
808
|
+
hits.append(rule)
|
|
809
|
+
return out, hits
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def scan_record(value):
|
|
813
|
+
"""A JSON-shaped value with every string in it scanned (#48)."""
|
|
814
|
+
if isinstance(value, str):
|
|
815
|
+
return scan_evidence(value)[0]
|
|
816
|
+
if isinstance(value, dict):
|
|
817
|
+
return {key: scan_record(item) for key, item in value.items()}
|
|
818
|
+
if isinstance(value, list):
|
|
819
|
+
return [scan_record(item) for item in value]
|
|
820
|
+
return value
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def redact(text):
|
|
824
|
+
"""Text from an agent's own output with anything token-shaped masked (#69).
|
|
825
|
+
|
|
826
|
+
A wake or nudge log merges a model's prose with its process's errors, and a
|
|
827
|
+
line of it went to the committed notices ledger, Telegram and e-mail. What
|
|
828
|
+
leaves the log is masked first.
|
|
829
|
+
"""
|
|
830
|
+
out = str(text or "")
|
|
831
|
+
for pattern in SECRET_PATTERNS:
|
|
832
|
+
out = re.sub(pattern, "[redacted]", out)
|
|
833
|
+
return out
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def record_actor_flags(root, actor, flags, reason):
|
|
837
|
+
"""Record a flag polyctl adds to an actor's argv, with why, each time that changes (#69)."""
|
|
838
|
+
from .storage import append_jsonl, read_jsonl
|
|
839
|
+
path = os.path.join(root, ".polyctl", "ledger", "actor-flags.jsonl")
|
|
840
|
+
try:
|
|
841
|
+
rows = read_jsonl(path)
|
|
842
|
+
except Exception:
|
|
843
|
+
rows = []
|
|
844
|
+
last = next((row for row in reversed(rows)
|
|
845
|
+
if isinstance(row, dict) and row.get("actor") == actor), None)
|
|
846
|
+
if last and last.get("flags") == list(flags) and last.get("reason") == reason:
|
|
847
|
+
return last
|
|
848
|
+
return append_jsonl(path, {"at": int(time.time()), "actor": actor,
|
|
849
|
+
"flags": list(flags), "reason": reason})
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def from_watchdog(name):
|
|
853
|
+
return name.startswith("watchdog-to-") or "-watchdog-to-" in name
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def write_mail(root, cfg, name, body, meta=None):
|
|
857
|
+
"""Every mail polyctl itself writes goes through here: envelope first, prose after.
|
|
858
|
+
|
|
859
|
+
The body stays Markdown — agents and people read it. The envelope is a small
|
|
860
|
+
front-matter block a program can read without parsing prose: kind, from, to,
|
|
861
|
+
slice, time, id. And every write is mirrored to `.polyctl/ledger/mail.jsonl`, so
|
|
862
|
+
that delivery-by-deletion stops deleting the record: a mailbox that was
|
|
863
|
+
emptied is still searchable, and "how long did that request stand" has an
|
|
864
|
+
answer after the file is gone.
|
|
865
|
+
"""
|
|
866
|
+
box = os.path.join(root, cfg.get("mailbox", "agent-mail"))
|
|
867
|
+
os.makedirs(box, exist_ok=True)
|
|
868
|
+
meta = dict(meta or {})
|
|
869
|
+
meta.setdefault("polyctl", 1)
|
|
870
|
+
meta.setdefault("at", time.strftime("%Y-%m-%dT%H:%M:%S%z"))
|
|
871
|
+
# The address is the role; the name beside it is display (#31).
|
|
872
|
+
meta.setdefault("from_role", role_of(meta.get("from"), cfg))
|
|
873
|
+
meta.setdefault("to_role", role_of(meta.get("to"), cfg))
|
|
874
|
+
order = ["polyctl", "id", "kind", "from", "from_role", "to", "to_role", "slice", "at"]
|
|
875
|
+
keys = [k for k in order if k in meta] + [k for k in meta if k not in order]
|
|
876
|
+
meta = scan_record(meta)
|
|
877
|
+
body = scan_evidence(body)[0] # scanned before it is written (#48)
|
|
878
|
+
# A handled message leaves its view but still owns its ID. Serialize allocation through
|
|
879
|
+
# recording so a concurrent writer cannot reuse an ID after the view vanishes (TRIAGE-2-MEDIUM).
|
|
880
|
+
from .storage import _exclusive_lock, open_new_text
|
|
881
|
+
with _exclusive_lock(os.path.join(root, ".polyctl", "mail", "write.lock")):
|
|
882
|
+
reserved = {row.get("id") for row in mail_log(root, limit=None) if isinstance(row, dict)}
|
|
883
|
+
reserved.update(row.get("id") for row in mail_store_rows(root))
|
|
884
|
+
store = os.path.join(root, ".polyctl", "mail", "store")
|
|
885
|
+
if os.path.isdir(store):
|
|
886
|
+
reserved.update(os.listdir(store))
|
|
887
|
+
stem, extension = os.path.splitext(name)
|
|
888
|
+
attempt = 1
|
|
889
|
+
while True:
|
|
890
|
+
if name in reserved:
|
|
891
|
+
attempt += 1
|
|
892
|
+
name = f"{stem}-{attempt}{extension}"
|
|
893
|
+
continue
|
|
894
|
+
try:
|
|
895
|
+
fh = open_new_text(os.path.join(box, name))
|
|
896
|
+
break
|
|
897
|
+
except FileExistsError:
|
|
898
|
+
attempt += 1
|
|
899
|
+
name = f"{stem}-{attempt}{extension}"
|
|
900
|
+
meta["id"] = name
|
|
901
|
+
if "id" not in keys:
|
|
902
|
+
keys.insert(1, "id")
|
|
903
|
+
head = "---\n" + "".join(f"{k}: {meta[k]}\n" for k in keys if meta[k] not in (None, "")) + "---\n"
|
|
904
|
+
with fh:
|
|
905
|
+
fh.write(head + body.lstrip("\n"))
|
|
906
|
+
summary = next((l.lstrip("# ").strip() for l in body.split("\n") if l.startswith("#")), "")
|
|
907
|
+
mail_ledger_append(root, {"event": "written", "id": name, "kind": meta.get("kind"),
|
|
908
|
+
"from": meta.get("from"), "to": meta.get("to"),
|
|
909
|
+
"slice": meta.get("slice"), "summary": summary[:200]})
|
|
910
|
+
# Named, never quoted: the log is read by programs a message's body was not written for (EVENTS-LOG).
|
|
911
|
+
emit_event(root, "mail-sent", {"message": name, "kind": meta.get("kind"), "from": meta.get("from"),
|
|
912
|
+
"to": meta.get("to")})
|
|
913
|
+
# Mail polyctl writes reaches the append-only store at once: it reached it at the next reconcile, and a view
|
|
914
|
+
# file removed before then took its message with it (MESSAGE-STORE-2). The view file stands whatever the
|
|
915
|
+
# store says; one the store cannot take now is taken at the next reconcile, and the doctor names a store
|
|
916
|
+
# it cannot read.
|
|
917
|
+
if mail_store_mode(root) == "append-only":
|
|
918
|
+
try:
|
|
919
|
+
ingest_mail(root, cfg)
|
|
920
|
+
except Exception:
|
|
921
|
+
pass
|
|
922
|
+
return name
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def mail_meta(path):
|
|
926
|
+
"""The envelope of a mail file, or {} for a file written without one."""
|
|
927
|
+
try:
|
|
928
|
+
head = open(path, errors="replace", encoding=UTF8).read(2000)
|
|
929
|
+
except OSError:
|
|
930
|
+
return {}
|
|
931
|
+
if not head.startswith("---\n"):
|
|
932
|
+
return {}
|
|
933
|
+
end = head.find("\n---", 4)
|
|
934
|
+
if end < 0:
|
|
935
|
+
return {}
|
|
936
|
+
out = {}
|
|
937
|
+
for line in head[4:end].split("\n"):
|
|
938
|
+
if ":" in line:
|
|
939
|
+
k, v = line.split(":", 1)
|
|
940
|
+
out[k.strip()] = v.strip()
|
|
941
|
+
return out
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
# ---- unread age escalates on the ladder (#30) --------------------------------------------
|
|
945
|
+
|
|
946
|
+
MAIL_CLASSES = ("fyi", "needs-read", "needs-decision", "urgent")
|
|
947
|
+
_DECISION_KINDS = language.words("decision-kinds") # in every language, whichever the project writes
|
|
948
|
+
_FYI_KINDS = language.words("fyi-kinds")
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def mail_class(name, meta=None, body=""):
|
|
952
|
+
"""What a message asks of whoever reads it: fyi, needs-read, needs-decision or urgent (#30).
|
|
953
|
+
|
|
954
|
+
Declared in the envelope (`class:`) when the writer said; otherwise read from
|
|
955
|
+
its kind and its headings, the way the watchdog already reads a request.
|
|
956
|
+
"""
|
|
957
|
+
meta = meta or {}
|
|
958
|
+
declared = str(meta.get("class") or "").strip().lower()
|
|
959
|
+
if declared in MAIL_CLASSES:
|
|
960
|
+
return declared
|
|
961
|
+
kind = str(meta.get("kind") or "").strip().lower()
|
|
962
|
+
if not kind:
|
|
963
|
+
found = re.match(r"^\d{8}-\d{4}-.+?-to-.+?-([a-z]+)-", name, re.I)
|
|
964
|
+
kind = found.group(1).lower() if found else ""
|
|
965
|
+
low = language.folded(body or "")
|
|
966
|
+
if kind in _DECISION_KINDS or any(mark in low for mark in language.lowered("decision")):
|
|
967
|
+
return "needs-decision"
|
|
968
|
+
return "fyi" if kind in _FYI_KINDS else "needs-read"
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def addressed_to(name, cfg, role):
|
|
972
|
+
"""Whether a message is for a role: the architect's inbox, or the implementer's; a person reads both."""
|
|
973
|
+
if role == "architect":
|
|
974
|
+
return to_architect(name, cfg)
|
|
975
|
+
if role == "implementer":
|
|
976
|
+
return not to_architect(name, cfg) and not to_reviewer(name, cfg) and not from_watchdog(name)
|
|
977
|
+
return True
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
def mail_seen(root, names, by):
|
|
981
|
+
"""Record, once each, that messages were put in front of their reader (#30).
|
|
982
|
+
|
|
983
|
+
Seeing is not handling: handling stays deletion, and a message seen and not
|
|
984
|
+
yet handled is still in the mailbox. What this separates is "nobody has been
|
|
985
|
+
shown it" from "someone is working on it".
|
|
986
|
+
"""
|
|
987
|
+
already = {row.get("id") for row in mail_log(root, 5000) if row.get("event") == "seen"}
|
|
988
|
+
fresh = [name for name in names if name not in already]
|
|
989
|
+
for name in fresh:
|
|
990
|
+
mail_ledger_append(root, {"event": "seen", "id": name, "by": by})
|
|
991
|
+
return fresh
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def unseen_messages(root, cfg):
|
|
995
|
+
"""Messages in the mailbox nobody has been shown, oldest first, with their class and age (#30)."""
|
|
996
|
+
box = cfg.get("mailbox", "agent-mail")
|
|
997
|
+
rows = mail_log(root, 5000)
|
|
998
|
+
seen = {row.get("id") for row in rows if row.get("event") == "seen"}
|
|
999
|
+
written = {row.get("id"): row.get("at") for row in rows if row.get("event") == "written"}
|
|
1000
|
+
out = []
|
|
1001
|
+
for name in mailbox(root, box):
|
|
1002
|
+
if name in seen or from_watchdog(name):
|
|
1003
|
+
continue
|
|
1004
|
+
path = os.path.join(root, box, name)
|
|
1005
|
+
# A message whose body cannot be read, or whose written time in the ledger is no number, is still one
|
|
1006
|
+
# nobody has been shown: it was dropped, and a decision request so read never climbed the ladder. It is
|
|
1007
|
+
# classed by its name and envelope, and aged by its file (UNREAD-AGE-2).
|
|
1008
|
+
try:
|
|
1009
|
+
with open(path, errors="replace", encoding=UTF8) as fh:
|
|
1010
|
+
body = fh.read(1_000_000)
|
|
1011
|
+
except OSError:
|
|
1012
|
+
body = ""
|
|
1013
|
+
try:
|
|
1014
|
+
at = float(written.get(name))
|
|
1015
|
+
except (TypeError, ValueError):
|
|
1016
|
+
try:
|
|
1017
|
+
at = os.path.getmtime(path)
|
|
1018
|
+
except OSError:
|
|
1019
|
+
continue # gone since the listing
|
|
1020
|
+
out.append({"id": name, "class": mail_class(name, mail_meta(path), body), "at": at,
|
|
1021
|
+
"age": max(0.0, time.time() - at)})
|
|
1022
|
+
return sorted(out, key=lambda message: message["at"])
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
# ---- nothing is deleted to prove it was handled (#80) ------------------------------------
|
|
1026
|
+
|
|
1027
|
+
MAIL_STORE_CHAIN = "polyctl-mail-store-row-v1"
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
def mail_store_mode(root):
|
|
1031
|
+
"""`append-only` when the project has switched its mailbox to the store, else `deletion` (#80)."""
|
|
1032
|
+
try:
|
|
1033
|
+
return settings.get(load_config(root, record=False), "mail.store")
|
|
1034
|
+
except Exception:
|
|
1035
|
+
return "deletion"
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def mail_store_rows(root):
|
|
1039
|
+
from .storage import read_chained_jsonl
|
|
1040
|
+
return [row for row in read_chained_jsonl(os.path.join(root, ".polyctl", "ledger", "mail-store.jsonl"), MAIL_STORE_CHAIN)
|
|
1041
|
+
if isinstance(row, dict)]
|
|
1042
|
+
|
|
1043
|
+
|
|
1044
|
+
def _mail_store_append(root, row):
|
|
1045
|
+
from .storage import append_chained_jsonl
|
|
1046
|
+
return append_chained_jsonl(os.path.join(root, ".polyctl", "ledger", "mail-store.jsonl"),
|
|
1047
|
+
scan_record(dict(row, at=int(time.time()))), MAIL_STORE_CHAIN)
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def _store_path(root, mid):
|
|
1051
|
+
return os.path.join(root, ".polyctl", "mail", "store", os.path.basename(mid))
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def ingest_mail(root, cfg):
|
|
1055
|
+
"""Take every message in the mailbox view into the append-only store, once (#80).
|
|
1056
|
+
|
|
1057
|
+
A body stored with no row - the row's append failed after the body landed, and the view was
|
|
1058
|
+
removed before the next reconcile - is taken from the store itself; only the view was read,
|
|
1059
|
+
and the message was gone from the queue with its body orphaned (MESSAGE-STORE-3).
|
|
1060
|
+
"""
|
|
1061
|
+
from .storage import replace_file_durably
|
|
1062
|
+
box = os.path.join(root, cfg.get("mailbox", "agent-mail"))
|
|
1063
|
+
known = {row.get("id") for row in mail_store_rows(root) if row.get("event") == "message"}
|
|
1064
|
+
views = set(os.listdir(box)) if os.path.isdir(box) else set()
|
|
1065
|
+
store = os.path.join(root, ".polyctl", "mail", "store")
|
|
1066
|
+
stored = set(os.listdir(store)) if os.path.isdir(store) else set()
|
|
1067
|
+
taken = []
|
|
1068
|
+
for name in sorted(views | stored):
|
|
1069
|
+
if name == "README.md" or not name.endswith(".md") or name in known:
|
|
1070
|
+
continue
|
|
1071
|
+
source = os.path.join(box, name) if name in views else _store_path(root, name)
|
|
1072
|
+
with open(source, "rb") as fh:
|
|
1073
|
+
data = fh.read()
|
|
1074
|
+
if name not in views and data.startswith(b"polyctl-mail-stub v1\n"):
|
|
1075
|
+
continue
|
|
1076
|
+
if name in views:
|
|
1077
|
+
replace_file_durably(_store_path(root, name), data)
|
|
1078
|
+
meta = mail_meta(source)
|
|
1079
|
+
_mail_store_append(root, {"event": "message", "id": name,
|
|
1080
|
+
"digest": "sha256:" + hashlib.sha256(data).hexdigest(),
|
|
1081
|
+
"from": meta.get("from"), "to": meta.get("to"), "kind": meta.get("kind")})
|
|
1082
|
+
taken.append(name)
|
|
1083
|
+
return taken
|
|
1084
|
+
|
|
1085
|
+
|
|
1086
|
+
def unhandled_messages(root):
|
|
1087
|
+
"""The unhandled queue, derived: every stored message with no handling record (#80)."""
|
|
1088
|
+
rows = mail_store_rows(root)
|
|
1089
|
+
handled = {row.get("id") for row in rows if row.get("event") == "handled"}
|
|
1090
|
+
return sorted({row.get("id") for row in rows if row.get("event") == "message"} - handled)
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def handle_message(root, cfg, mid, by, outcome):
|
|
1094
|
+
"""Record that a message was handled, by whom and how; its view file then goes, its record never (#80)."""
|
|
1095
|
+
if mid not in unhandled_messages(root):
|
|
1096
|
+
return False
|
|
1097
|
+
_mail_store_append(root, {"event": "handled", "id": mid, "by": by, "outcome": outcome})
|
|
1098
|
+
view = os.path.join(root, cfg.get("mailbox", "agent-mail"), mid)
|
|
1099
|
+
try:
|
|
1100
|
+
os.remove(view)
|
|
1101
|
+
except FileNotFoundError:
|
|
1102
|
+
pass
|
|
1103
|
+
return True
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
def message_body(root, mid):
|
|
1107
|
+
"""A stored message's bytes, read through a compaction stub when it has one (#80)."""
|
|
1108
|
+
import gzip
|
|
1109
|
+
try:
|
|
1110
|
+
with open(_store_path(root, mid), "rb") as fh:
|
|
1111
|
+
data = fh.read()
|
|
1112
|
+
except OSError:
|
|
1113
|
+
data = None
|
|
1114
|
+
if data is not None and data.startswith(b"polyctl-mail-stub v1\n"):
|
|
1115
|
+
archive = data.decode(UTF8).split("archive: ", 1)[1].strip()
|
|
1116
|
+
with gzip.open(os.path.join(root, archive), "rb") as fh:
|
|
1117
|
+
return fh.read()
|
|
1118
|
+
return data
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def sync_mail_view(root, cfg):
|
|
1122
|
+
"""Make the mailbox view the derived queue: restore what was removed unhandled, drop what was handled (#80).
|
|
1123
|
+
|
|
1124
|
+
An agent may delete a view file; that removes nothing. The message comes back
|
|
1125
|
+
until someone records handling it, because the actor deciding what was handled
|
|
1126
|
+
must not also be able to erase the question.
|
|
1127
|
+
"""
|
|
1128
|
+
from .storage import replace_file_durably
|
|
1129
|
+
box = os.path.join(root, cfg.get("mailbox", "agent-mail"))
|
|
1130
|
+
queue = set(unhandled_messages(root))
|
|
1131
|
+
restored, dropped = [], []
|
|
1132
|
+
for mid in sorted(queue):
|
|
1133
|
+
view = os.path.join(box, mid)
|
|
1134
|
+
if not os.path.exists(view):
|
|
1135
|
+
body = message_body(root, mid)
|
|
1136
|
+
if body is not None:
|
|
1137
|
+
replace_file_durably(view, body)
|
|
1138
|
+
restored.append(mid)
|
|
1139
|
+
stored = {row.get("id") for row in mail_store_rows(root) if row.get("event") == "message"}
|
|
1140
|
+
for name in (sorted(os.listdir(box)) if os.path.isdir(box) else []):
|
|
1141
|
+
if name in stored and name not in queue:
|
|
1142
|
+
os.remove(os.path.join(box, name))
|
|
1143
|
+
dropped.append(name)
|
|
1144
|
+
return restored, dropped
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
def compact_messages(root, days, now=None):
|
|
1148
|
+
"""Collapse stored bodies older than `days` to a stub with their digest and an archive pointer (#80)."""
|
|
1149
|
+
import gzip
|
|
1150
|
+
from .storage import replace_file_durably
|
|
1151
|
+
cutoff = (time.time() if now is None else now) - days * 86400
|
|
1152
|
+
compacted = []
|
|
1153
|
+
done = {row.get("id") for row in mail_store_rows(root) if row.get("event") == "compacted"}
|
|
1154
|
+
for row in mail_store_rows(root):
|
|
1155
|
+
mid = row.get("id")
|
|
1156
|
+
if row.get("event") != "message" or mid in done or float(row.get("at") or 0) > cutoff:
|
|
1157
|
+
continue
|
|
1158
|
+
path = _store_path(root, mid)
|
|
1159
|
+
try:
|
|
1160
|
+
with open(path, "rb") as fh:
|
|
1161
|
+
data = fh.read()
|
|
1162
|
+
except OSError:
|
|
1163
|
+
continue
|
|
1164
|
+
archive = f".polyctl/mail/archive/{mid}.gz"
|
|
1165
|
+
if data.startswith(b"polyctl-mail-stub v1\n"):
|
|
1166
|
+
# A compaction whose record did not land left its stub, and its archive holds the body: the stub
|
|
1167
|
+
# was read as the body and written over the archive, and the words were gone. The record is what
|
|
1168
|
+
# was missing (MESSAGE-STORE-2).
|
|
1169
|
+
_mail_store_append(root, {"event": "compacted", "id": mid, "archive": archive, "digest": row.get("digest")})
|
|
1170
|
+
compacted.append(mid)
|
|
1171
|
+
done.add(mid)
|
|
1172
|
+
continue
|
|
1173
|
+
os.makedirs(os.path.dirname(os.path.join(root, archive)), exist_ok=True)
|
|
1174
|
+
# The archive is written as durably as the stub that replaces the body: written plainly, a crash could
|
|
1175
|
+
# leave a stub pointing at an archive that never reached the disk (MESSAGE-STORE-2).
|
|
1176
|
+
replace_file_durably(os.path.join(root, archive), gzip.compress(data, mtime=0))
|
|
1177
|
+
replace_file_durably(path, f"polyctl-mail-stub v1\ndigest: {row.get('digest')}\narchive: {archive}\n".encode(UTF8))
|
|
1178
|
+
_mail_store_append(root, {"event": "compacted", "id": mid, "archive": archive, "digest": row.get("digest")})
|
|
1179
|
+
compacted.append(mid)
|
|
1180
|
+
done.add(mid) # two writers can each have stored it: it is compacted once
|
|
1181
|
+
return compacted
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
def room_search(text, root=None, limit=20):
|
|
1185
|
+
"""Stored messages in every registered project whose body mentions `text` (#80)."""
|
|
1186
|
+
needle = str(text or "").lower()
|
|
1187
|
+
found = []
|
|
1188
|
+
for project, path in recall_roots(root):
|
|
1189
|
+
try:
|
|
1190
|
+
rows = [row for row in mail_store_rows(path) if row.get("event") == "message"]
|
|
1191
|
+
except Exception:
|
|
1192
|
+
continue
|
|
1193
|
+
for row in rows:
|
|
1194
|
+
body = message_body(path, row["id"])
|
|
1195
|
+
if body is not None and needle in body.decode(UTF8, "replace").lower():
|
|
1196
|
+
found.append(dict(row, project=project))
|
|
1197
|
+
found.sort(key=lambda row: -float(row.get("at") or 0))
|
|
1198
|
+
return found[:limit]
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
# ---- the message store syncs to one private repository, never the product's remote (#83) --
|
|
1202
|
+
|
|
1203
|
+
def _remote_identity(url):
|
|
1204
|
+
"""(host, path) a git URL names, alike for its https, ssh and host:path forms; None for a path on this machine."""
|
|
1205
|
+
url = (url or "").strip()
|
|
1206
|
+
found = re.match(r"^[A-Za-z][A-Za-z0-9+.-]*://(?:[^@/]+@)?([^/:]+)(?::\d+)?/(.+)$", url)
|
|
1207
|
+
if not found and _scp_like(url):
|
|
1208
|
+
found = re.match(r"^(?:[^@]+@)?([^:]+):(.+)$", url)
|
|
1209
|
+
if not found:
|
|
1210
|
+
return None
|
|
1211
|
+
return found.group(1).lower(), found.group(2).strip("/").lower().removesuffix(".git") # .GIT too (MAIL-SYNC-3)
|
|
1212
|
+
|
|
1213
|
+
|
|
1214
|
+
def _scp_like(url):
|
|
1215
|
+
"""Whether git reads `url` as host:path over SSH: a colon before any slash, with or without a user (MAIL-SYNC-2)."""
|
|
1216
|
+
colon, slash = url.find(":"), url.find("/")
|
|
1217
|
+
return colon > 0 and not 0 <= slash < colon and not (os.name == "nt" and re.match(r"^[A-Za-z]:", url))
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
def _git_config_entries(root, pattern):
|
|
1221
|
+
"""(key, value) of every git config entry whose key matches `pattern`, from every file git reads (MAIL-SYNC-4).
|
|
1222
|
+
|
|
1223
|
+
git answers 1 when no key matches; any other failure is no answer, and is not taken for an empty one.
|
|
1224
|
+
"""
|
|
1225
|
+
try:
|
|
1226
|
+
result = subprocess.run([git_binary(), "config", "-z", "--get-regexp", pattern], cwd=root,
|
|
1227
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
|
|
1228
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
1229
|
+
raise RuntimeError(f"cannot read git config: {exc}") from exc
|
|
1230
|
+
if result.returncode == 1:
|
|
1231
|
+
return []
|
|
1232
|
+
if result.returncode:
|
|
1233
|
+
raise RuntimeError(f"cannot read git config: {result.stderr.decode(UTF8, 'replace').strip()[:240]}")
|
|
1234
|
+
return [tuple(record.partition("\n")[::2]) for record in result.stdout.decode(UTF8, "replace").split("\0")
|
|
1235
|
+
if record]
|
|
1236
|
+
|
|
1237
|
+
|
|
1238
|
+
def _git_rewrite(url, entries, rule):
|
|
1239
|
+
"""(url as git rewrites it, the rule's key, its prefix) by the url.<base>.<rule> entries among `entries`; else None.
|
|
1240
|
+
|
|
1241
|
+
git's alias_url: the longest prefix wins, and of two as long, the one whose base git read first. The
|
|
1242
|
+
first entry that matched was taken, and a refusal could name a place git would not push (MAIL-SYNC-6).
|
|
1243
|
+
"""
|
|
1244
|
+
rewrites = {} # each base with its prefixes, in the order git first reads it
|
|
1245
|
+
for key, value in entries:
|
|
1246
|
+
if key.startswith("url.") and key.endswith("." + rule):
|
|
1247
|
+
rewrites.setdefault(key[4:key.rindex(".")], []).append((key, value))
|
|
1248
|
+
best = None
|
|
1249
|
+
for base, rules in rewrites.items():
|
|
1250
|
+
for key, prefix in rules:
|
|
1251
|
+
if url.startswith(prefix) and (best is None or len(prefix) > len(best[2])):
|
|
1252
|
+
best = (base + url[len(prefix):], key, prefix)
|
|
1253
|
+
return best
|
|
1254
|
+
|
|
1255
|
+
|
|
1256
|
+
def _git_reads_otherwise(root, url):
|
|
1257
|
+
"""Why `git push url` would not reach `url` itself, or None (MAIL-SYNC-4).
|
|
1258
|
+
|
|
1259
|
+
git looks a word up as a remote's name, in every config file, before it reads it as a URL, and
|
|
1260
|
+
rewrites a URL by url.<base>.insteadOf and, when it pushes, url.<base>.pushInsteadOf: `mailbox`
|
|
1261
|
+
was judged a private directory while a pushInsteadOf sent it to another host, a remote's pushurl
|
|
1262
|
+
`origin` was pushed to as the remote origin, and a remote kept in ~/.gitconfig, which
|
|
1263
|
+
`git remote get-url` does not know, was taken for a directory.
|
|
1264
|
+
"""
|
|
1265
|
+
entries = _git_config_entries(root, r"^(remote\..+\.[^.]+|url\..+\.pushinsteadof)$")
|
|
1266
|
+
# What git pushes to is said before what it fetches from: a pushInsteadOf wins over an insteadOf when git
|
|
1267
|
+
# pushes, and the refusal named the fetch's rewrite, a place git would not push (MAIL-SYNC-5).
|
|
1268
|
+
pushed = _git_rewrite(url, entries, "pushinsteadof")
|
|
1269
|
+
if pushed:
|
|
1270
|
+
return f"git pushes {url} to {pushed[0]} ({pushed[1]} = {pushed[2]})"
|
|
1271
|
+
seen = _git_output(root, "ls-remote", "--get-url", url).decode(UTF8, "replace").strip()
|
|
1272
|
+
|
|
1273
|
+
def spelled(text): # a path git prints with its own separators is the same path
|
|
1274
|
+
text = text.replace("\\", "/").rstrip("/")
|
|
1275
|
+
return text.lower() if os.name == "nt" else text
|
|
1276
|
+
if spelled(seen) != spelled(url):
|
|
1277
|
+
return f"git reads {url} as {seen}" # a remote of that name, from any config file, or an insteadOf
|
|
1278
|
+
for key, _ in entries:
|
|
1279
|
+
if key.startswith("remote.") and key[7:key.rindex(".")] == url:
|
|
1280
|
+
return f"{url} is also the name of a remote ({key})"
|
|
1281
|
+
return None
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def _push_url(root, target):
|
|
1285
|
+
"""The one URL `git push target` reaches: the push URL of the remote `target` names, else `target` (MAIL-SYNC-3).
|
|
1286
|
+
|
|
1287
|
+
git takes a configured remote's name before a directory of that name, and pushes to its pushurl
|
|
1288
|
+
where one is set: `origin` was judged as a directory - private, no host asked, never compared with
|
|
1289
|
+
the product's remote - and `git push origin` took the mail to the product. A name is judged, read
|
|
1290
|
+
and pushed as the URL it names. A word that begins with `-` is an option to git, not a repository:
|
|
1291
|
+
`--upload-pack=<command>` ran the command on every `polyctl doctor`, and it is refused before git sees it.
|
|
1292
|
+
The URL is then pushed to and read as itself, so one git would read as another remote or rewrite
|
|
1293
|
+
is refused, naming what git reaches (MAIL-SYNC-4).
|
|
1294
|
+
"""
|
|
1295
|
+
if str(target or "").startswith("-"):
|
|
1296
|
+
raise RuntimeError(f"{target!r} is an option to git, not a repository")
|
|
1297
|
+
try:
|
|
1298
|
+
urls = _git_output(root, "remote", "get-url", "--push", "--all", target).decode(UTF8, "replace")
|
|
1299
|
+
except RuntimeError:
|
|
1300
|
+
urls = target # no remote of that name here: git reads it as a URL or a path
|
|
1301
|
+
else:
|
|
1302
|
+
urls = [url.strip() for url in urls.splitlines() if url.strip()]
|
|
1303
|
+
if len(urls) != 1:
|
|
1304
|
+
raise RuntimeError(f"the remote {target} pushes to {len(urls)} URLs; name one repository")
|
|
1305
|
+
if urls[0].startswith("-"):
|
|
1306
|
+
raise RuntimeError(f"the remote {target} names {urls[0]!r}, an option to git, not a repository")
|
|
1307
|
+
urls = urls[0]
|
|
1308
|
+
why = _git_reads_otherwise(root, urls)
|
|
1309
|
+
if why:
|
|
1310
|
+
raise RuntimeError(f"{why}; name the repository git reaches")
|
|
1311
|
+
return urls
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
def _github_says_private(url):
|
|
1315
|
+
"""True or False as GitHub answers for a repository on github.com itself; None for any other host, or no answer.
|
|
1316
|
+
|
|
1317
|
+
The host is the one git reaches: `github.com/<owner>/<repo>` matched anywhere in the URL let GitHub's
|
|
1318
|
+
answer for an unrelated repository send mail and governance to another host (MAIL-SYNC-3).
|
|
1319
|
+
"""
|
|
1320
|
+
identity = _remote_identity(url)
|
|
1321
|
+
found = identity and identity[0] == "github.com" and re.fullmatch(r"([^/]+)/([^/]+)", identity[1])
|
|
1322
|
+
gh = program_path("gh") if found else None # never one the current directory holds (WINDOWS-NO-CWD-EXEC)
|
|
1323
|
+
if not gh:
|
|
1324
|
+
return None
|
|
1325
|
+
try:
|
|
1326
|
+
answer = subprocess.run([gh, "api", f"repos/{found.group(1)}/{found.group(2)}", "--jq", ".private"],
|
|
1327
|
+
capture_output=True, text=True, encoding=UTF8, errors="replace", timeout=30)
|
|
1328
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
1329
|
+
return None
|
|
1330
|
+
return {"true": True, "false": False}.get(answer.stdout.strip())
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
def _url_is_private(root, url):
|
|
1334
|
+
"""True for a local path, or when the host says the repository is private; None when it cannot say (#83).
|
|
1335
|
+
|
|
1336
|
+
A URL is classified as git classifies it (MAIL-SYNC-2): `github.com:owner/repo.git`, an alias
|
|
1337
|
+
from ~/.ssh/config and `HTTPS://...` were taken for directories on this machine, and mail was
|
|
1338
|
+
pushed with no host asked. A host polyctl cannot ask - an alias, another server - is not confirmed.
|
|
1339
|
+
"""
|
|
1340
|
+
url = url or ""
|
|
1341
|
+
if url and not re.match(r"^[A-Za-z][A-Za-z0-9+.-]*://", url) and not _scp_like(url):
|
|
1342
|
+
return True # a directory on this machine publishes nothing
|
|
1343
|
+
return _github_says_private(url)
|
|
1344
|
+
|
|
1345
|
+
|
|
1346
|
+
def mail_ref_commit(root):
|
|
1347
|
+
"""Commit the message store under refs/polyctl/mail, scanned, parented on the last one; returns the commit (#83)."""
|
|
1348
|
+
store = os.path.join(root, ".polyctl", "mail")
|
|
1349
|
+
ledger = os.path.join(root, ".polyctl", "ledger", "mail-store.jsonl")
|
|
1350
|
+
entries = {}
|
|
1351
|
+
for directory, _, files in os.walk(store):
|
|
1352
|
+
for name in files:
|
|
1353
|
+
rel = os.path.relpath(os.path.join(directory, name), root).replace(os.sep, "/")
|
|
1354
|
+
with open(os.path.join(root, rel), "rb") as fh:
|
|
1355
|
+
data = fh.read()
|
|
1356
|
+
if name.endswith(".gz"):
|
|
1357
|
+
# A compacted body is scanned as every other record is before it leaves: an agent writes its
|
|
1358
|
+
# mail by hand, and a credential in it reached the private copy inside the archive (MAIL-SYNC-2).
|
|
1359
|
+
import gzip
|
|
1360
|
+
data = gzip.compress(scan_evidence(gzip.decompress(data).decode(UTF8, "replace"))[0].encode(UTF8),
|
|
1361
|
+
mtime=0)
|
|
1362
|
+
else:
|
|
1363
|
+
data = scan_evidence(data.decode(UTF8, "replace"))[0].encode(UTF8)
|
|
1364
|
+
entries[rel] = data
|
|
1365
|
+
if os.path.exists(ledger):
|
|
1366
|
+
with open(ledger, encoding=UTF8) as fh:
|
|
1367
|
+
entries[".polyctl/ledger/mail-store.jsonl"] = scan_evidence(fh.read())[0].encode(UTF8)
|
|
1368
|
+
tree = {}
|
|
1369
|
+
for rel, data in sorted(entries.items()):
|
|
1370
|
+
blob = subprocess.run([git_binary(), "hash-object", "-w", "--stdin"], cwd=root, input=data,
|
|
1371
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout.decode().strip()
|
|
1372
|
+
node = tree
|
|
1373
|
+
parts = rel.split("/")
|
|
1374
|
+
for part in parts[:-1]:
|
|
1375
|
+
node = node.setdefault(part, {})
|
|
1376
|
+
node[parts[-1]] = blob
|
|
1377
|
+
|
|
1378
|
+
def write(node):
|
|
1379
|
+
lines = [f"040000 tree {write(node[name])}\t{name}" if isinstance(node[name], dict)
|
|
1380
|
+
else f"100644 blob {node[name]}\t{name}" for name in sorted(node)]
|
|
1381
|
+
return subprocess.run([git_binary(), "mktree"], cwd=root, input=("\n".join(lines) + "\n").encode(UTF8),
|
|
1382
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout.decode().strip()
|
|
1383
|
+
|
|
1384
|
+
parent = git_text(root, "rev-parse", "--verify", "--quiet", "refs/polyctl/mail")
|
|
1385
|
+
tree_id = write(tree)
|
|
1386
|
+
if parent and git_text(root, "rev-parse", f"{parent}^{{tree}}") == tree_id:
|
|
1387
|
+
return parent
|
|
1388
|
+
argv = [git_binary(), "-c", "user.name=polyctl", "-c", "user.email=polyctl@localhost", "commit-tree", tree_id,
|
|
1389
|
+
"-m", f"polyctl mail store of {project_key(root)}"]
|
|
1390
|
+
if parent:
|
|
1391
|
+
argv += ["-p", parent]
|
|
1392
|
+
commit = subprocess.run(argv, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
1393
|
+
check=True).stdout.decode().strip()
|
|
1394
|
+
_git_output(root, "update-ref", "refs/polyctl/mail", commit)
|
|
1395
|
+
return commit
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def sync_mail(root, cfg):
|
|
1399
|
+
"""Push the store's ref to the one private repository named in mail.sync_repo, as refs/mail/<project> (#83).
|
|
1400
|
+
|
|
1401
|
+
The product's own remote is never a target, and a repository the host does not
|
|
1402
|
+
confirm private is refused, named and logged. Every record is scanned before it
|
|
1403
|
+
leaves. Returns (commit, where).
|
|
1404
|
+
"""
|
|
1405
|
+
target = (settings.get(cfg, "mail.sync_repo") or "").strip()
|
|
1406
|
+
if not target:
|
|
1407
|
+
raise RuntimeError("no mail.sync_repo is named; syncing is opt-in per project")
|
|
1408
|
+
target = _push_url(root, target)
|
|
1409
|
+
# The product's remote in any of its forms - https and git@ name one repository (MAIL-SYNC-2) - where it
|
|
1410
|
+
# fetches and where it pushes (MAIL-SYNC-3), every URL of each: `git push origin` reaches them all (MAIL-SYNC-4).
|
|
1411
|
+
origins = {line.strip() for line in (git_text(root, "remote", "get-url", "--all", "origin").splitlines()
|
|
1412
|
+
+ git_text(root, "remote", "get-url", "--push", "--all", "origin").splitlines())}
|
|
1413
|
+
# `git remote get-url` knows only the repository's own remotes; an origin kept in ~/.gitconfig is still where
|
|
1414
|
+
# `git push origin` goes (MAIL-SYNC-5), as git rewrites it: `product-short` with an insteadOf naming the product
|
|
1415
|
+
# was compared as written, and the mail went to the product (MAIL-SYNC-6).
|
|
1416
|
+
entries = _git_config_entries(root, r"^(remote\.origin\.(url|pushurl)|url\..+\.(insteadof|pushinsteadof))$")
|
|
1417
|
+
for key, value in entries:
|
|
1418
|
+
if key in ("remote.origin.url", "remote.origin.pushurl"):
|
|
1419
|
+
origins |= {value.strip()} | {rewritten[0].strip() for rewritten in (
|
|
1420
|
+
_git_rewrite(value, entries, "insteadof"), _git_rewrite(value, entries, "pushinsteadof")) if rewritten}
|
|
1421
|
+
same = any(origin and (target.rstrip("/").removesuffix(".git") == origin.rstrip("/").removesuffix(".git")
|
|
1422
|
+
or _remote_identity(target) is not None and _remote_identity(target) == _remote_identity(origin))
|
|
1423
|
+
for origin in origins)
|
|
1424
|
+
if same:
|
|
1425
|
+
raise RuntimeError("mail.sync_repo is this product's own remote; mail never goes there")
|
|
1426
|
+
private = _url_is_private(root, target)
|
|
1427
|
+
if private is not True:
|
|
1428
|
+
record_notice(root, "mail sync refused", f"{target} was not confirmed private", sent=False, key="mail-sync-refused")
|
|
1429
|
+
raise RuntimeError(f"not syncing mail to {target}: the host did not confirm it is private")
|
|
1430
|
+
commit = mail_ref_commit(root)
|
|
1431
|
+
ref = f"refs/mail/{project_key(root)}"
|
|
1432
|
+
_git_output(root, "push", target, f"refs/polyctl/mail:{ref}", timeout=300)
|
|
1433
|
+
return commit, f"{target} {ref}"
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
def mail_sync_state(root, cfg):
|
|
1437
|
+
"""(local, remote commit or None, problem or None) for `polyctl doctor` (#83).
|
|
1438
|
+
|
|
1439
|
+
`local` is the last synced commit while the store is as it synced, and "unsynced" once its
|
|
1440
|
+
ledger has moved past the copy in that commit: only `polyctl mail sync` moves the ref, so mail
|
|
1441
|
+
stored since, or a store never synced, read as up to date (MAIL-SYNC-2). Read, never written:
|
|
1442
|
+
the doctor commits nothing. A target that cannot be read is named, not taken for one behind.
|
|
1443
|
+
"""
|
|
1444
|
+
target = (settings.get(cfg, "mail.sync_repo") or "").strip()
|
|
1445
|
+
if not target:
|
|
1446
|
+
return None
|
|
1447
|
+
synced = git_text(root, "rev-parse", "--verify", "--quiet", "refs/polyctl/mail") or None
|
|
1448
|
+
try:
|
|
1449
|
+
with open(os.path.join(root, ".polyctl", "ledger", "mail-store.jsonl"), encoding=UTF8) as fh:
|
|
1450
|
+
now = scan_evidence(fh.read())[0]
|
|
1451
|
+
except OSError:
|
|
1452
|
+
now = None
|
|
1453
|
+
copy = None
|
|
1454
|
+
if synced and now is not None:
|
|
1455
|
+
try:
|
|
1456
|
+
copy = _git_output(root, "show", "refs/polyctl/mail:.polyctl/ledger/mail-store.jsonl").decode(UTF8, "replace")
|
|
1457
|
+
except RuntimeError:
|
|
1458
|
+
copy = None
|
|
1459
|
+
local = synced if now is None or copy == now else "unsynced"
|
|
1460
|
+
try:
|
|
1461
|
+
target = _push_url(root, target) # read where a sync would push (MAIL-SYNC-3)
|
|
1462
|
+
except RuntimeError as exc:
|
|
1463
|
+
return local, None, f"mail.sync_repo cannot be synced to: {exc}"
|
|
1464
|
+
try:
|
|
1465
|
+
remote = (_git_output(root, "ls-remote", target, f"refs/mail/{project_key(root)}", timeout=60)
|
|
1466
|
+
.decode(UTF8, "replace").split() or [None])[0]
|
|
1467
|
+
except RuntimeError as exc:
|
|
1468
|
+
return local, None, f"{target} cannot be read ({' '.join(str(exc).split())})"
|
|
1469
|
+
problem = None if _url_is_private(root, target) is True else f"{target} could not be verified private"
|
|
1470
|
+
return local, remote, problem
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
def mail_ledger_append(root, row):
|
|
1474
|
+
d = os.path.join(root, ".polyctl", "ledger")
|
|
1475
|
+
try:
|
|
1476
|
+
os.makedirs(d, exist_ok=True)
|
|
1477
|
+
row = dict(row)
|
|
1478
|
+
row.setdefault("at", int(time.time()))
|
|
1479
|
+
with open(os.path.join(d, "mail.jsonl"), "a", encoding=UTF8) as fh:
|
|
1480
|
+
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
1481
|
+
except OSError:
|
|
1482
|
+
pass
|
|
1483
|
+
|
|
1484
|
+
|
|
1485
|
+
def mail_log(root, limit=200):
|
|
1486
|
+
p = os.path.join(root, ".polyctl", "ledger", "mail.jsonl")
|
|
1487
|
+
if not os.path.exists(p):
|
|
1488
|
+
return []
|
|
1489
|
+
rows = []
|
|
1490
|
+
for line in open(p, errors="replace", encoding=UTF8):
|
|
1491
|
+
try:
|
|
1492
|
+
rows.append(json.loads(line))
|
|
1493
|
+
except ValueError:
|
|
1494
|
+
pass
|
|
1495
|
+
return rows if limit is None else rows[-limit:]
|
|
1496
|
+
|
|
1497
|
+
|
|
1498
|
+
def reconcile_mail_ledger(root, cfg):
|
|
1499
|
+
"""Mail the ledger saw written and the mailbox no longer has was consumed.
|
|
1500
|
+
|
|
1501
|
+
The architect and the implementer delete by hand; nothing hooks `rm`. So
|
|
1502
|
+
the watchdog closes the loop each cycle: an open id that is not on disk gets
|
|
1503
|
+
a `consumed` row, timed now — a lower bound on when it was read.
|
|
1504
|
+
"""
|
|
1505
|
+
if mail_store_mode(root) == "append-only":
|
|
1506
|
+
# Nothing is consumed by vanishing: take new mail in, and put the view back in line (#80).
|
|
1507
|
+
ingest_mail(root, cfg)
|
|
1508
|
+
sync_mail_view(root, cfg)
|
|
1509
|
+
return []
|
|
1510
|
+
box = os.path.join(root, cfg.get("mailbox", "agent-mail"))
|
|
1511
|
+
rows = mail_log(root, 5000)
|
|
1512
|
+
open_ids = {}
|
|
1513
|
+
for r in rows:
|
|
1514
|
+
if r.get("event") == "written":
|
|
1515
|
+
open_ids[r["id"]] = r
|
|
1516
|
+
elif r.get("event") == "consumed":
|
|
1517
|
+
open_ids.pop(r["id"], None)
|
|
1518
|
+
closed = []
|
|
1519
|
+
for mid, r in open_ids.items():
|
|
1520
|
+
if not os.path.exists(os.path.join(box, mid)):
|
|
1521
|
+
mail_ledger_append(root, {"event": "consumed", "id": mid, "kind": r.get("kind"),
|
|
1522
|
+
"from": r.get("from"), "to": r.get("to"),
|
|
1523
|
+
"stood_s": int(time.time()) - int(r.get("at") or time.time())})
|
|
1524
|
+
closed.append(mid)
|
|
1525
|
+
return closed
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
def mail_search(root, text, limit=50):
|
|
1529
|
+
"""Ledger rows and live files whose id, summary or body mention `text`."""
|
|
1530
|
+
t = text.lower()
|
|
1531
|
+
out, seen = [], set()
|
|
1532
|
+
for r in reversed(mail_log(root, 5000)):
|
|
1533
|
+
if r.get("event") != "written":
|
|
1534
|
+
continue
|
|
1535
|
+
hay = " ".join(str(r.get(k) or "") for k in ("id", "summary", "kind", "slice")).lower()
|
|
1536
|
+
if t in hay and r["id"] not in seen:
|
|
1537
|
+
seen.add(r["id"])
|
|
1538
|
+
out.append(r)
|
|
1539
|
+
if len(out) >= limit:
|
|
1540
|
+
return out
|
|
1541
|
+
return out
|