brainiac-cli 0.16.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. brain/__init__.py +39 -0
  2. brain/__main__.py +14 -0
  3. brain/_assets/AGENTS.md +564 -0
  4. brain/_assets/overlay/template/brand/brand-guide.md +24 -0
  5. brain/_assets/overlay/template/keywords/glossary.md +15 -0
  6. brain/_assets/overlay/template/people/roster.md +14 -0
  7. brain/_assets/overlay/template/voice/voice-profile.md +34 -0
  8. brain/_assets/routines/manifest.json +257 -0
  9. brain/_assets/scripts/brain-brief-mac.plist +59 -0
  10. brain/_assets/scripts/brain-brief.sh +74 -0
  11. brain/_assets/scripts/brain-synthesis-mac.plist +48 -0
  12. brain/_assets/scripts/brain-synthesis.sh +214 -0
  13. brain/_assets/scripts/install-brief-mac.sh +152 -0
  14. brain/_assets/scripts/install-brief-windows.ps1 +97 -0
  15. brain/_assets/scripts/register_tasks.py +386 -0
  16. brain/_assets/templates/company.md +21 -0
  17. brain/_assets/templates/concept.md +27 -0
  18. brain/_assets/templates/daily.md +20 -0
  19. brain/_assets/templates/decision.md +42 -0
  20. brain/_assets/templates/meeting.md +33 -0
  21. brain/_assets/templates/person.md +20 -0
  22. brain/_assets/templates/project.md +23 -0
  23. brain/_assets/templates/state-moc.md +40 -0
  24. brain/_version.py +12 -0
  25. brain/anchor.py +121 -0
  26. brain/audit.py +422 -0
  27. brain/backup.py +210 -0
  28. brain/brief.py +417 -0
  29. brain/capture.py +117 -0
  30. brain/chunk.py +249 -0
  31. brain/classification.py +134 -0
  32. brain/cli.py +1906 -0
  33. brain/config.py +368 -0
  34. brain/connect.py +362 -0
  35. brain/context.py +108 -0
  36. brain/core.py +3018 -0
  37. brain/doctor.py +1161 -0
  38. brain/egress.py +148 -0
  39. brain/embed.py +857 -0
  40. brain/encryption.py +217 -0
  41. brain/frontmatter.py +102 -0
  42. brain/golden_probe.py +678 -0
  43. brain/graph.py +369 -0
  44. brain/graphify.py +352 -0
  45. brain/index.py +1576 -0
  46. brain/ingest/__init__.py +19 -0
  47. brain/ingest/handlers/__init__.py +43 -0
  48. brain/ingest/handlers/base.py +95 -0
  49. brain/ingest/handlers/docx.py +78 -0
  50. brain/ingest/handlers/email.py +228 -0
  51. brain/ingest/handlers/html.py +142 -0
  52. brain/ingest/handlers/image.py +91 -0
  53. brain/ingest/handlers/pdf.py +99 -0
  54. brain/ingest/handlers/pptx.py +69 -0
  55. brain/ingest/handlers/tables.py +41 -0
  56. brain/ingest/handlers/text.py +43 -0
  57. brain/ingest/handlers/xlsx.py +100 -0
  58. brain/ingest/handlers/zip.py +163 -0
  59. brain/ingest/pipeline.py +839 -0
  60. brain/ingest/transcript.py +158 -0
  61. brain/init.py +870 -0
  62. brain/maintenance.py +2266 -0
  63. brain/mcp_adapter.py +217 -0
  64. brain/multihop.py +232 -0
  65. brain/notes.py +195 -0
  66. brain/overlay.py +183 -0
  67. brain/projection.py +79 -0
  68. brain/rerank.py +425 -0
  69. brain/snapshot.py +231 -0
  70. brain/update.py +743 -0
  71. brain/vectors.py +225 -0
  72. brainiac_cli-0.16.0.dist-info/METADATA +306 -0
  73. brainiac_cli-0.16.0.dist-info/RECORD +77 -0
  74. brainiac_cli-0.16.0.dist-info/WHEEL +5 -0
  75. brainiac_cli-0.16.0.dist-info/entry_points.txt +4 -0
  76. brainiac_cli-0.16.0.dist-info/licenses/LICENSE +202 -0
  77. brainiac_cli-0.16.0.dist-info/top_level.txt +1 -0
brain/maintenance.py ADDED
@@ -0,0 +1,2266 @@
1
+ """Outcomes-report shape + date-gate logic for the `brain` maintenance verbs
2
+ (CUT-03): `check` / `health` / `curate` / `integrity` / `promote-scan` / `maintain`.
3
+
4
+ Pure, dependency-free helpers — no I/O, no BrainCore import (mirrors brain.brief).
5
+ ``check``/``health``/``curate``/``integrity``/``promote-scan`` are the FOLDED
6
+ maintenance rituals from ``routines/manifest.json`` (per-task ``disposition``); ``maintain`` is
7
+ the single sanctioned host task (``brain-nightly``) that multiplexes the
8
+ weekly/monthly cadences into one OS scheduler entry via date gates, per
9
+ ``routines/manifest.json`` ``locked_counts``.
10
+
11
+ The three-bucket shape here is the generic, vault-agnostic structured
12
+ equivalent of the owner-vault scheduled-task outcomes contract's three-block
13
+ report (✅ Auto-remediated / ⚠ Action Required / 🚧 Blocked): a thin host-side
14
+ scheduled-task wrapper can render ``render_outcomes_markdown`` directly, or
15
+ consume the structured JSON.
16
+
17
+ MEM-03/AUT-02 (session s08) add two more pure pieces: recommendations
18
+ lifecycle (open -> aging -> surfaced -> resolved) file-format helpers, and
19
+ markdown renderers for the Sunday curation/promotion-scan findings. This
20
+ module still does NO I/O — ``BrainCore.maintain`` (src/brain/core.py) is the
21
+ one place that reads/writes `.brain/memory/{recommendations-open.jsonl,
22
+ recommendations-log.md,hot.md}`, guarded by an idempotency key so a re-run
23
+ never duplicates a hot-queue entry (ADR-0003 Ruling 5/HARDENED:codex).
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import datetime
28
+ import itertools
29
+ import json
30
+ import logging
31
+ import re
32
+ import shlex
33
+ from pathlib import Path
34
+ from typing import Any
35
+
36
+ _log = logging.getLogger("brain.maintenance")
37
+
38
+
39
+ def auto_fixed_item(verb: str, path: str, reason: str) -> dict[str, Any]:
40
+ return {"verb": verb, "path": path, "reason": reason}
41
+
42
+
43
+ def action_required_item(
44
+ finding: str, why: str, proposed: str, inspect: str
45
+ ) -> dict[str, Any]:
46
+ return {"finding": finding, "why": why, "proposed_action": proposed, "inspect": inspect}
47
+
48
+
49
+ def blocked_item(finding: str, blocking_on: str, retry_when: str) -> dict[str, Any]:
50
+ return {"finding": finding, "blocking_on": blocking_on, "retry_when": retry_when}
51
+
52
+
53
+ def framework_sync_finding(report: dict[str, Any]) -> dict[str, Any] | None:
54
+ """HYG-02 (ADR-0003 Ruling 5): shape a pre-computed
55
+ ``tools.framework_sync.audit()`` report into a Monday-health
56
+ ``action_required`` item, or ``None`` when the report is clean. Pure —
57
+ the actual file-hashing/reading I/O lives in ``tools/framework_sync.py``
58
+ and is invoked by ``BrainCore`` (host-only), never here. Never
59
+ auto-fixes: the proposed action is always "re-run package_clients.py"."""
60
+ if report.get("clean"):
61
+ return None
62
+ drift = report.get("skill_drift") or []
63
+ claude_md = report.get("claude_md_import") or {}
64
+ paths = [f"{d['skill']} [{d['mirror']}] {d.get('path') or d['reason']}" for d in drift[:5]]
65
+ parts = []
66
+ if drift:
67
+ parts.append(f"{len(drift)} skill-mirror file(s) diverged")
68
+ if not claude_md.get("ok"):
69
+ parts.append(f"CLAUDE.md: {claude_md.get('reason')}")
70
+ return action_required_item(
71
+ "; ".join(parts) or "framework-sync drift detected",
72
+ "the .claude/skills canonical tree, .agents/skills mirror, and/or "
73
+ "plugins/ marketplace copies have drifted apart (or CLAUDE.md's "
74
+ "@AGENTS.md import broke)",
75
+ "run `python3 tools/package_clients.py` to resync, then re-run health",
76
+ "; ".join(paths) or "CLAUDE.md",
77
+ )
78
+
79
+
80
+ def build_outcomes(
81
+ auto_fixed: list[dict[str, Any]] | None = None,
82
+ action_required: list[dict[str, Any]] | None = None,
83
+ blocked: list[dict[str, Any]] | None = None,
84
+ ) -> dict[str, Any]:
85
+ """The structured three-bucket disposition. Buckets are ALWAYS present
86
+ (possibly empty) so the shape is stable and grep/parse-friendly — mirrors
87
+ the vault outcomes contract's "(none)" convention."""
88
+ af = list(auto_fixed or [])
89
+ ar = list(action_required or [])
90
+ bl = list(blocked or [])
91
+ return {
92
+ "auto_fixed": af,
93
+ "action_required": ar,
94
+ "blocked": bl,
95
+ "counts": {"auto_fixed": len(af), "action_required": len(ar), "blocked": len(bl)},
96
+ }
97
+
98
+
99
+ def render_outcomes_markdown(outcomes: dict[str, Any]) -> str:
100
+ """Render the three-block markdown shape (✅/⚠/🚧), generic — no
101
+ vault-specific file paths or chain-script invocations baked in."""
102
+ lines: list[str] = []
103
+
104
+ af = outcomes.get("auto_fixed", [])
105
+ lines.append(f"## Auto-remediated this run ({len(af)} items)")
106
+ if af:
107
+ for it in af:
108
+ lines.append(f"- **[{it.get('verb')}]** `{it.get('path')}` — {it.get('reason')}")
109
+ else:
110
+ lines.append("(none)")
111
+ lines.append("")
112
+
113
+ ar = outcomes.get("action_required", [])
114
+ lines.append(f"## Action Required ({len(ar)} items)")
115
+ if ar:
116
+ for i, it in enumerate(ar, 1):
117
+ lines.append(f"**Finding {i}:** {it.get('finding')}")
118
+ lines.append(f"**Why it can't auto-fix:** {it.get('why')}")
119
+ lines.append(f"**Proposed action:** {it.get('proposed_action')}")
120
+ lines.append(f"**Inspect:** `{it.get('inspect')}`")
121
+ lines.append("")
122
+ else:
123
+ lines.append("(none)")
124
+ lines.append("")
125
+
126
+ bl = outcomes.get("blocked", [])
127
+ lines.append(f"## Blocked — external dependency ({len(bl)} items)")
128
+ if bl:
129
+ for i, it in enumerate(bl, 1):
130
+ lines.append(f"**Finding {i}:** {it.get('finding')}")
131
+ lines.append(f"**Blocking on:** {it.get('blocking_on')}")
132
+ lines.append(f"**Retry when:** {it.get('retry_when')}")
133
+ lines.append("")
134
+ else:
135
+ lines.append("(none)")
136
+
137
+ return "\n".join(lines).rstrip() + "\n"
138
+
139
+
140
+ # Date-gated branch names, in cadence order. "daily" is always due; the rest
141
+ # fire on the weekday/day-of-month the persistence budget assigns them so the
142
+ # WHOLE roster rides the one sanctioned OS task (`brain-nightly`) with ZERO
143
+ # extra scheduler entries (persistence-budget.md THE LOCK).
144
+ _MONDAY, _TUESDAY, _SUNDAY = 0, 1, 6
145
+ _WEEKLY_TRIGGER_WEEKDAY = {
146
+ "health": _MONDAY, "integrity": _TUESDAY, "digest": _SUNDAY, "golden": _SUNDAY,
147
+ }
148
+ _ALL_BRANCHES = ("daily", "health", "integrity", "digest", "graphify", "golden")
149
+
150
+
151
+ def _next_trigger(branch: str, last_run: datetime.date) -> datetime.date:
152
+ """First calendar date STRICTLY AFTER ``last_run`` on which ``branch`` is
153
+ next due (ADR-0003 Ruling d)."""
154
+ if branch == "daily":
155
+ return last_run + datetime.timedelta(days=1)
156
+ if branch == "graphify":
157
+ y, m = last_run.year, last_run.month + 1
158
+ if m > 12:
159
+ y, m = y + 1, 1
160
+ return datetime.date(y, m, 1)
161
+ weekday = _WEEKLY_TRIGGER_WEEKDAY[branch]
162
+ days_ahead = (weekday - last_run.weekday()) % 7
163
+ return last_run + datetime.timedelta(days=days_ahead or 7)
164
+
165
+
166
+ def maintain_branches(
167
+ today: datetime.date | None = None,
168
+ last_runs: dict[str, str | None] | None = None,
169
+ ) -> list[str]:
170
+ """Which date-gated branches are due for ``today`` (default: real today).
171
+
172
+ ADR-0003 Ruling 5/(d) — DUE-SINCE-LAST-RUN, not calendar-day-only: a
173
+ branch is due when ``today >= next_trigger(last_run)``. ``last_runs`` maps
174
+ branch -> its last SUCCESSFUL run date (ISO string) as persisted in
175
+ ``.brain/maintain-state.json``; a branch absent from the mapping (or
176
+ ``last_runs=None``, e.g. no state file yet — a brand-new install) has
177
+ never run and is due immediately, regardless of weekday. This is safe
178
+ because every branch is idempotent (a re-run finds nothing new to do).
179
+
180
+ A missed weekly/monthly branch (the laptop was off on its trigger day)
181
+ fires once on the next run that reaches or passes its next trigger date —
182
+ not once per day it stayed missed.
183
+
184
+ Mon -> health, Tue -> integrity, Sun -> digest, 1st-of-month -> graphify
185
+ (ADR-0003 Ruling 6/(a): `maintain` invokes a REAL, bounded graph build on
186
+ this date-gate — drift-gated, embedding-reuse, wall-clock-budgeted; see
187
+ `BrainCore.graphify`). "daily" (index sync + drain + recommendations-aging
188
+ scan) is due at most once per day.
189
+ """
190
+ d = today or datetime.date.today()
191
+ last_runs = last_runs or {}
192
+ due: list[str] = []
193
+ for branch in _ALL_BRANCHES:
194
+ raw = last_runs.get(branch)
195
+ if not raw:
196
+ due.append(branch) # never run -> due now
197
+ continue
198
+ last = raw if isinstance(raw, datetime.date) else datetime.date.fromisoformat(raw)
199
+ if last > d:
200
+ # A marker AFTER today is corrupt state (clock skew, or a
201
+ # `--today` date-gate test run against a live vault). Left alone
202
+ # it silently disables the branch until real time catches up, so
203
+ # treat it like never-run: due now, and the success rewrite of
204
+ # last_run to today's date self-heals the state file.
205
+ due.append(branch)
206
+ continue
207
+ if d >= _next_trigger(branch, last):
208
+ due.append(branch)
209
+ return due
210
+
211
+
212
+ # ---------------------------------------------------------------------------
213
+ # Recommendations lifecycle (MEM-03, ADR-0003 Ruling 5 "daily" unconditional
214
+ # fold). Ported pattern: the reference vault's `_recommendations_open.jsonl` /
215
+ # `_recommendations_log.md` (schema/pattern only, per Appendix B — never
216
+ # content). Lifecycle: open -> aging (implicit: an open entry past the aging
217
+ # threshold) -> surfaced (flipped + queued into hot.md, exactly once) ->
218
+ # resolved (removed from the open file, appended to the log as a closed
219
+ # record). Appending a NEW open entry, and resolving one, are both simple
220
+ # enough that no CLI verb exists yet — an agent/owner appends/edits the JSONL
221
+ # directly, the same convention as `hot.md` itself (docs/session-memory.md).
222
+ # ---------------------------------------------------------------------------
223
+ DEFAULT_RECOMMENDATION_AGING_DAYS = 14
224
+
225
+
226
+ def parse_recommendation_lines(text: str) -> list[dict[str, Any]]:
227
+ """Parse a ``recommendations-open.jsonl`` blob into entry dicts.
228
+
229
+ A blank or unparsable line is dropped, never raised — the aging fold is a
230
+ cheap unconditional maintain step and must never abort the run over one
231
+ corrupt line."""
232
+ out: list[dict[str, Any]] = []
233
+ for line in (text or "").splitlines():
234
+ line = line.strip()
235
+ if not line:
236
+ continue
237
+ try:
238
+ entry = json.loads(line)
239
+ except (ValueError, TypeError):
240
+ continue
241
+ if isinstance(entry, dict):
242
+ out.append(entry)
243
+ return out
244
+
245
+
246
+ def render_recommendation_lines(entries: list[dict[str, Any]]) -> str:
247
+ """Serialise entries back to the one-JSON-object-per-line file shape."""
248
+ if not entries:
249
+ return ""
250
+ return "\n".join(json.dumps(e, sort_keys=True) for e in entries) + "\n"
251
+
252
+
253
+ def recommendations_aging_scan(
254
+ entries: list[dict[str, Any]],
255
+ today: datetime.date,
256
+ aging_days: int = DEFAULT_RECOMMENDATION_AGING_DAYS,
257
+ ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
258
+ """Flip any ``status: open`` entry whose ``created`` is >= ``aging_days``
259
+ old to ``status: surfaced`` (+ ``surfaced_at``). Returns ``(updated,
260
+ newly_surfaced)`` — ``updated`` is the FULL list (for rewriting the JSONL),
261
+ ``newly_surfaced`` is only what changed THIS run (what the caller queues
262
+ into ``hot.md``). Idempotent by construction: an entry already
263
+ ``surfaced``/``resolved`` is left untouched and never re-emitted, so a
264
+ caller that reruns this scan never duplicates a hot-queue entry for the
265
+ same recommendation."""
266
+ updated: list[dict[str, Any]] = []
267
+ newly: list[dict[str, Any]] = []
268
+ for raw in entries:
269
+ entry = dict(raw)
270
+ if entry.get("status", "open") == "open":
271
+ age: int | None = None
272
+ try:
273
+ age = (today - datetime.date.fromisoformat(str(entry.get("created")))).days
274
+ except (TypeError, ValueError):
275
+ age = None
276
+ if age is not None and age >= aging_days:
277
+ entry["status"] = "surfaced"
278
+ entry["surfaced_at"] = today.isoformat()
279
+ newly.append(entry)
280
+ updated.append(entry)
281
+ return updated, newly
282
+
283
+
284
+ def render_recommendation_hot_entry(entry: dict[str, Any], today: datetime.date) -> str:
285
+ """One ``hot.md``-shaped dated entry (docs/session-memory.md format) for a
286
+ newly-aged recommendation."""
287
+ text = str(entry.get("text") or "").strip()
288
+ title = (text.splitlines()[0] if text else entry.get("id", "recommendation"))[:80]
289
+ return (
290
+ f"## {today.isoformat()} — Recommendation aged: {title}\n"
291
+ f"- **Context:** proposed {entry.get('created', '?')}, still open with no "
292
+ f"action (id: `{entry.get('id')}`).\n"
293
+ f"- **Question:** still worth doing — act on it, defer it, or drop it?\n"
294
+ f"- **Owner input needed:** resolve `{entry.get('id')}` in "
295
+ f"recommendations-open.jsonl (moves to recommendations-log.md once decided).\n"
296
+ )
297
+
298
+
299
+ def resolve_recommendation(
300
+ entries: list[dict[str, Any]], rec_id: str, resolution: str, today: datetime.date
301
+ ) -> tuple[list[dict[str, Any]], str | None]:
302
+ """Close out ``rec_id``: returns ``(remaining_entries, log_line)`` —
303
+ ``remaining_entries`` is the open list with ``rec_id`` removed (rewrite the
304
+ JSONL with it), ``log_line`` is the Markdown line to append to
305
+ ``recommendations-log.md`` (``None`` if ``rec_id`` was not found — no-op,
306
+ never raises)."""
307
+ remaining: list[dict[str, Any]] = []
308
+ resolved: dict[str, Any] | None = None
309
+ for entry in entries:
310
+ if entry.get("id") == rec_id and resolved is None:
311
+ resolved = entry
312
+ else:
313
+ remaining.append(entry)
314
+ if resolved is None:
315
+ return entries, None
316
+ log_line = (
317
+ f"## {today.isoformat()} — {resolved.get('text', '(no text)')} (resolved)\n"
318
+ f"- **Opened:** {resolved.get('created', '?')}\n"
319
+ f"- **Resolution:** {resolution}\n\n"
320
+ )
321
+ return remaining, log_line
322
+
323
+
324
+ # ---------------------------------------------------------------------------
325
+ # Sunday curation/promotion-scan hot-queue renderers (AUT-02). Pure markdown
326
+ # builders — ``BrainCore.maintain`` does the idempotent file I/O.
327
+ # ---------------------------------------------------------------------------
328
+ def render_curation_hot_entry(
329
+ stale_links: list[dict[str, Any]], revisit_sample: list[dict[str, Any]],
330
+ today: datetime.date,
331
+ ) -> str:
332
+ lines = [f"## {today.isoformat()} — Sunday curation scan"]
333
+ lines.append(
334
+ f"- **Context:** scheduled curation fold found {len(stale_links)} stale "
335
+ f"wikilink target(s) and a {len(revisit_sample)}-note revisit sample."
336
+ )
337
+ for s in stale_links[:10]:
338
+ target = s.get("target_text")
339
+ reason = s.get("reason")
340
+ frm = s.get("from", {}).get("id")
341
+ lines.append(f" - stale link: `{frm}` -> `{target}` ({reason})")
342
+ for r in revisit_sample[:10]:
343
+ lines.append(
344
+ f" - revisit: `{r.get('id')}` (last updated {r.get('updated')}, "
345
+ f"age {r.get('age_days')}d, score {r.get('score')})"
346
+ )
347
+ lines.append(
348
+ "- **Owner input needed:** review via `brain curate --json` or the "
349
+ "`.claude/skills/curation` skill for full detail."
350
+ )
351
+ return "\n".join(lines) + "\n"
352
+
353
+
354
+ # ---------------------------------------------------------------------------
355
+ # Autoresearch quarterly-poke visibility (HARDENED:claude, AUT-01). aut-04
356
+ # (session s11, after this one) is the skill that actually RUNS autoresearch
357
+ # and writes an evidence artifact under eval/runs/ each time; this helper is
358
+ # the pure staleness judgment the brief renders. No autoresearch run has ever
359
+ # landed at the time this module ships (s09 precedes s11) — ``last_run=None``
360
+ # (never run) is handled the same as "very stale", not as an error, so the
361
+ # quarterly convention is visible from day one instead of silently starting
362
+ # blind.
363
+ # ---------------------------------------------------------------------------
364
+ DEFAULT_AUTORESEARCH_STALE_DAYS = 90
365
+
366
+
367
+ def autoresearch_staleness(
368
+ last_run: datetime.date | None, today: datetime.date,
369
+ stale_days: int = DEFAULT_AUTORESEARCH_STALE_DAYS,
370
+ ) -> dict[str, Any]:
371
+ """Judge whether the quarterly autoresearch cadence looks alive.
372
+
373
+ ``last_run`` is the date of the newest ``eval/runs/autoresearch-*.json``
374
+ artifact (the caller does that file scan; this function is pure). Returns
375
+ ``never_run`` (no artifact found yet), ``age_days`` (``None`` if never
376
+ run), and ``stale`` (true when overdue — the brief only surfaces a line
377
+ when this is true, per the ~90-day threshold)."""
378
+ if last_run is None:
379
+ return {"never_run": True, "age_days": None, "last_run": None, "stale": True}
380
+ age_days = (today - last_run).days
381
+ return {
382
+ "never_run": False,
383
+ "age_days": age_days,
384
+ "last_run": last_run.isoformat(),
385
+ "stale": age_days > stale_days,
386
+ }
387
+
388
+
389
+ def render_graphify_hot_entry(candidates: list[dict[str, Any]], today: datetime.date) -> str:
390
+ """The monthly graphify-build hot-queue entry (GRF-01/GRF-02, ADR-0003
391
+ Ruling 6/(a)). ``candidates`` are already egress-gated INFERRED edges —
392
+ review-only, NEVER auto-written into a note body."""
393
+ lines = [f"## {today.isoformat()} — Monthly graphify discovery build"]
394
+ lines.append(
395
+ f"- **Context:** {len(candidates)} INFERRED link candidate(s) proposed "
396
+ "from embedding-neighbour similarity (discovery-only, non-authoritative)."
397
+ )
398
+ for c in candidates[:10]:
399
+ lines.append(
400
+ f" - `{c.get('from')}` <-> `{c.get('to')}` (score {c.get('score')}) — {c.get('reason')}"
401
+ )
402
+ lines.append(
403
+ "- **Owner input needed:** review via `brain graphify --json` and, if a "
404
+ "candidate is genuinely related, add the wikilink yourself — graphify "
405
+ "never writes a link into a note."
406
+ )
407
+ return "\n".join(lines) + "\n"
408
+
409
+
410
+ # ---------------------------------------------------------------------------
411
+ # FRESH-01 (2026-07-11) — drift-triggered graphify. The monthly date-gate
412
+ # (``maintain_branches``' "graphify" branch, `_next_trigger` above) is a
413
+ # FLOOR, not a GATE: a vault that doubles mid-month (measured on the owner's
414
+ # real vault — graph built at 1,169 notes, index at 2,239 sixteen days
415
+ # later) must not wait out the calendar for the next 1st-of-month. The daily
416
+ # fold instead measures corpus drift since the last graphify build and fires
417
+ # a BOUNDED rebuild early once drift crosses a threshold.
418
+ #
419
+ # ``core.graphify()`` runs synchronously IN-PROCESS (owner decision
420
+ # 2026-07-12, superseding the earlier subprocess-wrapper hardening) and its
421
+ # OWN branch bookkeeping (``_mark("graphify", ok)``) only persists at the END
422
+ # of a run — so a build that hangs (until the maintain run-lock recovers it)
423
+ # would never advance any cooldown on its own, and a naive drift check would
424
+ # re-fire a fresh build every single hourly maintain, forever. The
425
+ # ``_graphify_drift`` maintain-state marker is therefore ATTEMPT-keyed
426
+ # (``BrainCore._run_bounded_graphify`` persists ``last_attempt`` to disk
427
+ # BEFORE the build runs) and backs off exponentially (capped) on consecutive
428
+ # failures, resetting only after a build that actually publishes.
429
+ # ---------------------------------------------------------------------------
430
+ GRAPHIFY_DRIFT_PCT_ENV = "BRAIN_GRAPHIFY_DRIFT_PCT"
431
+ DEFAULT_GRAPHIFY_DRIFT_PCT = 0.15
432
+ GRAPHIFY_COOLDOWN_DAYS_ENV = "BRAIN_GRAPHIFY_COOLDOWN_DAYS"
433
+ DEFAULT_GRAPHIFY_COOLDOWN_DAYS = 2
434
+ GRAPHIFY_BACKOFF_MAX_MULTIPLIER = 8 # capped exponential backoff on consecutive overruns
435
+
436
+
437
+ def graphify_drift(manifest: dict[str, Any] | None, conn: Any) -> float:
438
+ """Corpus drift ratio since the persisted graphify ``manifest`` — the
439
+ SAME ``{"notes": {id: content_hash}, ...}`` shape ``BrainCore.graphify``
440
+ reads from/writes to ``graph_manifest_path``. Reuses the index's own
441
+ ``content_hash`` column via ``graphify.corpus_manifest`` (never
442
+ re-hashes note bodies, same embedding-reuse doctrine as the build
443
+ itself). ``(changed + added + removed) / len(old_notes)``.
444
+
445
+ No persisted manifest yet (fresh vault, or first-ever build) is treated
446
+ as full drift — ``1.0`` — so a brand-new vault is eligible on its very
447
+ first drift check (still subject to the same cooldown as every other
448
+ trigger)."""
449
+ from . import graphify as gmod
450
+
451
+ old_notes: dict[str, str] = (manifest or {}).get("notes") or {}
452
+ new_notes = gmod.corpus_manifest(conn)
453
+ if not old_notes:
454
+ # No (or empty) baseline: full drift ONLY if there is anything to
455
+ # build — an empty baseline over a still-empty corpus is 0.0, not a
456
+ # perpetual build/skip churn every cooldown (review finding [3]).
457
+ return 1.0 if new_notes else 0.0
458
+ changed = sum(1 for nid, h in old_notes.items() if nid in new_notes and new_notes[nid] != h)
459
+ removed = sum(1 for nid in old_notes if nid not in new_notes)
460
+ added = sum(1 for nid in new_notes if nid not in old_notes)
461
+ return (changed + added + removed) / len(old_notes)
462
+
463
+
464
+ def graphify_backoff_days(cooldown_days: int, consecutive_overruns: int) -> int:
465
+ """Capped exponential backoff (HARDENED correction c): each consecutive
466
+ overrun/failure doubles the effective cooldown, capped at
467
+ ``GRAPHIFY_BACKOFF_MAX_MULTIPLIER``x — a corpus that keeps timing out
468
+ backs off instead of re-attempting (and re-failing) a fresh bounded
469
+ build every single hourly maintain run forever."""
470
+ multiplier = min(2 ** max(0, consecutive_overruns), GRAPHIFY_BACKOFF_MAX_MULTIPLIER)
471
+ return cooldown_days * multiplier
472
+
473
+
474
+ def graphify_drift_marker_due(
475
+ marker: dict[str, Any] | None, today: datetime.date, cooldown_days: int,
476
+ ) -> bool:
477
+ """True iff enough time has passed since the last bounded-graphify
478
+ ATTEMPT (never "success" alone — HARDENED correction b) to allow
479
+ another one. ``marker`` is the persisted ``_graphify_drift`` maintain-
480
+ state entry; absent/never-attempted is always due. A corrupt
481
+ ``last_attempt`` (unparsable date) degrades to "due now" rather than
482
+ permanently wedging the trigger."""
483
+ last = marker.get("last_attempt") if marker else None
484
+ if not last:
485
+ return True
486
+ try:
487
+ last_date = datetime.date.fromisoformat(str(last))
488
+ except ValueError:
489
+ return True
490
+ overruns = int((marker or {}).get("consecutive_overruns", 0))
491
+ effective_cooldown = graphify_backoff_days(cooldown_days, overruns)
492
+ return (today - last_date).days >= effective_cooldown
493
+
494
+
495
+ def should_trigger_drift_graphify(
496
+ ratio: float, marker: dict[str, Any] | None, today: datetime.date, *,
497
+ drift_pct: float | None = None, cooldown_days: int | None = None,
498
+ ) -> bool:
499
+ """The daily fold's drift-trigger decision: drift over threshold AND the
500
+ attempt-keyed cooldown (with backoff) has elapsed. Pure — the caller
501
+ supplies ``ratio`` (from ``graphify_drift``) and ``marker`` (the loaded
502
+ ``_graphify_drift`` maintain-state entry). The monthly date-gate
503
+ (``maintain_branches``) remains the FLOOR trigger, independent of this
504
+ function — a maintain run ORs the two triggers together (session
505
+ context bundle FRESH-01)."""
506
+ import os as _os
507
+
508
+ pct = drift_pct if drift_pct is not None else float(
509
+ _os.environ.get(GRAPHIFY_DRIFT_PCT_ENV, DEFAULT_GRAPHIFY_DRIFT_PCT))
510
+ days = cooldown_days if cooldown_days is not None else int(
511
+ _os.environ.get(GRAPHIFY_COOLDOWN_DAYS_ENV, DEFAULT_GRAPHIFY_COOLDOWN_DAYS))
512
+ if ratio <= pct:
513
+ return False
514
+ return graphify_drift_marker_due(marker, today, days)
515
+
516
+
517
+ # ---------------------------------------------------------------------------
518
+ # WD-03 (2026-07-12) — Sunday cross-family golden-probe EXECUTION. Codex (the
519
+ # family that did NOT build the retrieval engine) shells the SAME `brain`
520
+ # CLI the probes exercise — this is cross-family EXECUTION of a deterministic
521
+ # scorer, correction 5: NEVER "independent verification" / "independent
522
+ # eyes" / "Codex grades retrieval". A shared retrieval bug is invisible to
523
+ # both invokers; only the INVOKER differs, not the measurement.
524
+ #
525
+ # Pure helpers only, here (parsing/validation/marker arithmetic) — the actual
526
+ # `codex exec` / self-run subprocess calls are host I/O and live on
527
+ # `BrainCore._run_golden_probe` (mirrors `_run_bounded_graphify`'s split).
528
+ # The 4 exit codes mirror `brain.golden_probe`'s own contract BY HAND (that
529
+ # module is deliberately engine-decoupled/stdlib-only and never imports
530
+ # anything from this package, so this fold never imports it either — see
531
+ # golden_probe.py's own VALID_TIERS for the same by-hand-sync precedent).
532
+ # ---------------------------------------------------------------------------
533
+ GOLDEN_EXIT_OK = 0
534
+ GOLDEN_EXIT_REGRESSION = 1
535
+ GOLDEN_EXIT_ACTION_REQUIRED = 2
536
+ GOLDEN_EXIT_TRANSIENT = 3
537
+ _GOLDEN_VALID_EXIT_CODES = (GOLDEN_EXIT_OK, GOLDEN_EXIT_REGRESSION,
538
+ GOLDEN_EXIT_ACTION_REQUIRED, GOLDEN_EXIT_TRANSIENT)
539
+ _GOLDEN_VALID_DISPOSITIONS = ("ok", "regression", "action_required", "transient")
540
+
541
+ GOLDEN_RETRY_BASE_MINUTES_ENV = "BRAIN_GOLDEN_RETRY_BASE_MINUTES"
542
+ # 6h base (re-review): the old 60m EQUALLED the hourly maintain cadence, so a
543
+ # run repeatedly killed mid-`codex exec` re-fired every hour despite the
544
+ # provisional backoff. golden is a WEEKLY branch — a base well above the
545
+ # cadence, escalating on consecutive failures (incl. kills), is the point.
546
+ DEFAULT_GOLDEN_RETRY_BASE_MINUTES = 360
547
+ GOLDEN_RETRY_MAX_MULTIPLIER = 8 # capped exponential backoff, same shape as graphify's
548
+
549
+ GOLDEN_CODEX_TIMEOUT_SECONDS_ENV = "BRAIN_GOLDEN_CODEX_TIMEOUT_SECONDS"
550
+ # HARD cap <=10min (correction 1) — strictly below the 2h maintain-lock stale
551
+ # window, so a wedged codex child can never itself become the reason a
552
+ # concurrent maintain run thinks the lock is abandoned.
553
+ DEFAULT_GOLDEN_CODEX_TIMEOUT_SECONDS = 600
554
+ MAX_GOLDEN_CODEX_TIMEOUT_SECONDS = 600
555
+
556
+
557
+ def golden_codex_timeout_seconds() -> int:
558
+ import os as _os
559
+
560
+ raw = int(_os.environ.get(GOLDEN_CODEX_TIMEOUT_SECONDS_ENV, DEFAULT_GOLDEN_CODEX_TIMEOUT_SECONDS))
561
+ return max(1, min(raw, MAX_GOLDEN_CODEX_TIMEOUT_SECONDS))
562
+
563
+
564
+ def golden_probes_path(vault: Path) -> Path:
565
+ """Per-vault probes file (WD-02) — absence is a loud SKIP, never an
566
+ error (session context bundle: 'skips loudly when codex is absent' /
567
+ here, when the probes file itself is absent)."""
568
+ return Path(vault) / "eval" / "golden-probes.json"
569
+
570
+
571
+ def build_codex_golden_prompt(probes_path: Path, vault: Path, python_exe: str) -> str:
572
+ """The FIXED instruction handed to `codex exec` (correction 1): run
573
+ ONLY the golden-probe scorer, read-only, and return ONLY its JSON — no
574
+ prose wrapper — so the caller's strict shape/range validation is
575
+ checking the scorer's own emitted document, not codex's summary of it.
576
+
577
+ ``python_exe`` (review fixes [2] + the re-review's OUTER-interpreter fix)
578
+ is the ABSOLUTE host interpreter that has ``brain`` importable
579
+ (``sys.executable`` from the running maintain). BOTH the outer
580
+ ``-m brain.golden_probe`` invocation AND the inner ``--brain-cmd`` use it:
581
+ a bare ``python3`` here would ``ModuleNotFoundError`` under codex's ambient
582
+ interpreter on a uv-tool/pipx-isolated brain install (the recommended
583
+ channels), so the codex leg would fail every Sunday to the degraded
584
+ self-run and cross-family EXECUTION would never actually happen."""
585
+ brain_cmd = shlex.join([python_exe, "-m", "brain.cli"])
586
+ return (
587
+ "Run exactly this command and reply with ONLY its stdout, verbatim, "
588
+ "and nothing else before or after it:\n\n"
589
+ f"{shlex.quote(python_exe)} -m brain.golden_probe {shlex.quote(str(probes_path))} "
590
+ f"--vault {shlex.quote(str(vault))} "
591
+ f"--brain-cmd {shlex.quote(brain_cmd)}\n\n"
592
+ "Do not modify any files. Do not run any other command. Do not "
593
+ "interpret, summarize, explain, or comment on the result — your "
594
+ "entire reply must be exactly that command's JSON stdout."
595
+ )
596
+
597
+
598
+ def parse_codex_final_message(stdout: str) -> str | None:
599
+ """Extract the text of the LAST `item.completed` event whose
600
+ `item.type == "agent_message"` from a `codex exec --json` JSONL stream
601
+ (correction 1): the stream interleaves thread/turn/tool-call/error
602
+ events, so a caller must never treat the first (or only) JSON-shaped
603
+ line as the answer — a run can emit an `item.type: "error"` info event
604
+ before its real final message. Returns ``None`` when no agent_message
605
+ event is found (or the stream is not JSONL at all); the caller treats
606
+ that as a codex-path failure and falls back to the self-run."""
607
+ last_text: str | None = None
608
+ for line in (stdout or "").splitlines():
609
+ line = line.strip()
610
+ if not line:
611
+ continue
612
+ try:
613
+ event = json.loads(line)
614
+ except ValueError:
615
+ continue
616
+ if not isinstance(event, dict):
617
+ continue
618
+ item = event.get("item")
619
+ if isinstance(item, dict) and item.get("type") == "agent_message":
620
+ text = item.get("text")
621
+ if isinstance(text, str):
622
+ last_text = text
623
+ return last_text
624
+
625
+
626
+ def validate_golden_probe_doc(doc: Any) -> str | None:
627
+ """Strict shape/range check on a parsed golden-probe result document
628
+ (from either the codex path or the self-run fallback) — the
629
+ "exit-0-with-garbage" trap (correction 1): a codex run can exit 0 while
630
+ its final message is empty prose, a truncated fragment, or a
631
+ well-formed-but-nonsensical object. Returns an error string, or
632
+ ``None`` when the doc is trustworthy enough to source a score from."""
633
+ if not isinstance(doc, dict):
634
+ return f"not a JSON object: {type(doc).__name__}"
635
+ if "disposition" not in doc or "exit_code" not in doc:
636
+ return "missing disposition/exit_code key(s)"
637
+ disposition = doc.get("disposition")
638
+ if disposition not in _GOLDEN_VALID_DISPOSITIONS:
639
+ return f"unrecognized disposition: {disposition!r}"
640
+ exit_code = doc.get("exit_code")
641
+ if isinstance(exit_code, bool) or exit_code not in _GOLDEN_VALID_EXIT_CODES:
642
+ return f"unrecognized exit_code: {exit_code!r}"
643
+ score = doc.get("score")
644
+ if score is not None:
645
+ if isinstance(score, bool) or not isinstance(score, (int, float)):
646
+ return f"score is not a number: {score!r}"
647
+ if not (0.0 <= float(score) <= 1.0):
648
+ return f"score out of [0,1]: {score!r}"
649
+ return None
650
+
651
+
652
+ def golden_retry_backoff_minutes(base_minutes: int, consecutive_transient: int) -> int:
653
+ """Capped exponential backoff, same shape as `graphify_backoff_days`:
654
+ each consecutive TRANSIENT failure doubles the effective wait, capped at
655
+ `GOLDEN_RETRY_MAX_MULTIPLIER`x, so a flaky codex/CLI backs off instead of
656
+ re-attempting (and re-failing) every single hourly maintain run."""
657
+ multiplier = min(2 ** max(0, consecutive_transient - 1), GOLDEN_RETRY_MAX_MULTIPLIER)
658
+ return base_minutes * multiplier
659
+
660
+
661
+ def golden_attempt_due(marker: dict[str, Any] | None, now: datetime.datetime) -> bool:
662
+ """True iff the persisted `_golden_attempt` marker's `next_retry_at` has
663
+ elapsed (or there is none yet — never attempted, or the last attempt
664
+ resolved deterministically and cleared it). A corrupt/unparsable
665
+ timestamp degrades to "due now" rather than permanently wedging the
666
+ branch (mirrors `graphify_drift_marker_due`'s same fail-open posture)."""
667
+ nxt = (marker or {}).get("next_retry_at")
668
+ if not nxt:
669
+ return True
670
+ try:
671
+ nxt_dt = datetime.datetime.fromisoformat(str(nxt).replace("Z", "+00:00"))
672
+ except ValueError:
673
+ return True
674
+ if nxt_dt.tzinfo is None:
675
+ nxt_dt = nxt_dt.replace(tzinfo=datetime.timezone.utc)
676
+ return now >= nxt_dt
677
+
678
+
679
+ def update_golden_attempt_marker(
680
+ marker: dict[str, Any] | None, now: datetime.datetime, *,
681
+ transient: bool, base_minutes: int | None = None,
682
+ ) -> dict[str, Any]:
683
+ """The `_golden_attempt` marker to persist AFTER an attempt (the caller
684
+ persists `last_attempt` itself BEFORE the shell-out, mirroring
685
+ `_run_bounded_graphify`'s crash-safety ordering). `transient` is True
686
+ ONLY for exit 3 — every other resolved outcome (ok/regression/
687
+ action_required) is a DETERMINISTIC answer and resets the backoff, since
688
+ the branch got its weekly answer whatever it was."""
689
+ import os as _os
690
+
691
+ base = base_minutes if base_minutes is not None else int(
692
+ _os.environ.get(GOLDEN_RETRY_BASE_MINUTES_ENV, DEFAULT_GOLDEN_RETRY_BASE_MINUTES))
693
+ prev = dict(marker or {})
694
+ consecutive = int(prev.get("consecutive_transient_failures", 0))
695
+ consecutive = consecutive + 1 if transient else 0
696
+ out = dict(prev)
697
+ out["consecutive_transient_failures"] = consecutive
698
+ if transient:
699
+ backoff_min = golden_retry_backoff_minutes(base, consecutive)
700
+ out["next_retry_at"] = (now + datetime.timedelta(minutes=backoff_min)).strftime(
701
+ "%Y-%m-%dT%H:%M:%SZ")
702
+ else:
703
+ out["next_retry_at"] = None
704
+ return out
705
+
706
+
707
+ def render_promote_scan_hot_entry(candidates: list[dict[str, Any]], today: datetime.date) -> str:
708
+ lines = [f"## {today.isoformat()} — Sunday promotion-scan"]
709
+ lines.append(
710
+ f"- **Context:** {len(candidates)} `raw/` source(s) not yet promoted "
711
+ "into a typed `brain/` note."
712
+ )
713
+ for c in candidates[:10]:
714
+ lines.append(f" - `{c.get('id')}` ({c.get('path')})")
715
+ lines.append(
716
+ "- **Owner input needed:** review for promotion (`brain capture` / "
717
+ "`brain write`) — promotion itself stays a human gate."
718
+ )
719
+ return "\n".join(lines) + "\n"
720
+
721
+
722
+ # ---------------------------------------------------------------------------
723
+ # Workspace sweep (WSP-01, 2026-07-11). A live working folder (e.g. an
724
+ # Obsidian vault's `99 Workspace/`) accumulates hundreds of session artifacts
725
+ # with no lifecycle. The sweep gives them one: a SETTLED file (mtime older
726
+ # than the age gate — nobody is editing it any more) is MOVED into the
727
+ # vault's `inbox/`, where the standard ingest drain archives the original
728
+ # immutably under `raw/originals/`, signs + writes the `raw/` note (with a
729
+ # filename-derived `document_date`), the next sync embeds it, and the
730
+ # monthly graphify wires it into the discovery graph. Content already
731
+ # ingested dedups by content hash (parked in the inbox duplicate dir), so
732
+ # the sweep is idempotent and never double-ingests.
733
+ #
734
+ # Scope rules (deliberately dumb): TOP-LEVEL FILES ONLY — subdirectories are
735
+ # other systems' machine state (skill packages, archives, trust runs) and are
736
+ # never touched; dotfiles are skipped. Actively-edited files have a fresh
737
+ # mtime and are skipped by the age gate. Configuration: the sweep only runs
738
+ # when dirs are configured ($BRAIN_WORKSPACE_SWEEP_DIRS, os.pathsep-separated;
739
+ # $BRAIN_WORKSPACE_SWEEP_AGE_DAYS, default 14) — no config, no sweep.
740
+ # ---------------------------------------------------------------------------
741
+ WORKSPACE_SWEEP_DIRS_ENV = "BRAIN_WORKSPACE_SWEEP_DIRS"
742
+ WORKSPACE_SWEEP_AGE_ENV = "BRAIN_WORKSPACE_SWEEP_AGE_DAYS"
743
+ WORKSPACE_SWEEP_DEFAULT_AGE_DAYS = 14
744
+
745
+
746
+ def workspace_sweep_config() -> tuple[list[tuple[Path, int | None]], int]:
747
+ """Configured sweep sources + default age gate. Empty list = disabled.
748
+
749
+ Each $BRAIN_WORKSPACE_SWEEP_DIRS entry is ``path`` or ``path=N`` — the
750
+ per-dir age override (2026-07-11, round-5 benchmark): a CAPTURE folder
751
+ (an Obsidian ``00 Inbox``, a meetings drop) holds FINAL documents that
752
+ settle in a day, while a WORKING folder needs the long gate so
753
+ in-progress files are never swept. One global age starved the capture
754
+ folders by a week; ``path=1`` fixes that per source."""
755
+ import os
756
+
757
+ raw = os.environ.get(WORKSPACE_SWEEP_DIRS_ENV, "").strip()
758
+ dirs: list[tuple[Path, int | None]] = []
759
+ for entry in raw.split(os.pathsep):
760
+ entry = entry.strip()
761
+ if not entry:
762
+ continue
763
+ path_part, sep, age_part = entry.rpartition("=")
764
+ if sep and age_part.isdigit():
765
+ # age 0 is legal: same-day capture sweep (a 15-minute
766
+ # write-settle guard still applies inside sweep_workspace).
767
+ dirs.append((Path(path_part).expanduser(), int(age_part)))
768
+ else:
769
+ dirs.append((Path(entry).expanduser(), None))
770
+ try:
771
+ age = int(os.environ.get(WORKSPACE_SWEEP_AGE_ENV, ""))
772
+ except ValueError:
773
+ age = WORKSPACE_SWEEP_DEFAULT_AGE_DAYS
774
+ if age < 1:
775
+ age = WORKSPACE_SWEEP_DEFAULT_AGE_DAYS
776
+ return dirs, age
777
+
778
+
779
+ def sweep_workspace(
780
+ dirs: list[Path] | list[tuple[Path, int | None]], inbox: Path, age_days: int,
781
+ now: float | None = None, dry_run: bool = False,
782
+ ) -> dict[str, Any]:
783
+ """Move settled top-level files from ``dirs`` into ``inbox``.
784
+
785
+ ``dirs`` entries are ``Path`` (use the global ``age_days``) or
786
+ ``(Path, age)`` tuples (per-dir override; ``None`` = global). Pure file
787
+ motion — classification/signing/dedup all happen downstream in the
788
+ ingest drain. Collisions uniquify (never clobber an inbox file).
789
+ Returns an honest report; a missing dir is reported, never raised."""
790
+ import time
791
+
792
+ from .ingest.pipeline import _move, _unique_dest
793
+
794
+ from .ingest.handlers import handler_for
795
+
796
+ base_now = now if now is not None else time.time()
797
+ report: dict[str, Any] = {
798
+ "swept": [], "skipped_active": 0, "skipped_unsupported": 0,
799
+ "missing_dirs": [], "errors": [],
800
+ "age_days": age_days, "dry_run": dry_run,
801
+ }
802
+ for entry in dirs:
803
+ d, dir_age = entry if isinstance(entry, tuple) else (entry, None)
804
+ eff_age = dir_age if dir_age is not None else age_days
805
+ # age 0 = capture-inbox mode: sweep same-day, but never a file
806
+ # younger than 15 minutes (write-settle guard against partial copies).
807
+ cutoff = base_now - (900.0 if eff_age == 0 else eff_age * 86400.0)
808
+ if not d.is_dir():
809
+ report["missing_dirs"].append(str(d))
810
+ continue
811
+ for p in sorted(d.iterdir()):
812
+ if not p.is_file() or p.name.startswith("."):
813
+ continue
814
+ if handler_for(p) is None:
815
+ # Machine artifacts (.py, .json, .tsv, …) have no ingest
816
+ # handler — sweeping them only detours through quarantine
817
+ # (measured: 344 on the first real sweep). Leave them where
818
+ # they live; the count keeps the skip honest.
819
+ report["skipped_unsupported"] += 1
820
+ continue
821
+ try:
822
+ mtime = p.stat().st_mtime
823
+ except OSError as exc:
824
+ report["errors"].append({"file": str(p), "error": str(exc)})
825
+ continue
826
+ if mtime > cutoff:
827
+ report["skipped_active"] += 1
828
+ continue
829
+ if dry_run:
830
+ report["swept"].append({"file": str(p), "would_move": True})
831
+ continue
832
+ try:
833
+ inbox.mkdir(parents=True, exist_ok=True)
834
+ _move(p, _unique_dest(inbox, p.name))
835
+ report["swept"].append({"file": str(p)})
836
+ except OSError as exc:
837
+ report["errors"].append({"file": str(p), "error": str(exc)})
838
+ return report
839
+
840
+
841
+ # ---------------------------------------------------------------------------
842
+ # CUT-02 — quarantine & duplicate retention fold. The ingest pipeline
843
+ # (``ingest/pipeline.py``) parks two kinds of leftovers under ``inbox/`` that
844
+ # used to grow forever with nobody looking:
845
+ # - ``_duplicate/`` — content that hashed identical to something already
846
+ # promoted into ``raw/``. Safe to prune after a grace period, but ONLY
847
+ # once the full provenance chain re-confirms the redundancy (see
848
+ # ``_verify_duplicate_provenance`` — HARDENED:codex, a manifest hit alone
849
+ # is not proof: the manifest's ``sha256`` key is the ORIGINAL file's hash,
850
+ # but a raw note's own ``sha256:`` frontmatter is the EXTRACTED
851
+ # MARKDOWN's hash, not the original's — the only thing that can prove
852
+ # "this parked file's bytes truly live elsewhere" is the archived
853
+ # original under ``raw/originals/`` the note's ``origin:`` points at).
854
+ # - ``_quarantine/`` — content that could NOT be safely processed (bad
855
+ # encoding, missing handler, collision, ...). NEVER auto-deleted: it may
856
+ # be the only copy. Instead it gets a non-destructive monthly triage
857
+ # summary queued to ``hot.md`` (see ``quarantine_triage_summary`` /
858
+ # ``render_quarantine_summary_hot_entry`` / ``quarantine_summary_due``).
859
+ # ---------------------------------------------------------------------------
860
+ DUPLICATE_RETENTION_DAYS_ENV = "BRAIN_DUPLICATE_RETENTION_DAYS"
861
+ DEFAULT_DUPLICATE_RETENTION_DAYS = 30
862
+ # Mirrors the sidecar convention ``ingest/pipeline.py``'s ``_process_claimed``
863
+ # writes alongside every parked duplicate (``<file>.duplicate-of.txt``).
864
+ _DUPLICATE_SIDECAR_SUFFIX = ".duplicate-of.txt"
865
+
866
+
867
+ def _sha256_file(path: Path) -> str:
868
+ """Streaming sha256 (reuses snapshot's chunked helper — review finding [5]:
869
+ a parked duplicate AND its archived original are BOTH hashed per aged
870
+ candidate, and raw/originals are exactly the large binaries — never load a
871
+ whole PDF/video into memory)."""
872
+ from .snapshot import _sha256_file as _stream_sha256_file
873
+
874
+ return _stream_sha256_file(path)
875
+
876
+
877
+ def _verify_duplicate_provenance(dup_file: Path, vault: Path, manifest: dict[str, str]) -> tuple[bool, str]:
878
+ """The FULL provenance chain a parked duplicate must clear before it is
879
+ ever deleted (HARDENED:codex — the naive "a manifest entry exists" guard
880
+ is insufficient, see the module docstring above):
881
+
882
+ 1. hash the PARKED file itself (its bytes are the original bytes, moved
883
+ verbatim into ``_duplicate/`` — never re-encoded);
884
+ 2. that hash must be a key in the ingest manifest (``sha256(original) ->
885
+ raw-note-id``);
886
+ 3. resolve the manifest's target ``raw/<id>.md`` note;
887
+ 4. read ITS ``origin:`` frontmatter — the archived-original's path;
888
+ 5. confirm that archived file still exists under ``raw/originals/`` AND
889
+ still hashes to the SAME original sha.
890
+
891
+ Only step 5 passing proves the parked bytes are truly redundant. Returns
892
+ ``(True, "verified")`` on success, else ``(False, <specific reason>)`` —
893
+ every failure reason is precise enough to explain a skip without
894
+ re-deriving it, since a failed chain is reported, never silently dropped."""
895
+ try:
896
+ dup_sha = _sha256_file(dup_file)
897
+ except OSError as exc:
898
+ return False, f"parked file unreadable: {exc}"
899
+ existing_id = manifest.get(dup_sha)
900
+ if existing_id is None:
901
+ return False, "no ingest-manifest entry for this file's content hash"
902
+ note_path = vault / "raw" / f"{existing_id}.md"
903
+ if not note_path.is_file():
904
+ return False, f"manifest target raw/{existing_id}.md does not exist"
905
+ from . import frontmatter as fm
906
+
907
+ try:
908
+ meta, _ = fm.parse_text(note_path.read_text(encoding="utf-8"))
909
+ except Exception as exc: # noqa: BLE001 — review finding [3]: a malformed raw
910
+ # note (invalid UTF-8 = UnicodeDecodeError, or a YAML parse error — both
911
+ # ValueError, not OSError) must SKIP this one candidate, never propagate
912
+ # and abort the whole fold (which would starve pruning of every other
913
+ # verifiable duplicate). Fail-safe: an unverifiable note is never pruned.
914
+ return False, f"could not read/parse raw/{existing_id}.md: {exc}"
915
+ origin = meta.get("origin")
916
+ if not origin:
917
+ return False, f"raw/{existing_id}.md has no origin: provenance"
918
+ archive_path = vault / str(origin)
919
+ if not archive_path.is_file():
920
+ return False, f"archived original '{origin}' is missing"
921
+ try:
922
+ archive_sha = _sha256_file(archive_path)
923
+ except OSError as exc:
924
+ return False, f"could not hash archived original: {exc}"
925
+ if archive_sha != dup_sha:
926
+ return False, "archived original's hash no longer matches the parked duplicate"
927
+ return True, "verified"
928
+
929
+
930
+ def retention_fold(vault: Path, today: datetime.date, *, dry_run: bool = False) -> dict[str, Any]:
931
+ """CUT-02: prune ``inbox/_duplicate/`` entries older than
932
+ ``BRAIN_DUPLICATE_RETENTION_DAYS`` (default 30, mtime-based). Safe by
933
+ construction ONLY because every candidate is re-verified through
934
+ ``_verify_duplicate_provenance`` right before deletion — a manifest hit
935
+ is necessary but not sufficient (see the module docstring); anything
936
+ that fails the chain is left in place and reported, never deleted on an
937
+ unverified chain. Deletes the parked file + its ``.duplicate-of.txt``
938
+ sidecar together, as one lot. Never touches ``inbox/_quarantine/`` (see
939
+ ``quarantine_triage_summary`` for that lifecycle instead)."""
940
+ import os
941
+ import time
942
+
943
+ from .ingest.pipeline import _load_manifest as _load_ingest_manifest
944
+
945
+ # Review finding [1]: a non-integer $BRAIN_DUPLICATE_RETENTION_DAYS must not
946
+ # raise (which would block the whole fold and silently disable pruning) —
947
+ # fall back to the default on any bad value.
948
+ try:
949
+ retention_days = int(os.environ.get(
950
+ DUPLICATE_RETENTION_DAYS_ENV, DEFAULT_DUPLICATE_RETENTION_DAYS))
951
+ except (TypeError, ValueError):
952
+ retention_days = DEFAULT_DUPLICATE_RETENTION_DAYS
953
+ dup_dir = vault / "inbox" / "_duplicate"
954
+ report: dict[str, Any] = {
955
+ "retention_days": retention_days, "dry_run": dry_run,
956
+ "considered": 0, "not_due": 0, "pruned": [], "skipped": [],
957
+ }
958
+ if not dup_dir.is_dir():
959
+ return report
960
+
961
+ manifest = _load_ingest_manifest(vault)
962
+ cutoff_ts = time.mktime(today.timetuple()) - retention_days * 86400.0
963
+
964
+ for p in sorted(dup_dir.iterdir()):
965
+ if not p.is_file() or p.name.endswith(_DUPLICATE_SIDECAR_SUFFIX):
966
+ continue # sidecars are handled alongside their primary file
967
+ report["considered"] += 1
968
+ try:
969
+ mtime = p.stat().st_mtime
970
+ except OSError as exc:
971
+ report["skipped"].append({"file": p.name, "kind": "stat", "reason": f"stat failed: {exc}"})
972
+ continue
973
+ if mtime > cutoff_ts:
974
+ report["not_due"] += 1
975
+ continue
976
+ # Per-candidate isolation (review finding [3]): any unexpected error in
977
+ # the verify chain skips THIS candidate, never aborts the fold.
978
+ try:
979
+ ok, reason = _verify_duplicate_provenance(p, vault, manifest)
980
+ except Exception as exc: # noqa: BLE001 — defensive; verify is fail-safe
981
+ report["skipped"].append(
982
+ {"file": p.name, "kind": "provenance", "reason": f"verify error: {exc}"})
983
+ continue
984
+ if not ok:
985
+ report["skipped"].append({"file": p.name, "kind": "provenance", "reason": reason})
986
+ continue
987
+ if dry_run:
988
+ report["pruned"].append({"file": p.name, "would_delete": True})
989
+ continue
990
+ # Delete the primary FIRST (the data-safe step, since provenance
991
+ # verified the bytes are archived). Review finding [2]: only a PRIMARY
992
+ # delete failure is a "not pruned" skip — a sidecar-unlink failure AFTER
993
+ # the primary is gone must NOT be misreported as a provenance failure
994
+ # (the file IS pruned); record the orphaned sidecar separately instead.
995
+ sidecar = dup_dir / f"{p.name}{_DUPLICATE_SIDECAR_SUFFIX}"
996
+ try:
997
+ p.unlink()
998
+ except OSError as exc:
999
+ report["skipped"].append(
1000
+ {"file": p.name, "kind": "delete", "reason": f"delete failed: {exc}"})
1001
+ continue
1002
+ entry: dict[str, Any] = {"file": p.name}
1003
+ try:
1004
+ sidecar.unlink(missing_ok=True)
1005
+ except OSError as exc:
1006
+ entry["orphaned_sidecar"] = f"{sidecar.name}: {exc}"
1007
+ report["pruned"].append(entry)
1008
+ return report
1009
+
1010
+
1011
+ def quarantine_summary_due(marker: dict[str, Any] | None, today: datetime.date) -> bool:
1012
+ """True when the monthly quarantine-triage summary is due: never fired,
1013
+ or last fired in an earlier calendar month than ``today`` — "due since
1014
+ last run" catch-up (mirrors every other monthly ``maintain`` branch),
1015
+ not a strict "only on the 1st" calendar check."""
1016
+ if not marker:
1017
+ return True
1018
+ return marker.get("last_month") != today.strftime("%Y-%m")
1019
+
1020
+
1021
+ def quarantine_triage_summary(vault: Path, today: datetime.date | None = None) -> dict[str, Any]:
1022
+ """A non-destructive snapshot of ``inbox/_quarantine/`` — counts by
1023
+ reason (the reason subdirectory ``_quarantine()`` files each candidate
1024
+ into, see ``ingest/pipeline.py``'s ``_quarantine``), oldest item's age in
1025
+ days, and total size. Never deletes anything: a quarantined file may be
1026
+ the only copy of its content (that's exactly why it never reached
1027
+ ``raw/`` in the first place)."""
1028
+ import time
1029
+
1030
+ qdir = vault / "inbox" / "_quarantine"
1031
+ result: dict[str, Any] = {
1032
+ "total": 0, "by_reason": {}, "oldest_age_days": None, "total_bytes": 0,
1033
+ }
1034
+ if not qdir.is_dir():
1035
+ return result
1036
+
1037
+ now_ts = time.mktime(today.timetuple()) if today else time.time()
1038
+ oldest_mtime: float | None = None
1039
+ for reason_dir in sorted(p for p in qdir.iterdir() if p.is_dir()):
1040
+ count = 0
1041
+ for f in reason_dir.rglob("*"):
1042
+ if not f.is_file() or f.name.endswith(".reason.txt"):
1043
+ continue
1044
+ count += 1
1045
+ try:
1046
+ st = f.stat()
1047
+ except OSError:
1048
+ continue
1049
+ result["total_bytes"] += st.st_size
1050
+ if oldest_mtime is None or st.st_mtime < oldest_mtime:
1051
+ oldest_mtime = st.st_mtime
1052
+ if count:
1053
+ result["by_reason"][reason_dir.name] = count
1054
+ result["total"] += count
1055
+ if oldest_mtime is not None:
1056
+ result["oldest_age_days"] = max(0, int((now_ts - oldest_mtime) // 86400))
1057
+ return result
1058
+
1059
+
1060
+ def render_quarantine_summary_hot_entry(summary: dict[str, Any], today: datetime.date) -> str:
1061
+ """The monthly quarantine-aging hot-queue entry — never silent, never a
1062
+ prompt to delete anything (see ``retention_fold``'s docstring for why
1063
+ quarantine is excluded from auto-pruning)."""
1064
+ lines = [f"## {today.isoformat()} — Monthly quarantine triage"]
1065
+ total = summary.get("total", 0)
1066
+ if not total:
1067
+ lines.append("- **Context:** `inbox/_quarantine/` is empty. Nothing to triage.")
1068
+ return "\n".join(lines) + "\n"
1069
+ size_mb = round(summary.get("total_bytes", 0) / (1024 * 1024), 1)
1070
+ lines.append(
1071
+ f"- **Context:** {total} quarantined file(s), {size_mb} MB total, "
1072
+ f"oldest {summary.get('oldest_age_days')} day(s) old."
1073
+ )
1074
+ for reason, count in sorted(summary.get("by_reason", {}).items(), key=lambda kv: -kv[1]):
1075
+ lines.append(f" - `{reason}`: {count}")
1076
+ lines.append(
1077
+ "- **Owner input needed:** review `inbox/_quarantine/` and fix or "
1078
+ "discard each reason bucket by hand — quarantined files are NEVER "
1079
+ "auto-deleted; a file here may be the only copy of its content."
1080
+ )
1081
+ return "\n".join(lines) + "\n"
1082
+
1083
+
1084
+ # ---------------------------------------------------------------------------
1085
+ # Self-organization folds (owner decision 2026-07-11): metadata, versioning,
1086
+ # PARA zoning and navigation are AUTOMATIC nightly maintenance, not user
1087
+ # input. Synthesis (writing new prose notes) remains session work — these
1088
+ # folds only manage METADATA and generated views, never note bodies.
1089
+ # ---------------------------------------------------------------------------
1090
+ _WIKILINK_RE = re.compile(r"\[\[([^\]|#]+)(?:\|.+?)?\]\]")
1091
+ _PARA_ZONES = ("projects", "areas", "resources", "archive")
1092
+ _VERSION_ID_RE = re.compile(r"^(?P<base>.+?)-v(?P<num>\d{1,3})$")
1093
+ _LEADING_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}-")
1094
+
1095
+
1096
+ def version_family_key(note_id: str) -> tuple[str, int] | None:
1097
+ """(family, version) for an id that names an explicit document version
1098
+ (``…-v12``), else None. The family key strips ONE leading capture-date
1099
+ prefix so re-captures of the same document line up
1100
+ (``2026-07-09-…-annex-v12`` and ``2026-05-27-…-annex-v10`` are one
1101
+ family). Deliberately conservative: only a trailing ``-v<digits>``
1102
+ counts — ``-v4-0``, ``-vf``, ``-vcomentada-26`` never chain."""
1103
+ m = _VERSION_ID_RE.match(note_id)
1104
+ if not m:
1105
+ return None
1106
+ base = _LEADING_DATE_RE.sub("", m.group("base"), count=1)
1107
+ return base, int(m.group("num"))
1108
+
1109
+
1110
+ def auto_version_chains(core: Any) -> dict[str, Any]:
1111
+ """VER-01: stamp supersession chains across explicit version families.
1112
+
1113
+ Groups indexed notes by ``version_family_key``, orders each family by
1114
+ (version, valid-date, id), and retires each predecessor via the AUDITED
1115
+ ``core.supersede`` path (both sides signed, journaled, invariant-checked
1116
+ — never a raw frontmatter poke). Idempotent: an already-retired
1117
+ predecessor is skipped; a predecessor already superseded by something
1118
+ OUTSIDE the computed chain (a human's manual call) freezes its family —
1119
+ reported, never overridden."""
1120
+ rows = core.index.conn.execute(
1121
+ "SELECT id, is_latest_version, superseded_by, "
1122
+ "COALESCE(NULLIF(effective_date,''), NULLIF(document_date,''), created) "
1123
+ "FROM notes").fetchall()
1124
+ families: dict[str, list[tuple[int, str, str, dict[str, str]]]] = {}
1125
+ for nid, ilv, sup_by, vdate in rows:
1126
+ key = version_family_key(str(nid))
1127
+ if key is None:
1128
+ continue
1129
+ fam, num = key
1130
+ families.setdefault(fam, []).append(
1131
+ (num, str(vdate or ""), str(nid),
1132
+ {"is_latest": str(ilv or ""), "superseded_by": str(sup_by or "")}))
1133
+
1134
+ report: dict[str, Any] = {"chained": [], "skipped_conflict": [], "errors": []}
1135
+ for fam, members in sorted(families.items()):
1136
+ if len(members) < 2:
1137
+ continue
1138
+ members.sort()
1139
+ ordered = [m[2] for m in members]
1140
+ meta = {m[2]: m[3] for m in members}
1141
+ # A manual chain pointing outside the computed order freezes the family.
1142
+ conflict = any(
1143
+ meta[nid]["superseded_by"] and meta[nid]["superseded_by"] != ordered[i + 1]
1144
+ for i, nid in enumerate(ordered[:-1]))
1145
+ if conflict:
1146
+ report["skipped_conflict"].append(fam)
1147
+ continue
1148
+ for old_id, new_id in zip(ordered[:-1], ordered[1:]):
1149
+ if meta[old_id]["superseded_by"] == new_id:
1150
+ continue # already chained — idempotent re-run
1151
+ try:
1152
+ core.supersede(old_id, new_id, reason="auto version-chain (nightly self-organization)")
1153
+ report["chained"].append({"old": old_id, "new": new_id, "family": fam})
1154
+ except Exception as exc: # noqa: BLE001 — one bad family never aborts the fold
1155
+ report["errors"].append({"family": fam, "old": old_id,
1156
+ "new": new_id, "error": str(exc)})
1157
+ break
1158
+ return report
1159
+
1160
+
1161
+ def auto_para(vault: Path) -> dict[str, Any]:
1162
+ """PAR-01: file brain/ notes into their PARA zone by METADATA, not by a
1163
+ human dragging files. Two deliberately small rules:
1164
+
1165
+ - ``type: project`` -> ``brain/projects/``
1166
+ - ``is_latest_version: false`` (retired by a supersession chain)
1167
+ -> ``brain/archive/``
1168
+
1169
+ Generated views (``type: index``/``moc``) and everything else stay where
1170
+ they are. Moves are by-id-safe: wikilinks target ids, not paths, and the
1171
+ next index sync reconciles paths."""
1172
+ from . import frontmatter as fm
1173
+
1174
+ brain_dir = vault / "brain"
1175
+ report: dict[str, Any] = {"moved": [], "errors": []}
1176
+ if not brain_dir.is_dir():
1177
+ return report
1178
+ for p in sorted(brain_dir.rglob("*.md")):
1179
+ if p.name in ("backlinks.md", "catalog.md", "index.md"):
1180
+ continue
1181
+ try:
1182
+ meta, _ = fm.parse_text(p.read_text(encoding="utf-8"))
1183
+ except Exception as exc: # noqa: BLE001
1184
+ report["errors"].append({"file": str(p), "error": str(exc)})
1185
+ continue
1186
+ ntype = str(meta.get("type") or "")
1187
+ if ntype in ("index", "moc"):
1188
+ continue
1189
+ retired = str(meta.get("is_latest_version")).lower() == "false"
1190
+ dest_zone = ("archive" if retired
1191
+ else "projects" if ntype == "project" else None)
1192
+ if dest_zone is None or p.parent.name == dest_zone:
1193
+ continue
1194
+ dest_dir = brain_dir / dest_zone
1195
+ dest_dir.mkdir(parents=True, exist_ok=True)
1196
+ dest = dest_dir / p.name
1197
+ if dest.exists():
1198
+ report["errors"].append({"file": str(p), "error": f"collision at {dest}"})
1199
+ continue
1200
+ p.rename(dest)
1201
+ report["moved"].append({"id": str(meta.get("id") or p.stem),
1202
+ "to": f"brain/{dest_zone}/"})
1203
+ return report
1204
+
1205
+
1206
+ def refresh_navigation(vault: Path) -> dict[str, Any]:
1207
+ """NAV-01: regenerate the human navigation surfaces nightly —
1208
+ ``brain/backlinks.md`` + one ``catalog.md`` per PARA zone. Byte-compatible
1209
+ with ``tools/validate.py --backlinks --catalogs`` (same formats, both
1210
+ deterministic, no wall-clock timestamps), so either producer yields a
1211
+ no-op diff over an unchanged vault."""
1212
+ from . import frontmatter as fm
1213
+
1214
+ brain_dir = vault / "brain"
1215
+ raw_dir = vault / "raw"
1216
+ notes: list[dict[str, Any]] = []
1217
+ for base, zone in ((raw_dir, "raw"), (brain_dir, "brain")):
1218
+ if not base.is_dir():
1219
+ continue
1220
+ for p in sorted(base.rglob("*.md")):
1221
+ if p.name in ("backlinks.md", "catalog.md"):
1222
+ continue
1223
+ try:
1224
+ meta, body = fm.parse_text(p.read_text(encoding="utf-8"))
1225
+ except Exception: # noqa: BLE001 — a broken note never kills navigation
1226
+ continue
1227
+ notes.append({"meta": meta, "body": body, "path": p, "zone": zone})
1228
+
1229
+ ids = {n["meta"].get("id") for n in notes if n["meta"].get("id")}
1230
+ backlinks: dict[str, set[str]] = {}
1231
+ for n in notes:
1232
+ src = n["meta"].get("id")
1233
+ for m in _WIKILINK_RE.finditer(n["body"]):
1234
+ target = m.group(1).strip()
1235
+ if target in ids:
1236
+ backlinks.setdefault(target, set()).add(src)
1237
+
1238
+ title_by_id = {n["meta"].get("id"): n["meta"].get("title", n["meta"].get("id"))
1239
+ for n in notes}
1240
+ lines = [
1241
+ "---", "id: backlinks", "title: \"Backlinks (generated)\"",
1242
+ "type: index", "classification: Internal", "---", "",
1243
+ "# Backlinks (generated — do not hand-edit)", "",
1244
+ ]
1245
+ for tgt in sorted(backlinks):
1246
+ lines.append(f"## [[{tgt}]]")
1247
+ for s in sorted(x for x in backlinks[tgt] if x):
1248
+ lines.append(f"- [[{s}|{title_by_id.get(s, s)}]]")
1249
+ lines.append("")
1250
+ brain_dir.mkdir(parents=True, exist_ok=True)
1251
+ (brain_dir / "backlinks.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
1252
+
1253
+ by_zone: dict[str, list[dict]] = {z: [] for z in _PARA_ZONES}
1254
+ for n in notes:
1255
+ if n["zone"] != "brain":
1256
+ continue
1257
+ rel = n["path"].relative_to(brain_dir).parts
1258
+ if len(rel) > 1 and rel[0] in by_zone:
1259
+ by_zone[rel[0]].append(n)
1260
+ for zone in _PARA_ZONES:
1261
+ zone_dir = brain_dir / zone
1262
+ zone_dir.mkdir(parents=True, exist_ok=True)
1263
+ cat = [
1264
+ "---", f"id: catalog-{zone}",
1265
+ f"title: \"{zone.capitalize()} catalog (generated)\"",
1266
+ "type: index", "classification: Internal", "---", "",
1267
+ f"# {zone.capitalize()} catalog (generated — do not hand-edit)", "",
1268
+ "| id | title | type | updated | classification |",
1269
+ "|---|---|---|---|---|",
1270
+ ]
1271
+ for n in sorted(by_zone[zone], key=lambda n: n["meta"].get("id") or ""):
1272
+ meta = n["meta"]
1273
+ cat.append(
1274
+ f"| [[{meta.get('id', '')}]] | {meta.get('title', '')} | "
1275
+ f"{meta.get('type', '')} | {meta.get('updated', '')} | {meta.get('classification', '')} |")
1276
+ cat.append("")
1277
+ (zone_dir / "catalog.md").write_text("\n".join(cat) + "\n", encoding="utf-8")
1278
+
1279
+ return {"backlink_targets": len(backlinks),
1280
+ "catalog_counts": {z: len(by_zone[z]) for z in _PARA_ZONES}}
1281
+
1282
+
1283
+ # ---------------------------------------------------------------------------
1284
+ # Decision-capture nudge (DEC-01, 2026-07-11). Measured failure, G&P
1285
+ # benchmark round 6: a real perimeter decision lived FIVE DAYS in a slide
1286
+ # deck ("decided 6-Jul", "decision taken") without a `type: decision` note —
1287
+ # so every decision-first agent was confidently stale, and the
1288
+ # decision-layer-authoritative rule amplified the gap. This fold closes the
1289
+ # loop: every maintain run scans RECENTLY captured non-decision notes for
1290
+ # decision language and queues each hit ONCE to hot.md as a decision-note
1291
+ # candidate. A nudge, not a writer — capturing the decision note stays
1292
+ # owner/synthesis work (P-10 human gate), so false positives cost one
1293
+ # hot-queue line, never a wrong decision record.
1294
+ # ---------------------------------------------------------------------------
1295
+ _DECISION_LANGUAGE_RE = re.compile(
1296
+ r"(?<![a-z])("
1297
+ r"decided(?:\s+on)?\s+\d|decided:\s|decision\s+(?:was\s+)?taken|"
1298
+ r"we\s+(?:have\s+)?decided|formally\s+approved|approved\s+on\s+\d|"
1299
+ r"signed\s+off\s+on|sign-off\s+given|"
1300
+ r"foi\s+decidido|decidiu-se|decis[aã]o\s+tomada|aprovado\s+em\s+\d"
1301
+ r")", re.IGNORECASE)
1302
+ DECISION_CAPTURE_LOOKBACK_DAYS = 3
1303
+ DECISION_CAPTURE_MAX_CANDIDATES = 10
1304
+
1305
+
1306
+ def decision_capture_scan(
1307
+ conn: Any, today: datetime.date,
1308
+ lookback_days: int = DECISION_CAPTURE_LOOKBACK_DAYS,
1309
+ limit: int = DECISION_CAPTURE_MAX_CANDIDATES,
1310
+ ) -> list[dict[str, Any]]:
1311
+ """Notes captured within ``lookback_days`` whose body carries decision
1312
+ language but whose ``type`` is not ``decision``. Returns at most
1313
+ ``limit`` candidates (id, date, phrase, snippet), newest first. Pure
1314
+ read — no writes, no egress concern (consumers queue to host-only
1315
+ hot.md)."""
1316
+ since = (today - datetime.timedelta(days=lookback_days)).isoformat()
1317
+ # Retired version-family members (is_latest_version: false) are excluded:
1318
+ # every sibling of a versioned deck repeats the same decision language —
1319
+ # only the family head is a meaningful capture candidate (live run
1320
+ # 2026-07-11: retired 6pager versions crowded the candidate cap).
1321
+ rows = conn.execute(
1322
+ "SELECT id, type, body, "
1323
+ "COALESCE(NULLIF(effective_date,''), NULLIF(document_date,''), created) "
1324
+ "FROM notes WHERE type != 'decision' AND created >= ? "
1325
+ "AND COALESCE(is_latest_version,'') != 'false' "
1326
+ "ORDER BY created DESC", (since,)).fetchall()
1327
+ out: list[dict[str, Any]] = []
1328
+ for nid, ntype, body, vdate in rows:
1329
+ m = _DECISION_LANGUAGE_RE.search(body or "")
1330
+ if not m:
1331
+ continue
1332
+ start = max(0, m.start() - 80)
1333
+ snippet = " ".join((body[start:m.end() + 120]).split())
1334
+ out.append({"id": str(nid), "type": str(ntype or ""),
1335
+ "date": str(vdate or ""), "phrase": m.group(0),
1336
+ "snippet": snippet})
1337
+ if len(out) >= limit:
1338
+ break
1339
+ return out
1340
+
1341
+
1342
+ def render_decision_capture_hot_entry(c: dict[str, Any], today: datetime.date) -> str:
1343
+ return "\n".join([
1344
+ f"## {today.isoformat()} — decision-capture candidate: `{c['id']}`",
1345
+ f"- **Context:** a freshly captured source (valid date {c.get('date') or '?'}) "
1346
+ f"carries decision language (“{c['phrase']}”) but no `type: decision` "
1347
+ f"note records it.",
1348
+ f"- **Snippet:** …{c['snippet']}…",
1349
+ "- **Owner input needed:** if this is a real decision, capture it as a "
1350
+ "`type: decision` note (and `brain supersede` whatever it reverses); "
1351
+ "if not, ignore — this entry never repeats for this note.",
1352
+ ]) + "\n"
1353
+
1354
+
1355
+ # ---------------------------------------------------------------------------
1356
+ # WATCHDOG-01 (2026-07-11): the two sanctioned scheduled tasks watch EACH
1357
+ # OTHER, so a dead task is caught by the live one instead of by a human
1358
+ # reading logs. Direction 1 (here): the hourly maintain umbrella checks the
1359
+ # synthesis heartbeat (written by scripts/brain-synthesis.sh after every
1360
+ # vault pass). Direction 2: the weekly synthesis session's prompt starts
1361
+ # with `brain status`/`brain doctor`, which surface the maintain heartbeat.
1362
+ # ---------------------------------------------------------------------------
1363
+ SYNTHESIS_STATE_ENV = "BRAIN_SYNTHESIS_STATE"
1364
+ SYNTHESIS_STALE_DAYS = 8 # weekly task + one day of grace
1365
+
1366
+
1367
+ def _load_synthesis_entry(
1368
+ vault: Path, state_path: Path | None = None,
1369
+ ) -> tuple[Path, dict[str, Any] | None]:
1370
+ """Shared ``synthesis-state.json`` resolution + read + per-vault lookup
1371
+ (fix for review finding [8] — this state-file/read/lookup sequence was
1372
+ duplicated between ``synthesis_heartbeat_finding`` and
1373
+ ``latest_synthesis_cost``). Returns ``(resolved_path, entry)`` — entry is
1374
+ ``None`` on any absence/parse failure/missing-vault-entry (never
1375
+ raises); the path is still returned so a caller (the watchdog finding)
1376
+ can report where it looked."""
1377
+ import os
1378
+
1379
+ path = state_path or Path(
1380
+ os.environ.get(SYNTHESIS_STATE_ENV, "")
1381
+ or Path.home() / ".brain" / "synthesis-state.json")
1382
+ try:
1383
+ state = json.loads(path.read_text(encoding="utf-8"))
1384
+ except (OSError, ValueError):
1385
+ return path, None
1386
+ entry = state.get(str(vault))
1387
+ return path, (entry if isinstance(entry, dict) else None)
1388
+
1389
+
1390
+ def synthesis_heartbeat_finding(
1391
+ vault: Path, today: datetime.date,
1392
+ state_path: Path | None = None,
1393
+ stale_days: int = SYNTHESIS_STALE_DAYS,
1394
+ ) -> dict[str, Any] | None:
1395
+ """An ``action_required`` finding when this vault's last SUCCESSFUL
1396
+ synthesis run is older than ``stale_days`` (or attempts keep failing) —
1397
+ ``None`` when healthy, unknown, or synthesis simply isn't set up here.
1398
+
1399
+ Fail-quiet on absence by design: no state file means the synthesis task
1400
+ has never run on this host (not installed, or first week) — flagging
1401
+ that would nag every non-synthesis install forever. Once a heartbeat
1402
+ EXISTS for this vault, silence longer than the cadence is a real
1403
+ failure signal."""
1404
+ path, entry = _load_synthesis_entry(vault, state_path)
1405
+ if entry is None:
1406
+ return None
1407
+ last_ok = entry.get("last_success")
1408
+ last_try = entry.get("last_attempt")
1409
+ ref = last_ok or last_try
1410
+ if not ref:
1411
+ return None
1412
+ try:
1413
+ age = (today - datetime.date.fromisoformat(str(ref)[:10])).days
1414
+ except ValueError:
1415
+ return None
1416
+ failing = entry.get("rc", 0) != 0 and last_ok != last_try
1417
+ if age <= stale_days and not failing:
1418
+ return None
1419
+ what = (f"last SUCCESSFUL synthesis for this vault was {ref} "
1420
+ f"({age}d ago; cadence is weekly)" if not failing else
1421
+ f"synthesis attempts are FAILING (last attempt {last_try}, "
1422
+ f"rc={entry.get('rc')}; last success {last_ok or 'never'})")
1423
+ return action_required_item(
1424
+ f"brain-synthesis watchdog: {what}",
1425
+ "the weekly synthesis session keeps the state/MOC layer current; "
1426
+ "a silent death re-opens the decision-staleness gap the 2026-07 "
1427
+ "benchmark exposed",
1428
+ "check ~/.brain/logs/synthesis-*.log and `launchctl list "
1429
+ "com.brainiac.synthesis`; re-run scripts/brain-synthesis.sh manually "
1430
+ "to confirm the fix",
1431
+ str(path))
1432
+
1433
+
1434
+ def latest_synthesis_cost(vault: Path, state_path: Path | None = None) -> float | None:
1435
+ """OBS-04 lift: the most recently METERED ``est_cost_usd`` for this vault
1436
+ from ``synthesis-state.json`` (written by ``scripts/brain-synthesis.sh``
1437
+ from the ``claude -p --output-format json`` structured usage stream —
1438
+ NEVER scraped human-readable text, per HARDENED correction 4: a scraped
1439
+ format drifts silently to a wrong zero). Absent, unparseable, or a bare
1440
+ ``0`` all record as ``None`` — a real cost of exactly zero is
1441
+ indistinguishable from "never measured" and the caller must not read a
1442
+ zero as a healthy trend point."""
1443
+ _, entry = _load_synthesis_entry(vault, state_path)
1444
+ if entry is None:
1445
+ return None
1446
+ cost = entry.get("est_cost_usd")
1447
+ return float(cost) if isinstance(cost, (int, float)) and cost > 0 else None
1448
+
1449
+
1450
+ # ---------------------------------------------------------------------------
1451
+ # WD-01 (2026-07-12) — off-host watchdog of last resort. If launchd itself
1452
+ # dies (or its plists get wiped), the brain-nightly umbrella — and the
1453
+ # synthesis-heartbeat check running INSIDE it (WATCHDOG-01, commit d28c0ce)
1454
+ # — die with it. This EXTENDS that shipped pattern rather than rebuilding
1455
+ # it: `offhost_watchdog_findings` reuses `synthesis_heartbeat_finding` and
1456
+ # `health_trend` UNCHANGED, adding only the one check neither of those
1457
+ # covers — "is `maintain` itself still firing at all".
1458
+ #
1459
+ # Freshness is keyed on the LATEST health-history record's `ts` (a precise
1460
+ # ISO datetime, written every maintain run), never on `maintain-state.json`'s
1461
+ # `last_run` (an ISO DATE only — it advances in whole-day steps, so it
1462
+ # cannot express a >26h threshold: the first value it could ever cross is
1463
+ # 48h). This is the correction that makes an hourly-cadence watchdog
1464
+ # meaningful at sub-day granularity.
1465
+ #
1466
+ # LOCAL-first (owner decision 2026-07-12): the off-host CLOUD leg (a Claude
1467
+ # `/schedule` routine reading this remotely, weekly) is DEFERRED. Verified
1468
+ # (not assumed) via the `schedule` skill's own documentation: a `/schedule`
1469
+ # cloud routine "cannot access local files, local services, or local
1470
+ # environment variables" — there is no remote-export transport yet to get
1471
+ # this vault's local state to it. What ships now is the LOCAL check +
1472
+ # macOS-push failsafe (the SAME `fire_notification` osascript channel
1473
+ # OBS-02 already uses — deliberately not a new outbound/remote channel).
1474
+ # See docs/operations/wd01-offhost-watchdog-spec.md.
1475
+ # ---------------------------------------------------------------------------
1476
+ OFFHOST_DAILY_STALE_HOURS_ENV = "BRAIN_OFFHOST_DAILY_STALE_HOURS"
1477
+ DEFAULT_OFFHOST_DAILY_STALE_HOURS = 26
1478
+
1479
+
1480
+ def _union_by_run_id(*record_lists: list[dict[str, Any]]) -> list[dict[str, Any]]:
1481
+ """Union health records across the main history + sparse sidecar, deduped
1482
+ by ``run_id`` (falling back to ``ts`` then object identity). Later lists
1483
+ win on a key collision — callers pass ``(history, sparse)`` so a sidecar
1484
+ record supersedes the same-id main-history one. Fix [8]: the ONE
1485
+ union-dedup both the off-host watchdog's freshness check and
1486
+ ``health_trend``'s golden lookback share, instead of two copies."""
1487
+ merged: dict[str, dict[str, Any]] = {}
1488
+ for r in itertools.chain.from_iterable(record_lists):
1489
+ rid = str(r.get("run_id") or r.get("ts") or id(r))
1490
+ merged[rid] = r
1491
+ return list(merged.values())
1492
+
1493
+
1494
+ def offhost_watchdog_findings(
1495
+ vault: Path, now: datetime.datetime | None = None,
1496
+ *, daily_stale_hours: float | None = None,
1497
+ ) -> list[str]:
1498
+ """Human-readable breach strings for THIS vault — an empty list means
1499
+ healthy (silent); any entry means fire the failsafe notification.
1500
+
1501
+ The failsafe's PURPOSE is "maintain/health is DEAD" — so it is scoped to
1502
+ STALENESS ONLY (fix [4]); the transient blocked/regression fold was
1503
+ DROPPED because that is OBS-02's on-host job and folding it here fired an
1504
+ un-deduped, hourly-re-firing notification that duplicated the on-host
1505
+ alarm. Two independent staleness checks (one absent signal never hides
1506
+ another):
1507
+
1508
+ 1. **Heartbeat freshness** — the latest health-history record (main +
1509
+ sparse union) is older than ``daily_stale_hours`` (default 26h — an
1510
+ hourly cadence + grace). NO history at all (fresh install, or
1511
+ genuinely never run) is SILENT, not a breach — mirrors
1512
+ `synthesis_heartbeat_finding`'s fail-quiet-on-absence: nothing to
1513
+ watch yet is not "went quiet". If NO record has a parseable ``ts`` at
1514
+ all, freshness cannot be determined — that IS a breach ("cannot
1515
+ determine health freshness"; fix [6]), never silent-healthy. When some
1516
+ records parse, the newest VALID record drives freshness: a lone
1517
+ corrupt/lexically-large ts cannot be mistaken for "latest" and mask a
1518
+ fresh valid one (re-review [825]), and a PERSISTENT corruption ages the
1519
+ youngest valid record past the window so it still breaches on staleness.
1520
+ 2. **Synthesis heartbeat** — delegates to `synthesis_heartbeat_finding`
1521
+ (WATCHDOG-01) UNCHANGED — itself a staleness (last-success-too-old)
1522
+ check.
1523
+ """
1524
+ import os as _os
1525
+
1526
+ now = now or datetime.datetime.now(datetime.timezone.utc)
1527
+ stale_hours = daily_stale_hours if daily_stale_hours is not None else float(
1528
+ _os.environ.get(OFFHOST_DAILY_STALE_HOURS_ENV, DEFAULT_OFFHOST_DAILY_STALE_HOURS))
1529
+ findings: list[str] = []
1530
+
1531
+ merged = _union_by_run_id(read_health_history(vault), read_sparse_history(vault))
1532
+ if merged:
1533
+ # Pick the newest record by PARSED ts, not a lexical max over the raw
1534
+ # strings (re-review): a corrupt but lexically-large ts on a NON-newest
1535
+ # record ("2026-07-12T25:00:00Z") would otherwise be selected as
1536
+ # "latest" and force a false freshness breach while a genuinely fresh
1537
+ # valid record exists. Parse every ts; the newest VALID record drives
1538
+ # freshness. Only if NO record has a parseable ts is it a breach.
1539
+ parsed = []
1540
+ for r in merged:
1541
+ try:
1542
+ parsed.append((datetime.datetime.strptime(
1543
+ str(r.get("ts") or ""), "%Y-%m-%dT%H:%M:%SZ",
1544
+ ).replace(tzinfo=datetime.timezone.utc), r))
1545
+ except ValueError:
1546
+ continue
1547
+ if not parsed:
1548
+ findings.append(
1549
+ f"{vault}: no health-history record has a parseable ts — cannot "
1550
+ f"determine health freshness (a breach, not silent-healthy)")
1551
+ else:
1552
+ latest_dt, latest = max(parsed, key=lambda t: t[0])
1553
+ age_hours = (now - latest_dt).total_seconds() / 3600.0
1554
+ if age_hours > stale_hours:
1555
+ findings.append(
1556
+ f"{vault}: no health-history record in {age_hours:.1f}h "
1557
+ f"(> {stale_hours:.0f}h) — the hourly maintain umbrella "
1558
+ f"(and whatever schedules it) may have died")
1559
+
1560
+ synth = synthesis_heartbeat_finding(vault, now.date())
1561
+ if synth is not None:
1562
+ findings.append(f"{vault}: {synth['finding']}")
1563
+
1564
+ return findings
1565
+
1566
+
1567
+ # ---------------------------------------------------------------------------
1568
+ # OBS-01 — health-metrics history (health-history.jsonl). Every ``maintain``
1569
+ # run appends ONE record (schema below) so trend questions ("worse than last
1570
+ # week?") have something to answer from instead of amnesia. HARDENED
1571
+ # corrections applied throughout (see s02 context bundle):
1572
+ # 1. time-based (7-CALENDAR-DAY) baseline for high-frequency metrics, never
1573
+ # "last 7 records" (that's ~7 hours on an hourly cadence); sparse weekly
1574
+ # metrics (golden_score, synthesis_cost_usd) compare against the
1575
+ # trailing non-null observation regardless of window.
1576
+ # 2. ONE final immutable append per run, built from a single run-context
1577
+ # object (``results``) that a later branch (s07's golden-eval fold) can
1578
+ # still fold into via ``results["golden"] = {...}`` before this append —
1579
+ # never a JSONL line rewrite.
1580
+ # 3. concurrency-safe append+rotation under a DEDICATED short-lived lock
1581
+ # (the coarse 2h maintain-lock can legitimately let two runs overlap);
1582
+ # monotonically-named archive segments; every record carries a unique
1583
+ # ``run_id`` the reader dedups on, tolerating one trailing partial line.
1584
+ # 4. every new maintain-state marker this session touches is ``_``-prefixed
1585
+ # (core.py:1973-1977 treats a bare key as a due-branch name); cost is
1586
+ # metered from the structured usage stream only.
1587
+ # 5. a PER-METRIC daily-bucket reducer (never one generic "representative")
1588
+ # so a single-hour blocked/latency spike survives bucketing.
1589
+ # ---------------------------------------------------------------------------
1590
+ HEALTH_HISTORY_MAX_BYTES_ENV = "BRAIN_HEALTH_HISTORY_MAX_BYTES"
1591
+ DEFAULT_HEALTH_HISTORY_MAX_BYTES = 1_000_000
1592
+ HEALTH_HISTORY_LOCK_STALE_SECONDS = 30.0
1593
+ # Fix [6]: bound the archive re-read + add retention pruning. 14 days
1594
+ # comfortably covers health_trend's 7-day trailing baseline plus a weekly
1595
+ # sparse-metric (golden_score/synthesis_cost_usd) lookback; retention is a
1596
+ # much longer, separate knob (mirrors scripts/brain-synthesis.sh's
1597
+ # `find -mtime +N -delete` posture for its own out-json captures).
1598
+ HEALTH_HISTORY_READ_WINDOW_DAYS_ENV = "BRAIN_HEALTH_HISTORY_READ_WINDOW_DAYS"
1599
+ DEFAULT_HEALTH_HISTORY_READ_WINDOW_DAYS = 14
1600
+ HEALTH_ARCHIVE_RETENTION_DAYS_ENV = "BRAIN_HEALTH_ARCHIVE_RETENTION_DAYS"
1601
+ DEFAULT_HEALTH_ARCHIVE_RETENTION_DAYS = 90
1602
+
1603
+
1604
+ def new_health_run_id() -> str:
1605
+ """A short, unique-enough id stamped on every health-history record so a
1606
+ reader merging the live file + rotated archives can dedup instead of
1607
+ double-counting a record two racing writers might otherwise both see."""
1608
+ import time
1609
+ import uuid
1610
+
1611
+ return f"{int(time.time() * 1000):x}-{uuid.uuid4().hex[:8]}"
1612
+
1613
+
1614
+ def _count_files(dir_path: Path) -> int:
1615
+ """Recursive file count under ``dir_path`` — 0 if it does not exist yet
1616
+ (a fresh vault has no ``_quarantine``/``_duplicate`` dir at all)."""
1617
+ if not dir_path.is_dir():
1618
+ return 0
1619
+ return sum(1 for p in dir_path.rglob("*") if p.is_file())
1620
+
1621
+
1622
+ def collect_health_metrics(
1623
+ core: Any, *, outcomes: dict[str, Any], results: dict[str, Any],
1624
+ run_id: str, ts: str | None = None,
1625
+ ) -> dict[str, Any]:
1626
+ """Build ONE health-history record (schema in the s02 context bundle) from
1627
+ already-computed run state. ``results`` is the SAME run-context dict
1628
+ ``BrainCore.maintain`` accumulates branch outputs into — this is the
1629
+ "structured partial-result hook" a later golden-eval branch (s07) folds
1630
+ into: it need only set ``results["golden"] = {"score": ...}`` before this
1631
+ is called, no JSONL rewrite required. Never raises on a missing piece —
1632
+ every field degrades to ``None`` rather than aborting the run's own
1633
+ health-history append."""
1634
+ import datetime as _dt
1635
+ import time as _time
1636
+
1637
+ status: dict[str, Any] = {}
1638
+ try:
1639
+ status = core.status()
1640
+ except Exception: # noqa: BLE001 — a broken status() must not break history
1641
+ status = {}
1642
+ idx = status.get("index") if isinstance(status.get("index"), dict) else {}
1643
+ snap = status.get("snapshot") if isinstance(status.get("snapshot"), dict) else {}
1644
+
1645
+ selftest_ms: float | None = None
1646
+ try:
1647
+ t0 = _time.perf_counter()
1648
+ core.hybrid_search("brain", k=1)
1649
+ selftest_ms = round((_time.perf_counter() - t0) * 1000, 1)
1650
+ except Exception: # noqa: BLE001 — probe failure is just a null latency point
1651
+ selftest_ms = None
1652
+
1653
+ vault = Path(core.vault)
1654
+ counts = outcomes.get("counts", {}) if isinstance(outcomes.get("counts"), dict) else {}
1655
+ decision_candidates = None
1656
+ dc = results.get("decision_capture")
1657
+ if isinstance(dc, dict):
1658
+ decision_candidates = dc.get("candidates")
1659
+ golden = results.get("golden") if isinstance(results.get("golden"), dict) else {}
1660
+
1661
+ return {
1662
+ "ts": ts or _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
1663
+ "run_id": run_id,
1664
+ "notes": idx.get("notes") if isinstance(idx, dict) else None,
1665
+ "chunks": idx.get("chunks") if isinstance(idx, dict) else None,
1666
+ "snapshot_gen": snap.get("generation") if isinstance(snap, dict) else None,
1667
+ "snapshot_age_s": snap.get("age_seconds") if isinstance(snap, dict) else None,
1668
+ "quarantine": _count_files(vault / "inbox" / "_quarantine"),
1669
+ "duplicate": _count_files(vault / "inbox" / "_duplicate"),
1670
+ "selftest_ms": selftest_ms,
1671
+ "action_required": counts.get("action_required", 0),
1672
+ "blocked": counts.get("blocked", 0),
1673
+ "decision_candidates": decision_candidates,
1674
+ "golden_score": golden.get("score"),
1675
+ "synthesis_cost_usd": latest_synthesis_cost(vault),
1676
+ }
1677
+
1678
+
1679
+ def _acquire_health_history_lock(
1680
+ lock_path: Path, *, stale_after: float = HEALTH_HISTORY_LOCK_STALE_SECONDS,
1681
+ ) -> None:
1682
+ """Best-effort exclusive lock scoped ONLY to the tiny append+rotate
1683
+ critical section (correction 3) — deliberately separate from
1684
+ ``BrainCore._acquire_maintain_lock``: that lock's 2h auto-break lets two
1685
+ ``maintain`` runs overlap by design, so append/rotation needs its own
1686
+ much-shorter-lived lock or two overlapping runs could both decide to
1687
+ rotate onto the same archive name. Blocks briefly (busy-wait), self-heals
1688
+ a lock older than ``stale_after`` (a crash mid-critical-section), and
1689
+ never blocks indefinitely."""
1690
+ import os
1691
+ import time
1692
+
1693
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
1694
+ while True:
1695
+ try:
1696
+ fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
1697
+ os.write(fd, str(time.time()).encode("ascii"))
1698
+ os.close(fd)
1699
+ return
1700
+ except FileExistsError:
1701
+ try:
1702
+ age = time.time() - lock_path.stat().st_mtime
1703
+ except OSError:
1704
+ age = stale_after + 1
1705
+ if age > stale_after:
1706
+ lock_path.unlink(missing_ok=True)
1707
+ continue
1708
+ time.sleep(0.05)
1709
+
1710
+
1711
+ def _release_health_history_lock(lock_path: Path) -> None:
1712
+ lock_path.unlink(missing_ok=True)
1713
+
1714
+
1715
+ def _rotate_health_history(path: Path, archive_dir: Path) -> str:
1716
+ """Move the current file to a MONOTONICALLY-NAMED, create-exclusive
1717
+ archive segment — never overwrites an existing segment even if two
1718
+ rotations somehow land in the same millisecond."""
1719
+ import os
1720
+ import time
1721
+
1722
+ archive_dir.mkdir(parents=True, exist_ok=True)
1723
+ for attempt in range(1000):
1724
+ stamp = f"{int(time.time() * 1000):x}-{attempt:03d}"
1725
+ dest = archive_dir / f"health-history-{stamp}.jsonl"
1726
+ try:
1727
+ fd = os.open(str(dest), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
1728
+ os.close(fd)
1729
+ except FileExistsError:
1730
+ continue
1731
+ os.replace(path, dest)
1732
+ return str(dest)
1733
+ raise RuntimeError("could not allocate a unique health-history archive segment")
1734
+
1735
+
1736
+ def _prune_old_files(dir_path: Path, pattern: str, retention_days: int) -> None:
1737
+ """Delete files under ``dir_path`` matching ``pattern`` whose mtime is
1738
+ older than ``retention_days``. Best-effort: a file that vanishes mid-scan
1739
+ or resists deletion is skipped, never raised. Mirrors
1740
+ ``scripts/brain-synthesis.sh``'s ``find -mtime +N -delete`` posture. One
1741
+ shared implementation for the two near-identical mtime pruners this
1742
+ session added — the health-archive and the notify-marker cleanups (review
1743
+ finding [8])."""
1744
+ import time
1745
+
1746
+ if not dir_path.is_dir():
1747
+ return
1748
+ cutoff = time.time() - retention_days * 86400.0
1749
+ for p in dir_path.glob(pattern):
1750
+ try:
1751
+ if p.stat().st_mtime < cutoff:
1752
+ p.unlink(missing_ok=True)
1753
+ except OSError:
1754
+ continue
1755
+
1756
+
1757
+ def _prune_health_archive(archive_dir: Path, retention_days: int) -> None:
1758
+ """Delete rotated ``health-history-*.jsonl`` segments older than
1759
+ ``retention_days`` (fix [6] retention companion)."""
1760
+ _prune_old_files(archive_dir, "health-history-*.jsonl", retention_days)
1761
+
1762
+
1763
+ def append_health_record(
1764
+ vault: Path, record: dict[str, Any], *, max_bytes: int | None = None,
1765
+ archive_retention_days: int | None = None,
1766
+ ) -> dict[str, Any]:
1767
+ """Append ONE JSONL record under the dedicated health-history lock,
1768
+ rotating to an archive segment first if the live file would cross
1769
+ ``max_bytes`` (~1MB default, env-overridable). Never raises past a
1770
+ caller — a health-history write failure is reported by the caller as a
1771
+ ``blocked`` item, never allowed to fail the whole maintain run.
1772
+
1773
+ Also prunes archive segments past ``archive_retention_days`` (default
1774
+ 90, env-overridable — fix [6]) every call: cheap (a small glob under the
1775
+ lock already held) and keeps the archive dir from growing forever."""
1776
+ import os
1777
+
1778
+ from . import config as _config
1779
+
1780
+ limit = max_bytes if max_bytes is not None else int(
1781
+ os.environ.get(HEALTH_HISTORY_MAX_BYTES_ENV, DEFAULT_HEALTH_HISTORY_MAX_BYTES))
1782
+ retention = archive_retention_days if archive_retention_days is not None else int(
1783
+ os.environ.get(HEALTH_ARCHIVE_RETENTION_DAYS_ENV, DEFAULT_HEALTH_ARCHIVE_RETENTION_DAYS))
1784
+ path = _config.health_history_path(vault)
1785
+ archive_dir = _config.health_archive_dir(vault)
1786
+ lock_path = _config.health_history_lock_path(vault)
1787
+ path.parent.mkdir(parents=True, exist_ok=True)
1788
+
1789
+ _acquire_health_history_lock(lock_path)
1790
+ try:
1791
+ line = json.dumps(record, sort_keys=True)
1792
+ rotated = None
1793
+ if path.is_file() and path.stat().st_size + len(line) + 1 > limit:
1794
+ rotated = _rotate_health_history(path, archive_dir)
1795
+ with path.open("a", encoding="utf-8") as fh:
1796
+ fh.write(line + "\n")
1797
+ _append_sparse_metrics(_config.health_sparse_path(vault), record)
1798
+ _prune_health_archive(archive_dir, retention)
1799
+ return {"appended": True, "rotated": rotated}
1800
+ finally:
1801
+ _release_health_history_lock(lock_path)
1802
+
1803
+
1804
+ # The sidecar mirrors ONLY the GENUINELY sparse metric — ``golden_score``,
1805
+ # which is null on every record until the (quarterly-cadence) golden-eval
1806
+ # branch produces one. ``synthesis_cost_usd`` is deliberately NOT here: it is
1807
+ # the PERSISTED last metered cost (``latest_synthesis_cost``), non-null on
1808
+ # every hourly record once synthesis has run once — mirroring it would grow
1809
+ # the never-rotated sidecar unbounded (review finding [0]), and nothing
1810
+ # trend-compares it anyway (only ``golden_score`` has a sparse check).
1811
+ SPARSE_METRICS = ("golden_score",)
1812
+
1813
+
1814
+ def _append_sparse_metrics(sparse_path: Path, record: dict[str, Any]) -> None:
1815
+ """Mirror a record's non-null sparse metrics into the never-rotated
1816
+ sidecar (review finding [7]). No-op when the record carries none — so the
1817
+ sidecar only ever gains a line on the golden-eval (quarterly) cadence.
1818
+ Called under the same append lock as the main history write; BEST-EFFORT
1819
+ (review finding [2]): a sidecar write failure must NOT propagate and fail
1820
+ the main history append that already succeeded — the golden point also
1821
+ lives in the main record, and ``health_trend`` unions the two sources."""
1822
+ sparse = {k: record.get(k) for k in SPARSE_METRICS if record.get(k) is not None}
1823
+ if not sparse:
1824
+ return
1825
+ sparse["ts"] = record.get("ts")
1826
+ sparse["run_id"] = record.get("run_id")
1827
+ try:
1828
+ sparse_path.parent.mkdir(parents=True, exist_ok=True)
1829
+ with sparse_path.open("a", encoding="utf-8") as fh:
1830
+ fh.write(json.dumps(sparse, sort_keys=True) + "\n")
1831
+ except OSError:
1832
+ _log.warning("[health] sparse sidecar append failed (main record kept): %s",
1833
+ sparse_path)
1834
+
1835
+
1836
+ def read_sparse_history(vault: Path) -> list[dict[str, Any]]:
1837
+ """Full (never-windowed) sparse-metric history from the sidecar — tiny by
1838
+ construction (review finding [7]). De-duplicated by ``run_id`` and sorted
1839
+ by ``ts``; tolerant of a trailing partial line. Empty list when the
1840
+ sidecar does not exist yet."""
1841
+ from . import config as _config
1842
+
1843
+ path = _config.health_sparse_path(vault)
1844
+ try:
1845
+ text = path.read_text(encoding="utf-8")
1846
+ except OSError:
1847
+ return []
1848
+ records: dict[str, dict[str, Any]] = {}
1849
+ for rec in parse_recommendation_lines(text):
1850
+ key = rec.get("run_id") or f"__no_run_id__{rec.get('ts')}"
1851
+ records[key] = rec
1852
+ return sorted(records.values(), key=lambda r: str(r.get("ts") or ""))
1853
+
1854
+
1855
+ def read_health_history(
1856
+ vault: Path, *, window_days: int | None = None,
1857
+ ) -> list[dict[str, Any]]:
1858
+ """Merge the live ``health-history.jsonl`` with RECENT rotated archive
1859
+ segments — bounded to the last ``window_days`` by file mtime (default
1860
+ 14, env-overridable via ``$BRAIN_HEALTH_HISTORY_READ_WINDOW_DAYS``; fix
1861
+ [6] — re-reading and re-parsing EVERY archive segment on every hourly
1862
+ run does not scale as segments accumulate). De-duplicated by ``run_id``
1863
+ and sorted by ``ts``. Read-only — safe to call from ``health_trend`` on
1864
+ every run without touching state. The live file is always included
1865
+ regardless of age (it is small until it next rotates)."""
1866
+ import os
1867
+ import time
1868
+
1869
+ from . import config as _config
1870
+
1871
+ win = window_days if window_days is not None else int(
1872
+ os.environ.get(HEALTH_HISTORY_READ_WINDOW_DAYS_ENV, DEFAULT_HEALTH_HISTORY_READ_WINDOW_DAYS))
1873
+ cutoff = time.time() - win * 86400.0
1874
+
1875
+ records: dict[str, dict[str, Any]] = {}
1876
+ paths: list[Path] = []
1877
+ archive_dir = _config.health_archive_dir(vault)
1878
+ if archive_dir.is_dir():
1879
+ for p in sorted(archive_dir.glob("health-history-*.jsonl")):
1880
+ try:
1881
+ if p.stat().st_mtime >= cutoff:
1882
+ paths.append(p)
1883
+ except OSError:
1884
+ continue
1885
+ live = _config.health_history_path(vault)
1886
+ if live.is_file():
1887
+ paths.append(live)
1888
+ for p in paths:
1889
+ try:
1890
+ text = p.read_text(encoding="utf-8")
1891
+ except OSError:
1892
+ continue
1893
+ for rec in parse_recommendation_lines(text):
1894
+ key = rec.get("run_id") or f"__no_run_id__{p}__{rec.get('ts')}"
1895
+ records[key] = rec
1896
+ return sorted(records.values(), key=lambda r: str(r.get("ts") or ""))
1897
+
1898
+
1899
+ DEFAULT_LATENCY_REGRESSION_PCT = 0.50
1900
+ DEFAULT_QUARANTINE_REGRESSION_PCT = 0.25
1901
+ DEFAULT_GOLDEN_REGRESSION_PCT = 0.05
1902
+ HEALTH_TREND_MIN_DAYS = 7
1903
+ HEALTH_TREND_MIN_BASELINE_DAYS = 2
1904
+
1905
+ # Correction 5 — per-metric daily-bucket reducer. A single generic
1906
+ # "representative" (e.g. always "last") would average/suppress a real
1907
+ # single-hour spike; each metric family gets the reducer that keeps that
1908
+ # spike visible after bucketing hourly records into one-per-day.
1909
+ _DAILY_REDUCERS: dict[str, str] = {
1910
+ "notes": "last", "chunks": "last",
1911
+ "snapshot_gen": "last", "snapshot_age_s": "last",
1912
+ "quarantine": "last", "duplicate": "last", "decision_candidates": "last",
1913
+ "selftest_ms": "median",
1914
+ "blocked": "max", "action_required": "max",
1915
+ "golden_score": "last_non_null", "synthesis_cost_usd": "last_non_null",
1916
+ }
1917
+
1918
+
1919
+ def _record_date(rec: dict[str, Any]) -> datetime.date | None:
1920
+ ts = str(rec.get("ts") or "")
1921
+ try:
1922
+ return datetime.date.fromisoformat(ts[:10])
1923
+ except ValueError:
1924
+ return None
1925
+
1926
+
1927
+ def _bucket_daily(history: list[dict[str, Any]], metric: str) -> dict[datetime.date, Any]:
1928
+ """One representative value per calendar day for ``metric``, per its
1929
+ schema reducer (see ``_DAILY_REDUCERS``)."""
1930
+ reducer = _DAILY_REDUCERS.get(metric, "last")
1931
+ per_day: dict[datetime.date, list[Any]] = {}
1932
+ for rec in history:
1933
+ d = _record_date(rec)
1934
+ if d is None:
1935
+ continue
1936
+ per_day.setdefault(d, []).append(rec.get(metric))
1937
+
1938
+ out: dict[datetime.date, Any] = {}
1939
+ for d, values in per_day.items():
1940
+ if reducer == "median":
1941
+ nums = sorted(x for x in values if isinstance(x, (int, float)))
1942
+ if not nums:
1943
+ v = None
1944
+ else:
1945
+ mid = len(nums) // 2
1946
+ v = nums[mid] if len(nums) % 2 else (nums[mid - 1] + nums[mid]) / 2
1947
+ elif reducer == "max":
1948
+ nums = [x for x in values if isinstance(x, (int, float))]
1949
+ v = max(nums) if nums else None
1950
+ elif reducer == "last_non_null":
1951
+ v = next((x for x in reversed(values) if x is not None), None)
1952
+ else: # "last" — gauge counts (end-of-day snapshot)
1953
+ v = values[-1]
1954
+ out[d] = v
1955
+ return out
1956
+
1957
+
1958
+ def health_trend(
1959
+ history: list[dict[str, Any]], today: datetime.date, *,
1960
+ sparse_history: list[dict[str, Any]] | None = None,
1961
+ latency_regression_pct: float | None = None,
1962
+ quarantine_regression_pct: float | None = None,
1963
+ golden_regression_pct: float | None = None,
1964
+ ) -> list[dict[str, Any]]:
1965
+ """Week-over-week regression findings. Each finding is
1966
+ ``{metric, severity, current, baseline, delta_pct, summary}``.
1967
+
1968
+ - ``blocked`` fires immediately from the LATEST record alone (no
1969
+ baseline needed — any blocked>0 is already actionable).
1970
+ - ``selftest_ms``/``quarantine`` (high-frequency, appended every hourly
1971
+ run) are daily-bucketed and compared against a trailing-median
1972
+ baseline, but ONLY once >= ``HEALTH_TREND_MIN_DAYS`` calendar days of
1973
+ history exist AND the baseline has >= ``HEALTH_TREND_MIN_BASELINE_DAYS``
1974
+ non-null days — otherwise these two checks silently skip (never a
1975
+ false regression from a too-thin history; correction 1).
1976
+ - ``golden_score`` (sparse — null on nearly every hourly record) compares
1977
+ the latest non-null value against the PREVIOUS non-null value
1978
+ regardless of window/day-count (correction 1). A null on either side
1979
+ skips the check — a null is "absent", never a "-100%" drop.
1980
+ """
1981
+ import os
1982
+
1983
+ lat_pct = latency_regression_pct if latency_regression_pct is not None else float(
1984
+ os.environ.get("BRAIN_HEALTH_LATENCY_REGRESSION_PCT", DEFAULT_LATENCY_REGRESSION_PCT))
1985
+ quar_pct = quarantine_regression_pct if quarantine_regression_pct is not None else float(
1986
+ os.environ.get("BRAIN_HEALTH_QUARANTINE_REGRESSION_PCT", DEFAULT_QUARANTINE_REGRESSION_PCT))
1987
+ gold_pct = golden_regression_pct if golden_regression_pct is not None else float(
1988
+ os.environ.get("BRAIN_HEALTH_GOLDEN_REGRESSION_PCT", DEFAULT_GOLDEN_REGRESSION_PCT))
1989
+
1990
+ findings: list[dict[str, Any]] = []
1991
+ if not history:
1992
+ return findings
1993
+
1994
+ ordered = sorted(history, key=lambda r: str(r.get("ts") or ""))
1995
+
1996
+ latest = ordered[-1]
1997
+ blocked_now = latest.get("blocked")
1998
+ if isinstance(blocked_now, (int, float)) and blocked_now > 0:
1999
+ findings.append({
2000
+ "metric": "blocked", "severity": "regression",
2001
+ "current": blocked_now, "baseline": 0, "delta_pct": None,
2002
+ "summary": f"{int(blocked_now)} blocked finding(s) in the latest maintain run",
2003
+ })
2004
+
2005
+ dates_present = sorted({d for r in ordered if (d := _record_date(r)) is not None})
2006
+ span_ok = bool(dates_present) and (today - dates_present[0]).days >= HEALTH_TREND_MIN_DAYS
2007
+
2008
+ def _high_freq(metric: str, pct: float, label: str) -> None:
2009
+ buckets = _bucket_daily(ordered, metric)
2010
+ if not buckets:
2011
+ return
2012
+ days_sorted = sorted(buckets)
2013
+ current = buckets[days_sorted[-1]]
2014
+ baseline_vals = [buckets[d] for d in days_sorted[:-1] if buckets[d] is not None]
2015
+ if current is None or not span_ok or len(baseline_vals) < HEALTH_TREND_MIN_BASELINE_DAYS:
2016
+ return
2017
+ base_sorted = sorted(baseline_vals)
2018
+ mid = len(base_sorted) // 2
2019
+ baseline = (base_sorted[mid] if len(base_sorted) % 2
2020
+ else (base_sorted[mid - 1] + base_sorted[mid]) / 2)
2021
+ if not baseline: # 0 or None — a % delta off a zero baseline is meaningless
2022
+ return
2023
+ delta = (current - baseline) / baseline
2024
+ if delta > pct:
2025
+ findings.append({
2026
+ "metric": metric, "severity": "regression",
2027
+ "current": current, "baseline": baseline,
2028
+ "delta_pct": round(delta * 100, 1),
2029
+ "summary": f"{label}: {current} vs trailing baseline {baseline} "
2030
+ f"(+{round(delta * 100, 1)}%, threshold +{round(pct * 100)}%)",
2031
+ })
2032
+
2033
+ _high_freq("selftest_ms", lat_pct, "search self-test latency regressed")
2034
+ _high_freq("quarantine", quar_pct, "quarantine growth")
2035
+
2036
+ # Fix [7]+[2]: draw golden observations from the UNION of the never-windowed
2037
+ # sparse sidecar AND the windowed main history, deduped by run_id. The
2038
+ # sidecar carries points that have aged out of the 14-day window (golden
2039
+ # scores land quarterly); the main history covers a point a transient
2040
+ # sidecar-write failure (finding [2]) dropped that is still in-window. Only
2041
+ # a point BOTH absent from the sidecar AND aged out of the window is lost —
2042
+ # a double fault on a quarterly metric.
2043
+ golden_records = [r for r in _union_by_run_id(ordered, sparse_history or [])
2044
+ if r.get("golden_score") is not None]
2045
+ golden_points = [r["golden_score"]
2046
+ for r in sorted(golden_records, key=lambda r: str(r.get("ts") or ""))]
2047
+ if len(golden_points) >= 2:
2048
+ prev_score, cur_score = golden_points[-2], golden_points[-1]
2049
+ if (isinstance(prev_score, (int, float)) and isinstance(cur_score, (int, float))
2050
+ and prev_score):
2051
+ delta = (cur_score - prev_score) / prev_score
2052
+ if delta < -gold_pct:
2053
+ findings.append({
2054
+ "metric": "golden_score", "severity": "regression",
2055
+ "current": cur_score, "baseline": prev_score,
2056
+ "delta_pct": round(delta * 100, 1),
2057
+ "summary": f"golden retrieval score regressed: {cur_score} vs "
2058
+ f"previous {prev_score} ({round(delta * 100, 1)}%, "
2059
+ f"threshold -{round(gold_pct * 100)}%)",
2060
+ })
2061
+
2062
+ return findings
2063
+
2064
+
2065
+ # ---------------------------------------------------------------------------
2066
+ # OBS-02 — macOS degradation alarm. A trend regression, a tripped watchdog
2067
+ # finding, or blocked>0 fires a single osascript notification per finding per
2068
+ # day (dedup marker under ``.brain/notify-sent/``); ``BRAIN_NOTIFY=off`` kills
2069
+ # it; non-macOS degrades to a log-only no-op (the Windows installer path is a
2070
+ # separate, already-covered surface — WD-01's cloud push is the other leg).
2071
+ # ---------------------------------------------------------------------------
2072
+ NOTIFY_ENV = "BRAIN_NOTIFY"
2073
+ NOTIFY_MARKER_RETENTION_DAYS_ENV = "BRAIN_NOTIFY_MARKER_RETENTION_DAYS"
2074
+ DEFAULT_NOTIFY_MARKER_RETENTION_DAYS = 30
2075
+
2076
+
2077
+ def degradation_findings(
2078
+ outcomes: dict[str, Any], trend_findings: list[dict[str, Any]],
2079
+ ) -> list[tuple[str, str]]:
2080
+ """``(dedup_key, display_text)`` pairs, in priority order: blocked>0 (from
2081
+ ``outcomes`` — the ritual-level count, distinct from ``health_trend``'s
2082
+ own per-record ``blocked`` metric check so a blocked item is never
2083
+ reported twice), the synthesis watchdog (if tripped this run), then every
2084
+ trend regression. Pure — no I/O, no dedup bookkeeping (that's
2085
+ ``should_notify``).
2086
+
2087
+ The KEY is a STABLE per-finding-identity string (``"blocked"``,
2088
+ ``"synthesis-watchdog"``, ``"trend:<metric>"``); the TEXT is the
2089
+ human-readable notification body, which DELIBERATELY carries the
2090
+ fluctuating measured values (``540ms vs baseline 480ms``). Review finding
2091
+ [1]: the dedup marker must hash the KEY, never the value-bearing text —
2092
+ otherwise the same ongoing regression, whose daily-median ``current``
2093
+ shifts each hourly run, hashes a different marker every hour and re-fires
2094
+ a fresh notification hourly, defeating "one notification per finding per
2095
+ day."
2096
+
2097
+ Fix [3]: ``health_trend`` ALSO appends its own ``metric: "blocked"`` entry
2098
+ to ``trend_findings`` whenever the latest record shows blocked>0 — that
2099
+ entry is skipped here so the SAME blocked condition never produces two
2100
+ findings in one run."""
2101
+ pairs: list[tuple[str, str]] = []
2102
+ blocked = (outcomes.get("counts") or {}).get("blocked", 0)
2103
+ if blocked:
2104
+ pairs.append(("blocked", f"{blocked} blocked finding(s) this maintain run"))
2105
+ for item in outcomes.get("action_required") or []:
2106
+ finding = str(item.get("finding", ""))
2107
+ if finding.startswith("brain-synthesis watchdog:"):
2108
+ pairs.append(("synthesis-watchdog", finding))
2109
+ for f in trend_findings:
2110
+ metric = str(f.get("metric") or "")
2111
+ if metric == "blocked":
2112
+ continue # already reported via outcomes.counts above
2113
+ text = str(f.get("summary") or f"{metric} regression")
2114
+ pairs.append((f"trend:{metric}", text))
2115
+ return pairs
2116
+
2117
+
2118
+ def _notify_marker_dir(vault: Path) -> Path:
2119
+ from . import config as _config
2120
+
2121
+ return _config.brain_runtime_dir(vault) / "notify-sent"
2122
+
2123
+
2124
+ def _notify_marker_path(vault: Path, key: str, today: datetime.date) -> Path:
2125
+ """Marker path for a STABLE dedup ``key`` (not the value-bearing display
2126
+ text — review finding [1]). One marker per key per day."""
2127
+ import hashlib
2128
+
2129
+ digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
2130
+ return _notify_marker_dir(vault) / f"{today.isoformat()}-{digest}.marker"
2131
+
2132
+
2133
+ def _prune_notify_markers(vault: Path, retention_days: int | None = None) -> None:
2134
+ """Delete per-day dedup markers older than ``retention_days`` (default
2135
+ 30, env-overridable — fix [7]): one marker file accumulates per unique
2136
+ finding per day forever otherwise. Best-effort, never raises."""
2137
+ import os
2138
+
2139
+ retention = retention_days if retention_days is not None else int(
2140
+ os.environ.get(NOTIFY_MARKER_RETENTION_DAYS_ENV, DEFAULT_NOTIFY_MARKER_RETENTION_DAYS))
2141
+ _prune_old_files(_notify_marker_dir(vault), "*.marker", retention)
2142
+
2143
+
2144
+ def should_notify(vault: Path, key: str, today: datetime.date) -> bool:
2145
+ """True iff this ``key`` has NOT yet been surfaced today. PURE READ — no
2146
+ marker write (review finding [1]: the check used to write the marker
2147
+ eagerly). Pair with ``mark_notified``."""
2148
+ return not _notify_marker_path(vault, key, today).exists()
2149
+
2150
+
2151
+ def mark_notified(vault: Path, key: str, today: datetime.date) -> str:
2152
+ """ATOMICALLY claim the per-day dedup marker for ``key`` via an
2153
+ ``O_CREAT | O_EXCL`` create. Returns:
2154
+
2155
+ - ``"claimed"`` — this call created the marker (we own the notification);
2156
+ - ``"exists"`` — the marker already existed, so an earlier run today (or a
2157
+ concurrently-overlapping maintain) already surfaced this key;
2158
+ - ``"unwritable"`` — the marker dir could not be written.
2159
+
2160
+ The exclusive create closes the check-then-write TOCTOU (review finding
2161
+ [1]): the coarse 2h maintain-lock auto-break can let two maintains overlap
2162
+ (the health-history append is locked for exactly this reason), and a
2163
+ plain ``should_notify`` read + later write let both fire the same banner.
2164
+ Now only ONE overlapping run wins the create; the other sees ``"exists"``
2165
+ and skips.
2166
+
2167
+ ``"unwritable"`` is best-effort (review finding [1] sibling): on a
2168
+ read-only/full ``.brain`` dedup state cannot be persisted by ANY design —
2169
+ the caller still fires (the finding must not be lost) and it may re-surface
2170
+ next run until the dir recovers. The durable WARNING ``[degradation]`` log
2171
+ line fires regardless, so the finding is never lost."""
2172
+ import os
2173
+
2174
+ marker = _notify_marker_path(vault, key, today)
2175
+ try:
2176
+ marker.parent.mkdir(parents=True, exist_ok=True)
2177
+ fd = os.open(str(marker), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
2178
+ except FileExistsError:
2179
+ return "exists"
2180
+ except OSError:
2181
+ _log.warning("[degradation] dedup marker unwritable (%s); "
2182
+ "finding may re-surface next run", marker.parent)
2183
+ return "unwritable"
2184
+ try:
2185
+ os.write(fd, key.encode("utf-8"))
2186
+ finally:
2187
+ os.close(fd)
2188
+ return "claimed"
2189
+
2190
+
2191
+ def fire_notification(text: str, *, title: str = "Brainiac health") -> str:
2192
+ """Best-effort ``osascript`` notification on macOS. Returns
2193
+ ``"skipped (non-macOS)"`` off Darwin and ``"failed: …"`` on a send error,
2194
+ never raises — a GUI ping is pure convenience on top of the durable log
2195
+ line the caller always emits. Slowing or failing the maintain run over a
2196
+ notification is never acceptable."""
2197
+ import platform
2198
+ import subprocess
2199
+
2200
+ if platform.system() != "Darwin":
2201
+ return "skipped (non-macOS)"
2202
+ try:
2203
+ result = subprocess.run(
2204
+ ["osascript", "-e",
2205
+ f'display notification {json.dumps(text)} with title {json.dumps(title)}'],
2206
+ check=False, capture_output=True, timeout=5,
2207
+ )
2208
+ if result.returncode != 0:
2209
+ return "failed: osascript rc={}".format(result.returncode)
2210
+ return "notified"
2211
+ except Exception as exc: # noqa: BLE001 — a notification failure is cosmetic
2212
+ return f"failed: {type(exc).__name__}"
2213
+
2214
+
2215
+ def pending_notifications(
2216
+ vault: Path, outcomes: dict[str, Any], trend_findings: list[dict[str, Any]],
2217
+ today: datetime.date,
2218
+ ) -> list[tuple[str, str]]:
2219
+ """The ``(key, text)`` findings still pending notification for TODAY —
2220
+ empty when ``BRAIN_NOTIFY=off`` or when every candidate was already
2221
+ surfaced today (dedup by stable KEY). Pure read (does not mark) — pair
2222
+ with ``fire_and_mark_notifications``. Also opportunistically prunes old
2223
+ markers (fix [7]) since this is called once per maintain run."""
2224
+ import os
2225
+
2226
+ _prune_notify_markers(vault)
2227
+ if os.environ.get(NOTIFY_ENV, "").strip().lower() == "off":
2228
+ return []
2229
+ return [(k, t) for (k, t) in degradation_findings(outcomes, trend_findings)
2230
+ if should_notify(vault, k, today)]
2231
+
2232
+
2233
+ def fire_and_mark_notifications(
2234
+ vault: Path, findings: list[tuple[str, str]], today: datetime.date,
2235
+ ) -> list[str]:
2236
+ """Surface each pending ``(key, text)`` finding, then persist its per-day
2237
+ dedup marker. Returns the display texts surfaced this call.
2238
+
2239
+ Two review fixes converge here:
2240
+
2241
+ - **Fix [4] / durable half of [2]:** a WARNING log line is emitted for
2242
+ every finding FIRST, independent of the GUI channel and platform, so a
2243
+ failed ``osascript`` (headless/non-Aqua launchd) or a non-macOS host
2244
+ never means the degradation went unrecorded. The GUI ``osascript`` ping
2245
+ is best-effort convenience on top; its result is intentionally ignored.
2246
+ - **Fix [2]:** the marker is written REGARDLESS of the GUI send result, so
2247
+ the finding is surfaced at most ONCE per key per day. ``osascript`` on a
2248
+ headless/non-Aqua launchd session is PERMANENTLY unavailable, not
2249
+ transient — the prior "withhold the marker on failure and retry next
2250
+ run" behavior just respawned a subprocess that can never succeed, every
2251
+ hour, forever. The durable WARNING log above is the channel that
2252
+ guarantees the finding is never lost; the once-a-day GUI ping is not."""
2253
+ sent: list[str] = []
2254
+ for key, text in findings:
2255
+ # Claim the day's marker FIRST (atomic O_EXCL). "exists" means a
2256
+ # concurrently-overlapping maintain already surfaced this key today —
2257
+ # skip, so one condition never double-fires (finding [1]). "claimed"
2258
+ # and "unwritable" both fire: "unwritable" is the degraded read-only
2259
+ # `.brain` case where the finding must still reach the durable log
2260
+ # (finding [0] tradeoff — it may re-surface next run).
2261
+ if mark_notified(vault, key, today) == "exists":
2262
+ continue
2263
+ _log.warning("[degradation] %s", text)
2264
+ fire_notification(text) # best-effort GUI ping; result ignored by design
2265
+ sent.append(text)
2266
+ return sent