brainiac-cli 0.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brain/__init__.py +39 -0
- brain/__main__.py +14 -0
- brain/_assets/AGENTS.md +564 -0
- brain/_assets/overlay/template/brand/brand-guide.md +24 -0
- brain/_assets/overlay/template/keywords/glossary.md +15 -0
- brain/_assets/overlay/template/people/roster.md +14 -0
- brain/_assets/overlay/template/voice/voice-profile.md +34 -0
- brain/_assets/routines/manifest.json +257 -0
- brain/_assets/scripts/brain-brief-mac.plist +59 -0
- brain/_assets/scripts/brain-brief.sh +74 -0
- brain/_assets/scripts/brain-synthesis-mac.plist +48 -0
- brain/_assets/scripts/brain-synthesis.sh +214 -0
- brain/_assets/scripts/install-brief-mac.sh +152 -0
- brain/_assets/scripts/install-brief-windows.ps1 +97 -0
- brain/_assets/scripts/register_tasks.py +386 -0
- brain/_assets/templates/company.md +21 -0
- brain/_assets/templates/concept.md +27 -0
- brain/_assets/templates/daily.md +20 -0
- brain/_assets/templates/decision.md +42 -0
- brain/_assets/templates/meeting.md +33 -0
- brain/_assets/templates/person.md +20 -0
- brain/_assets/templates/project.md +23 -0
- brain/_assets/templates/state-moc.md +40 -0
- brain/_version.py +12 -0
- brain/anchor.py +121 -0
- brain/audit.py +422 -0
- brain/backup.py +210 -0
- brain/brief.py +417 -0
- brain/capture.py +117 -0
- brain/chunk.py +249 -0
- brain/classification.py +134 -0
- brain/cli.py +1906 -0
- brain/config.py +368 -0
- brain/connect.py +362 -0
- brain/context.py +108 -0
- brain/core.py +3018 -0
- brain/doctor.py +1161 -0
- brain/egress.py +148 -0
- brain/embed.py +857 -0
- brain/encryption.py +217 -0
- brain/frontmatter.py +102 -0
- brain/golden_probe.py +678 -0
- brain/graph.py +369 -0
- brain/graphify.py +352 -0
- brain/index.py +1576 -0
- brain/ingest/__init__.py +19 -0
- brain/ingest/handlers/__init__.py +43 -0
- brain/ingest/handlers/base.py +95 -0
- brain/ingest/handlers/docx.py +78 -0
- brain/ingest/handlers/email.py +228 -0
- brain/ingest/handlers/html.py +142 -0
- brain/ingest/handlers/image.py +91 -0
- brain/ingest/handlers/pdf.py +99 -0
- brain/ingest/handlers/pptx.py +69 -0
- brain/ingest/handlers/tables.py +41 -0
- brain/ingest/handlers/text.py +43 -0
- brain/ingest/handlers/xlsx.py +100 -0
- brain/ingest/handlers/zip.py +163 -0
- brain/ingest/pipeline.py +839 -0
- brain/ingest/transcript.py +158 -0
- brain/init.py +870 -0
- brain/maintenance.py +2266 -0
- brain/mcp_adapter.py +217 -0
- brain/multihop.py +232 -0
- brain/notes.py +195 -0
- brain/overlay.py +183 -0
- brain/projection.py +79 -0
- brain/rerank.py +425 -0
- brain/snapshot.py +231 -0
- brain/update.py +743 -0
- brain/vectors.py +225 -0
- brainiac_cli-0.16.0.dist-info/METADATA +306 -0
- brainiac_cli-0.16.0.dist-info/RECORD +77 -0
- brainiac_cli-0.16.0.dist-info/WHEEL +5 -0
- brainiac_cli-0.16.0.dist-info/entry_points.txt +4 -0
- brainiac_cli-0.16.0.dist-info/licenses/LICENSE +202 -0
- brainiac_cli-0.16.0.dist-info/top_level.txt +1 -0
brain/core.py
ADDED
|
@@ -0,0 +1,3018 @@
|
|
|
1
|
+
"""BrainCore — the engine. Importable, but NOT the integration surface.
|
|
2
|
+
|
|
3
|
+
CRITICAL CONTRACT: the read verbs here (``search``/``get``/``recent``) return
|
|
4
|
+
**UNFILTERED** results. The deny-by-default classification filter lives in the
|
|
5
|
+
CLI (brain.cli), applied as the final stage before stdout. Importing BrainCore
|
|
6
|
+
in-process therefore BYPASSES the egress filter — by design. This is exactly why
|
|
7
|
+
the filter is an egress-decision mechanism, not containment: real containment is
|
|
8
|
+
workspace projection (brain.projection) + the host/VM trust split.
|
|
9
|
+
|
|
10
|
+
The write verb (``write_note``) is a HOST-BROKER privilege: it appends to the
|
|
11
|
+
Ed25519 audit chain (CORE-03) and fails closed if no signing key resolves.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from . import classification
|
|
19
|
+
from . import config
|
|
20
|
+
from . import frontmatter
|
|
21
|
+
from .audit import AuditChain, KeyUnavailable
|
|
22
|
+
from .index import BrainIndex, Hit
|
|
23
|
+
from .notes import load_note, safe_slug, sha256_text
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _contained_in(target: Path, base: Path) -> bool:
|
|
27
|
+
"""True iff RESOLVED ``target`` is strictly inside RESOLVED ``base``.
|
|
28
|
+
|
|
29
|
+
Uses Path.relative_to on resolved paths — never string-prefix checks
|
|
30
|
+
(sibling-directory bypass, e.g. ``vault-x`` matching ``vault``). Resolving
|
|
31
|
+
also follows symlinks, so a symlink inside the vault pointing outside it
|
|
32
|
+
fails containment. Path.resolve() is non-strict, so a not-yet-existing
|
|
33
|
+
target (draft-capture writes NEW files) resolves fine.
|
|
34
|
+
"""
|
|
35
|
+
target = target.resolve()
|
|
36
|
+
base = base.resolve()
|
|
37
|
+
if target == base:
|
|
38
|
+
return False
|
|
39
|
+
try:
|
|
40
|
+
target.relative_to(base)
|
|
41
|
+
except ValueError:
|
|
42
|
+
return False
|
|
43
|
+
return True
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _stamp_draft_frontmatter(content: str, note_id: str, is_source: bool) -> str:
|
|
47
|
+
"""Return ``content`` with draft markers ensured (idempotent, non-clobbering).
|
|
48
|
+
|
|
49
|
+
Guarantees the staged file carries frontmatter with an ``id``, ``status:
|
|
50
|
+
draft`` and ``provenance.trust: untrusted`` so (a) the host drain's
|
|
51
|
+
``load_note`` can read it and (b) any reader can see it is an uncommitted,
|
|
52
|
+
untrusted draft. Existing keys are never overwritten — capture is additive.
|
|
53
|
+
"""
|
|
54
|
+
meta, body = frontmatter.parse_text(content)
|
|
55
|
+
if not content.startswith("---") or not meta:
|
|
56
|
+
# No (or unparseable) frontmatter — synthesise a minimal block.
|
|
57
|
+
dtype = "source" if is_source else "note"
|
|
58
|
+
return (
|
|
59
|
+
f"---\nid: {note_id}\ntype: {dtype}\nstatus: draft\n"
|
|
60
|
+
f"provenance.trust: untrusted\n---\n\n{content.lstrip()}\n"
|
|
61
|
+
)
|
|
62
|
+
block, after = content.split("---", 2)[1], content.split("---", 2)[2]
|
|
63
|
+
additions = []
|
|
64
|
+
if "id" not in meta:
|
|
65
|
+
additions.append(f"id: {note_id}")
|
|
66
|
+
if "status" not in meta:
|
|
67
|
+
additions.append("status: draft")
|
|
68
|
+
if "provenance.trust" not in meta:
|
|
69
|
+
additions.append("provenance.trust: untrusted")
|
|
70
|
+
if not additions:
|
|
71
|
+
return content
|
|
72
|
+
new_block = block.rstrip("\n") + "\n" + "\n".join(additions) + "\n"
|
|
73
|
+
return f"---{new_block}---{after}"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class RoleError(RuntimeError):
|
|
77
|
+
"""A host-broker operation was attempted from the read+draft-only VM leg.
|
|
78
|
+
|
|
79
|
+
The VM leg (``role=vm``) may never write notes, mutate/WAL the index, publish
|
|
80
|
+
a snapshot, or resolve a signing key. These ops fail with RoleError BEFORE
|
|
81
|
+
any signing-key resolution or index write is attempted (S06 hard guarantee).
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class BrainCore:
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
vault: str | Path | None = None,
|
|
89
|
+
index: BrainIndex | None = None,
|
|
90
|
+
audit_log: str | Path | None = None,
|
|
91
|
+
*,
|
|
92
|
+
role: str | None = None,
|
|
93
|
+
) -> None:
|
|
94
|
+
self.role = config.role(role)
|
|
95
|
+
self.vault = config.vault_root(vault)
|
|
96
|
+
if index is not None:
|
|
97
|
+
self.index = index
|
|
98
|
+
elif self.role == config.ROLE_VM:
|
|
99
|
+
# VM leg reads ONLY the published read-only snapshot — never the
|
|
100
|
+
# authoritative writable index, never WAL.
|
|
101
|
+
self.index = BrainIndex(db_path=config.snapshot_db_path(self.vault),
|
|
102
|
+
read_only=True)
|
|
103
|
+
else:
|
|
104
|
+
self.index = BrainIndex(db_path=config.index_path(self.vault))
|
|
105
|
+
if self.role == config.ROLE_VM:
|
|
106
|
+
# No signing surface AT ALL on the VM: the audit chain (and thus
|
|
107
|
+
# resolve_signing_key) is simply not constructed here.
|
|
108
|
+
self.audit = None
|
|
109
|
+
else:
|
|
110
|
+
log = Path(audit_log) if audit_log else config.default_audit_log(self.vault)
|
|
111
|
+
self.audit = AuditChain(log)
|
|
112
|
+
|
|
113
|
+
def _require_host(self, op: str) -> None:
|
|
114
|
+
if self.role != config.ROLE_HOST:
|
|
115
|
+
raise RoleError(
|
|
116
|
+
f"role={self.role!r} may not {op}; this is a host-broker privilege "
|
|
117
|
+
"(the VM leg is read + draft only). Run on the host."
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# -- read verbs (UNFILTERED — see module docstring) -------------------
|
|
121
|
+
def search(self, query: str, k: int = 10) -> list[Hit]:
|
|
122
|
+
return self.index.search(query, k)
|
|
123
|
+
|
|
124
|
+
def source_freshness(self, newest_hit_date: str, max_tier: str) -> dict[str, Any]:
|
|
125
|
+
"""RET-09 freshness signal: count + newest date of notes whose
|
|
126
|
+
valid-time date is strictly newer than ``newest_hit_date``, at the
|
|
127
|
+
caller's egress cap. See ``BrainIndex.freshness``."""
|
|
128
|
+
return self.index.freshness(newest_hit_date, max_tier)
|
|
129
|
+
|
|
130
|
+
def hybrid_search(
|
|
131
|
+
self, query: str, k: int = 10, *, rerank: bool = False, rerank_top: int = 15,
|
|
132
|
+
rrf_k: int = 60,
|
|
133
|
+
) -> list[Hit]:
|
|
134
|
+
"""Fused RRF(k) BM25 + dense retrieval (RET-01), optional skippable
|
|
135
|
+
reranker (RET-02). UNFILTERED — the CLI applies the egress gate."""
|
|
136
|
+
return self.index.hybrid_search(
|
|
137
|
+
query, k=k, rerank=rerank, rerank_top=rerank_top, rrf_k=rrf_k,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
def hybrid_search_graph(
|
|
141
|
+
self, query: str, k: int = 10, *, rerank: bool = False, rerank_top: int = 15,
|
|
142
|
+
rrf_k: int = 60, depth: int = 2, graph_weight: float = 0.5,
|
|
143
|
+
seed_flat_top: int = 3, flat_pool: int = 30, return_trace: bool = False,
|
|
144
|
+
):
|
|
145
|
+
"""Gated graph-augmented multi-hop retrieval (RET-06).
|
|
146
|
+
|
|
147
|
+
Single-hop queries pass through to ``hybrid_search`` UNCHANGED (the gate
|
|
148
|
+
does not fire); multi-hop-shaped queries (>= 2 named non-hub entities)
|
|
149
|
+
get a wikilink-graph expansion fused into the flat ranking. DISCOVERY-
|
|
150
|
+
ONLY (RET-03): the graph never overrides an authoritative flat hit. See
|
|
151
|
+
``brain.multihop``. UNFILTERED — the CLI applies the egress gate."""
|
|
152
|
+
return self.index.hybrid_search_graph(
|
|
153
|
+
query, k=k, rerank=rerank, rerank_top=rerank_top, rrf_k=rrf_k,
|
|
154
|
+
depth=depth, graph_weight=graph_weight, seed_flat_top=seed_flat_top,
|
|
155
|
+
flat_pool=flat_pool, return_trace=return_trace,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def search_multi(
|
|
159
|
+
self, queries: "list[str]", k: int = 10, *, rerank: bool = False,
|
|
160
|
+
rerank_top: int = 15, rrf_k: int = 60, per_query_k: int | None = None,
|
|
161
|
+
rerank_fused: bool = False, fused_pool: int = 20,
|
|
162
|
+
) -> list[Hit]:
|
|
163
|
+
"""Multi-query fan-out (RET-05) — the AGENTIC retrieval primitive.
|
|
164
|
+
|
|
165
|
+
Run ``hybrid_search`` for EACH query variant and Reciprocal-Rank-Fuse the
|
|
166
|
+
result lists into one ranking. This is the recovery for cross-boundary
|
|
167
|
+
misses (query-language ≠ document-language; query-vocabulary ≠
|
|
168
|
+
note-vocabulary): an agent issues the original query PLUS reformulations
|
|
169
|
+
(e.g. a cross-lingual rephrase, a synonym expansion, a HyDE answer) and
|
|
170
|
+
this fuses them. A PT query and its EN rephrase reach the same EN-content
|
|
171
|
+
note through different legs; RRF promotes the note that appears across
|
|
172
|
+
lists. Empirically this TIES Smart Connections on monolingual PT
|
|
173
|
+
(0.736 vs 0.750) where any single query trails it by ~0.10 — and brain
|
|
174
|
+
already beats SC on EN / cross-lingual / temporal / multi-hop, so fan-out
|
|
175
|
+
closes the one stratum that single-query retrieval lost. See
|
|
176
|
+
docs/operations/s10-agentic-retrieval-analysis.md.
|
|
177
|
+
|
|
178
|
+
The caller supplies the variants (the agent/LLM generates them — brain
|
|
179
|
+
stays model-agnostic and offline). A single-element list degrades exactly
|
|
180
|
+
to ``hybrid_search``. UNFILTERED — the CLI applies the egress gate.
|
|
181
|
+
"""
|
|
182
|
+
from dataclasses import replace
|
|
183
|
+
|
|
184
|
+
variants = [q for q in (queries or []) if q and q.strip()]
|
|
185
|
+
if not variants:
|
|
186
|
+
return []
|
|
187
|
+
if len(variants) == 1:
|
|
188
|
+
return self.hybrid_search(
|
|
189
|
+
variants[0], k=k, rerank=rerank, rerank_top=rerank_top, rrf_k=rrf_k
|
|
190
|
+
)
|
|
191
|
+
# Per-query depth is deliberately SHALLOW (≈ k, not a wide over-fetch).
|
|
192
|
+
# RRF over wide per-query lists lets a noise doc present in BOTH lists at
|
|
193
|
+
# low rank (e.g. PT@50 + EN@60) out-accumulate a gold present in only ONE
|
|
194
|
+
# list at high rank (e.g. EN@5) — measured: per_query_k 20→80 drops
|
|
195
|
+
# monolingual_pt fan-out recall 0.736→0.625. Keep each variant's
|
|
196
|
+
# contribution to its genuine top hits. Tunable via per_query_k.
|
|
197
|
+
pk = per_query_k or max(k, 20)
|
|
198
|
+
fused: dict[str, list] = {} # id -> [fused_score, Hit]
|
|
199
|
+
for q in variants:
|
|
200
|
+
hits = self.hybrid_search(
|
|
201
|
+
q, k=pk, rerank=rerank, rerank_top=rerank_top, rrf_k=rrf_k
|
|
202
|
+
)
|
|
203
|
+
for rank, h in enumerate(hits, start=1):
|
|
204
|
+
contrib = 1.0 / (rrf_k + rank)
|
|
205
|
+
cur = fused.get(h.id)
|
|
206
|
+
if cur is None:
|
|
207
|
+
fused[h.id] = [contrib, h]
|
|
208
|
+
else:
|
|
209
|
+
cur[0] += contrib
|
|
210
|
+
ranked = sorted(fused.values(), key=lambda t: -t[0])
|
|
211
|
+
# Stamp the fused score so any downstream re-sort preserves fan-out order.
|
|
212
|
+
fused_hits = [replace(h, score=s) for s, h in ranked]
|
|
213
|
+
|
|
214
|
+
# POST-FUSION RERANK (RET-05b) — fan-out maximises deep RECALL (golds the
|
|
215
|
+
# single query missed surface at ranks 11-20), but answer generation reads
|
|
216
|
+
# only the TOP few, where wide recall + RRF + a zone prior inject noise.
|
|
217
|
+
# The cross-encoder reorders the wide fused POOL against the ORIGINAL query
|
|
218
|
+
# (variants[0]) so brain's recall@20 advantage is converted into top-k
|
|
219
|
+
# PRECISION. Without this, fan-out wins recall@20 but loses precision@5 to
|
|
220
|
+
# SC's whole-note embeddings (measured: answer-grounded eval, S10). The
|
|
221
|
+
# rerank is SKIPPABLE (offline/no model -> identity, never an error).
|
|
222
|
+
if rerank_fused and fused_hits:
|
|
223
|
+
fused_hits = self.index._apply_rerank(
|
|
224
|
+
variants[0], fused_hits, None, fused_pool
|
|
225
|
+
)
|
|
226
|
+
# _apply_rerank REORDERS but keeps each hit's (fused) score, so a
|
|
227
|
+
# downstream re-sort by score would undo the rerank. Re-stamp a strictly
|
|
228
|
+
# descending score that encodes the post-rerank RANK, so the cross-encoder
|
|
229
|
+
# order survives any {path: score} round-trip (e.g. the eval harness).
|
|
230
|
+
n = len(fused_hits)
|
|
231
|
+
fused_hits = [replace(h, score=float(n - i)) for i, h in enumerate(fused_hits)]
|
|
232
|
+
return fused_hits[:k]
|
|
233
|
+
|
|
234
|
+
def grep(self, pattern: str, *, k: int = 20, regex: bool = False) -> list[dict[str, Any]]:
|
|
235
|
+
"""Lexical-first scan over note bodies — no embedding (RET-04)."""
|
|
236
|
+
return self.index.grep(pattern, k=k, regex=regex)
|
|
237
|
+
|
|
238
|
+
def bases_query(
|
|
239
|
+
self, filters: dict[str, str] | None = None, *, k: int = 50,
|
|
240
|
+
latest_only: bool = False, as_of: str | None = None,
|
|
241
|
+
) -> list[dict[str, Any]]:
|
|
242
|
+
"""Structured frontmatter view over indexed columns — no embedding (RET-04).
|
|
243
|
+
TMP-02: ``latest_only``/``as_of`` are temporal views (Latest Only / As Of)."""
|
|
244
|
+
return self.index.bases_query(filters, k=k, latest_only=latest_only, as_of=as_of)
|
|
245
|
+
|
|
246
|
+
def dossier(self, query: str, k: int = 12) -> dict[str, Any]:
|
|
247
|
+
"""RET-10: the ONE-CALL retrieval sweep — what a careful agent
|
|
248
|
+
orchestrates by hand (decision layer + corroborating sources +
|
|
249
|
+
contradiction check + version noise handling), composed engine-side
|
|
250
|
+
so even a minimal-path harness gets the full sweep deterministically.
|
|
251
|
+
|
|
252
|
+
Motivation (2026-07-11 benchmark series close): on the same
|
|
253
|
+
substrate, the remaining quality gap between harnesses was
|
|
254
|
+
ORCHESTRATION BREADTH — one agent cross-checked newer sources
|
|
255
|
+
against the decision layer and caught superseded thinking; the
|
|
256
|
+
other walked the minimal path and could not see contradictions off
|
|
257
|
+
it. This verb makes the sweep the minimal path.
|
|
258
|
+
|
|
259
|
+
Returns (UNFILTERED — callers apply the egress gate):
|
|
260
|
+
- ``decisions``: hits with ``type: decision`` (the authority
|
|
261
|
+
layer), each carrying a ``tensions`` list — NEWER-dated,
|
|
262
|
+
non-decision hits from the same sweep (a proposal/deck that
|
|
263
|
+
post-dates the recorded decision: report the tension, never
|
|
264
|
+
promote the proposal).
|
|
265
|
+
- ``sources``: the remaining live hits (material under
|
|
266
|
+
consideration).
|
|
267
|
+
- ``retired_excluded``: hits dropped because a supersession chain
|
|
268
|
+
retired them (``is_latest_version: false``) — version noise the
|
|
269
|
+
sweep already handled.
|
|
270
|
+
"""
|
|
271
|
+
# A DEEP candidate pool: decision notes are scarce and often rank
|
|
272
|
+
# below big source documents on broad queries — the decision layer
|
|
273
|
+
# must never come back empty just because the top-k was crowded
|
|
274
|
+
# (measured on the live corpus: decisions at rank ~30 on a broad
|
|
275
|
+
# decision-state query). Scanning deeper is one indexed query.
|
|
276
|
+
pool = [h.to_dict() for h in self.hybrid_search(query, k=max(k * 2, 60))]
|
|
277
|
+
live = [h for h in pool if h.get("is_latest_version") != "false"]
|
|
278
|
+
retired_excluded = len(pool) - len(live)
|
|
279
|
+
decisions = [h for h in live if h.get("type") == "decision"]
|
|
280
|
+
# RET-10b: MERGE a targeted BM25 probe over the decision layer — the
|
|
281
|
+
# decision layer must never come back empty just because a phrasing
|
|
282
|
+
# shift pushed decision notes below the semantic pool (measured live:
|
|
283
|
+
# a rewording emptied the layer while the notes plainly existed).
|
|
284
|
+
seen_ids = {d["id"] for d in decisions}
|
|
285
|
+
for h in self.index.decision_layer_hits(query, k=max(5, k // 2)):
|
|
286
|
+
hd = h.to_dict()
|
|
287
|
+
if hd["id"] not in seen_ids:
|
|
288
|
+
decisions.append(hd)
|
|
289
|
+
seen_ids.add(hd["id"])
|
|
290
|
+
decisions = decisions[:max(5, k // 2)]
|
|
291
|
+
sources = [h for h in live if h.get("type") != "decision"][:k]
|
|
292
|
+
for d in decisions:
|
|
293
|
+
d_date = d.get("date") or ""
|
|
294
|
+
d["tensions"] = [
|
|
295
|
+
{"id": s["id"], "date": s.get("date", ""), "type": s.get("type", "")}
|
|
296
|
+
for s in sources
|
|
297
|
+
if d_date and s.get("date") and s["date"] > d_date
|
|
298
|
+
]
|
|
299
|
+
return {
|
|
300
|
+
"query": query,
|
|
301
|
+
"decisions": decisions,
|
|
302
|
+
"sources": sources,
|
|
303
|
+
"retired_excluded": retired_excluded,
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
def graph_expand(
|
|
307
|
+
self, seeds: list[str], *, depth: int = 2, k: int = 10, use_ppr: bool = True,
|
|
308
|
+
use_inferred: bool = False,
|
|
309
|
+
) -> dict[str, Any]:
|
|
310
|
+
"""On-demand wikilink-BFS + PPR — DISCOVERY-ONLY (RET-03).
|
|
311
|
+
|
|
312
|
+
``use_inferred`` (GRF-01, ADR-0003 Ruling 6, "Optional"): fold the
|
|
313
|
+
published graphify build's INFERRED edges in as extra traversal
|
|
314
|
+
input. HOST-ONLY read of the graphify artifact — on the VM leg this
|
|
315
|
+
is silently ignored (degrades to the plain wikilink graph) rather
|
|
316
|
+
than reaching for a host-only runtime artifact through the shared
|
|
317
|
+
mount, mirroring the session-memory host-only-by-contract posture
|
|
318
|
+
(ADR-0003 Ruling 4)."""
|
|
319
|
+
extra_edges = None
|
|
320
|
+
if use_inferred and self.role == config.ROLE_HOST:
|
|
321
|
+
from . import graphify as gmod
|
|
322
|
+
|
|
323
|
+
extra_edges = gmod.read_published_inferred_edges(
|
|
324
|
+
config.graph_json_path(self.vault))
|
|
325
|
+
return self.index.graph_expand(
|
|
326
|
+
seeds, depth=depth, k=k, use_ppr=use_ppr, extra_edges=extra_edges)
|
|
327
|
+
|
|
328
|
+
def get(self, note_id: str) -> dict[str, Any] | None:
|
|
329
|
+
return self.index.get(note_id)
|
|
330
|
+
|
|
331
|
+
def recent(self, limit: int = 10) -> list[dict[str, Any]]:
|
|
332
|
+
return self.index.recent(limit)
|
|
333
|
+
|
|
334
|
+
# -- VM-side capture (read + DRAFT only; NO sign, NO index, NO WAL) ---
|
|
335
|
+
def capture_inbox_dir(self) -> Path:
|
|
336
|
+
return config.capture_inbox_dir(self.vault)
|
|
337
|
+
|
|
338
|
+
def draft_capture(
|
|
339
|
+
self, content: str, *, ident: str | None = None, is_source: bool = False
|
|
340
|
+
) -> dict[str, Any]:
|
|
341
|
+
"""Stage a candidate note as a plain DRAFT — the ONE write a VM leg may do.
|
|
342
|
+
|
|
343
|
+
This is the VM-side capture verb (AGENTS.md §5/§6). It writes a plain
|
|
344
|
+
Markdown file into the writable ``capture-inbox/`` on the shared mount and
|
|
345
|
+
stamps ``status: draft`` + ``provenance.trust: untrusted``. It NEVER:
|
|
346
|
+
signs the audit chain, opens the index, writes WAL, or resolves a signing
|
|
347
|
+
key. The draft is NOT authoritative and is NOT surfaced by ``search``
|
|
348
|
+
until the HOST drains it (drain-on-invoke -> sign + index + snapshot).
|
|
349
|
+
|
|
350
|
+
Available on BOTH legs (host + VM) — it is the only quasi-write a VM holds.
|
|
351
|
+
"""
|
|
352
|
+
meta, _body = frontmatter.parse_text(content)
|
|
353
|
+
note_id = ident or (str(meta.get("id")) if meta and meta.get("id") else None)
|
|
354
|
+
if not note_id:
|
|
355
|
+
# deterministic fallback id from content hash
|
|
356
|
+
note_id = "draft-" + sha256_text(content)[:12]
|
|
357
|
+
# C-1 trust boundary: the id comes from --id or untrusted YAML and
|
|
358
|
+
# becomes a path — refuse anything but a bare slug (fail closed).
|
|
359
|
+
note_id = safe_slug(note_id)
|
|
360
|
+
staged = _stamp_draft_frontmatter(content, note_id, is_source)
|
|
361
|
+
inbox = self.capture_inbox_dir()
|
|
362
|
+
inbox.mkdir(parents=True, exist_ok=True)
|
|
363
|
+
target = inbox / f"{note_id}.md"
|
|
364
|
+
# Belt over the slug check: the resolved target (symlinks followed)
|
|
365
|
+
# must stay inside the inbox.
|
|
366
|
+
if not _contained_in(target, inbox):
|
|
367
|
+
raise ValueError(f"draft target escapes capture inbox: {note_id!r}")
|
|
368
|
+
target.write_text(staged, encoding="utf-8")
|
|
369
|
+
return {
|
|
370
|
+
"draft": str(target),
|
|
371
|
+
"id": note_id,
|
|
372
|
+
"signed": False,
|
|
373
|
+
"indexed": False,
|
|
374
|
+
"authoritative": False,
|
|
375
|
+
"note": "draft staged; host drain-on-invoke will sign + index + snapshot",
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
# -- maintenance (HOST-broker only) ----------------------------------
|
|
379
|
+
def rebuild(self) -> dict[str, Any]:
|
|
380
|
+
self._require_host("rebuild the index")
|
|
381
|
+
return self.index.rebuild(self.vault)
|
|
382
|
+
|
|
383
|
+
def embedder_pending(self) -> bool:
|
|
384
|
+
"""True when the index's stored dense vectors were built with a
|
|
385
|
+
DIFFERENT embedder than the one the live runtime would use now (S02/
|
|
386
|
+
CS-01) — e.g. a cold-start install built the index with the offline
|
|
387
|
+
``hash`` placeholder to avoid a network model download. Read-only,
|
|
388
|
+
cheap (no download): :meth:`BrainIndex.model_matches` only compares
|
|
389
|
+
recorded meta strings against the constructed (not yet loaded)
|
|
390
|
+
embedder's ``model_id``/``dim``."""
|
|
391
|
+
return not self.index.model_matches()
|
|
392
|
+
|
|
393
|
+
def warmup(self) -> dict[str, Any]:
|
|
394
|
+
"""HOST-ONLY (S02/CS-01): resolve + download the live auto-embedder's
|
|
395
|
+
model weights now, instead of on the first real semantic search.
|
|
396
|
+
|
|
397
|
+
huggingface_hub prints its own progress bar to stderr during the
|
|
398
|
+
download (never stdout — keeps ``--json`` output parseable) and
|
|
399
|
+
already file-locks the blob it is writing
|
|
400
|
+
(``huggingface_hub.file_download.WeakFileLock``), so a concurrent
|
|
401
|
+
warmup / first-search / nightly-maintenance embed racing on the same
|
|
402
|
+
cache directory cannot corrupt it — see the closeout note; no extra
|
|
403
|
+
locking is added here.
|
|
404
|
+
|
|
405
|
+
Does NOT rebuild the index. If the index was built with a placeholder
|
|
406
|
+
embedder (``embedder_pending()`` was True), run `brain sync` (or
|
|
407
|
+
`brain rebuild`) afterward — `BrainIndex.sync`'s existing model-
|
|
408
|
+
mismatch guard will do a full, now-offline (model already cached)
|
|
409
|
+
re-embed automatically."""
|
|
410
|
+
self._require_host("warm up the embedding model (download)")
|
|
411
|
+
import os
|
|
412
|
+
import time
|
|
413
|
+
|
|
414
|
+
from .embed import get_embedder, model_cache_ready
|
|
415
|
+
|
|
416
|
+
embedder = get_embedder(os.environ.get("BRAIN_EMBEDDER", "auto"))
|
|
417
|
+
was_cached = model_cache_ready(embedder)
|
|
418
|
+
t0 = time.monotonic()
|
|
419
|
+
embedder.embed("warmup") # triggers the real load/download if needed
|
|
420
|
+
elapsed = time.monotonic() - t0
|
|
421
|
+
return {
|
|
422
|
+
"model_id": embedder.model_id,
|
|
423
|
+
"already_cached": bool(was_cached),
|
|
424
|
+
"elapsed_s": round(elapsed, 2),
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
def drafts_dir(self) -> Path:
|
|
428
|
+
return self.vault / ".brain" / "drafts"
|
|
429
|
+
|
|
430
|
+
def _draft_sources(self) -> list[Path]:
|
|
431
|
+
"""Both draft drop locations, drained on the host: the legacy
|
|
432
|
+
``.brain/drafts/`` and the VM-facing ``capture-inbox/``."""
|
|
433
|
+
dirs = [self.drafts_dir(), self.capture_inbox_dir()]
|
|
434
|
+
seen: set[str] = set()
|
|
435
|
+
out: list[Path] = []
|
|
436
|
+
for d in dirs:
|
|
437
|
+
key = str(d.resolve()) if d.exists() else str(d)
|
|
438
|
+
if key in seen:
|
|
439
|
+
continue
|
|
440
|
+
seen.add(key)
|
|
441
|
+
out.append(d)
|
|
442
|
+
return out
|
|
443
|
+
|
|
444
|
+
def drain_drafts(self) -> dict[str, Any]:
|
|
445
|
+
"""drain-on-invoke (HOST only): promote pending capture drafts.
|
|
446
|
+
|
|
447
|
+
The incremental indexer IS the capture drain. The host picks up each
|
|
448
|
+
draft in ``.brain/drafts/`` AND ``capture-inbox/`` (the VM-facing drop),
|
|
449
|
+
signs + writes it into ``raw/`` (if a source) or ``brain/resources/`` (if
|
|
450
|
+
a note) via the audited host-broker ``write_note``, then removes the
|
|
451
|
+
draft. Idempotent and cheap: empty drop dirs are a no-op. Fails CLOSED —
|
|
452
|
+
if no signing key resolves, drafts are LEFT in place (never promoted
|
|
453
|
+
unsigned) and reported as skipped.
|
|
454
|
+
|
|
455
|
+
This is NOT a dedicated scheduled task and NOT a daemon: it runs as the
|
|
456
|
+
first step of any host ``sync`` invocation. There is no capture daemon
|
|
457
|
+
and no dedicated drain task — the ONE sanctioned scheduled task is the
|
|
458
|
+
ux-02 brief/digest, which doubles as the guaranteed daily drain floor.
|
|
459
|
+
"""
|
|
460
|
+
self._require_host("drain capture drafts (sign + index)")
|
|
461
|
+
promoted: list[str] = []
|
|
462
|
+
skipped: list[dict[str, str]] = []
|
|
463
|
+
any_dir = False
|
|
464
|
+
for ddir in self._draft_sources():
|
|
465
|
+
if not ddir.is_dir():
|
|
466
|
+
continue
|
|
467
|
+
any_dir = True
|
|
468
|
+
for draft in sorted(ddir.glob("*.md")):
|
|
469
|
+
note = load_note(draft, self.vault)
|
|
470
|
+
if note is None:
|
|
471
|
+
skipped.append({"draft": draft.name, "reason": "no-frontmatter"})
|
|
472
|
+
continue
|
|
473
|
+
# C-2 trust boundary: note.id comes from attacker-controlled
|
|
474
|
+
# draft frontmatter — refuse non-slug ids (fail closed, draft
|
|
475
|
+
# left in place, never signed).
|
|
476
|
+
try:
|
|
477
|
+
nid = safe_slug(note.id)
|
|
478
|
+
except ValueError as exc:
|
|
479
|
+
skipped.append({"draft": draft.name, "reason": f"unsafe-id (fail-closed): {exc}"})
|
|
480
|
+
continue
|
|
481
|
+
# raw source -> raw/<id>.md ; otherwise a brain note -> resources/.
|
|
482
|
+
if note.type == "source" or note.zone == "raw":
|
|
483
|
+
rel, subtree = f"raw/{nid}.md", "raw"
|
|
484
|
+
else:
|
|
485
|
+
rel, subtree = f"brain/resources/{nid}.md", "brain/resources"
|
|
486
|
+
# M-3: a duplicate id would write a second file sharing the
|
|
487
|
+
# same frontmatter `id`, which crashes the next sync on the
|
|
488
|
+
# UNIQUE index constraint (H-2). Check both the live index and
|
|
489
|
+
# the target path before promoting; report as skipped rather
|
|
490
|
+
# than write the collision.
|
|
491
|
+
dest_path = self.vault / rel
|
|
492
|
+
try:
|
|
493
|
+
already_indexed = self.index.get(nid) is not None
|
|
494
|
+
except Exception:
|
|
495
|
+
already_indexed = False
|
|
496
|
+
if already_indexed or dest_path.exists():
|
|
497
|
+
skipped.append({"draft": draft.name,
|
|
498
|
+
"reason": f"duplicate-id: {nid!r} already exists"})
|
|
499
|
+
continue
|
|
500
|
+
content = draft.read_text(encoding="utf-8")
|
|
501
|
+
try:
|
|
502
|
+
self.write_note(rel, content,
|
|
503
|
+
reason=f"drain-on-invoke promote {draft.name}",
|
|
504
|
+
subtree=subtree)
|
|
505
|
+
except KeyUnavailable:
|
|
506
|
+
skipped.append({"draft": draft.name, "reason": "no-signing-key (fail-closed)"})
|
|
507
|
+
continue
|
|
508
|
+
except ValueError as exc:
|
|
509
|
+
skipped.append({"draft": draft.name, "reason": f"unsafe-path (fail-closed): {exc}"})
|
|
510
|
+
continue
|
|
511
|
+
draft.unlink()
|
|
512
|
+
promoted.append(rel)
|
|
513
|
+
if not any_dir:
|
|
514
|
+
return {"promoted": 0, "skipped": 0, "details": [], "reason": "no-drafts-dir"}
|
|
515
|
+
return {
|
|
516
|
+
"promoted": len(promoted),
|
|
517
|
+
"skipped": len(skipped),
|
|
518
|
+
"details": {"promoted": promoted, "skipped": skipped},
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
def ingest_dropzone(self, *, dry_run: bool = False) -> dict[str, Any]:
|
|
522
|
+
"""HOST-only: drain ``<vault>/inbox/`` (ADR-0003 Ruling 1 / ING-01).
|
|
523
|
+
|
|
524
|
+
Refused on the VM leg BEFORE any filesystem side effect (no key
|
|
525
|
+
lookup, no processing-dir claim, no archive/WAL write) — the same
|
|
526
|
+
fail-closed shape as ``drain_drafts``/``write_note``. Idempotent and
|
|
527
|
+
cheap when the inbox is empty or absent (a directory listing)."""
|
|
528
|
+
self._require_host("ingest the drop zone")
|
|
529
|
+
from .ingest.pipeline import run_ingest
|
|
530
|
+
|
|
531
|
+
return run_ingest(self, dry_run=dry_run)
|
|
532
|
+
|
|
533
|
+
def ingest_transcript(
|
|
534
|
+
self, path: str | Path, *, origin: str, language: str | None = None,
|
|
535
|
+
document_date: str | None = None, classification: str = "Internal",
|
|
536
|
+
) -> dict[str, Any]:
|
|
537
|
+
"""HOST-only: promote one transcript ``.md`` file into ``vault/raw/``
|
|
538
|
+
with explicit provenance (ADR-0003 Ruling 1 companion / ING-04).
|
|
539
|
+
|
|
540
|
+
``origin`` is the source audio/video file path, or the literal
|
|
541
|
+
string ``"verbal"`` — the generic drop-zone (``ingest_dropzone``)
|
|
542
|
+
cannot express this fact on its own (its own ``origin`` always points
|
|
543
|
+
at an archived COPY of the dropped file). Refused on the VM leg
|
|
544
|
+
BEFORE any filesystem side effect, same fail-closed shape as
|
|
545
|
+
``ingest_dropzone``/``write_note``."""
|
|
546
|
+
self._require_host("ingest a transcript")
|
|
547
|
+
from .ingest.transcript import ingest_transcript as _ingest_transcript
|
|
548
|
+
|
|
549
|
+
return _ingest_transcript(
|
|
550
|
+
self, path, origin=origin, language=language,
|
|
551
|
+
document_date=document_date, classification=classification,
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
def sync(self, *, drain: bool = True, publish: bool = False) -> dict[str, Any]:
|
|
555
|
+
"""Incremental index reconcile (IDX-03), draining capture drafts AND
|
|
556
|
+
the ingestion drop zone first.
|
|
557
|
+
|
|
558
|
+
HOST-broker only (it mutates the index). ``drain`` runs the host capture
|
|
559
|
+
drain + inbox ingest drain before reconciling (ADR-0003 Ruling 1
|
|
560
|
+
amendment: the ingest drain fires on every host ``sync``, not only the
|
|
561
|
+
nightly `maintain` floor); ``publish`` additionally republishes the
|
|
562
|
+
read-only snapshot so a VM session's next read sees the just-committed
|
|
563
|
+
note (closing the capture loop). Set ``drain=False`` only for a host
|
|
564
|
+
read-only reconcile."""
|
|
565
|
+
self._require_host("sync (mutate) the index")
|
|
566
|
+
drain_res = self.drain_drafts() if drain else {"promoted": 0, "skipped": 0, "drain": "off"}
|
|
567
|
+
if drain:
|
|
568
|
+
try:
|
|
569
|
+
ingest_res = self.ingest_dropzone()
|
|
570
|
+
except Exception as exc:
|
|
571
|
+
# C2: run_ingest's own per-file retry/quarantine machinery
|
|
572
|
+
# isolates a single poison file WITHOUT raising, but this is
|
|
573
|
+
# the last-resort backstop for anything that still escapes it
|
|
574
|
+
# (e.g. a manifest/failures-file I/O error). ingest_dropzone
|
|
575
|
+
# ran BEFORE index.sync with no try/except, so any escaping
|
|
576
|
+
# exception aborted index reconciliation and snapshot
|
|
577
|
+
# publication on every subsequent sync — one bad drop must
|
|
578
|
+
# never abort index maintenance.
|
|
579
|
+
ingest_res = {"processed": [], "error": f"{type(exc).__name__}: {exc}"}
|
|
580
|
+
else:
|
|
581
|
+
ingest_res = {"processed": [], "reason": "drain-off"}
|
|
582
|
+
idx_res = self.index.sync(self.vault)
|
|
583
|
+
idx_res["drain"] = drain_res
|
|
584
|
+
idx_res["ingest"] = ingest_res
|
|
585
|
+
if publish:
|
|
586
|
+
idx_res["snapshot"] = self.publish_snapshot()
|
|
587
|
+
return idx_res
|
|
588
|
+
|
|
589
|
+
def publish_snapshot(self, dest: str | Path | None = None) -> dict[str, Any]:
|
|
590
|
+
"""Publish a read-only, generation-stamped snapshot of the authoritative
|
|
591
|
+
host index (atomic). The VM mounts this read-only; it never writes the
|
|
592
|
+
authoritative DB. HOST-broker only."""
|
|
593
|
+
self._require_host("publish a snapshot")
|
|
594
|
+
from .snapshot import publish_snapshot as _publish
|
|
595
|
+
|
|
596
|
+
dest_dir = Path(dest) if dest else config.snapshot_dir(self.vault)
|
|
597
|
+
return _publish(self.index.db_path, dest_dir).to_dict()
|
|
598
|
+
|
|
599
|
+
def restore_index_from_snapshot(
|
|
600
|
+
self, *, force: bool = False, dry_run: bool = False
|
|
601
|
+
) -> dict[str, Any]:
|
|
602
|
+
"""Fast index recovery: replace the live index with the published snapshot.
|
|
603
|
+
|
|
604
|
+
The snapshot is a complete, read-consistent copy of the authoritative
|
|
605
|
+
index, so restoring from it is O(seconds) — the safe alternative to a
|
|
606
|
+
full re-embed ``rebuild`` when the live index is corrupt or empty (e.g.
|
|
607
|
+
an interrupted rebuild left a half-written DB). HOST-broker only.
|
|
608
|
+
|
|
609
|
+
Guards: refuses a missing/empty/unreadable snapshot; refuses to clobber a
|
|
610
|
+
live index that holds MORE notes than the snapshot (the snapshot is
|
|
611
|
+
older — ``sync``/``rebuild`` instead) unless ``force``; backs up the
|
|
612
|
+
current index (reversible ``.pre-restore-*.bak``) before overwriting; and
|
|
613
|
+
verifies the note count post-restore.
|
|
614
|
+
"""
|
|
615
|
+
import datetime as _dt
|
|
616
|
+
import shutil as _sh
|
|
617
|
+
import sqlite3 as _sq
|
|
618
|
+
|
|
619
|
+
self._require_host("restore the index from a snapshot")
|
|
620
|
+
idx = config.index_path(self.vault)
|
|
621
|
+
snap = config.snapshot_db_path(self.vault)
|
|
622
|
+
|
|
623
|
+
def _count(p: Path):
|
|
624
|
+
if not p.exists():
|
|
625
|
+
return None # absent
|
|
626
|
+
try:
|
|
627
|
+
c = _sq.connect(f"file:{p}?mode=ro", uri=True)
|
|
628
|
+
try:
|
|
629
|
+
return int(c.execute("SELECT count(*) FROM notes").fetchone()[0])
|
|
630
|
+
finally:
|
|
631
|
+
c.close()
|
|
632
|
+
except Exception:
|
|
633
|
+
return -1 # present but unreadable/corrupt
|
|
634
|
+
|
|
635
|
+
snap_n = _count(snap)
|
|
636
|
+
if snap_n is None:
|
|
637
|
+
raise FileNotFoundError(f"no snapshot to restore from: {snap}")
|
|
638
|
+
if snap_n <= 0:
|
|
639
|
+
raise ValueError(
|
|
640
|
+
f"snapshot has {snap_n} notes — refusing to restore an empty/corrupt "
|
|
641
|
+
f"snapshot ({snap})")
|
|
642
|
+
live_n = _count(idx)
|
|
643
|
+
|
|
644
|
+
if live_n is not None and live_n > snap_n and not force:
|
|
645
|
+
raise ValueError(
|
|
646
|
+
f"live index has {live_n} notes but the snapshot has only {snap_n} — "
|
|
647
|
+
f"restoring would LOSE {live_n - snap_n} note(s). The snapshot is older; "
|
|
648
|
+
f"run `brain sync`/`rebuild` instead, or pass --force to override.")
|
|
649
|
+
|
|
650
|
+
plan: dict[str, Any] = {
|
|
651
|
+
"index": str(idx), "snapshot": str(snap),
|
|
652
|
+
"snapshot_notes": snap_n, "live_notes_before": live_n,
|
|
653
|
+
}
|
|
654
|
+
if dry_run:
|
|
655
|
+
plan["dry_run"] = True
|
|
656
|
+
return plan
|
|
657
|
+
|
|
658
|
+
config.ensure_index_dir(self.vault)
|
|
659
|
+
backup = None
|
|
660
|
+
if idx.exists():
|
|
661
|
+
stamp = _dt.datetime.now().strftime("%Y%m%dT%H%M%S")
|
|
662
|
+
backup = idx.with_name(idx.name + f".pre-restore-{stamp}.bak")
|
|
663
|
+
_sh.move(str(idx), str(backup))
|
|
664
|
+
for suf in ("-wal", "-shm"): # stale sqlite sidecars would mask the copy
|
|
665
|
+
side = idx.with_name(idx.name + suf)
|
|
666
|
+
if side.exists():
|
|
667
|
+
side.unlink()
|
|
668
|
+
_sh.copy2(str(snap), str(idx))
|
|
669
|
+
|
|
670
|
+
live_after = _count(idx)
|
|
671
|
+
if live_after != snap_n:
|
|
672
|
+
raise RuntimeError(
|
|
673
|
+
f"post-restore verification failed: index has {live_after} notes, "
|
|
674
|
+
f"expected {snap_n} (backup preserved at {backup})")
|
|
675
|
+
plan.update({"restored": True, "live_notes_after": live_after,
|
|
676
|
+
"backup": str(backup) if backup else None})
|
|
677
|
+
return plan
|
|
678
|
+
|
|
679
|
+
def status(self, snapshot_dest: str | Path | None = None, today: Any = None) -> dict[str, Any]:
|
|
680
|
+
"""Report index stats + snapshot generation/age (available on BOTH legs —
|
|
681
|
+
the VM uses it to tell whether its read-only view is fresh or stale, and
|
|
682
|
+
how many drafts are pending)."""
|
|
683
|
+
from . import __version__
|
|
684
|
+
from .index import SCHEMA_VERSION
|
|
685
|
+
from .snapshot import snapshot_status
|
|
686
|
+
|
|
687
|
+
dest_dir = Path(snapshot_dest) if snapshot_dest else config.snapshot_dir(self.vault)
|
|
688
|
+
out: dict[str, Any] = {"vault": str(self.vault), "role": self.role}
|
|
689
|
+
try:
|
|
690
|
+
out["index"] = self.index.stats()
|
|
691
|
+
except Exception as exc: # index/snapshot not built yet
|
|
692
|
+
out["index"] = {"error": f"{type(exc).__name__}: {exc}"}
|
|
693
|
+
# ADR-0004 Ruling 2/8: surface the version everywhere a skew could
|
|
694
|
+
# implicate a failure. `index_newer_than_binary` flags the direction
|
|
695
|
+
# `sync()` cannot silently absorb — an on-disk schema_version GREATER
|
|
696
|
+
# than this binary's SCHEMA_VERSION means an older `brain` met newer
|
|
697
|
+
# state and must not rebuild it downward.
|
|
698
|
+
stored_schema = out["index"].get("schema_version") if isinstance(out.get("index"), dict) else None
|
|
699
|
+
index_newer = False
|
|
700
|
+
if stored_schema is not None:
|
|
701
|
+
try:
|
|
702
|
+
index_newer = int(stored_schema) > SCHEMA_VERSION
|
|
703
|
+
except (TypeError, ValueError):
|
|
704
|
+
index_newer = False
|
|
705
|
+
out["version"] = {
|
|
706
|
+
"package_version": __version__,
|
|
707
|
+
"index_schema_version": stored_schema,
|
|
708
|
+
"binary_schema_version": SCHEMA_VERSION,
|
|
709
|
+
"index_newer_than_binary": index_newer,
|
|
710
|
+
}
|
|
711
|
+
# LIVE embedder surfacing (S11). ``index.embed_model`` above is INDEX
|
|
712
|
+
# METADATA — the model the index was BUILT with; it does NOT prove which
|
|
713
|
+
# embedder would answer a query right now. On a partial install
|
|
714
|
+
# (onnxruntime missing) get_embedder() degrades to HashEmbedder while the
|
|
715
|
+
# metadata still says e5-small. Surface the model_id of the embedder
|
|
716
|
+
# actually constructed, and flag a mismatch loudly so a silent semantic
|
|
717
|
+
# downgrade is visible in `brain status`/`brain health`.
|
|
718
|
+
try:
|
|
719
|
+
live_id = self.index.embedder.model_id
|
|
720
|
+
recorded = out.get("index", {}).get("embed_model")
|
|
721
|
+
matches = recorded is None or recorded == live_id
|
|
722
|
+
out["live_embedder"] = {
|
|
723
|
+
"model_id": live_id,
|
|
724
|
+
"is_hash_fallback": live_id == "hash-v1",
|
|
725
|
+
"matches_index_metadata": matches,
|
|
726
|
+
}
|
|
727
|
+
# `embedder: ready|pending` (S02/CS-01) — the cold-start-friendly
|
|
728
|
+
# summary a human/agent actually wants from `brain status`: is
|
|
729
|
+
# semantic search fully live right now, or is there a deferred
|
|
730
|
+
# download/re-embed step still owed? A deliberate explicit-hash
|
|
731
|
+
# choice ($BRAIN_EMBEDDER=hash) is "ready" (nothing IS pending —
|
|
732
|
+
# same "deliberate, not a fault" posture as `brain doctor`);
|
|
733
|
+
# otherwise pending means either the model isn't cached yet
|
|
734
|
+
# (`brain warmup` needed) or the index still carries placeholder
|
|
735
|
+
# vectors from a cold-start install (`brain sync` needed after).
|
|
736
|
+
import os
|
|
737
|
+
|
|
738
|
+
from .embed import ONNX_MODEL_SIZE_HINT, model_cache_ready
|
|
739
|
+
|
|
740
|
+
explicit_hash = os.environ.get("BRAIN_EMBEDDER", "").strip().lower() == "hash"
|
|
741
|
+
cached = model_cache_ready(self.index.embedder)
|
|
742
|
+
pending = (not explicit_hash) and (not matches or cached is False)
|
|
743
|
+
out["embedder"] = {
|
|
744
|
+
"state": "pending" if pending else "ready",
|
|
745
|
+
"model_id": live_id,
|
|
746
|
+
"cached": cached,
|
|
747
|
+
"index_matches": matches,
|
|
748
|
+
}
|
|
749
|
+
if pending and cached is False:
|
|
750
|
+
out["embedder"]["download_size_hint"] = ONNX_MODEL_SIZE_HINT
|
|
751
|
+
except Exception as exc:
|
|
752
|
+
out["live_embedder"] = {"error": f"{type(exc).__name__}: {exc}"}
|
|
753
|
+
out["embedder"] = {"state": "error", "error": f"{type(exc).__name__}: {exc}"}
|
|
754
|
+
out["snapshot"] = snapshot_status(dest_dir)
|
|
755
|
+
# Mirror of the index check above (Ruling 2's directional fail-fast):
|
|
756
|
+
# a snapshot schema_version GREATER than this binary's SCHEMA_VERSION
|
|
757
|
+
# means an old CLI is reading state a newer engine produced — the CLI
|
|
758
|
+
# command layer (not this report) is what must refuse; this just makes
|
|
759
|
+
# the condition visible before it bites (Ruling 8).
|
|
760
|
+
snap_schema = out["snapshot"].get("schema_version")
|
|
761
|
+
snapshot_newer = False
|
|
762
|
+
if snap_schema is not None:
|
|
763
|
+
try:
|
|
764
|
+
snapshot_newer = int(snap_schema) > SCHEMA_VERSION
|
|
765
|
+
except (TypeError, ValueError):
|
|
766
|
+
snapshot_newer = False
|
|
767
|
+
out["version"]["snapshot_schema_version"] = snap_schema
|
|
768
|
+
out["version"]["snapshot_newer_than_binary"] = snapshot_newer
|
|
769
|
+
out["pending_drafts"] = self._count_pending_drafts()
|
|
770
|
+
# ADR-0003 Ruling 5/d + HARDENED:premortem — surface `brain maintain`'s
|
|
771
|
+
# own heartbeat (a stale `daily` branch or a repeatedly-failing branch)
|
|
772
|
+
# so a broken nightly is visible here too, not only via the
|
|
773
|
+
# session-start hook's stale-nightly line.
|
|
774
|
+
out["maintain_heartbeat"] = self._maintain_heartbeat_summary(today=today)
|
|
775
|
+
out["graph"] = self._graph_status()
|
|
776
|
+
return out
|
|
777
|
+
|
|
778
|
+
def _count_pending_drafts(self) -> int:
|
|
779
|
+
n = 0
|
|
780
|
+
for ddir in self._draft_sources():
|
|
781
|
+
if ddir.is_dir():
|
|
782
|
+
n += len(list(ddir.glob("*.md")))
|
|
783
|
+
return n
|
|
784
|
+
|
|
785
|
+
# -- write verb (HOST-BROKER ONLY; audited; fails closed) ------------
|
|
786
|
+
def write_note(
|
|
787
|
+
self, rel_path: str, content: str, reason: str = "", *,
|
|
788
|
+
subtree: str | None = None,
|
|
789
|
+
) -> dict[str, Any]:
|
|
790
|
+
"""Write a note to the vault and append a signed audit-chain entry.
|
|
791
|
+
|
|
792
|
+
Fails closed in BOTH directions:
|
|
793
|
+
- if no signing key resolves (KeyUnavailable), nothing is written;
|
|
794
|
+
- the chain records the write ATTEMPT first, then the OUTCOME. If the
|
|
795
|
+
file write raises after signing (disk full, permission), a compensating
|
|
796
|
+
``write_failed`` entry is appended so the chain never claims a write
|
|
797
|
+
that didn't land (F-06). The original exception is re-raised.
|
|
798
|
+
|
|
799
|
+
Containment (C-2): the RESOLVED target (symlinks followed) must stay
|
|
800
|
+
inside the vault, and — when ``subtree`` is given (e.g. ``"raw"`` or
|
|
801
|
+
``"brain/resources"`` on the drain/capture paths) — inside that
|
|
802
|
+
SPECIFIC subtree, so a traversal-laden rel_path can never earn an
|
|
803
|
+
Ed25519 signature over an overwrite elsewhere. Refused BEFORE signing.
|
|
804
|
+
|
|
805
|
+
HOST-broker only: refused on the VM leg BEFORE any signing-key
|
|
806
|
+
resolution (the VM never holds the audit key).
|
|
807
|
+
"""
|
|
808
|
+
self._require_host("write notes (sign + commit)")
|
|
809
|
+
target = self.vault / rel_path
|
|
810
|
+
if not _contained_in(target, self.vault):
|
|
811
|
+
raise ValueError(f"write target escapes vault: {rel_path}")
|
|
812
|
+
if subtree is not None and not _contained_in(target, self.vault / subtree):
|
|
813
|
+
raise ValueError(f"write target escapes {subtree!r} subtree: {rel_path}")
|
|
814
|
+
target = target.resolve()
|
|
815
|
+
# Append the signed audit entry FIRST; if signing fails, nothing is written.
|
|
816
|
+
try:
|
|
817
|
+
entry = self.audit.append(
|
|
818
|
+
verb="write", path=rel_path,
|
|
819
|
+
reason=reason or f"write_note {rel_path} sha256={sha256_text(content)[:12]}",
|
|
820
|
+
)
|
|
821
|
+
except KeyUnavailable:
|
|
822
|
+
raise # fail closed — no unsigned writes
|
|
823
|
+
try:
|
|
824
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
825
|
+
target.write_text(content, encoding="utf-8")
|
|
826
|
+
except Exception as exc:
|
|
827
|
+
# The signed "write" entry is already in the chain; record the failure
|
|
828
|
+
# so verify-audit shows the attempt did not complete.
|
|
829
|
+
try:
|
|
830
|
+
self.audit.append(
|
|
831
|
+
verb="write_failed", path=rel_path,
|
|
832
|
+
reason=f"file write failed after signing: {type(exc).__name__}: {exc}",
|
|
833
|
+
)
|
|
834
|
+
except KeyUnavailable:
|
|
835
|
+
pass # key vanished mid-op; the original error is what matters
|
|
836
|
+
raise
|
|
837
|
+
return {"written": str(target), "audit": entry}
|
|
838
|
+
|
|
839
|
+
# -- supersession (TMP-02, ADR-0003 Ruling 2/8) — HOST-broker only ----
|
|
840
|
+
_SUPERSEDE_JOURNAL = "supersede-pending.json"
|
|
841
|
+
|
|
842
|
+
def _supersede_journal_path(self) -> Path:
|
|
843
|
+
return config.brain_runtime_dir(self.vault) / self._SUPERSEDE_JOURNAL
|
|
844
|
+
|
|
845
|
+
def _recover_pending_supersede(self) -> dict[str, Any] | None:
|
|
846
|
+
"""HOST-only. If a prior ``supersede`` was interrupted between its two
|
|
847
|
+
signed writes, roll the completed side back to its pre-transaction
|
|
848
|
+
content (itself a fresh signed write) and clear the journal — so a crash
|
|
849
|
+
mid-transaction can never leave a signed half-chain. Runs at the top of
|
|
850
|
+
every ``supersede`` call before any new write is attempted."""
|
|
851
|
+
path = self._supersede_journal_path()
|
|
852
|
+
if not path.exists():
|
|
853
|
+
return None
|
|
854
|
+
import json
|
|
855
|
+
|
|
856
|
+
try:
|
|
857
|
+
journal = json.loads(path.read_text(encoding="utf-8"))
|
|
858
|
+
except (OSError, ValueError):
|
|
859
|
+
path.unlink(missing_ok=True)
|
|
860
|
+
return {"recovered": False, "reason": "unreadable journal, discarded"}
|
|
861
|
+
result: dict[str, Any] = {"recovered": True, "stage": journal.get("stage")}
|
|
862
|
+
if journal.get("stage") == "old_written":
|
|
863
|
+
self.write_note(
|
|
864
|
+
journal["old_rel"], journal["old_before"],
|
|
865
|
+
reason=f"supersede-rollback: {journal['old_id']} -> "
|
|
866
|
+
f"{journal['new_id']} (interrupted mid-transaction)",
|
|
867
|
+
)
|
|
868
|
+
result["action"] = "rolled_back_old"
|
|
869
|
+
path.unlink(missing_ok=True)
|
|
870
|
+
return result
|
|
871
|
+
|
|
872
|
+
def supersede(self, old_id: str, new_id: str, *, reason: str = "") -> dict[str, Any]:
|
|
873
|
+
"""Retire ``old_id`` in favour of ``new_id`` — both sides of the version
|
|
874
|
+
chain, written through the audited ``write_note`` path (ADR-0003 Ruling
|
|
875
|
+
2/8). HOST-broker only.
|
|
876
|
+
|
|
877
|
+
Refuses BEFORE any signing-key resolution / WAL / index mutation when:
|
|
878
|
+
- ``role != host``;
|
|
879
|
+
- either id does not resolve to an on-disk note, or ``old_id == new_id``;
|
|
880
|
+
- ``old_id`` is already superseded (chain invariant: no re-superseding an
|
|
881
|
+
already-superseded note);
|
|
882
|
+
- ``new_id`` itself already carries ``is_latest_version: false`` (would
|
|
883
|
+
make it a "latest" that is simultaneously retired — refuse creating a
|
|
884
|
+
second latest);
|
|
885
|
+
- the successor's OWN frontmatter has no explicit ``classification`` —
|
|
886
|
+
per the ADR ruling, classification is NEVER inherited implicitly
|
|
887
|
+
across a supersession.
|
|
888
|
+
|
|
889
|
+
Atomicity: a pending-operation journal at
|
|
890
|
+
``.brain/supersede-pending.json`` is written before either note write and
|
|
891
|
+
cleared after both succeed. A crash between the two signed writes leaves
|
|
892
|
+
a journal that the NEXT ``supersede`` call rolls back (restores the old
|
|
893
|
+
note, then proceeds) before doing anything else — never a signed
|
|
894
|
+
half-chain (HARDENED:codex).
|
|
895
|
+
"""
|
|
896
|
+
self._require_host("supersede notes (writes both sides of a version chain)")
|
|
897
|
+
self._recover_pending_supersede()
|
|
898
|
+
|
|
899
|
+
if old_id == new_id:
|
|
900
|
+
raise ValueError("supersede: a note may not supersede itself")
|
|
901
|
+
old_row = self.index.get(old_id)
|
|
902
|
+
new_row = self.index.get(new_id)
|
|
903
|
+
if not old_row:
|
|
904
|
+
raise ValueError(f"supersede: old note not found: {old_id}")
|
|
905
|
+
if not new_row:
|
|
906
|
+
raise ValueError(f"supersede: new note not found: {new_id}")
|
|
907
|
+
|
|
908
|
+
old_path, new_path = Path(old_row["path"]), Path(new_row["path"])
|
|
909
|
+
old_before = old_path.read_text(encoding="utf-8")
|
|
910
|
+
new_before = new_path.read_text(encoding="utf-8")
|
|
911
|
+
old_meta, _ = frontmatter.parse_text(old_before)
|
|
912
|
+
new_meta, _ = frontmatter.parse_text(new_before)
|
|
913
|
+
|
|
914
|
+
# -- chain invariants + classification ruling (refused before any write) --
|
|
915
|
+
if old_meta.get("superseded_by") or str(old_meta.get("is_latest_version", "")).strip().lower() == "false":
|
|
916
|
+
raise ValueError(f"supersede: {old_id!r} is already superseded — no re-superseding")
|
|
917
|
+
if str(new_meta.get("is_latest_version", "")).strip().lower() == "false":
|
|
918
|
+
raise ValueError(
|
|
919
|
+
f"supersede: {new_id!r} is itself already retired "
|
|
920
|
+
"(is_latest_version: false) — refusing to create a second latest"
|
|
921
|
+
)
|
|
922
|
+
if not str(new_meta.get("classification") or "").strip():
|
|
923
|
+
raise ValueError(
|
|
924
|
+
f"supersede: successor {new_id!r} has no explicit classification — "
|
|
925
|
+
"classification is never inherited across a supersession (ADR-0003 Ruling 2b)"
|
|
926
|
+
)
|
|
927
|
+
|
|
928
|
+
import datetime as _dt
|
|
929
|
+
|
|
930
|
+
today = _dt.date.today().isoformat()
|
|
931
|
+
old_rel = old_path.relative_to(self.vault).as_posix()
|
|
932
|
+
new_rel = new_path.relative_to(self.vault).as_posix()
|
|
933
|
+
|
|
934
|
+
journal_path = self._supersede_journal_path()
|
|
935
|
+
journal_path.parent.mkdir(parents=True, exist_ok=True)
|
|
936
|
+
import json
|
|
937
|
+
|
|
938
|
+
journal = {
|
|
939
|
+
"stage": "starting", "old_id": old_id, "new_id": new_id,
|
|
940
|
+
"old_rel": old_rel, "new_rel": new_rel,
|
|
941
|
+
"old_before": old_before, "new_before": new_before,
|
|
942
|
+
}
|
|
943
|
+
journal_path.write_text(json.dumps(journal), encoding="utf-8")
|
|
944
|
+
|
|
945
|
+
old_after = frontmatter.set_keys(old_before, {
|
|
946
|
+
"superseded_by": new_id, "superseded_date": today, "is_latest_version": False,
|
|
947
|
+
})
|
|
948
|
+
new_after = frontmatter.set_keys(new_before, {
|
|
949
|
+
"previous_version": old_id, "is_latest_version": True,
|
|
950
|
+
})
|
|
951
|
+
|
|
952
|
+
old_write = self.write_note(
|
|
953
|
+
old_rel, old_after,
|
|
954
|
+
reason=reason or f"supersede: {old_id} -> {new_id} (retiring {old_id})",
|
|
955
|
+
)
|
|
956
|
+
journal["stage"] = "old_written"
|
|
957
|
+
journal_path.write_text(json.dumps(journal), encoding="utf-8")
|
|
958
|
+
|
|
959
|
+
new_write = self.write_note(
|
|
960
|
+
new_rel, new_after,
|
|
961
|
+
reason=reason or f"supersede: {old_id} -> {new_id} (new head {new_id})",
|
|
962
|
+
)
|
|
963
|
+
journal_path.unlink(missing_ok=True)
|
|
964
|
+
|
|
965
|
+
sync_res = self.sync(drain=False)
|
|
966
|
+
return {
|
|
967
|
+
"old_id": old_id, "new_id": new_id,
|
|
968
|
+
"old_write": old_write, "new_write": new_write,
|
|
969
|
+
"reindexed": {"added": sync_res.get("added", 0), "updated": sync_res.get("updated", 0)},
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
def verify_audit(self) -> dict[str, Any]:
|
|
973
|
+
# HOST-broker only: verify() derives the public key via the resolved
|
|
974
|
+
# signing key — the VM leg must never resolve a key.
|
|
975
|
+
self._require_host("verify the audit chain (resolves the signing key)")
|
|
976
|
+
return self.audit.verify()
|
|
977
|
+
|
|
978
|
+
# -- off-host anchor + encrypted backup (HOST-broker only; SEC-03) ----
|
|
979
|
+
def anchor_chain(self, anchor_dir: str | Path) -> dict[str, Any]:
|
|
980
|
+
"""Publish the signed chain head to an OFF-HOST append-only store."""
|
|
981
|
+
self._require_host("anchor the audit chain off-host")
|
|
982
|
+
from . import anchor as _anchor
|
|
983
|
+
|
|
984
|
+
return _anchor.anchor(self.audit.log_path, Path(anchor_dir))
|
|
985
|
+
|
|
986
|
+
def verify_anchor(self, anchor_dir: str | Path) -> dict[str, Any]:
|
|
987
|
+
"""Verify the live chain against the off-host anchor (detect rewrite)."""
|
|
988
|
+
self._require_host("verify the off-host anchor")
|
|
989
|
+
from . import anchor as _anchor
|
|
990
|
+
|
|
991
|
+
return _anchor.verify_against_anchor(self.audit.log_path, Path(anchor_dir))
|
|
992
|
+
|
|
993
|
+
def backup(self, dest_dir: str | Path, *, encrypt: bool = True) -> dict[str, Any]:
|
|
994
|
+
"""Create an encrypted off-device backup of the Markdown truth."""
|
|
995
|
+
self._require_host("create an off-device backup")
|
|
996
|
+
from . import backup as _backup
|
|
997
|
+
|
|
998
|
+
return _backup.create_backup(self.vault, Path(dest_dir), encrypt=encrypt).to_dict()
|
|
999
|
+
|
|
1000
|
+
def restore(self, archive: str | Path, dest_dir: str | Path) -> dict[str, Any]:
|
|
1001
|
+
"""Restore (and decrypt) a backup archive into ``dest_dir``."""
|
|
1002
|
+
self._require_host("restore a backup")
|
|
1003
|
+
from . import backup as _backup
|
|
1004
|
+
|
|
1005
|
+
return _backup.restore_backup(Path(archive), Path(dest_dir))
|
|
1006
|
+
|
|
1007
|
+
# -- daily-use UX layer (UX-01 / UX-02) --------------------------------
|
|
1008
|
+
|
|
1009
|
+
def capture(
|
|
1010
|
+
self,
|
|
1011
|
+
content: str,
|
|
1012
|
+
*,
|
|
1013
|
+
note_id: str | None = None,
|
|
1014
|
+
note_type: str | None = None,
|
|
1015
|
+
classification: str | None = None,
|
|
1016
|
+
reason: str = "",
|
|
1017
|
+
) -> dict[str, Any]:
|
|
1018
|
+
"""Unified capture verb (UX-01).
|
|
1019
|
+
|
|
1020
|
+
HOST path: enforce frontmatter → write_note (sign + audit) → incremental
|
|
1021
|
+
sync → note immediately retrievable.
|
|
1022
|
+
VM path: enforce frontmatter → draft_capture (capture-inbox/, unsigned,
|
|
1023
|
+
unindexed) → host drain-on-invoke picks it up on the next run.
|
|
1024
|
+
|
|
1025
|
+
No signing key is ever touched on the VM path. The VM drops an untrusted
|
|
1026
|
+
draft; the host validates, signs, and indexes it on drain-on-invoke.
|
|
1027
|
+
"""
|
|
1028
|
+
from . import capture as cap_mod
|
|
1029
|
+
|
|
1030
|
+
override: dict[str, Any] = {}
|
|
1031
|
+
if note_id:
|
|
1032
|
+
override["id"] = note_id
|
|
1033
|
+
if note_type:
|
|
1034
|
+
override["type"] = note_type
|
|
1035
|
+
if classification:
|
|
1036
|
+
override["classification"] = classification
|
|
1037
|
+
|
|
1038
|
+
enforced = cap_mod.enforce(content, override=override or None)
|
|
1039
|
+
|
|
1040
|
+
if self.role == config.ROLE_HOST:
|
|
1041
|
+
meta, _body = frontmatter.parse_text(enforced)
|
|
1042
|
+
nid = safe_slug(meta.get("id", "capture")) # same C-1/C-2 trust boundary
|
|
1043
|
+
ntype = str(meta.get("type", "note"))
|
|
1044
|
+
if ntype == "source":
|
|
1045
|
+
rel, subtree = f"raw/{nid}.md", "raw"
|
|
1046
|
+
else:
|
|
1047
|
+
rel, subtree = f"brain/resources/{nid}.md", "brain/resources"
|
|
1048
|
+
write_res = self.write_note(rel, enforced, reason=reason or f"capture {nid}",
|
|
1049
|
+
subtree=subtree)
|
|
1050
|
+
sync_res = self.sync(drain=False) # note already written; just reconcile
|
|
1051
|
+
return {
|
|
1052
|
+
"id": nid,
|
|
1053
|
+
"path": write_res["written"],
|
|
1054
|
+
"signed": True,
|
|
1055
|
+
"indexed": True,
|
|
1056
|
+
"role": "host",
|
|
1057
|
+
"sync": {
|
|
1058
|
+
"added": sync_res.get("added", 0),
|
|
1059
|
+
"updated": sync_res.get("updated", 0),
|
|
1060
|
+
},
|
|
1061
|
+
}
|
|
1062
|
+
else:
|
|
1063
|
+
res = self.draft_capture(enforced, ident=None, is_source=False)
|
|
1064
|
+
return {
|
|
1065
|
+
"id": res["id"],
|
|
1066
|
+
"draft": res["draft"],
|
|
1067
|
+
"signed": False,
|
|
1068
|
+
"indexed": False,
|
|
1069
|
+
"role": "vm",
|
|
1070
|
+
"note": "draft in capture-inbox/; host drain-on-invoke will sign + index",
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
def brief(
|
|
1074
|
+
self, *, max_recent: int = 5, drain: bool = True,
|
|
1075
|
+
max_tier: str = classification.DEFAULT_MAX_TIER,
|
|
1076
|
+
) -> dict[str, Any]:
|
|
1077
|
+
"""Generate the morning brief (UX-02).
|
|
1078
|
+
|
|
1079
|
+
Drains pending captures first (HOST only) — making this the guaranteed
|
|
1080
|
+
daily drain FLOOR when run as the scheduled task. Always reports the
|
|
1081
|
+
pending count BEFORE the drain attempt so a stalled drain is visible
|
|
1082
|
+
next morning via the tripwire line.
|
|
1083
|
+
|
|
1084
|
+
VM leg: reports pending count + index stats (read-only view) but cannot
|
|
1085
|
+
drain (no signing key).
|
|
1086
|
+
|
|
1087
|
+
The recent-notes list is routed through the SAME egress.apply_gate
|
|
1088
|
+
chokepoint as every other read verb (H-1) — a summary surface must not
|
|
1089
|
+
leak titles/paths/classification of withheld-tier notes.
|
|
1090
|
+
"""
|
|
1091
|
+
from . import brief as brief_mod
|
|
1092
|
+
from . import egress
|
|
1093
|
+
from .snapshot import snapshot_status
|
|
1094
|
+
|
|
1095
|
+
pending_before = self._count_pending_drafts()
|
|
1096
|
+
drain_res: dict[str, Any] = {"promoted": 0, "skipped": 0}
|
|
1097
|
+
|
|
1098
|
+
if self.role == config.ROLE_HOST and drain:
|
|
1099
|
+
try:
|
|
1100
|
+
drain_res = self.drain_drafts()
|
|
1101
|
+
except Exception as exc:
|
|
1102
|
+
drain_res = {"promoted": 0, "skipped": 0, "error": str(exc)}
|
|
1103
|
+
|
|
1104
|
+
try:
|
|
1105
|
+
stats = self.index.stats()
|
|
1106
|
+
except Exception:
|
|
1107
|
+
stats = {"notes": 0, "chunks": 0}
|
|
1108
|
+
|
|
1109
|
+
try:
|
|
1110
|
+
recent = self.recent(limit=max_recent)
|
|
1111
|
+
except Exception:
|
|
1112
|
+
recent = []
|
|
1113
|
+
|
|
1114
|
+
surfaced, egress_report = egress.apply_gate(recent, max_tier=max_tier)
|
|
1115
|
+
|
|
1116
|
+
snap = snapshot_status(config.snapshot_dir(self.vault))
|
|
1117
|
+
age_hours: float | None = None
|
|
1118
|
+
if snap.get("snapshot") == "present" and snap.get("age_seconds") is not None:
|
|
1119
|
+
age_hours = snap["age_seconds"] / 3600
|
|
1120
|
+
|
|
1121
|
+
result = brief_mod.build_brief(
|
|
1122
|
+
index_stats=stats,
|
|
1123
|
+
recent_notes=surfaced,
|
|
1124
|
+
pending_before_drain=pending_before,
|
|
1125
|
+
drain_result=drain_res,
|
|
1126
|
+
snapshot_age_hours=age_hours,
|
|
1127
|
+
max_recent=max_recent,
|
|
1128
|
+
)
|
|
1129
|
+
result["egress"] = egress_report
|
|
1130
|
+
return result
|
|
1131
|
+
|
|
1132
|
+
def digest(
|
|
1133
|
+
self, *, days: int = 7, max_tier: str = classification.DEFAULT_MAX_TIER,
|
|
1134
|
+
) -> dict[str, Any]:
|
|
1135
|
+
"""Generate the weekly digest (UX-02).
|
|
1136
|
+
|
|
1137
|
+
Shows notes from the past ``days`` days. Available on both host and VM
|
|
1138
|
+
legs (read-only; reads from the index/snapshot in use for this role).
|
|
1139
|
+
|
|
1140
|
+
The recent-notes list is gated through egress.apply_gate before it is
|
|
1141
|
+
built into the digest (H-1) — same chokepoint as every other read verb.
|
|
1142
|
+
"""
|
|
1143
|
+
from . import brief as brief_mod
|
|
1144
|
+
from . import egress
|
|
1145
|
+
|
|
1146
|
+
try:
|
|
1147
|
+
stats = self.index.stats()
|
|
1148
|
+
except Exception:
|
|
1149
|
+
stats = {"notes": 0, "chunks": 0}
|
|
1150
|
+
|
|
1151
|
+
try:
|
|
1152
|
+
recent = self.recent(limit=500)
|
|
1153
|
+
except Exception:
|
|
1154
|
+
recent = []
|
|
1155
|
+
|
|
1156
|
+
surfaced, egress_report = egress.apply_gate(recent, max_tier=max_tier)
|
|
1157
|
+
|
|
1158
|
+
result = brief_mod.build_digest(
|
|
1159
|
+
index_stats=stats, recent_notes=surfaced, days=days
|
|
1160
|
+
)
|
|
1161
|
+
result["egress"] = egress_report
|
|
1162
|
+
return result
|
|
1163
|
+
|
|
1164
|
+
def _autoresearch_status(self, today: Any) -> dict[str, Any]:
|
|
1165
|
+
"""Maintenance-visibility line data (HARDENED:claude, AUT-01): scan
|
|
1166
|
+
``eval/runs/autoresearch-*.json`` for the newest ``captured``
|
|
1167
|
+
timestamp and judge staleness via the pure
|
|
1168
|
+
``maintenance.autoresearch_staleness`` helper. No autoresearch run has
|
|
1169
|
+
landed at this session (aut-04 is session s11, after this one) — a
|
|
1170
|
+
missing/unreadable artifact is treated as ``never_run``, never an
|
|
1171
|
+
error, so the brief still renders."""
|
|
1172
|
+
import datetime as _dt
|
|
1173
|
+
import json
|
|
1174
|
+
|
|
1175
|
+
from . import maintenance as maint
|
|
1176
|
+
|
|
1177
|
+
runs_dir = Path(__file__).resolve().parents[2] / "eval" / "runs"
|
|
1178
|
+
latest: _dt.datetime | None = None
|
|
1179
|
+
try:
|
|
1180
|
+
for p in runs_dir.glob("autoresearch-*.json"):
|
|
1181
|
+
try:
|
|
1182
|
+
data = json.loads(p.read_text(encoding="utf-8"))
|
|
1183
|
+
ts = _dt.datetime.fromisoformat(str(data.get("captured")))
|
|
1184
|
+
except Exception:
|
|
1185
|
+
continue
|
|
1186
|
+
if latest is None or ts > latest:
|
|
1187
|
+
latest = ts
|
|
1188
|
+
except Exception:
|
|
1189
|
+
pass
|
|
1190
|
+
return maint.autoresearch_staleness(latest.date() if latest else None, today)
|
|
1191
|
+
|
|
1192
|
+
def brief_html(
|
|
1193
|
+
self, *, max_recent: int = 5, drain: bool = True,
|
|
1194
|
+
max_tier: str = classification.VM_DEFAULT_MAX_TIER, today: Any = None,
|
|
1195
|
+
) -> dict[str, Any]:
|
|
1196
|
+
"""Render + write the branded HTML morning brief (AUT-01, ADR-0003
|
|
1197
|
+
Ruling c) to ``.brain/brief/``. HOST-ONLY: this writes a FILE, a new
|
|
1198
|
+
egress surface the stdout gate does not cover, so every section is
|
|
1199
|
+
composed from data already routed through ``egress.apply_gate`` at
|
|
1200
|
+
``max_tier`` (default Internal) before it reaches the pure renderer
|
|
1201
|
+
(``brain.brief.render_brief_html``), which does no I/O of its own —
|
|
1202
|
+
gathering + gating happens entirely here.
|
|
1203
|
+
"""
|
|
1204
|
+
import datetime as _dt
|
|
1205
|
+
|
|
1206
|
+
from . import brief as brief_mod
|
|
1207
|
+
from . import egress
|
|
1208
|
+
from . import maintenance as maint
|
|
1209
|
+
from . import overlay as ov
|
|
1210
|
+
|
|
1211
|
+
self._require_host("write the HTML morning brief")
|
|
1212
|
+
d = today or _dt.date.today()
|
|
1213
|
+
flt = classification.ClassificationFilter(max_tier=max_tier)
|
|
1214
|
+
|
|
1215
|
+
base = self.brief(max_recent=max_recent, drain=drain, max_tier=max_tier)
|
|
1216
|
+
|
|
1217
|
+
try:
|
|
1218
|
+
stale_links = self.index.stale_wikilink_targets()
|
|
1219
|
+
except Exception:
|
|
1220
|
+
stale_links = []
|
|
1221
|
+
stale_links = [
|
|
1222
|
+
s for s in stale_links
|
|
1223
|
+
if flt.allows((s.get("from") or {}).get("classification"))
|
|
1224
|
+
and (s.get("target") is None or flt.allows((s.get("target") or {}).get("classification")))
|
|
1225
|
+
]
|
|
1226
|
+
|
|
1227
|
+
try:
|
|
1228
|
+
revisit_sample = self.index.revisit_sample(today=d, k=10)
|
|
1229
|
+
except Exception:
|
|
1230
|
+
revisit_sample = []
|
|
1231
|
+
revisit_sample, _ = egress.apply_gate(revisit_sample, max_tier=max_tier)
|
|
1232
|
+
|
|
1233
|
+
open_recs: list[dict[str, Any]] = []
|
|
1234
|
+
try:
|
|
1235
|
+
open_path = config.recommendations_open_path(self.vault)
|
|
1236
|
+
if open_path.exists():
|
|
1237
|
+
open_recs = maint.parse_recommendation_lines(open_path.read_text(encoding="utf-8"))
|
|
1238
|
+
except Exception:
|
|
1239
|
+
open_recs = []
|
|
1240
|
+
|
|
1241
|
+
hot_head: list[str] = []
|
|
1242
|
+
try:
|
|
1243
|
+
hot_path = self._hot_md_path()
|
|
1244
|
+
if hot_path.exists():
|
|
1245
|
+
hot_head = brief_mod.parse_hot_entries(hot_path.read_text(encoding="utf-8"))[-5:]
|
|
1246
|
+
except Exception:
|
|
1247
|
+
hot_head = []
|
|
1248
|
+
|
|
1249
|
+
autoresearch = self._autoresearch_status(d)
|
|
1250
|
+
brand = ov.resolve_brand(self.vault)
|
|
1251
|
+
|
|
1252
|
+
html_text = brief_mod.render_brief_html(
|
|
1253
|
+
base, stale_links=stale_links, revisit_sample=revisit_sample,
|
|
1254
|
+
open_recommendations=open_recs, hot_head=hot_head,
|
|
1255
|
+
autoresearch=autoresearch, brand=brand,
|
|
1256
|
+
)
|
|
1257
|
+
|
|
1258
|
+
out_dir = config.brief_dir(self.vault)
|
|
1259
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
1260
|
+
dated = out_dir / f"brief-{d.isoformat()}.html"
|
|
1261
|
+
latest = out_dir / "brief-latest.html"
|
|
1262
|
+
dated.write_text(html_text, encoding="utf-8")
|
|
1263
|
+
latest.write_text(html_text, encoding="utf-8")
|
|
1264
|
+
return {"path": str(dated), "latest_path": str(latest), "bytes": len(html_text)}
|
|
1265
|
+
|
|
1266
|
+
def digest_html(
|
|
1267
|
+
self, *, days: int = 7, max_tier: str = classification.VM_DEFAULT_MAX_TIER,
|
|
1268
|
+
today: Any = None,
|
|
1269
|
+
) -> dict[str, Any]:
|
|
1270
|
+
"""Render + write the branded HTML weekly digest (AUT-03, ADR-0003
|
|
1271
|
+
Ruling c) to ``.brain/brief/``. HOST-ONLY (writes a file). Notes are
|
|
1272
|
+
already routed through ``egress.apply_gate`` inside ``self.digest()``
|
|
1273
|
+
before the pure renderer (``brain.brief.render_digest_html``) formats
|
|
1274
|
+
them — the renderer performs no I/O."""
|
|
1275
|
+
import datetime as _dt
|
|
1276
|
+
|
|
1277
|
+
from . import brief as brief_mod
|
|
1278
|
+
from . import overlay as ov
|
|
1279
|
+
|
|
1280
|
+
self._require_host("write the HTML weekly digest")
|
|
1281
|
+
d = today or _dt.date.today()
|
|
1282
|
+
|
|
1283
|
+
base = self.digest(days=days, max_tier=max_tier)
|
|
1284
|
+
brand = ov.resolve_brand(self.vault)
|
|
1285
|
+
html_text = brief_mod.render_digest_html(base, brand=brand)
|
|
1286
|
+
|
|
1287
|
+
out_dir = config.brief_dir(self.vault)
|
|
1288
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
1289
|
+
dated = out_dir / f"digest-{d.isoformat()}.html"
|
|
1290
|
+
latest = out_dir / "digest-latest.html"
|
|
1291
|
+
dated.write_text(html_text, encoding="utf-8")
|
|
1292
|
+
latest.write_text(html_text, encoding="utf-8")
|
|
1293
|
+
return {"path": str(dated), "latest_path": str(latest), "bytes": len(html_text)}
|
|
1294
|
+
|
|
1295
|
+
# -- maintenance rituals (CUT-03) --------------------------------------
|
|
1296
|
+
# check / health / curate / integrity / promote-scan + the `maintain`
|
|
1297
|
+
# umbrella. Per routines/manifest.json (disposition field) these are WRITE rituals
|
|
1298
|
+
# (regen index, sign+drain, query the audit chain) -> HOST-broker only,
|
|
1299
|
+
# never runnable under BRAIN_ROLE=vm. Content-listing returns here
|
|
1300
|
+
# (curate/integrity/promote_scan) are UNFILTERED by design (module
|
|
1301
|
+
# contract, see top of file) — brain.cli applies the egress gate before
|
|
1302
|
+
# surfacing, exactly like the read verbs.
|
|
1303
|
+
|
|
1304
|
+
def check(self, *, dry_run: bool = False) -> dict[str, Any]:
|
|
1305
|
+
"""daily-check fold: index reconcile + drain drafts + freshness status
|
|
1306
|
+
(task-disposition.md row 1). ``dry_run`` skips the mutation and reports
|
|
1307
|
+
status only — still a real read against the live index."""
|
|
1308
|
+
from . import maintenance as maint
|
|
1309
|
+
|
|
1310
|
+
self._require_host("run the check ritual")
|
|
1311
|
+
auto_fixed: list[dict[str, Any]] = []
|
|
1312
|
+
action_required: list[dict[str, Any]] = []
|
|
1313
|
+
blocked: list[dict[str, Any]] = []
|
|
1314
|
+
|
|
1315
|
+
sync_res: dict[str, Any] | None = None
|
|
1316
|
+
if not dry_run:
|
|
1317
|
+
sync_res = self.sync(drain=True, publish=False)
|
|
1318
|
+
added = sync_res.get("added", 0)
|
|
1319
|
+
updated = sync_res.get("updated", 0)
|
|
1320
|
+
deleted = sync_res.get("deleted", 0)
|
|
1321
|
+
if added or updated or deleted:
|
|
1322
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
1323
|
+
"sync", str(self.vault),
|
|
1324
|
+
f"index reconciled +{added} ~{updated} -{deleted}"))
|
|
1325
|
+
drain = sync_res.get("drain", {}) or {}
|
|
1326
|
+
if drain.get("promoted"):
|
|
1327
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
1328
|
+
"drain", str(self.capture_inbox_dir()),
|
|
1329
|
+
f"drained {drain['promoted']} pending capture(s)"))
|
|
1330
|
+
details = drain.get("details", {})
|
|
1331
|
+
skipped_list = details.get("skipped", []) if isinstance(details, dict) else []
|
|
1332
|
+
for skip in skipped_list:
|
|
1333
|
+
reason = skip.get("reason", "")
|
|
1334
|
+
draft_path = str(self.capture_inbox_dir() / skip.get("draft", ""))
|
|
1335
|
+
if "no-signing-key" in reason:
|
|
1336
|
+
blocked.append(maint.blocked_item(
|
|
1337
|
+
f"capture draft {skip.get('draft')} could not be drained",
|
|
1338
|
+
"no audit signing key resolved",
|
|
1339
|
+
"signing key configured (Keychain/env), then re-run check"))
|
|
1340
|
+
else:
|
|
1341
|
+
action_required.append(maint.action_required_item(
|
|
1342
|
+
f"capture draft {skip.get('draft')} could not be drained",
|
|
1343
|
+
reason or "unrecognised draft frontmatter",
|
|
1344
|
+
"fix the draft's frontmatter, then re-run check",
|
|
1345
|
+
draft_path))
|
|
1346
|
+
|
|
1347
|
+
status_res = self.status()
|
|
1348
|
+
return {
|
|
1349
|
+
"ritual": "check", "dry_run": dry_run,
|
|
1350
|
+
"sync": sync_res, "status": status_res,
|
|
1351
|
+
"outcomes": maint.build_outcomes(auto_fixed, action_required, blocked),
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
def health(self) -> dict[str, Any]:
|
|
1355
|
+
"""health fold: index/snapshot status + audit-chain verify + a
|
|
1356
|
+
substrate self-test probe (task-disposition.md row 2). Entirely
|
|
1357
|
+
READ-ONLY — safe to run under a caller's --dry-run posture too."""
|
|
1358
|
+
from . import maintenance as maint
|
|
1359
|
+
|
|
1360
|
+
self._require_host("run the health ritual")
|
|
1361
|
+
action_required: list[dict[str, Any]] = []
|
|
1362
|
+
blocked: list[dict[str, Any]] = []
|
|
1363
|
+
|
|
1364
|
+
status_res = self.status()
|
|
1365
|
+
|
|
1366
|
+
audit_res: dict[str, Any] | None = None
|
|
1367
|
+
try:
|
|
1368
|
+
audit_res = self.verify_audit()
|
|
1369
|
+
if audit_res.get("status") not in ("ok", "empty"):
|
|
1370
|
+
action_required.append(maint.action_required_item(
|
|
1371
|
+
f"audit chain status={audit_res.get('status')} "
|
|
1372
|
+
f"({len(audit_res.get('errors', []))} error(s))",
|
|
1373
|
+
"chain tamper/break needs human judgment, never auto-repaired",
|
|
1374
|
+
"inspect the chain errors; re-link from the last-good entry",
|
|
1375
|
+
str(self.audit.log_path) if self.audit else "audit chain"))
|
|
1376
|
+
except Exception as exc:
|
|
1377
|
+
blocked.append(maint.blocked_item(
|
|
1378
|
+
"could not verify the audit chain",
|
|
1379
|
+
f"{type(exc).__name__}: {exc}",
|
|
1380
|
+
"signing key configured (Keychain/env), then re-run health"))
|
|
1381
|
+
|
|
1382
|
+
selftest: dict[str, Any] = {"probe_ok": False}
|
|
1383
|
+
live = status_res.get("live_embedder", {})
|
|
1384
|
+
try:
|
|
1385
|
+
hits = self.hybrid_search("brain", k=1)
|
|
1386
|
+
ix = status_res.get("index", {})
|
|
1387
|
+
selftest = {
|
|
1388
|
+
"probe_ok": True, "result_count": len(hits),
|
|
1389
|
+
"vector_backend": ix.get("vector_backend"),
|
|
1390
|
+
"embed_model": ix.get("embed_model"),
|
|
1391
|
+
# LIVE embedder actually in use (S11) — distinct from the index's
|
|
1392
|
+
# recorded embed_model metadata above.
|
|
1393
|
+
"live_embedder": live.get("model_id"),
|
|
1394
|
+
"hash_fallback": bool(live.get("is_hash_fallback")),
|
|
1395
|
+
}
|
|
1396
|
+
except Exception as exc:
|
|
1397
|
+
blocked.append(maint.blocked_item(
|
|
1398
|
+
"retrieval self-test probe raised",
|
|
1399
|
+
f"{type(exc).__name__}: {exc}",
|
|
1400
|
+
"investigate the embedder/vector-backend, then re-run health"))
|
|
1401
|
+
# A live HashEmbedder on an index built with a real model is a silent
|
|
1402
|
+
# semantic downgrade — surface it as ACTION REQUIRED, not a pass.
|
|
1403
|
+
if live.get("is_hash_fallback") and not live.get("matches_index_metadata"):
|
|
1404
|
+
action_required.append(maint.action_required_item(
|
|
1405
|
+
"live embedder is the non-semantic HashEmbedder but the index was "
|
|
1406
|
+
f"built with {status_res.get('index', {}).get('embed_model')!r}",
|
|
1407
|
+
"retrieval quality is effectively random on this install "
|
|
1408
|
+
"(onnxruntime/tokenizers missing or the e5-small model absent)",
|
|
1409
|
+
"install the 'corporate' extras (onnxruntime + tokenizers) or the "
|
|
1410
|
+
"bundled model; set BRAIN_REQUIRE_REAL_EMBEDDER=1 to fail closed",
|
|
1411
|
+
"brain status --json (live_embedder block)"))
|
|
1412
|
+
|
|
1413
|
+
return {
|
|
1414
|
+
"ritual": "health", "status": status_res, "audit": audit_res,
|
|
1415
|
+
"selftest": selftest,
|
|
1416
|
+
"outcomes": maint.build_outcomes([], action_required, blocked),
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
def _framework_sync_finding(self) -> dict[str, Any] | None:
|
|
1420
|
+
"""HYG-02 (ADR-0003 Ruling 5): the Monday health branch also runs the
|
|
1421
|
+
framework-sync drift audit (canonical .claude/skills/ vs the
|
|
1422
|
+
.agents/skills/ + plugins/ mirrors, plus CLAUDE.md's @AGENTS.md
|
|
1423
|
+
import) — reported as a health finding, NEVER auto-fixed. Best-effort
|
|
1424
|
+
and silent when not applicable: ``tools/framework_sync.py`` is a
|
|
1425
|
+
dev-only script that lives in the profile-a-brain source checkout,
|
|
1426
|
+
not the installed package, so a generic installed vault (no sibling
|
|
1427
|
+
``tools/`` tree) simply has nothing to compare and skips."""
|
|
1428
|
+
from . import maintenance as maint
|
|
1429
|
+
|
|
1430
|
+
repo_root = Path(__file__).resolve().parents[2]
|
|
1431
|
+
fsync_path = repo_root / "tools" / "framework_sync.py"
|
|
1432
|
+
if not fsync_path.is_file():
|
|
1433
|
+
return None
|
|
1434
|
+
import sys as _sys
|
|
1435
|
+
tools_dir = str(repo_root / "tools")
|
|
1436
|
+
inserted = tools_dir not in _sys.path
|
|
1437
|
+
if inserted:
|
|
1438
|
+
_sys.path.insert(0, tools_dir)
|
|
1439
|
+
try:
|
|
1440
|
+
import framework_sync as fsync
|
|
1441
|
+
report = fsync.audit()
|
|
1442
|
+
except Exception as exc:
|
|
1443
|
+
return maint.action_required_item(
|
|
1444
|
+
f"framework-sync drift audit raised: {type(exc).__name__}: {exc}",
|
|
1445
|
+
"could not compare .claude/skills/ against its mirrors",
|
|
1446
|
+
"inspect tools/framework_sync.py, then re-run health",
|
|
1447
|
+
str(fsync_path))
|
|
1448
|
+
finally:
|
|
1449
|
+
if inserted and tools_dir in _sys.path:
|
|
1450
|
+
_sys.path.remove(tools_dir)
|
|
1451
|
+
return maint.framework_sync_finding(report)
|
|
1452
|
+
|
|
1453
|
+
def curate(
|
|
1454
|
+
self, *, dry_run: bool = False, k: int = 50, today: Any = None,
|
|
1455
|
+
) -> dict[str, Any]:
|
|
1456
|
+
"""curation fold (task-disposition.md row 4, extended by AUT-02): the
|
|
1457
|
+
refresh-index sub-step folds to ``sync``; unclassified-notes lint,
|
|
1458
|
+
stale-wikilink-target detection, and an age x centrality revisit
|
|
1459
|
+
sample now run directly against the brain index (ADR-0003 Ruling 5 —
|
|
1460
|
+
this SUPERSEDES the curation skill's previously-documented "no brain
|
|
1461
|
+
equivalent, G3 not shipped" framing for those two checks). Orphan/
|
|
1462
|
+
contradiction/callout lint stay vault-structure overlay tooling with
|
|
1463
|
+
NO brain equivalent (G4 RETIRE) — those still route through
|
|
1464
|
+
``.claude/skills/curation``. UNFILTERED findings — the CLI
|
|
1465
|
+
egress-gates every one before surfacing."""
|
|
1466
|
+
from . import maintenance as maint
|
|
1467
|
+
|
|
1468
|
+
self._require_host("run the curate ritual")
|
|
1469
|
+
auto_fixed: list[dict[str, Any]] = []
|
|
1470
|
+
|
|
1471
|
+
sync_res: dict[str, Any] | None = None
|
|
1472
|
+
if not dry_run:
|
|
1473
|
+
sync_res = self.sync(drain=False)
|
|
1474
|
+
added = sync_res.get("added", 0)
|
|
1475
|
+
updated = sync_res.get("updated", 0)
|
|
1476
|
+
deleted = sync_res.get("deleted", 0)
|
|
1477
|
+
if added or updated or deleted:
|
|
1478
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
1479
|
+
"sync", str(self.vault),
|
|
1480
|
+
f"refresh-index +{added} ~{updated} -{deleted}"))
|
|
1481
|
+
|
|
1482
|
+
unclassified = self.index.unclassified_notes(k=k)
|
|
1483
|
+
stale_links = self.index.stale_wikilink_targets()
|
|
1484
|
+
revisit_sample = self.index.revisit_sample(today=today, k=10)
|
|
1485
|
+
return {
|
|
1486
|
+
"ritual": "curate", "dry_run": dry_run, "sync": sync_res,
|
|
1487
|
+
"unclassified_notes": unclassified, # UNFILTERED
|
|
1488
|
+
"stale_links": stale_links, # UNFILTERED
|
|
1489
|
+
"revisit_sample": revisit_sample, # UNFILTERED
|
|
1490
|
+
"overlay_only_skipped": {
|
|
1491
|
+
"orphans": "vault-structure overlay, no brain equivalent (RETIRE)",
|
|
1492
|
+
"contradictions": "vault-structure overlay, no brain equivalent (RETIRE)",
|
|
1493
|
+
"callouts": "vault-structure overlay, no brain equivalent (RETIRE)",
|
|
1494
|
+
},
|
|
1495
|
+
"auto_fixed": auto_fixed,
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
def integrity(self, *, min_score: float = 0.95, k: int = 5) -> dict[str, Any]:
|
|
1499
|
+
"""integrity-scan fold (task-disposition.md row 3): audit-chain verify
|
|
1500
|
+
+ a corpus-wide near-dup scan directly over the brain vector backend
|
|
1501
|
+
(brain-cli-gaps.md G1 — no SC/MCP round-trip). READ-ONLY. UNFILTERED
|
|
1502
|
+
``near_dup_pairs`` — the CLI egress-gates BOTH members of every pair
|
|
1503
|
+
before surfacing (G1's explicit requirement)."""
|
|
1504
|
+
from . import maintenance as maint
|
|
1505
|
+
|
|
1506
|
+
self._require_host("run the integrity ritual")
|
|
1507
|
+
blocked: list[dict[str, Any]] = []
|
|
1508
|
+
|
|
1509
|
+
audit_res: dict[str, Any] | None = None
|
|
1510
|
+
try:
|
|
1511
|
+
audit_res = self.verify_audit()
|
|
1512
|
+
except Exception as exc:
|
|
1513
|
+
blocked.append(maint.blocked_item(
|
|
1514
|
+
"could not verify the audit chain",
|
|
1515
|
+
f"{type(exc).__name__}: {exc}",
|
|
1516
|
+
"signing key configured (Keychain/env), then re-run integrity"))
|
|
1517
|
+
|
|
1518
|
+
audit_issue: dict[str, Any] | None = None
|
|
1519
|
+
if audit_res and audit_res.get("status") not in ("ok", "empty"):
|
|
1520
|
+
audit_issue = maint.action_required_item(
|
|
1521
|
+
f"audit chain status={audit_res.get('status')} "
|
|
1522
|
+
f"({len(audit_res.get('errors', []))} error(s))",
|
|
1523
|
+
"chain tamper/break needs human judgment, never auto-repaired",
|
|
1524
|
+
"inspect the chain errors; re-link from the last-good entry",
|
|
1525
|
+
str(self.audit.log_path) if self.audit else "audit chain")
|
|
1526
|
+
|
|
1527
|
+
# M-2: `verify()` above only checks linkage + signatures over the
|
|
1528
|
+
# entries PRESENT in the log — deleting the tail (never re-signing)
|
|
1529
|
+
# still verifies "ok". Folding the off-host anchor check in here is
|
|
1530
|
+
# what actually detects a truncated tail (chain_shorter_than_anchor).
|
|
1531
|
+
adir = config.anchor_dir()
|
|
1532
|
+
if adir is None:
|
|
1533
|
+
if audit_issue is None:
|
|
1534
|
+
audit_issue = maint.action_required_item(
|
|
1535
|
+
"no off-host anchor configured (BRAIN_ANCHOR_DIR unset)",
|
|
1536
|
+
"verify() alone gives NO tail-truncation guarantee — "
|
|
1537
|
+
"deleting recent audit-log lines still verifies ok",
|
|
1538
|
+
"run `brain anchor --anchor-dir <off-host-dir>` on a "
|
|
1539
|
+
"schedule, then set BRAIN_ANCHOR_DIR so integrity/maintain "
|
|
1540
|
+
"can check it",
|
|
1541
|
+
str(self.audit.log_path) if self.audit else "audit chain")
|
|
1542
|
+
else:
|
|
1543
|
+
try:
|
|
1544
|
+
anchor_res = self.verify_anchor(adir)
|
|
1545
|
+
except Exception as exc:
|
|
1546
|
+
blocked.append(maint.blocked_item(
|
|
1547
|
+
"could not verify the off-host anchor",
|
|
1548
|
+
f"{type(exc).__name__}: {exc}",
|
|
1549
|
+
"check BRAIN_ANCHOR_DIR is reachable, then re-run integrity"))
|
|
1550
|
+
else:
|
|
1551
|
+
if anchor_res.get("status") == "divergence":
|
|
1552
|
+
audit_issue = maint.action_required_item(
|
|
1553
|
+
f"audit chain diverges from off-host anchor "
|
|
1554
|
+
f"({len(anchor_res.get('divergences', []))} divergence(s))",
|
|
1555
|
+
"tail truncation or a silent rewrite is possible — "
|
|
1556
|
+
"human judgment, never auto-repaired",
|
|
1557
|
+
"inspect anchor divergences; treat the chain as "
|
|
1558
|
+
"compromised from the first divergent entry_count",
|
|
1559
|
+
anchor_res.get("anchor_log", str(adir)))
|
|
1560
|
+
|
|
1561
|
+
try:
|
|
1562
|
+
pairs = self.index.near_dup(min_score=min_score, k=k)
|
|
1563
|
+
except Exception as exc:
|
|
1564
|
+
pairs = []
|
|
1565
|
+
blocked.append(maint.blocked_item(
|
|
1566
|
+
"near-dup scan raised",
|
|
1567
|
+
f"{type(exc).__name__}: {exc}",
|
|
1568
|
+
"investigate the embedder/vector-backend, then re-run integrity"))
|
|
1569
|
+
|
|
1570
|
+
return {
|
|
1571
|
+
"ritual": "integrity", "min_score": min_score,
|
|
1572
|
+
"audit": audit_res, "audit_issue": audit_issue,
|
|
1573
|
+
"near_dup_pairs": pairs, # UNFILTERED
|
|
1574
|
+
"blocked": blocked,
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
def promote_scan(self, *, k: int = 50) -> dict[str, Any]:
|
|
1578
|
+
"""promotion-scan fold (task-disposition.md row 5 — ON-INVOKE triage;
|
|
1579
|
+
promotion itself stays a P-10 human gate). Candidates: ``raw/`` zone
|
|
1580
|
+
sources not yet promoted into a typed ``brain/`` note. UNFILTERED — the
|
|
1581
|
+
CLI egress-gates the candidate list before surfacing."""
|
|
1582
|
+
self._require_host("run the promote-scan ritual")
|
|
1583
|
+
candidates = self.index.bases_query({"zone": "raw"}, k=k)
|
|
1584
|
+
return {
|
|
1585
|
+
"ritual": "promote-scan",
|
|
1586
|
+
"candidates": candidates, # UNFILTERED
|
|
1587
|
+
"pending_drafts": self._count_pending_drafts(),
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
def graphify(
|
|
1591
|
+
self, *, force: bool = False, dry_run: bool = False, today: Any = None,
|
|
1592
|
+
max_tier: str = classification.VM_DEFAULT_MAX_TIER, candidate_limit: int = 20,
|
|
1593
|
+
) -> dict[str, Any]:
|
|
1594
|
+
"""GRF-01: build the derived, non-authoritative discovery graph
|
|
1595
|
+
(ADR-0003 Ruling 6/(a) — supersedes the earlier "documented only"
|
|
1596
|
+
disposition). HOST-ONLY: reads the writable index + vectors, writes
|
|
1597
|
+
runtime artifacts under ``.brain/graph/``.
|
|
1598
|
+
|
|
1599
|
+
Bounded three ways (Ruling a, ground 2): a corpus-manifest DRIFT GATE
|
|
1600
|
+
(``brain.graphify.manifest_unchanged``) skips the rebuild in
|
|
1601
|
+
milliseconds when nothing changed (bypass with ``force``); INFERRED
|
|
1602
|
+
edges reuse vectors ALREADY in the index (never re-embeds); the
|
|
1603
|
+
caller times the build and flags ``action_required`` past the
|
|
1604
|
+
5-minute soft budget (target <=60s at the current corpus scale).
|
|
1605
|
+
|
|
1606
|
+
Publication is ATOMIC (HARDENED:codex): the artifact is built and
|
|
1607
|
+
schema/cap-validated BEFORE anything touches disk; only a validated
|
|
1608
|
+
build replaces the published ``graph.json`` (temp-file + ``os.replace``
|
|
1609
|
+
— atomic on POSIX and Windows same-volume). A build that raises, or
|
|
1610
|
+
fails validation, writes a SEPARATE ``BUILD_FAILED.json`` marker and
|
|
1611
|
+
the published ``graph.json`` (if any) is left completely untouched —
|
|
1612
|
+
a partial/failed build is never mistaken for a valid publish.
|
|
1613
|
+
|
|
1614
|
+
Candidate surfacing is egress-gated HERE (before assembly into either
|
|
1615
|
+
the CLI's own output or a maintain hot-queue entry) — the same
|
|
1616
|
+
doctrine ``graph_expand`` already applies: a withheld note must never
|
|
1617
|
+
leak via the graph surface. The full graph.json artifact itself is
|
|
1618
|
+
NOT per-item gated (a host-only, gitignored, never-published runtime
|
|
1619
|
+
cache — same "egress is the budget, not at-rest" doctrine as the
|
|
1620
|
+
writable index and ``.brain/memory/``)."""
|
|
1621
|
+
import datetime as _dt
|
|
1622
|
+
import json as _json
|
|
1623
|
+
import os as _os
|
|
1624
|
+
import time as _time
|
|
1625
|
+
|
|
1626
|
+
from . import egress
|
|
1627
|
+
from . import graphify as gmod
|
|
1628
|
+
from . import maintenance as maint
|
|
1629
|
+
from .graph import build_graph
|
|
1630
|
+
|
|
1631
|
+
self._require_host("build the graphify discovery graph")
|
|
1632
|
+
# [S04 fix 3/4] `today` is threaded EXPLICITLY (the CLI `--as-of` flag,
|
|
1633
|
+
# which maintain's bounded child passes) — never from an ambient env
|
|
1634
|
+
# var, which would silently leak a stale date into a manual
|
|
1635
|
+
# `brain graphify`. A manual run leaves today=None and uses today's date.
|
|
1636
|
+
d = today if today is not None else _dt.date.today()
|
|
1637
|
+
graph_dir = config.graph_dir(self.vault)
|
|
1638
|
+
graph_dir.mkdir(parents=True, exist_ok=True)
|
|
1639
|
+
manifest_path = config.graph_manifest_path(self.vault)
|
|
1640
|
+
graph_path = config.graph_json_path(self.vault)
|
|
1641
|
+
marker_path = config.graph_build_failed_marker_path(self.vault)
|
|
1642
|
+
|
|
1643
|
+
conn = self.index.conn
|
|
1644
|
+
new_manifest = gmod.corpus_manifest(conn)
|
|
1645
|
+
|
|
1646
|
+
old_state: dict[str, Any] = {}
|
|
1647
|
+
try:
|
|
1648
|
+
old_state = _json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
1649
|
+
except (OSError, ValueError):
|
|
1650
|
+
old_state = {}
|
|
1651
|
+
|
|
1652
|
+
if not force and gmod.manifest_unchanged(old_state, new_manifest):
|
|
1653
|
+
return {
|
|
1654
|
+
"ritual": "graphify", "skipped": "unchanged",
|
|
1655
|
+
"generation": old_state.get("generation"),
|
|
1656
|
+
"built_at": old_state.get("built_at"),
|
|
1657
|
+
"note_count": len(new_manifest),
|
|
1658
|
+
"published": False,
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
t0 = _time.monotonic()
|
|
1662
|
+
try:
|
|
1663
|
+
link_graph = build_graph(conn)
|
|
1664
|
+
built = gmod.build_graph_artifact(conn, self.index.backend, link_graph, today=d)
|
|
1665
|
+
except Exception as exc:
|
|
1666
|
+
marker_path.write_text(_json.dumps({
|
|
1667
|
+
"status": "build_failed", "error": f"{type(exc).__name__}: {exc}",
|
|
1668
|
+
"attempted_at": d.isoformat(),
|
|
1669
|
+
}, indent=2), encoding="utf-8")
|
|
1670
|
+
return {
|
|
1671
|
+
"ritual": "graphify", "status": "build_failed", "published": False,
|
|
1672
|
+
"error": f"{type(exc).__name__}: {exc}", "marker": str(marker_path),
|
|
1673
|
+
}
|
|
1674
|
+
duration = _time.monotonic() - t0
|
|
1675
|
+
|
|
1676
|
+
generation = int(old_state.get("generation") or 0) + 1
|
|
1677
|
+
artifact = {
|
|
1678
|
+
"schema_version": gmod.GRAPH_SCHEMA_VERSION,
|
|
1679
|
+
"generation": generation,
|
|
1680
|
+
"built_at": d.isoformat(),
|
|
1681
|
+
"authoritative": False,
|
|
1682
|
+
"provenance": gmod.PROVENANCE,
|
|
1683
|
+
**built,
|
|
1684
|
+
"build": {
|
|
1685
|
+
"duration_seconds": round(duration, 3),
|
|
1686
|
+
"budget_seconds": gmod.DEFAULT_BUDGET_SECONDS,
|
|
1687
|
+
"action_required_seconds": gmod.ACTION_REQUIRED_SECONDS,
|
|
1688
|
+
"action_required": duration > gmod.ACTION_REQUIRED_SECONDS,
|
|
1689
|
+
},
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
ok, problems = gmod.validate_artifact(artifact)
|
|
1693
|
+
if not ok:
|
|
1694
|
+
marker_path.write_text(_json.dumps({
|
|
1695
|
+
"status": "invalid_artifact", "problems": problems,
|
|
1696
|
+
"attempted_at": d.isoformat(),
|
|
1697
|
+
}, indent=2), encoding="utf-8")
|
|
1698
|
+
return {
|
|
1699
|
+
"ritual": "graphify", "status": "invalid_artifact", "published": False,
|
|
1700
|
+
"problems": problems, "marker": str(marker_path),
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
candidates = gmod.top_candidates(artifact["edges"], limit=candidate_limit)
|
|
1704
|
+
node_lookup = {n["id"]: n for n in artifact["nodes"]}
|
|
1705
|
+
touched_ids = {c["from"] for c in candidates} | {c["to"] for c in candidates}
|
|
1706
|
+
touched_nodes = [node_lookup[i] for i in touched_ids if i in node_lookup]
|
|
1707
|
+
surfaced_nodes, cand_report = egress.apply_gate(touched_nodes, max_tier=max_tier)
|
|
1708
|
+
surfaced_ids = {n["id"] for n in surfaced_nodes}
|
|
1709
|
+
gated_candidates = [
|
|
1710
|
+
c for c in candidates if c["from"] in surfaced_ids and c["to"] in surfaced_ids
|
|
1711
|
+
]
|
|
1712
|
+
|
|
1713
|
+
if dry_run:
|
|
1714
|
+
return {
|
|
1715
|
+
"ritual": "graphify", "dry_run": True, "published": False,
|
|
1716
|
+
"generation": generation, "corpus": artifact["corpus"],
|
|
1717
|
+
"build": artifact["build"], "candidates": gated_candidates,
|
|
1718
|
+
"egress": cand_report,
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
tmp_graph = graph_path.with_suffix(graph_path.suffix + ".tmp")
|
|
1722
|
+
tmp_graph.write_text(_json.dumps(artifact, indent=2, sort_keys=True), encoding="utf-8")
|
|
1723
|
+
_os.replace(tmp_graph, graph_path)
|
|
1724
|
+
marker_path.unlink(missing_ok=True) # a prior failure marker is now stale
|
|
1725
|
+
|
|
1726
|
+
new_state = {"generation": generation, "built_at": d.isoformat(), "notes": new_manifest}
|
|
1727
|
+
tmp_manifest = manifest_path.with_suffix(manifest_path.suffix + ".tmp")
|
|
1728
|
+
tmp_manifest.write_text(_json.dumps(new_state, indent=2, sort_keys=True), encoding="utf-8")
|
|
1729
|
+
_os.replace(tmp_manifest, manifest_path)
|
|
1730
|
+
|
|
1731
|
+
return {
|
|
1732
|
+
"ritual": "graphify", "dry_run": False, "published": True,
|
|
1733
|
+
"generation": generation, "path": str(graph_path),
|
|
1734
|
+
"corpus": artifact["corpus"], "build": artifact["build"],
|
|
1735
|
+
"candidates": gated_candidates, "egress": cand_report,
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
def _graph_status(self) -> dict[str, Any]:
|
|
1739
|
+
"""``brain status`` surfacing of the graphify build's generation/age
|
|
1740
|
+
(GRF-02) — reads the SAME manifest ``graphify()`` writes; never
|
|
1741
|
+
builds, never mutates."""
|
|
1742
|
+
import datetime as _dt
|
|
1743
|
+
import json as _json
|
|
1744
|
+
|
|
1745
|
+
try:
|
|
1746
|
+
state = _json.loads(config.graph_manifest_path(self.vault).read_text(encoding="utf-8"))
|
|
1747
|
+
except (OSError, ValueError):
|
|
1748
|
+
return {"status": "never_built"}
|
|
1749
|
+
built_at = state.get("built_at")
|
|
1750
|
+
age_days = None
|
|
1751
|
+
if built_at:
|
|
1752
|
+
try:
|
|
1753
|
+
age_days = (_dt.date.today() - _dt.date.fromisoformat(built_at)).days
|
|
1754
|
+
except ValueError:
|
|
1755
|
+
age_days = None
|
|
1756
|
+
return {
|
|
1757
|
+
"status": "ok", "generation": state.get("generation"),
|
|
1758
|
+
"built_at": built_at, "age_days": age_days,
|
|
1759
|
+
"note_count": len(state.get("notes") or {}),
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
def _run_bounded_graphify(
|
|
1763
|
+
self, *, force: bool, dry_run: bool, today: Any, state: dict[str, Any],
|
|
1764
|
+
reason: str, builder: Any = None,
|
|
1765
|
+
) -> dict[str, Any]:
|
|
1766
|
+
"""FRESH-01: run a graphify build IN-PROCESS, attempt-bounded — the ONE
|
|
1767
|
+
path both the monthly date-gate FLOOR and the drift-triggered fold
|
|
1768
|
+
route through.
|
|
1769
|
+
|
|
1770
|
+
OWNER DECISION 2026-07-11 (SUPERSEDES the earlier
|
|
1771
|
+
``[HARDENED:codex-verify-r1]`` subprocess-wrapper design): the build
|
|
1772
|
+
runs via ``self.graphify()`` in-process, against THIS BrainCore's OWN
|
|
1773
|
+
index — never a re-invoked ``brain.cli`` child. The subprocess approach
|
|
1774
|
+
was reverted after review because it (a) re-resolved its index from the
|
|
1775
|
+
environment, so an embedded/injected index built the WRONG corpus and
|
|
1776
|
+
unit tests touched the real machine index; (b) needed JSON-over-stdout
|
|
1777
|
+
parsing that could latch onto a JSON-shaped noise line; and (c) split
|
|
1778
|
+
the ``_graphify_drift`` marker across two processes, causing
|
|
1779
|
+
double-counted/clobbered backoff. Its one benefit — a hard KILL of a
|
|
1780
|
+
hypothetical C-extension stall — is traded for simplicity and
|
|
1781
|
+
correctness: a real corpus builds in ~60s, and the attempt-marker below
|
|
1782
|
+
makes a stall non-fatal without a kill.
|
|
1783
|
+
|
|
1784
|
+
Bounding is the ATTEMPT-keyed ``_graphify_drift`` marker + capped
|
|
1785
|
+
exponential backoff (``maintenance.graphify_backoff_days``):
|
|
1786
|
+
- ``last_attempt`` is persisted BEFORE the build (HARDENED correction
|
|
1787
|
+
b), so even a build that hangs (and whose ``maintain`` is later
|
|
1788
|
+
killed) leaves an attempt on disk — the next maintain respects the
|
|
1789
|
+
cooldown and does NOT re-fire within it.
|
|
1790
|
+
- ANY non-publishing, non-skipped, non-preview outcome (an exception, a
|
|
1791
|
+
non-dict, or an in-process ``build_failed``/``invalid_artifact``)
|
|
1792
|
+
bumps ``consecutive_overruns`` and keeps ``build.action_required`` so
|
|
1793
|
+
the alarm layer sees it. ``consecutive_overruns`` resets to 0 only on
|
|
1794
|
+
a build that actually publishes or cleanly skips.
|
|
1795
|
+
- A dry-run is a PREVIEW: returned as-is (neither pass nor fail for
|
|
1796
|
+
backoff); no state is persisted under dry_run.
|
|
1797
|
+
|
|
1798
|
+
``builder`` is a test-only injection point — a callable
|
|
1799
|
+
``(force, dry_run, today) -> result dict`` standing in for
|
|
1800
|
+
``self.graphify`` so tests drive published/skipped/failed/raising
|
|
1801
|
+
outcomes without a real vector build. Defaults to ``self.graphify``
|
|
1802
|
+
at the HOST egress default tier (owner ruling 2026-07-10: the host
|
|
1803
|
+
default is the full vault — graphify's own signature default is the
|
|
1804
|
+
conservative VM tier, which would silently drop hot-queue candidates
|
|
1805
|
+
touching Confidential/Restricted/MNPI notes; review finding [1])."""
|
|
1806
|
+
if builder is not None:
|
|
1807
|
+
build = builder
|
|
1808
|
+
else:
|
|
1809
|
+
def build(*, force: bool, dry_run: bool, today: Any) -> dict[str, Any]:
|
|
1810
|
+
return self.graphify(
|
|
1811
|
+
force=force, dry_run=dry_run, today=today,
|
|
1812
|
+
max_tier=classification.DEFAULT_MAX_TIER)
|
|
1813
|
+
|
|
1814
|
+
# `state` is this process's live dict — the marker in it is
|
|
1815
|
+
# authoritative (in-process design; no cross-process copy to re-read).
|
|
1816
|
+
marker = dict(state.get("_graphify_drift") or {})
|
|
1817
|
+
marker["last_attempt"] = today.isoformat()
|
|
1818
|
+
marker["last_reason"] = reason
|
|
1819
|
+
state["_graphify_drift"] = marker
|
|
1820
|
+
if not dry_run:
|
|
1821
|
+
self._save_maintain_state(state) # ATTEMPT persisted BEFORE the build
|
|
1822
|
+
|
|
1823
|
+
def _bump_and_persist() -> None:
|
|
1824
|
+
marker["consecutive_overruns"] = int(marker.get("consecutive_overruns", 0)) + 1
|
|
1825
|
+
marker["last_overrun"] = today.isoformat()
|
|
1826
|
+
state["_graphify_drift"] = marker
|
|
1827
|
+
if not dry_run:
|
|
1828
|
+
self._save_maintain_state(state)
|
|
1829
|
+
|
|
1830
|
+
try:
|
|
1831
|
+
result = build(force=force, dry_run=dry_run, today=today)
|
|
1832
|
+
except Exception as exc: # noqa: BLE001 — a build error is a failure, never propagate
|
|
1833
|
+
_bump_and_persist()
|
|
1834
|
+
return {"ritual": "graphify", "invoked": True, "published": False,
|
|
1835
|
+
"reason": reason, "status": "build_error",
|
|
1836
|
+
"error": f"{type(exc).__name__}: {exc}",
|
|
1837
|
+
"build": {"action_required": True}}
|
|
1838
|
+
|
|
1839
|
+
if not isinstance(result, dict):
|
|
1840
|
+
_bump_and_persist()
|
|
1841
|
+
return {"ritual": "graphify", "invoked": True, "published": False,
|
|
1842
|
+
"reason": reason, "status": "bad_result",
|
|
1843
|
+
"build": {"action_required": True}}
|
|
1844
|
+
result["invoked"] = True
|
|
1845
|
+
result["reason"] = reason
|
|
1846
|
+
|
|
1847
|
+
# A dry-run is a PREVIEW (published False by design): neither pass nor
|
|
1848
|
+
# fail for backoff, and no state persisted under dry_run.
|
|
1849
|
+
if dry_run or result.get("dry_run"):
|
|
1850
|
+
return result
|
|
1851
|
+
if result.get("published") or result.get("skipped"):
|
|
1852
|
+
marker["consecutive_overruns"] = 0
|
|
1853
|
+
marker["last_success"] = today.isoformat()
|
|
1854
|
+
state["_graphify_drift"] = marker
|
|
1855
|
+
self._save_maintain_state(state)
|
|
1856
|
+
return result
|
|
1857
|
+
|
|
1858
|
+
# published False, not skipped, not a preview: an in-process
|
|
1859
|
+
# build_failed/invalid_artifact — a failure for backoff/escalation.
|
|
1860
|
+
_bump_and_persist()
|
|
1861
|
+
result.setdefault("build", {})["action_required"] = True
|
|
1862
|
+
result.setdefault("status", "build_not_published")
|
|
1863
|
+
return result
|
|
1864
|
+
|
|
1865
|
+
def _run_golden_probe(
|
|
1866
|
+
self, *, probes_path: Path, timeout_seconds: int | None = None,
|
|
1867
|
+
codex_call: Any = None, self_call: Any = None,
|
|
1868
|
+
) -> dict[str, Any]:
|
|
1869
|
+
"""WD-03: the Sunday cross-family golden-probe EXECUTION. Codex (the
|
|
1870
|
+
family that did NOT build the retrieval engine) shells the SAME
|
|
1871
|
+
``brain`` CLI the probes exercise — this is cross-family EXECUTION of
|
|
1872
|
+
a deterministic scorer, correction 5: NEVER "independent
|
|
1873
|
+
verification" / "Codex grades retrieval". A shared retrieval bug is
|
|
1874
|
+
invisible to both invokers; only the INVOKER differs, not the
|
|
1875
|
+
measurement.
|
|
1876
|
+
|
|
1877
|
+
Codex runs READ-ONLY (``--sandbox read-only``) and ONLY executes the
|
|
1878
|
+
scorer; it never asserts a decision. Any parse/shape/range failure,
|
|
1879
|
+
non-zero codex exit, or timeout falls back to running the probe
|
|
1880
|
+
runner directly (subprocess) and returns ``{"runner": "self",
|
|
1881
|
+
"degraded": True}`` — NEVER a codex-sourced score from unvalidated
|
|
1882
|
+
output (exit-0-with-garbage is the trap this two-stage validation
|
|
1883
|
+
exists for).
|
|
1884
|
+
|
|
1885
|
+
``codex_call``/``self_call`` are test-only injection points —
|
|
1886
|
+
``(argv: list[str], timeout: int) -> (returncode, stdout, stderr)`` —
|
|
1887
|
+
standing in for the two subprocess invocations so tests never spawn a
|
|
1888
|
+
real codex (or python) child process. Production default shells out
|
|
1889
|
+
via ``subprocess.run``."""
|
|
1890
|
+
import json as _json
|
|
1891
|
+
import shlex as _shlex
|
|
1892
|
+
import subprocess as _subprocess
|
|
1893
|
+
import sys as _sys
|
|
1894
|
+
|
|
1895
|
+
from . import maintenance as maint
|
|
1896
|
+
|
|
1897
|
+
timeout = timeout_seconds if timeout_seconds is not None else maint.golden_codex_timeout_seconds()
|
|
1898
|
+
# PATH-independent brain command (review fix [2]): under launchd's
|
|
1899
|
+
# minimal PATH a bare `brain` isn't resolvable, so pin golden_probe to
|
|
1900
|
+
# the SAME interpreter running maintain via `-m brain.cli`. Used by
|
|
1901
|
+
# BOTH the codex-exec prompt's runner and the self-run fallback below.
|
|
1902
|
+
brain_cmd = _shlex.join([_sys.executable, "-m", "brain.cli"])
|
|
1903
|
+
|
|
1904
|
+
def _default_call(argv: list[str], to: int) -> tuple[int, str, str]:
|
|
1905
|
+
try:
|
|
1906
|
+
proc = _subprocess.run(argv, capture_output=True, text=True, timeout=to)
|
|
1907
|
+
return proc.returncode, proc.stdout, proc.stderr
|
|
1908
|
+
except _subprocess.TimeoutExpired as exc:
|
|
1909
|
+
return -1, "", f"timeout after {to}s: {exc}"
|
|
1910
|
+
except OSError as exc: # e.g. `codex` not on PATH
|
|
1911
|
+
return -1, "", f"{type(exc).__name__}: {exc}"
|
|
1912
|
+
|
|
1913
|
+
codex_call = codex_call or _default_call
|
|
1914
|
+
self_call = self_call or _default_call
|
|
1915
|
+
|
|
1916
|
+
# Pass the ABSOLUTE interpreter (has `brain` importable) so BOTH the
|
|
1917
|
+
# codex prompt's outer `-m brain.golden_probe` AND its inner
|
|
1918
|
+
# `--brain-cmd` are PATH-independent (re-review: a bare outer `python3`
|
|
1919
|
+
# ModuleNotFound'd on uv/pipx installs).
|
|
1920
|
+
prompt = maint.build_codex_golden_prompt(probes_path, Path(self.vault), _sys.executable)
|
|
1921
|
+
codex_argv = [
|
|
1922
|
+
"codex", "exec", "--skip-git-repo-check", "--sandbox", "read-only",
|
|
1923
|
+
"-C", str(self.vault), "--json", prompt,
|
|
1924
|
+
]
|
|
1925
|
+
codex_error: str | None = None
|
|
1926
|
+
rc, stdout, stderr = codex_call(codex_argv, timeout)
|
|
1927
|
+
if rc == 0:
|
|
1928
|
+
final_text = maint.parse_codex_final_message(stdout)
|
|
1929
|
+
if final_text is None:
|
|
1930
|
+
codex_error = "no agent_message event in codex --json stream"
|
|
1931
|
+
else:
|
|
1932
|
+
try:
|
|
1933
|
+
doc: Any = _json.loads(final_text)
|
|
1934
|
+
except ValueError as exc:
|
|
1935
|
+
codex_error = f"final message is not JSON: {exc}"
|
|
1936
|
+
doc = None
|
|
1937
|
+
if codex_error is None:
|
|
1938
|
+
shape_err = maint.validate_golden_probe_doc(doc)
|
|
1939
|
+
if shape_err:
|
|
1940
|
+
codex_error = f"invalid golden-probe doc: {shape_err}"
|
|
1941
|
+
else:
|
|
1942
|
+
return {
|
|
1943
|
+
"score": doc.get("score"), "disposition": doc.get("disposition"),
|
|
1944
|
+
"exit_code": doc.get("exit_code"), "runner": "codex", "degraded": False,
|
|
1945
|
+
}
|
|
1946
|
+
else:
|
|
1947
|
+
codex_error = f"codex exec exited {rc}: {(stderr or stdout or '').strip()[:300]}"
|
|
1948
|
+
|
|
1949
|
+
# -- fall back to the self-run (subprocess, same probes file/vault) --
|
|
1950
|
+
self_argv = [
|
|
1951
|
+
_sys.executable, "-m", "brain.golden_probe", str(probes_path),
|
|
1952
|
+
"--vault", str(self.vault), "--brain-cmd", brain_cmd,
|
|
1953
|
+
]
|
|
1954
|
+
rc2, stdout2, stderr2 = self_call(self_argv, timeout)
|
|
1955
|
+
try:
|
|
1956
|
+
doc2: Any = _json.loads(stdout2)
|
|
1957
|
+
except ValueError:
|
|
1958
|
+
doc2 = None
|
|
1959
|
+
shape_err2 = (maint.validate_golden_probe_doc(doc2) if doc2 is not None
|
|
1960
|
+
else f"non-JSON self-run output (rc={rc2}): "
|
|
1961
|
+
f"{(stderr2 or stdout2 or '').strip()[:300]}")
|
|
1962
|
+
if shape_err2:
|
|
1963
|
+
return {
|
|
1964
|
+
"score": None, "disposition": "transient", "exit_code": maint.GOLDEN_EXIT_TRANSIENT,
|
|
1965
|
+
"runner": "self", "degraded": True,
|
|
1966
|
+
"error": f"self-run also failed: {shape_err2} (codex: {codex_error})",
|
|
1967
|
+
}
|
|
1968
|
+
return {
|
|
1969
|
+
"score": doc2.get("score"), "disposition": doc2.get("disposition"),
|
|
1970
|
+
"exit_code": doc2.get("exit_code"), "runner": "self", "degraded": True,
|
|
1971
|
+
"codex_error": codex_error,
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
# -- maintain: lock + state-file helpers (ADR-0003 Ruling 5/d, HARDENED:codex) --
|
|
1975
|
+
def _acquire_maintain_lock(
|
|
1976
|
+
self, lock_path: Path, *, stale_after_seconds: float = 2 * 3600,
|
|
1977
|
+
) -> dict[str, Any] | None:
|
|
1978
|
+
"""Best-effort single-runner lock. Returns the lock-info dict on
|
|
1979
|
+
success, or ``None`` if another live-looking ``maintain`` run holds it
|
|
1980
|
+
(caller should skip the run, never block/wait). A lock older than
|
|
1981
|
+
``stale_after_seconds`` — far beyond the ADR's ~60s/5min graphify
|
|
1982
|
+
budget — is treated as an abandoned crash and broken automatically."""
|
|
1983
|
+
import json as _json
|
|
1984
|
+
import os as _os
|
|
1985
|
+
import time as _time
|
|
1986
|
+
|
|
1987
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1988
|
+
info = {"pid": _os.getpid(), "started": _time.time()}
|
|
1989
|
+
try:
|
|
1990
|
+
fd = _os.open(str(lock_path), _os.O_CREAT | _os.O_EXCL | _os.O_WRONLY)
|
|
1991
|
+
with _os.fdopen(fd, "w") as fh:
|
|
1992
|
+
fh.write(_json.dumps(info))
|
|
1993
|
+
return info
|
|
1994
|
+
except FileExistsError:
|
|
1995
|
+
pass
|
|
1996
|
+
existing = self._read_maintain_lock(lock_path)
|
|
1997
|
+
started = existing.get("started")
|
|
1998
|
+
if isinstance(started, (int, float)) and (_time.time() - started) > stale_after_seconds:
|
|
1999
|
+
lock_path.unlink(missing_ok=True)
|
|
2000
|
+
return self._acquire_maintain_lock(lock_path, stale_after_seconds=stale_after_seconds)
|
|
2001
|
+
return None
|
|
2002
|
+
|
|
2003
|
+
def _read_maintain_lock(self, lock_path: Path) -> dict[str, Any]:
|
|
2004
|
+
import json as _json
|
|
2005
|
+
|
|
2006
|
+
try:
|
|
2007
|
+
return _json.loads(lock_path.read_text(encoding="utf-8"))
|
|
2008
|
+
except (OSError, ValueError):
|
|
2009
|
+
return {}
|
|
2010
|
+
|
|
2011
|
+
def _release_maintain_lock(self, lock_path: Path) -> None:
|
|
2012
|
+
lock_path.unlink(missing_ok=True)
|
|
2013
|
+
|
|
2014
|
+
def _load_maintain_state(self) -> dict[str, Any]:
|
|
2015
|
+
import json as _json
|
|
2016
|
+
|
|
2017
|
+
path = config.maintain_state_path(self.vault)
|
|
2018
|
+
try:
|
|
2019
|
+
state = _json.loads(path.read_text(encoding="utf-8"))
|
|
2020
|
+
except (OSError, ValueError):
|
|
2021
|
+
return {}
|
|
2022
|
+
return state if isinstance(state, dict) else {}
|
|
2023
|
+
|
|
2024
|
+
def _save_maintain_state(self, state: dict[str, Any]) -> None:
|
|
2025
|
+
"""Atomic write (tmp + replace) so a crash mid-write never corrupts
|
|
2026
|
+
the file the session-start hook and ``brain status`` both read."""
|
|
2027
|
+
import json as _json
|
|
2028
|
+
import os as _os
|
|
2029
|
+
|
|
2030
|
+
path = config.maintain_state_path(self.vault)
|
|
2031
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
2032
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
2033
|
+
tmp.write_text(_json.dumps(state, indent=2, sort_keys=True), encoding="utf-8")
|
|
2034
|
+
_os.replace(tmp, path)
|
|
2035
|
+
|
|
2036
|
+
def _maintain_heartbeat_summary(self, today: Any = None) -> dict[str, Any]:
|
|
2037
|
+
"""``brain status`` surfacing (HARDENED:premortem) — a heartbeat older
|
|
2038
|
+
than 48h on the ``daily`` branch, or any branch with >=2 consecutive
|
|
2039
|
+
failures, is flagged. Reads the SAME file `maintain` writes and the
|
|
2040
|
+
session-start hook already reads (one file, two consumers)."""
|
|
2041
|
+
import datetime as _dt
|
|
2042
|
+
|
|
2043
|
+
state = self._load_maintain_state()
|
|
2044
|
+
if not state:
|
|
2045
|
+
return {"status": "no-record", "note": "no maintain-state.json yet — brain maintain has not run"}
|
|
2046
|
+
today = today or _dt.date.today()
|
|
2047
|
+
branches: dict[str, Any] = {}
|
|
2048
|
+
stale, repeated_failures = [], []
|
|
2049
|
+
for branch, entry in state.items():
|
|
2050
|
+
if str(branch).startswith("_"):
|
|
2051
|
+
continue # [S04 fix 7] marker, not a branch (mirrors maintain()'s own filter)
|
|
2052
|
+
if not isinstance(entry, dict):
|
|
2053
|
+
continue
|
|
2054
|
+
last_run = entry.get("last_run")
|
|
2055
|
+
age_hours: float | None = None
|
|
2056
|
+
if last_run:
|
|
2057
|
+
try:
|
|
2058
|
+
age_hours = (today - _dt.date.fromisoformat(last_run)).days * 24
|
|
2059
|
+
except ValueError:
|
|
2060
|
+
age_hours = None
|
|
2061
|
+
branches[branch] = {
|
|
2062
|
+
"last_run": last_run,
|
|
2063
|
+
"last_attempt": entry.get("last_attempt"),
|
|
2064
|
+
"status": entry.get("status"),
|
|
2065
|
+
"consecutive_failures": entry.get("consecutive_failures", 0),
|
|
2066
|
+
"age_hours": age_hours,
|
|
2067
|
+
}
|
|
2068
|
+
if branch == "daily" and (entry.get("failed") or (age_hours is not None and age_hours > 48)):
|
|
2069
|
+
stale.append(branch)
|
|
2070
|
+
if int(entry.get("consecutive_failures", 0)) >= 2:
|
|
2071
|
+
repeated_failures.append(branch)
|
|
2072
|
+
overall = "stale" if stale else ("repeated_failures" if repeated_failures else "ok")
|
|
2073
|
+
return {
|
|
2074
|
+
"status": overall, "stale_branches": stale,
|
|
2075
|
+
"repeated_failure_branches": repeated_failures, "branches": branches,
|
|
2076
|
+
}
|
|
2077
|
+
|
|
2078
|
+
def _hot_md_path(self) -> Path:
|
|
2079
|
+
return config.memory_dir(self.vault) / "hot.md"
|
|
2080
|
+
|
|
2081
|
+
def _append_hot_once(self, key: str, entry_md: str) -> bool:
|
|
2082
|
+
"""Append ``entry_md`` to ``hot.md`` guarded by an idempotency-key
|
|
2083
|
+
HTML comment; a no-op (returns ``False``) if the key is already
|
|
2084
|
+
present — the per-branch/per-run-date idempotency guard for every
|
|
2085
|
+
scheduled fold that queues a hot-queue entry (HARDENED:codex)."""
|
|
2086
|
+
path = self._hot_md_path()
|
|
2087
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
2088
|
+
marker = f"<!-- idempotency-key: {key} -->"
|
|
2089
|
+
existing = path.read_text(encoding="utf-8") if path.exists() else ""
|
|
2090
|
+
if marker in existing:
|
|
2091
|
+
return False
|
|
2092
|
+
with path.open("a", encoding="utf-8") as fh:
|
|
2093
|
+
if existing and not existing.endswith("\n"):
|
|
2094
|
+
fh.write("\n")
|
|
2095
|
+
fh.write(marker + "\n" + entry_md.rstrip("\n") + "\n\n")
|
|
2096
|
+
return True
|
|
2097
|
+
|
|
2098
|
+
def _daily_note_fold(self, today: Any, brief_result: Any = None) -> dict[str, Any]:
|
|
2099
|
+
"""Daily fold: create today's ``type: daily`` note exactly once per day.
|
|
2100
|
+
|
|
2101
|
+
Second-brain parity with the old Obsidian ``80 Daily`` habit, done the
|
|
2102
|
+
native way — a dated note with the standard template sections, seeded
|
|
2103
|
+
from the morning brief already built this run. Idempotent via
|
|
2104
|
+
``self.get`` (never a second copy). Host verb: ``capture`` signs +
|
|
2105
|
+
indexes. Defaults to Confidential (a personal work log carries deal
|
|
2106
|
+
detail, matching the Daily-zone migration floor)."""
|
|
2107
|
+
note_id = f"daily-{today.isoformat()}"
|
|
2108
|
+
if self.get(note_id) is not None:
|
|
2109
|
+
return {"created": False, "id": note_id}
|
|
2110
|
+
lines = [f"# {today.isoformat()} ({today.strftime('%A')})", "", "## Session Summary"]
|
|
2111
|
+
try:
|
|
2112
|
+
for n in (brief_result or {}).get("recent_notes") or []:
|
|
2113
|
+
title = (n.get("title") or n.get("id") or "").strip() if isinstance(n, dict) else str(n).strip()
|
|
2114
|
+
if title:
|
|
2115
|
+
lines.append(f"- {title}")
|
|
2116
|
+
except Exception:
|
|
2117
|
+
pass # seeding is best-effort; the empty note is still valid
|
|
2118
|
+
lines += ["", "## Work Done", "", "## Open Threads", "", "## Next Session", ""]
|
|
2119
|
+
body = "\n".join(lines).rstrip() + "\n"
|
|
2120
|
+
self.capture(body, note_id=note_id, note_type="daily",
|
|
2121
|
+
classification="Confidential",
|
|
2122
|
+
reason="brain-nightly daily-note fold")
|
|
2123
|
+
return {"created": True, "id": note_id}
|
|
2124
|
+
|
|
2125
|
+
def _recommendations_aging_fold(self, today: Any) -> dict[str, Any]:
|
|
2126
|
+
"""MEM-03 unconditional daily fold: surface any open recommendation
|
|
2127
|
+
older than the aging threshold into ``hot.md``, exactly once per
|
|
2128
|
+
recommendation (idempotent both at the JSONL-status level and via the
|
|
2129
|
+
hot.md idempotency key)."""
|
|
2130
|
+
from . import maintenance as maint
|
|
2131
|
+
|
|
2132
|
+
open_path = config.recommendations_open_path(self.vault)
|
|
2133
|
+
if not open_path.exists():
|
|
2134
|
+
return {"scanned": 0, "surfaced": 0, "appended_to_hot": 0}
|
|
2135
|
+
|
|
2136
|
+
entries = maint.parse_recommendation_lines(open_path.read_text(encoding="utf-8"))
|
|
2137
|
+
updated, newly = maint.recommendations_aging_scan(entries, today)
|
|
2138
|
+
appended = 0
|
|
2139
|
+
for entry in newly:
|
|
2140
|
+
key = f"rec:{entry.get('id')}"
|
|
2141
|
+
entry_md = maint.render_recommendation_hot_entry(entry, today)
|
|
2142
|
+
if self._append_hot_once(key, entry_md):
|
|
2143
|
+
appended += 1
|
|
2144
|
+
if newly:
|
|
2145
|
+
open_path.write_text(maint.render_recommendation_lines(updated), encoding="utf-8")
|
|
2146
|
+
return {"scanned": len(entries), "surfaced": len(newly), "appended_to_hot": appended}
|
|
2147
|
+
|
|
2148
|
+
def maintain(
|
|
2149
|
+
self, *, dry_run: bool = False, today: Any = None,
|
|
2150
|
+
min_score: float = 0.95, near_dup_k: int = 5,
|
|
2151
|
+
graphify_runner: Any = None, golden_runner: Any = None,
|
|
2152
|
+
) -> dict[str, Any]:
|
|
2153
|
+
"""The umbrella — THE single sanctioned host task (``brain-nightly``,
|
|
2154
|
+
persistence-budget.md THE LOCK). Runs ``sync --publish`` + ``brief`` +
|
|
2155
|
+
the recommendations-aging fold (skipped under ``dry_run`` — no
|
|
2156
|
+
mutation, no signing), then the date-gated branches: Mon->health,
|
|
2157
|
+
Tue->integrity, Sun->digest (+curate's stale-link/revisit scan
|
|
2158
|
+
+promote-scan, AUT-02), 1st-of-month->graphify (ADR-0003 Ruling 6/(a):
|
|
2159
|
+
a REAL, bounded graph build — drift-gated, embedding-reuse, wall-clock
|
|
2160
|
+
budgeted; this SUPERSEDES the earlier "documented only" disposition).
|
|
2161
|
+
``health``/``integrity`` are READ-ONLY by construction, so they run
|
|
2162
|
+
for REAL even under ``--dry-run`` — only the mutating/signing half is
|
|
2163
|
+
skipped.
|
|
2164
|
+
|
|
2165
|
+
ADR-0003 Ruling d/HARDENED:codex: a single-runner lock skips (never
|
|
2166
|
+
blocks) a concurrent run; branch due-ness reads
|
|
2167
|
+
``.brain/maintain-state.json`` (due-since-last-run catch-up, not
|
|
2168
|
+
calendar-day-only); each branch runs in its own try/except so one
|
|
2169
|
+
crash never aborts the rest of the run, and a branch's marker only
|
|
2170
|
+
advances to ``today`` on SUCCESS — a crash leaves it due next time,
|
|
2171
|
+
safely, because every branch (and every hot-queue write) is
|
|
2172
|
+
idempotent. HOST-broker.
|
|
2173
|
+
|
|
2174
|
+
FRESH-01 (2026-07-11): the 1st-of-month graphify date-gate is a
|
|
2175
|
+
FLOOR, not a gate — the daily fold ALSO measures corpus drift since
|
|
2176
|
+
the last build (``maintenance.graphify_drift``) and fires the same
|
|
2177
|
+
bounded build early once drift crosses ``BRAIN_GRAPHIFY_DRIFT_PCT``
|
|
2178
|
+
(default 15%) and its own attempt-keyed cooldown
|
|
2179
|
+
(``BRAIN_GRAPHIFY_COOLDOWN_DAYS``, default 2 days, capped
|
|
2180
|
+
exponential backoff on overruns) has elapsed. Both the monthly floor
|
|
2181
|
+
and the drift trigger execute through ONE attempt-bounded path
|
|
2182
|
+
(``_run_bounded_graphify``) that runs ``self.graphify()`` IN-PROCESS
|
|
2183
|
+
against this BrainCore's own index (owner decision 2026-07-11,
|
|
2184
|
+
superseding the earlier subprocess wrapper — see
|
|
2185
|
+
``_run_bounded_graphify``). ``graphify_runner`` is test-only dependency
|
|
2186
|
+
injection: a ``(force, dry_run, today) -> result dict`` builder that
|
|
2187
|
+
stands in for ``self.graphify`` — production leaves it ``None``.
|
|
2188
|
+
|
|
2189
|
+
WD-03 (2026-07-12): Sun->golden — cross-family EXECUTION (never
|
|
2190
|
+
"verification") of the WD-02 golden-probe scorer via ``codex exec``
|
|
2191
|
+
(read-only, validated, self-run-fallback on any failure — see
|
|
2192
|
+
``_run_golden_probe``), gated by its own ``_golden_attempt``
|
|
2193
|
+
next-retry marker so a transient failure backs off instead of
|
|
2194
|
+
re-invoking codex every hourly run. ``golden_runner`` is test-only
|
|
2195
|
+
dependency injection: a ``(probes_path) -> result dict`` callable
|
|
2196
|
+
standing in for ``self._run_golden_probe`` — production leaves it
|
|
2197
|
+
``None``."""
|
|
2198
|
+
from . import maintenance as maint
|
|
2199
|
+
import datetime as _dt
|
|
2200
|
+
import os as _os
|
|
2201
|
+
|
|
2202
|
+
self._require_host("run the maintain umbrella")
|
|
2203
|
+
d = today or _dt.date.today()
|
|
2204
|
+
|
|
2205
|
+
lock_path = config.maintain_lock_path(self.vault)
|
|
2206
|
+
lock_info = self._acquire_maintain_lock(lock_path)
|
|
2207
|
+
if lock_info is None:
|
|
2208
|
+
held = self._read_maintain_lock(lock_path)
|
|
2209
|
+
return {
|
|
2210
|
+
"ritual": "maintain", "dry_run": dry_run, "date": d.isoformat(),
|
|
2211
|
+
"skipped": "locked",
|
|
2212
|
+
"note": f"another maintain run holds the lock (pid={held.get('pid')}, "
|
|
2213
|
+
f"started={held.get('started')}) — skipping this run",
|
|
2214
|
+
"outcomes": maint.build_outcomes(),
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
try:
|
|
2218
|
+
state = self._load_maintain_state()
|
|
2219
|
+
last_runs = {
|
|
2220
|
+
k: (v.get("last_run") if isinstance(v, dict) else v)
|
|
2221
|
+
for k, v in state.items() if not str(k).startswith("_")
|
|
2222
|
+
}
|
|
2223
|
+
branches = maint.maintain_branches(d, last_runs=last_runs)
|
|
2224
|
+
|
|
2225
|
+
results: dict[str, Any] = {}
|
|
2226
|
+
auto_fixed: list[dict[str, Any]] = []
|
|
2227
|
+
action_required: list[dict[str, Any]] = []
|
|
2228
|
+
blocked: list[dict[str, Any]] = []
|
|
2229
|
+
|
|
2230
|
+
def _mark(branch: str, ok: bool, error: str | None = None) -> None:
|
|
2231
|
+
if dry_run:
|
|
2232
|
+
return
|
|
2233
|
+
prev = state.get(branch) if isinstance(state.get(branch), dict) else {}
|
|
2234
|
+
entry = dict(prev)
|
|
2235
|
+
entry["last_attempt"] = d.isoformat()
|
|
2236
|
+
if ok:
|
|
2237
|
+
entry["last_run"] = d.isoformat()
|
|
2238
|
+
entry["status"] = "ok"
|
|
2239
|
+
entry["failed"] = False
|
|
2240
|
+
entry["consecutive_failures"] = 0
|
|
2241
|
+
entry.pop("error", None)
|
|
2242
|
+
else:
|
|
2243
|
+
entry["status"] = "failed"
|
|
2244
|
+
entry["failed"] = True
|
|
2245
|
+
entry["consecutive_failures"] = int(prev.get("consecutive_failures", 0)) + 1
|
|
2246
|
+
entry["error"] = error
|
|
2247
|
+
state[branch] = entry
|
|
2248
|
+
|
|
2249
|
+
# -- unconditional daily work (sync/drain/publish/brief + recs) --
|
|
2250
|
+
if dry_run:
|
|
2251
|
+
results["status"] = self.status()
|
|
2252
|
+
else:
|
|
2253
|
+
# WSP-01 workspace sweep — BEFORE sync, so the ingest drain
|
|
2254
|
+
# inside sync picks up what the sweep just staged (settled
|
|
2255
|
+
# workspace files gain their lifecycle in the same nightly:
|
|
2256
|
+
# sweep -> inbox -> ingest -> raw/ -> index+embed -> snapshot).
|
|
2257
|
+
# No-op unless $BRAIN_WORKSPACE_SWEEP_DIRS is configured.
|
|
2258
|
+
sweep_dirs, sweep_age = maint.workspace_sweep_config()
|
|
2259
|
+
if sweep_dirs:
|
|
2260
|
+
try:
|
|
2261
|
+
sweep_res = maint.sweep_workspace(
|
|
2262
|
+
sweep_dirs, self.vault / "inbox", sweep_age)
|
|
2263
|
+
results["workspace_sweep"] = sweep_res
|
|
2264
|
+
if sweep_res["swept"]:
|
|
2265
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2266
|
+
"workspace-sweep", str(self.vault / "inbox"),
|
|
2267
|
+
f"swept {len(sweep_res['swept'])} settled "
|
|
2268
|
+
f"workspace file(s) into inbox/ "
|
|
2269
|
+
f"(age>{sweep_age}d)"))
|
|
2270
|
+
except Exception as exc:
|
|
2271
|
+
blocked.append(maint.blocked_item(
|
|
2272
|
+
f"workspace sweep failed: {exc}",
|
|
2273
|
+
"filesystem", "next maintain run"))
|
|
2274
|
+
try:
|
|
2275
|
+
# First pass WITHOUT publish: the self-organization folds
|
|
2276
|
+
# below mutate metadata/paths, and the snapshot must carry
|
|
2277
|
+
# their result — publish happens in the second pass.
|
|
2278
|
+
sync_res = self.sync(drain=True, publish=False)
|
|
2279
|
+
results["sync"] = sync_res
|
|
2280
|
+
added = sync_res.get("added", 0)
|
|
2281
|
+
updated = sync_res.get("updated", 0)
|
|
2282
|
+
deleted = sync_res.get("deleted", 0)
|
|
2283
|
+
if added or updated or deleted:
|
|
2284
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2285
|
+
"sync", str(self.vault),
|
|
2286
|
+
f"index reconciled +{added} ~{updated} -{deleted}"))
|
|
2287
|
+
drain = sync_res.get("drain", {}) or {}
|
|
2288
|
+
if drain.get("promoted"):
|
|
2289
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2290
|
+
"drain", str(self.capture_inbox_dir()),
|
|
2291
|
+
f"drained {drain['promoted']} pending capture(s)"))
|
|
2292
|
+
|
|
2293
|
+
# -- self-organization folds (owner decision 2026-07-11:
|
|
2294
|
+
# metadata, versioning, PARA and navigation are automatic,
|
|
2295
|
+
# never user-gated). Each fold is independent — one
|
|
2296
|
+
# failure never aborts the others or the publish.
|
|
2297
|
+
try:
|
|
2298
|
+
vres = maint.auto_version_chains(self)
|
|
2299
|
+
results["version_chains"] = vres
|
|
2300
|
+
if vres["chained"]:
|
|
2301
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2302
|
+
"version-chain", str(self.vault),
|
|
2303
|
+
f"stamped {len(vres['chained'])} supersession "
|
|
2304
|
+
f"link(s) across explicit version families"))
|
|
2305
|
+
for fam in vres["skipped_conflict"]:
|
|
2306
|
+
action_required.append(maint.action_required_item(
|
|
2307
|
+
f"version family '{fam}' has a manual chain that "
|
|
2308
|
+
f"disagrees with the computed order",
|
|
2309
|
+
"auto-chaining never overrides a human supersede",
|
|
2310
|
+
"inspect the family and fix the chain with "
|
|
2311
|
+
"`brain supersede` if the manual link is wrong",
|
|
2312
|
+
fam))
|
|
2313
|
+
except Exception as exc:
|
|
2314
|
+
blocked.append(maint.blocked_item(
|
|
2315
|
+
f"auto version-chain fold failed: {exc}",
|
|
2316
|
+
"index/write path", "next maintain run"))
|
|
2317
|
+
try:
|
|
2318
|
+
pres = maint.auto_para(Path(self.vault))
|
|
2319
|
+
results["auto_para"] = pres
|
|
2320
|
+
if pres["moved"]:
|
|
2321
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2322
|
+
"auto-para", str(Path(self.vault) / "brain"),
|
|
2323
|
+
f"filed {len(pres['moved'])} note(s) into their "
|
|
2324
|
+
f"PARA zone by metadata"))
|
|
2325
|
+
except Exception as exc:
|
|
2326
|
+
blocked.append(maint.blocked_item(
|
|
2327
|
+
f"auto-PARA fold failed: {exc}",
|
|
2328
|
+
"filesystem", "next maintain run"))
|
|
2329
|
+
try:
|
|
2330
|
+
nres = maint.refresh_navigation(Path(self.vault))
|
|
2331
|
+
results["navigation"] = nres
|
|
2332
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2333
|
+
"navigation", str(Path(self.vault) / "brain"),
|
|
2334
|
+
f"regenerated backlinks ({nres['backlink_targets']} "
|
|
2335
|
+
f"targets) + {len(nres['catalog_counts'])} zone catalogs"))
|
|
2336
|
+
except Exception as exc:
|
|
2337
|
+
blocked.append(maint.blocked_item(
|
|
2338
|
+
f"navigation refresh failed: {exc}",
|
|
2339
|
+
"filesystem", "next maintain run"))
|
|
2340
|
+
|
|
2341
|
+
# CUT-02: duplicate-retention prune — safe only because
|
|
2342
|
+
# every candidate is re-verified through the full
|
|
2343
|
+
# provenance chain (manifest -> raw note -> archived
|
|
2344
|
+
# original) right before deletion; see
|
|
2345
|
+
# `maint.retention_fold`'s docstring. Gated to run at most
|
|
2346
|
+
# ONCE PER DAY via the `_`-prefixed `_retention` marker
|
|
2347
|
+
# (review finding [4]: the maintain umbrella fires hourly,
|
|
2348
|
+
# and re-hashing the permanently-unverifiable residue every
|
|
2349
|
+
# hour is ~24x wasted whole-file I/O over ~1k parked files).
|
|
2350
|
+
_ret_marker = state.get("_retention")
|
|
2351
|
+
_ret_marker = _ret_marker if isinstance(_ret_marker, dict) else {}
|
|
2352
|
+
if _ret_marker.get("last_run") != d.isoformat():
|
|
2353
|
+
try:
|
|
2354
|
+
ret_res = maint.retention_fold(Path(self.vault), d)
|
|
2355
|
+
results["retention"] = ret_res
|
|
2356
|
+
if not dry_run:
|
|
2357
|
+
state["_retention"] = {"last_run": d.isoformat()}
|
|
2358
|
+
if ret_res["pruned"]:
|
|
2359
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2360
|
+
"duplicate-retention", str(Path(self.vault) / "inbox" / "_duplicate"),
|
|
2361
|
+
f"pruned {len(ret_res['pruned'])} duplicate(s) older "
|
|
2362
|
+
f"than {ret_res['retention_days']}d "
|
|
2363
|
+
f"(provenance-verified)"))
|
|
2364
|
+
if ret_res["skipped"]:
|
|
2365
|
+
# Distinguish "kept, chain unverifiable" (the
|
|
2366
|
+
# designed conservative outcome) from a real
|
|
2367
|
+
# delete/stat failure (review finding [2]): the
|
|
2368
|
+
# message must not tell the owner a file failed
|
|
2369
|
+
# provenance when it was actually a delete error.
|
|
2370
|
+
prov = [s for s in ret_res["skipped"]
|
|
2371
|
+
if s.get("kind") == "provenance"]
|
|
2372
|
+
err = [s for s in ret_res["skipped"]
|
|
2373
|
+
if s.get("kind") in ("delete", "stat")]
|
|
2374
|
+
if prov:
|
|
2375
|
+
action_required.append(maint.action_required_item(
|
|
2376
|
+
f"{len(prov)} aged duplicate(s) kept — their "
|
|
2377
|
+
"provenance chain does not verify",
|
|
2378
|
+
"an unverifiable duplicate is never auto-deleted "
|
|
2379
|
+
"(its archived original may be missing/changed)",
|
|
2380
|
+
"inspect inbox/_duplicate and the referenced "
|
|
2381
|
+
"manifest/raw/originals entries",
|
|
2382
|
+
str(Path(self.vault) / "inbox" / "_duplicate")))
|
|
2383
|
+
if err:
|
|
2384
|
+
action_required.append(maint.action_required_item(
|
|
2385
|
+
f"{len(err)} aged duplicate(s) could not be "
|
|
2386
|
+
"stat'd/deleted (filesystem error)",
|
|
2387
|
+
"a real I/O/permission error, NOT a provenance "
|
|
2388
|
+
"failure — the file was not removed",
|
|
2389
|
+
"check inbox/_duplicate permissions/mount",
|
|
2390
|
+
str(Path(self.vault) / "inbox" / "_duplicate")))
|
|
2391
|
+
except Exception as exc:
|
|
2392
|
+
blocked.append(maint.blocked_item(
|
|
2393
|
+
f"duplicate-retention fold failed: {exc}",
|
|
2394
|
+
"filesystem/manifest read", "next maintain run"))
|
|
2395
|
+
|
|
2396
|
+
# CUT-02: monthly quarantine triage summary — NEVER
|
|
2397
|
+
# deletes; queues a hot.md summary at most once per ISO
|
|
2398
|
+
# month (idempotency key), gated by the `_`-prefixed
|
|
2399
|
+
# `_quarantine_summary` marker (mirrors `_graphify_drift`).
|
|
2400
|
+
try:
|
|
2401
|
+
q_marker = state.get("_quarantine_summary")
|
|
2402
|
+
q_marker = q_marker if isinstance(q_marker, dict) else None
|
|
2403
|
+
if maint.quarantine_summary_due(q_marker, d):
|
|
2404
|
+
q_summary = maint.quarantine_triage_summary(Path(self.vault), d)
|
|
2405
|
+
results["quarantine_summary"] = q_summary
|
|
2406
|
+
if q_summary["total"]:
|
|
2407
|
+
self._append_hot_once(
|
|
2408
|
+
f"quarantine-summary:{d.strftime('%Y-%m')}",
|
|
2409
|
+
maint.render_quarantine_summary_hot_entry(q_summary, d),
|
|
2410
|
+
)
|
|
2411
|
+
state["_quarantine_summary"] = {"last_month": d.strftime("%Y-%m")}
|
|
2412
|
+
except Exception as exc:
|
|
2413
|
+
blocked.append(maint.blocked_item(
|
|
2414
|
+
f"quarantine triage summary failed: {exc}",
|
|
2415
|
+
"filesystem read", "next maintain run"))
|
|
2416
|
+
|
|
2417
|
+
# DEC-01 decision-capture nudge — after the sync so
|
|
2418
|
+
# freshly ingested notes are already indexed. Queues each
|
|
2419
|
+
# candidate to hot.md ONCE (idempotency key = note id);
|
|
2420
|
+
# capturing the decision note stays a human/synthesis gate.
|
|
2421
|
+
try:
|
|
2422
|
+
dcands = maint.decision_capture_scan(self.index.conn, d)
|
|
2423
|
+
results["decision_capture"] = {"candidates": len(dcands)}
|
|
2424
|
+
for c in dcands:
|
|
2425
|
+
if self._append_hot_once(
|
|
2426
|
+
f"decision-capture:{c['id']}",
|
|
2427
|
+
maint.render_decision_capture_hot_entry(c, d),
|
|
2428
|
+
):
|
|
2429
|
+
action_required.append(maint.action_required_item(
|
|
2430
|
+
f"possible uncaptured decision in `{c['id']}` "
|
|
2431
|
+
f"(“{c['phrase']}”)",
|
|
2432
|
+
"recording a decision note is a human gate — "
|
|
2433
|
+
"the fold only nudges",
|
|
2434
|
+
"review the hot.md entry; if real, capture a "
|
|
2435
|
+
"type: decision note (+ supersede what it reverses)",
|
|
2436
|
+
c["id"]))
|
|
2437
|
+
except Exception as exc:
|
|
2438
|
+
blocked.append(maint.blocked_item(
|
|
2439
|
+
f"decision-capture scan failed: {exc}",
|
|
2440
|
+
"index read", "next maintain run"))
|
|
2441
|
+
|
|
2442
|
+
# WATCHDOG-01: the hourly umbrella watches the weekly
|
|
2443
|
+
# synthesis task's heartbeat (the reverse watch lives in
|
|
2444
|
+
# the synthesis prompt: doctor-first). Queued to hot.md
|
|
2445
|
+
# at most once per ISO week.
|
|
2446
|
+
try:
|
|
2447
|
+
wd = maint.synthesis_heartbeat_finding(Path(self.vault), d)
|
|
2448
|
+
if wd is not None:
|
|
2449
|
+
action_required.append(wd)
|
|
2450
|
+
week = d.isocalendar()
|
|
2451
|
+
self._append_hot_once(
|
|
2452
|
+
f"synthesis-watchdog:{week[0]}-W{week[1]}",
|
|
2453
|
+
f"## {d.isoformat()} — synthesis watchdog\n"
|
|
2454
|
+
f"- **Finding:** {wd['finding']}\n"
|
|
2455
|
+
f"- **Owner input needed:** {wd['proposed_action']}\n")
|
|
2456
|
+
except Exception as exc:
|
|
2457
|
+
blocked.append(maint.blocked_item(
|
|
2458
|
+
f"synthesis watchdog failed: {exc}",
|
|
2459
|
+
"state file read", "next maintain run"))
|
|
2460
|
+
|
|
2461
|
+
# Second pass: reconcile the folds' mutations + publish.
|
|
2462
|
+
sync2 = self.sync(drain=False, publish=True)
|
|
2463
|
+
results["sync_publish"] = {
|
|
2464
|
+
k: sync2.get(k) for k in ("added", "updated", "deleted")}
|
|
2465
|
+
snap = sync2.get("snapshot")
|
|
2466
|
+
if snap:
|
|
2467
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2468
|
+
"snapshot", str(snap.get("snapshot_db", "")),
|
|
2469
|
+
f"published snapshot gen {snap.get('generation')}"))
|
|
2470
|
+
try:
|
|
2471
|
+
results["brief"] = self.brief(drain=False)
|
|
2472
|
+
except Exception as exc:
|
|
2473
|
+
blocked.append(maint.blocked_item(
|
|
2474
|
+
"could not generate the morning brief",
|
|
2475
|
+
f"{type(exc).__name__}: {exc}",
|
|
2476
|
+
"re-run after the underlying error is fixed"))
|
|
2477
|
+
try:
|
|
2478
|
+
results["brief_html"] = self.brief_html(drain=False, today=d)
|
|
2479
|
+
except Exception as exc:
|
|
2480
|
+
blocked.append(maint.blocked_item(
|
|
2481
|
+
"could not write the HTML morning brief",
|
|
2482
|
+
f"{type(exc).__name__}: {exc}",
|
|
2483
|
+
"re-run after the underlying error is fixed"))
|
|
2484
|
+
rec_res = self._recommendations_aging_fold(d)
|
|
2485
|
+
results["recommendations_aging"] = rec_res
|
|
2486
|
+
if rec_res.get("surfaced"):
|
|
2487
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2488
|
+
"recommendations-aging", str(config.recommendations_open_path(self.vault)),
|
|
2489
|
+
f"surfaced {rec_res['surfaced']} aged recommendation(s) into hot.md"))
|
|
2490
|
+
try:
|
|
2491
|
+
# Opt-in (default off): folding note-creation into the
|
|
2492
|
+
# maintain loop must not change maintain's note-count
|
|
2493
|
+
# invariant for vaults that don't want a daily journal.
|
|
2494
|
+
import os as _os
|
|
2495
|
+
dn = (self._daily_note_fold(d, results.get("brief"))
|
|
2496
|
+
if _os.environ.get("BRAIN_DAILY_NOTE") else {"created": False, "skipped": "BRAIN_DAILY_NOTE unset"})
|
|
2497
|
+
results["daily_note"] = dn
|
|
2498
|
+
if dn.get("created"):
|
|
2499
|
+
auto_fixed.append(maint.auto_fixed_item(
|
|
2500
|
+
"daily-note", str(self.vault),
|
|
2501
|
+
f"created daily note {dn['id']}"))
|
|
2502
|
+
except Exception as exc:
|
|
2503
|
+
blocked.append(maint.blocked_item(
|
|
2504
|
+
"could not create today's daily note",
|
|
2505
|
+
f"{type(exc).__name__}: {exc}",
|
|
2506
|
+
"create it manually with tools/brain_daily.py"))
|
|
2507
|
+
_mark("daily", True)
|
|
2508
|
+
except Exception as exc:
|
|
2509
|
+
blocked.append(maint.blocked_item(
|
|
2510
|
+
"daily branch (sync/brief/recommendations-aging) raised",
|
|
2511
|
+
f"{type(exc).__name__}: {exc}",
|
|
2512
|
+
"re-run maintain after the underlying error is fixed"))
|
|
2513
|
+
_mark("daily", False, f"{type(exc).__name__}: {exc}")
|
|
2514
|
+
|
|
2515
|
+
# FRESH-01 — drift-triggered graphify check (2026-07-11). Runs
|
|
2516
|
+
# every maintain, REGARDLESS of dry_run and independent of the
|
|
2517
|
+
# daily branch's own try/except (a corpus-drift READ never needs
|
|
2518
|
+
# the daily fold's sync to have succeeded — ``self.index.conn``
|
|
2519
|
+
# already reflects whatever the index currently holds): the
|
|
2520
|
+
# monthly date-gate below remains the FLOOR trigger; this is the
|
|
2521
|
+
# inverse — a vault that drifts past the threshold rebuilds
|
|
2522
|
+
# early instead of waiting out the calendar. Computed once here
|
|
2523
|
+
# so the unified graphify block below never double-builds on a
|
|
2524
|
+
# day that is BOTH drift-triggered and the monthly floor.
|
|
2525
|
+
graphify_drift_triggered = False
|
|
2526
|
+
try:
|
|
2527
|
+
import json as _json
|
|
2528
|
+
|
|
2529
|
+
try:
|
|
2530
|
+
old_manifest = _json.loads(
|
|
2531
|
+
config.graph_manifest_path(self.vault).read_text(encoding="utf-8"))
|
|
2532
|
+
except (OSError, ValueError):
|
|
2533
|
+
old_manifest = None
|
|
2534
|
+
drift_ratio = maint.graphify_drift(old_manifest, self.index.conn)
|
|
2535
|
+
drift_marker = state.get("_graphify_drift")
|
|
2536
|
+
drift_marker = drift_marker if isinstance(drift_marker, dict) else None
|
|
2537
|
+
# A manifest that has NEVER existed has no baseline to have
|
|
2538
|
+
# drifted FROM (the pure ratio function still reports 1.0 for
|
|
2539
|
+
# it — a defined, degenerate "unknown" signal) — that
|
|
2540
|
+
# first-ever-build case is already the monthly floor's job
|
|
2541
|
+
# (a "graphify" branch absent from maintain-state is due
|
|
2542
|
+
# immediately, ADR-0003 Ruling d/``maintain_branches``). The
|
|
2543
|
+
# drift trigger only fires once there IS an established
|
|
2544
|
+
# baseline to measure real drift against.
|
|
2545
|
+
graphify_drift_triggered = bool(old_manifest) and maint.should_trigger_drift_graphify(
|
|
2546
|
+
drift_ratio, drift_marker, d)
|
|
2547
|
+
results["graphify_drift"] = {
|
|
2548
|
+
"ratio": round(drift_ratio, 4), "triggered": graphify_drift_triggered,
|
|
2549
|
+
"has_baseline": bool(old_manifest)}
|
|
2550
|
+
except Exception as exc:
|
|
2551
|
+
blocked.append(maint.blocked_item(
|
|
2552
|
+
f"graphify drift check failed: {exc}",
|
|
2553
|
+
"index read", "next maintain run"))
|
|
2554
|
+
|
|
2555
|
+
if "health" in branches:
|
|
2556
|
+
try:
|
|
2557
|
+
h = self.health()
|
|
2558
|
+
fsync_finding = self._framework_sync_finding()
|
|
2559
|
+
if fsync_finding is not None:
|
|
2560
|
+
h["outcomes"]["action_required"].append(fsync_finding)
|
|
2561
|
+
results["health"] = h
|
|
2562
|
+
action_required += h["outcomes"]["action_required"]
|
|
2563
|
+
blocked += h["outcomes"]["blocked"]
|
|
2564
|
+
_mark("health", True)
|
|
2565
|
+
except Exception as exc:
|
|
2566
|
+
blocked.append(maint.blocked_item(
|
|
2567
|
+
"health branch raised",
|
|
2568
|
+
f"{type(exc).__name__}: {exc}",
|
|
2569
|
+
"re-run maintain after the underlying error is fixed"))
|
|
2570
|
+
_mark("health", False, f"{type(exc).__name__}: {exc}")
|
|
2571
|
+
|
|
2572
|
+
if "integrity" in branches:
|
|
2573
|
+
try:
|
|
2574
|
+
i = self.integrity(min_score=min_score, k=near_dup_k)
|
|
2575
|
+
results["integrity"] = i
|
|
2576
|
+
blocked += i.get("blocked", [])
|
|
2577
|
+
if i.get("audit_issue"):
|
|
2578
|
+
action_required.append(i["audit_issue"])
|
|
2579
|
+
if i.get("near_dup_pairs"):
|
|
2580
|
+
# near_dup_pairs are UNFILTERED here; `maintain` reports only
|
|
2581
|
+
# the raw count (egress applies at the standalone `integrity`
|
|
2582
|
+
# verb, which is where a caller actually inspects pair content).
|
|
2583
|
+
action_required.append(maint.action_required_item(
|
|
2584
|
+
f"{len(i['near_dup_pairs'])} near-duplicate pair(s) found "
|
|
2585
|
+
f">= {min_score}",
|
|
2586
|
+
"de-dup is a human merge/keep judgment, never auto-merged",
|
|
2587
|
+
"run `brain integrity --json` for the gated pair list and review",
|
|
2588
|
+
"near-dup scan"))
|
|
2589
|
+
_mark("integrity", True)
|
|
2590
|
+
except Exception as exc:
|
|
2591
|
+
blocked.append(maint.blocked_item(
|
|
2592
|
+
"integrity branch raised",
|
|
2593
|
+
f"{type(exc).__name__}: {exc}",
|
|
2594
|
+
"re-run maintain after the underlying error is fixed"))
|
|
2595
|
+
_mark("integrity", False, f"{type(exc).__name__}: {exc}")
|
|
2596
|
+
|
|
2597
|
+
if "digest" in branches:
|
|
2598
|
+
try:
|
|
2599
|
+
results["digest"] = self.digest(days=7)
|
|
2600
|
+
curate_res = self.curate(dry_run=dry_run, today=d)
|
|
2601
|
+
promote_res = self.promote_scan()
|
|
2602
|
+
results["curate"] = curate_res
|
|
2603
|
+
results["promote_scan"] = promote_res
|
|
2604
|
+
if not dry_run:
|
|
2605
|
+
if curate_res.get("stale_links") or curate_res.get("revisit_sample"):
|
|
2606
|
+
self._append_hot_once(
|
|
2607
|
+
f"maintain:curate:{d.isoformat()}",
|
|
2608
|
+
maint.render_curation_hot_entry(
|
|
2609
|
+
curate_res["stale_links"], curate_res["revisit_sample"], d),
|
|
2610
|
+
)
|
|
2611
|
+
if promote_res.get("candidates"):
|
|
2612
|
+
self._append_hot_once(
|
|
2613
|
+
f"maintain:promote-scan:{d.isoformat()}",
|
|
2614
|
+
maint.render_promote_scan_hot_entry(promote_res["candidates"], d),
|
|
2615
|
+
)
|
|
2616
|
+
results["digest_html"] = self.digest_html(days=7, today=d)
|
|
2617
|
+
_mark("digest", True)
|
|
2618
|
+
except Exception as exc:
|
|
2619
|
+
blocked.append(maint.blocked_item(
|
|
2620
|
+
"digest branch (digest/curate/promote-scan) raised",
|
|
2621
|
+
f"{type(exc).__name__}: {exc}",
|
|
2622
|
+
"re-run maintain after the underlying error is fixed"))
|
|
2623
|
+
_mark("digest", False, f"{type(exc).__name__}: {exc}")
|
|
2624
|
+
|
|
2625
|
+
if "golden" in branches:
|
|
2626
|
+
try:
|
|
2627
|
+
probes_path = maint.golden_probes_path(Path(self.vault))
|
|
2628
|
+
if not probes_path.is_file():
|
|
2629
|
+
# Absent probes file — skip LOUDLY (never a silent
|
|
2630
|
+
# pass, never an error): mark done so this doesn't
|
|
2631
|
+
# re-check every hour, but next Sunday re-checks.
|
|
2632
|
+
results["golden"] = {
|
|
2633
|
+
"score": None, "runner": None, "degraded": False,
|
|
2634
|
+
"skipped": "no probes file",
|
|
2635
|
+
}
|
|
2636
|
+
action_required.append(maint.action_required_item(
|
|
2637
|
+
f"golden-probe branch skipped: no probes file at {probes_path}",
|
|
2638
|
+
"WD-03 cross-family execution needs a per-vault "
|
|
2639
|
+
"eval/golden-probes.json (WD-02) to score",
|
|
2640
|
+
"author a probes file (see `brain-golden-probe --help` "
|
|
2641
|
+
"/ docs/operations/s06-evidence.md)",
|
|
2642
|
+
str(probes_path)))
|
|
2643
|
+
_mark("golden", True)
|
|
2644
|
+
else:
|
|
2645
|
+
marker = state.get("_golden_attempt")
|
|
2646
|
+
marker = marker if isinstance(marker, dict) else None
|
|
2647
|
+
now_dt = _dt.datetime.now(_dt.timezone.utc)
|
|
2648
|
+
if not maint.golden_attempt_due(marker, now_dt):
|
|
2649
|
+
results["golden"] = {
|
|
2650
|
+
"score": None, "runner": None, "degraded": False,
|
|
2651
|
+
"skipped": "cooldown",
|
|
2652
|
+
"next_retry_at": (marker or {}).get("next_retry_at"),
|
|
2653
|
+
}
|
|
2654
|
+
# Deliberately no `_mark` call: the branch stays
|
|
2655
|
+
# due (still Sunday, or a missed catch-up), but
|
|
2656
|
+
# NO codex/self invocation happens this hour —
|
|
2657
|
+
# the whole point of the next_retry_at gate.
|
|
2658
|
+
elif dry_run:
|
|
2659
|
+
# Fix [3]: a --dry-run PREVIEW must NOT spawn the
|
|
2660
|
+
# golden runner (real codex/self subprocess, up to
|
|
2661
|
+
# ~600s+600s) or burn codex quota — report what
|
|
2662
|
+
# WOULD run and persist NO marker.
|
|
2663
|
+
results["golden"] = {
|
|
2664
|
+
"score": None, "runner": None, "degraded": False,
|
|
2665
|
+
"dry_run": True, "would_run": True,
|
|
2666
|
+
"probes_path": str(probes_path),
|
|
2667
|
+
}
|
|
2668
|
+
_mark("golden", True)
|
|
2669
|
+
else:
|
|
2670
|
+
# Persist a PROVISIONAL next_retry_at (short
|
|
2671
|
+
# backoff) BEFORE the shell-out (fix [1], mirrors
|
|
2672
|
+
# `_run_bounded_graphify`'s attempt-persisted-
|
|
2673
|
+
# before-build ordering). `golden_attempt_due`
|
|
2674
|
+
# keys the cooldown on `next_retry_at` alone, so a
|
|
2675
|
+
# run KILLED mid-`codex exec` (reboot/OOM/launchd
|
|
2676
|
+
# timeout) — leaving no clean return to write the
|
|
2677
|
+
# outcome-based value — still backs off next hour
|
|
2678
|
+
# instead of re-storming codex. Overwritten with
|
|
2679
|
+
# the outcome-based value on a clean return below.
|
|
2680
|
+
base_min = int(_os.environ.get(
|
|
2681
|
+
maint.GOLDEN_RETRY_BASE_MINUTES_ENV,
|
|
2682
|
+
maint.DEFAULT_GOLDEN_RETRY_BASE_MINUTES))
|
|
2683
|
+
# ESCALATING provisional (re-review): optimistically
|
|
2684
|
+
# count THIS attempt as a failure and back off on the
|
|
2685
|
+
# incremented count, so a REPEATEDLY killed run backs
|
|
2686
|
+
# off progressively (6h → 12h → …) instead of a flat
|
|
2687
|
+
# base every time. A clean return below recomputes the
|
|
2688
|
+
# authoritative marker from the PRE-attempt count
|
|
2689
|
+
# `orig_n` (so a real transient isn't double-counted,
|
|
2690
|
+
# and a success resets to 0).
|
|
2691
|
+
orig_n = int((marker or {}).get("consecutive_transient_failures", 0))
|
|
2692
|
+
prov_n = orig_n + 1
|
|
2693
|
+
pre_marker = dict(marker or {})
|
|
2694
|
+
pre_marker["last_attempt"] = now_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
2695
|
+
pre_marker["consecutive_transient_failures"] = prov_n
|
|
2696
|
+
pre_marker["next_retry_at"] = (
|
|
2697
|
+
now_dt + _dt.timedelta(
|
|
2698
|
+
minutes=maint.golden_retry_backoff_minutes(base_min, prov_n))
|
|
2699
|
+
).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
2700
|
+
state["_golden_attempt"] = pre_marker
|
|
2701
|
+
self._save_maintain_state(state)
|
|
2702
|
+
runner = golden_runner or self._run_golden_probe
|
|
2703
|
+
g = runner(probes_path=probes_path)
|
|
2704
|
+
results["golden"] = g
|
|
2705
|
+
exit_code = g.get("exit_code")
|
|
2706
|
+
transient = exit_code not in (
|
|
2707
|
+
maint.GOLDEN_EXIT_OK, maint.GOLDEN_EXIT_REGRESSION,
|
|
2708
|
+
maint.GOLDEN_EXIT_ACTION_REQUIRED)
|
|
2709
|
+
state["_golden_attempt"] = maint.update_golden_attempt_marker(
|
|
2710
|
+
{**pre_marker, "consecutive_transient_failures": orig_n},
|
|
2711
|
+
now_dt, transient=transient)
|
|
2712
|
+
self._save_maintain_state(state)
|
|
2713
|
+
if transient:
|
|
2714
|
+
blocked.append(maint.blocked_item(
|
|
2715
|
+
f"golden-probe run was transient (runner="
|
|
2716
|
+
f"{g.get('runner')}): {g.get('error') or g.get('codex_error') or 'no deterministic result'}",
|
|
2717
|
+
"the brain CLI itself failed/emitted non-JSON, or "
|
|
2718
|
+
"codex could not be validated and the self-run "
|
|
2719
|
+
"fallback also failed",
|
|
2720
|
+
"bounded backoff will retry automatically once "
|
|
2721
|
+
"the cooldown elapses"))
|
|
2722
|
+
_mark("golden", False, "transient")
|
|
2723
|
+
else:
|
|
2724
|
+
if exit_code == maint.GOLDEN_EXIT_ACTION_REQUIRED:
|
|
2725
|
+
action_required.append(maint.action_required_item(
|
|
2726
|
+
f"golden-probe run is config-invalid (score="
|
|
2727
|
+
f"{g.get('score')})",
|
|
2728
|
+
"a deterministic problem in the probes file/vault "
|
|
2729
|
+
"anchors — never retried before next Sunday",
|
|
2730
|
+
"fix the probes file, then re-run "
|
|
2731
|
+
"`brain-golden-probe` manually to confirm",
|
|
2732
|
+
str(probes_path)))
|
|
2733
|
+
elif exit_code == maint.GOLDEN_EXIT_REGRESSION:
|
|
2734
|
+
action_required.append(maint.action_required_item(
|
|
2735
|
+
f"golden-probe regression: score {g.get('score')}",
|
|
2736
|
+
"retrieval quality regressed below the "
|
|
2737
|
+
"probes-file threshold",
|
|
2738
|
+
"run the autoresearch skill or review recent "
|
|
2739
|
+
"promotions/curation findings",
|
|
2740
|
+
str(probes_path)))
|
|
2741
|
+
# A persistent degraded/self-run state means
|
|
2742
|
+
# cross-family EXECUTION is not actually
|
|
2743
|
+
# happening (codex unavailable/unvalidated) —
|
|
2744
|
+
# surfaced every degraded run, not just once,
|
|
2745
|
+
# so it naturally reads as "persistent" in
|
|
2746
|
+
# hot.md/action_required if it keeps recurring
|
|
2747
|
+
# (ponytail: no extra streak-counter state).
|
|
2748
|
+
if g.get("degraded"):
|
|
2749
|
+
action_required.append(maint.action_required_item(
|
|
2750
|
+
"golden-probe ran in DEGRADED (self) mode — "
|
|
2751
|
+
f"codex execution unavailable/unvalidated: "
|
|
2752
|
+
f"{g.get('codex_error')}",
|
|
2753
|
+
"cross-family EXECUTION requires codex to "
|
|
2754
|
+
"actually run the scorer; a persistent "
|
|
2755
|
+
"degraded state means that isn't happening",
|
|
2756
|
+
"check codex CLI availability/auth on this host",
|
|
2757
|
+
str(probes_path)))
|
|
2758
|
+
_mark("golden", True)
|
|
2759
|
+
except Exception as exc:
|
|
2760
|
+
# Fix [1]: a raise mid-branch (before the pre-shell-out
|
|
2761
|
+
# save, or in the runner/marker-update path) must still
|
|
2762
|
+
# leave a next_retry_at so the next hourly maintain backs
|
|
2763
|
+
# off instead of re-invoking codex every hour. The
|
|
2764
|
+
# pre-shell-out save usually already wrote one; this is the
|
|
2765
|
+
# belt for a raise BEFORE it lands.
|
|
2766
|
+
if not dry_run:
|
|
2767
|
+
try:
|
|
2768
|
+
cur = state.get("_golden_attempt")
|
|
2769
|
+
cur = dict(cur) if isinstance(cur, dict) else {}
|
|
2770
|
+
now_dt = _dt.datetime.now(_dt.timezone.utc)
|
|
2771
|
+
# Refresh the backoff when it is ABSENT *or already
|
|
2772
|
+
# ELAPSED*: an elapsed next_retry_at is exactly why
|
|
2773
|
+
# this branch was due, so leaving it in place keeps
|
|
2774
|
+
# `golden_attempt_due` True and re-storms codex every
|
|
2775
|
+
# hour. Only a still-FUTURE value (the provisional
|
|
2776
|
+
# save already landed) is left alone. Parsing mirrors
|
|
2777
|
+
# `golden_attempt_due` so "present" means the same
|
|
2778
|
+
# thing on both sides of the gate.
|
|
2779
|
+
existing = cur.get("next_retry_at")
|
|
2780
|
+
existing_dt = None
|
|
2781
|
+
if existing:
|
|
2782
|
+
try:
|
|
2783
|
+
existing_dt = _dt.datetime.fromisoformat(
|
|
2784
|
+
str(existing).replace("Z", "+00:00"))
|
|
2785
|
+
if existing_dt.tzinfo is None:
|
|
2786
|
+
existing_dt = existing_dt.replace(
|
|
2787
|
+
tzinfo=_dt.timezone.utc)
|
|
2788
|
+
except ValueError:
|
|
2789
|
+
existing_dt = None
|
|
2790
|
+
if existing_dt is None or existing_dt <= now_dt:
|
|
2791
|
+
try:
|
|
2792
|
+
base_min = int(_os.environ.get(
|
|
2793
|
+
maint.GOLDEN_RETRY_BASE_MINUTES_ENV,
|
|
2794
|
+
maint.DEFAULT_GOLDEN_RETRY_BASE_MINUTES))
|
|
2795
|
+
except ValueError:
|
|
2796
|
+
base_min = maint.DEFAULT_GOLDEN_RETRY_BASE_MINUTES
|
|
2797
|
+
cur["last_attempt"] = now_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
2798
|
+
cur["next_retry_at"] = (
|
|
2799
|
+
now_dt + _dt.timedelta(minutes=base_min)
|
|
2800
|
+
).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
2801
|
+
state["_golden_attempt"] = cur
|
|
2802
|
+
self._save_maintain_state(state)
|
|
2803
|
+
except Exception: # noqa: BLE001 — backoff persist is best-effort
|
|
2804
|
+
pass
|
|
2805
|
+
blocked.append(maint.blocked_item(
|
|
2806
|
+
"golden branch raised",
|
|
2807
|
+
f"{type(exc).__name__}: {exc}",
|
|
2808
|
+
"re-run maintain after the underlying error is fixed"))
|
|
2809
|
+
_mark("golden", False, f"{type(exc).__name__}: {exc}")
|
|
2810
|
+
|
|
2811
|
+
# FRESH-01: the monthly date-gate (floor) and the drift trigger
|
|
2812
|
+
# (early rebuild) are ORed into ONE build — both routed through the
|
|
2813
|
+
# SAME attempt-bounded in-process path (`_run_bounded_graphify`;
|
|
2814
|
+
# owner decision 2026-07-12). A day that is both the monthly floor
|
|
2815
|
+
# AND drift-triggered still builds exactly once.
|
|
2816
|
+
monthly_due = "graphify" in branches
|
|
2817
|
+
# The ATTEMPT-keyed cooldown (with capped exponential backoff)
|
|
2818
|
+
# gates the MONTHLY floor too (review finding [0]): a failing
|
|
2819
|
+
# 1st-of-month build leaves its branch due, and without this gate
|
|
2820
|
+
# it would re-fire a full rebuild every hourly maintain, unbounded.
|
|
2821
|
+
# The floor OBLIGATION survives (the branch stays due until a
|
|
2822
|
+
# publish advances it) — only the RETRY CADENCE backs off.
|
|
2823
|
+
import os as _os
|
|
2824
|
+
|
|
2825
|
+
_drift_marker = state.get("_graphify_drift")
|
|
2826
|
+
_drift_marker = _drift_marker if isinstance(_drift_marker, dict) else None
|
|
2827
|
+
attempt_allowed = maint.graphify_drift_marker_due(
|
|
2828
|
+
_drift_marker, d, int(_os.environ.get(
|
|
2829
|
+
maint.GRAPHIFY_COOLDOWN_DAYS_ENV, maint.DEFAULT_GRAPHIFY_COOLDOWN_DAYS)))
|
|
2830
|
+
if (monthly_due or graphify_drift_triggered) and not attempt_allowed:
|
|
2831
|
+
# Deferred by the attempt-keyed (backed-off) cooldown — leave a
|
|
2832
|
+
# breadcrumb so a skipped build is explainable, never silent.
|
|
2833
|
+
results["graphify"] = {
|
|
2834
|
+
"ritual": "graphify", "invoked": False, "published": False,
|
|
2835
|
+
"status": "cooldown_deferred",
|
|
2836
|
+
"note": "a recent failed attempt is backing off; the build "
|
|
2837
|
+
"retries once the (exponential) cooldown elapses",
|
|
2838
|
+
}
|
|
2839
|
+
if (monthly_due or graphify_drift_triggered) and attempt_allowed:
|
|
2840
|
+
reason = "drift" if graphify_drift_triggered else "monthly-floor"
|
|
2841
|
+
try:
|
|
2842
|
+
g = self._run_bounded_graphify(
|
|
2843
|
+
force=False, dry_run=dry_run, today=d, state=state,
|
|
2844
|
+
reason=reason, builder=graphify_runner)
|
|
2845
|
+
results["graphify"] = g
|
|
2846
|
+
if not dry_run and g.get("published") and g.get("candidates"):
|
|
2847
|
+
# Best-effort (review finding [4]): a hot.md write
|
|
2848
|
+
# failure must not skip the `_mark` below — the build
|
|
2849
|
+
# PUBLISHED, and leaving the branch due would trigger a
|
|
2850
|
+
# redundant rebuild next run. Surface it instead.
|
|
2851
|
+
try:
|
|
2852
|
+
self._append_hot_once(
|
|
2853
|
+
f"maintain:graphify:{d.isoformat()}",
|
|
2854
|
+
maint.render_graphify_hot_entry(g["candidates"], d),
|
|
2855
|
+
)
|
|
2856
|
+
except Exception as hot_exc: # noqa: BLE001
|
|
2857
|
+
action_required.append(maint.action_required_item(
|
|
2858
|
+
"graphify hot-queue entry could not be written",
|
|
2859
|
+
f"{type(hot_exc).__name__}: {hot_exc}",
|
|
2860
|
+
"check .brain/memory/hot.md writability; the "
|
|
2861
|
+
"graph itself published fine",
|
|
2862
|
+
"graphify hot-queue"))
|
|
2863
|
+
build_info = g.get("build") or {}
|
|
2864
|
+
published = bool(g.get("published"))
|
|
2865
|
+
skipped = bool(g.get("skipped"))
|
|
2866
|
+
# Key the "benign, never alarm" suppression on the RESULT's
|
|
2867
|
+
# own dry_run flag: a genuine dry-run PREVIEW sets
|
|
2868
|
+
# ``g["dry_run"] = True`` (published False by design) and must
|
|
2869
|
+
# not alarm as a FAILURE; but a REAL failure under `brain
|
|
2870
|
+
# maintain --dry-run` returns a failure result with NO
|
|
2871
|
+
# dry_run flag, and the preview MUST still surface it.
|
|
2872
|
+
result_is_preview = bool(g.get("dry_run"))
|
|
2873
|
+
dur = build_info.get("duration_seconds")
|
|
2874
|
+
dur_suffix = f" ({dur}s)" if dur is not None else ""
|
|
2875
|
+
if result_is_preview:
|
|
2876
|
+
# Never a failure alarm — but a soft-budget breach in the
|
|
2877
|
+
# preview is exactly the pre-flight signal `--dry-run`
|
|
2878
|
+
# exists to show (review finding [5]): keep it.
|
|
2879
|
+
if build_info.get("action_required"):
|
|
2880
|
+
dur_txt = f"{dur}s" if dur is not None else "an unknown duration"
|
|
2881
|
+
action_required.append(maint.action_required_item(
|
|
2882
|
+
f"graphify dry-run build took {dur_txt} "
|
|
2883
|
+
f"(> {build_info.get('action_required_seconds')}s soft budget)",
|
|
2884
|
+
"the PREVIEW build exceeded the soft wall-clock "
|
|
2885
|
+
"budget — the real scheduled build likely will too",
|
|
2886
|
+
"investigate corpus scale / vector backend before "
|
|
2887
|
+
"the next scheduled build",
|
|
2888
|
+
"graphify build"))
|
|
2889
|
+
elif not published and not skipped:
|
|
2890
|
+
# A failed in-process build MUST append a `blocked` item
|
|
2891
|
+
# too — OBS-02's alarm keys off `blocked` count, so a
|
|
2892
|
+
# failed graph build must never read as a clean run just
|
|
2893
|
+
# because it also has an action_required.
|
|
2894
|
+
blocked.append(maint.blocked_item(
|
|
2895
|
+
f"graphify build ({reason}, status={g.get('status', 'unknown')}) "
|
|
2896
|
+
f"failed to complete{dur_suffix}",
|
|
2897
|
+
"the in-process graph build raised, returned a bad "
|
|
2898
|
+
"result, or failed to build/validate "
|
|
2899
|
+
"(build_failed/invalid_artifact)",
|
|
2900
|
+
"capped exponential backoff will retry automatically "
|
|
2901
|
+
"once the cooldown elapses"))
|
|
2902
|
+
action_required.append(maint.action_required_item(
|
|
2903
|
+
f"graphify build ({reason}, status={g.get('status', 'unknown')}) "
|
|
2904
|
+
f"failed to complete{dur_suffix}",
|
|
2905
|
+
"the in-process graph build raised, returned a bad "
|
|
2906
|
+
"result, or failed to build/validate",
|
|
2907
|
+
"inspect the result's error/status detail and "
|
|
2908
|
+
".brain/graph/BUILD_FAILED.json; capped exponential "
|
|
2909
|
+
"backoff will retry automatically once the cooldown "
|
|
2910
|
+
"elapses",
|
|
2911
|
+
"graphify build"))
|
|
2912
|
+
elif published and build_info.get("action_required"):
|
|
2913
|
+
# published, but slower than the soft budget —
|
|
2914
|
+
# informational only: this build already succeeded, so
|
|
2915
|
+
# never claim a retry that will not happen.
|
|
2916
|
+
dur_txt = f"{dur}s" if dur is not None else "an unknown duration"
|
|
2917
|
+
action_required.append(maint.action_required_item(
|
|
2918
|
+
f"graphify build ({reason}) published but took {dur_txt} "
|
|
2919
|
+
f"(> {build_info.get('action_required_seconds')}s soft budget)",
|
|
2920
|
+
"the graph build exceeded its soft wall-clock budget "
|
|
2921
|
+
"but completed and published successfully",
|
|
2922
|
+
"investigate corpus scale / vector backend before the "
|
|
2923
|
+
"next scheduled build — no retry is needed, this "
|
|
2924
|
+
"build already succeeded",
|
|
2925
|
+
"graphify build"))
|
|
2926
|
+
if result_is_preview:
|
|
2927
|
+
pass # a genuine dry-run preview marks nothing
|
|
2928
|
+
elif published or skipped:
|
|
2929
|
+
_mark("graphify", True)
|
|
2930
|
+
elif monthly_due:
|
|
2931
|
+
# the monthly FLOOR was due and the build failed — leave
|
|
2932
|
+
# it due (never silently drop the floor obligation).
|
|
2933
|
+
_mark("graphify", False, g.get("status", "build_failed"))
|
|
2934
|
+
except Exception as exc: # noqa: BLE001 — backstop for the maintain-
|
|
2935
|
+
# SIDE handling (e.g. the hot.md append, or a disk error in
|
|
2936
|
+
# `_run_bounded_graphify`'s own state write). The build's OWN
|
|
2937
|
+
# outcome + backoff are fully owned by `_run_bounded_graphify`
|
|
2938
|
+
# (it catches build errors internally and records/persists the
|
|
2939
|
+
# `_graphify_drift` marker), so this handler MUST NOT touch the
|
|
2940
|
+
# backoff marker: doing so would double-count a build failure,
|
|
2941
|
+
# or worse, penalize a build that actually PUBLISHED when the
|
|
2942
|
+
# post-publish hot.md write is what raised. It only surfaces
|
|
2943
|
+
# the failure so it is never silent.
|
|
2944
|
+
blocked.append(maint.blocked_item(
|
|
2945
|
+
"graphify branch raised (maintain-side handling)",
|
|
2946
|
+
f"{type(exc).__name__}: {exc}",
|
|
2947
|
+
"re-run maintain after the underlying error is fixed"))
|
|
2948
|
+
|
|
2949
|
+
# -- OBS-01/02/04: ONE final health-history append per run
|
|
2950
|
+
# (HARDENED correction 2 — never appended right after the second
|
|
2951
|
+
# sync, so this record carries health/integrity/digest/graphify
|
|
2952
|
+
# outcomes too; ``results`` is the structured hook a later
|
|
2953
|
+
# golden-eval branch folds into via ``results["golden"]``).
|
|
2954
|
+
# HOST-broker, and skipped (read-only collection only) under
|
|
2955
|
+
# ``dry_run`` — no append, no notification, no state mutation.
|
|
2956
|
+
pre_outcomes = maint.build_outcomes(auto_fixed, action_required, blocked)
|
|
2957
|
+
health_record: dict[str, Any] | None = None
|
|
2958
|
+
trend_findings: list[dict[str, Any]] = []
|
|
2959
|
+
notifications: list[str] = []
|
|
2960
|
+
try:
|
|
2961
|
+
health_record = maint.collect_health_metrics(
|
|
2962
|
+
self, outcomes=pre_outcomes, results=results,
|
|
2963
|
+
run_id=maint.new_health_run_id())
|
|
2964
|
+
if not dry_run:
|
|
2965
|
+
maint.append_health_record(Path(self.vault), health_record)
|
|
2966
|
+
except Exception as exc:
|
|
2967
|
+
blocked.append(maint.blocked_item(
|
|
2968
|
+
f"health-history/trend/notify fold failed: {exc}",
|
|
2969
|
+
"metrics collection or file I/O", "next maintain run"))
|
|
2970
|
+
|
|
2971
|
+
# Fix [2]: compute trend + fire notifications from the POST-fold
|
|
2972
|
+
# outcomes (built fresh here, AFTER the except above may have
|
|
2973
|
+
# just appended its own blocked_item) — never from the frozen
|
|
2974
|
+
# ``pre_outcomes`` snapshot. Otherwise a health-fold failure
|
|
2975
|
+
# (e.g. `.brain` full/read-only) reports blocked>0 in the run's
|
|
2976
|
+
# own outcomes yet never raises the alarm OBS-02 exists for,
|
|
2977
|
+
# because the notify call used to sit INSIDE the same try block
|
|
2978
|
+
# that just failed and was skipped entirely.
|
|
2979
|
+
if not dry_run:
|
|
2980
|
+
post_outcomes = maint.build_outcomes(auto_fixed, action_required, blocked)
|
|
2981
|
+
try:
|
|
2982
|
+
history = maint.read_health_history(Path(self.vault))
|
|
2983
|
+
sparse_history = maint.read_sparse_history(Path(self.vault))
|
|
2984
|
+
trend_findings = maint.health_trend(
|
|
2985
|
+
history, d, sparse_history=sparse_history)
|
|
2986
|
+
except Exception: # noqa: BLE001 — trend is best-effort; the
|
|
2987
|
+
pass # blocked count alone still drives the alarm below.
|
|
2988
|
+
try:
|
|
2989
|
+
candidates = maint.pending_notifications(
|
|
2990
|
+
Path(self.vault), post_outcomes, trend_findings, d)
|
|
2991
|
+
notifications = maint.fire_and_mark_notifications(
|
|
2992
|
+
Path(self.vault), candidates, d)
|
|
2993
|
+
except Exception: # noqa: BLE001 — a notify-path failure is cosmetic,
|
|
2994
|
+
pass # never allowed to fail the maintain run itself.
|
|
2995
|
+
results["health_history"] = health_record
|
|
2996
|
+
results["health_trend"] = trend_findings
|
|
2997
|
+
results["notifications"] = notifications
|
|
2998
|
+
|
|
2999
|
+
if not dry_run:
|
|
3000
|
+
# In-process graphify (owner decision 2026-07-11): the
|
|
3001
|
+
# `_graphify_drift` marker is written by `_run_bounded_graphify`
|
|
3002
|
+
# within THIS process, into THIS `state` dict, so `state` already
|
|
3003
|
+
# holds the authoritative marker — no cross-process re-merge is
|
|
3004
|
+
# needed (the subprocess-era re-read that could revert an
|
|
3005
|
+
# in-memory backoff bump is gone with the subprocess). Two
|
|
3006
|
+
# overlapping maintain PROCESSES under a broken 2h stale-lock
|
|
3007
|
+
# last-writer-win their branch stamps, as they always have — a
|
|
3008
|
+
# pre-existing maintain limitation, not something graphify adds.
|
|
3009
|
+
self._save_maintain_state(state)
|
|
3010
|
+
|
|
3011
|
+
return {
|
|
3012
|
+
"ritual": "maintain", "dry_run": dry_run, "date": d.isoformat(),
|
|
3013
|
+
"weekday": d.strftime("%A"), "branches_due": branches,
|
|
3014
|
+
"results": results,
|
|
3015
|
+
"outcomes": maint.build_outcomes(auto_fixed, action_required, blocked),
|
|
3016
|
+
}
|
|
3017
|
+
finally:
|
|
3018
|
+
self._release_maintain_lock(lock_path)
|