sidegraph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sidegraph/__init__.py +37 -0
- sidegraph/anchoring.py +246 -0
- sidegraph/bootstrap/__init__.py +49 -0
- sidegraph/bootstrap/apply.py +603 -0
- sidegraph/bootstrap/catalog.py +92 -0
- sidegraph/bootstrap/cli.py +827 -0
- sidegraph/bootstrap/integrations.py +184 -0
- sidegraph/bootstrap/model.py +277 -0
- sidegraph/bootstrap/planner.py +400 -0
- sidegraph/bootstrap/proof.py +103 -0
- sidegraph/bootstrap/review.py +331 -0
- sidegraph/bootstrap/scan.py +289 -0
- sidegraph/capture.py +1794 -0
- sidegraph/cli.py +1902 -0
- sidegraph/config.py +148 -0
- sidegraph/doc_import.py +2099 -0
- sidegraph/doctor.py +1429 -0
- sidegraph/domains.py +902 -0
- sidegraph/engine/__init__.py +7 -0
- sidegraph/engine/reader.py +353 -0
- sidegraph/gitio.py +572 -0
- sidegraph/host/__init__.py +7 -0
- sidegraph/host/hooks.py +770 -0
- sidegraph/importer.py +239 -0
- sidegraph/okf.py +471 -0
- sidegraph/profiles.py +459 -0
- sidegraph/retrieval.py +1657 -0
- sidegraph/schema.py +386 -0
- sidegraph/server.py +2608 -0
- sidegraph/store.py +3363 -0
- sidegraph/sync.py +885 -0
- sidegraph/verify.py +1040 -0
- sidegraph/viz/__init__.py +4 -0
- sidegraph/viz/assets/vis-network.min.js +33 -0
- sidegraph/viz/model.py +248 -0
- sidegraph/viz/render.py +110 -0
- sidegraph/viz/template.html +131 -0
- sidegraph-0.1.0.dist-info/METADATA +392 -0
- sidegraph-0.1.0.dist-info/RECORD +42 -0
- sidegraph-0.1.0.dist-info/WHEEL +4 -0
- sidegraph-0.1.0.dist-info/entry_points.txt +18 -0
- sidegraph-0.1.0.dist-info/licenses/LICENSE +201 -0
sidegraph/doctor.py
ADDED
|
@@ -0,0 +1,1429 @@
|
|
|
1
|
+
"""Advisory curation lint — the advisory half of ``sidegraph-doctor``.
|
|
2
|
+
|
|
3
|
+
See design/superpowers/specs/2026-07-23-sidegraph-doctor-design.md. Strict conformance
|
|
4
|
+
lives in ``verify.py`` (pinned ``Violation`` codes, CI-gateable); this module is the
|
|
5
|
+
curation-quality complement: findings are advice, not contract breaches, and the CLI exit
|
|
6
|
+
code ignores them unless ``--check`` escalates.
|
|
7
|
+
|
|
8
|
+
**PURE READ**, same reasoning as ``verify.py``'s module docstring: never construct
|
|
9
|
+
:class:`sidegraph.store.Store` (its ``__init__`` can write — index refresh, legacy
|
|
10
|
+
migration). Canonical files are read directly via ``verify``'s loading helpers. The one
|
|
11
|
+
non-canonical source is ``index.db``, opened read-only for binding statuses only (Task 2)
|
|
12
|
+
— statuses are volatile, index-only state that canonical ``bindings/*.json`` files
|
|
13
|
+
deliberately do not carry (docs/reference/store-format.md).
|
|
14
|
+
|
|
15
|
+
A file that fails to parse yields NO finding here: surfacing corruption is
|
|
16
|
+
``verify_snapshot``'s job (``parse-error``), ``sidegraph-doctor`` always runs both
|
|
17
|
+
halves, and a second copy of the same complaint would be noise.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import re
|
|
24
|
+
import sqlite3
|
|
25
|
+
import subprocess
|
|
26
|
+
import time
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from datetime import UTC, datetime, timedelta
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Literal, NamedTuple
|
|
31
|
+
|
|
32
|
+
from ulid import ULID
|
|
33
|
+
|
|
34
|
+
from .schema import DecisionStatus, canonicalize
|
|
35
|
+
from .store import _STAMPING_MARKER_NAME, _TERMINAL_DECISION_STATUSES
|
|
36
|
+
from .verify import _check_archive_dir, _iter_json_files, _load_raw_json, _run_git
|
|
37
|
+
|
|
38
|
+
# Finding codes — pinned constants (design: "Advisory checks — pinned finding codes").
|
|
39
|
+
# ``--json`` output and every test assert against these exact strings.
|
|
40
|
+
DANGLING_RECORD = "dangling-record"
|
|
41
|
+
DEGRADED_BINDING = "degraded-binding"
|
|
42
|
+
ORPHANED_BINDING = "orphaned-binding"
|
|
43
|
+
STALE_PROPOSAL = "stale-proposal"
|
|
44
|
+
UNREFERENCED_ENTITY = "unreferenced-entity"
|
|
45
|
+
EXPIRED_OPEN_VALIDITY = "expired-open-validity"
|
|
46
|
+
DUPLICATE_ENTITY = "duplicate-entity"
|
|
47
|
+
CODE_DRIFT = "code-drift"
|
|
48
|
+
STALE_INSTRUCTIONS = "stale-instructions"
|
|
49
|
+
UNRATIFIED_ACCEPT = "unratified-accept"
|
|
50
|
+
|
|
51
|
+
# Name reported in ``CurationReport.skipped`` when index.db is absent or unusable — the
|
|
52
|
+
# binding-status check has no canonical source to fall back on (statuses are index-only).
|
|
53
|
+
BINDING_STATUS_CHECK = "binding-status"
|
|
54
|
+
|
|
55
|
+
NEVER_SURFACED = "never-surfaced"
|
|
56
|
+
|
|
57
|
+
# Same name as the finding code (matching BINDING_STATUS_CHECK's own precedent) — reported
|
|
58
|
+
# in ``CurationReport.skipped`` when index.db is absent or unusable: shows/queries are
|
|
59
|
+
# index-only derived state (design/superpowers/specs/2026-07-25-retrieval-telemetry-design.md)
|
|
60
|
+
# with no canonical fallback.
|
|
61
|
+
NEVER_SURFACED_CHECK = "never-surfaced"
|
|
62
|
+
|
|
63
|
+
# Terminal decision statuses, as strings -- imported from store.py rather than hand-rolled
|
|
64
|
+
# here so D4's "live" complement always matches _TERMINAL_DECISION_STATUSES exactly (it
|
|
65
|
+
# includes DEPRECATED; an earlier draft of the design spec got this wrong by hand-listing
|
|
66
|
+
# "superseded/rejected" and missing it). DecisionStatus is a StrEnum, so its members compare
|
|
67
|
+
# equal to the plain strings this module reads out of raw JSON.
|
|
68
|
+
_TERMINAL_DECISION_STATUS_VALUES = frozenset(s.value for s in _TERMINAL_DECISION_STATUSES)
|
|
69
|
+
|
|
70
|
+
# Statuses under which a decision is still "open" memory, i.e. a curation target. A
|
|
71
|
+
# superseded/rejected/deprecated record awaiting compaction is deliberately NOT flagged
|
|
72
|
+
# by dangling/expired checks (design: narrower than viz's dangling flag, which marks
|
|
73
|
+
# every record node). DERIVED as the terminal complement, never hand-listed (drift→supersede
|
|
74
|
+
# review M1: the hand-listed pair would silently diverge on a sixth DecisionStatus, the
|
|
75
|
+
# exact hazard the comment above already warns about).
|
|
76
|
+
_OPEN_DECISION_STATUSES = frozenset(s.value for s in DecisionStatus) - (
|
|
77
|
+
_TERMINAL_DECISION_STATUS_VALUES
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
# Entity-name prefixes exempt from unreferenced-entity: ``domain:*`` is the structural
|
|
81
|
+
# pair of a Domain row, minted at acceptance and legitimately bindingless until a Tier-1
|
|
82
|
+
# decision binds there; ``community:*`` is derived and no longer committed, but legacy
|
|
83
|
+
# stores may still carry decayed leftovers (store-format: "legacy committed entries decay
|
|
84
|
+
# lazily") and flagging those would tell users to hand-delete what sync retires itself.
|
|
85
|
+
_STRUCTURAL_ENTITY_PREFIXES = ("domain:", "community:")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True)
|
|
89
|
+
class Finding:
|
|
90
|
+
"""One curation finding. Same field shape as ``verify.Violation``, a distinct type on
|
|
91
|
+
purpose: findings are advice, violations are contract breaches — the CLI must never
|
|
92
|
+
mix the two lists."""
|
|
93
|
+
|
|
94
|
+
code: str
|
|
95
|
+
path: str
|
|
96
|
+
detail: str
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(frozen=True)
|
|
100
|
+
class CurationReport:
|
|
101
|
+
"""Everything one ``curate`` pass produced: findings plus the names of checks that
|
|
102
|
+
could not run (``skipped`` — e.g. ``binding-status`` without a usable index.db)."""
|
|
103
|
+
|
|
104
|
+
findings: list[Finding] = field(default_factory=list)
|
|
105
|
+
skipped: list[str] = field(default_factory=list)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# -- canonical-file loading (silent-skip: parse errors are verify's territory) -------------
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _iter_records(store_dir: Path, subdir: str) -> list[tuple[Path, dict]]:
|
|
112
|
+
"""(path, payload) for every parseable JSON-object file in ``<store>/<subdir>``,
|
|
113
|
+
sorted by path (findings inherit this order — deterministic output)."""
|
|
114
|
+
out: list[tuple[Path, dict]] = []
|
|
115
|
+
for path in _iter_json_files(store_dir / subdir):
|
|
116
|
+
payload, err = _load_raw_json(path)
|
|
117
|
+
if err is None and isinstance(payload, dict):
|
|
118
|
+
out.append((path, payload))
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _binding_entries(store_dir: Path) -> dict[str, list[dict]]:
|
|
123
|
+
"""record_id -> committed anchor entries, from ``bindings/<record_id>.json`` (each a
|
|
124
|
+
JSON LIST of identity dicts, no status — see store-format). Non-list payloads and
|
|
125
|
+
non-dict entries are skipped under the same silent-skip policy."""
|
|
126
|
+
out: dict[str, list[dict]] = {}
|
|
127
|
+
for path in _iter_json_files(store_dir / "bindings"):
|
|
128
|
+
payload, err = _load_raw_json(path)
|
|
129
|
+
if err is None and isinstance(payload, list):
|
|
130
|
+
out[path.stem] = [e for e in payload if isinstance(e, dict)]
|
|
131
|
+
return out
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# -- individual checks ---------------------------------------------------------------------
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _check_dangling_records(
|
|
138
|
+
decisions: list[tuple[Path, dict]],
|
|
139
|
+
facts: list[tuple[Path, dict]],
|
|
140
|
+
bindings: dict[str, list[dict]],
|
|
141
|
+
decision_status_by_id: dict[str, str],
|
|
142
|
+
) -> list[Finding]:
|
|
143
|
+
"""dangling-record: an OPEN record (decision proposed|accepted; fact valid_to null)
|
|
144
|
+
whose committed anchor set is absent or empty -- AND, for a fact only, whose
|
|
145
|
+
``supports`` also fails to resolve to a LIVE decision (design D4/D5/D6, external review
|
|
146
|
+
Defect B).
|
|
147
|
+
|
|
148
|
+
Decisions (D7, unchanged): a decision has no ``supports`` field, so "an anchor alone is
|
|
149
|
+
reachability" is still the whole test for that branch.
|
|
150
|
+
|
|
151
|
+
Facts are narrower, because the OLD anchor-only rule contradicted the write path's own
|
|
152
|
+
gate (``capture.py``'s reachability check accepts a standalone fact with EITHER an
|
|
153
|
+
anchor OR a supports id) and retrieval renders such a fact as inline evidence for any
|
|
154
|
+
decision that reaches a bucket -- i.e. every LIVE one (see ``retrieval.rank_decisions``
|
|
155
|
+
and its ``evidence=False`` calls for the terminal exception, D6). An anchorless fact is
|
|
156
|
+
dangling unless at least one ``supports`` id resolves to a LIVE (``proposed``/
|
|
157
|
+
``accepted``) decision:
|
|
158
|
+
|
|
159
|
+
- no supports at all -> reported, generic wording.
|
|
160
|
+
- every supports id fails to resolve to ANY known decision (hot or archived) -> reported
|
|
161
|
+
with D5's distinct wording -- a genuinely broken reference, a different problem from
|
|
162
|
+
simply having none, and the message says so instead of leaving the reader to open the
|
|
163
|
+
file.
|
|
164
|
+
- every supports id resolves, but only to TERMINAL decisions -> reported with D6's
|
|
165
|
+
wording: such a fact survives only in ``list_facts``/a direct ``facts_for_decision``
|
|
166
|
+
lookup, never in task-seeded retrieval, so "was reachable once, decayed" is exactly
|
|
167
|
+
what a curation check should say.
|
|
168
|
+
- a MIX of a missing id and a terminal id (no live id at all): the missing-id wording
|
|
169
|
+
wins -- a broken reference is worth naming over "merely" terminal.
|
|
170
|
+
"""
|
|
171
|
+
open_records: list[tuple[Path, dict, str]] = [
|
|
172
|
+
(p, d, "decision") for p, d in decisions if d.get("status") in _OPEN_DECISION_STATUSES
|
|
173
|
+
] + [(p, f, "fact") for p, f in facts if f.get("valid_to") is None]
|
|
174
|
+
findings: list[Finding] = []
|
|
175
|
+
for path, rec, kind in open_records:
|
|
176
|
+
rid = rec.get("id")
|
|
177
|
+
if not isinstance(rid, str):
|
|
178
|
+
continue
|
|
179
|
+
if bindings.get(rid):
|
|
180
|
+
continue # an anchor set alone is reachability, for either kind
|
|
181
|
+
if kind == "decision":
|
|
182
|
+
findings.append(
|
|
183
|
+
Finding(
|
|
184
|
+
DANGLING_RECORD,
|
|
185
|
+
str(path),
|
|
186
|
+
f"open {kind} has no committed anchor set "
|
|
187
|
+
f"(bindings/{rid}.json absent or empty)",
|
|
188
|
+
)
|
|
189
|
+
)
|
|
190
|
+
continue
|
|
191
|
+
|
|
192
|
+
raw_supports = rec.get("supports")
|
|
193
|
+
supports = (
|
|
194
|
+
[s for s in raw_supports if isinstance(s, str)]
|
|
195
|
+
if isinstance(raw_supports, list)
|
|
196
|
+
else []
|
|
197
|
+
)
|
|
198
|
+
statuses = [decision_status_by_id.get(sid) for sid in supports]
|
|
199
|
+
# D4's "live" is the complement of _TERMINAL_DECISION_STATUS_VALUES, checked
|
|
200
|
+
# directly against it (not a second, separately-maintained "open" set) so the two
|
|
201
|
+
# can never drift apart: a resolved, non-terminal status IS live by definition.
|
|
202
|
+
if any(s is not None and s not in _TERMINAL_DECISION_STATUS_VALUES for s in statuses):
|
|
203
|
+
continue # D4: reachable via a live supporting decision
|
|
204
|
+
if not supports:
|
|
205
|
+
detail = "no anchor set and no supporting decision"
|
|
206
|
+
elif any(s is None for s in statuses):
|
|
207
|
+
detail = "supports a decision that does not exist"
|
|
208
|
+
else:
|
|
209
|
+
detail = (
|
|
210
|
+
"supporting decision is superseded/rejected/deprecated -- reachable only "
|
|
211
|
+
"via list_facts, not task-seeded retrieval"
|
|
212
|
+
)
|
|
213
|
+
findings.append(Finding(DANGLING_RECORD, str(path), detail))
|
|
214
|
+
return findings
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _ulid_datetime(rid: str) -> datetime | None:
|
|
218
|
+
"""Creation time embedded in a ULID id, or None when the id isn't a parseable ULID
|
|
219
|
+
(hand-edited store; age is unknowable and the record silently doesn't flag)."""
|
|
220
|
+
try:
|
|
221
|
+
return ULID.from_str(rid).datetime
|
|
222
|
+
except (ValueError, TypeError):
|
|
223
|
+
return None
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _check_stale_proposals(
|
|
227
|
+
decisions: list[tuple[Path, dict]],
|
|
228
|
+
domains: list[tuple[Path, dict]],
|
|
229
|
+
stale_days: int,
|
|
230
|
+
now: datetime,
|
|
231
|
+
) -> list[Finding]:
|
|
232
|
+
"""stale-proposal: a proposed decision/domain whose ULID id timestamp is STRICTLY
|
|
233
|
+
older than ``stale_days`` — a rotting ratification queue."""
|
|
234
|
+
cutoff = now - timedelta(days=stale_days)
|
|
235
|
+
findings: list[Finding] = []
|
|
236
|
+
|
|
237
|
+
def check(path: Path, rec: dict, id_key: str, kind: str) -> None:
|
|
238
|
+
if rec.get("status") != "proposed":
|
|
239
|
+
return
|
|
240
|
+
rid = rec.get(id_key)
|
|
241
|
+
created = _ulid_datetime(rid) if isinstance(rid, str) else None
|
|
242
|
+
if created is None or created >= cutoff:
|
|
243
|
+
return
|
|
244
|
+
findings.append(
|
|
245
|
+
Finding(
|
|
246
|
+
STALE_PROPOSAL,
|
|
247
|
+
str(path),
|
|
248
|
+
f"{kind} proposed {(now - created).days} days ago (threshold {stale_days} days)",
|
|
249
|
+
)
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
for path, rec in decisions:
|
|
253
|
+
check(path, rec, "id", "decision")
|
|
254
|
+
for path, rec in domains:
|
|
255
|
+
check(path, rec, "domain_id", "domain")
|
|
256
|
+
return findings
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _check_unreferenced_entities(
|
|
260
|
+
entities: list[tuple[Path, dict]], bindings: dict[str, list[dict]]
|
|
261
|
+
) -> list[Finding]:
|
|
262
|
+
"""unreferenced-entity: a hot entity no committed anchor set references (bindings
|
|
263
|
+
files persist when their record is archived, so historical references still count).
|
|
264
|
+
Structural ``domain:*`` / ``community:*`` entities are exempt."""
|
|
265
|
+
referenced = {e.get("entity_id") for entries in bindings.values() for e in entries}
|
|
266
|
+
findings: list[Finding] = []
|
|
267
|
+
for path, ent in entities:
|
|
268
|
+
eid = ent.get("entity_id")
|
|
269
|
+
name = ent.get("canonical_name", "")
|
|
270
|
+
if not isinstance(eid, str) or not isinstance(name, str):
|
|
271
|
+
continue
|
|
272
|
+
if name.startswith(_STRUCTURAL_ENTITY_PREFIXES) or eid in referenced:
|
|
273
|
+
continue
|
|
274
|
+
findings.append(
|
|
275
|
+
Finding(
|
|
276
|
+
UNREFERENCED_ENTITY,
|
|
277
|
+
str(path),
|
|
278
|
+
f"entity {name!r} is referenced by no committed anchor set",
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
return findings
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _entity_identity_key(ent: dict) -> tuple[str, str, str | None] | None:
|
|
285
|
+
"""The logical identity a duplicate-entity group is keyed on. MUST mirror
|
|
286
|
+
``Store.find_entity`` exactly for concrete entities (``store.py:1365-1367`` at the time
|
|
287
|
+
of writing): ``canonicalize(descriptor.name if descriptor else canonical_name)`` paired
|
|
288
|
+
with ``descriptor.file_path if descriptor else None`` — not a looser
|
|
289
|
+
``canonicalize(canonical_name)``, or this check and the lookup it describes would
|
|
290
|
+
disagree about what a duplicate even is (design D5).
|
|
291
|
+
|
|
292
|
+
Abstract identity is ``kind`` + ``canonical_name`` (mirrors ``find_abstract_entity``).
|
|
293
|
+
One edge accepted rather than redesigned (design §3's own note): ``find_entity`` itself
|
|
294
|
+
is kind-blind, so a stray abstract row can satisfy a ``file_path=None`` concrete lookup
|
|
295
|
+
at runtime — but grouping here is kind-partitioned, so that pair would NOT be reported
|
|
296
|
+
as a duplicate-entity group. Not this check's problem to solve; named so it isn't
|
|
297
|
+
mistaken for a bug later."""
|
|
298
|
+
kind = ent.get("kind")
|
|
299
|
+
canonical_name = ent.get("canonical_name", "")
|
|
300
|
+
if not isinstance(canonical_name, str):
|
|
301
|
+
return None # malformed -- see below
|
|
302
|
+
if kind == "abstract":
|
|
303
|
+
return ("abstract", canonical_name, None)
|
|
304
|
+
descriptor = ent.get("descriptor")
|
|
305
|
+
if isinstance(descriptor, dict):
|
|
306
|
+
name = descriptor.get("name", canonical_name)
|
|
307
|
+
file_path = descriptor.get("file_path")
|
|
308
|
+
else:
|
|
309
|
+
name = canonical_name
|
|
310
|
+
file_path = None
|
|
311
|
+
# ``None`` for any shape whose identity cannot be computed, so the caller skips it.
|
|
312
|
+
# Returning here rather than letting canonicalize() raise is the module's stated policy
|
|
313
|
+
# -- one corrupt file must never abort the whole pass -- and it is what the sibling
|
|
314
|
+
# _check_unreferenced_entities enforces with its own isinstance guard. Without it an
|
|
315
|
+
# entity whose canonical_name is null took every other check down with it (branch
|
|
316
|
+
# review, Low-1). ``verify`` reports the malformed file itself; this check stays quiet.
|
|
317
|
+
if not isinstance(name, str) or not isinstance(file_path, str | None):
|
|
318
|
+
return None
|
|
319
|
+
return ("concrete", canonicalize(name), file_path)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _check_duplicate_entities(
|
|
323
|
+
entities: list[tuple[Path, dict]], bindings: dict[str, list[dict]]
|
|
324
|
+
) -> list[Finding]:
|
|
325
|
+
"""duplicate-entity: two or more committed entity files share one logical identity.
|
|
326
|
+
|
|
327
|
+
LEGAL, not corruption (design D2): two branches each minting the same name produce two
|
|
328
|
+
ULIDs -> two files -> a clean git merge, and a UNIQUE index would brick the store on
|
|
329
|
+
exactly that path (see ``test_store_opens_with_merged_duplicate.py``) rather than guard
|
|
330
|
+
it. But it IS ambiguous: lookups now deterministically resolve to the lowest
|
|
331
|
+
``entity_id`` (D4), so every OTHER id in the group is reliably unreachable by name --
|
|
332
|
+
reported once per group, at the WINNER's file, naming every id and its binding count
|
|
333
|
+
(the count is what makes "which one should absorb the others" an answerable question,
|
|
334
|
+
design D5).
|
|
335
|
+
|
|
336
|
+
``bindings`` is keyed by RECORD id (``bindings/<record_id>.json``, each a list of anchor
|
|
337
|
+
entries — see ``_binding_entries``), not by entity id, so counting "how many bindings
|
|
338
|
+
does this entity have" means flattening every record's entries first (same precondition
|
|
339
|
+
``_check_unreferenced_entities`` computes for its own ``referenced`` set)."""
|
|
340
|
+
binding_count_by_entity: dict[str, int] = {}
|
|
341
|
+
for entries in bindings.values():
|
|
342
|
+
for entry in entries:
|
|
343
|
+
eid = entry.get("entity_id")
|
|
344
|
+
if isinstance(eid, str):
|
|
345
|
+
binding_count_by_entity[eid] = binding_count_by_entity.get(eid, 0) + 1
|
|
346
|
+
|
|
347
|
+
groups: dict[tuple[str, str, str | None], list[tuple[Path, dict]]] = {}
|
|
348
|
+
for path, ent in entities:
|
|
349
|
+
eid = ent.get("entity_id")
|
|
350
|
+
if not isinstance(eid, str):
|
|
351
|
+
continue
|
|
352
|
+
key = _entity_identity_key(ent)
|
|
353
|
+
if key is None:
|
|
354
|
+
continue
|
|
355
|
+
groups.setdefault(key, []).append((path, ent))
|
|
356
|
+
|
|
357
|
+
findings: list[Finding] = []
|
|
358
|
+
for members in groups.values():
|
|
359
|
+
if len(members) < 2:
|
|
360
|
+
continue
|
|
361
|
+
members_sorted = sorted(members, key=lambda pair: pair[1]["entity_id"])
|
|
362
|
+
winner_path, _winner = members_sorted[0]
|
|
363
|
+
detail_ids = ", ".join(
|
|
364
|
+
f"{e['entity_id']} ({binding_count_by_entity.get(e['entity_id'], 0)} bindings)"
|
|
365
|
+
for _p, e in members_sorted
|
|
366
|
+
)
|
|
367
|
+
findings.append(
|
|
368
|
+
Finding(
|
|
369
|
+
DUPLICATE_ENTITY,
|
|
370
|
+
# The real path, root included -- every other file-anchored finding in this
|
|
371
|
+
# module reports str(path), and a hand-built "entities/<id>.json" made this
|
|
372
|
+
# one the odd row out for --json consumers (branch review, Low-2).
|
|
373
|
+
str(winner_path),
|
|
374
|
+
f"{len(members_sorted)} entities share one logical identity; lookups "
|
|
375
|
+
f"resolve to the lowest id: {detail_ids}",
|
|
376
|
+
)
|
|
377
|
+
)
|
|
378
|
+
return sorted(findings, key=lambda f: f.path)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _parse_aware_iso(ts: object) -> datetime | None:
|
|
382
|
+
"""Parse an ISO timestamp; None for non-strings, unparseable strings, and NAIVE
|
|
383
|
+
timestamps (schema requires AwareDatetime — a naive one is verify's territory, and
|
|
384
|
+
comparing it against an aware ``now`` would raise)."""
|
|
385
|
+
if not isinstance(ts, str):
|
|
386
|
+
return None
|
|
387
|
+
try:
|
|
388
|
+
parsed = datetime.fromisoformat(ts)
|
|
389
|
+
except ValueError:
|
|
390
|
+
return None
|
|
391
|
+
return parsed if parsed.tzinfo is not None else None
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _check_expired_open(decisions: list[tuple[Path, dict]], now: datetime) -> list[Finding]:
|
|
395
|
+
"""expired-open-validity: a decision closed-by-time (valid_to STRICTLY past) whose
|
|
396
|
+
status is still open — never superseded/rejected. Decisions only: Fact.status is
|
|
397
|
+
deprecated-unused and a fact with valid_to set is closed by convention."""
|
|
398
|
+
findings: list[Finding] = []
|
|
399
|
+
for path, d in decisions:
|
|
400
|
+
if d.get("status") not in _OPEN_DECISION_STATUSES:
|
|
401
|
+
continue
|
|
402
|
+
valid_to = _parse_aware_iso(d.get("valid_to"))
|
|
403
|
+
if valid_to is None or valid_to >= now:
|
|
404
|
+
continue
|
|
405
|
+
findings.append(
|
|
406
|
+
Finding(
|
|
407
|
+
EXPIRED_OPEN_VALIDITY,
|
|
408
|
+
str(path),
|
|
409
|
+
f"valid_to {d['valid_to']} is past but status is still {d['status']!r}",
|
|
410
|
+
)
|
|
411
|
+
)
|
|
412
|
+
return findings
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _check_binding_statuses(store_dir: Path) -> tuple[list[Finding], list[str]]:
|
|
416
|
+
"""degraded-binding / orphaned-binding: binding rows whose last RECORDED status has
|
|
417
|
+
decayed. Statuses live only in the derived ``index.db`` (canonical bindings files
|
|
418
|
+
carry identity, no status; sync flips statuses; a reload resets them to live), so
|
|
419
|
+
this check reads the index — strictly read-only — and reports "as of last sync".
|
|
420
|
+
|
|
421
|
+
``mode=ro`` cannot write and, on this store's DELETE journal, creates no sidecar files.
|
|
422
|
+
The old ``immutable`` connection flag was dropped deliberately: the coverage journal
|
|
423
|
+
makes ``index.db`` a write target on every tool call, and that flag disables locking
|
|
424
|
+
and change detection — sound only on a quiescent file. See the coverage-telemetry
|
|
425
|
+
design, D8.
|
|
426
|
+
|
|
427
|
+
An absent, corrupt, or pre-``anchor_bindings`` index.db is a SKIPPED check, never an
|
|
428
|
+
operational error — only the canonical store gates exit 1 (design ruling)."""
|
|
429
|
+
index_path = store_dir / "index.db"
|
|
430
|
+
if not index_path.is_file():
|
|
431
|
+
return [], [BINDING_STATUS_CHECK]
|
|
432
|
+
try:
|
|
433
|
+
conn = sqlite3.connect(f"file:{index_path}?mode=ro", uri=True)
|
|
434
|
+
try:
|
|
435
|
+
rows = conn.execute("SELECT record_id, entity_id, data FROM anchor_bindings").fetchall()
|
|
436
|
+
finally:
|
|
437
|
+
conn.close()
|
|
438
|
+
except sqlite3.Error:
|
|
439
|
+
return [], [BINDING_STATUS_CHECK]
|
|
440
|
+
|
|
441
|
+
code_by_status = {"degraded": DEGRADED_BINDING, "orphaned": ORPHANED_BINDING}
|
|
442
|
+
findings: list[Finding] = []
|
|
443
|
+
for record_id, entity_id, data in rows:
|
|
444
|
+
try:
|
|
445
|
+
payload = json.loads(data)
|
|
446
|
+
except (json.JSONDecodeError, TypeError):
|
|
447
|
+
continue
|
|
448
|
+
status = payload.get("status") if isinstance(payload, dict) else None
|
|
449
|
+
if not isinstance(status, str):
|
|
450
|
+
continue
|
|
451
|
+
code = code_by_status.get(status)
|
|
452
|
+
if code is None:
|
|
453
|
+
continue
|
|
454
|
+
findings.append(
|
|
455
|
+
Finding(
|
|
456
|
+
code,
|
|
457
|
+
f"bindings/{record_id}.json",
|
|
458
|
+
f"binding {record_id} -> {entity_id} is {status} as of last sync",
|
|
459
|
+
)
|
|
460
|
+
)
|
|
461
|
+
return findings, []
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _check_never_surfaced(store_dir: Path) -> tuple[list[Finding], list[str]]:
|
|
465
|
+
"""never-surfaced: decisions anchored where people repeatedly work that have never once
|
|
466
|
+
reached a render (spec D5).
|
|
467
|
+
|
|
468
|
+
Reads the index — strictly read-only, same ``mode=ro`` contract as
|
|
469
|
+
``_check_binding_statuses`` — because show and query counts are derived state and live
|
|
470
|
+
nowhere else. An absent or unusable index.db is SKIPPED, never an error: a store nobody
|
|
471
|
+
has read yet has no evidence of dead memory, and saying otherwise would flag a healthy
|
|
472
|
+
store on its first day.
|
|
473
|
+
|
|
474
|
+
Only ``0 shows / N>0 queries`` is reported. ``0 shows / 0 queries`` means nobody worked
|
|
475
|
+
in that area — an absence of occasion, not a defect — and reporting it would bury the
|
|
476
|
+
real signal under every decision in the store.
|
|
477
|
+
"""
|
|
478
|
+
index_path = store_dir / "index.db"
|
|
479
|
+
if not index_path.is_file():
|
|
480
|
+
return [], [NEVER_SURFACED_CHECK]
|
|
481
|
+
try:
|
|
482
|
+
conn = sqlite3.connect(f"file:{index_path}?mode=ro", uri=True)
|
|
483
|
+
try:
|
|
484
|
+
shows = dict(conn.execute("SELECT record_id, shows FROM retrieval_shows").fetchall())
|
|
485
|
+
queries = dict(conn.execute("SELECT seed, queries FROM retrieval_seeds").fetchall())
|
|
486
|
+
finally:
|
|
487
|
+
conn.close()
|
|
488
|
+
except sqlite3.Error:
|
|
489
|
+
return [], [NEVER_SURFACED_CHECK]
|
|
490
|
+
|
|
491
|
+
# entity_id -> file path, resolved from the CANONICAL entities (no index needed) — same
|
|
492
|
+
# source _check_unreferenced_entities already reads.
|
|
493
|
+
file_by_entity: dict[str, str] = {}
|
|
494
|
+
for _path, ent in _iter_records(store_dir, "entities"):
|
|
495
|
+
eid = ent.get("entity_id")
|
|
496
|
+
descriptor = ent.get("descriptor")
|
|
497
|
+
file_path = descriptor.get("file_path") if isinstance(descriptor, dict) else None
|
|
498
|
+
if isinstance(eid, str) and isinstance(file_path, str):
|
|
499
|
+
file_by_entity[eid] = file_path
|
|
500
|
+
|
|
501
|
+
bindings = _binding_entries(store_dir)
|
|
502
|
+
findings: list[Finding] = []
|
|
503
|
+
for path, decision in _iter_records(store_dir, "decisions"):
|
|
504
|
+
if decision.get("status") != "accepted":
|
|
505
|
+
continue
|
|
506
|
+
rid = decision.get("id")
|
|
507
|
+
if not isinstance(rid, str) or shows.get(rid, 0) != 0:
|
|
508
|
+
continue
|
|
509
|
+
anchored_paths = sorted(
|
|
510
|
+
{
|
|
511
|
+
file_by_entity[entry["entity_id"]]
|
|
512
|
+
for entry in bindings.get(rid, [])
|
|
513
|
+
if entry.get("entity_id") in file_by_entity
|
|
514
|
+
}
|
|
515
|
+
)
|
|
516
|
+
if not anchored_paths:
|
|
517
|
+
continue
|
|
518
|
+
total_queries = sum(queries.get(p, 0) for p in anchored_paths)
|
|
519
|
+
if total_queries == 0:
|
|
520
|
+
continue
|
|
521
|
+
findings.append(
|
|
522
|
+
Finding(
|
|
523
|
+
NEVER_SURFACED,
|
|
524
|
+
str(path),
|
|
525
|
+
f"anchored to {', '.join(anchored_paths)}, queried {total_queries} times, "
|
|
526
|
+
"never surfaced — check its anchors or the ranking",
|
|
527
|
+
)
|
|
528
|
+
)
|
|
529
|
+
return findings, []
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _file_by_entity(entities: list[tuple[Path, dict]]) -> dict[str, str]:
|
|
533
|
+
"""entity_id -> descriptor.file_path, for every concrete entity that has one. Abstract
|
|
534
|
+
entities carry ``descriptor: null`` and drop out naturally (design D5) -- exactly the
|
|
535
|
+
same shape ``_check_never_surfaced`` builds for its own file-coverage join, but taking
|
|
536
|
+
an already-loaded ``entities`` list (``curate`` loads it once, up front) rather than
|
|
537
|
+
re-reading the canonical directory a second time."""
|
|
538
|
+
out: dict[str, str] = {}
|
|
539
|
+
for _path, ent in entities:
|
|
540
|
+
eid = ent.get("entity_id")
|
|
541
|
+
descriptor = ent.get("descriptor")
|
|
542
|
+
file_path = descriptor.get("file_path") if isinstance(descriptor, dict) else None
|
|
543
|
+
if isinstance(eid, str) and isinstance(file_path, str):
|
|
544
|
+
out[eid] = file_path
|
|
545
|
+
return out
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _binding_status_by_key(store_dir: Path) -> dict[tuple[str, str], str]:
|
|
549
|
+
"""(record_id, entity_id) -> last-recorded binding status, read from index.db —
|
|
550
|
+
read-only, same ``mode=ro`` contract as ``_check_binding_statuses`` (design D5: "copy
|
|
551
|
+
the _check_binding_statuses precedent", doctor.py:405 at spec-writing time). Binding
|
|
552
|
+
status has no canonical form (store-format) so this is the only place it can be read
|
|
553
|
+
from. Returns ``{}`` when index.db is absent/unusable — code-drift's orphaned-binding
|
|
554
|
+
exclusion simply doesn't filter anything in that case; a missing/corrupt index does NOT
|
|
555
|
+
by itself skip the code-drift check (git availability is what gates that, see
|
|
556
|
+
``_check_code_drift``), so no separate ``skipped`` entry is added here."""
|
|
557
|
+
index_path = store_dir / "index.db"
|
|
558
|
+
if not index_path.is_file():
|
|
559
|
+
return {}
|
|
560
|
+
try:
|
|
561
|
+
conn = sqlite3.connect(f"file:{index_path}?mode=ro", uri=True)
|
|
562
|
+
try:
|
|
563
|
+
rows = conn.execute("SELECT record_id, entity_id, data FROM anchor_bindings").fetchall()
|
|
564
|
+
finally:
|
|
565
|
+
conn.close()
|
|
566
|
+
except sqlite3.Error:
|
|
567
|
+
return {}
|
|
568
|
+
out: dict[tuple[str, str], str] = {}
|
|
569
|
+
for record_id, entity_id, data in rows:
|
|
570
|
+
try:
|
|
571
|
+
payload = json.loads(data)
|
|
572
|
+
except (json.JSONDecodeError, TypeError):
|
|
573
|
+
continue
|
|
574
|
+
status = payload.get("status") if isinstance(payload, dict) else None
|
|
575
|
+
if isinstance(status, str):
|
|
576
|
+
out[(record_id, entity_id)] = status
|
|
577
|
+
return out
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _git_diff_name_only(
|
|
581
|
+
repo_root: Path, commit: str, paths: list[str], *, timeout: float | None = None
|
|
582
|
+
) -> list[str]:
|
|
583
|
+
"""``git diff --name-only <commit>..HEAD -- <paths>``, from ``repo_root`` (design D5).
|
|
584
|
+
Raises ``ValueError`` on a non-zero exit -- mirrors ``verify._git_diff_name_status``'s
|
|
585
|
+
own raise-on-failure contract -- so every call site wraps this the same way that
|
|
586
|
+
module's callers wrap ITS git helpers; ``_scan_code_drift`` is the one caller here, and
|
|
587
|
+
it turns any failure into a failed batch (rendered as a skipped-note finding) rather
|
|
588
|
+
than letting it propagate. ``timeout`` (drift→supersede D1) bounds the subprocess; the
|
|
589
|
+
curate()/CLI path passes none, exactly today's behavior."""
|
|
590
|
+
result = _run_git(
|
|
591
|
+
["diff", "--name-only", f"{commit}..HEAD", "--", *paths], cwd=repo_root, timeout=timeout
|
|
592
|
+
)
|
|
593
|
+
if result.returncode != 0:
|
|
594
|
+
raise ValueError(f"git diff {commit}..HEAD failed: {result.stderr.strip()}")
|
|
595
|
+
return [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
_CODE_DRIFT_GIT_NOTE = "code-drift: git unavailable — check skipped"
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
class DriftEntry(NamedTuple):
|
|
602
|
+
"""One live, commit-stamped decision whose anchored file(s) changed since capture —
|
|
603
|
+
the fields ``_check_code_drift`` needs to render today's finding exactly, plus the
|
|
604
|
+
``record_id`` the drift cache (sync.refresh_code_drift_cache) keys on.
|
|
605
|
+
# see design/superpowers/specs/2026-07-30-drift-supersede-affordance-design.md (D1)
|
|
606
|
+
"""
|
|
607
|
+
|
|
608
|
+
record_path: str
|
|
609
|
+
record_id: str
|
|
610
|
+
files: list[str]
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
class DriftBatch(NamedTuple):
|
|
614
|
+
"""One distinct ``provenance.commit``'s diff outcome, in ``by_commit`` iteration order
|
|
615
|
+
— findings order IS this order, including a failed batch's git-note emitted in place
|
|
616
|
+
(the mid-list interleaving the round-1 review pinned as a refactor constraint)."""
|
|
617
|
+
|
|
618
|
+
commit: str
|
|
619
|
+
failed: bool
|
|
620
|
+
entries: list[DriftEntry]
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
class CodeDriftScan(NamedTuple):
|
|
624
|
+
"""The structured result the drift→supersede surfaces consume (D1). ``active`` False
|
|
625
|
+
reproduces the check's silent contract (nothing commit-stamped AND anchored);
|
|
626
|
+
``repo_root_failed`` is the no-batch-ran case (one note, distinct from per-batch
|
|
627
|
+
failures); ``head`` is resolved ONLY by the load-inclusive :func:`scan_code_drift`
|
|
628
|
+
wrapper (never on the ``curate()`` path — its git-call count must not change) and is
|
|
629
|
+
cache provenance only, it gates nothing."""
|
|
630
|
+
|
|
631
|
+
batches: list[DriftBatch]
|
|
632
|
+
unstamped: int
|
|
633
|
+
active: bool
|
|
634
|
+
repo_root_failed: bool
|
|
635
|
+
head: str | None
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _scan_code_drift(
|
|
639
|
+
decisions: list[tuple[Path, dict]],
|
|
640
|
+
bindings: dict[str, list[dict]],
|
|
641
|
+
entities: list[tuple[Path, dict]],
|
|
642
|
+
store_dir: Path,
|
|
643
|
+
repo_root: Path | None,
|
|
644
|
+
*,
|
|
645
|
+
deadline: float | None = None,
|
|
646
|
+
resolve_head: bool = False,
|
|
647
|
+
) -> CodeDriftScan:
|
|
648
|
+
"""The pure scan half of the code-drift check (D1): record → committed bindings →
|
|
649
|
+
entity ``file_path`` join, orphaned-binding exclusion, one batched diff per distinct
|
|
650
|
+
commit. Never raises; every git failure lands in a batch's ``failed`` flag (or
|
|
651
|
+
``repo_root_failed``). ``deadline`` is a TOTAL budget in seconds (round-2 N2, never
|
|
652
|
+
per-call): each subprocess gets the remaining time, and once exhausted every un-run
|
|
653
|
+
batch is marked ``failed`` without spawning anything — hook cost is bounded by the
|
|
654
|
+
deadline itself, not by ``distinct_commits × per-call-timeout``. EVERY git call this
|
|
655
|
+
scan makes — root resolution, the batch diffs, and (``resolve_head=True``, the
|
|
656
|
+
:func:`scan_code_drift` wrapper only — never the ``curate()`` path, N6) the final
|
|
657
|
+
``rev-parse HEAD`` — draws from the ONE budget (code review round 1, I-1/M-1: the
|
|
658
|
+
root used to be resolved through ``verify._find_repo_root`` with no timeout at all,
|
|
659
|
+
then re-resolved a second time in the wrapper)."""
|
|
660
|
+
file_by_entity = _file_by_entity(entities)
|
|
661
|
+
status_by_key = _binding_status_by_key(store_dir)
|
|
662
|
+
|
|
663
|
+
unstamped = 0
|
|
664
|
+
by_commit: dict[str, list[tuple[Path, str, set[str]]]] = {}
|
|
665
|
+
for path, d in decisions:
|
|
666
|
+
if d.get("status") not in _OPEN_DECISION_STATUSES:
|
|
667
|
+
continue
|
|
668
|
+
prov = d.get("provenance")
|
|
669
|
+
commit = prov.get("commit") if isinstance(prov, dict) else None
|
|
670
|
+
rid = d.get("id")
|
|
671
|
+
if not isinstance(commit, str) or not commit:
|
|
672
|
+
unstamped += 1
|
|
673
|
+
continue
|
|
674
|
+
if not isinstance(rid, str):
|
|
675
|
+
continue
|
|
676
|
+
paths: set[str] = set()
|
|
677
|
+
for entry in bindings.get(rid, []):
|
|
678
|
+
eid = entry.get("entity_id")
|
|
679
|
+
if not isinstance(eid, str):
|
|
680
|
+
continue
|
|
681
|
+
if status_by_key.get((rid, eid)) == "orphaned":
|
|
682
|
+
continue
|
|
683
|
+
fp = file_by_entity.get(eid)
|
|
684
|
+
if fp:
|
|
685
|
+
paths.add(fp)
|
|
686
|
+
if paths:
|
|
687
|
+
by_commit.setdefault(commit, []).append((path, rid, paths))
|
|
688
|
+
|
|
689
|
+
if not by_commit:
|
|
690
|
+
return CodeDriftScan([], unstamped, False, False, None)
|
|
691
|
+
|
|
692
|
+
started = time.monotonic()
|
|
693
|
+
|
|
694
|
+
def _remaining() -> float | None:
|
|
695
|
+
if deadline is None:
|
|
696
|
+
return None
|
|
697
|
+
return deadline - (time.monotonic() - started)
|
|
698
|
+
|
|
699
|
+
if repo_root is None:
|
|
700
|
+
# Inline rather than `verify._find_repo_root` so the call draws on the scan
|
|
701
|
+
# budget — the helper has no timeout seam, and this used to be the one unbounded
|
|
702
|
+
# git call on the hook paths (review I-1). Same command, same cwd, same
|
|
703
|
+
# `.resolve()`; a non-zero exit or any failure is the same repo_root_failed scan
|
|
704
|
+
# the helper's raise produced.
|
|
705
|
+
try:
|
|
706
|
+
result = _run_git(["rev-parse", "--show-toplevel"], cwd=store_dir, timeout=_remaining())
|
|
707
|
+
if result.returncode != 0:
|
|
708
|
+
return CodeDriftScan([], unstamped, True, True, None)
|
|
709
|
+
repo_root = Path(result.stdout.strip()).resolve()
|
|
710
|
+
except (ValueError, OSError, subprocess.TimeoutExpired):
|
|
711
|
+
return CodeDriftScan([], unstamped, True, True, None)
|
|
712
|
+
|
|
713
|
+
batches: list[DriftBatch] = []
|
|
714
|
+
for commit, items in by_commit.items():
|
|
715
|
+
remaining = _remaining()
|
|
716
|
+
if remaining is not None and remaining <= 0:
|
|
717
|
+
batches.append(DriftBatch(commit, True, []))
|
|
718
|
+
continue
|
|
719
|
+
all_paths = sorted({p for _path, _rid, paths in items for p in paths})
|
|
720
|
+
try:
|
|
721
|
+
changed = set(_git_diff_name_only(repo_root, commit, all_paths, timeout=remaining))
|
|
722
|
+
except (ValueError, OSError, subprocess.TimeoutExpired):
|
|
723
|
+
batches.append(DriftBatch(commit, True, []))
|
|
724
|
+
continue
|
|
725
|
+
entries = [
|
|
726
|
+
DriftEntry(str(path), rid, drifted)
|
|
727
|
+
for path, rid, paths in items
|
|
728
|
+
if (drifted := sorted(paths & changed))
|
|
729
|
+
]
|
|
730
|
+
batches.append(DriftBatch(commit, False, entries))
|
|
731
|
+
|
|
732
|
+
head: str | None = None
|
|
733
|
+
if resolve_head:
|
|
734
|
+
remaining = _remaining()
|
|
735
|
+
if remaining is None or remaining > 0:
|
|
736
|
+
try:
|
|
737
|
+
result = _run_git(["rev-parse", "HEAD"], cwd=repo_root, timeout=remaining)
|
|
738
|
+
if result.returncode == 0:
|
|
739
|
+
head = result.stdout.strip() or None
|
|
740
|
+
except (ValueError, OSError, subprocess.TimeoutExpired):
|
|
741
|
+
head = None
|
|
742
|
+
return CodeDriftScan(batches, unstamped, True, False, head)
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def scan_code_drift(
|
|
746
|
+
store_dir: str | Path,
|
|
747
|
+
repo_root: Path | None = None,
|
|
748
|
+
*,
|
|
749
|
+
deadline: float | None = None,
|
|
750
|
+
) -> CodeDriftScan:
|
|
751
|
+
"""Load-inclusive public entry point for the drift→supersede surfaces (D1): loads the
|
|
752
|
+
three record sets itself (the same helpers ``curate()`` uses — no ``Store``, this
|
|
753
|
+
module stays pure-read) and asks the scan to resolve ``head`` (``git rev-parse
|
|
754
|
+
HEAD``) for cache provenance. ``resolve_head`` is set ONLY here, never on the
|
|
755
|
+
``curate()`` path (round-2 N6), and an inactive store still makes zero git calls
|
|
756
|
+
(N5). Every git call — root resolution, diffs, head — draws on the ONE ``deadline``
|
|
757
|
+
(review I-1/M-1), so hook cost is bounded by the deadline itself."""
|
|
758
|
+
root = Path(store_dir)
|
|
759
|
+
decisions = _iter_records(root, "decisions")
|
|
760
|
+
entities = _iter_records(root, "entities")
|
|
761
|
+
bindings = _binding_entries(root)
|
|
762
|
+
return _scan_code_drift(
|
|
763
|
+
decisions, bindings, entities, root, repo_root, deadline=deadline, resolve_head=True
|
|
764
|
+
)
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def _check_code_drift(
|
|
768
|
+
decisions: list[tuple[Path, dict]],
|
|
769
|
+
bindings: dict[str, list[dict]],
|
|
770
|
+
entities: list[tuple[Path, dict]],
|
|
771
|
+
store_dir: Path,
|
|
772
|
+
repo_root: Path | None,
|
|
773
|
+
) -> list[Finding]:
|
|
774
|
+
"""code-drift: a LIVE decision's anchored file has changed since its capture-time HEAD
|
|
775
|
+
(design D5) -- the UPDATE-half curation candidate list this whole wave exists for.
|
|
776
|
+
|
|
777
|
+
"Live" is the same open-record set every other check in this module reasons about
|
|
778
|
+
(:data:`_OPEN_DECISION_STATUSES` -- proposed/accepted; a superseded/rejected/deprecated
|
|
779
|
+
decision is historical, not a drift target). For each live decision whose
|
|
780
|
+
``provenance.commit`` is set, this resolves record -> committed bindings -> entities ->
|
|
781
|
+
``descriptor.file_path`` (a pure in-memory join over already-loaded canonical JSON),
|
|
782
|
+
excluding any binding index.db records as ``orphaned`` (:func:`_binding_status_by_key`
|
|
783
|
+
-- a degraded binding is still included, same as ``valid_decisions_for_entity``'s own
|
|
784
|
+
live/degraded-vs-orphaned split). Records with NO commit (every pre-wave record, by
|
|
785
|
+
design -- commits are never backfilled) are counted once, not silently dropped:
|
|
786
|
+
``"N record(s) predate commit stamping — not checkable"``.
|
|
787
|
+
|
|
788
|
+
That unstamped-count note (and the whole check) stays SILENT when there is not one
|
|
789
|
+
single commit-stamped, checkable decision anywhere in the store: every decision written
|
|
790
|
+
before this wave shipped lacks ``provenance.commit`` by construction (§5/§6, "pre-wave
|
|
791
|
+
records are never code-drift-checkable"), so an unconditional note would surface on
|
|
792
|
+
every pre-existing store forever, permanently costing it a clean ``sidegraph-doctor``
|
|
793
|
+
run for a wave it never opted into. The check only starts talking once it has at least
|
|
794
|
+
one commit-stamped, anchored decision to actually check — at which point the unstamped
|
|
795
|
+
count rides alongside as informational context about the rest of the store.
|
|
796
|
+
|
|
797
|
+
Anchored file sets are grouped by their decision's ``provenance.commit`` so ``git diff``
|
|
798
|
+
runs ONCE per distinct commit (batched), never once per decision. Never raises and never
|
|
799
|
+
silently reports zero on a git failure (design ruling; C1's silent-zero shape rejected):
|
|
800
|
+
a failed batch yields NO drift findings for that commit plus one
|
|
801
|
+
:data:`_CODE_DRIFT_GIT_NOTE` finding, IN PLACE in the batch order, so a git-less run
|
|
802
|
+
reads as "check skipped", never as a clean pass.
|
|
803
|
+
|
|
804
|
+
Since the drift→supersede wave this is a pure renderer over :func:`_scan_code_drift`
|
|
805
|
+
(D1) — the scan owns the join and the git mechanics; this function owns only today's
|
|
806
|
+
exact finding shapes and order, which the pre-wave test suite pins unchanged.
|
|
807
|
+
"""
|
|
808
|
+
scan = _scan_code_drift(decisions, bindings, entities, store_dir, repo_root)
|
|
809
|
+
if not scan.active:
|
|
810
|
+
# Nothing in the store is commit-stamped AND anchored -- the check has literally
|
|
811
|
+
# nothing to say, unstamped count included (see this function's docstring).
|
|
812
|
+
return []
|
|
813
|
+
|
|
814
|
+
findings: list[Finding] = []
|
|
815
|
+
if scan.unstamped:
|
|
816
|
+
findings.append(
|
|
817
|
+
Finding(
|
|
818
|
+
CODE_DRIFT,
|
|
819
|
+
str(store_dir),
|
|
820
|
+
f"{scan.unstamped} record(s) predate commit stamping — not checkable",
|
|
821
|
+
)
|
|
822
|
+
)
|
|
823
|
+
if scan.repo_root_failed:
|
|
824
|
+
findings.append(Finding(CODE_DRIFT, str(store_dir), _CODE_DRIFT_GIT_NOTE))
|
|
825
|
+
return findings
|
|
826
|
+
for batch in scan.batches:
|
|
827
|
+
if batch.failed:
|
|
828
|
+
findings.append(Finding(CODE_DRIFT, str(store_dir), _CODE_DRIFT_GIT_NOTE))
|
|
829
|
+
continue
|
|
830
|
+
for entry in batch.entries:
|
|
831
|
+
findings.append(
|
|
832
|
+
Finding(
|
|
833
|
+
CODE_DRIFT,
|
|
834
|
+
entry.record_path,
|
|
835
|
+
f"anchored file(s) changed since capture (commit {batch.commit[:8]}): "
|
|
836
|
+
+ ", ".join(entry.files),
|
|
837
|
+
)
|
|
838
|
+
)
|
|
839
|
+
return findings
|
|
840
|
+
|
|
841
|
+
|
|
842
|
+
# -- entry point ---------------------------------------------------------------------------
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
# Files a host auto-injects into every session's turn-zero context. The whitepaper's
|
|
846
|
+
# §12.6 finding is why this check exists: that file wins by position, so a superseded
|
|
847
|
+
# rule surviving in it outranks the store's correction by default.
|
|
848
|
+
_INSTRUCTION_FILES = ("CLAUDE.md", "AGENTS.md", "GEMINI.md", ".cursorrules")
|
|
849
|
+
|
|
850
|
+
# A phrase must be this distinctive before a match means anything: short titles ("Use
|
|
851
|
+
# SQLite", "Retry policy") collide with ordinary prose and would make the check noise.
|
|
852
|
+
_MIN_PHRASE_CHARS = 28
|
|
853
|
+
_MIN_PHRASE_WORDS = 5
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def _normalize_phrase(text: str) -> str:
|
|
857
|
+
"""Lowercased, whitespace-collapsed, punctuation-light form for substring matching.
|
|
858
|
+
|
|
859
|
+
Deliberately NOT a semantic comparison: this check reports a VERBATIM-ish quotation
|
|
860
|
+
of a superseded record's own wording, which is a fact a script can establish. Whether
|
|
861
|
+
the file still *teaches* the abandoned rule (rather than narrating its history) is the
|
|
862
|
+
human judgment the finding asks for — see the finding's wording.
|
|
863
|
+
"""
|
|
864
|
+
lowered = re.sub(r"[`*_#>\[\]()]", " ", text.lower())
|
|
865
|
+
return re.sub(r"\s+", " ", lowered).strip()
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def _check_stale_instructions(
|
|
869
|
+
decisions: list[tuple[Path, dict]], repo_root: Path | None
|
|
870
|
+
) -> list[Finding]:
|
|
871
|
+
"""stale-instructions: an auto-injected agent-instructions file still carries the
|
|
872
|
+
distinctive wording of a SUPERSEDED decision.
|
|
873
|
+
|
|
874
|
+
Advisory and deliberately conservative: title-length and word-count floors, superseded
|
|
875
|
+
records only (an accepted record's wording SHOULD appear there), and a wording that
|
|
876
|
+
names the possibility rather than asserting a contradiction. ``repo_root`` is the
|
|
877
|
+
directory the instructions files are looked up in — the store's parent in the standard
|
|
878
|
+
layout; no git call, no search.
|
|
879
|
+
"""
|
|
880
|
+
if repo_root is None:
|
|
881
|
+
return []
|
|
882
|
+
phrases: list[tuple[str, str]] = [] # (normalized phrase, record id)
|
|
883
|
+
for _path, rec in decisions:
|
|
884
|
+
if rec.get("status") != "superseded":
|
|
885
|
+
continue
|
|
886
|
+
rid = rec.get("id")
|
|
887
|
+
title = rec.get("title")
|
|
888
|
+
if not isinstance(rid, str) or not isinstance(title, str):
|
|
889
|
+
continue
|
|
890
|
+
norm = _normalize_phrase(title)
|
|
891
|
+
if len(norm) >= _MIN_PHRASE_CHARS and len(norm.split()) >= _MIN_PHRASE_WORDS:
|
|
892
|
+
phrases.append((norm, rid))
|
|
893
|
+
if not phrases:
|
|
894
|
+
return []
|
|
895
|
+
|
|
896
|
+
findings: list[Finding] = []
|
|
897
|
+
for name in _INSTRUCTION_FILES:
|
|
898
|
+
f = repo_root / name
|
|
899
|
+
try:
|
|
900
|
+
if not f.is_file():
|
|
901
|
+
continue
|
|
902
|
+
haystack = _normalize_phrase(f.read_text(encoding="utf-8"))
|
|
903
|
+
except (OSError, UnicodeDecodeError):
|
|
904
|
+
continue
|
|
905
|
+
for norm, rid in phrases:
|
|
906
|
+
if norm in haystack:
|
|
907
|
+
findings.append(
|
|
908
|
+
Finding(
|
|
909
|
+
STALE_INSTRUCTIONS,
|
|
910
|
+
str(f),
|
|
911
|
+
f"quotes the wording of superseded decision {rid} "
|
|
912
|
+
f'("{norm[:60]}…") — an auto-injected file reaches the agent '
|
|
913
|
+
"before retrieval does, so verify it no longer teaches the "
|
|
914
|
+
"abandoned rule",
|
|
915
|
+
)
|
|
916
|
+
)
|
|
917
|
+
return findings
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def _read_stamping_marker(store_dir: Path) -> datetime | None:
|
|
921
|
+
"""The store's creation marker (``store.py``'s ``_STAMPING_MARKER_NAME``, written
|
|
922
|
+
once — and only once — by ``Store._ensure_stamping_marker`` the moment a store is
|
|
923
|
+
found genuinely new), or ``None`` when it's absent: every store predating that
|
|
924
|
+
feature, and any store that already held a record of any kind (hot or archived) the
|
|
925
|
+
first time a stamping version opened it, never has this file. Read directly off disk
|
|
926
|
+
(this module never constructs ``Store`` — see the module docstring).
|
|
927
|
+
|
|
928
|
+
A missing, unreadable, or unparseable marker degrades to ``None`` the same silent-skip
|
|
929
|
+
way every other read in this module does. Unlike most such reads, this one is NOT
|
|
930
|
+
covered elsewhere: ``verify_snapshot`` has no check on this file at all (measured,
|
|
931
|
+
review round 2, Minor 4 — a prior version of this docstring claimed otherwise; it was
|
|
932
|
+
wrong). A hand-edited, corrupt, or future-dated marker therefore silently re-opens the
|
|
933
|
+
exact blind window this whole mechanism exists to close — ``_check_unratified_accepts``
|
|
934
|
+
falls back to stamp-only scoping (or ``[]`` with no stamps either) with no finding, no
|
|
935
|
+
warning, anywhere, ever pointing at the marker itself as the reason. Recorded here as a
|
|
936
|
+
known gap, not fixed in this round: closing it is a `verify.py` change (a new pinned
|
|
937
|
+
``Violation`` code for an unparseable/naive/future marker), a different module's
|
|
938
|
+
contract, out of scope for this fix."""
|
|
939
|
+
try:
|
|
940
|
+
text = (store_dir / _STAMPING_MARKER_NAME).read_text(encoding="utf-8")
|
|
941
|
+
except (OSError, UnicodeDecodeError):
|
|
942
|
+
return None
|
|
943
|
+
return _parse_aware_iso(text.strip())
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
# Tier-0 entity kinds only the propose path ever binds fresh onto a superseding record --
|
|
947
|
+
# review Major 1 (fix round 2). ``supersede_decision``/``supersede_fact`` never creates
|
|
948
|
+
# either: with anchors omitted it copies the predecessor's bindings verbatim (never a NEW
|
|
949
|
+
# one), and with explicit anchors it has no ``tags``/``initiative`` parameter at all to bind
|
|
950
|
+
# one from. Only ``add_decision`` (source ``"human"``), ``doc_import`` (source
|
|
951
|
+
# ``"doc-import"``) and the propose/``auto_accept`` path (source ``"agent"``) ever mint
|
|
952
|
+
# these, so for an agent-sourced, stamp-less, ``supersedes``-bearing record, one of these
|
|
953
|
+
# present and NOT already on the predecessor is unambiguous: the propose path wrote it.
|
|
954
|
+
_PROPOSE_ONLY_TIER0_PREFIXES = ("tag:", "initiative:")
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def _tier0_tag_or_initiative_entity_ids(
|
|
958
|
+
record_id: str | None,
|
|
959
|
+
bindings: dict[str, list[dict]],
|
|
960
|
+
entity_canonical_by_id: dict[str, str],
|
|
961
|
+
) -> set[str]:
|
|
962
|
+
"""``entity_id``s of ``record_id``'s own committed tier-0 ``tag:``/``initiative:``
|
|
963
|
+
bindings -- ``set()`` for ``None``/an unknown id/a record with no bindings at all,
|
|
964
|
+
which correctly makes "absent from the predecessor" true by default when the
|
|
965
|
+
predecessor itself cannot be resolved (never a false negative on the propose-path
|
|
966
|
+
signal)."""
|
|
967
|
+
if not isinstance(record_id, str):
|
|
968
|
+
return set()
|
|
969
|
+
out: set[str] = set()
|
|
970
|
+
for b in bindings.get(record_id, []):
|
|
971
|
+
if not isinstance(b, dict) or b.get("tier") != 0:
|
|
972
|
+
continue
|
|
973
|
+
entity_id = b.get("entity_id")
|
|
974
|
+
if not isinstance(entity_id, str):
|
|
975
|
+
continue
|
|
976
|
+
name = entity_canonical_by_id.get(entity_id)
|
|
977
|
+
if isinstance(name, str) and name.startswith(_PROPOSE_ONLY_TIER0_PREFIXES):
|
|
978
|
+
out.add(entity_id)
|
|
979
|
+
return out
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
def _has_propose_only_signal(
|
|
983
|
+
rec: dict,
|
|
984
|
+
bindings: dict[str, list[dict]],
|
|
985
|
+
entity_canonical_by_id: dict[str, str],
|
|
986
|
+
) -> bool:
|
|
987
|
+
"""True iff ``rec`` (a ``supersedes``-bearing, accepted, agent-sourced, stamp-less
|
|
988
|
+
record) carries at least one field only the propose/``auto_accept`` path can produce —
|
|
989
|
+
review Major 1: ``layer``, a ``provenance.ref``, or a tier-0 ``tag:``/``initiative:``
|
|
990
|
+
binding not already on its predecessor. One-directional by construction (see the
|
|
991
|
+
constant above's comment): presence proves propose wrote it; absence does NOT prove
|
|
992
|
+
``supersede_decision``/``supersede_fact`` wrote it -- a minimal propose draft (no
|
|
993
|
+
layer/ref/tags) is genuinely indistinguishable from a genuine successor, and stays
|
|
994
|
+
excluded either way (see the check's own docstring)."""
|
|
995
|
+
if rec.get("layer"):
|
|
996
|
+
return True
|
|
997
|
+
provenance = rec.get("provenance")
|
|
998
|
+
if isinstance(provenance, dict) and provenance.get("ref"):
|
|
999
|
+
return True
|
|
1000
|
+
successor_tags = _tier0_tag_or_initiative_entity_ids(
|
|
1001
|
+
rec.get("id"), bindings, entity_canonical_by_id
|
|
1002
|
+
)
|
|
1003
|
+
if not successor_tags:
|
|
1004
|
+
return False
|
|
1005
|
+
predecessor_tags = _tier0_tag_or_initiative_entity_ids(
|
|
1006
|
+
rec.get("supersedes"), bindings, entity_canonical_by_id
|
|
1007
|
+
)
|
|
1008
|
+
return bool(successor_tags - predecessor_tags)
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
def _check_unratified_accepts(
|
|
1012
|
+
decisions: list[tuple[Path, dict]],
|
|
1013
|
+
facts: list[tuple[Path, dict]],
|
|
1014
|
+
store_dir: Path,
|
|
1015
|
+
bindings: dict[str, list[dict]],
|
|
1016
|
+
entities: list[tuple[Path, dict]],
|
|
1017
|
+
) -> list[Finding]:
|
|
1018
|
+
"""unratified-accept: an ACCEPTED, agent-sourced record carrying no ratifier stamp —
|
|
1019
|
+
the signature `SIDEGRAPH_AUTO_ACCEPT=on` leaves behind.
|
|
1020
|
+
|
|
1021
|
+
That switch is per-environment (one developer's shell) but writes into the SHARED
|
|
1022
|
+
committed store, so the ratification gate can be bypassed for everyone by a setting
|
|
1023
|
+
nobody else sees. Three of four practitioner reviewers flagged it independently; this
|
|
1024
|
+
makes it auditable in a PR diff and in CI instead of trust-based.
|
|
1025
|
+
|
|
1026
|
+
The signature is genuinely ambiguous for records ratified before the stamp existed, so
|
|
1027
|
+
the check is SCOPED to the period where it can discriminate. The scope start is the
|
|
1028
|
+
EARLIER of: the store's earliest ``ratified_at`` stamp, and its creation marker
|
|
1029
|
+
(``stamping_live_since`` — see ``_read_stamping_marker``/``Store._ensure_stamping_marker``),
|
|
1030
|
+
when either exists. Records older than that scope start are skipped entirely.
|
|
1031
|
+
|
|
1032
|
+
Without a creation marker (every store predating that feature), this is exactly the
|
|
1033
|
+
original rule: a store with no stamps at all produces no findings, because nothing
|
|
1034
|
+
in it can discriminate. Measured before shipping — unscoped, this check emitted 239
|
|
1035
|
+
findings on this project's own store (every agent-sourced record predating the
|
|
1036
|
+
feature), which is not a signal, it is a wall; that wall stays shut for any store
|
|
1037
|
+
that has never been opened by a marker-writing version.
|
|
1038
|
+
|
|
1039
|
+
With a creation marker, "a store with no stamps at all produces no findings" is no
|
|
1040
|
+
longer universally true, by design: the marker records the instant stamping became
|
|
1041
|
+
live for THAT store, so records created at/after it are judgeable whether or not any
|
|
1042
|
+
ratification has happened yet. This closes a blind window the stamp-only rule left
|
|
1043
|
+
open — a store adopting an auto-ratification policy from scratch has never ratified
|
|
1044
|
+
anything, so it had no stamp to scope from and produced zero findings forever, even
|
|
1045
|
+
after a later ratification finally armed the check (every existing record already
|
|
1046
|
+
predated that first stamp). Taking the EARLIER of the two bounds when both exist
|
|
1047
|
+
means a store that was created new and only later ratified something does not lose
|
|
1048
|
+
coverage of its own first window either.
|
|
1049
|
+
|
|
1050
|
+
The finding's wording still says the signature is not proof, because an advisory that
|
|
1051
|
+
overstates is one people learn to skip.
|
|
1052
|
+
|
|
1053
|
+
Deliberately DOES NOT see a supersession successor (a record with ``supersedes`` set)
|
|
1054
|
+
UNLESS it carries a propose-only signal — fix round 1, 2026-09-15, measured live: both
|
|
1055
|
+
of this project's own store's findings at the time were ``supersede_decision``/
|
|
1056
|
+
``supersede_fact`` successors, not ``SIDEGRAPH_AUTO_ACCEPT=on``. (Precisely: all of
|
|
1057
|
+
them were DECISION successors. The fact half of this exclusion is defensive only --
|
|
1058
|
+
``supersede_fact`` always stamps ``source="human"``, so a fact successor never reaches
|
|
1059
|
+
this agent-sourced check at all, and ``DraftFact`` carries no ``supersedes`` field, so
|
|
1060
|
+
the propose path cannot write one either. No current writer produces the shape the
|
|
1061
|
+
fact half excludes; it is kept so the rule does not silently depend on that staying
|
|
1062
|
+
true -- fix round 2 re-review, Minor B.) Reversal
|
|
1063
|
+
(``supersede_decision``/``supersede_fact``) has no ratification queue to bypass in the
|
|
1064
|
+
first place — it writes its successor ``accepted`` immediately, unconditionally, by
|
|
1065
|
+
design (CLAUDE.md invariant #2's "reversal = new record with supersedes"), and its
|
|
1066
|
+
``provenance.source`` is legitimately ``"agent"`` (that tool's own docstring names an
|
|
1067
|
+
agent-initiated caller as an anticipated shape, and every real successor in this
|
|
1068
|
+
project's own store already is exactly that). Left unscoped, that reproduces this
|
|
1069
|
+
check's whole signature — accepted, agent-sourced, no ratifier stamp — on every
|
|
1070
|
+
superseding record, with no relation to the env var this check exists to catch.
|
|
1071
|
+
|
|
1072
|
+
Narrowed in fix round 2, 2026-09-15 (review Major 1): fix round 1 excluded EVERY
|
|
1073
|
+
``supersedes``-bearing record on the claim that a ``supersede_decision`` successor and
|
|
1074
|
+
an auto-accepted superseding ``propose_decisions`` draft are "genuinely
|
|
1075
|
+
indistinguishable" — measurement refuted that for a real, non-trivial share of drafts.
|
|
1076
|
+
``_has_propose_only_signal`` checks three fields only the propose/``auto_accept`` path
|
|
1077
|
+
can produce and ``supersede_decision``/``supersede_fact`` structurally cannot: a
|
|
1078
|
+
``layer`` (no parameter on either supersede tool), a ``provenance.ref`` (same), or a
|
|
1079
|
+
tier-0 ``tag:``/``initiative:`` binding not already on the predecessor (the supersede
|
|
1080
|
+
tools either inherit the predecessor's bindings verbatim or resolve explicit anchors
|
|
1081
|
+
with no ``tags``/``initiative`` parameter to mint one from). A ``supersedes``-bearing
|
|
1082
|
+
record is now excluded ONLY when none of the three is present — one-directional and
|
|
1083
|
+
zero-false-positive by construction: presence proves the propose path wrote it, but a
|
|
1084
|
+
MINIMAL superseding draft (no layer/ref/tags) still carries none of the three and stays
|
|
1085
|
+
excluded, correctly indistinguishable from a genuine successor, exactly as before.
|
|
1086
|
+
|
|
1087
|
+
Known, accepted gap this still leaves (record the trade-off, don't pretend it isn't
|
|
1088
|
+
one): a minimal superseding ``propose_decisions`` draft — no ``layer``, no ``ref``, no
|
|
1089
|
+
new tier-0 binding — landed via ``SIDEGRAPH_AUTO_ACCEPT=on`` is still excluded, because
|
|
1090
|
+
it is genuinely indistinguishable, from the canonical record's own fields, from a
|
|
1091
|
+
legitimate ``supersede_decision``/``supersede_fact`` successor (neither carries any
|
|
1092
|
+
field naming which MCP tool wrote it). This is a real, deliberate, NARROWER blind spot
|
|
1093
|
+
than fix round 1 shipped, not an oversight — the alternative (still flagging every
|
|
1094
|
+
supersession) is the measured, guaranteed-noisy status quo this fix round exists to
|
|
1095
|
+
remove, and an advisory nobody trusts protects nothing.
|
|
1096
|
+
"""
|
|
1097
|
+
stamps = [
|
|
1098
|
+
parsed
|
|
1099
|
+
for _p, rec in [*decisions, *facts]
|
|
1100
|
+
if (parsed := _parse_aware_iso(rec.get("ratified_at"))) is not None
|
|
1101
|
+
]
|
|
1102
|
+
marker_stamp = _read_stamping_marker(store_dir)
|
|
1103
|
+
scope_candidates = [s for s in (marker_stamp, min(stamps) if stamps else None) if s is not None]
|
|
1104
|
+
if not scope_candidates:
|
|
1105
|
+
return [] # neither a creation marker nor any stamp exists; nothing to scope from
|
|
1106
|
+
scope_start = min(scope_candidates)
|
|
1107
|
+
entity_canonical_by_id = {
|
|
1108
|
+
e["entity_id"]: e["canonical_name"]
|
|
1109
|
+
for _p, e in entities
|
|
1110
|
+
if isinstance(e.get("entity_id"), str) and isinstance(e.get("canonical_name"), str)
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
findings: list[Finding] = []
|
|
1114
|
+
for path, rec in [*decisions, *facts]:
|
|
1115
|
+
if rec.get("status") != DecisionStatus.ACCEPTED.value:
|
|
1116
|
+
continue
|
|
1117
|
+
provenance = rec.get("provenance")
|
|
1118
|
+
source = provenance.get("source") if isinstance(provenance, dict) else None
|
|
1119
|
+
if source != "agent" or rec.get("ratified_at"):
|
|
1120
|
+
continue
|
|
1121
|
+
if rec.get("supersedes") and not _has_propose_only_signal(
|
|
1122
|
+
rec, bindings, entity_canonical_by_id
|
|
1123
|
+
):
|
|
1124
|
+
continue # a reversal's successor -- no ratify queue exists for it to bypass
|
|
1125
|
+
created = _parse_aware_iso(rec.get("valid_from"))
|
|
1126
|
+
if created is None or created < scope_start:
|
|
1127
|
+
continue # predates observable stamping (or the store's own creation) here
|
|
1128
|
+
findings.append(
|
|
1129
|
+
Finding(
|
|
1130
|
+
UNRATIFIED_ACCEPT,
|
|
1131
|
+
str(path),
|
|
1132
|
+
"accepted with provenance.source='agent' and no ratifier stamp — the "
|
|
1133
|
+
"signature of SIDEGRAPH_AUTO_ACCEPT=on (the ratification gate bypassed "
|
|
1134
|
+
"for the whole shared store by one environment's setting). Records "
|
|
1135
|
+
"ratified before the ratifier stamp existed look identical, so verify "
|
|
1136
|
+
"rather than assume; if the switch is on, decide whether that is the "
|
|
1137
|
+
"policy you want committed",
|
|
1138
|
+
)
|
|
1139
|
+
)
|
|
1140
|
+
return findings
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
# -- D5: auto-ratification audit (informational lines, not findings; design D5) ------------
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _hot_plus_archive_records(store_dir: Path) -> dict[str, dict[str, dict]]:
|
|
1147
|
+
"""{"decisions": {id: payload}, "facts": {id: payload}, "domains": {domain_id: payload}}:
|
|
1148
|
+
hot canonical files plus ``archive/*.jsonl``, HOT winning on an id present in both
|
|
1149
|
+
(mirrors ``verify.py``'s own ``{**archive.decisions, **decisions_raw}`` precedent, also
|
|
1150
|
+
mirrored at this module's own ``curate()`` above). Facts are never archived
|
|
1151
|
+
(``Store.compact``: "facts/ stays entirely hot regardless of status"), so they are hot
|
|
1152
|
+
only. Read-only: archive violations are ``verify``'s territory and are silently ignored
|
|
1153
|
+
here, the same silent-skip policy ``_iter_records`` already uses. No ``Finding``, no
|
|
1154
|
+
``Store()``, no write.
|
|
1155
|
+
# see design/superpowers/specs/2026-09-11-auto-ratification-policy-design.md D5
|
|
1156
|
+
"""
|
|
1157
|
+
archive = _check_archive_dir(store_dir / "archive")
|
|
1158
|
+
|
|
1159
|
+
def hot(subdir: str, key: str) -> dict[str, dict]:
|
|
1160
|
+
return {
|
|
1161
|
+
rec[key]: rec
|
|
1162
|
+
for _p, rec in _iter_records(store_dir, subdir)
|
|
1163
|
+
if isinstance(rec.get(key), str)
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
return {
|
|
1167
|
+
"decisions": {**archive.decisions, **hot("decisions", "id")},
|
|
1168
|
+
"facts": hot("facts", "id"),
|
|
1169
|
+
"domains": {**archive.domains, **hot("domains", "domain_id")},
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
@dataclass(frozen=True)
|
|
1174
|
+
class _AutoShareTally:
|
|
1175
|
+
"""One kind's classification tallies (D5, the pre-flight review's Q5 table).
|
|
1176
|
+
``auto_total``/``human_total`` are the share line's numerator/denominator (records
|
|
1177
|
+
EVER ratified, whatever their current status); ``*_retired`` are the supersede-rate
|
|
1178
|
+
numerators (auto/human records whose CURRENT status is in the kind's retired set);
|
|
1179
|
+
``excluded`` counts stamp-less records created after the store's earliest stamp --
|
|
1180
|
+
neither numerator nor denominator, but named in the line."""
|
|
1181
|
+
|
|
1182
|
+
auto_total: int = 0
|
|
1183
|
+
auto_retired: int = 0
|
|
1184
|
+
human_total: int = 0
|
|
1185
|
+
human_retired: int = 0
|
|
1186
|
+
excluded: int = 0
|
|
1187
|
+
|
|
1188
|
+
|
|
1189
|
+
# The three record kinds the D5 audit classifies (mirrors the store's own dirs). Private
|
|
1190
|
+
# and prefixed on purpose: this codebase's own "kind" already means `DecisionKind`
|
|
1191
|
+
# (adr/lesson/constraint/gotcha), and `capture._auto_ratify(kind=...)` uses a third
|
|
1192
|
+
# vocabulary again ("fact"/"domain"/a decision kind) -- an unprefixed `Kind` here would
|
|
1193
|
+
# read as a fourth, public meaning (review round 1, Minor 3).
|
|
1194
|
+
_RecordKind = Literal["decisions", "facts", "domains"]
|
|
1195
|
+
|
|
1196
|
+
# Per-kind "retired" statuses for the supersede-rate numerator (design D5, Ruling Z):
|
|
1197
|
+
# decisions/facts retire only by supersession; a domain retires via EITHER
|
|
1198
|
+
# ratify_domains(drop=...) OR Store.supersede_domain (rev 10 corrected rev 9's claim that
|
|
1199
|
+
# drop was the only one -- both are legitimate retirements of an accepted domain, so the
|
|
1200
|
+
# line's group label is "domains retired", not "domains dropped").
|
|
1201
|
+
_RETIRED_STATUSES: dict[_RecordKind, frozenset[str]] = {
|
|
1202
|
+
"decisions": frozenset({DecisionStatus.SUPERSEDED.value}),
|
|
1203
|
+
"facts": frozenset({DecisionStatus.SUPERSEDED.value}),
|
|
1204
|
+
"domains": frozenset({"dropped", "superseded"}),
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def _is_auto_stamped(rec: dict) -> bool:
|
|
1209
|
+
"""True iff ``rec`` carries a genuine ``auto:``-prefixed ratifier stamp: a parseable
|
|
1210
|
+
``ratified_at`` AND a ``ratified_by`` starting ``auto:`` (design D5, Q5's "auto"
|
|
1211
|
+
classification). Both the print gate in ``_auto_share_lines`` and the classifier
|
|
1212
|
+
below use this SAME predicate, so a record can never open the gate without also
|
|
1213
|
+
feeding the ``min()`` that computes the earliest stamp -- ``min()`` is therefore
|
|
1214
|
+
never called over an empty generator."""
|
|
1215
|
+
return (
|
|
1216
|
+
_parse_aware_iso(rec.get("ratified_at")) is not None
|
|
1217
|
+
and isinstance(rb := rec.get("ratified_by"), str)
|
|
1218
|
+
and rb.startswith("auto:")
|
|
1219
|
+
)
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
def _tally_kind(kind: _RecordKind, records: dict[str, dict], earliest: datetime) -> _AutoShareTally:
|
|
1223
|
+
"""Classify every record of one kind (D5, Q5's table) by ``ratified_at`` PRESENCE,
|
|
1224
|
+
never by ``ratified_by`` -- a real human ratify can legitimately store
|
|
1225
|
+
``ratified_by=None`` (no resolvable git identity, ``store.py:521-545``). ``earliest``
|
|
1226
|
+
is the store's earliest stamp taken over hot PLUS archive, ALL three kinds (mirrors
|
|
1227
|
+
``_check_unratified_accepts``'s scoping rule, widened): a stamp-less accepted/retired
|
|
1228
|
+
record created BEFORE ``earliest`` is legacy-human (a numerator hit when retired) --
|
|
1229
|
+
residual contamination the design accepts, since a pre-stamp retired record might
|
|
1230
|
+
never have been accepted at all and looks identical by canonical fields. Created ON
|
|
1231
|
+
OR AFTER ``earliest``, it is excluded instead: stamping was already live when the
|
|
1232
|
+
record was created, so a stamp-less accepted record bypassed the ratify path
|
|
1233
|
+
entirely and is neither auto- nor human-ratified.
|
|
1234
|
+
|
|
1235
|
+
Deliberately NOT widened by the store's creation marker (``stamping_live_since``),
|
|
1236
|
+
unlike ``_check_unratified_accepts`` -- considered and rejected for this doctor-blind-
|
|
1237
|
+
window fix, and left as a KNOWN, RECORDED inconsistency, not a closed question (review
|
|
1238
|
+
round 2, Minor 3 -- the round-1 justification below was incomplete, not wrong: it only
|
|
1239
|
+
covered PRINTING, not CLASSIFICATION).
|
|
1240
|
+
|
|
1241
|
+
For PRINTING: this audit's own print gate already requires at least one genuine
|
|
1242
|
+
``auto:``-stamped record to exist anywhere before it prints a single line
|
|
1243
|
+
(``_auto_share_lines``), so a store that has merely adopted stamping without ever
|
|
1244
|
+
actually stamping anything stays silent either way -- there is no blind window in
|
|
1245
|
+
WHETHER the lines print.
|
|
1246
|
+
|
|
1247
|
+
For CLASSIFICATION, there is one, and it is live today: measured (fix round 2) -- take
|
|
1248
|
+
a new store (marker present, `_check_unratified_accepts` armed from creation), one
|
|
1249
|
+
``propose(auto_accept=True)`` bypass record, and one LATER decision that gets
|
|
1250
|
+
auto-ratified (arming this audit's own print gate). The bypass record predates the
|
|
1251
|
+
store's earliest actual stamp (that later ratification), so THIS function classifies
|
|
1252
|
+
it as legacy-human -- `auto share: decisions 1/2 (50%) ... vs human 0/1`, where it
|
|
1253
|
+
lands in ``human_total``, the supersede rate's DENOMINATOR (fix round 2 re-review,
|
|
1254
|
+
Minor C: an earlier wording called this "a numerator hit", which it is not -- the
|
|
1255
|
+
record is accepted, not retired, and only a retired record reaches a numerator) -- in
|
|
1256
|
+
the exact same ``curate()`` run where `_check_unratified_accepts` flags that
|
|
1257
|
+
SAME record as an unratified accept. Two checks in one report disagreeing about the
|
|
1258
|
+
same record is a real inconsistency, not a hypothetical one.
|
|
1259
|
+
|
|
1260
|
+
The ruling stands anyway: folding the marker into `earliest` here would additionally
|
|
1261
|
+
misdescribe this function's own "excluded" bucket, whose printed wording is anchored to
|
|
1262
|
+
a real, OBSERVED event ("after the first stamp excluded") -- the marker only proves
|
|
1263
|
+
stamping CAPABILITY existed, not that any stamping ever ran, so reusing it here needs
|
|
1264
|
+
new wording, not just a wider `earliest`, which stays out of scope for a fix this
|
|
1265
|
+
function did not have a BUG in (D5's numbers are informational, not a security signal).
|
|
1266
|
+
The inconsistency above is the documented cost of that scope decision, not evidence
|
|
1267
|
+
against it."""
|
|
1268
|
+
retired = _RETIRED_STATUSES[kind]
|
|
1269
|
+
accepted_or_retired = {DecisionStatus.ACCEPTED.value} | retired
|
|
1270
|
+
# _AutoShareTally is frozen -- accumulate in plain locals first, build it once at the
|
|
1271
|
+
# end.
|
|
1272
|
+
auto_total = 0
|
|
1273
|
+
auto_retired = 0
|
|
1274
|
+
human_total = 0
|
|
1275
|
+
human_retired = 0
|
|
1276
|
+
excluded = 0
|
|
1277
|
+
for record_id, rec in records.items():
|
|
1278
|
+
status = rec.get("status")
|
|
1279
|
+
if _parse_aware_iso(rec.get("ratified_at")) is not None:
|
|
1280
|
+
if _is_auto_stamped(rec):
|
|
1281
|
+
auto_total += 1
|
|
1282
|
+
if status in retired:
|
|
1283
|
+
auto_retired += 1
|
|
1284
|
+
else:
|
|
1285
|
+
human_total += 1
|
|
1286
|
+
if status in retired:
|
|
1287
|
+
human_retired += 1
|
|
1288
|
+
continue
|
|
1289
|
+
if status not in accepted_or_retired:
|
|
1290
|
+
continue # ignored: proposed / rejected / deprecated -- not classifiable
|
|
1291
|
+
created = (
|
|
1292
|
+
_ulid_datetime(record_id)
|
|
1293
|
+
if kind == "domains"
|
|
1294
|
+
else _parse_aware_iso(rec.get("valid_from"))
|
|
1295
|
+
)
|
|
1296
|
+
if created is None or created >= earliest:
|
|
1297
|
+
excluded += 1
|
|
1298
|
+
continue
|
|
1299
|
+
human_total += 1 # legacy-human: unstamped, predates observable stamping
|
|
1300
|
+
if status in retired:
|
|
1301
|
+
human_retired += 1
|
|
1302
|
+
return _AutoShareTally(
|
|
1303
|
+
auto_total=auto_total,
|
|
1304
|
+
auto_retired=auto_retired,
|
|
1305
|
+
human_total=human_total,
|
|
1306
|
+
human_retired=human_retired,
|
|
1307
|
+
excluded=excluded,
|
|
1308
|
+
)
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def _fmt_share(n: int, d: int) -> str:
|
|
1312
|
+
"""``n/d (p%)`` with an integer floor percentage, or ``n/a`` for a zero denominator
|
|
1313
|
+
(design D5: "zero records of a kind prints n/a, never a division")."""
|
|
1314
|
+
return f"{n}/{d} ({100 * n // d}%)" if d else "n/a"
|
|
1315
|
+
|
|
1316
|
+
|
|
1317
|
+
def _auto_share_lines(store_dir: str | Path) -> list[str]:
|
|
1318
|
+
"""The two D5 informational lines (``auto share`` / ``auto supersede rate``), or
|
|
1319
|
+
``[]`` when no ``auto:``-prefixed stamp exists anywhere (hot plus archive, any kind)
|
|
1320
|
+
-- the print gate that keeps a manual deployment's doctor output byte-identical (D4:
|
|
1321
|
+
"manual deployments see zero render diff"). Pure read: no ``Finding``, no
|
|
1322
|
+
``Store()``, no write.
|
|
1323
|
+
# see design/superpowers/specs/2026-09-11-auto-ratification-policy-design.md D5
|
|
1324
|
+
"""
|
|
1325
|
+
records = _hot_plus_archive_records(Path(store_dir))
|
|
1326
|
+
all_records = [rec for kind_records in records.values() for rec in kind_records.values()]
|
|
1327
|
+
if not any(_is_auto_stamped(rec) for rec in all_records):
|
|
1328
|
+
return []
|
|
1329
|
+
earliest = min(
|
|
1330
|
+
parsed
|
|
1331
|
+
for rec in all_records
|
|
1332
|
+
if (parsed := _parse_aware_iso(rec.get("ratified_at"))) is not None
|
|
1333
|
+
)
|
|
1334
|
+
tallies = {kind: _tally_kind(kind, records[kind], earliest) for kind in _RETIRED_STATUSES}
|
|
1335
|
+
excluded = sum(t.excluded for t in tallies.values())
|
|
1336
|
+
|
|
1337
|
+
share = ", ".join(
|
|
1338
|
+
f"{kind} {_fmt_share(t.auto_total, t.auto_total + t.human_total)}"
|
|
1339
|
+
for kind, t in tallies.items()
|
|
1340
|
+
)
|
|
1341
|
+
df_auto_n = tallies["decisions"].auto_retired + tallies["facts"].auto_retired
|
|
1342
|
+
df_auto_d = tallies["decisions"].auto_total + tallies["facts"].auto_total
|
|
1343
|
+
df_human_n = tallies["decisions"].human_retired + tallies["facts"].human_retired
|
|
1344
|
+
df_human_d = tallies["decisions"].human_total + tallies["facts"].human_total
|
|
1345
|
+
dom = tallies["domains"]
|
|
1346
|
+
|
|
1347
|
+
return [
|
|
1348
|
+
f"auto share: {share} ({excluded} stamp-less record(s) after the first stamp excluded)",
|
|
1349
|
+
"auto supersede rate: decisions+facts superseded "
|
|
1350
|
+
f"{_fmt_share(df_auto_n, df_auto_d)} vs human {_fmt_share(df_human_n, df_human_d)}; "
|
|
1351
|
+
f"domains retired {_fmt_share(dom.auto_retired, dom.auto_total)} vs human "
|
|
1352
|
+
f"{_fmt_share(dom.human_retired, dom.human_total)} "
|
|
1353
|
+
f"({excluded} stamp-less record(s) after the first stamp excluded)",
|
|
1354
|
+
]
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def curate(
|
|
1358
|
+
store_dir: str | Path,
|
|
1359
|
+
*,
|
|
1360
|
+
stale_days: int = 30,
|
|
1361
|
+
now: datetime | None = None,
|
|
1362
|
+
repo_root: Path | None = None,
|
|
1363
|
+
) -> CurationReport:
|
|
1364
|
+
"""Run every advisory curation check over the store at ``store_dir``.
|
|
1365
|
+
|
|
1366
|
+
Pure read (module docstring). ``now`` is injectable for tests and defaults to
|
|
1367
|
+
``datetime.now(UTC)``. Findings are ordered by check (the spec's listing order),
|
|
1368
|
+
then by file path within each check — deterministic output.
|
|
1369
|
+
|
|
1370
|
+
``repo_root`` (design D5, additive): the git repository ``code-drift`` diffs against.
|
|
1371
|
+
``None`` (the default -- ``sidegraph-doctor`` passes nothing) resolves it the same way
|
|
1372
|
+
``verify._find_repo_root`` does, from ``store_dir`` itself (the store may be nested
|
|
1373
|
+
inside the repo). Passing it explicitly is mainly a test seam -- it skips that
|
|
1374
|
+
resolution and any git-availability failure it could raise.
|
|
1375
|
+
# see design/superpowers/specs/2026-07-23-sidegraph-doctor-design.md
|
|
1376
|
+
# see design/superpowers/specs/2026-07-30-staleness-machinery-design.md (D5)
|
|
1377
|
+
"""
|
|
1378
|
+
root = Path(store_dir)
|
|
1379
|
+
now = now or datetime.now(UTC)
|
|
1380
|
+
decisions = _iter_records(root, "decisions")
|
|
1381
|
+
facts = _iter_records(root, "facts")
|
|
1382
|
+
domains = _iter_records(root, "domains")
|
|
1383
|
+
entities = _iter_records(root, "entities")
|
|
1384
|
+
bindings = _binding_entries(root)
|
|
1385
|
+
|
|
1386
|
+
# decision id -> status, HOT ids winning over archived ones (mirrors verify.py's own
|
|
1387
|
+
# `{**archive.decisions, **decisions_raw}` precedent) -- resolution must include archived
|
|
1388
|
+
# decisions too, not just hot ones: `compact` moves terminal decisions into
|
|
1389
|
+
# `archive/*.jsonl`, and without this a supports-only fact whose decision was archived
|
|
1390
|
+
# would misreport "does not exist" (D5's wording) about a decision that is sitting right
|
|
1391
|
+
# there. The verdict doesn't change (an archived decision is terminal by construction, so
|
|
1392
|
+
# such a fact is correctly reported either way) -- only the WORDING does, which is
|
|
1393
|
+
# exactly the failure D5's distinct wording exists to prevent.
|
|
1394
|
+
archived = _check_archive_dir(root / "archive")
|
|
1395
|
+
decision_status_by_id: dict[str, str] = {
|
|
1396
|
+
did: status
|
|
1397
|
+
for did, payload in archived.decisions.items()
|
|
1398
|
+
if isinstance(status := payload.get("status"), str)
|
|
1399
|
+
}
|
|
1400
|
+
decision_status_by_id.update(
|
|
1401
|
+
{
|
|
1402
|
+
d["id"]: status
|
|
1403
|
+
for _, d in decisions
|
|
1404
|
+
if isinstance(d.get("id"), str) and isinstance(status := d.get("status"), str)
|
|
1405
|
+
}
|
|
1406
|
+
)
|
|
1407
|
+
|
|
1408
|
+
findings: list[Finding] = []
|
|
1409
|
+
findings += _check_dangling_records(decisions, facts, bindings, decision_status_by_id)
|
|
1410
|
+
status_findings, skipped = _check_binding_statuses(root)
|
|
1411
|
+
findings += status_findings
|
|
1412
|
+
findings += _check_stale_proposals(decisions, domains, stale_days, now)
|
|
1413
|
+
findings += _check_unreferenced_entities(entities, bindings)
|
|
1414
|
+
findings += _check_duplicate_entities(entities, bindings)
|
|
1415
|
+
findings += _check_expired_open(decisions, now)
|
|
1416
|
+
surfaced_findings, surfaced_skipped = _check_never_surfaced(root)
|
|
1417
|
+
findings += surfaced_findings
|
|
1418
|
+
skipped += surfaced_skipped
|
|
1419
|
+
findings += _check_unratified_accepts(decisions, facts, root, bindings, entities)
|
|
1420
|
+
# Appended LAST (design D5: "appended at the END of curate()'s listing order").
|
|
1421
|
+
findings += _check_code_drift(decisions, bindings, entities, root, repo_root)
|
|
1422
|
+
# Instructions files sit beside the store in the standard layout (`<repo>/.sidegraph`),
|
|
1423
|
+
# so the store's own parent is the lookup root. Deliberately NOT a git call: curate()'s
|
|
1424
|
+
# git-call count is pinned by test_curate_git_call_count_unchanged_by_refactor, and
|
|
1425
|
+
# this advisory check is not worth a subprocess on the hook paths. A store configured
|
|
1426
|
+
# outside its repository simply finds no instructions file — the check stays silent
|
|
1427
|
+
# rather than guessing where one might live.
|
|
1428
|
+
findings += _check_stale_instructions(decisions, repo_root or root.parent)
|
|
1429
|
+
return CurationReport(findings=findings, skipped=skipped)
|