sidegraph 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2099 @@
1
+ """Decision-shaped markdown -> anchored decisions (portable core, no LLM, no key).
2
+
3
+ Importer #2 (mirrors ``importer.py``'s shape/report/CLI patterns exactly). A markdown file
4
+ qualifies as **decision-shaped** when it has an H1 title AND at least one known
5
+ decision-section heading (ADR/Nygard style: Context/Decision/Status/Consequences/
6
+ Rejected/Alternatives/... or this project's own spec style: Trigger/Design/Residuals/
7
+ Scope notes/User decisions). Anchors come from backtick-quoted mentions in the doc, never
8
+ guessed, plus the document's own file-level node when present. See
9
+ ``docs/reference/cli.md#importing-decision-shaped-markdown---docs``.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import hashlib
15
+ import os
16
+ import re
17
+ from collections.abc import Sequence
18
+ from dataclasses import dataclass
19
+ from datetime import UTC, datetime
20
+ from pathlib import Path
21
+ from typing import Literal
22
+
23
+ from pydantic import BaseModel, Field, computed_field
24
+
25
+ from .anchoring import resolve_and_bind
26
+ from .capture import (
27
+ AutoEligibility,
28
+ RatifyPolicy,
29
+ _anchor_signal,
30
+ _auto_ratify,
31
+ auto_ratify_eligible,
32
+ redact,
33
+ )
34
+ from .engine.reader import GraphifyReader
35
+ from .profiles import GENERIC_ADR_DIALECT, FlowProfile, ReaderDialect, get_profile
36
+ from .schema import (
37
+ AnchorBinding,
38
+ Decision,
39
+ DecisionKind,
40
+ DecisionStatus,
41
+ Descriptor,
42
+ Provenance,
43
+ slugify,
44
+ )
45
+ from .store import Store
46
+ from .verify import _find_repo_root
47
+
48
+ # Cap on frequency-ranked, resolved mention-anchors per doc (design §2) — separate from,
49
+ # and in addition to, the doc's own file-node anchor (always added when present).
50
+ _MAX_MENTION_ANCHORS = 3
51
+
52
+ _TITLE_LIMIT = 120
53
+ # Fix-wave B, B3: 600 chars amputated the "why" mid-word ("Secu", "over-en", "deadli" were all
54
+ # observed against a real ADR corpus) — raised to 2000 (default; overridable per run via
55
+ # `import_docs(section_limit=...)` / `sidegraph-import --docs --section-limit N`, min 200 —
56
+ # see `_truncate_section`). The title limit is untouched: a title is a lead-in, not the
57
+ # recorded reasoning, so it stays a plain 120-char slice, no word-boundary/marker logic.
58
+ _SECTION_LIMIT = 2000
59
+ _MIN_SECTION_LIMIT = 200
60
+ # Appended when a section body is cut at `section_limit` so a downstream reader (human or
61
+ # agent) KNOWS the text was truncated, instead of silently reading a sentence that just stops
62
+ # — the eval's own root-cause finding was that a silent mid-word cut gets misread as complete.
63
+ _TRUNCATION_MARKER = " …[truncated]"
64
+
65
+ # Known decision-section headings (design §1), matched case-insensitively as a PREFIX of
66
+ # the heading text (verified against a real-world ADR/spec corpus during pre-release
67
+ # calibration: the exact-heading rule qualified only 4/11 decision docs there; prefix
68
+ # matching — real headings look like "Design — track the observed community…", "Scope notes /
69
+ # residuals (documented, not solved here)", "User decisions (this brainstorm)" — plus the
70
+ # bold-metadata gate below (see :data:`_BOLD_META_RE`) takes that to 11/11, while still
71
+ # correctly excluding every task-plan file (headings like "## Tasks" don't start with
72
+ # anything known, and this project's plan template's bold metadata is "**Goal:**", not
73
+ # Trigger/Status). Prefix, not substring: substring matching let "Decision" false-positive-
74
+ # match inside "User decisions" ("decisions" contains "decision"), silently stealing
75
+ # `choice` from the wrong (gate-only) section — a heading has to actually START WITH the
76
+ # keyword to count.
77
+ #
78
+ # Most-specific-first ordering matters within `_CHOICE_HEADINGS`: MADR's template carries
79
+ # BOTH "Decision Drivers" and "Decision Outcome" — both start with bare "decision", so
80
+ # without a more specific keyword tried first, `_first_matching_section` (doc order, first
81
+ # KEYWORD match wins) would pick whichever section happens to come first in the document,
82
+ # which for MADR is "Decision Drivers" (context, not the choice). "decision outcome" is
83
+ # tried before the bare "decision" fallback so the outcome section always wins when both
84
+ # exist. Same story for `_REJECTED_HEADINGS`: MADR's "Considered Options" heading didn't
85
+ # match any existing keyword at all (word order differs from "options considered") until
86
+ # added explicitly.
87
+ _CONTEXT_HEADINGS = GENERIC_ADR_DIALECT.context
88
+ _CHOICE_HEADINGS = GENERIC_ADR_DIALECT.choice
89
+ _REJECTED_HEADINGS = GENERIC_ADR_DIALECT.rejected
90
+ # Fix-wave B, B2: real ADRs carry a genuine "## Consequences" H2 (Positive/Negative/Risks
91
+ # subsections) — split out of `_GATE_ONLY_HEADINGS` into its own extraction keyword tuple so
92
+ # `Decision.consequences` (a schema field the pre-B2 mapping never populated at all) actually
93
+ # gets filled from it, the same "first matching known heading" mechanism `choice`/`rejected`
94
+ # already use.
95
+ _CONSEQUENCES_HEADINGS = GENERIC_ADR_DIALECT.consequences
96
+ _GATE_ONLY_HEADINGS = GENERIC_ADR_DIALECT.gate_only
97
+ _KNOWN_SECTION_HEADINGS = GENERIC_ADR_DIALECT.all_headings()
98
+
99
+ _HEADING_RE = re.compile(r"^(#{1,3})\s+(.*)$")
100
+ _FENCE_RE = re.compile(r"^\s*```")
101
+ # Blank templates carry every qualifying section heading by construction, so the section
102
+ # gate can't reject them — the angle-bracket placeholder left in the H1 ("<Decision
103
+ # title>") is the deterministic tell (found live on an ADR corpus's `_ADR-template.md`).
104
+ # Backtick spans are stripped first so a real title quoting a generic (`Dict<str, Node>`)
105
+ # still qualifies; an unbackticked generic in an H1 is a miss we accept — a skipped doc
106
+ # beats an imported blank form (false-positive imports are worse than missed specs).
107
+ _BACKTICK_SPAN_RE = re.compile(r"`[^`]*`")
108
+ _PLACEHOLDER_RE = re.compile(r"<[^<>]+>")
109
+ _FRONTMATTER_RE = re.compile(r"^---[ \t]*\n(.*?\n)---[ \t]*\n?", re.DOTALL)
110
+ _FRONTMATTER_STATUS_RE = re.compile(r"^\s*status\s*:\s*(.+?)\s*$", re.IGNORECASE)
111
+
112
+ # BUG G — document-TEMPLATE tells (see :func:`_is_template_doc` / :func:`_placeholder_dominated`).
113
+ # A template is a skeleton meant to be COPIED and filled in, never a decision — yet blank ADR/
114
+ # SAD templates carry real-word H1s (`# ADR - Architecture Decision Record`) plus every
115
+ # qualifying section heading, so the decision-shaped gate and the H1-placeholder guard
116
+ # (:data:`_PLACEHOLDER_RE`, which only catches an angle-bracket IN the H1) both waved them
117
+ # through — one even landed ACCEPTED off a template's own `**Status:** APPROVED` line,
118
+ # polluting the store with a fake accepted decision. Calibrated against a real architecture
119
+ # corpus: all 3 of its ADR/SAD templates are `*template*.md` AND carry `type: template`
120
+ # frontmatter, while its 11 real ADRs are neither (their `type:` is adr/appendix/annex).
121
+ # Conservative by design — prefer MISSING a template (importing a real doc) over rejecting a
122
+ # real ADR: the filename/`type` tells are near-zero-false-positive, and the placeholder-body
123
+ # tell fires only when placeholder stubs DOMINATE the qualifying section text.
124
+ _FRONTMATTER_TYPE_RE = re.compile(r"^\s*type\s*:\s*(.+?)\s*$", re.IGNORECASE)
125
+ # Non-link square-bracket stub: `[...]` NOT followed by `(` (a markdown link `[text](url)`)
126
+ # or `[` (a reference link `[text][ref]`) — a bare `[decision here]`/`[added]` guidance stub,
127
+ # not real linked content.
128
+ _BRACKET_PLACEHOLDER_RE = re.compile(r"\[[^\[\]\n]+\](?![(\[])")
129
+ # Bare authoring markers / date-number stubs left unfilled in a blank template.
130
+ _PLACEHOLDER_MARKER_RE = re.compile(r"\b(?:TODO|TBD|FIXME|XXX|NNN|YYYY(?:-MM-DD)?)\b")
131
+ # A Context/Decision body counts as blank-template when placeholder text makes up at least
132
+ # this fraction of its (backtick-stripped) characters.
133
+ _PLACEHOLDER_DOMINANCE = 0.5
134
+
135
+ # Bold metadata lines right after the H1 (this project's own spec-style convention — real
136
+ # specs open with "**Date:** ... **Status:** ... **Trigger:** ..." right after the title).
137
+ # Distinct from YAML frontmatter (a real "---" fence):
138
+ # this is prose-adjacent metadata, not a machine block, but it is common and regular
139
+ # enough in this corpus to parse deliberately (mid-task spec refinement).
140
+ #
141
+ # Adjudicated gate rule (mid-task refinement, second pass, live-corpus verified 11/11 specs
142
+ # / 0/10 plans): a bold **Trigger:** OR **Status:** line is an ADDITIONAL qualifying signal
143
+ # alongside the known-heading-prefix gate (Nygard ADRs carry Status as a first-class field,
144
+ # so a bold Status line is a strong decision-doc signal on its own). **Date:** alone NEVER
145
+ # qualifies — every note, dated or not, has a date; using it as a gate would let arbitrary
146
+ # prose in. The known-section VOCABULARY is deliberately NOT widened with generic
147
+ # engineering-doc headings like "Purpose"/"Scope"/"Architecture"/"Components" to catch the
148
+ # remaining stragglers some other way: those headings are endemic to READMEs and ordinary
149
+ # module docs too, and a false-positive import (some unrelated doc misread as a decision)
150
+ # is worse than a missed one (which the interactive/agent capture path still covers). Do
151
+ # not pad this list without re-running the live-corpus check both ways (specs qualifying
152
+ # AND plans/READMEs staying excluded).
153
+ _BOLD_META_RE = re.compile(r"^\*\*(Trigger|Status|Date):\*\*\s*(.*)$", re.IGNORECASE)
154
+
155
+ # ANY bold "**Key:** value" lead line, not just Trigger/Status/Date — real specs carry
156
+ # non-qualifying bold keys right alongside the qualifying ones (e.g. "**Related:**",
157
+ # "**Covers:**", "**Informed by:**", "**Builds on:**", as seen in real specs alongside the
158
+ # qualifying keys). This is used ONLY to decide
159
+ # what counts as "prose" for the first-paragraph fallback (see `_strip_bold_lead_lines`) —
160
+ # it must NEVER widen the qualifying gate itself, which stays exactly `_BOLD_META_RE`
161
+ # (Trigger/Status only, per the adjudicated rule above).
162
+ _BOLD_ANY_RE = re.compile(r"^\*\*([A-Za-z][^*]*):\*\*\s*(.*)$")
163
+
164
+ # Fix-wave B, B2: bold-line pseudo-headings for `rejected`/`consequences` content that lives
165
+ # INSIDE a narrative section rather than under its own real H2/H3 heading — the calibrated
166
+ # finding against a real ADR corpus: none of its 8 imported ADRs had a literal "## Rejected"
167
+ # heading (rejection reasoning instead sits in prose, e.g. "**Rejected — emitter-side
168
+ # auto-create.** A create-if-missing behaviour..." and, as a list item,
169
+ # "- **Rejected (antipattern) — a centralised...** A single service..."). Unlike
170
+ # `_BOLD_META_RE`/`_BOLD_ANY_RE` (the H1 lead-block ``**Key:** value`` convention, colon
171
+ # required), these pseudo-headings have NO colon — the label runs straight into an em-dash or
172
+ # a closing `**`, and a leading `- ` list marker is common, so it's stripped rather than
173
+ # required or rejected.
174
+ #
175
+ # Deliberately narrow (Rejected/Alternative(s)/Considered for `rejected`; Consequence(s) for
176
+ # `consequences`) rather than a generic "any bold lead-in", mirroring the same
177
+ # false-positive-avoidance stance as `_BOLD_META_RE`'s Trigger/Status-only gate: a
178
+ # **Chosen:**-style bold lead-in must never be misread as a rejection.
179
+ _BOLD_PSEUDO_REJECTED_RE = re.compile(
180
+ r"^-?\s*\*\*(Rejected|Alternatives?|Considered)\b", re.IGNORECASE
181
+ )
182
+ _BOLD_PSEUDO_CONSEQUENCES_RE = re.compile(r"^-?\s*\*\*(Consequences?)\b", re.IGNORECASE)
183
+ # Detects the START of ANY bold pseudo-heading line (not just the two keyword sets above) so
184
+ # a multi-item block with no blank line between entries (e.g. "- **Chosen:** ...\n-
185
+ # **Rejected:** ...", observed live) stops capturing the PRIOR block at the right line rather
186
+ # than swallowing the next one — the same role `_BOLD_ANY_RE` plays for `_bold_lead_blocks`.
187
+ _BOLD_PSEUDO_ANY_RE = re.compile(r"^-?\s*\*\*[A-Za-z]")
188
+
189
+ # "known source extension" allowlist for the path-like mention classifier (design §2).
190
+ _SOURCE_EXTENSIONS = (
191
+ ".py",
192
+ ".ts",
193
+ ".tsx",
194
+ ".js",
195
+ ".jsx",
196
+ ".go",
197
+ ".rs",
198
+ ".java",
199
+ ".rb",
200
+ ".c",
201
+ ".cc",
202
+ ".cpp",
203
+ ".h",
204
+ ".hpp",
205
+ ".cs",
206
+ ".php",
207
+ ".swift",
208
+ ".kt",
209
+ ".scala",
210
+ ".md",
211
+ ".json",
212
+ ".yaml",
213
+ ".yml",
214
+ ".toml",
215
+ ".sh",
216
+ ".sql",
217
+ )
218
+ _SNAKE_RE = re.compile(r"^[a-z][a-z0-9]*(?:_[a-z0-9]+)+$")
219
+ _CAMEL_RE = re.compile(r"^[A-Za-z][a-z0-9]*(?:[A-Z][a-z0-9]*)+$")
220
+ _BACKTICK_RE = re.compile(r"`([^`\n]+)`")
221
+
222
+
223
+ class ParsedDoc(BaseModel):
224
+ """One decision-shaped doc's field mapping (design §1). ``title``/``context``/
225
+ ``choice``/``rejected``/``consequences`` are already redacted+truncated — ready to write
226
+ onto a :class:`~sidegraph.schema.Decision` verbatim. ``frontmatter_status`` is the raw
227
+ (unredacted, untruncated) value of either a YAML-frontmatter ``status:`` key OR a bold
228
+ ``**Status:**`` metadata line right after the H1 (YAML frontmatter wins if somehow both
229
+ are present) — it is never persisted; the caller uses it only to decide whether this
230
+ doc is a superseded/deprecated history doc that should not be (re-)imported as live
231
+ memory, AND (fix-wave B, B1) whether it reads as a draft/proposed/pending-review doc
232
+ that must land ``proposed`` regardless of ``--propose``. ``suggested_kind`` (fix-wave B,
233
+ B4) is likewise never persisted as-is: a per-document signal (currently just "the doc
234
+ has a qualifying 'Root cause' section" -> :class:`~sidegraph.schema.DecisionKind.LESSON`)
235
+ the caller applies UNLESS the run's own ``--kind`` was explicitly given, which always
236
+ wins."""
237
+
238
+ title: str
239
+ context: str
240
+ choice: str
241
+ rejected: str | None = None
242
+ consequences: str | None = None
243
+ frontmatter_status: str | None = None
244
+ suggested_kind: DecisionKind | None = None
245
+ # Oneshot+granularity spec D2: set on split-produced CHILD records only — the
246
+ # slugified H3 heading this record was cut from. The import loop keys the child's
247
+ # provenance/idempotency on the effective ref `f"{rel_path}#{fragment}"`; None (the
248
+ # default, and always the parent's value) keeps the bare-path ref unchanged.
249
+ fragment: str | None = None
250
+
251
+
252
+ class DocWriteRequest(BaseModel):
253
+ parsed: ParsedDoc
254
+ rel_path: str
255
+ source_hash: str
256
+ status: DecisionStatus
257
+ anchors: tuple[Descriptor, ...] = ()
258
+ file_anchor: Descriptor | None = None
259
+ anchors_skipped: tuple[dict, ...] = ()
260
+ graph_version: str | None = None
261
+ tags: tuple[str, ...] = ()
262
+ # A write may ratify a pending proposal at the same ref ONLY when the caller states that a
263
+ # human asked for it (Bootstrap's `accept` action). Default False so an unattended importer
264
+ # rerun can never bypass the ratification queue — see
265
+ # design/superpowers/specs/2026-08-02-bootstrap-release-readiness-design.md §3.1.
266
+ ratify_matching_proposal: bool = False
267
+
268
+ @computed_field # type: ignore[prop-decorator]
269
+ @property
270
+ def ref(self) -> str:
271
+ if self.parsed.fragment is None:
272
+ return self.rel_path
273
+ return f"{self.rel_path}#{self.parsed.fragment}"
274
+
275
+
276
+ class DocWriteResult(BaseModel):
277
+ action: Literal["written", "superseded", "skipped-existing", "skipped-unanchorable"]
278
+ decision_id: str | None = None
279
+
280
+
281
+ @dataclass(frozen=True)
282
+ class _DocDisposition:
283
+ action: Literal["written", "superseded", "skipped-existing", "skipped-unanchorable"]
284
+ accepted_ancestor: Decision | None
285
+ pending_proposal: Decision | None
286
+ supersedable: tuple[Decision, ...]
287
+ inherit_bindings_from: str | None
288
+
289
+
290
+ class DocImportReport(BaseModel):
291
+ """Counts (+ dry-run-only listing) for one ``import_docs`` run. Mirrors
292
+ ``importer.ImportReport``'s shape, widened for doc-import's extra outcomes: a doc can
293
+ be *superseded* (edited since its last import) as well as freshly *imported*, and can
294
+ be skipped for four distinct reasons (existing/unanchorable/not-decision-shaped/
295
+ unparseable) plus a fifth doc-import-only reason (superseded/deprecated frontmatter —
296
+ a history doc, never (re-)imported as live) and a sixth (a document TEMPLATE — see
297
+ ``skipped_template``)."""
298
+
299
+ imported: int = 0
300
+ superseded: int = 0
301
+ skipped_existing: int = 0
302
+ skipped_unanchorable: int = 0
303
+ skipped_not_decision: int = 0
304
+ skipped_superseded_frontmatter: int = 0
305
+ # BUG G: a document TEMPLATE (skeleton to copy/fill) — detected by a "template" filename,
306
+ # a `type: template` frontmatter field, or a placeholder-dominated Context/Decision body
307
+ # (see `_is_template_doc`/`_placeholder_dominated`). Counted distinctly from a plain
308
+ # not-decision-shaped skip: blank templates were importing as false-positive decisions
309
+ # (one even landing ACCEPTED off the template's own `**Status:** APPROVED` line).
310
+ skipped_template: int = 0
311
+ # A doc that IS decision-shaped (has the H1 + qualifying section/bold-metadata signal)
312
+ # but whose `choice` came back empty even after every fallback (design refinement (c) —
313
+ # see `_parse_decision_doc_with_reason`/`_deepest_choice_fallback`) — never written as a
314
+ # Decision with an empty choice, counted separately from a plain not-decision-shaped
315
+ # skip so the CLI/dry-run listing can tell the two apart.
316
+ skipped_unparseable: int = 0
317
+ # Of the decisions counted above as imported/superseded (dry-run or real), how many
318
+ # landed `proposed` because their source status read as draft/pending/under-review-like
319
+ # (fix-wave B, B1) — regardless of whether `--propose` was also passed. Counted
320
+ # separately from a plain `--propose` run so the report line can say WHY, e.g. "N landed
321
+ # proposed (source status: draft)".
322
+ status_derived_proposed: int = 0
323
+ # Of the decisions counted above, how many landed `rejected` because their source
324
+ # status read as turned down — regardless of `--propose`. Before this counter such a
325
+ # doc landed `accepted`, i.e. the store asserted the team had adopted what it refused.
326
+ status_derived_rejected: int = 0
327
+ # design D7.1 (staleness-machinery wave, E8 hygiene): a file enumerated from `paths`
328
+ # (directly, or via directory recursion) whose repo-relative path matches NONE of the
329
+ # active profile's `ingest_globs` — never parsed, never anchor-resolved, just skipped
330
+ # and counted. Distinct from every other skip reason above: those all fire AFTER a doc
331
+ # was at least read; this one fires before the file is opened at all. Zero unless
332
+ # `any_doc=True` was NOT passed and at least one enumerated file falls outside the
333
+ # profile's globs (the CLI's `--any-doc` flag restores today's no-filter behavior).
334
+ skipped_outside_profile: int = 0
335
+ # E3 (design note §7, review Major 4/Blocker N1): a split-produced parent that is
336
+ # never written because it is degenerate — either an echo of its own `context` (rule
337
+ # 1) or empty even after every fallback (rule 2, which previously discarded the whole
338
+ # file's children along with it). The children stand alone either way; this counts the
339
+ # suppressed parent, once per file where either rule fired.
340
+ skipped_degenerate_parent: int = 0
341
+ # Auto-ratification policy (design D2/D6) -- additive/defaulted, same contract as
342
+ # importer.ImportReport's own pair: incremented/appended by the post-write auto block
343
+ # in _import_one_record, always empty/zero under `manual` or when a written record
344
+ # was ineligible.
345
+ auto_ratified: int = 0
346
+ auto_ratify_failures: list[str] = Field(default_factory=list) # ["<decision id>: <reason>"]
347
+ # [{"file_path", "ref", "title", "action": "imported"|"superseded", "anchors_skipped"},
348
+ # ...] — one item per RECORD (a split-capable dialect can emit several per file);
349
+ # `ref` is the effective ref (`path` or `path#fragment`), `file_path` the real path.
350
+ dry_run: list[dict] = Field(default_factory=list)
351
+
352
+ def by_file(self) -> dict[str, int]:
353
+ """Per-file breakdown of the dry-run listing, for the CLI's ``--dry-run`` printer."""
354
+ out: dict[str, int] = {}
355
+ for item in self.dry_run:
356
+ fp = item["file_path"] or "<unknown>"
357
+ out[fp] = out.get(fp, 0) + 1
358
+ return out
359
+
360
+
361
+ # -- parser (design §1) ------------------------------------------------------------------
362
+
363
+
364
+ def _strip_frontmatter(text: str) -> tuple[str, str | None]:
365
+ """Split a leading ``---``-delimited YAML-ish frontmatter block off ``text`` and pull
366
+ its ``status:`` value, if any. Returns ``(remaining_text, status_or_None)``. Only a
367
+ genuine frontmatter block counts (a real ``---`` fence at byte 0) — this project's own
368
+ bold-metadata-line convention (``**Status:** ...`` right under the H1) is NOT
369
+ frontmatter and is intentionally not parsed here (it is prose, not machine metadata;
370
+ see design §1's literal "status frontmatter" wording)."""
371
+ if not text.startswith("---"):
372
+ return text, None
373
+ m = _FRONTMATTER_RE.match(text)
374
+ if not m:
375
+ return text, None
376
+ status = None
377
+ for line in m.group(1).splitlines():
378
+ mm = _FRONTMATTER_STATUS_RE.match(line)
379
+ if mm:
380
+ status = mm.group(1).strip().strip("\"'")
381
+ return text[m.end() :], status
382
+
383
+
384
+ def _frontmatter_type(text: str) -> str | None:
385
+ """Raw ``type:`` value of a leading ``---``-delimited YAML-frontmatter block, or ``None``
386
+ (BUG G — a ``type: template`` doc is a skeleton, never a decision). Mirrors
387
+ :func:`_strip_frontmatter`'s parse; only a genuine ``---`` fence at byte 0 counts, and a
388
+ key repeated more than once keeps the last occurrence."""
389
+ if not text.startswith("---"):
390
+ return None
391
+ m = _FRONTMATTER_RE.match(text)
392
+ if not m:
393
+ return None
394
+ type_value: str | None = None
395
+ for line in m.group(1).splitlines():
396
+ mm = _FRONTMATTER_TYPE_RE.match(line)
397
+ if mm:
398
+ type_value = mm.group(1).strip().strip("\"'")
399
+ return type_value
400
+
401
+
402
+ def _is_template_doc(rel_path: str, frontmatter_type: str | None) -> bool:
403
+ """True when the doc is a TEMPLATE by its cheapest, near-zero-false-positive tells (BUG
404
+ G): a filename whose STEM is ``template`` as a whole word (``ADR-template.md``,
405
+ ``SAD-template.md``, ``_ADR-template.md``, ``template.md``) OR an explicit
406
+ ``type: template`` frontmatter field. Both hold for every template in the calibration
407
+ corpus and for none of its real ADRs. The filename tell is anchored to the stem's END (a
408
+ ``-``/``_`` boundary then ``template``) rather than a bare substring so a REAL ADR that
409
+ merely mentions templates in its name -- ``ADR-012-email-template-engine.md`` -- is NOT
410
+ dropped (review finding N1)."""
411
+ stem = rel_path.rsplit("/", 1)[-1].rsplit(".", 1)[0].lower()
412
+ if stem == "template" or stem.endswith("-template") or stem.endswith("_template"):
413
+ return True
414
+ return frontmatter_type is not None and frontmatter_type.strip().lower() == "template"
415
+
416
+
417
+ def _placeholder_dominated(text: str) -> bool:
418
+ """True when placeholder stubs (``<...>``, non-link ``[...]``, TODO/TBD/NNN/YYYY markers)
419
+ make up at least :data:`_PLACEHOLDER_DOMINANCE` of ``text``'s non-backtick characters (BUG
420
+ G — a blank-template Context/Decision body left unfilled). Backtick spans are stripped
421
+ first so a real ADR quoting a generic (`` `Dict<str, Node>` ``) never counts; empty text
422
+ is never dominated."""
423
+ stripped = _BACKTICK_SPAN_RE.sub("", text).strip()
424
+ total = len(stripped)
425
+ if total == 0:
426
+ return False
427
+ placeholder_chars = 0
428
+
429
+ def _blank(m: re.Match[str]) -> str:
430
+ nonlocal placeholder_chars
431
+ placeholder_chars += len(m.group(0))
432
+ return " " * len(m.group(0))
433
+
434
+ # Blank each <...>/[...] span out (same length in spaces) so a marker sitting INSIDE one
435
+ # (e.g. `<TBD>`) is not counted twice, then tally the bare markers left in the remainder.
436
+ remainder = _BRACKET_PLACEHOLDER_RE.sub(_blank, _PLACEHOLDER_RE.sub(_blank, stripped))
437
+ for mk in _PLACEHOLDER_MARKER_RE.finditer(remainder):
438
+ placeholder_chars += len(mk.group(0))
439
+ return placeholder_chars / total >= _PLACEHOLDER_DOMINANCE
440
+
441
+
442
+ def _heading_positions(lines: list[str]) -> list[tuple[int, int, str]]:
443
+ """``[(line_index, level, heading_text), ...]`` outside fenced code blocks — a line
444
+ inside a ```` ``` ````-fenced block (e.g. a Python ``# comment``) must never be mistaken
445
+ for a markdown heading."""
446
+ out: list[tuple[int, int, str]] = []
447
+ in_fence = False
448
+ for i, line in enumerate(lines):
449
+ if _FENCE_RE.match(line):
450
+ in_fence = not in_fence
451
+ continue
452
+ if in_fence:
453
+ continue
454
+ m = _HEADING_RE.match(line)
455
+ if m:
456
+ out.append((i, len(m.group(1)), m.group(2).rstrip("#").strip()))
457
+ return out
458
+
459
+
460
+ def _first_paragraph(lines: list[str]) -> str:
461
+ """First contiguous block of non-blank lines, space-joined — the ``choice`` fallback
462
+ when neither a Decision nor a Design section exists (design §1)."""
463
+ started = False
464
+ out: list[str] = []
465
+ for line in lines:
466
+ if line.strip() == "":
467
+ if started:
468
+ break
469
+ continue
470
+ started = True
471
+ out.append(line.strip())
472
+ return " ".join(out).strip()
473
+
474
+
475
+ def _bold_lead_blocks(lines: list[str], key_re: re.Pattern[str]) -> list[tuple[str, str, int, int]]:
476
+ """Walk ``lines`` (expected to be a doc's "lead" block, between the H1 and the first
477
+ heading) for ``**Key:** value`` blocks whose key matches ``key_re``. Returns
478
+ ``[(key_lower, joined_value, start_idx, end_idx_exclusive), ...]``.
479
+
480
+ A value's continuation runs across physical lines until a blank line, the next bold
481
+ lead line (:data:`_BOLD_ANY_RE` — ANY key, not just ``key_re``'s, so one key's
482
+ continuation never swallows the next key's line), or a heading — real ``**Trigger:**``
483
+ values routinely wrap 3-4 physical lines (observed in a real multi-line trigger
484
+ description): capturing only the first physical line left ``context``/``choice``
485
+ truncated mid-sentence, and — worse
486
+ — let the WRAPPED continuation (starting mid-sentence, lowercase) leak into the
487
+ first-paragraph ``choice`` fallback when no Decision/Design section existed.
488
+ """
489
+ out: list[tuple[str, str, int, int]] = []
490
+ i = 0
491
+ n = len(lines)
492
+ while i < n:
493
+ m = key_re.match(lines[i].strip())
494
+ if m is None:
495
+ i += 1
496
+ continue
497
+ start = i
498
+ key = m.group(1).lower()
499
+ value_parts = [m.group(2).strip()]
500
+ i += 1
501
+ while i < n:
502
+ nxt = lines[i]
503
+ if nxt.strip() == "" or _BOLD_ANY_RE.match(nxt.strip()) or _HEADING_RE.match(nxt):
504
+ break
505
+ value_parts.append(nxt.strip())
506
+ i += 1
507
+ out.append((key, " ".join(p for p in value_parts if p).strip(), start, i))
508
+ return out
509
+
510
+
511
+ def _bold_pseudo_heading_blocks(lines: list[str], key_re: re.Pattern[str]) -> list[str]:
512
+ """Bold-line pseudo-headings ANYWHERE in ``lines`` (a full document body — unlike
513
+ :func:`_bold_lead_blocks`, these appear inside narrative section bodies, not just the H1
514
+ lead block) whose line start matches ``key_re``. Returns the joined block text (the
515
+ matching line plus every following line up to a blank line, a markdown heading, or the
516
+ next bold pseudo-heading of ANY kind — :data:`_BOLD_PSEUDO_ANY_RE`), one entry per match,
517
+ in document order.
518
+
519
+ A leading ``- `` list marker is stripped; the ``**...**`` markup itself is kept — unlike
520
+ ``**Trigger:** value``, there is no colon separating a machine key from its value here
521
+ (real examples: ``"**Rejected — emitter-side auto-create.** A create-if-missing
522
+ behaviour..."``), so the bold span is part of the meaningful content, not a label to
523
+ discard (fix-wave B, B2).
524
+
525
+ Fence-aware (review follow-up, Minor 2): a line inside a ```` ``` ````-fenced block
526
+ (e.g. a style guide QUOTING this very convention as a worked example for future ADR
527
+ authors) is never scanned as a real pseudo-heading — the same ``_FENCE_RE`` toggle
528
+ :func:`_heading_positions` uses, so a fenced example is skipped exactly like a fenced
529
+ ``#`` comment is already skipped for real headings."""
530
+ out: list[str] = []
531
+ i = 0
532
+ n = len(lines)
533
+ in_fence = False
534
+ while i < n:
535
+ if _FENCE_RE.match(lines[i]):
536
+ in_fence = not in_fence
537
+ i += 1
538
+ continue
539
+ if in_fence:
540
+ i += 1
541
+ continue
542
+ stripped = lines[i].strip()
543
+ if key_re.match(stripped) is None:
544
+ i += 1
545
+ continue
546
+ block = [re.sub(r"^-\s*", "", stripped)]
547
+ i += 1
548
+ while i < n:
549
+ nxt = lines[i]
550
+ if _FENCE_RE.match(nxt):
551
+ # A fence boundary ends this block too — never let it span into (or
552
+ # swallow the toggle of) a fenced region; re-processed by the outer loop.
553
+ break
554
+ if (
555
+ nxt.strip() == ""
556
+ or _HEADING_RE.match(nxt)
557
+ or _BOLD_PSEUDO_ANY_RE.match(nxt.strip())
558
+ ):
559
+ break
560
+ block.append(nxt.strip())
561
+ i += 1
562
+ out.append(" ".join(p for p in block if p).strip())
563
+ return out
564
+
565
+
566
+ def _extract_bold_metadata(lines: list[str]) -> dict[str, str]:
567
+ """``{"trigger"|"status"|"date": value, ...}`` from bold ``**Key:** value`` lines (see
568
+ :data:`_BOLD_META_RE`) among ``lines`` — expected to be the "lead" block between the H1
569
+ and the first heading, where this project's own specs put them. Values spanning several
570
+ wrapped physical lines are joined (see :func:`_bold_lead_blocks`). A key repeated more
571
+ than once keeps the LAST occurrence (simplest deterministic rule; not expected in
572
+ practice — this is metadata, not a section)."""
573
+ return {key: value for key, value, _, _ in _bold_lead_blocks(lines, _BOLD_META_RE)}
574
+
575
+
576
+ def _strip_bold_lead_lines(lines: list[str]) -> list[str]:
577
+ """Remove every ``**Key:** value`` lead block (key line + wrapped continuation lines),
578
+ for ANY bold key — not just the qualifying Trigger/Status/Date (:data:`_BOLD_META_RE`)
579
+ — from ``lines`` before it is scanned for the first-paragraph ``choice`` fallback.
580
+
581
+ Design refinement (b): real lead blocks routinely carry non-qualifying bold keys right
582
+ alongside the qualifying ones — ``**Related:**``, ``**Covers:**``, ``**Informed by:**``
583
+ (observed on real specs during pre-release calibration). Stripping only
584
+ Trigger/Status/Date left those lines as the "first paragraph" in any doc with no real
585
+ Decision/Design section — the raw ``**Related:** [...]`` markup became ``choice``
586
+ verbatim. This only changes what counts as "prose" for the fallback scan; the
587
+ QUALIFYING gate (:data:`_BOLD_META_RE`, Trigger/Status only) is untouched.
588
+ """
589
+ strip_indices: set[int] = set()
590
+ for _, _, start, end in _bold_lead_blocks(lines, _BOLD_ANY_RE):
591
+ strip_indices.update(range(start, end))
592
+ return [ln for i, ln in enumerate(lines) if i not in strip_indices]
593
+
594
+
595
+ def _parse_structure(
596
+ text: str,
597
+ ) -> tuple[str | None, list[tuple[str, str]], str, dict[str, str]]:
598
+ """``(h1_text, sections, first_paragraph_after_h1, bold_metadata)``. ``h1_text`` is
599
+ ``None`` when no H1 exists at all (freeform doc by default — see E1 below for the one
600
+ profile-gated exception). ``sections`` is ``[(heading_text, body_text), ...]`` for
601
+ every H2/H3 (and any further H1) heading found AFTER the H1 — or, when there is no H1
602
+ at all, every H1/H2/H3 heading in the WHOLE document (E1, design note §5: a
603
+ ``title_pattern``-synthesized title stands in for a missing H1, so the sections it
604
+ would have introduced must still be visible; :func:`_split_section_children` already
605
+ reads the same "no H1 -> start of file" convention independently, via its own
606
+ ``h1_idx = -1`` default). A section's body runs to the next heading at or above ITS
607
+ OWN level, so a deeper nested heading (H3 under an H2) is folded INTO its parent's
608
+ body rather than truncating it to empty — this project's own spec style routinely puts
609
+ an umbrella "## Design" heading directly above a run of "### 1. ...", "### 2. ..."
610
+ numbered subsections with no prose of its own between the H2 and its first H3 child;
611
+ without this rule, ``choice`` extraction (which keys off a "## Design"/"##
612
+ Decision(s)" match) would see an empty body for exactly that heading and silently fall
613
+ through to the wrong fallback. Each H3 still appears as its OWN entry too (its body
614
+ stops at the next heading of any level <=3, i.e. unchanged for H3s specifically) — so a
615
+ keyword match against a nested H3 heading still works. ``bold_metadata`` is extracted
616
+ from the same "lead" lines ``first_paragraph`` comes from (see
617
+ :func:`_extract_bold_metadata`)."""
618
+ lines = text.split("\n")
619
+ headings = _heading_positions(lines)
620
+ h1 = next(((i, t) for i, lvl, t in headings if lvl == 1), None)
621
+ h1_idx, h1_text = h1 if h1 is not None else (-1, None)
622
+ rest = [(i, lvl, t) for i, lvl, t in headings if i > h1_idx]
623
+
624
+ lead_end = rest[0][0] if rest else len(lines)
625
+ lead_lines = lines[h1_idx + 1 : lead_end]
626
+ bold_metadata = _extract_bold_metadata(lead_lines)
627
+ # The "first paragraph" fallback means actual prose, not the bold metadata block those
628
+ # lines sit right above (design refinement (a): a doc with only bold **Status:**/
629
+ # **Date:** lines and a real opening paragraph must fall back to THAT paragraph, not to
630
+ # "**Date:** 2026-... **Status:** Approved" verbatim) — strip EVERY bold lead line (any
631
+ # key, with its wrapped continuation), not just the qualifying Trigger/Status/Date ones
632
+ # (design refinement (b) — see :func:`_strip_bold_lead_lines`), before scanning for the
633
+ # first non-blank prose block.
634
+ prose_lead_lines = _strip_bold_lead_lines(lead_lines)
635
+ first_paragraph = _first_paragraph(prose_lead_lines)
636
+
637
+ sections: list[tuple[str, str]] = []
638
+ for idx, (pos, lvl, heading_text) in enumerate(rest):
639
+ body_start = pos + 1
640
+ body_end = len(lines)
641
+ for later_pos, later_lvl, _ in rest[idx + 1 :]:
642
+ if later_lvl <= lvl:
643
+ body_end = later_pos
644
+ break
645
+ sections.append((heading_text, "\n".join(lines[body_start:body_end]).strip()))
646
+
647
+ return h1_text, sections, first_paragraph, bold_metadata
648
+
649
+
650
+ # Numbered section headings ("## 1. Context", "## 6.1 Security / privacy",
651
+ # "## IV. Alternatives") are endemic to arc42-style templates — genkovich's sad.md numbers every
652
+ # section ("1. Introduction and goals" ... "11. Risks and technical debt"). spec.md numbers its
653
+ # sections the same way upstream, but spec.md is a requirements document that R3 excludes from
654
+ # ingestion (see profiles.py), so sad.md alone is what this strip exists to read today. The
655
+ # keyword match is a PREFIX match, so "context" does not match "1. Context" and every section of
656
+ # such a document is invisible. Strip the enumerator before matching only; the heading text
657
+ # callers see is unchanged.
658
+ # Decimal form: a single level must carry its dot ("1. Context"), so a heading that merely
659
+ # STARTS with a year or count ("2024 in review") is untouched; a multi-level enumerator needs
660
+ # no trailing dot, because real templates write "6.1 Security / privacy" and "2.1 Jobs To Be
661
+ # Done" without one. The roman form always requires the dot, so "Impact" can never be read as
662
+ # "I" + "mpact".
663
+ _ENUMERATOR_RE = re.compile(r"^\s*(?:\d+(?:\.\d+)+\.?|\d+\.|[IVXLCDM]+\.)\s+")
664
+
665
+
666
+ def _heading_has_keyword(heading: str, keyword: str) -> bool:
667
+ """Case-insensitive PREFIX match: ``heading`` must START WITH ``keyword``, after
668
+ stripping a leading numeric/roman enumerator (see :data:`_ENUMERATOR_RE`) — for this
669
+ comparison only; the heading text stored and returned everywhere else is untouched (see
670
+ the module-level comment above :data:`_CONTEXT_HEADINGS` for why prefix, not substring
671
+ or exact)."""
672
+ normalized = _ENUMERATOR_RE.sub("", heading.strip())
673
+ return normalized.lower().startswith(keyword.lower())
674
+
675
+
676
+ def _matches_any(heading: str, keywords: tuple[str, ...]) -> bool:
677
+ return any(_heading_has_keyword(heading, kw) for kw in keywords)
678
+
679
+
680
+ def _first_matching_section(
681
+ sections: list[tuple[str, str]], keywords: tuple[str, ...]
682
+ ) -> str | None:
683
+ """The body of the first section (by KEYWORD priority, then document order) whose
684
+ heading starts with one of ``keywords`` — implements "first of [A, B]" from design §1:
685
+ every section is checked against the higher-priority keyword before any section is
686
+ checked against the next one."""
687
+ for kw in keywords:
688
+ for heading, body in sections:
689
+ if _heading_has_keyword(heading, kw):
690
+ return body
691
+ return None
692
+
693
+
694
+ _WHITESPACE_RE = re.compile(r"\s")
695
+
696
+
697
+ def _last_whitespace_index(s: str) -> int:
698
+ """Index of the LAST whitespace character (any of ``\\s`` — space, tab, newline, ...)
699
+ in ``s``, or ``-1`` when none exists. Review follow-up (Minor 3): a plain
700
+ ``s.rfind(" ")`` only finds a literal ASCII space, so text whose sole whitespace near a
701
+ cut point is a newline (no space at all — e.g. a section with no blank-line-separated
702
+ prose, just wrapped lines) fell through to the hard-cut fallback and amputated mid-word
703
+ anyway."""
704
+ idx = -1
705
+ for m in _WHITESPACE_RE.finditer(s):
706
+ idx = m.start()
707
+ return idx
708
+
709
+
710
+ def _truncate_section(text: str, limit: int) -> str:
711
+ """Cap ``text`` (already redacted+stripped) at ``limit`` chars, cutting at a word
712
+ boundary and appending :data:`_TRUNCATION_MARKER` when it's actually cut (fix-wave B,
713
+ B3) — never a silent mid-word amputation ("Secu", "over-en", "deadli" were all observed
714
+ against a real ADR corpus at the old 600-char cap). Falls back to a hard cut at
715
+ ``limit`` only when there is no whitespace at all in the first ``limit`` chars (a single
716
+ very long token, e.g. a URL)."""
717
+ if len(text) <= limit:
718
+ return text
719
+ cut = text[:limit]
720
+ last_ws = _last_whitespace_index(cut)
721
+ if last_ws > 0:
722
+ cut = cut[:last_ws]
723
+ return cut.rstrip() + _TRUNCATION_MARKER
724
+
725
+
726
+ def _deepest_choice_fallback(sections: list[tuple[str, str]]) -> str:
727
+ """Design refinement (c): when a doc's lead block is ALL bold metadata (no real prose
728
+ paragraph survives :func:`_strip_bold_lead_lines`) AND no Decision/Design section
729
+ exists either, ``choice`` would otherwise come back empty — the first-paragraph
730
+ fallback and the keyword-matched-section fallback both missed. Fall through one level
731
+ deeper: the first non-empty body of ANY section, in document order, regardless of
732
+ whether its heading matches a known keyword (real examples: a "## Purpose" section, a
733
+ "## Research conclusions..." section — neither matches a known keyword). Returns
734
+ ``""`` (never ``None``) when every section body is empty too — the caller treats that as
735
+ unparseable."""
736
+ for _, body in sections:
737
+ if body.strip():
738
+ return body
739
+ return ""
740
+
741
+
742
+ def _synthesize_title(rel_path: str, title_pattern: str | None) -> str | None:
743
+ """E1 (design note §5): a fallback title for a doc with no H1, from ``title_pattern``'s
744
+ named ``name``/``doc`` groups matched against ``rel_path`` — ``f"{name} ({doc})"``.
745
+ ``None`` when no pattern was supplied (every profile's default) or it doesn't match
746
+ ``rel_path`` — the caller then rejects the doc exactly as it did before E1 existed; the
747
+ pattern doubles as this fallback's own scope guard, so a profile can never synthesize a
748
+ title for a path outside what it actually ingests."""
749
+ if title_pattern is None:
750
+ return None
751
+ match = re.search(title_pattern, rel_path)
752
+ if match is None:
753
+ return None
754
+ return f"{match.group('name')} ({match.group('doc')})"
755
+
756
+
757
+ def _choice_is_context_echo(context: str, choice: str, rel_path: str) -> bool:
758
+ """E3 rule 1 (design note §7, review Major 4): true when a split-produced parent's
759
+ ``choice`` is nothing but its own ``context`` echoed back — the exact
760
+ predicate ``bootstrap/planner.py``'s ``_choice_is_context_fallback`` already applies to
761
+ every bootstrap candidate (``planner.py:88-91``), duplicated here rather than imported
762
+ (core must never import the bootstrap seam — see ``test_flow_profile.py``'s
763
+ portable-core contract tests) so both write paths agree on the same records. ``context``
764
+ is the field's FINAL stamped value (already carrying the trailing ``"imported from
765
+ <rel_path>"`` suffix); it is stripped back off before comparing."""
766
+ suffix = f"\n\nimported from {rel_path}"
767
+ stripped_context = context.removesuffix(suffix).strip()
768
+ return bool(stripped_context) and choice.strip() == stripped_context
769
+
770
+
771
+ def _split_section_children(
772
+ lines: list[str], dialect: ReaderDialect
773
+ ) -> tuple[str, str, list[tuple[str, str]]] | None:
774
+ """The split target and its H3 children (oneshot+granularity spec D2, review F1).
775
+
776
+ Selection is DOC-ORDER-first (unlike `_first_matching_section`'s keyword-priority
777
+ rule — a split target is a location, not a ranked field source): the first heading
778
+ AFTER the H1, of any recognized level LEVEL <= 2 (E4, design note §7, review Major
779
+ N2 — a level-3 prefix match is a leaf, never a container of its own children, so it
780
+ is skipped and the scan continues to a possible later level-<=2 match; `_HEADING_RE`
781
+ treats nothing deeper than H3 as a heading at all, so an `####` between the split
782
+ heading and its first H3 folds into the intro), whose text prefix-matches a
783
+ `split_choice` keyword. Children are headings of level 3 whose line index lies
784
+ strictly between the split heading's line and the next heading of level <= 2 (or
785
+ EOF) — an H3 under any OTHER H2 is never a child (the naive positional reading
786
+ manufactured spurious children; T16 pins the negative). Each child's body runs to
787
+ the next recognized heading, matching `_parse_structure`'s own H3 body rule. Returns
788
+ ``(split_heading_text, intro_text_before_first_child, children)`` or ``None`` when
789
+ `split_choice` is empty or nothing matches.
790
+ """
791
+ if not dialect.split_choice:
792
+ return None
793
+ headings = _heading_positions(lines)
794
+ # Candidates start AFTER the first H1 (code review F14): `_parse_structure`'s
795
+ # sections exclude the H1 and anything above it, and "the first section" must mean
796
+ # the same population here — an H1 that happens to prefix-match a split keyword is
797
+ # a title, not a split target.
798
+ h1_idx = next((i for i, lvl, _ in headings if lvl == 1), -1)
799
+ split = next(
800
+ (
801
+ (i, lvl, text)
802
+ for i, lvl, text in headings
803
+ # E4: level <= 2 only — a level-3 match is skipped, not a stop (skip-and-
804
+ # continue, round-3 Info); the scan keeps looking for a later level-<=2 match.
805
+ if i > h1_idx and lvl <= 2 and _matches_any(text, dialect.split_choice)
806
+ ),
807
+ None,
808
+ )
809
+ if split is None:
810
+ return None
811
+ split_idx, _split_lvl, split_text = split
812
+ after = [(i, lvl, t) for i, lvl, t in headings if i > split_idx]
813
+ section_end = next((i for i, lvl, _ in after if lvl <= 2), len(lines))
814
+ children: list[tuple[str, str]] = []
815
+ for pos, (i, lvl, text) in enumerate(after):
816
+ # Containment is guarded TWICE, and each guard is independently sufficient (code
817
+ # review F10): the `i >= section_end` skip here, and the `min(..., section_end)`
818
+ # clamp below (which empties an out-of-section body so the empty-choice rule
819
+ # drops the child). Do not delete one believing T16 covers it — T16 bites only
820
+ # against the fully naive reading with neither guard.
821
+ if i >= section_end or lvl != 3:
822
+ continue
823
+ body_end = next((j for j, _, _ in after[pos + 1 :] if j > i), len(lines))
824
+ body_end = min(body_end, section_end)
825
+ children.append((text, "\n".join(lines[i + 1 : body_end]).strip()))
826
+ intro_end = min((i for i, _, _ in after), default=len(lines))
827
+ intro = "\n".join(lines[split_idx + 1 : intro_end]).strip()
828
+ return split_text, intro, children
829
+
830
+
831
+ def _parse_decision_doc_with_reason(
832
+ path_text: str,
833
+ rel_path: str,
834
+ section_limit: int = _SECTION_LIMIT,
835
+ *,
836
+ dialect: ReaderDialect = GENERIC_ADR_DIALECT,
837
+ title_pattern: str | None = None,
838
+ ) -> tuple[ParsedDoc | None, str | None]:
839
+ """Single-doc compatibility wrapper over :func:`_parse_decision_docs_with_reason` —
840
+ returns the PARENT record only. With a split-capable dialect the parent's `choice` is
841
+ the first matching NON-split section (spec F6: the exclusion lives in the parser, and
842
+ `parse_decision_doc`'s contract changes for split profiles by design — the AD content
843
+ moves to the child records the plural API returns)."""
844
+ docs, reason = _parse_decision_docs_with_reason(
845
+ path_text, rel_path, section_limit, dialect=dialect, title_pattern=title_pattern
846
+ )
847
+ return (docs[0] if docs else None), reason
848
+
849
+
850
+ def _parse_decision_docs_with_reason(
851
+ path_text: str,
852
+ rel_path: str,
853
+ section_limit: int = _SECTION_LIMIT,
854
+ *,
855
+ dialect: ReaderDialect = GENERIC_ADR_DIALECT,
856
+ title_pattern: str | None = None,
857
+ ) -> tuple[list[ParsedDoc], str | None]:
858
+ """Core of :func:`parse_decision_doc`, plus a skip ``reason`` for the caller
859
+ (``import_docs``) to count separately: ``None`` when the doc simply isn't
860
+ decision-shaped (no H1, or no qualifying section/bold-metadata signal at all);
861
+ ``"template"`` (BUG G) when it is a document TEMPLATE (a "template" filename, a
862
+ ``type: template`` frontmatter field, or a placeholder-dominated Context/Decision body —
863
+ see :func:`_is_template_doc`/:func:`_placeholder_dominated`), short-circuited before any
864
+ status handling so a template's own ``**Status:** APPROVED`` can never drive acceptance;
865
+ ``"unparseable"`` when it IS decision-shaped but every ``choice`` source — the known
866
+ Decision/Design section, the first prose paragraph, AND the deepest any-section
867
+ fallback (:func:`_deepest_choice_fallback`) — came back empty. A Decision is never
868
+ written with an empty ``choice`` (design refinement (c)).
869
+
870
+ ``section_limit`` (fix-wave B, B3) is the per-field cap applied to ``context``/
871
+ ``choice``/``rejected``/``consequences`` AFTER redaction, at a word boundary (see
872
+ :func:`_truncate_section`) — the caller (``import_docs``/the CLI's ``--section-limit``)
873
+ controls it; this function trusts whatever it's given.
874
+ """
875
+ body, fm_status = _strip_frontmatter(path_text)
876
+ fm_type = _frontmatter_type(path_text)
877
+ lines = body.split("\n")
878
+ h1_text, sections, first_paragraph, bold_meta = _parse_structure(body)
879
+ if h1_text is None:
880
+ # E1 (design note §5): a `title_pattern` match synthesizes a title in place of the
881
+ # missing H1 and parsing proceeds exactly as if that were the H1 — no match (or no
882
+ # pattern at all, every profile's default) rejects exactly as before E1 existed.
883
+ h1_text = _synthesize_title(rel_path, title_pattern)
884
+ if h1_text is None:
885
+ return [], None
886
+
887
+ # Oneshot+granularity spec D2: computed BEFORE the placeholder probe so probe and
888
+ # extraction see the same (excluded) choice pool (review F6). The exclusion is
889
+ # conditional on >= 1 child (review F7): a split keyword whose section has no H3s is
890
+ # ordinary choice material, byte-identical to a dialect without split_choice.
891
+ split = _split_section_children(lines, dialect)
892
+ split_children: list[tuple[str, str]] = []
893
+ split_intro = ""
894
+ if split is not None and split[2]:
895
+ split_heading, split_intro, split_children = split
896
+ # Exclusion is keyed on (heading, body) PAIRS, not heading text alone (code
897
+ # review F3): a duplicate heading text elsewhere in the doc — a second section
898
+ # named like the split target, or a child H3 named like a later real H2 — must
899
+ # not knock an unrelated section out of the parent's choice pool. The split
900
+ # section's own entry is identified by its heading plus the folded body
901
+ # _parse_structure gives it; a same-named section with different content stays.
902
+ split_body = next((b for h, b in sections if h == split_heading), None)
903
+ excluded_pairs = {(split_heading, split_body)} | set(split_children)
904
+ choice_sections = [(h, b) for h, b in sections if (h, b) not in excluded_pairs]
905
+ else:
906
+ choice_sections = sections
907
+ # BUG G: a document TEMPLATE (skeleton to copy/fill) is never a decision — and a
908
+ # template's own `**Status:** APPROVED` must never drive acceptance, so this
909
+ # short-circuits BEFORE any status handling (and before the H1-placeholder guard, so a
910
+ # `*template*.md` file is counted as "template", not "not-decision-shaped"). The
911
+ # filename/`type: template` tells are near-zero-false-positive; the placeholder-body
912
+ # tell below catches a renamed copy the first two miss.
913
+ if _is_template_doc(rel_path, fm_type):
914
+ return [], "template"
915
+ if _PLACEHOLDER_RE.search(_BACKTICK_SPAN_RE.sub("", h1_text)):
916
+ return [], None # blank template, not a record — see _PLACEHOLDER_RE
917
+ known_headings = dialect.all_headings()
918
+ has_known_section = any(_matches_any(h, known_headings) for h, _ in sections)
919
+ has_qualifying_bold_meta = "trigger" in bold_meta or "status" in bold_meta
920
+ if not has_known_section and not has_qualifying_bold_meta:
921
+ return [], None
922
+
923
+ # BUG G: a decision-shaped doc whose Context AND Decision sections are still mostly
924
+ # placeholder stubs is a blank-template copy the filename/frontmatter tells missed. Probe
925
+ # the same section sources the extraction below uses, pre-redact (redaction can never
926
+ # introduce placeholder markup, so raw is the honest signal).
927
+ context_probe = _first_matching_section(sections, dialect.context) or bold_meta.get("trigger")
928
+ choice_probe = (
929
+ _first_matching_section(choice_sections, dialect.choice) or split_intro or first_paragraph
930
+ )
931
+ if _placeholder_dominated(f"{context_probe or ''}\n{choice_probe or ''}"):
932
+ return [], "template"
933
+
934
+ title_clean, _ = redact(h1_text)
935
+ title = title_clean.strip()[:_TITLE_LIMIT]
936
+
937
+ context_body = _first_matching_section(sections, dialect.context) or bold_meta.get("trigger")
938
+ if context_body:
939
+ clean, _ = redact(context_body)
940
+ clean = _truncate_section(clean.strip(), section_limit)
941
+ context = f"{clean}\n\nimported from {rel_path}" if clean else f"imported from {rel_path}"
942
+ else:
943
+ context = f"imported from {rel_path}"
944
+
945
+ choice_body = (
946
+ _first_matching_section(choice_sections, dialect.choice) or split_intro or first_paragraph
947
+ )
948
+ choice_clean, _ = redact(choice_body or "")
949
+ choice = _truncate_section(choice_clean.strip(), section_limit)
950
+ if not choice:
951
+ choice_clean, _ = redact(_deepest_choice_fallback(choice_sections))
952
+ choice = _truncate_section(choice_clean.strip(), section_limit)
953
+ # E3 rule 2 (design note §7, review Blocker N1): this early return used to fire BEFORE
954
+ # children were built, so a split with N children yielded ZERO records whenever the
955
+ # parent's own choice fell through every fallback to empty — 22 real child decisions
956
+ # silently discarded across 4 archived files on the calibration corpus. When a split
957
+ # produced children, they stand alone instead (the parent is simply never built below —
958
+ # `choice` stays falsy, so the `if choice:` guard around `parent = ParsedDoc(...)`
959
+ # skips it, and the caller counts it via `skipped_degenerate_parent`); a doc with NO
960
+ # split children keeps today's behavior exactly.
961
+ if not choice and not split_children:
962
+ return [], "unparseable"
963
+
964
+ # `rejected`/`consequences` (fix-wave B, B2): a real heading match wins when present;
965
+ # calibrated against a real ADR corpus where NONE of 8 imported ADRs had a literal
966
+ # "## Rejected" heading, so falling straight through to "None, nothing captured" (the
967
+ # pre-B2 behavior) left the schema's headline field null on every record. The bold
968
+ # pseudo-heading fallback (see :data:`_BOLD_PSEUDO_REJECTED_RE`/
969
+ # :func:`_bold_pseudo_heading_blocks`) recovers what that corpus's prose actually marks
970
+ # up; multiple matches (a doc can carry more than one "**Rejected...**" block) are
971
+ # joined, not just the first.
972
+ rejected_body = _first_matching_section(sections, dialect.rejected)
973
+ if not rejected_body or not rejected_body.strip():
974
+ pseudo = _bold_pseudo_heading_blocks(lines, _BOLD_PSEUDO_REJECTED_RE)
975
+ rejected_body = "\n\n".join(pseudo) if pseudo else None
976
+ rejected: str | None = None
977
+ if rejected_body and rejected_body.strip():
978
+ rejected_clean, _ = redact(rejected_body)
979
+ rejected = _truncate_section(rejected_clean.strip(), section_limit) or None
980
+
981
+ consequences_body = _first_matching_section(sections, dialect.consequences)
982
+ if not consequences_body or not consequences_body.strip():
983
+ pseudo = _bold_pseudo_heading_blocks(lines, _BOLD_PSEUDO_CONSEQUENCES_RE)
984
+ consequences_body = "\n\n".join(pseudo) if pseudo else None
985
+ consequences: str | None = None
986
+ if consequences_body and consequences_body.strip():
987
+ consequences_clean, _ = redact(consequences_body)
988
+ consequences = _truncate_section(consequences_clean.strip(), section_limit) or None
989
+
990
+ frontmatter_status = fm_status if fm_status is not None else bold_meta.get("status")
991
+
992
+ # B4: a single conservative kind signal — a doc whose qualifying sections include
993
+ # "Root cause" reads as a lesson (a root-cause doc IS a lesson), never overriding an
994
+ # explicit `--kind` (the caller's job — see ParsedDoc's docstring).
995
+ suggested_kind = (
996
+ DecisionKind.LESSON
997
+ if any(_heading_has_keyword(h, "root cause") for h, _ in sections)
998
+ else None
999
+ )
1000
+
1001
+ # E3 rule 2's degenerate-empty-shape case reaches here with `choice == ""` (the early
1002
+ # "unparseable" return above was bypassed because `split_children` is non-empty) — the
1003
+ # parent is simply never built; `import_docs` infers the omission (no record with
1004
+ # `fragment is None` in the returned list) and counts `skipped_degenerate_parent`.
1005
+ parent = (
1006
+ ParsedDoc(
1007
+ title=title,
1008
+ context=context,
1009
+ choice=choice,
1010
+ rejected=rejected,
1011
+ consequences=consequences,
1012
+ frontmatter_status=frontmatter_status,
1013
+ suggested_kind=suggested_kind,
1014
+ )
1015
+ if choice
1016
+ else None
1017
+ )
1018
+
1019
+ # Children (spec D2): one ParsedDoc per H3 under the split section. Fields: title =
1020
+ # the H3 heading; choice = its body; context = the SAME matched context section the
1021
+ # parent uses, suffixed with the child's EFFECTIVE ref (review F13 — an agent reading
1022
+ # the record can find the exact section); rejected/consequences stay on the parent
1023
+ # (doc-level sections, not duplicated N times). Fragments: `schema.slugify` (the
1024
+ # existing slugifier — review F8), collisions get `-2`, `-3`, … in doc order, an
1025
+ # empty slug falls back to `section-<n>`. A child whose body redacts/truncates to
1026
+ # empty is not emitted (a Decision is never written with an empty choice — same rule
1027
+ # as refinement (c) above).
1028
+ children_docs: list[ParsedDoc] = []
1029
+ if split_children:
1030
+ raw_context_clean = ""
1031
+ if context_body:
1032
+ clean, _ = redact(context_body)
1033
+ raw_context_clean = _truncate_section(clean.strip(), section_limit)
1034
+ # Uniqueness is a POST-CONDITION over the fragments actually emitted, not an
1035
+ # inference from a per-base counter (code review F1, BLOCKING): a heading whose
1036
+ # slug is naturally `<base>-2` collides with the second occurrence of `<base>`
1037
+ # under counting — two children then share one effective ref and the import loop
1038
+ # "supersedes" one AD with its sibling on a fresh store, flip-flopping every run
1039
+ # (measured: total 4→6→8 across three imports of an unchanged file). The loop
1040
+ # below advances the ordinal until the fragment is genuinely free, which also
1041
+ # closes the same hole for the `section-<n>` fallback.
1042
+ used_fragments: set[str] = set()
1043
+ for n, (child_heading, child_body) in enumerate(split_children, start=1):
1044
+ child_choice_clean, _ = redact(child_body)
1045
+ child_choice = _truncate_section(child_choice_clean.strip(), section_limit)
1046
+ if not child_choice:
1047
+ continue
1048
+ child_title_clean, _ = redact(child_heading)
1049
+ base = slugify(child_title_clean) or f"section-{n}"
1050
+ fragment, k = base, 1
1051
+ while fragment in used_fragments:
1052
+ k += 1
1053
+ fragment = f"{base}-{k}"
1054
+ used_fragments.add(fragment)
1055
+ effective_ref = f"{rel_path}#{fragment}"
1056
+ child_context = (
1057
+ f"{raw_context_clean}\n\nimported from {effective_ref}"
1058
+ if raw_context_clean
1059
+ else f"imported from {effective_ref}"
1060
+ )
1061
+ children_docs.append(
1062
+ ParsedDoc(
1063
+ title=child_title_clean.strip()[:_TITLE_LIMIT],
1064
+ context=child_context,
1065
+ choice=child_choice,
1066
+ rejected=None,
1067
+ consequences=None,
1068
+ frontmatter_status=frontmatter_status,
1069
+ suggested_kind=suggested_kind,
1070
+ fragment=fragment,
1071
+ )
1072
+ )
1073
+
1074
+ records = ([parent] if parent is not None else []) + children_docs
1075
+ if not records:
1076
+ # Every split child ALSO redacted/truncated to empty (an extreme edge case — the
1077
+ # parent's own choice already had to be empty to reach this branch) — nothing
1078
+ # usable came out of this doc at all, same "unparseable" reason a non-split doc
1079
+ # gets for the identical outcome.
1080
+ return [], "unparseable"
1081
+ return records, None
1082
+
1083
+
1084
+ def parse_decision_docs(
1085
+ text: str,
1086
+ rel_path: str,
1087
+ section_limit: int = _SECTION_LIMIT,
1088
+ *,
1089
+ dialect: ReaderDialect = GENERIC_ADR_DIALECT,
1090
+ title_pattern: str | None = None,
1091
+ ) -> tuple[list[ParsedDoc], str | None]:
1092
+ return _parse_decision_docs_with_reason(
1093
+ text, rel_path, section_limit=section_limit, dialect=dialect, title_pattern=title_pattern
1094
+ )
1095
+
1096
+
1097
+ def parse_decision_doc(
1098
+ path_text: str,
1099
+ rel_path: str,
1100
+ section_limit: int = _SECTION_LIMIT,
1101
+ *,
1102
+ dialect: ReaderDialect = GENERIC_ADR_DIALECT,
1103
+ title_pattern: str | None = None,
1104
+ ) -> ParsedDoc | None:
1105
+ """Parse one markdown file into a :class:`ParsedDoc`, or ``None`` when it is not
1106
+ decision-shaped (design §1, refined: needs an H1 title AND — at least one known
1107
+ decision-section heading (prefix match) OR a bold ``**Trigger:**``/``**Status:**``
1108
+ metadata line right after the H1 (see the adjudication note above
1109
+ :data:`_BOLD_META_RE` — a bold ``**Date:**`` line alone is NOT a qualifying signal);
1110
+ freeform docs, and task-plan/README/module-doc files whose only headings are things
1111
+ like "## Tasks" or "## Purpose"/"## Architecture", are never imported) OR its
1112
+ ``choice`` would come back empty even after every fallback (see
1113
+ :func:`_parse_decision_doc_with_reason` — ``import_docs`` distinguishes this
1114
+ ``"unparseable"`` case from a plain not-decision-shaped skip via its own ``reason``
1115
+ counter; this public wrapper collapses both to ``None`` for callers that only care
1116
+ whether a usable :class:`ParsedDoc` came out).
1117
+
1118
+ ``path_text`` is the RAW file content (not pre-redacted — every field this function
1119
+ derives is redacted internally, redact-then-truncate order, mirroring
1120
+ ``importer._make_title``'s rationale: truncating a raw secret first can leave a
1121
+ partial, non-matching fragment that a later redact pass would miss). ``rel_path`` is
1122
+ the doc's path as it should be stamped into ``context`` and (by the caller)
1123
+ ``provenance.ref`` — never redacted (a path is not a secret). ``section_limit``
1124
+ (fix-wave B, B3; default :data:`_SECTION_LIMIT`, 2000) is the per-field char cap applied
1125
+ to ``context``/``choice``/``rejected``/``consequences`` AFTER redaction, at a word
1126
+ boundary with a trailing :data:`_TRUNCATION_MARKER` when actually cut — never the old
1127
+ silent mid-word cut.
1128
+
1129
+ Field mapping (the PARENT record — with a ``split_choice`` dialect this function
1130
+ returns only the parent; the per-H3 child records come from
1131
+ :func:`_parse_decision_docs_with_reason`):
1132
+ - ``title`` = H1 text, redacted, truncated to :data:`_TITLE_LIMIT` chars (plain char
1133
+ slice — a title is a lead-in, not recorded reasoning, so it gets no word-boundary/
1134
+ marker treatment). E1: when the doc has no H1 at all, ``title_pattern`` — a regex
1135
+ with named ``name``/``doc`` groups, matched against ``rel_path`` — supplies a
1136
+ synthesized ``"{name} ({doc})"`` title instead; ``None`` (the default, byte-identical
1137
+ to before E1 existed) or no match rejects the doc exactly as a missing H1 always has.
1138
+ - ``context`` = first of [Context, Trigger, Root cause] section text (redacted,
1139
+ truncated to ``section_limit``) + ``"\\n\\nimported from <rel_path>"``; when
1140
+ none of those sections exist, falls back to a bold ``**Trigger:**`` metadata line
1141
+ (continuation lines included — see :func:`_bold_lead_blocks`); when that's absent
1142
+ too, ``context`` is just the "imported from" line (context is a required field —
1143
+ never empty).
1144
+ - ``choice`` = first of [Decision Outcome, Decisions, Decision, Design] section text;
1145
+ when none exist, the first prose paragraph right after the H1 (with ALL bold lead
1146
+ lines stripped, any key — see :func:`_strip_bold_lead_lines`); when that's empty too,
1147
+ the first non-empty body of ANY section (:func:`_deepest_choice_fallback`). Redacted,
1148
+ truncated. Never empty: a doc that still comes back with nothing usable is skipped
1149
+ entirely (see :func:`_parse_decision_doc_with_reason`).
1150
+ - ``rejected`` = first of [Rejected, Alternatives, "Considered alternatives",
1151
+ "Considered options", "Options considered"] section text; when no such REAL heading
1152
+ exists, falls back to bold pseudo-heading blocks (fix-wave B, B2 — e.g. a bare
1153
+ ``"**Rejected — ..."`` paragraph or ``"- **Rejected (antipattern) — ..."`` list item,
1154
+ see :data:`_BOLD_PSEUDO_REJECTED_RE`), joined when more than one exists. Redacted,
1155
+ truncated; ``None`` when nothing was found either way.
1156
+ - ``consequences`` = first of [Consequences] section text, same bold pseudo-heading
1157
+ fallback as ``rejected`` when no real heading exists (fix-wave B, B2 — this field was
1158
+ never populated by the mapping before B2). Redacted, truncated; ``None`` when absent.
1159
+ - ``frontmatter_status`` = the raw value of a leading YAML-frontmatter ``status:`` key,
1160
+ or (when that's absent) a bold ``**Status:**`` metadata line — the caller
1161
+ (``import_docs``) decides whether "superseded"/"deprecated" in it means "skip, this
1162
+ is a history doc" (design §1's kind/status bullet), and (fix-wave B, B1) whether a
1163
+ draft/proposed/pending/under-review-like value means the imported decision must land
1164
+ ``proposed`` regardless of ``--propose``.
1165
+ - ``suggested_kind`` — see :class:`ParsedDoc`'s docstring (fix-wave B, B4).
1166
+
1167
+ ``kind`` is NOT part of this return value's APPLIED kind: ``suggested_kind`` is only a
1168
+ signal; the run's own ``--kind``, when explicitly given, always overrides it — applied
1169
+ by the caller when constructing the :class:`~sidegraph.schema.Decision`.
1170
+ """
1171
+ parsed, _ = _parse_decision_doc_with_reason(
1172
+ path_text,
1173
+ rel_path,
1174
+ section_limit=section_limit,
1175
+ dialect=dialect,
1176
+ title_pattern=title_pattern,
1177
+ )
1178
+ return parsed
1179
+
1180
+
1181
+ def _is_historical_status(status: str | None) -> bool:
1182
+ """True when a frontmatter ``status:`` value marks the doc as history (design §1:
1183
+ "history docs are not re-imported as live"). Substring, case-insensitive — covers
1184
+ "Superseded", "superseded by ADR-9999", "Deprecated in favor of...", etc."""
1185
+ if not status:
1186
+ return False
1187
+ s = status.lower()
1188
+ return "superseded" in s or "deprecated" in s
1189
+
1190
+
1191
+ # Fix-wave B, B1: substrings (case-insensitive) that mark a frontmatter/bold `status` value
1192
+ # as "not yet ratified" — calibrated against a real ADR corpus where every source ADR
1193
+ # carries `status: draft` (YAML frontmatter) alongside a `status_note` like "proposed,
1194
+ # pending EACL review". The recommended `--docs` import path was landing all of these
1195
+ # `accepted` regardless (a real distortion: an agent querying the store saw ratified
1196
+ # decisions where the corpus says draft-pending-review).
1197
+ _DRAFT_STATUS_MARKERS = ("draft", "propos", "pending", "under review")
1198
+
1199
+
1200
+ # A doc the team TURNED DOWN. Deliberately NOT folded into _DRAFT_STATUS_MARKERS
1201
+ # (v0.2-scope item 7 says "add the marker", and that is the trap): a rejected ADR is not
1202
+ # "proposed, awaiting review" — labelling it so would push a settled no into the human
1203
+ # ratification queue and invite someone to accept what was already refused. It is also not
1204
+ # history to skip (_is_historical_status): a rejected proposal, with its reasons, is
1205
+ # exactly the "tried before, abandoned because…" the store exists to keep. It lands
1206
+ # `rejected` — closed, retrievable, truthfully labelled.
1207
+ # Deliberately NOT a bare substring, unlike _DRAFT_STATUS_MARKERS. The two are not
1208
+ # symmetric in consequence (review finding 3): a false `proposed` lands in the human
1209
+ # ratification queue where someone sees and fixes it, while a false `rejected` lands
1210
+ # CLOSED — invisible to retrieval, so nobody ever notices the record that quietly stopped
1211
+ # being memory. Measured false positives on the bare substring: "Accepted (rejected
1212
+ # alternative: gRPC)", "not rejected", "rejection criteria defined" all landed rejected.
1213
+ _REJECTED_WORD_RE = re.compile(r"(?<!\w)(?<!not )(?<!un-)(?<!un)reject(?:ed)?(?!\w)")
1214
+ _PARENTHETICAL_RE = re.compile(r"\([^)]*\)")
1215
+
1216
+
1217
+ def _is_rejected_status(status: str | None) -> bool:
1218
+ """True when a frontmatter/bold ``status`` value reads as TURNED DOWN.
1219
+
1220
+ Word-boundary match on ``reject``/``rejected``, after parenthesised asides are
1221
+ stripped and with ``not``/``un`` negations excluded. So:
1222
+
1223
+ - ``rejected``, ``Rejected in favour of ADR-9999``, ``proposed, then rejected`` → True
1224
+ - ``Accepted (rejected alternative: gRPC)`` → False (the aside is about an option)
1225
+ - ``not rejected``, ``un-rejected`` → False
1226
+ - ``rejection criteria defined`` → False (``rejection`` is a different word)
1227
+
1228
+ Checked BEFORE :func:`_is_draft_like_status`: a value carrying both readings is
1229
+ terminal, and the terminal one wins."""
1230
+ if not status:
1231
+ return False
1232
+ return bool(_REJECTED_WORD_RE.search(_PARENTHETICAL_RE.sub(" ", status.lower())))
1233
+
1234
+
1235
+ def _is_draft_like_status(status: str | None) -> bool:
1236
+ """True when a frontmatter/bold ``status`` value reads as not-yet-ratified (see
1237
+ :data:`_DRAFT_STATUS_MARKERS`). The caller (``import_docs``) uses this to force the
1238
+ imported decision ``proposed`` REGARDLESS of ``--propose`` — an explicit
1239
+ accepted/approved value, or an absent/unrecognized one, takes the normal
1240
+ ``--propose``-controlled default path unchanged (design: B1's override exists only to
1241
+ stop a genuine distortion, not to second-guess every status string)."""
1242
+ if not status:
1243
+ return False
1244
+ s = status.lower()
1245
+ return any(marker in s for marker in _DRAFT_STATUS_MARKERS)
1246
+
1247
+
1248
+ # -- mention anchors (design §2) ----------------------------------------------------------
1249
+
1250
+
1251
+ def _is_path_like(token: str) -> bool:
1252
+ """A real path never contains whitespace — a backticked prose phrase can still contain
1253
+ a "/" (e.g. `` `see the docs/ folder for details` ``) or end in what looks like an
1254
+ extension by coincidence; rejecting any token with whitespace up front kills that false
1255
+ positive before the "/" / extension checks ever run."""
1256
+ if any(ch.isspace() for ch in token):
1257
+ return False
1258
+ if "/" in token:
1259
+ return True
1260
+ lowered = token.lower()
1261
+ return lowered.endswith(_SOURCE_EXTENSIONS)
1262
+
1263
+
1264
+ def _is_identifier_like(token: str) -> bool:
1265
+ """CamelCase or snake_case, >=4 chars (design §2). A single ordinary word (no case
1266
+ transition, no underscore) never matches either pattern, which is exactly what "not a
1267
+ bare common word" requires — no separate stopword list needed."""
1268
+ if len(token) < 4:
1269
+ return False
1270
+ return bool(_SNAKE_RE.match(token) or _CAMEL_RE.match(token))
1271
+
1272
+
1273
+ def _classify_token(token: str) -> str | None:
1274
+ """``"path" | "identifier" | None`` — path-like takes priority per design §2's
1275
+ ordering (a token like ``anchor_binding.py`` is a path, not an identifier)."""
1276
+ if _is_path_like(token):
1277
+ return "path"
1278
+ if _is_identifier_like(token):
1279
+ return "identifier"
1280
+ return None
1281
+
1282
+
1283
+ def extract_mention_tokens(text: str) -> list[str]:
1284
+ """Backtick-quoted tokens from ``text``, filtered to path-like/identifier-like
1285
+ (design §2 — a token that is neither, e.g. plain prose emphasis, is dropped outright),
1286
+ deduped, and frequency-ranked (most-mentioned first; ties keep first-appearance order,
1287
+ since ``sorted`` is stable and tokens are collected in that order to begin with).
1288
+
1289
+ Pure tokenizer — no graph dependency: it never decides whether a token actually
1290
+ resolves against the graph (that is ``import_docs``'s job, via ``reader``). Callers
1291
+ that care about the "redaction before any use" invariant (design §1) must pass
1292
+ already-redacted ``text``; this function does not redact.
1293
+ """
1294
+ counts: dict[str, int] = {}
1295
+ order: list[str] = []
1296
+ for m in _BACKTICK_RE.finditer(text):
1297
+ token = m.group(1).strip()
1298
+ if not token or _classify_token(token) is None:
1299
+ continue
1300
+ if token not in counts:
1301
+ order.append(token)
1302
+ counts[token] = 0
1303
+ counts[token] += 1
1304
+ return sorted(order, key=lambda t: -counts[t])
1305
+
1306
+
1307
+ def _file_node_descriptor(path: str, reader: GraphifyReader) -> Descriptor | None:
1308
+ """The exact file-level node for ``path`` — never guesses a member: only a node whose
1309
+ label equals the path's basename counts, and it must independently confirm
1310
+ ``resolved`` via ``reader.resolve`` (not just "a node with this name exists somewhere
1311
+ in this file" — never bind an anchor whose resolution isn't itself confirmed clean).
1312
+ Shared by the path-like mention branch and the doc's own always-added file-node anchor
1313
+ (design §2's "Plus always" bullet) — both are literally the same lookup.
1314
+ """
1315
+ nodes = reader.nodes_in_file(path)
1316
+ if not nodes:
1317
+ return None
1318
+ basename = path.rsplit("/", 1)[-1]
1319
+ match = next((n for n in nodes if n.name == basename), None)
1320
+ if match is None:
1321
+ return None
1322
+ ref = Descriptor(name=match.name, file_path=path)
1323
+ if reader.resolve(ref).status != "resolved":
1324
+ return None
1325
+ return ref
1326
+
1327
+
1328
+ def _select_mention_anchors(
1329
+ text: str, reader: GraphifyReader
1330
+ ) -> tuple[list[Descriptor], list[dict]]:
1331
+ """Frequency-ranked mention tokens -> up to :data:`_MAX_MENTION_ANCHORS` resolved
1332
+ anchors (design §2). Never guesses: a path-like token with no matching file node, or
1333
+ an identifier that resolves ambiguous/unresolved, is reported in the second return
1334
+ value (``[{"name", "reason", "candidates"}, ...]`` — reuses the
1335
+ ``anchoring.AnchorResolution`` shape for identifiers; ``"unresolved"``/``[]`` for a
1336
+ path-like miss) and simply doesn't count toward the cap — scanning continues to the
1337
+ next-ranked token rather than stopping at the first miss.
1338
+ """
1339
+ anchors: list[Descriptor] = []
1340
+ skipped: list[dict] = []
1341
+ for token in extract_mention_tokens(text):
1342
+ if len(anchors) >= _MAX_MENTION_ANCHORS:
1343
+ break
1344
+ if _classify_token(token) == "path":
1345
+ ref = _file_node_descriptor(token, reader)
1346
+ if ref is not None:
1347
+ anchors.append(ref)
1348
+ else:
1349
+ skipped.append({"name": token, "reason": "unresolved", "candidates": []})
1350
+ else:
1351
+ ref = Descriptor(name=token)
1352
+ result = reader.resolve(ref)
1353
+ if result.status == "resolved":
1354
+ anchors.append(ref)
1355
+ else:
1356
+ skipped.append(
1357
+ {
1358
+ "name": token,
1359
+ "reason": result.status,
1360
+ "candidates": result.candidates[:5],
1361
+ }
1362
+ )
1363
+ return anchors, skipped
1364
+
1365
+
1366
+ # -- import (design §3/§4) -----------------------------------------------------------------
1367
+
1368
+
1369
+ def _collect_markdown_files(paths: Sequence[str | Path]) -> list[str]:
1370
+ """Expand ``paths`` (each a file or a directory to recurse) into a sorted, deterministic
1371
+ list of ``*.md`` file path strings. A directory contributes every ``*.md`` under it
1372
+ (recursive); a bare file is taken as-is regardless of extension (the parser itself
1373
+ rejects anything that isn't decision-shaped, so an explicit non-``.md`` file is simply
1374
+ very likely to come back unparseable — no special-cased rejection needed here). Paths
1375
+ that don't exist are silently omitted (the CLI validates existence up front so this
1376
+ only matters to direct callers of ``import_docs``)."""
1377
+ out: list[str] = []
1378
+ for p in paths:
1379
+ path = Path(p)
1380
+ if path.is_dir():
1381
+ out.extend(str(f) for f in sorted(path.rglob("*.md")))
1382
+ elif path.is_file():
1383
+ out.append(str(path))
1384
+ return out
1385
+
1386
+
1387
+ def _glob_to_regex(pattern: str) -> re.Pattern[str]:
1388
+ """Translate one repo-relative ingest glob (single ``*``/``?`` wildcards, never
1389
+ crossing a ``/`` — the same semantics ``Path.cwd().glob()`` already uses for
1390
+ profile-only discovery, see ``cli._import_docs_mode``) into a fully-anchored regex
1391
+ (design D7.1).
1392
+
1393
+ Neither ``fnmatch.translate`` nor ``PurePath.match`` fits: ``fnmatch``'s ``*`` crosses
1394
+ ``/`` (a doc several directories deeper than a glob intends would still match), and
1395
+ ``PurePath.match`` right-anchors instead of fully anchoring a RELATIVE pattern (a file
1396
+ under an unrelated leading directory, e.g. ``other/docs/adr/x.md``, would wrongly match
1397
+ ``docs/adr/*.md``) — this enforcement needs neither looseness.
1398
+
1399
+ Raises ``ValueError`` naming the feature when ``pattern`` contains ``**`` (recursive
1400
+ descent) or a ``[seq]`` character class (code review CORRECTION-5): neither is
1401
+ supported by this translator, and silently mistranslating either — ``**`` would
1402
+ flatten to two independent single-segment ``*``s; ``[seq]`` would become a literal,
1403
+ escaped substring match via the ``else`` branch below — would diverge from real glob
1404
+ semantics with no signal beyond a mysteriously-empty import. Loud and immediate (the
1405
+ very first time the pattern is ever used) beats that.
1406
+ """
1407
+ if "**" in pattern:
1408
+ raise ValueError(f"unsupported glob feature '**' (recursive descent) in {pattern!r}")
1409
+ if "[" in pattern or "]" in pattern:
1410
+ raise ValueError(f"unsupported glob feature '[seq]' (character class) in {pattern!r}")
1411
+ parts = []
1412
+ for ch in pattern:
1413
+ if ch == "*":
1414
+ parts.append("[^/]*")
1415
+ elif ch == "?":
1416
+ parts.append("[^/]")
1417
+ else:
1418
+ parts.append(re.escape(ch))
1419
+ return re.compile("".join(parts) + r"\Z")
1420
+
1421
+
1422
+ def _matches_ingest_globs(rel_path: str, globs: tuple[str, ...]) -> bool:
1423
+ """True iff ``rel_path`` (repo-relative) matches at least one of ``globs`` (design
1424
+ D7.1) — the per-file gate ``import_docs`` applies at enumeration, before a file
1425
+ outside the active profile's declared scope is ever opened."""
1426
+ return any(_glob_to_regex(g).fullmatch(rel_path) for g in globs)
1427
+
1428
+
1429
+ def _is_live_tree_path(rel_path: str, profile: FlowProfile) -> bool:
1430
+ """I2 (R1 improvement wave §2) trigger predicate, shared by both write paths
1431
+ (``import_docs`` below and ``bootstrap/planner.py``'s ``plan_sources``): True iff the
1432
+ ON-DISK, repo-root-relative ``rel_path`` sits in ``profile``'s LIVE tree.
1433
+
1434
+ Three clauses, all required (review Minor 5 — direct, not proxy):
1435
+
1436
+ 1. ``profile.ref_normalize`` is set — a profile with no archive-move convention has no
1437
+ live/archived distinction to make, so this is always ``False`` for it (every
1438
+ profile but openspec today).
1439
+ 2. The on-disk ``rel_path`` does NOT already contain the archive segment
1440
+ ``ref_normalize``'s own pattern rewrites — implemented as "the pattern doesn't
1441
+ match the on-disk path" (not merely "doesn't end in a date-stamped dir"), so a
1442
+ non-dated archive dir still contains the literal archive segment and is excluded
1443
+ the same way.
1444
+ 3. ``rel_path`` matches ``profile.ingest_globs`` — closes the ``--any-doc`` stray-doc
1445
+ hole: a doc reached via ``--any-doc`` from outside the profile's own tree is never
1446
+ stamped, even if it happens to dodge clause 2.
1447
+ """
1448
+ if profile.ref_normalize is None:
1449
+ return False
1450
+ pattern, _replacement = profile.ref_normalize
1451
+ if re.search(pattern, rel_path):
1452
+ return False
1453
+ return _matches_ingest_globs(rel_path, profile.ingest_globs)
1454
+
1455
+
1456
+ def _glob_repo_root() -> Path:
1457
+ """The repo root the D7.1 import-glob gate normalizes against — resolved from the
1458
+ process's current directory the same way D5 resolves ``sidegraph-doctor``'s
1459
+ (``verify._find_repo_root``: ``git rev-parse --show-toplevel``, wrapped never-raise).
1460
+ ``ingest_globs`` are repo-root-relative patterns, never cwd-relative — a caller
1461
+ running ``import_docs`` from a subdirectory (or passing a path via a symlinked
1462
+ prefix) must still normalize against the SAME root a glob like ``docs/adr/*.md``
1463
+ means. Git missing, no repo, or any other failure all fall back to the bare (but
1464
+ still ``.resolve()``-d) cwd itself, so a caller already running from the repo root —
1465
+ the common case — behaves exactly as before this fix (BLOCKING-1, code review)."""
1466
+ try:
1467
+ return _find_repo_root(Path.cwd())
1468
+ except (ValueError, OSError):
1469
+ return Path.cwd().resolve()
1470
+
1471
+
1472
+ def import_docs(
1473
+ store: Store,
1474
+ reader: GraphifyReader,
1475
+ paths: Sequence[str | Path],
1476
+ *,
1477
+ kind: str | None = None,
1478
+ propose: bool = False,
1479
+ dry_run: bool = False,
1480
+ limit: int | None = None,
1481
+ tags: list[str] | None = None,
1482
+ section_limit: int = _SECTION_LIMIT,
1483
+ profile: str = "generic-adr",
1484
+ any_doc: bool = False,
1485
+ ratify_policy: RatifyPolicy = RatifyPolicy.MANUAL,
1486
+ ) -> DocImportReport:
1487
+ """One :class:`~sidegraph.schema.Decision` per decision-shaped markdown file under
1488
+ ``paths`` (each a file or a directory recursed for ``*.md``; see design §4).
1489
+
1490
+ ``kind`` (fix-wave B, B4): ``None`` (the default) means "auto" — per document,
1491
+ :attr:`ParsedDoc.suggested_kind` is used when the parser found a signal (currently: a
1492
+ qualifying "Root cause" section -> ``lesson``), else ``adr``. An explicit string (any
1493
+ valid :class:`~sidegraph.schema.DecisionKind` value, including ``"adr"``) always wins
1494
+ for every document this run, exactly like before B4 — the heuristic only fires when the
1495
+ caller didn't ask for a specific kind.
1496
+
1497
+ ``section_limit`` (fix-wave B, B3, default :data:`_SECTION_LIMIT`): forwarded to
1498
+ :func:`parse_decision_doc` as the per-field word-boundary truncation cap.
1499
+
1500
+ Pipeline per file: ``parse_decision_doc`` -> historical-frontmatter skip -> idempotency
1501
+ lookup (``provenance.source == "doc-import"``, ``provenance.ref == <rel_path>``, title-
1502
+ agnostic — see ``Store.find_decisions_by_ref``) -> mention-anchor resolution (only when
1503
+ about to write; a doc that's ``skipped_existing`` never touches the graph) -> write.
1504
+
1505
+ **Idempotency & evolution (design §3, append-only, no schema change; pending-proposal
1506
+ dedup fix).** The idempotency check compares the fresh parse against EVERY currently
1507
+ open (``accepted`` OR ``proposed``) decision at the same ``provenance.ref`` — not just
1508
+ one of them (see ``Store.find_decisions_by_ref``'s docstring for the regression this
1509
+ fixes: comparing against only a single, scan-order-picked record let a re-run miss an
1510
+ already-pending proposal with matching content and duplicate it every run). A doc whose
1511
+ ``(title, choice, rejected)`` are byte-identical to ANY open record at that ref is
1512
+ ``skipped_existing`` — no anchor resolution even attempted for it, and nothing is
1513
+ written or touched, regardless of which of the (at most two) open records matched.
1514
+
1515
+ A doc at the same ``ref`` whose parsed content differs from ALL open records (the doc
1516
+ was edited again) gets a fresh :class:`~sidegraph.schema.Decision`. Its ``supersedes``
1517
+ always points at the ACCEPTED ancestor, if one is open — never at a still-pending
1518
+ proposal — so that a chain of edit/``--propose`` cycles always resolves back to the one
1519
+ real predecessor a ratify-accept needs to close. If a proposal was still pending at
1520
+ this ref (``edit-again-while-pending``), that stale, never-ratified draft is closed
1521
+ immediately as ``rejected`` (via ``Store.drop``) as part of this same write — closing
1522
+ an unratified draft is NOT the ratify-guarded case ``Store.ratify`` exists for (nothing
1523
+ reviewed/accepted it yet), so there is nothing to defer; it simply never matched the
1524
+ new parse and is superseded-in-spirit by the new proposal/decision being written here.
1525
+
1526
+ **Status-derived ``proposed`` override (fix-wave B, B1):** when the doc's own extracted
1527
+ status (YAML frontmatter ``status:`` or a bold ``**Status:**`` line) reads as
1528
+ draft/proposed/pending/under-review-like (:func:`_is_draft_like_status`), the decision
1529
+ lands ``proposed`` REGARDLESS of ``--propose`` — calibrated against a real ADR corpus
1530
+ where every source doc says ``status: draft`` / "proposed, pending EACL review" but the
1531
+ unconditional-``accepted`` default was silently promoting all of them past review. An
1532
+ explicit accepted/approved status, or an absent/unrecognized one, takes the normal
1533
+ ``--propose``-controlled default path, unchanged. ``DocImportReport.status_derived_proposed``
1534
+ counts how many decisions this run landed ``proposed`` for this reason specifically (the
1535
+ CLI prints it as its own report line).
1536
+
1537
+ **``--propose`` rule (design §3, option (b) — ratify-time supersession):** ``propose``
1538
+ (or the status-derived override above — either one landing the successor ``proposed``
1539
+ has the same effect here) controls the successor's ``status`` (``proposed`` vs
1540
+ ``accepted``) same as a fresh import, AND gates WHEN the ACCEPTED ancestor (if any)
1541
+ closes. Without either (or on a doc with no accepted ancestor at all),
1542
+ ``store.add_decision`` closes the ancestor immediately in the same transaction as before
1543
+ (the same mechanism
1544
+ ``server._supersede_decision_impl`` uses) — nothing changed there. WHEN the successor
1545
+ lands ``proposed`` (``--propose`` or the status-derived override) on an edited doc, it
1546
+ carries ``supersedes=<ancestor_id>``
1547
+ but the ancestor is left exactly as it was (typically still ``accepted``) — an accepted
1548
+ decision must never be silently closed by a mere proposal that hasn't been reviewed
1549
+ yet. The ancestor only closes once a human actually ratifies the successor:
1550
+ ``Store.ratify`` closes any still-open (``accepted``/``proposed``) predecessor it
1551
+ supersedes in the same operation (valid_to + status=superseded) — this is shared,
1552
+ generic machinery, not doc-import-specific, so it applies identically whether
1553
+ ratification comes through the MCP ``ratify``/``ratify_decisions`` tools or
1554
+ ``sidegraph-ratify``. Dropping the proposal instead (``Store.drop``) never touches the
1555
+ ancestor — it stays untouched, exactly as if the edit had never been proposed. History
1556
+ is preserved either way; the only change from the old rule is WHEN an accepted record
1557
+ closes.
1558
+
1559
+ **Anchors (design §2)** are always freshly resolved from the CURRENT doc content and
1560
+ CURRENT graph on every run (never inherited from a superseded predecessor, unlike
1561
+ ``supersede_decision``'s default) — the whole point of doc-import is that the doc's
1562
+ mentions are the source of truth. A doc with NO anchor at all (no resolved mention AND
1563
+ no doc-level file node) is skipped entirely and counted ``skipped_unanchorable`` —
1564
+ consistent with importer #1, never written with zero anchors.
1565
+
1566
+ ``dry_run=True`` runs the full pipeline (parse -> frontmatter -> idempotency -> anchor
1567
+ resolution) so counts reflect what WOULD happen, but writes nothing;
1568
+ ``DocImportReport.dry_run`` carries ``{"file_path", "ref", "title", "action":
1569
+ "imported"|"superseded", "anchors_skipped"}`` per RECORD (a split-capable dialect can
1570
+ emit several per file). ``tags`` (free text, slugified, empty slugs
1571
+ dropped) bind durable ``tag:<slug>`` entities on every written decision, same mechanism
1572
+ as ``capture.py``'s tags.
1573
+
1574
+ **Import glob enforcement (design D7.1, staleness-machinery wave, E8 hygiene):** every
1575
+ enumerated file (from an explicit ``--docs PATH`` — a file or, recursed, a directory —
1576
+ the CLI's explicit-path branch bypasses profile scoping entirely, so this is the only
1577
+ layer that ever sees individual files) is matched against the active ``profile``'s
1578
+ ``ingest_globs`` (:func:`_matches_ingest_globs`) BEFORE it is even opened. A file
1579
+ outside every glob is skipped and counted ``skipped_outside_profile`` — never parsed,
1580
+ never anchor-resolved. ``any_doc=True`` restores the pre-D7.1 no-filter behavior (the
1581
+ CLI's ``--any-doc`` flag). Profile-only discovery (no explicit ``--docs PATH``) is
1582
+ unaffected either way: those files already came FROM ``profile.ingest_globs``, so the
1583
+ check is a structural no-op there.
1584
+
1585
+ Normalization is against the REPO ROOT (:func:`_glob_repo_root`), never the process's
1586
+ current directory (BLOCKING-1, code review): ``ingest_globs`` are repo-relative
1587
+ patterns, so a caller running from a subdirectory, or passing an absolute path (or one
1588
+ reached through a symlinked prefix) from anywhere but the repo root, must still
1589
+ resolve to the same repo-relative form a glob like ``docs/adr/*.md`` means. Both the
1590
+ file and the repo root are ``.resolve()``-d before the comparison (the macOS
1591
+ ``/tmp`` -> ``/private/tmp`` symlink case).
1592
+
1593
+ **E1/E2/E3 (openspec profile design, design/superpowers/specs/2026-08-06-openspec-
1594
+ profile-design.md).** E1: ``active_profile.title_pattern`` is threaded into the parse
1595
+ call so a doc with no H1 still qualifies when its path matches (§5). E2:
1596
+ ``active_profile.normalized_rel_path(rel_path)`` — not the on-disk path — is what gets
1597
+ parsed and so stamped into ``context``/``provenance.ref``, so a doc re-imported after a
1598
+ flow's own archive-style move dedupes by ref instead of duplicating (§6); the on-disk
1599
+ path stays in play for file I/O, the anchor lookup, and the dry-run/``by_file()``
1600
+ listings. E3: a split-produced parent that is either an echo of its own ``context`` or
1601
+ empty even after every fallback is never written — ``DocImportReport.
1602
+ skipped_degenerate_parent`` counts it, once per file, and its children stand alone (§7).
1603
+
1604
+ ``ratify_policy`` (default ``RatifyPolicy.MANUAL``): the resolved
1605
+ ``SIDEGRAPH_RATIFY_POLICY`` value (design D1), sampled once by the CLI shell
1606
+ immediately before this call and passed down unchanged. ``_import_one_record``'s own
1607
+ post-write block stamps a written record ``auto:<policy>`` via the shared
1608
+ ``_auto_ratify`` helper when it landed ``proposed`` (``lands_proposed``, not the raw
1609
+ ``propose`` flag — a ``status: rejected`` doc imported with ``--propose`` lands
1610
+ REJECTED with ``action == "written"`` and must never be handed to a ratify
1611
+ transition) and passes the same D3 gate every other write path uses. This seam (the
1612
+ per-record adapter) is deliberately the ONLY hook site — the shared
1613
+ ``apply_doc_candidate``/``_apply_doc_state`` write path stays untouched, since
1614
+ Bootstrap's ``bootstrap/apply.py`` calls it directly and must never auto-ratify.
1615
+ # see design/superpowers/specs/2026-09-11-auto-ratification-policy-design.md D1/D2/D3
1616
+ """
1617
+ # None = "auto per document" (B4); an explicit kind (including "adr") always wins.
1618
+ decision_kind_override = DecisionKind(kind) if kind is not None else None
1619
+ graph_version = reader.graph_version()
1620
+ active_profile = get_profile(profile)
1621
+ dialect = active_profile.dialect
1622
+ tag_slugs = [s for s in (slugify(t) for t in (tags or [])) if s and s != "redacted"]
1623
+
1624
+ files = _collect_markdown_files(paths)
1625
+ process = files[:limit] if limit is not None else files
1626
+
1627
+ report = DocImportReport()
1628
+ # Resolved ONCE per run, not per file (BLOCKING-1) — repo-root-relative, never
1629
+ # cwd-relative. Always computed now (I2, R1 improvement wave §2): the live-tree
1630
+ # trigger predicate below needs a repo-root-relative on-disk path REGARDLESS of
1631
+ # `any_doc` — its own glob-match clause is independent of the D7.1 enforcement gate
1632
+ # below, which stays `any_doc`-gated exactly as before.
1633
+ repo_root = _glob_repo_root()
1634
+
1635
+ for rel_path in process:
1636
+ # D7.1: enforcement runs FIRST, before the file is even read — an out-of-profile
1637
+ # doc costs nothing beyond the glob check itself. Both sides are `.resolve()`-d
1638
+ # (follows symlinks, absolute-izes a relative path against the CURRENT cwd) before
1639
+ # computing the repo-root-relative form a glob like `docs/adr/*.md` actually means
1640
+ # — see `_glob_repo_root`'s docstring for why cwd alone isn't enough.
1641
+ glob_check_path = os.path.relpath(Path(rel_path).resolve(), repo_root)
1642
+ if not any_doc and not _matches_ingest_globs(glob_check_path, active_profile.ingest_globs):
1643
+ report.skipped_outside_profile += 1
1644
+ continue
1645
+
1646
+ source_bytes = Path(rel_path).read_bytes()
1647
+ source_hash = hashlib.sha256(source_bytes).hexdigest()
1648
+ raw_text = source_bytes.decode().replace("\r\n", "\n").replace("\r", "\n")
1649
+ # E2 (design note §6): the NORMALIZED path — not the on-disk `rel_path` — is what
1650
+ # gets stamped into `context`/children's `effective_ref`/`provenance.ref` (via
1651
+ # `DocWriteRequest.rel_path` below), so an unedited doc re-imported after an
1652
+ # archive-style move dedupes by ref instead of duplicating. The on-disk `rel_path`
1653
+ # stays in play for exactly two things: file I/O (already done, above) and the
1654
+ # anchor lookup (`anchor_lookup_path`, below) — plus the dry-run/`by_file()`
1655
+ # listings, which deliberately show what the operator would open (review N5).
1656
+ normalized_rel_path = active_profile.normalized_rel_path(rel_path)
1657
+ docs, reason = _parse_decision_docs_with_reason(
1658
+ raw_text,
1659
+ normalized_rel_path,
1660
+ section_limit=section_limit,
1661
+ dialect=dialect,
1662
+ title_pattern=active_profile.title_pattern,
1663
+ )
1664
+ if not docs:
1665
+ if reason == "template":
1666
+ report.skipped_template += 1
1667
+ elif reason == "unparseable":
1668
+ report.skipped_unparseable += 1
1669
+ else:
1670
+ report.skipped_not_decision += 1
1671
+ continue
1672
+ # Historical status is a DOC-level property (frontmatter) — checked once per file;
1673
+ # children inherit the parent's frontmatter_status by construction.
1674
+ if _is_historical_status(docs[0].frontmatter_status):
1675
+ report.skipped_superseded_frontmatter += 1
1676
+ continue
1677
+
1678
+ # E3 rule 1 (design note §7, review Major 4): doc_import-only — bootstrap already
1679
+ # refuses an echo-choice parent via its own `_choice_is_context_fallback` check, so
1680
+ # only this write path has the gap to close. Rule 2 (the empty-choice case)
1681
+ # already happened inside the parser when it fires: `docs` simply has no record
1682
+ # with `fragment is None` (no parent) while its children are present. Both cases
1683
+ # share one counter; rule 1 additionally drops the echoed parent from `docs` before
1684
+ # any record from this file reaches `_import_one_record`.
1685
+ has_parent = docs[0].fragment is None
1686
+ has_children = any(d.fragment is not None for d in docs)
1687
+ if not has_parent and has_children:
1688
+ report.skipped_degenerate_parent += 1
1689
+ elif (
1690
+ has_parent
1691
+ and has_children
1692
+ and _choice_is_context_echo(docs[0].context, docs[0].choice, normalized_rel_path)
1693
+ ):
1694
+ report.skipped_degenerate_parent += 1
1695
+ docs = docs[1:]
1696
+
1697
+ # I2 (R1 improvement wave §2, Blocker 1 — decide-then-stamp): the in-flight note is
1698
+ # appended to every SURVIVING record's `context` — parent AND children alike (review
1699
+ # Major 2) — only AFTER the degenerate-parent decision above has already run against
1700
+ # the note-free context. Stamping any earlier would corrupt
1701
+ # `_choice_is_context_echo`'s own strip-and-compare (measured: flips the E3 rule 1
1702
+ # refusal from 11 hits to 0 on the live tree) — the ONLY correct order is
1703
+ # decide-then-stamp. `test_openspec_bootstrap_apply_live_path_note_matches_importer`
1704
+ # (test_bootstrap_apply.py) is the cross-path stamp-equality pin for a LIVE doc — the
1705
+ # profile wave's own T12 only exercises an archived doc, where neither write path
1706
+ # stamps at all, so it can't catch a one-sided or differently-placed note here; if
1707
+ # the live twin above goes red, the bug is in this sequencing.
1708
+ if active_profile.in_flight_note and _is_live_tree_path(glob_check_path, active_profile):
1709
+ note = active_profile.in_flight_note
1710
+ docs = [d.model_copy(update={"context": f"{d.context}\n\n{note}"}) for d in docs]
1711
+
1712
+ # Doc-level anchor context, computed once per file: the redacted whole text feeds
1713
+ # the PARENT's mention anchors (unchanged); the file-node anchor is shared by every
1714
+ # record from this file. Children take their mention anchors from their OWN
1715
+ # redacted title+choice text instead — a per-AD record should bind to the symbols
1716
+ # its own block names, not to the whole document's top-3.
1717
+ clean_whole, _ = redact(raw_text)
1718
+ # Doc-node anchor lookup needs a REPO-RELATIVE path to match the graph's
1719
+ # `source_file` — a caller (the CLI's `--docs` in particular) may pass an absolute
1720
+ # path straight through; normalize it against the cwd here, just for this lookup
1721
+ # (provenance.ref/idempotency stay keyed on whatever path the caller actually
1722
+ # passed, unchanged — the CLI's absolute-path warning depends on this rationale).
1723
+ # Deliberately the ON-DISK path (E2): anchor lookups match the graph's actual
1724
+ # `source_file`, which never moves just because a flow archives a change folder.
1725
+ anchor_lookup_path = rel_path
1726
+ if os.path.isabs(anchor_lookup_path):
1727
+ anchor_lookup_path = os.path.relpath(anchor_lookup_path, os.getcwd())
1728
+ file_anchor = _file_node_descriptor(anchor_lookup_path, reader)
1729
+
1730
+ for parsed in docs:
1731
+ _import_one_record(
1732
+ parsed,
1733
+ normalized_rel_path,
1734
+ file_path=rel_path,
1735
+ report=report,
1736
+ store=store,
1737
+ reader=reader,
1738
+ source_hash=source_hash,
1739
+ graph_version=graph_version,
1740
+ clean_whole=clean_whole,
1741
+ file_anchor=file_anchor,
1742
+ tag_slugs=tag_slugs,
1743
+ decision_kind_override=decision_kind_override,
1744
+ propose=propose,
1745
+ dry_run=dry_run,
1746
+ ratify_policy=ratify_policy,
1747
+ )
1748
+
1749
+ return report
1750
+
1751
+
1752
+ def _import_one_record(
1753
+ parsed: ParsedDoc,
1754
+ rel_path: str,
1755
+ *,
1756
+ file_path: str,
1757
+ report: DocImportReport,
1758
+ store: Store,
1759
+ reader: GraphifyReader,
1760
+ source_hash: str,
1761
+ graph_version: str | None,
1762
+ clean_whole: str,
1763
+ file_anchor: Descriptor | None,
1764
+ tag_slugs: list[str],
1765
+ decision_kind_override: DecisionKind | None,
1766
+ propose: bool,
1767
+ dry_run: bool,
1768
+ ratify_policy: RatifyPolicy,
1769
+ ) -> None:
1770
+ """Compatibility adapter from importer options/reporting to the shared write seam.
1771
+
1772
+ ``rel_path`` (E2, design note §6) is the NORMALIZED path — it drives
1773
+ ``DocWriteRequest.rel_path``/``.ref`` and so ``context``'s "imported from" stamp and
1774
+ ``provenance.ref``. ``file_path`` is the real, on-disk path (review N5) — used ONLY for
1775
+ the dry-run listing's ``"file_path"`` entry, which shows the operator what they'd
1776
+ actually open; every other on-disk use (file I/O, the anchor lookup) already happened
1777
+ in the caller before this record-level adapter runs."""
1778
+ status_rejected = _is_rejected_status(parsed.frontmatter_status)
1779
+ status_derived = not status_rejected and _is_draft_like_status(parsed.frontmatter_status)
1780
+ lands_proposed = not status_rejected and (propose or status_derived)
1781
+ if status_rejected:
1782
+ status = DecisionStatus.REJECTED
1783
+ else:
1784
+ status = DecisionStatus.PROPOSED if lands_proposed else DecisionStatus.ACCEPTED
1785
+ parsed_for_write = parsed.model_copy(
1786
+ update={"suggested_kind": decision_kind_override or parsed.suggested_kind}
1787
+ )
1788
+ preflight = DocWriteRequest(
1789
+ parsed=parsed_for_write,
1790
+ rel_path=rel_path,
1791
+ source_hash=source_hash,
1792
+ status=status,
1793
+ file_anchor=file_anchor,
1794
+ graph_version=graph_version,
1795
+ tags=tuple(tag_slugs),
1796
+ )
1797
+ preflight_disposition = _classify_doc_candidate(store, preflight)
1798
+ if preflight_disposition.action == "skipped-existing":
1799
+ report.skipped_existing += 1
1800
+ return
1801
+
1802
+ mention_source = clean_whole if parsed.fragment is None else f"{parsed.title}\n{parsed.choice}"
1803
+ anchors, anchors_skipped = _select_mention_anchors(mention_source, reader)
1804
+ request = preflight.model_copy(
1805
+ update={"anchors": tuple(anchors), "anchors_skipped": tuple(anchors_skipped)}
1806
+ )
1807
+
1808
+ if dry_run:
1809
+ disposition = _classify_doc_candidate(store, request)
1810
+ result = DocWriteResult(action=disposition.action)
1811
+ else:
1812
+ result = apply_doc_candidate(store, reader, request)
1813
+
1814
+ # Auto-ratify (design D2/D3, spec rev 9 Ruling V): gated on `lands_proposed`, NOT the
1815
+ # raw `propose` flag — a `status: rejected` doc lands REJECTED even with `--propose`
1816
+ # (`action == "written"`), and handing that to _auto_ratify would report a spurious
1817
+ # "not proposed" failure. `result.action == "written"` excludes a superseding write
1818
+ # (D2 defers the predecessor's close to a successful Store.ratify, never to an
1819
+ # unratified auto write); `result.decision_id is not None` narrows for mypy and is
1820
+ # never False here (a dry run never reaches this branch at all: `not dry_run` is a
1821
+ # declared equivalent mutant — the dry-run DocWriteResult above carries no
1822
+ # decision_id, so _anchor_signal(store, None) would be ineligible anyway).
1823
+ if (
1824
+ lands_proposed
1825
+ and not status_derived
1826
+ and not dry_run
1827
+ and result.action == "written"
1828
+ and result.decision_id is not None
1829
+ ):
1830
+ decision = store.get_decision(result.decision_id)
1831
+ assert decision is not None # just written by apply_doc_candidate, above
1832
+ live_tier12, ambiguous_or_orphan_only = _anchor_signal(store, result.decision_id)
1833
+ signal = AutoEligibility(
1834
+ kind=decision.kind.value,
1835
+ live_tier12=live_tier12,
1836
+ ambiguous_or_orphan_only=ambiguous_or_orphan_only,
1837
+ pipeline_clean=True,
1838
+ has_provenance=True,
1839
+ domain_anchored=False,
1840
+ has_supersedes=decision.supersedes is not None,
1841
+ )
1842
+ if auto_ratify_eligible(signal, ratify_policy):
1843
+ outcome = _auto_ratify(store, result.decision_id, signal.kind, ratify_policy)
1844
+ if outcome.ratified_by is not None:
1845
+ report.auto_ratified += 1
1846
+ if outcome.error is not None:
1847
+ report.auto_ratify_failures.append(f"{result.decision_id}: {outcome.error}")
1848
+
1849
+ if result.action == "skipped-existing":
1850
+ report.skipped_existing += 1
1851
+ return
1852
+ if result.action == "skipped-unanchorable":
1853
+ report.skipped_unanchorable += 1
1854
+ return
1855
+ if result.action == "superseded":
1856
+ report.superseded += 1
1857
+ else:
1858
+ report.imported += 1
1859
+ if status_derived:
1860
+ report.status_derived_proposed += 1
1861
+ if status_rejected:
1862
+ report.status_derived_rejected += 1
1863
+ if dry_run:
1864
+ report.dry_run.append(
1865
+ {
1866
+ "file_path": file_path,
1867
+ "ref": request.ref,
1868
+ "title": parsed.title,
1869
+ "action": "superseded" if result.action == "superseded" else "imported",
1870
+ "anchors_skipped": anchors_skipped,
1871
+ }
1872
+ )
1873
+
1874
+
1875
+ _DOC_WRITE_STATUSES = (
1876
+ DecisionStatus.ACCEPTED,
1877
+ DecisionStatus.PROPOSED,
1878
+ DecisionStatus.REJECTED,
1879
+ )
1880
+
1881
+
1882
+ def _content_matches(request: DocWriteRequest, decisions: Sequence[Decision]) -> list[Decision]:
1883
+ parsed = request.parsed
1884
+ return [
1885
+ decision
1886
+ for decision in decisions
1887
+ if (
1888
+ decision.title,
1889
+ decision.context,
1890
+ decision.choice,
1891
+ decision.rejected,
1892
+ decision.consequences,
1893
+ )
1894
+ == (
1895
+ parsed.title,
1896
+ parsed.context,
1897
+ parsed.choice,
1898
+ parsed.rejected,
1899
+ parsed.consequences,
1900
+ )
1901
+ ]
1902
+
1903
+
1904
+ def _preferred_content_match(
1905
+ request: DocWriteRequest, decisions: Sequence[Decision]
1906
+ ) -> Decision | None:
1907
+ matches = _content_matches(request, decisions)
1908
+ open_matches = [decision for decision in matches if decision.status != DecisionStatus.REJECTED]
1909
+ rejected_matches = [
1910
+ decision for decision in matches if decision.status == DecisionStatus.REJECTED
1911
+ ]
1912
+ return next(iter(open_matches), None) or next(iter(rejected_matches), None)
1913
+
1914
+
1915
+ def _classify_doc_state(
1916
+ store: Store, request: DocWriteRequest, existing: Sequence[Decision]
1917
+ ) -> _DocDisposition:
1918
+ match = _preferred_content_match(request, existing)
1919
+ status_rejected = request.status == DecisionStatus.REJECTED
1920
+ needs_status_change = match is not None and status_rejected != (
1921
+ match.status == DecisionStatus.REJECTED
1922
+ )
1923
+ supersedable = tuple(
1924
+ decision for decision in existing if decision.status != DecisionStatus.REJECTED
1925
+ )
1926
+ accepted_ancestor = next(
1927
+ (decision for decision in existing if decision.status == DecisionStatus.ACCEPTED), None
1928
+ )
1929
+ pending_proposal = next(
1930
+ (decision for decision in existing if decision.status == DecisionStatus.PROPOSED), None
1931
+ )
1932
+
1933
+ if match is not None and not needs_status_change:
1934
+ return _DocDisposition(
1935
+ action="skipped-existing",
1936
+ accepted_ancestor=accepted_ancestor,
1937
+ pending_proposal=pending_proposal,
1938
+ supersedable=supersedable,
1939
+ inherit_bindings_from=None,
1940
+ )
1941
+
1942
+ inherit_bindings_from = None
1943
+ if not request.anchors and request.file_anchor is None:
1944
+ donor = match if needs_status_change else None
1945
+ if donor is None and status_rejected and supersedable:
1946
+ donor = supersedable[0]
1947
+ if donor is not None and store.bindings_for_record(donor.id):
1948
+ inherit_bindings_from = donor.id
1949
+ else:
1950
+ return _DocDisposition(
1951
+ action="skipped-unanchorable",
1952
+ accepted_ancestor=accepted_ancestor,
1953
+ pending_proposal=pending_proposal,
1954
+ supersedable=supersedable,
1955
+ inherit_bindings_from=None,
1956
+ )
1957
+
1958
+ return _DocDisposition(
1959
+ action="superseded" if supersedable else "written",
1960
+ accepted_ancestor=accepted_ancestor,
1961
+ pending_proposal=pending_proposal,
1962
+ supersedable=supersedable,
1963
+ inherit_bindings_from=inherit_bindings_from,
1964
+ )
1965
+
1966
+
1967
+ def _classify_doc_candidate(store: Store, request: DocWriteRequest) -> _DocDisposition:
1968
+ existing = store.find_decisions_by_ref("doc-import", request.ref, statuses=_DOC_WRITE_STATUSES)
1969
+ return _classify_doc_state(store, request, existing)
1970
+
1971
+
1972
+ def _apply_doc_bindings(
1973
+ *,
1974
+ store: Store,
1975
+ reader: GraphifyReader,
1976
+ request: DocWriteRequest,
1977
+ decision_id: str,
1978
+ inherit_bindings_from: str | None,
1979
+ ) -> None:
1980
+ """Idempotently apply every binding requested for one document decision."""
1981
+ for anchor in request.anchors:
1982
+ resolve_and_bind(decision_id, anchor, reader, store)
1983
+ for inherited in (
1984
+ store.bindings_for_record(inherit_bindings_from) if inherit_bindings_from else []
1985
+ ):
1986
+ store.add_binding(
1987
+ AnchorBinding(
1988
+ record_id=decision_id,
1989
+ entity_id=inherited.entity_id,
1990
+ tier=inherited.tier,
1991
+ weight=inherited.weight,
1992
+ relation=inherited.relation,
1993
+ status=inherited.status,
1994
+ )
1995
+ )
1996
+ if request.file_anchor is not None:
1997
+ resolve_and_bind(decision_id, request.file_anchor, reader, store)
1998
+ for slug in request.tags:
1999
+ tag_entity = store.get_or_create_abstract_entity(f"tag:{slug}")
2000
+ store.add_binding(
2001
+ AnchorBinding(record_id=decision_id, entity_id=tag_entity.entity_id, tier=0)
2002
+ )
2003
+
2004
+
2005
+ def _apply_doc_state(
2006
+ *,
2007
+ store: Store,
2008
+ reader: GraphifyReader,
2009
+ request: DocWriteRequest,
2010
+ existing: Sequence[Decision],
2011
+ decision_kind: DecisionKind,
2012
+ ) -> DocWriteResult:
2013
+ disposition = _classify_doc_state(store, request, existing)
2014
+ if disposition.action == "skipped-unanchorable":
2015
+ return DocWriteResult(action=disposition.action)
2016
+ if disposition.action == "skipped-existing":
2017
+ match = _preferred_content_match(request, existing)
2018
+ if match is None: # Defensive: classification can skip only a content match.
2019
+ raise RuntimeError("document classification lost its existing content match")
2020
+ if (
2021
+ request.ratify_matching_proposal
2022
+ and request.status == DecisionStatus.ACCEPTED
2023
+ and match.status == DecisionStatus.PROPOSED
2024
+ ):
2025
+ match, _cascaded = store.ratify(match.id)
2026
+ _apply_doc_bindings(
2027
+ store=store,
2028
+ reader=reader,
2029
+ request=request,
2030
+ decision_id=match.id,
2031
+ inherit_bindings_from=None,
2032
+ )
2033
+ return DocWriteResult(action=disposition.action, decision_id=match.id)
2034
+
2035
+ if disposition.pending_proposal is not None:
2036
+ store.drop(disposition.pending_proposal.id)
2037
+
2038
+ match = _preferred_content_match(request, existing)
2039
+ needs_status_change = match is not None and (request.status == DecisionStatus.REJECTED) != (
2040
+ match.status == DecisionStatus.REJECTED
2041
+ )
2042
+ decision = Decision(
2043
+ title=request.parsed.title,
2044
+ kind=decision_kind,
2045
+ status=request.status,
2046
+ context=request.parsed.context,
2047
+ choice=request.parsed.choice,
2048
+ rejected=request.parsed.rejected,
2049
+ consequences=request.parsed.consequences,
2050
+ valid_from=datetime.now(UTC),
2051
+ valid_to=datetime.now(UTC) if request.status == DecisionStatus.REJECTED else None,
2052
+ supersedes=(
2053
+ disposition.accepted_ancestor.id
2054
+ if disposition.accepted_ancestor is not None
2055
+ else (match.id if needs_status_change and match is not None else None)
2056
+ ),
2057
+ provenance=Provenance(
2058
+ source="doc-import",
2059
+ author="sidegraph-import",
2060
+ ref=request.ref,
2061
+ graph_version=request.graph_version,
2062
+ ),
2063
+ )
2064
+ close_predecessor = not (
2065
+ request.status == DecisionStatus.PROPOSED and disposition.accepted_ancestor is not None
2066
+ )
2067
+ store.add_decision(decision, close_predecessor=close_predecessor)
2068
+ _apply_doc_bindings(
2069
+ store=store,
2070
+ reader=reader,
2071
+ request=request,
2072
+ decision_id=decision.id,
2073
+ inherit_bindings_from=disposition.inherit_bindings_from,
2074
+ )
2075
+
2076
+ return DocWriteResult(
2077
+ action=disposition.action,
2078
+ decision_id=decision.id,
2079
+ )
2080
+
2081
+
2082
+ def apply_doc_candidate(
2083
+ store: Store, reader: GraphifyReader, request: DocWriteRequest
2084
+ ) -> DocWriteResult:
2085
+ if request.status not in (
2086
+ DecisionStatus.ACCEPTED,
2087
+ DecisionStatus.PROPOSED,
2088
+ DecisionStatus.REJECTED,
2089
+ ):
2090
+ raise ValueError("document import can write only accepted, proposed, or rejected")
2091
+ decision_kind = request.parsed.suggested_kind or DecisionKind.ADR
2092
+ existing = store.find_decisions_by_ref("doc-import", request.ref, statuses=_DOC_WRITE_STATUSES)
2093
+ return _apply_doc_state(
2094
+ store=store,
2095
+ reader=reader,
2096
+ request=request,
2097
+ existing=existing,
2098
+ decision_kind=decision_kind,
2099
+ )