devkb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. devkb/__init__.py +11 -0
  2. devkb/audit.py +173 -0
  3. devkb/bench/add_packs.py +52 -0
  4. devkb/bench/cli_tasks.py +105 -0
  5. devkb/bench/confirm_tasks.py +149 -0
  6. devkb/bench/module_truth.py +249 -0
  7. devkb/bench/oco_tasks.py +216 -0
  8. devkb/bench/pipeline.py +313 -0
  9. devkb/bench/process_tasks.py +135 -0
  10. devkb/bench/report.py +74 -0
  11. devkb/bench/run.py +174 -0
  12. devkb/bench/truth_check.py +148 -0
  13. devkb/bench/workflow_report.py +64 -0
  14. devkb/bench/workflow_run.py +206 -0
  15. devkb/bench/workflow_tasks.py +227 -0
  16. devkb/build.py +258 -0
  17. devkb/build_symbols.py +1394 -0
  18. devkb/call_recall.py +159 -0
  19. devkb/cli.py +53 -0
  20. devkb/coverage.py +155 -0
  21. devkb/edge_truth.py +272 -0
  22. devkb/extractors/backlog.py +113 -0
  23. devkb/extractors/component_subsystem.py +116 -0
  24. devkb/extractors/configuration.py +267 -0
  25. devkb/extractors/files_tests.py +220 -0
  26. devkb/extractors/flows.py +117 -0
  27. devkb/extractors/lifecycle.py +258 -0
  28. devkb/extractors/members.py +160 -0
  29. devkb/extractors/pull_requests.py +271 -0
  30. devkb/extractors/releases.py +49 -0
  31. devkb/extractors/source_baseline.py +66 -0
  32. devkb/extractors/spec_docs.py +220 -0
  33. devkb/extractors/symbol_tests.py +131 -0
  34. devkb/extractors/test_inventory.py +38 -0
  35. devkb/extractors/ticket_similarity.py +193 -0
  36. devkb/extractors/tickets_git.py +151 -0
  37. devkb/extractors/warnings.py +230 -0
  38. devkb/extractors/work_units.py +90 -0
  39. devkb/github.py +515 -0
  40. devkb/gitutil.py +16 -0
  41. devkb/index.py +51 -0
  42. devkb/init.py +178 -0
  43. devkb/mcp_server.py +216 -0
  44. devkb/model.py +221 -0
  45. devkb/module_file_bridge.py +57 -0
  46. devkb/ontology.md +178 -0
  47. devkb/packs.py +462 -0
  48. devkb/process_api.py +1601 -0
  49. devkb/project_context.py +191 -0
  50. devkb/query.py +251 -0
  51. devkb/service.py +324 -0
  52. devkb/startup.py +103 -0
  53. devkb/test_discovery.py +76 -0
  54. devkb/ts_symbols.py +291 -0
  55. devkb-0.2.0.dist-info/METADATA +162 -0
  56. devkb-0.2.0.dist-info/RECORD +61 -0
  57. devkb-0.2.0.dist-info/WHEEL +5 -0
  58. devkb-0.2.0.dist-info/entry_points.txt +2 -0
  59. devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
  60. devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
  61. devkb-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,220 @@
1
+ """File cross-reference + test-marker extractor (devkb Phase 2, ODSF-032).
2
+
3
+ Ports the two remaining halves of an earlier traceability script's
4
+ logic that Phase 1 (tickets_git.py) didn't already cover: `_git_files_changed`
5
+ (commit -> files touched) and `_find_test_markers` (which test files/functions
6
+ mention a ticket), generalized off OCO's hardcoded `OCO-` prefix strip to this
7
+ project's own ctx.ticket_prefix.
8
+
9
+ Takes the already-built tickets_git graph as an argument instead of round-
10
+ tripping through a previous .devkb/edges.jsonl build on disk (the roadmap's
11
+ literal "input: .devkb/edges.jsonl's REFERENCES" wording) -- same information
12
+ (which commits reference which ticket), but no ordering/staleness risk on a
13
+ project's very first build, when no prior .devkb/ store exists yet.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ from pathlib import Path
19
+
20
+ from devkb.extractors.tickets_git import _run
21
+ from devkb.test_discovery import list_test_files
22
+ from devkb.model import DevKBGraph
23
+ from devkb.project_context import ProjectContext
24
+
25
+ _TEST_FUNC_RE = re.compile(r"^\s*def (test_\w+)\s*\(")
26
+
27
+ # Ticket-mediated File->TestCase fan-out is inherently coarse (every file a
28
+ # ticket's commits touched gets linked to every test marker that ticket has),
29
+ # so a ticket with an unusually large touched-file set produces a
30
+ # disproportionate number of low-signal edges. Confirmed live against OCO:
31
+ # one ticket's commit list included a mass file-move (~650 files after fixing
32
+ # the git --grep anchoring bug below, ~20k before) -- past a certain size the
33
+ # "touched by this ticket" relationship stops being a meaningful coverage
34
+ # signal regardless of anchoring. p90 of OCO's own real per-ticket
35
+ # touched-file counts is 28; 50 keeps the vast majority of normal tickets
36
+ # while excluding the long tail.
37
+ _MAX_TOUCHED_FILES_FOR_TESTED_BY = 50
38
+
39
+ # ODSF-314. A commit touching more files than this is a bulk change (an archive rename, a mass reformat): in OCO 38 of
40
+ # 5,132 commits produced 40% of every TOUCHES edge, and "commit X touched file F" says nothing about F. Such a commit
41
+ # keeps its node, with `files_changed` and `bulk`, but emits no TOUCHES.
42
+ BULK_COMMIT_FILES = 200
43
+
44
+ # ODSF-314. Ticket-marker TESTED_BY ("a test names ticket T, so it covers every file T's commits touched") measured
45
+ # 0.7% precision on OCO (600 sampled edges; the test body called something in the file in 4). It is kept, as the
46
+ # owner chose, but only from files that hold code, under a lower cap, with a confidence that says what it is.
47
+ _MARKER_CAP = 20
48
+ _MARKER_CONFIDENCE = 0.2
49
+ _CODE_FILE = re.compile(r"\.(py|ts|tsx|js|jsx|mjs)$")
50
+ _TEST_PATH = re.compile(r"(^|/)(tests?/|test_[^/]*$|[^/]*_test\.py$|[^/]*\.(test|spec)\.[jt]sx?$|conftest\.py$)")
51
+
52
+
53
+ def is_bulk(files: list[str] | set[str]) -> bool:
54
+ return len(files) > BULK_COMMIT_FILES
55
+
56
+
57
+ def _files_touched(commit_hash: str, repo_root: Path) -> list[str]:
58
+ # --root: without it, diff-tree on a commit with no parent (the repo's
59
+ # very first commit) has nothing to diff against and silently returns no
60
+ # files -- a real gap in the traceability.py logic this ports (same
61
+ # command, minus --root), just never exercised there against a root
62
+ # commit. Safe to always pass: it only changes root-commit behavior,
63
+ # normal commits with a parent are unaffected.
64
+ out = _run(["git", "diff-tree", "--no-commit-id", "-r", "--root", "--name-only", commit_hash], repo_root)
65
+ return [f for f in out.splitlines() if f]
66
+
67
+
68
+ def _load_test_files(
69
+ tests_dir: Path, repo_root: Path, files: list[Path] | None = None,
70
+ ) -> list[tuple[str, str, list[str]]]:
71
+ """Reads every test file from disk exactly once -- (rel_path, full_text,
72
+ lines) per file. Callers scan this in-memory list per ticket instead of
73
+ re-reading disk per ticket (see _scan_test_files' own docstring for why
74
+ that distinction matters).
75
+
76
+ `files` (ODSF-304), when given, replaces the `tests_dir` walk: the caller
77
+ has already found every test file in the repo, wherever it lives."""
78
+ if files is None:
79
+ if not tests_dir.exists():
80
+ return []
81
+ files = sorted(tests_dir.rglob("*.py"))
82
+ out: list[tuple[str, str, list[str]]] = []
83
+ for py in files:
84
+ try:
85
+ content = py.read_text(errors="replace")
86
+ except OSError:
87
+ continue
88
+ out.append((str(py.relative_to(repo_root)), content, content.splitlines()))
89
+ return out
90
+
91
+
92
+ def _scan_test_files(
93
+ ticket: str, test_files: list[tuple[str, str, list[str]]],
94
+ ) -> list[tuple[str, int, str | None, str]]:
95
+ """Deliberately narrower than traceability.py::_find_test_markers, which
96
+ also matched `ticket.*<short-id>` and `mark.*<ticket>` (e.g. bare `100`
97
+ near the word "ticket"). That's fine for a human-read CLI report — a
98
+ reader visually discounts an obvious false positive — but wrong for a
99
+ persisted graph edge with no human filter: confirmed live against OCO,
100
+ those two loose patterns alone pushed "tickets with a test marker" past
101
+ 1,100 of ~1,265, most of it short-numeric-id noise (a 2-3 digit ticket
102
+ number is a common substring in test code for unrelated reasons — ports,
103
+ counts, HTTP statuses). The full anchored ticket id alone is still real,
104
+ substantial signal (OCO's own tests/ conventions actively encourage
105
+ `@pytest.mark.ticket("OCO-NNN")`/`test_OCO_NNN_*`), just without the
106
+ noise floor. \\b-anchored for the same reason as tickets_git.py's own
107
+ _git_commits fix: "OCO-100" must not match inside "OCO-1000".
108
+ Returns (path, line 1-indexed, enclosing test function name if found,
109
+ matched line text) per hit -- the enclosing-function walk is new here
110
+ (traceability.py's own CLI report didn't need a stable per-hit key;
111
+ devkb's TestCase natural_key does)."""
112
+ pattern = re.compile(rf"\b{re.escape(ticket)}\b")
113
+ hits: list[tuple[str, int, str | None, str]] = []
114
+ for path, content, lines in test_files:
115
+ if not pattern.search(content):
116
+ continue
117
+ for i, line in enumerate(lines):
118
+ if not pattern.search(line):
119
+ continue
120
+ func = None
121
+ for j in range(i, -1, -1):
122
+ m = _TEST_FUNC_RE.match(lines[j])
123
+ if m:
124
+ func = m.group(1)
125
+ break
126
+ hits.append((path, i + 1, func, line.strip()))
127
+ return hits
128
+
129
+
130
+ def _find_test_markers(
131
+ ticket: str, tests_dir: Path, repo_root: Path,
132
+ ) -> list[tuple[str, int, str | None, str]]:
133
+ """Single-ticket convenience wrapper over _load_test_files/_scan_test_files.
134
+ extract() itself below calls _load_test_files once and _scan_test_files
135
+ per ticket instead of this -- confirmed live that this straight port's
136
+ original per-ticket-rglob-and-read version never finished a build against
137
+ OCO (~1,300+ archived tickets x 184 test files = hundreds of thousands of
138
+ file reads) inside a 2-minute window."""
139
+ return _scan_test_files(ticket, _load_test_files(tests_dir, repo_root))
140
+
141
+
142
+ def extract(
143
+ ctx: ProjectContext, tickets_graph: DevKBGraph,
144
+ precomputed_touches: dict[str, list[str]] | None = None,
145
+ ) -> DevKBGraph:
146
+ """precomputed_touches (commit hash -> touched file paths), when given,
147
+ lets a caller that already ran `git diff-tree` for a commit (BL-1's
148
+ source_baseline.py, which walks ALL commits, not just ticket-referenced
149
+ ones -- its own commit set is normally a superset of this function's)
150
+ skip paying that subprocess cost twice for the same commit. Correctness
151
+ is unaffected either way -- DevKBGraph dedupes -- this is purely a
152
+ performance path."""
153
+ g = DevKBGraph()
154
+ precomputed_touches = precomputed_touches or {}
155
+
156
+ seen_commits: set[str] = set()
157
+ for commit in tickets_graph.nodes("Commit"):
158
+ h = commit.natural_key
159
+ if h in seen_commits:
160
+ continue
161
+ seen_commits.add(h)
162
+ touched = precomputed_touches.get(h)
163
+ if touched is None:
164
+ touched = _files_touched(h, ctx.repo_root)
165
+ if is_bulk(touched):
166
+ g.add_node("Commit", h, source="git-diff-tree", files_changed=len(touched), bulk=True)
167
+ continue
168
+ for path in touched:
169
+ g.add_node("File", path, source="git-diff-tree")
170
+ g.add_edge(
171
+ "TOUCHES", "Commit", h, "File", path,
172
+ source="git-diff-tree", confidence=1.0,
173
+ evidence_ref=f"git diff-tree --name-only {h[:10]}",
174
+ )
175
+
176
+ # Ticket -> its touched files, precomputed once so the per-marker loop
177
+ # below doesn't re-walk REFERENCES/TOUCHES edges per hit.
178
+ files_by_ticket: dict[str, set[str]] = {}
179
+ commit_to_ticket = {e.src: e.dst for e in tickets_graph.edges("REFERENCES")}
180
+ for edge in g.edges("TOUCHES"):
181
+ ticket = commit_to_ticket.get(edge.src)
182
+ if ticket:
183
+ files_by_ticket.setdefault(ticket, set()).add(edge.dst)
184
+
185
+ # ODSF-304: every tracked python test file, not only those under tests/.
186
+ test_files = _load_test_files(
187
+ ctx.tests_dir, ctx.repo_root,
188
+ files=[ctx.repo_root / rel for rel in list_test_files(ctx.repo_root, ctx.tests_dir) if rel.endswith(".py")],
189
+ )
190
+ for ticket_node in tickets_graph.nodes("Ticket"):
191
+ ticket = ticket_node.natural_key
192
+ touched_files = files_by_ticket.get(ticket, set())
193
+ # Past _MAX_TOUCHED_FILES_FOR_TESTED_BY, "this ticket's commits
194
+ # touched file X" stops being a meaningful per-file signal (a
195
+ # mass-rename/move ticket touches everything, not "everything this
196
+ # ticket is about") -- the TestCase node below still gets created
197
+ # either way, only the fan-out edges are skipped for an oversized
198
+ # ticket, so the marker itself is never silently dropped.
199
+ code_files = {f for f in touched_files if _CODE_FILE.search(f) and not _TEST_PATH.search(f)}
200
+ link_files = code_files if len(code_files) <= _MARKER_CAP else set()
201
+ for path, line, func, snippet in _scan_test_files(ticket, test_files):
202
+ test_key = f"{path}:{line}"
203
+ g.add_node(
204
+ "TestCase", test_key, source="tests",
205
+ file=path, line=line, function=func, ticket=ticket, snippet=snippet,
206
+ )
207
+ # File -> TestCase is ticket-mediated (same coarse grain
208
+ # traceability.py's own report used: a test that mentions the
209
+ # ticket "covers" every file that ticket's commits touched, not
210
+ # a content-level coverage analysis) -- includes the test file
211
+ # itself if a commit also touched it, which is correct, not
212
+ # redundant: that commit really did change the test.
213
+ for file_path in link_files:
214
+ g.add_edge(
215
+ "TESTED_BY", "File", file_path, "TestCase", test_key,
216
+ source="test-marker-grep", confidence=_MARKER_CONFIDENCE,
217
+ evidence_ref=f"{path}:{line} references {ticket}", extra={"kind": "ticket-marker"},
218
+ )
219
+
220
+ return g
@@ -0,0 +1,117 @@
1
+ """Flow + STEP: fully automated touch-point BFS (devkb BL-12,
2
+ KNOWLEDGE_SPEC.md Section 4/4a/5, resolution of Open question 3).
3
+
4
+ The original draft made `Flow` a curated node type (a human or an
5
+ LLM-assisted job proposes an ordered chain, a human confirms). Dropped per
6
+ direct instruction: fully automated, no human/LLM step, same rigor as
7
+ `CALLS`/`DEFINED_IN`. A touch point (Section 4a -- a graph location
8
+ entered from OUTSIDE the call graph itself) seeds a mechanical BFS over
9
+ real `CALLS`/`CALLS_API` edges; the result is stored, not curated.
10
+
11
+ Runs deliberately LAST in the build order: needs BL-7's `Endpoint` nodes
12
+ (one touch-point kind) and the full cross-file `CALLS`/`CALLS_API` graph
13
+ already merged, or a BFS over an incomplete call graph would produce
14
+ truncated/wrong flows. Post-merge, same shape as BL-4/BL-6/BL-11.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ from collections import deque
19
+
20
+ from devkb.model import DevKBGraph
21
+ from devkb.project_context import ProjectContext
22
+
23
+ # Allow-list, extensible -- same discipline as every other allow-list in
24
+ # this codebase (_HTTP_METHODS, the logger-level list, etc.).
25
+ _SCHEDULER_DECORATOR_PATTERNS = ("celery_app.task", "shared_task", "scheduler.scheduled_job")
26
+ _MAX_BFS_DEPTH = 50 # engineering safety valve, not a design choice
27
+
28
+
29
+ def _is_ui_handler_name(short_name: str) -> bool:
30
+ if short_name.startswith("on") and len(short_name) > 2 and short_name[2].isupper():
31
+ return True
32
+ return short_name.startswith("handle") and len(short_name) > 6 and short_name[6].isupper()
33
+
34
+
35
+ def _detect_touch_points(graph: DevKBGraph) -> list[tuple[str, str]]:
36
+ """[(touch_point_kind, start_symbol_natural_key), ...] -- an Endpoint
37
+ touch point resolves immediately to its own owner_symbol here, so
38
+ every touch point downstream is uniformly a Symbol to BFS from."""
39
+ touch_points: list[tuple[str, str]] = []
40
+
41
+ for ep in graph.nodes("Endpoint"):
42
+ owner = ep.props.get("owner_symbol")
43
+ if owner: # excludes BL-10's wildcard middleware markers (method="*", no owner)
44
+ touch_points.append(("endpoint", owner))
45
+
46
+ called_targets = {e.dst for e in graph.edges("CALLS")} | {e.dst for e in graph.edges("CALLS_API")}
47
+
48
+ for sym in graph.nodes("Symbol"):
49
+ short = sym.natural_key.rsplit("::", 1)[-1].rsplit(".", 1)[-1]
50
+
51
+ if (sym.source == "ast-ts" and _is_ui_handler_name(short)
52
+ and sym.natural_key not in called_targets):
53
+ touch_points.append(("ui_handler", sym.natural_key))
54
+
55
+ decorators = sym.props.get("decorators") or []
56
+ if any(pat in d for d in decorators for pat in _SCHEDULER_DECORATOR_PATTERNS):
57
+ touch_points.append(("scheduled", sym.natural_key))
58
+
59
+ if sym.props.get("is_daemon_entry"):
60
+ touch_points.append(("daemon_entry", sym.natural_key))
61
+
62
+ return touch_points
63
+
64
+
65
+ def _bfs(start_symbol: str, adjacency: dict[str, list[tuple[str, float]]]) -> list[tuple[str, int, float]]:
66
+ """[(natural_key, sequence, weakest_link_confidence), ...] in BFS visit
67
+ order, NOT including start_symbol itself (the Flow node's own STEP 0
68
+ already represents the touch point). Cycle-guarded via `visited`,
69
+ depth-capped as a safety valve against runaway/cyclic call graphs."""
70
+ visited = {start_symbol}
71
+ queue: deque[tuple[str, float, int]] = deque([(start_symbol, 1.0, 0)])
72
+ steps: list[tuple[str, int, float]] = []
73
+ seq = 0
74
+ while queue:
75
+ current, path_confidence, depth = queue.popleft()
76
+ if depth >= _MAX_BFS_DEPTH:
77
+ continue
78
+ for neighbor, edge_confidence in adjacency.get(current, []):
79
+ if neighbor in visited:
80
+ continue
81
+ visited.add(neighbor)
82
+ weakest = min(path_confidence, edge_confidence)
83
+ seq += 1
84
+ steps.append((neighbor, seq, weakest))
85
+ queue.append((neighbor, weakest, depth + 1))
86
+ return steps
87
+
88
+
89
+ def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
90
+ g = DevKBGraph()
91
+ touch_points = _detect_touch_points(graph)
92
+ if not touch_points:
93
+ return g
94
+
95
+ adjacency: dict[str, list[tuple[str, float]]] = {}
96
+ for verb in ("CALLS", "CALLS_API"):
97
+ for e in graph.edges(verb):
98
+ adjacency.setdefault(e.src, []).append((e.dst, e.confidence))
99
+
100
+ for kind, start_symbol in touch_points:
101
+ steps = _bfs(start_symbol, adjacency)
102
+ flow_key = f"flow:{kind}:{start_symbol}"
103
+ g.add_node("Flow", flow_key, source="touch-point-bfs", touch_point_kind=kind, step_count=len(steps) + 1)
104
+
105
+ g.add_edge(
106
+ "STEP", "Flow", flow_key, "Symbol", start_symbol,
107
+ source="touch-point-bfs", confidence=1.0,
108
+ evidence_ref=f"{kind} touch point", extra={"sequence": 0},
109
+ )
110
+ for target_key, seq, confidence in steps:
111
+ g.add_edge(
112
+ "STEP", "Flow", flow_key, "Symbol", target_key,
113
+ source="touch-point-bfs", confidence=confidence,
114
+ evidence_ref=f"reached from {kind} touch point {start_symbol} via CALLS/CALLS_API",
115
+ extra={"sequence": seq},
116
+ )
117
+ return g
@@ -0,0 +1,258 @@
1
+ """Requirement + Evidence, CHILD_OF / DERIVED_FROM / IMPLEMENTS / EVIDENCED_BY
2
+ (devkb BL-14, ADR-SDP-DEVKB-002, ODSF-230).
3
+
4
+ Links the renovated lifecycle into the graph, so "which stories implement this
5
+ spec" and "what evidence supports this" are graph questions:
6
+
7
+ - **Evidence** nodes from every evidence pack under `.onboarding/evidence/<EVP>/`:
8
+ one per `ev_` record in `manifest.json` `evidence[]` (REA's ids -- ODSF never
9
+ computes one) and one per `of_` record in `odsf-findings.jsonl`. `confidence`
10
+ is copied exactly as recorded and never upgraded. Only ids, confidence and a
11
+ short label land here: the pack is the record, not the graph (ADR §Consequences).
12
+ - a ticket's `## Trace` rows: `Requirement` -> a `Requirement` node + `DERIVED_FROM`;
13
+ `Epic` -> `CHILD_OF` (a STORY to an EPIC only); `Specification` -> `IMPLEMENTS`, resolved to a Spec node
14
+ by its path or its `spec_id`.
15
+ - an `ev_`/`of_` id named in a ticket or a Spec file -> `EVIDENCED_BY`.
16
+
17
+ Every edge is added only when both ends exist in the graph. A reference that
18
+ cannot be resolved is reported on stderr at build and never guessed -- the same
19
+ rule as `DOCUMENTS`. Deterministic and regex-derived, no LLM call. Post-merge:
20
+ needs the merged graph's Ticket nodes and `spec_docs`' Spec nodes, so it runs
21
+ after both (`devkb/build.py`).
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import re
27
+ import sys
28
+ from pathlib import Path
29
+ from typing import Any
30
+
31
+ from devkb.extractors.tickets_git import _iter_ticket_files, _parse_metadata
32
+ from devkb.model import DevKBGraph
33
+ from devkb.project_context import ProjectContext
34
+
35
+ EVIDENCE_DIR = Path(".onboarding") / "evidence"
36
+
37
+ _EV_ID_RE = re.compile(r"^ev_[0-9a-f]{64}$")
38
+ _OF_ID_RE = re.compile(r"^of_[0-9a-f]{64}$")
39
+ _CITED_ID_RE = re.compile(r"(?<![0-9A-Za-z_])(?:ev|of)_[0-9a-f]{64}(?![0-9A-Za-z_])")
40
+ _REQ_RE = re.compile(r"\bREQ-\d+\b")
41
+ _TRACE_HEADER_RE = re.compile(r"^##\s+Trace\s*$", re.MULTILINE)
42
+ _NEXT_SECTION_RE = re.compile(r"^##\s", re.MULTILINE)
43
+ _TRACE_ROW_RE = re.compile(r"^\|\s*(Requirement|Specification|Epic)\s*\|\s*(.*?)\s*\|\s*$", re.MULTILINE)
44
+ # A Trace row nobody has filled in yet says so; that is not an unresolved reference.
45
+ _PLACEHOLDERS = frozenset({"", "-", "—", "–", "tbd", "n/a", "na", "none"})
46
+ _LABEL_MAX = 120
47
+
48
+
49
+ def _trace_rows(text: str) -> list[tuple[str, str]]:
50
+ """`(link, value)` for the Requirement/Specification/Epic rows of `## Trace`."""
51
+ m = _TRACE_HEADER_RE.search(text)
52
+ if not m:
53
+ return []
54
+ rest = text[m.end():]
55
+ nxt = _NEXT_SECTION_RE.search(rest)
56
+ section = rest[:nxt.start()] if nxt else rest
57
+ return [(link, value) for link, value in _TRACE_ROW_RE.findall(section)]
58
+
59
+
60
+ def _label(*parts: Any) -> str:
61
+ text = " ".join(str(p) for p in parts if p)
62
+ return text[:_LABEL_MAX]
63
+
64
+
65
+ def _subject(rec: dict[str, Any]) -> dict[str, Any]:
66
+ subject = rec.get("subject")
67
+ return subject if isinstance(subject, dict) else {}
68
+
69
+
70
+ def _read_pack(pack_dir: Path) -> tuple[str, list[dict[str, str]]] | str:
71
+ """`(pack_id, records)` for a well-formed pack, else the reason it is not.
72
+
73
+ All-or-nothing: one malformed record skips the whole pack, so a pack never
74
+ lands half-indexed and looking complete."""
75
+ try:
76
+ manifest = json.loads((pack_dir / "manifest.json").read_text(encoding="utf-8"))
77
+ except (OSError, ValueError) as exc:
78
+ return f"manifest.json unreadable ({exc.__class__.__name__})"
79
+ if not isinstance(manifest, dict) or not isinstance(manifest.get("evidence", []), list):
80
+ return "manifest.json is not a pack manifest"
81
+ pack_id = manifest.get("id")
82
+ if not isinstance(pack_id, str) or not pack_id:
83
+ return "manifest.json has no id"
84
+
85
+ records: list[dict[str, str]] = []
86
+ for i, rec in enumerate(manifest.get("evidence", [])):
87
+ if not isinstance(rec, dict) or not _EV_ID_RE.match(str(rec.get("evidence_id", ""))) \
88
+ or not isinstance(rec.get("confidence"), str):
89
+ return f"manifest.json evidence[{i}] has no valid ev_ id and confidence"
90
+ subject = _subject(rec)
91
+ records.append({
92
+ "id": rec["evidence_id"], "origin": "rea", "confidence": rec["confidence"],
93
+ "label": _label(rec.get("operation"), subject.get("alias") or subject.get("name")),
94
+ })
95
+
96
+ findings = pack_dir / "odsf-findings.jsonl"
97
+ if findings.exists():
98
+ try:
99
+ lines = findings.read_text(encoding="utf-8").splitlines()
100
+ except OSError as exc:
101
+ return f"odsf-findings.jsonl unreadable ({exc.__class__.__name__})"
102
+ for n, line in enumerate(lines, 1):
103
+ if not line.strip():
104
+ continue
105
+ try:
106
+ rec = json.loads(line)
107
+ except ValueError:
108
+ return f"odsf-findings.jsonl line {n} is not JSON"
109
+ if not isinstance(rec, dict) or not _OF_ID_RE.match(str(rec.get("id", ""))) \
110
+ or not isinstance(rec.get("confidence"), str):
111
+ return f"odsf-findings.jsonl line {n} has no valid of_ id and confidence"
112
+ subject = _subject(rec)
113
+ records.append({
114
+ "id": rec["id"], "origin": "odsf", "confidence": rec["confidence"],
115
+ "label": _label(rec.get("kind"), subject.get("name") or subject.get("local_path")),
116
+ })
117
+ return pack_id, records
118
+
119
+
120
+ def _add_evidence(ctx: ProjectContext, g: DevKBGraph, report: list[str]) -> set[str]:
121
+ root = ctx.repo_root / EVIDENCE_DIR
122
+ known: dict[str, str] = {} # id -> confidence as first recorded
123
+ if not root.is_dir():
124
+ return set()
125
+ for pack_dir in sorted(p for p in root.iterdir() if p.is_dir()):
126
+ rel = str(pack_dir.relative_to(ctx.repo_root))
127
+ result = _read_pack(pack_dir)
128
+ if isinstance(result, str):
129
+ report.append(f"{rel}: evidence pack skipped -- {result}")
130
+ continue
131
+ pack_id, records = result
132
+ for rec in records:
133
+ if rec["id"] in known:
134
+ # first pack (sorted) to record an id owns it; never re-labelled
135
+ if known[rec["id"]] != rec["confidence"]:
136
+ report.append(f"{rel}: {rec['id'][:15]}… recorded as {rec['confidence']!r}, already "
137
+ f"{known[rec['id']]!r} in an earlier pack -- earlier record kept")
138
+ continue
139
+ known[rec["id"]] = rec["confidence"]
140
+ g.add_node(
141
+ "Evidence", rec["id"], source="evidence-pack",
142
+ confidence=rec["confidence"], origin=rec["origin"],
143
+ pack=pack_id, pack_dir=rel, label=rec["label"] or rec["id"][:15],
144
+ )
145
+ return set(known)
146
+
147
+
148
+ def _spec_lookup(graph: DevKBGraph) -> tuple[set[str], dict[str, list[str]]]:
149
+ paths: set[str] = set()
150
+ by_id: dict[str, list[str]] = {}
151
+ for spec in graph.nodes("Spec"):
152
+ paths.add(spec.natural_key)
153
+ spec_id = spec.props.get("spec_id")
154
+ if spec_id:
155
+ by_id.setdefault(spec_id, []).append(spec.natural_key)
156
+ return paths, {k: sorted(v) for k, v in by_id.items()}
157
+
158
+
159
+ def _resolve_specs(value: str, paths: set[str], by_id: dict[str, list[str]],
160
+ where: str, report: list[str]) -> list[str]:
161
+ out: list[str] = []
162
+ for token in re.split(r"[,\s]+", value.replace("`", "")):
163
+ if token.lower() in _PLACEHOLDERS:
164
+ continue
165
+ if token in paths:
166
+ out.append(token)
167
+ continue
168
+ matches = by_id.get(token, [])
169
+ if len(matches) == 1:
170
+ out.append(matches[0])
171
+ elif matches:
172
+ report.append(f"{where}: Specification {token} is ambiguous ({', '.join(matches)}) -- not linked")
173
+ else:
174
+ report.append(f"{where}: Specification {token} is not a Spec in the graph -- not linked")
175
+ return out
176
+
177
+
178
+ def _cite_evidence(g: DevKBGraph, src_type: str, src: str, text: str, where: str,
179
+ known: set[str], report: list[str]) -> None:
180
+ for ev_id in sorted(set(_CITED_ID_RE.findall(text))):
181
+ if ev_id not in known:
182
+ report.append(f"{where}: cites {ev_id[:15]}… which is in no evidence pack -- not linked")
183
+ continue
184
+ g.add_edge(
185
+ "EVIDENCED_BY", src_type, src, "Evidence", ev_id,
186
+ source="evidence-cite", confidence=1.0, evidence_ref=f"{where} cites {ev_id}",
187
+ )
188
+
189
+
190
+ def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
191
+ g = DevKBGraph()
192
+ report: list[str] = []
193
+ real_tickets = {t.natural_key: t.props.get("ticket_type") for t in graph.nodes("Ticket")}
194
+ epic_re = re.compile(rf"\b{re.escape(ctx.ticket_prefix)}-\d+\b")
195
+ known_evidence = _add_evidence(ctx, g, report)
196
+ spec_paths, spec_by_id = _spec_lookup(graph)
197
+
198
+ for path in _iter_ticket_files(ctx):
199
+ try:
200
+ text = path.read_text(errors="replace")
201
+ except OSError:
202
+ continue
203
+ ticket = _parse_metadata(text, path)["ticket"]
204
+ if ticket not in real_tickets:
205
+ continue
206
+ where = str(path.relative_to(ctx.repo_root))
207
+ trace_ref = f"{where}#Trace"
208
+
209
+ for link, value in _trace_rows(text):
210
+ if value.strip().lower() in _PLACEHOLDERS:
211
+ continue
212
+ if link == "Requirement":
213
+ reqs = _REQ_RE.findall(value)
214
+ if not reqs:
215
+ report.append(f"{where}: Requirement {value!r} names no REQ-NNN -- not linked")
216
+ for req in reqs:
217
+ g.add_node("Requirement", req, source="ticket-trace")
218
+ g.add_edge("DERIVED_FROM", "Ticket", ticket, "Requirement", req,
219
+ source="ticket-trace", confidence=1.0, evidence_ref=trace_ref)
220
+ elif link == "Epic":
221
+ epics = epic_re.findall(value)
222
+ if not epics:
223
+ report.append(f"{where}: Epic {value!r} names no {ctx.ticket_prefix}-NNN -- not linked")
224
+ for epic in epics:
225
+ if epic == ticket:
226
+ report.append(f"{where}: Epic {epic} is the ticket itself -- not linked")
227
+ continue
228
+ if epic not in real_tickets:
229
+ report.append(f"{where}: Epic {epic} is not a Ticket in the graph -- not linked")
230
+ continue
231
+ # ADR-SDP-DEVKB-002 §3: CHILD_OF is Story -> Epic, nothing else.
232
+ if real_tickets[ticket] != "STORY" or real_tickets[epic] != "EPIC":
233
+ report.append(f"{where}: Epic row links {real_tickets[ticket]} {ticket} to "
234
+ f"{real_tickets[epic]} {epic}; CHILD_OF is STORY -> EPIC -- not linked")
235
+ continue
236
+ g.add_edge("CHILD_OF", "Ticket", ticket, "Ticket", epic,
237
+ source="ticket-trace", confidence=1.0, evidence_ref=trace_ref)
238
+ else: # Specification
239
+ for spec in _resolve_specs(value, spec_paths, spec_by_id, where, report):
240
+ g.add_edge("IMPLEMENTS", "Ticket", ticket, "Spec", spec,
241
+ source="ticket-trace", confidence=1.0, evidence_ref=trace_ref)
242
+
243
+ _cite_evidence(g, "Ticket", ticket, text, where, known_evidence, report)
244
+
245
+ for spec_path in sorted(spec_paths):
246
+ try:
247
+ text = (ctx.repo_root / spec_path).read_text(errors="replace")
248
+ except OSError:
249
+ continue
250
+ _cite_evidence(g, "Spec", spec_path, text, spec_path, known_evidence, report)
251
+
252
+ if report:
253
+ lines = sorted(set(report))
254
+ print(f"devkb build: lifecycle: {len(lines)} unresolved reference(s), none guessed:",
255
+ file=sys.stderr)
256
+ for line in lines:
257
+ print(f" {line}", file=sys.stderr)
258
+ return g