devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""File cross-reference + test-marker extractor (devkb Phase 2, ODSF-032).
|
|
2
|
+
|
|
3
|
+
Ports the two remaining halves of an earlier traceability script's
|
|
4
|
+
logic that Phase 1 (tickets_git.py) didn't already cover: `_git_files_changed`
|
|
5
|
+
(commit -> files touched) and `_find_test_markers` (which test files/functions
|
|
6
|
+
mention a ticket), generalized off OCO's hardcoded `OCO-` prefix strip to this
|
|
7
|
+
project's own ctx.ticket_prefix.
|
|
8
|
+
|
|
9
|
+
Takes the already-built tickets_git graph as an argument instead of round-
|
|
10
|
+
tripping through a previous .devkb/edges.jsonl build on disk (the roadmap's
|
|
11
|
+
literal "input: .devkb/edges.jsonl's REFERENCES" wording) -- same information
|
|
12
|
+
(which commits reference which ticket), but no ordering/staleness risk on a
|
|
13
|
+
project's very first build, when no prior .devkb/ store exists yet.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from devkb.extractors.tickets_git import _run
|
|
21
|
+
from devkb.test_discovery import list_test_files
|
|
22
|
+
from devkb.model import DevKBGraph
|
|
23
|
+
from devkb.project_context import ProjectContext
|
|
24
|
+
|
|
25
|
+
_TEST_FUNC_RE = re.compile(r"^\s*def (test_\w+)\s*\(")
|
|
26
|
+
|
|
27
|
+
# Ticket-mediated File->TestCase fan-out is inherently coarse (every file a
|
|
28
|
+
# ticket's commits touched gets linked to every test marker that ticket has),
|
|
29
|
+
# so a ticket with an unusually large touched-file set produces a
|
|
30
|
+
# disproportionate number of low-signal edges. Confirmed live against OCO:
|
|
31
|
+
# one ticket's commit list included a mass file-move (~650 files after fixing
|
|
32
|
+
# the git --grep anchoring bug below, ~20k before) -- past a certain size the
|
|
33
|
+
# "touched by this ticket" relationship stops being a meaningful coverage
|
|
34
|
+
# signal regardless of anchoring. p90 of OCO's own real per-ticket
|
|
35
|
+
# touched-file counts is 28; 50 keeps the vast majority of normal tickets
|
|
36
|
+
# while excluding the long tail.
|
|
37
|
+
_MAX_TOUCHED_FILES_FOR_TESTED_BY = 50
|
|
38
|
+
|
|
39
|
+
# ODSF-314. A commit touching more files than this is a bulk change (an archive rename, a mass reformat): in OCO 38 of
|
|
40
|
+
# 5,132 commits produced 40% of every TOUCHES edge, and "commit X touched file F" says nothing about F. Such a commit
|
|
41
|
+
# keeps its node, with `files_changed` and `bulk`, but emits no TOUCHES.
|
|
42
|
+
BULK_COMMIT_FILES = 200
|
|
43
|
+
|
|
44
|
+
# ODSF-314. Ticket-marker TESTED_BY ("a test names ticket T, so it covers every file T's commits touched") measured
|
|
45
|
+
# 0.7% precision on OCO (600 sampled edges; the test body called something in the file in 4). It is kept, as the
|
|
46
|
+
# owner chose, but only from files that hold code, under a lower cap, with a confidence that says what it is.
|
|
47
|
+
_MARKER_CAP = 20
|
|
48
|
+
_MARKER_CONFIDENCE = 0.2
|
|
49
|
+
_CODE_FILE = re.compile(r"\.(py|ts|tsx|js|jsx|mjs)$")
|
|
50
|
+
_TEST_PATH = re.compile(r"(^|/)(tests?/|test_[^/]*$|[^/]*_test\.py$|[^/]*\.(test|spec)\.[jt]sx?$|conftest\.py$)")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def is_bulk(files: list[str] | set[str]) -> bool:
|
|
54
|
+
return len(files) > BULK_COMMIT_FILES
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _files_touched(commit_hash: str, repo_root: Path) -> list[str]:
|
|
58
|
+
# --root: without it, diff-tree on a commit with no parent (the repo's
|
|
59
|
+
# very first commit) has nothing to diff against and silently returns no
|
|
60
|
+
# files -- a real gap in the traceability.py logic this ports (same
|
|
61
|
+
# command, minus --root), just never exercised there against a root
|
|
62
|
+
# commit. Safe to always pass: it only changes root-commit behavior,
|
|
63
|
+
# normal commits with a parent are unaffected.
|
|
64
|
+
out = _run(["git", "diff-tree", "--no-commit-id", "-r", "--root", "--name-only", commit_hash], repo_root)
|
|
65
|
+
return [f for f in out.splitlines() if f]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _load_test_files(
|
|
69
|
+
tests_dir: Path, repo_root: Path, files: list[Path] | None = None,
|
|
70
|
+
) -> list[tuple[str, str, list[str]]]:
|
|
71
|
+
"""Reads every test file from disk exactly once -- (rel_path, full_text,
|
|
72
|
+
lines) per file. Callers scan this in-memory list per ticket instead of
|
|
73
|
+
re-reading disk per ticket (see _scan_test_files' own docstring for why
|
|
74
|
+
that distinction matters).
|
|
75
|
+
|
|
76
|
+
`files` (ODSF-304), when given, replaces the `tests_dir` walk: the caller
|
|
77
|
+
has already found every test file in the repo, wherever it lives."""
|
|
78
|
+
if files is None:
|
|
79
|
+
if not tests_dir.exists():
|
|
80
|
+
return []
|
|
81
|
+
files = sorted(tests_dir.rglob("*.py"))
|
|
82
|
+
out: list[tuple[str, str, list[str]]] = []
|
|
83
|
+
for py in files:
|
|
84
|
+
try:
|
|
85
|
+
content = py.read_text(errors="replace")
|
|
86
|
+
except OSError:
|
|
87
|
+
continue
|
|
88
|
+
out.append((str(py.relative_to(repo_root)), content, content.splitlines()))
|
|
89
|
+
return out
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _scan_test_files(
|
|
93
|
+
ticket: str, test_files: list[tuple[str, str, list[str]]],
|
|
94
|
+
) -> list[tuple[str, int, str | None, str]]:
|
|
95
|
+
"""Deliberately narrower than traceability.py::_find_test_markers, which
|
|
96
|
+
also matched `ticket.*<short-id>` and `mark.*<ticket>` (e.g. bare `100`
|
|
97
|
+
near the word "ticket"). That's fine for a human-read CLI report — a
|
|
98
|
+
reader visually discounts an obvious false positive — but wrong for a
|
|
99
|
+
persisted graph edge with no human filter: confirmed live against OCO,
|
|
100
|
+
those two loose patterns alone pushed "tickets with a test marker" past
|
|
101
|
+
1,100 of ~1,265, most of it short-numeric-id noise (a 2-3 digit ticket
|
|
102
|
+
number is a common substring in test code for unrelated reasons — ports,
|
|
103
|
+
counts, HTTP statuses). The full anchored ticket id alone is still real,
|
|
104
|
+
substantial signal (OCO's own tests/ conventions actively encourage
|
|
105
|
+
`@pytest.mark.ticket("OCO-NNN")`/`test_OCO_NNN_*`), just without the
|
|
106
|
+
noise floor. \\b-anchored for the same reason as tickets_git.py's own
|
|
107
|
+
_git_commits fix: "OCO-100" must not match inside "OCO-1000".
|
|
108
|
+
Returns (path, line 1-indexed, enclosing test function name if found,
|
|
109
|
+
matched line text) per hit -- the enclosing-function walk is new here
|
|
110
|
+
(traceability.py's own CLI report didn't need a stable per-hit key;
|
|
111
|
+
devkb's TestCase natural_key does)."""
|
|
112
|
+
pattern = re.compile(rf"\b{re.escape(ticket)}\b")
|
|
113
|
+
hits: list[tuple[str, int, str | None, str]] = []
|
|
114
|
+
for path, content, lines in test_files:
|
|
115
|
+
if not pattern.search(content):
|
|
116
|
+
continue
|
|
117
|
+
for i, line in enumerate(lines):
|
|
118
|
+
if not pattern.search(line):
|
|
119
|
+
continue
|
|
120
|
+
func = None
|
|
121
|
+
for j in range(i, -1, -1):
|
|
122
|
+
m = _TEST_FUNC_RE.match(lines[j])
|
|
123
|
+
if m:
|
|
124
|
+
func = m.group(1)
|
|
125
|
+
break
|
|
126
|
+
hits.append((path, i + 1, func, line.strip()))
|
|
127
|
+
return hits
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _find_test_markers(
|
|
131
|
+
ticket: str, tests_dir: Path, repo_root: Path,
|
|
132
|
+
) -> list[tuple[str, int, str | None, str]]:
|
|
133
|
+
"""Single-ticket convenience wrapper over _load_test_files/_scan_test_files.
|
|
134
|
+
extract() itself below calls _load_test_files once and _scan_test_files
|
|
135
|
+
per ticket instead of this -- confirmed live that this straight port's
|
|
136
|
+
original per-ticket-rglob-and-read version never finished a build against
|
|
137
|
+
OCO (~1,300+ archived tickets x 184 test files = hundreds of thousands of
|
|
138
|
+
file reads) inside a 2-minute window."""
|
|
139
|
+
return _scan_test_files(ticket, _load_test_files(tests_dir, repo_root))
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def extract(
|
|
143
|
+
ctx: ProjectContext, tickets_graph: DevKBGraph,
|
|
144
|
+
precomputed_touches: dict[str, list[str]] | None = None,
|
|
145
|
+
) -> DevKBGraph:
|
|
146
|
+
"""precomputed_touches (commit hash -> touched file paths), when given,
|
|
147
|
+
lets a caller that already ran `git diff-tree` for a commit (BL-1's
|
|
148
|
+
source_baseline.py, which walks ALL commits, not just ticket-referenced
|
|
149
|
+
ones -- its own commit set is normally a superset of this function's)
|
|
150
|
+
skip paying that subprocess cost twice for the same commit. Correctness
|
|
151
|
+
is unaffected either way -- DevKBGraph dedupes -- this is purely a
|
|
152
|
+
performance path."""
|
|
153
|
+
g = DevKBGraph()
|
|
154
|
+
precomputed_touches = precomputed_touches or {}
|
|
155
|
+
|
|
156
|
+
seen_commits: set[str] = set()
|
|
157
|
+
for commit in tickets_graph.nodes("Commit"):
|
|
158
|
+
h = commit.natural_key
|
|
159
|
+
if h in seen_commits:
|
|
160
|
+
continue
|
|
161
|
+
seen_commits.add(h)
|
|
162
|
+
touched = precomputed_touches.get(h)
|
|
163
|
+
if touched is None:
|
|
164
|
+
touched = _files_touched(h, ctx.repo_root)
|
|
165
|
+
if is_bulk(touched):
|
|
166
|
+
g.add_node("Commit", h, source="git-diff-tree", files_changed=len(touched), bulk=True)
|
|
167
|
+
continue
|
|
168
|
+
for path in touched:
|
|
169
|
+
g.add_node("File", path, source="git-diff-tree")
|
|
170
|
+
g.add_edge(
|
|
171
|
+
"TOUCHES", "Commit", h, "File", path,
|
|
172
|
+
source="git-diff-tree", confidence=1.0,
|
|
173
|
+
evidence_ref=f"git diff-tree --name-only {h[:10]}",
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# Ticket -> its touched files, precomputed once so the per-marker loop
|
|
177
|
+
# below doesn't re-walk REFERENCES/TOUCHES edges per hit.
|
|
178
|
+
files_by_ticket: dict[str, set[str]] = {}
|
|
179
|
+
commit_to_ticket = {e.src: e.dst for e in tickets_graph.edges("REFERENCES")}
|
|
180
|
+
for edge in g.edges("TOUCHES"):
|
|
181
|
+
ticket = commit_to_ticket.get(edge.src)
|
|
182
|
+
if ticket:
|
|
183
|
+
files_by_ticket.setdefault(ticket, set()).add(edge.dst)
|
|
184
|
+
|
|
185
|
+
# ODSF-304: every tracked python test file, not only those under tests/.
|
|
186
|
+
test_files = _load_test_files(
|
|
187
|
+
ctx.tests_dir, ctx.repo_root,
|
|
188
|
+
files=[ctx.repo_root / rel for rel in list_test_files(ctx.repo_root, ctx.tests_dir) if rel.endswith(".py")],
|
|
189
|
+
)
|
|
190
|
+
for ticket_node in tickets_graph.nodes("Ticket"):
|
|
191
|
+
ticket = ticket_node.natural_key
|
|
192
|
+
touched_files = files_by_ticket.get(ticket, set())
|
|
193
|
+
# Past _MAX_TOUCHED_FILES_FOR_TESTED_BY, "this ticket's commits
|
|
194
|
+
# touched file X" stops being a meaningful per-file signal (a
|
|
195
|
+
# mass-rename/move ticket touches everything, not "everything this
|
|
196
|
+
# ticket is about") -- the TestCase node below still gets created
|
|
197
|
+
# either way, only the fan-out edges are skipped for an oversized
|
|
198
|
+
# ticket, so the marker itself is never silently dropped.
|
|
199
|
+
code_files = {f for f in touched_files if _CODE_FILE.search(f) and not _TEST_PATH.search(f)}
|
|
200
|
+
link_files = code_files if len(code_files) <= _MARKER_CAP else set()
|
|
201
|
+
for path, line, func, snippet in _scan_test_files(ticket, test_files):
|
|
202
|
+
test_key = f"{path}:{line}"
|
|
203
|
+
g.add_node(
|
|
204
|
+
"TestCase", test_key, source="tests",
|
|
205
|
+
file=path, line=line, function=func, ticket=ticket, snippet=snippet,
|
|
206
|
+
)
|
|
207
|
+
# File -> TestCase is ticket-mediated (same coarse grain
|
|
208
|
+
# traceability.py's own report used: a test that mentions the
|
|
209
|
+
# ticket "covers" every file that ticket's commits touched, not
|
|
210
|
+
# a content-level coverage analysis) -- includes the test file
|
|
211
|
+
# itself if a commit also touched it, which is correct, not
|
|
212
|
+
# redundant: that commit really did change the test.
|
|
213
|
+
for file_path in link_files:
|
|
214
|
+
g.add_edge(
|
|
215
|
+
"TESTED_BY", "File", file_path, "TestCase", test_key,
|
|
216
|
+
source="test-marker-grep", confidence=_MARKER_CONFIDENCE,
|
|
217
|
+
evidence_ref=f"{path}:{line} references {ticket}", extra={"kind": "ticket-marker"},
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
return g
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Flow + STEP: fully automated touch-point BFS (devkb BL-12,
|
|
2
|
+
KNOWLEDGE_SPEC.md Section 4/4a/5, resolution of Open question 3).
|
|
3
|
+
|
|
4
|
+
The original draft made `Flow` a curated node type (a human or an
|
|
5
|
+
LLM-assisted job proposes an ordered chain, a human confirms). Dropped per
|
|
6
|
+
direct instruction: fully automated, no human/LLM step, same rigor as
|
|
7
|
+
`CALLS`/`DEFINED_IN`. A touch point (Section 4a -- a graph location
|
|
8
|
+
entered from OUTSIDE the call graph itself) seeds a mechanical BFS over
|
|
9
|
+
real `CALLS`/`CALLS_API` edges; the result is stored, not curated.
|
|
10
|
+
|
|
11
|
+
Runs deliberately LAST in the build order: needs BL-7's `Endpoint` nodes
|
|
12
|
+
(one touch-point kind) and the full cross-file `CALLS`/`CALLS_API` graph
|
|
13
|
+
already merged, or a BFS over an incomplete call graph would produce
|
|
14
|
+
truncated/wrong flows. Post-merge, same shape as BL-4/BL-6/BL-11.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from collections import deque
|
|
19
|
+
|
|
20
|
+
from devkb.model import DevKBGraph
|
|
21
|
+
from devkb.project_context import ProjectContext
|
|
22
|
+
|
|
23
|
+
# Allow-list, extensible -- same discipline as every other allow-list in
|
|
24
|
+
# this codebase (_HTTP_METHODS, the logger-level list, etc.).
|
|
25
|
+
_SCHEDULER_DECORATOR_PATTERNS = ("celery_app.task", "shared_task", "scheduler.scheduled_job")
|
|
26
|
+
_MAX_BFS_DEPTH = 50 # engineering safety valve, not a design choice
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _is_ui_handler_name(short_name: str) -> bool:
|
|
30
|
+
if short_name.startswith("on") and len(short_name) > 2 and short_name[2].isupper():
|
|
31
|
+
return True
|
|
32
|
+
return short_name.startswith("handle") and len(short_name) > 6 and short_name[6].isupper()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _detect_touch_points(graph: DevKBGraph) -> list[tuple[str, str]]:
|
|
36
|
+
"""[(touch_point_kind, start_symbol_natural_key), ...] -- an Endpoint
|
|
37
|
+
touch point resolves immediately to its own owner_symbol here, so
|
|
38
|
+
every touch point downstream is uniformly a Symbol to BFS from."""
|
|
39
|
+
touch_points: list[tuple[str, str]] = []
|
|
40
|
+
|
|
41
|
+
for ep in graph.nodes("Endpoint"):
|
|
42
|
+
owner = ep.props.get("owner_symbol")
|
|
43
|
+
if owner: # excludes BL-10's wildcard middleware markers (method="*", no owner)
|
|
44
|
+
touch_points.append(("endpoint", owner))
|
|
45
|
+
|
|
46
|
+
called_targets = {e.dst for e in graph.edges("CALLS")} | {e.dst for e in graph.edges("CALLS_API")}
|
|
47
|
+
|
|
48
|
+
for sym in graph.nodes("Symbol"):
|
|
49
|
+
short = sym.natural_key.rsplit("::", 1)[-1].rsplit(".", 1)[-1]
|
|
50
|
+
|
|
51
|
+
if (sym.source == "ast-ts" and _is_ui_handler_name(short)
|
|
52
|
+
and sym.natural_key not in called_targets):
|
|
53
|
+
touch_points.append(("ui_handler", sym.natural_key))
|
|
54
|
+
|
|
55
|
+
decorators = sym.props.get("decorators") or []
|
|
56
|
+
if any(pat in d for d in decorators for pat in _SCHEDULER_DECORATOR_PATTERNS):
|
|
57
|
+
touch_points.append(("scheduled", sym.natural_key))
|
|
58
|
+
|
|
59
|
+
if sym.props.get("is_daemon_entry"):
|
|
60
|
+
touch_points.append(("daemon_entry", sym.natural_key))
|
|
61
|
+
|
|
62
|
+
return touch_points
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _bfs(start_symbol: str, adjacency: dict[str, list[tuple[str, float]]]) -> list[tuple[str, int, float]]:
|
|
66
|
+
"""[(natural_key, sequence, weakest_link_confidence), ...] in BFS visit
|
|
67
|
+
order, NOT including start_symbol itself (the Flow node's own STEP 0
|
|
68
|
+
already represents the touch point). Cycle-guarded via `visited`,
|
|
69
|
+
depth-capped as a safety valve against runaway/cyclic call graphs."""
|
|
70
|
+
visited = {start_symbol}
|
|
71
|
+
queue: deque[tuple[str, float, int]] = deque([(start_symbol, 1.0, 0)])
|
|
72
|
+
steps: list[tuple[str, int, float]] = []
|
|
73
|
+
seq = 0
|
|
74
|
+
while queue:
|
|
75
|
+
current, path_confidence, depth = queue.popleft()
|
|
76
|
+
if depth >= _MAX_BFS_DEPTH:
|
|
77
|
+
continue
|
|
78
|
+
for neighbor, edge_confidence in adjacency.get(current, []):
|
|
79
|
+
if neighbor in visited:
|
|
80
|
+
continue
|
|
81
|
+
visited.add(neighbor)
|
|
82
|
+
weakest = min(path_confidence, edge_confidence)
|
|
83
|
+
seq += 1
|
|
84
|
+
steps.append((neighbor, seq, weakest))
|
|
85
|
+
queue.append((neighbor, weakest, depth + 1))
|
|
86
|
+
return steps
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
|
|
90
|
+
g = DevKBGraph()
|
|
91
|
+
touch_points = _detect_touch_points(graph)
|
|
92
|
+
if not touch_points:
|
|
93
|
+
return g
|
|
94
|
+
|
|
95
|
+
adjacency: dict[str, list[tuple[str, float]]] = {}
|
|
96
|
+
for verb in ("CALLS", "CALLS_API"):
|
|
97
|
+
for e in graph.edges(verb):
|
|
98
|
+
adjacency.setdefault(e.src, []).append((e.dst, e.confidence))
|
|
99
|
+
|
|
100
|
+
for kind, start_symbol in touch_points:
|
|
101
|
+
steps = _bfs(start_symbol, adjacency)
|
|
102
|
+
flow_key = f"flow:{kind}:{start_symbol}"
|
|
103
|
+
g.add_node("Flow", flow_key, source="touch-point-bfs", touch_point_kind=kind, step_count=len(steps) + 1)
|
|
104
|
+
|
|
105
|
+
g.add_edge(
|
|
106
|
+
"STEP", "Flow", flow_key, "Symbol", start_symbol,
|
|
107
|
+
source="touch-point-bfs", confidence=1.0,
|
|
108
|
+
evidence_ref=f"{kind} touch point", extra={"sequence": 0},
|
|
109
|
+
)
|
|
110
|
+
for target_key, seq, confidence in steps:
|
|
111
|
+
g.add_edge(
|
|
112
|
+
"STEP", "Flow", flow_key, "Symbol", target_key,
|
|
113
|
+
source="touch-point-bfs", confidence=confidence,
|
|
114
|
+
evidence_ref=f"reached from {kind} touch point {start_symbol} via CALLS/CALLS_API",
|
|
115
|
+
extra={"sequence": seq},
|
|
116
|
+
)
|
|
117
|
+
return g
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Requirement + Evidence, CHILD_OF / DERIVED_FROM / IMPLEMENTS / EVIDENCED_BY
|
|
2
|
+
(devkb BL-14, ADR-SDP-DEVKB-002, ODSF-230).
|
|
3
|
+
|
|
4
|
+
Links the renovated lifecycle into the graph, so "which stories implement this
|
|
5
|
+
spec" and "what evidence supports this" are graph questions:
|
|
6
|
+
|
|
7
|
+
- **Evidence** nodes from every evidence pack under `.onboarding/evidence/<EVP>/`:
|
|
8
|
+
one per `ev_` record in `manifest.json` `evidence[]` (REA's ids -- ODSF never
|
|
9
|
+
computes one) and one per `of_` record in `odsf-findings.jsonl`. `confidence`
|
|
10
|
+
is copied exactly as recorded and never upgraded. Only ids, confidence and a
|
|
11
|
+
short label land here: the pack is the record, not the graph (ADR §Consequences).
|
|
12
|
+
- a ticket's `## Trace` rows: `Requirement` -> a `Requirement` node + `DERIVED_FROM`;
|
|
13
|
+
`Epic` -> `CHILD_OF` (a STORY to an EPIC only); `Specification` -> `IMPLEMENTS`, resolved to a Spec node
|
|
14
|
+
by its path or its `spec_id`.
|
|
15
|
+
- an `ev_`/`of_` id named in a ticket or a Spec file -> `EVIDENCED_BY`.
|
|
16
|
+
|
|
17
|
+
Every edge is added only when both ends exist in the graph. A reference that
|
|
18
|
+
cannot be resolved is reported on stderr at build and never guessed -- the same
|
|
19
|
+
rule as `DOCUMENTS`. Deterministic and regex-derived, no LLM call. Post-merge:
|
|
20
|
+
needs the merged graph's Ticket nodes and `spec_docs`' Spec nodes, so it runs
|
|
21
|
+
after both (`devkb/build.py`).
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import re
|
|
27
|
+
import sys
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
from devkb.extractors.tickets_git import _iter_ticket_files, _parse_metadata
|
|
32
|
+
from devkb.model import DevKBGraph
|
|
33
|
+
from devkb.project_context import ProjectContext
|
|
34
|
+
|
|
35
|
+
EVIDENCE_DIR = Path(".onboarding") / "evidence"
|
|
36
|
+
|
|
37
|
+
_EV_ID_RE = re.compile(r"^ev_[0-9a-f]{64}$")
|
|
38
|
+
_OF_ID_RE = re.compile(r"^of_[0-9a-f]{64}$")
|
|
39
|
+
_CITED_ID_RE = re.compile(r"(?<![0-9A-Za-z_])(?:ev|of)_[0-9a-f]{64}(?![0-9A-Za-z_])")
|
|
40
|
+
_REQ_RE = re.compile(r"\bREQ-\d+\b")
|
|
41
|
+
_TRACE_HEADER_RE = re.compile(r"^##\s+Trace\s*$", re.MULTILINE)
|
|
42
|
+
_NEXT_SECTION_RE = re.compile(r"^##\s", re.MULTILINE)
|
|
43
|
+
_TRACE_ROW_RE = re.compile(r"^\|\s*(Requirement|Specification|Epic)\s*\|\s*(.*?)\s*\|\s*$", re.MULTILINE)
|
|
44
|
+
# A Trace row nobody has filled in yet says so; that is not an unresolved reference.
|
|
45
|
+
_PLACEHOLDERS = frozenset({"", "-", "—", "–", "tbd", "n/a", "na", "none"})
|
|
46
|
+
_LABEL_MAX = 120
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _trace_rows(text: str) -> list[tuple[str, str]]:
|
|
50
|
+
"""`(link, value)` for the Requirement/Specification/Epic rows of `## Trace`."""
|
|
51
|
+
m = _TRACE_HEADER_RE.search(text)
|
|
52
|
+
if not m:
|
|
53
|
+
return []
|
|
54
|
+
rest = text[m.end():]
|
|
55
|
+
nxt = _NEXT_SECTION_RE.search(rest)
|
|
56
|
+
section = rest[:nxt.start()] if nxt else rest
|
|
57
|
+
return [(link, value) for link, value in _TRACE_ROW_RE.findall(section)]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _label(*parts: Any) -> str:
|
|
61
|
+
text = " ".join(str(p) for p in parts if p)
|
|
62
|
+
return text[:_LABEL_MAX]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _subject(rec: dict[str, Any]) -> dict[str, Any]:
|
|
66
|
+
subject = rec.get("subject")
|
|
67
|
+
return subject if isinstance(subject, dict) else {}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _read_pack(pack_dir: Path) -> tuple[str, list[dict[str, str]]] | str:
|
|
71
|
+
"""`(pack_id, records)` for a well-formed pack, else the reason it is not.
|
|
72
|
+
|
|
73
|
+
All-or-nothing: one malformed record skips the whole pack, so a pack never
|
|
74
|
+
lands half-indexed and looking complete."""
|
|
75
|
+
try:
|
|
76
|
+
manifest = json.loads((pack_dir / "manifest.json").read_text(encoding="utf-8"))
|
|
77
|
+
except (OSError, ValueError) as exc:
|
|
78
|
+
return f"manifest.json unreadable ({exc.__class__.__name__})"
|
|
79
|
+
if not isinstance(manifest, dict) or not isinstance(manifest.get("evidence", []), list):
|
|
80
|
+
return "manifest.json is not a pack manifest"
|
|
81
|
+
pack_id = manifest.get("id")
|
|
82
|
+
if not isinstance(pack_id, str) or not pack_id:
|
|
83
|
+
return "manifest.json has no id"
|
|
84
|
+
|
|
85
|
+
records: list[dict[str, str]] = []
|
|
86
|
+
for i, rec in enumerate(manifest.get("evidence", [])):
|
|
87
|
+
if not isinstance(rec, dict) or not _EV_ID_RE.match(str(rec.get("evidence_id", ""))) \
|
|
88
|
+
or not isinstance(rec.get("confidence"), str):
|
|
89
|
+
return f"manifest.json evidence[{i}] has no valid ev_ id and confidence"
|
|
90
|
+
subject = _subject(rec)
|
|
91
|
+
records.append({
|
|
92
|
+
"id": rec["evidence_id"], "origin": "rea", "confidence": rec["confidence"],
|
|
93
|
+
"label": _label(rec.get("operation"), subject.get("alias") or subject.get("name")),
|
|
94
|
+
})
|
|
95
|
+
|
|
96
|
+
findings = pack_dir / "odsf-findings.jsonl"
|
|
97
|
+
if findings.exists():
|
|
98
|
+
try:
|
|
99
|
+
lines = findings.read_text(encoding="utf-8").splitlines()
|
|
100
|
+
except OSError as exc:
|
|
101
|
+
return f"odsf-findings.jsonl unreadable ({exc.__class__.__name__})"
|
|
102
|
+
for n, line in enumerate(lines, 1):
|
|
103
|
+
if not line.strip():
|
|
104
|
+
continue
|
|
105
|
+
try:
|
|
106
|
+
rec = json.loads(line)
|
|
107
|
+
except ValueError:
|
|
108
|
+
return f"odsf-findings.jsonl line {n} is not JSON"
|
|
109
|
+
if not isinstance(rec, dict) or not _OF_ID_RE.match(str(rec.get("id", ""))) \
|
|
110
|
+
or not isinstance(rec.get("confidence"), str):
|
|
111
|
+
return f"odsf-findings.jsonl line {n} has no valid of_ id and confidence"
|
|
112
|
+
subject = _subject(rec)
|
|
113
|
+
records.append({
|
|
114
|
+
"id": rec["id"], "origin": "odsf", "confidence": rec["confidence"],
|
|
115
|
+
"label": _label(rec.get("kind"), subject.get("name") or subject.get("local_path")),
|
|
116
|
+
})
|
|
117
|
+
return pack_id, records
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _add_evidence(ctx: ProjectContext, g: DevKBGraph, report: list[str]) -> set[str]:
|
|
121
|
+
root = ctx.repo_root / EVIDENCE_DIR
|
|
122
|
+
known: dict[str, str] = {} # id -> confidence as first recorded
|
|
123
|
+
if not root.is_dir():
|
|
124
|
+
return set()
|
|
125
|
+
for pack_dir in sorted(p for p in root.iterdir() if p.is_dir()):
|
|
126
|
+
rel = str(pack_dir.relative_to(ctx.repo_root))
|
|
127
|
+
result = _read_pack(pack_dir)
|
|
128
|
+
if isinstance(result, str):
|
|
129
|
+
report.append(f"{rel}: evidence pack skipped -- {result}")
|
|
130
|
+
continue
|
|
131
|
+
pack_id, records = result
|
|
132
|
+
for rec in records:
|
|
133
|
+
if rec["id"] in known:
|
|
134
|
+
# first pack (sorted) to record an id owns it; never re-labelled
|
|
135
|
+
if known[rec["id"]] != rec["confidence"]:
|
|
136
|
+
report.append(f"{rel}: {rec['id'][:15]}… recorded as {rec['confidence']!r}, already "
|
|
137
|
+
f"{known[rec['id']]!r} in an earlier pack -- earlier record kept")
|
|
138
|
+
continue
|
|
139
|
+
known[rec["id"]] = rec["confidence"]
|
|
140
|
+
g.add_node(
|
|
141
|
+
"Evidence", rec["id"], source="evidence-pack",
|
|
142
|
+
confidence=rec["confidence"], origin=rec["origin"],
|
|
143
|
+
pack=pack_id, pack_dir=rel, label=rec["label"] or rec["id"][:15],
|
|
144
|
+
)
|
|
145
|
+
return set(known)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _spec_lookup(graph: DevKBGraph) -> tuple[set[str], dict[str, list[str]]]:
|
|
149
|
+
paths: set[str] = set()
|
|
150
|
+
by_id: dict[str, list[str]] = {}
|
|
151
|
+
for spec in graph.nodes("Spec"):
|
|
152
|
+
paths.add(spec.natural_key)
|
|
153
|
+
spec_id = spec.props.get("spec_id")
|
|
154
|
+
if spec_id:
|
|
155
|
+
by_id.setdefault(spec_id, []).append(spec.natural_key)
|
|
156
|
+
return paths, {k: sorted(v) for k, v in by_id.items()}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _resolve_specs(value: str, paths: set[str], by_id: dict[str, list[str]],
|
|
160
|
+
where: str, report: list[str]) -> list[str]:
|
|
161
|
+
out: list[str] = []
|
|
162
|
+
for token in re.split(r"[,\s]+", value.replace("`", "")):
|
|
163
|
+
if token.lower() in _PLACEHOLDERS:
|
|
164
|
+
continue
|
|
165
|
+
if token in paths:
|
|
166
|
+
out.append(token)
|
|
167
|
+
continue
|
|
168
|
+
matches = by_id.get(token, [])
|
|
169
|
+
if len(matches) == 1:
|
|
170
|
+
out.append(matches[0])
|
|
171
|
+
elif matches:
|
|
172
|
+
report.append(f"{where}: Specification {token} is ambiguous ({', '.join(matches)}) -- not linked")
|
|
173
|
+
else:
|
|
174
|
+
report.append(f"{where}: Specification {token} is not a Spec in the graph -- not linked")
|
|
175
|
+
return out
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _cite_evidence(g: DevKBGraph, src_type: str, src: str, text: str, where: str,
|
|
179
|
+
known: set[str], report: list[str]) -> None:
|
|
180
|
+
for ev_id in sorted(set(_CITED_ID_RE.findall(text))):
|
|
181
|
+
if ev_id not in known:
|
|
182
|
+
report.append(f"{where}: cites {ev_id[:15]}… which is in no evidence pack -- not linked")
|
|
183
|
+
continue
|
|
184
|
+
g.add_edge(
|
|
185
|
+
"EVIDENCED_BY", src_type, src, "Evidence", ev_id,
|
|
186
|
+
source="evidence-cite", confidence=1.0, evidence_ref=f"{where} cites {ev_id}",
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
|
|
191
|
+
g = DevKBGraph()
|
|
192
|
+
report: list[str] = []
|
|
193
|
+
real_tickets = {t.natural_key: t.props.get("ticket_type") for t in graph.nodes("Ticket")}
|
|
194
|
+
epic_re = re.compile(rf"\b{re.escape(ctx.ticket_prefix)}-\d+\b")
|
|
195
|
+
known_evidence = _add_evidence(ctx, g, report)
|
|
196
|
+
spec_paths, spec_by_id = _spec_lookup(graph)
|
|
197
|
+
|
|
198
|
+
for path in _iter_ticket_files(ctx):
|
|
199
|
+
try:
|
|
200
|
+
text = path.read_text(errors="replace")
|
|
201
|
+
except OSError:
|
|
202
|
+
continue
|
|
203
|
+
ticket = _parse_metadata(text, path)["ticket"]
|
|
204
|
+
if ticket not in real_tickets:
|
|
205
|
+
continue
|
|
206
|
+
where = str(path.relative_to(ctx.repo_root))
|
|
207
|
+
trace_ref = f"{where}#Trace"
|
|
208
|
+
|
|
209
|
+
for link, value in _trace_rows(text):
|
|
210
|
+
if value.strip().lower() in _PLACEHOLDERS:
|
|
211
|
+
continue
|
|
212
|
+
if link == "Requirement":
|
|
213
|
+
reqs = _REQ_RE.findall(value)
|
|
214
|
+
if not reqs:
|
|
215
|
+
report.append(f"{where}: Requirement {value!r} names no REQ-NNN -- not linked")
|
|
216
|
+
for req in reqs:
|
|
217
|
+
g.add_node("Requirement", req, source="ticket-trace")
|
|
218
|
+
g.add_edge("DERIVED_FROM", "Ticket", ticket, "Requirement", req,
|
|
219
|
+
source="ticket-trace", confidence=1.0, evidence_ref=trace_ref)
|
|
220
|
+
elif link == "Epic":
|
|
221
|
+
epics = epic_re.findall(value)
|
|
222
|
+
if not epics:
|
|
223
|
+
report.append(f"{where}: Epic {value!r} names no {ctx.ticket_prefix}-NNN -- not linked")
|
|
224
|
+
for epic in epics:
|
|
225
|
+
if epic == ticket:
|
|
226
|
+
report.append(f"{where}: Epic {epic} is the ticket itself -- not linked")
|
|
227
|
+
continue
|
|
228
|
+
if epic not in real_tickets:
|
|
229
|
+
report.append(f"{where}: Epic {epic} is not a Ticket in the graph -- not linked")
|
|
230
|
+
continue
|
|
231
|
+
# ADR-SDP-DEVKB-002 §3: CHILD_OF is Story -> Epic, nothing else.
|
|
232
|
+
if real_tickets[ticket] != "STORY" or real_tickets[epic] != "EPIC":
|
|
233
|
+
report.append(f"{where}: Epic row links {real_tickets[ticket]} {ticket} to "
|
|
234
|
+
f"{real_tickets[epic]} {epic}; CHILD_OF is STORY -> EPIC -- not linked")
|
|
235
|
+
continue
|
|
236
|
+
g.add_edge("CHILD_OF", "Ticket", ticket, "Ticket", epic,
|
|
237
|
+
source="ticket-trace", confidence=1.0, evidence_ref=trace_ref)
|
|
238
|
+
else: # Specification
|
|
239
|
+
for spec in _resolve_specs(value, spec_paths, spec_by_id, where, report):
|
|
240
|
+
g.add_edge("IMPLEMENTS", "Ticket", ticket, "Spec", spec,
|
|
241
|
+
source="ticket-trace", confidence=1.0, evidence_ref=trace_ref)
|
|
242
|
+
|
|
243
|
+
_cite_evidence(g, "Ticket", ticket, text, where, known_evidence, report)
|
|
244
|
+
|
|
245
|
+
for spec_path in sorted(spec_paths):
|
|
246
|
+
try:
|
|
247
|
+
text = (ctx.repo_root / spec_path).read_text(errors="replace")
|
|
248
|
+
except OSError:
|
|
249
|
+
continue
|
|
250
|
+
_cite_evidence(g, "Spec", spec_path, text, spec_path, known_evidence, report)
|
|
251
|
+
|
|
252
|
+
if report:
|
|
253
|
+
lines = sorted(set(report))
|
|
254
|
+
print(f"devkb build: lifecycle: {len(lines)} unresolved reference(s), none guessed:",
|
|
255
|
+
file=sys.stderr)
|
|
256
|
+
for line in lines:
|
|
257
|
+
print(f" {line}", file=sys.stderr)
|
|
258
|
+
return g
|