devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""Spec + DOCUMENTS (devkb BL-13, promoted from `devkb/ROADMAP.md`'s own
|
|
2
|
+
Phase 4 step 1 -- KNOWLEDGE_SPEC.md Section 4/5).
|
|
3
|
+
|
|
4
|
+
Deterministic, regex-derived, no LLM call -- the cheap half of Phase 4
|
|
5
|
+
(`devkb/answer.py`'s LLM-synthesis half stays out of scope, Section 9).
|
|
6
|
+
Closes the audit's "no node descriptions from specification documents"
|
|
7
|
+
finding: every ADR/design/onboarding markdown file becomes a real `Spec`
|
|
8
|
+
node, `DOCUMENTS` linking it to any `Ticket` it actually mentions.
|
|
9
|
+
|
|
10
|
+
Needs the FULL merged graph (real `Ticket` nodes from Phase 1) so a
|
|
11
|
+
`DOCUMENTS` edge only ever points at a ticket confirmed to exist -- a
|
|
12
|
+
stale/renamed/typo'd ticket-id mention in an old doc gets skipped, not
|
|
13
|
+
guessed into a dangling edge. Post-merge, same shape as BL-4/BL-6/BL-11.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import hashlib
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
from devkb.model import DevKBGraph
|
|
21
|
+
from devkb.project_context import DEFAULT_SPEC_DIRS, ProjectContext
|
|
22
|
+
|
|
23
|
+
_HEADING_RE = re.compile(r"^#\s+(.+)$", re.MULTILINE)
|
|
24
|
+
_RELATED_LINE_RE = re.compile(r"^\*?\*?Related.*?:\*?\*?\s*(.+)$", re.MULTILINE | re.IGNORECASE)
|
|
25
|
+
|
|
26
|
+
# The two closed vocabularies ADR-SDP-SPEC-001 Section 3 defines. Closed on
|
|
27
|
+
# purpose: an open one is how the sibling OCO checkout ended up carrying
|
|
28
|
+
# `Accepted`, `Proposed`, `PROPOSED`, `DRAFT` and nine lines of prose in the
|
|
29
|
+
# same field. A declared value outside these sets is not honoured -- it falls
|
|
30
|
+
# back to the path rule below rather than inventing a fifth state.
|
|
31
|
+
_DOC_TYPES = frozenset({"specification", "adr", "design", "onboarding", "index"})
|
|
32
|
+
_STATUSES = frozenset({"draft", "review", "accepted", "superseded"})
|
|
33
|
+
|
|
34
|
+
# The front-matter block, only when it opens the file. The schema is
|
|
35
|
+
# deliberately flat -- scalars, `null`, and inline lists -- so this is parsed
|
|
36
|
+
# here rather than with PyYAML: yaml is present in this dev environment but
|
|
37
|
+
# absent from `pms-backend/requirements.txt`, and `scripts/deployment.sh`
|
|
38
|
+
# installs a fresh clone from that file. An extractor that works on this
|
|
39
|
+
# machine and dies on a clean checkout is worse than one with no front-matter.
|
|
40
|
+
_FRONT_MATTER_RE = re.compile(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*\r?\n", re.DOTALL)
|
|
41
|
+
_FM_KEY_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*):[ \t]*(.*)$")
|
|
42
|
+
# A `#` opens a comment only at the start of the value or after whitespace,
|
|
43
|
+
# so a title containing one (`# 3 of 9 journeys`) survives.
|
|
44
|
+
_FM_COMMENT_RE = re.compile(r"(?:^|\s)#.*$")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _scalar(raw: str) -> str | list[str] | None:
|
|
48
|
+
value = raw.strip()
|
|
49
|
+
# A quoted scalar is taken whole: inside quotes a `#` is literal, not a
|
|
50
|
+
# comment. Stripping comments first truncated any title that contained one
|
|
51
|
+
# ("Only 1 of 9 journeys reach #done" became "Only 1 of 9 journeys reach").
|
|
52
|
+
if value[:1] in ("'", '"'):
|
|
53
|
+
closing = value.find(value[0], 1)
|
|
54
|
+
if closing != -1:
|
|
55
|
+
return value[1:closing]
|
|
56
|
+
value = _FM_COMMENT_RE.sub("", value).strip()
|
|
57
|
+
if value in ("", "null", "~"):
|
|
58
|
+
return None
|
|
59
|
+
if value.startswith("[") and value.endswith("]"):
|
|
60
|
+
inner = value[1:-1].strip()
|
|
61
|
+
return [p.strip().strip("'\"") for p in inner.split(",") if p.strip()] if inner else []
|
|
62
|
+
return value.strip("'\"")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def parse_front_matter(text: str) -> tuple[dict[str, str | list[str] | None], str]:
|
|
66
|
+
"""Split a document into its declared front-matter and its body.
|
|
67
|
+
|
|
68
|
+
Returns `({}, text)` unchanged when there is no block, which is the common
|
|
69
|
+
case and must stay the cheap one: most of the corpus will carry no
|
|
70
|
+
front-matter for a long time, and such a document has to remain a
|
|
71
|
+
first-class member rather than quietly dropping out of the index.
|
|
72
|
+
|
|
73
|
+
Only top-level `key: value` lines are read. Anything indented belongs to a
|
|
74
|
+
nested structure this schema does not have, and is skipped rather than
|
|
75
|
+
guessed at.
|
|
76
|
+
"""
|
|
77
|
+
match = _FRONT_MATTER_RE.match(text)
|
|
78
|
+
if not match:
|
|
79
|
+
return {}, text
|
|
80
|
+
fields: dict[str, str | list[str] | None] = {}
|
|
81
|
+
for line in match.group(1).splitlines():
|
|
82
|
+
if not line.strip() or line[:1] in (" ", "\t", "#", "-"):
|
|
83
|
+
continue
|
|
84
|
+
key_match = _FM_KEY_RE.match(line)
|
|
85
|
+
if key_match:
|
|
86
|
+
fields[key_match.group(1).strip().lower()] = _scalar(key_match.group(2))
|
|
87
|
+
return fields, text[match.end():]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _declared(fields: dict[str, str | list[str] | None], key: str, allowed: frozenset[str]) -> str:
|
|
91
|
+
value = fields.get(key)
|
|
92
|
+
if isinstance(value, str) and value.strip().lower() in allowed:
|
|
93
|
+
return value.strip().lower()
|
|
94
|
+
return ""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _doc_type(rel_path: str, fields: dict[str, str | list[str] | None] | None = None) -> str:
|
|
98
|
+
"""What a document *is*, preferring its own declaration over its location.
|
|
99
|
+
|
|
100
|
+
ODSF-128: the path rule below is a fallthrough, not a classification --
|
|
101
|
+
`design` is what anything outside `docs/` and `.onboarding/` gets by
|
|
102
|
+
default. That made `backlogs/PMS-NEXT-ERA-SPECIFICATION.md`, this project's
|
|
103
|
+
actual product specification, sort last in the BA's injection ranking and
|
|
104
|
+
be the first thing dropped at the cap. A document can now say what it is.
|
|
105
|
+
|
|
106
|
+
The fallthrough is kept exactly as it was for every document that declares
|
|
107
|
+
nothing, so this change cannot shrink an existing corpus.
|
|
108
|
+
"""
|
|
109
|
+
declared = _declared(fields or {}, "type", _DOC_TYPES)
|
|
110
|
+
if declared:
|
|
111
|
+
return declared
|
|
112
|
+
if rel_path.startswith("docs/adr/"):
|
|
113
|
+
return "adr"
|
|
114
|
+
if rel_path.startswith(".onboarding/"):
|
|
115
|
+
return "onboarding"
|
|
116
|
+
if rel_path.startswith("docs/"):
|
|
117
|
+
# A top-level `docs/*.md` is the project's read-facing specification
|
|
118
|
+
# set -- requirement/technical/test documents and the cross-cutting
|
|
119
|
+
# references beside them. Typed apart from `design` because they are a
|
|
120
|
+
# different altitude: `design/` is per-screen and per-sprint working
|
|
121
|
+
# material, these consolidate it. Both are Spec nodes; only the
|
|
122
|
+
# `doc_type` property distinguishes them, so this adds no node type and
|
|
123
|
+
# no edge verb (ADR-SDP-DEVKB-001's schema gate does not apply).
|
|
124
|
+
return "specification"
|
|
125
|
+
return "design"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _spec_id(rel_path: str, fields: dict[str, str | list[str] | None]) -> str:
|
|
129
|
+
"""The id a ticket's `## Trace` `Specification` row names this document by
|
|
130
|
+
(ODSF-230, ADR-SDP-DEVKB-002 §3 `IMPLEMENTS`).
|
|
131
|
+
|
|
132
|
+
The front-matter `id` when the document declares one (`SPEC-ODSF-003`),
|
|
133
|
+
else the filename stem before its first `_` -- the ADR-SDP-SPEC-001 naming
|
|
134
|
+
convention (`SPEC-ODSF-003_<slug>.md`, `ADR-SDP-DEVKB-002_<slug>.md`). A
|
|
135
|
+
property, not a key: the natural key stays the path, so no existing uid moves.
|
|
136
|
+
"""
|
|
137
|
+
declared = fields.get("id")
|
|
138
|
+
if isinstance(declared, str) and declared.strip():
|
|
139
|
+
return declared.strip()
|
|
140
|
+
return rel_path.rsplit("/", 1)[-1].removesuffix(".md").split("_", 1)[0]
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _title(text: str) -> str | None:
|
|
144
|
+
m = _HEADING_RE.search(text)
|
|
145
|
+
return m.group(1).strip() if m else None
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _spec_files(ctx: ProjectContext) -> list[str]:
|
|
149
|
+
"""The markdown this project calls its specifications.
|
|
150
|
+
|
|
151
|
+
ODSF-102: the roots come from the project rather than from this file. They
|
|
152
|
+
used to be three hardcoded globs, which meant a project keeping its specs
|
|
153
|
+
anywhere else was invisible to its own BA -- ODSF's whole specification set
|
|
154
|
+
is under `backlogs/`, so its graph held 2 Spec nodes while OCO, whose layout
|
|
155
|
+
happens to match those globs, held 187.
|
|
156
|
+
|
|
157
|
+
A root ending in `/**` is scanned recursively; without it, top level only.
|
|
158
|
+
That distinction is not decoration: `design` was recursive and `docs/adr`
|
|
159
|
+
deliberately was not (its siblings under `docs/` are per-owner working
|
|
160
|
+
areas, not specifications of the system), and a project that declares
|
|
161
|
+
nothing must behave exactly as it did before.
|
|
162
|
+
|
|
163
|
+
A declared root that does not exist is skipped, so a config naming a
|
|
164
|
+
directory the project has not created yet does not break the build.
|
|
165
|
+
"""
|
|
166
|
+
out: set[str] = set()
|
|
167
|
+
for root in (ctx.spec_dirs or DEFAULT_SPEC_DIRS):
|
|
168
|
+
recursive = root.endswith("/**")
|
|
169
|
+
base = ctx.repo_root / (root[:-3] if recursive else root)
|
|
170
|
+
if not base.is_dir():
|
|
171
|
+
continue
|
|
172
|
+
for path in (base.rglob("*.md") if recursive else base.glob("*.md")):
|
|
173
|
+
if path.is_file():
|
|
174
|
+
out.add(str(path.relative_to(ctx.repo_root)))
|
|
175
|
+
return sorted(out)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
|
|
179
|
+
g = DevKBGraph()
|
|
180
|
+
real_tickets = {t.natural_key for t in graph.nodes("Ticket")}
|
|
181
|
+
ticket_re = re.compile(rf"{re.escape(ctx.ticket_prefix)}-\d+")
|
|
182
|
+
|
|
183
|
+
for rel_path in _spec_files(ctx):
|
|
184
|
+
try:
|
|
185
|
+
text = (ctx.repo_root / rel_path).read_text(errors="replace")
|
|
186
|
+
except OSError:
|
|
187
|
+
continue
|
|
188
|
+
|
|
189
|
+
fields, body = parse_front_matter(text)
|
|
190
|
+
title = _title(text) or rel_path
|
|
191
|
+
# Hashed over the BODY, not the whole file, so that editing front-matter
|
|
192
|
+
# -- appending a ticket id to `tickets:`, bumping `updated:` -- is not
|
|
193
|
+
# reported as the document's content having drifted. Those edits are
|
|
194
|
+
# bookkeeping about the document; only the body is the document.
|
|
195
|
+
content_sha = hashlib.sha256(body.encode("utf-8", "replace")).hexdigest()[:16]
|
|
196
|
+
g.add_node(
|
|
197
|
+
"Spec", rel_path, source="spec-scan", title=title,
|
|
198
|
+
doc_type=_doc_type(rel_path, fields),
|
|
199
|
+
# "" means the document declares no status, which is not the same
|
|
200
|
+
# as declaring `draft`. Only an explicit `superseded` withdraws a
|
|
201
|
+
# document from the BA's corpus, so silence never excludes anything.
|
|
202
|
+
status=_declared(fields, "status", _STATUSES),
|
|
203
|
+
content_sha=content_sha,
|
|
204
|
+
spec_id=_spec_id(rel_path, fields),
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
mentioned: set[str] = set()
|
|
208
|
+
for related_line in _RELATED_LINE_RE.findall(text):
|
|
209
|
+
mentioned.update(ticket_re.findall(related_line))
|
|
210
|
+
mentioned.update(ticket_re.findall(text))
|
|
211
|
+
|
|
212
|
+
for ticket_id in mentioned:
|
|
213
|
+
if ticket_id not in real_tickets:
|
|
214
|
+
continue # a stale/typo'd/renamed mention -- skip, don't guess
|
|
215
|
+
g.add_edge(
|
|
216
|
+
"DOCUMENTS", "Spec", rel_path, "Ticket", ticket_id,
|
|
217
|
+
source="spec-scan", confidence=0.85,
|
|
218
|
+
evidence_ref=f"{rel_path} mentions {ticket_id}",
|
|
219
|
+
)
|
|
220
|
+
return g
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Content-based test linkage (devkb BL-4, body/legs model).
|
|
2
|
+
|
|
3
|
+
Phase 2's TESTED_BY needs a ticket ID literally mentioned near the test --
|
|
4
|
+
doubly ticket-dependent (a ticket to find the commit, a ticket mention in
|
|
5
|
+
the test). This resolves real evidence instead: a test function's own
|
|
6
|
+
imports matched against Symbol natural keys (BL-3), scoped to calls that
|
|
7
|
+
function's own body actually makes -- not "the file imports it somewhere,"
|
|
8
|
+
which would just move Phase 2's coarseness from ticket-level to file-level
|
|
9
|
+
instead of removing it. Additive alongside Phase 2's existing
|
|
10
|
+
0.7-confidence ticket-mediated edges, never a replacement.
|
|
11
|
+
|
|
12
|
+
Needs BL-3's Symbol nodes already in `graph` (build.py merges the symbols
|
|
13
|
+
cache before calling this) -- an empty/missing Symbol set just means zero
|
|
14
|
+
edges here, not an error, same "can't check yet" framing used elsewhere in
|
|
15
|
+
devkb for a missing input.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import ast
|
|
20
|
+
|
|
21
|
+
from devkb.build_symbols import (
|
|
22
|
+
EXPR, _calls_in_body, _import_descriptors, _import_targets, _imports, _module_paths, _py_literals,
|
|
23
|
+
_list_python_files, _resolve_cross_file_symbol, _resolve_via_literals, collect_reexports, register_reexports,
|
|
24
|
+
)
|
|
25
|
+
from devkb.model import DevKBGraph
|
|
26
|
+
from devkb.project_context import ProjectContext
|
|
27
|
+
from devkb.test_discovery import list_test_files
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _test_functions(tree: ast.AST) -> list[tuple[str, int, set[str], set[tuple[str | None, str]]]]:
|
|
31
|
+
"""[(name, lineno, {short names this function's own body calls}), ...]
|
|
32
|
+
-- reuses BL-3's own body-scoped call scanner (build_symbols.py) so a
|
|
33
|
+
nested helper's calls don't get misattributed to the outer test. Only
|
|
34
|
+
the short callee name is used here (not _calls_in_body's receiver,
|
|
35
|
+
which BL-3b's own cross-file CALLS resolution needs but this doesn't --
|
|
36
|
+
a test's own import-name match is already the precise signal)."""
|
|
37
|
+
out: list[tuple[str, int, set[str], set[tuple[str | None, str]]]] = []
|
|
38
|
+
for node in ast.walk(tree):
|
|
39
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"):
|
|
40
|
+
pairs = set(_calls_in_body(node))
|
|
41
|
+
out.append((node.name, node.lineno, {short for _receiver, short in pairs}, pairs))
|
|
42
|
+
return out
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
|
|
46
|
+
g = DevKBGraph()
|
|
47
|
+
by_short_name: dict[str, list[str]] = {}
|
|
48
|
+
for s in graph.nodes("Symbol"):
|
|
49
|
+
short = s.natural_key.rsplit("::", 1)[-1].rsplit(".", 1)[-1]
|
|
50
|
+
by_short_name.setdefault(short, []).append(s.natural_key)
|
|
51
|
+
if not by_short_name:
|
|
52
|
+
return g
|
|
53
|
+
# path -> [{"qualified_name": ...}], the shape _resolve_cross_file_symbol reads (ODSF-305).
|
|
54
|
+
symbols_by_file: dict[str, list[dict]] = {}
|
|
55
|
+
for s in graph.nodes("Symbol"):
|
|
56
|
+
path, _, qualified = s.natural_key.partition("::")
|
|
57
|
+
symbols_by_file.setdefault(path, []).append({"qualified_name": qualified})
|
|
58
|
+
# `from pkg import f` where `pkg/__init__.py` re-exports f from a submodule
|
|
59
|
+
register_reexports(symbols_by_file, collect_reexports(ctx.repo_root, _list_python_files(ctx.repo_root)))
|
|
60
|
+
|
|
61
|
+
# ODSF-304: every tracked python test file, wherever it lives.
|
|
62
|
+
for py in (ctx.repo_root / rel for rel in list_test_files(ctx.repo_root, ctx.tests_dir) if rel.endswith(".py")):
|
|
63
|
+
try:
|
|
64
|
+
text = py.read_text(errors="replace")
|
|
65
|
+
tree = ast.parse(text, filename=str(py))
|
|
66
|
+
except (SyntaxError, OSError, ValueError):
|
|
67
|
+
continue
|
|
68
|
+
rel = str(py.relative_to(ctx.repo_root))
|
|
69
|
+
imported = _imports(tree)
|
|
70
|
+
tests = _test_functions(tree)
|
|
71
|
+
if not tests:
|
|
72
|
+
continue
|
|
73
|
+
|
|
74
|
+
receivers, bare = _import_targets(_import_descriptors(tree), rel)
|
|
75
|
+
literals = _py_literals(tree)
|
|
76
|
+
for name, lineno, calls, pairs in tests:
|
|
77
|
+
resolved: set[str] = set()
|
|
78
|
+
# ODSF-305: `module.func()` and aliased or relative imports.
|
|
79
|
+
for receiver, short in pairs:
|
|
80
|
+
if receiver and "." in receiver and receiver != EXPR: # `pkg.mod.f()` after `import pkg.mod`
|
|
81
|
+
root, rest = receiver.split(".", 1)
|
|
82
|
+
modules = [f"{m}.{rest}" for m in receivers.get(root, [])]
|
|
83
|
+
hit = next((h for h in (_resolve_cross_file_symbol(m, rel, short, symbols_by_file) for m in modules) if h), None)
|
|
84
|
+
if hit:
|
|
85
|
+
resolved.add(hit)
|
|
86
|
+
continue
|
|
87
|
+
if receiver:
|
|
88
|
+
modules = receivers.get(receiver) or ([receiver] if receiver.isidentifier() else [])
|
|
89
|
+
hit = None
|
|
90
|
+
for module in modules:
|
|
91
|
+
hit = _resolve_cross_file_symbol(module, rel, short, symbols_by_file)
|
|
92
|
+
if hit:
|
|
93
|
+
break
|
|
94
|
+
if hit is None and receiver not in receivers:
|
|
95
|
+
hit = _resolve_via_literals(literals, short, symbols_by_file)
|
|
96
|
+
if hit:
|
|
97
|
+
resolved.add(hit)
|
|
98
|
+
elif short in bare:
|
|
99
|
+
module, original = bare[short]
|
|
100
|
+
hit = _resolve_cross_file_symbol(module, rel, original, symbols_by_file)
|
|
101
|
+
if hit:
|
|
102
|
+
resolved.add(hit)
|
|
103
|
+
for local_name, dotted_module in imported:
|
|
104
|
+
if local_name not in calls:
|
|
105
|
+
continue
|
|
106
|
+
candidates = by_short_name.get(local_name, [])
|
|
107
|
+
if not candidates:
|
|
108
|
+
continue
|
|
109
|
+
target = None
|
|
110
|
+
if dotted_module:
|
|
111
|
+
for guess in _module_paths(dotted_module):
|
|
112
|
+
hit = next((c for c in candidates if c.startswith(f"{guess}::")), None)
|
|
113
|
+
if hit:
|
|
114
|
+
target = hit
|
|
115
|
+
break
|
|
116
|
+
if target is None and len(candidates) == 1:
|
|
117
|
+
target = candidates[0]
|
|
118
|
+
if target:
|
|
119
|
+
resolved.add(target)
|
|
120
|
+
|
|
121
|
+
if not resolved:
|
|
122
|
+
continue
|
|
123
|
+
test_key = f"{rel}:{lineno}"
|
|
124
|
+
g.add_node("TestCase", test_key, source="ast-import-resolution", file=rel, line=lineno, function=name)
|
|
125
|
+
for symbol_key in resolved:
|
|
126
|
+
g.add_edge(
|
|
127
|
+
"TESTED_BY", "Symbol", symbol_key, "TestCase", test_key,
|
|
128
|
+
source="ast-import-resolution", confidence=0.95,
|
|
129
|
+
evidence_ref=f"{name} (line {lineno}) imports and calls a name resolving to {symbol_key}",
|
|
130
|
+
)
|
|
131
|
+
return g
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""TestCase for every test function, not only those that name a ticket (ODSF-304).
|
|
2
|
+
|
|
3
|
+
`files_tests.py` creates a TestCase only where a ticket id sits near a test, so
|
|
4
|
+
a project whose tests never mention tickets (ODSF's own, and nearly all of
|
|
5
|
+
another's) had almost no TestCase nodes at all. This inventories every test
|
|
6
|
+
function that `devkb.test_discovery` finds and links it to its file with
|
|
7
|
+
DEFINED_IN (ADR-SDP-DEVKB-003).
|
|
8
|
+
|
|
9
|
+
Node identity is `path:line`, the same key `files_tests.py` uses, so a test
|
|
10
|
+
that is both inventoried and ticket-marked is one node carrying both sets of
|
|
11
|
+
properties (DevKBGraph merges props on a repeated key).
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from devkb.model import DevKBGraph
|
|
16
|
+
from devkb.project_context import ProjectContext
|
|
17
|
+
from devkb.test_discovery import list_test_files, tests_in_file
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def extract(ctx: ProjectContext) -> DevKBGraph:
|
|
21
|
+
g = DevKBGraph()
|
|
22
|
+
for rel in list_test_files(ctx.repo_root, ctx.tests_dir):
|
|
23
|
+
framework = "pytest" if rel.endswith(".py") else "js"
|
|
24
|
+
cases = tests_in_file(ctx.repo_root, rel)
|
|
25
|
+
if not cases:
|
|
26
|
+
continue
|
|
27
|
+
g.add_node("File", rel, source="test-inventory")
|
|
28
|
+
for name, line in cases:
|
|
29
|
+
key = f"{rel}:{line}"
|
|
30
|
+
g.add_node(
|
|
31
|
+
"TestCase", key, source="test-inventory",
|
|
32
|
+
file=rel, line=line, function=name, framework=framework,
|
|
33
|
+
)
|
|
34
|
+
g.add_edge(
|
|
35
|
+
"DEFINED_IN", "TestCase", key, "File", rel,
|
|
36
|
+
source="test-inventory", confidence=1.0, evidence_ref=f"{rel}:{line}",
|
|
37
|
+
)
|
|
38
|
+
return g
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""Threshold-free ticket similarity (devkb BL-11, KNOWLEDGE_SPEC.md Section
|
|
2
|
+
5, resolution of Open question 4).
|
|
3
|
+
|
|
4
|
+
The original draft gated SIMILAR_TO behind a build-time overlap threshold
|
|
5
|
+
("what fraction counts as similar enough to surface") -- dropped per
|
|
6
|
+
direct pushback: a guessed constant baked into the graph build is the
|
|
7
|
+
wrong place to make that call. This creates the edge for ANY nonzero
|
|
8
|
+
touched-file overlap between two tickets' own `REFERENCES`-linked commits'
|
|
9
|
+
`TOUCHES`-linked files, and stores the real overlap
|
|
10
|
+
(`shared_file_count`/`overlap_score`) as edge data -- filtering ("show me
|
|
11
|
+
only the top 3" / "only above 0.3") moves to query time, the same shape
|
|
12
|
+
`list_nodes()`'s own `q` search already uses (rank at the edge, don't
|
|
13
|
+
gate at the build).
|
|
14
|
+
|
|
15
|
+
Needs the FULL merged graph (Ticket/Commit/File nodes, REFERENCES/TOUCHES
|
|
16
|
+
edges from Phase 1/BL-1, already shipped) -- a post-merge derivation pass,
|
|
17
|
+
not a new source, so it runs from `graph` itself like BL-4/BL-6, not as a
|
|
18
|
+
`graphs[]` entry in build.py.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from itertools import combinations
|
|
23
|
+
from math import log
|
|
24
|
+
|
|
25
|
+
from devkb.model import DevKBGraph
|
|
26
|
+
|
|
27
|
+
#: ODSF-314: neighbours kept per ticket; process_api's index keeps 8, so more is never read.
|
|
28
|
+
TOP_NEIGHBOURS = 8
|
|
29
|
+
from devkb.project_context import ProjectContext
|
|
30
|
+
|
|
31
|
+
# A heartbeat/status file touched by nearly every commit regardless of what
|
|
32
|
+
# the commit is actually about carries ZERO discriminating signal for "are
|
|
33
|
+
# these two tickets related" -- this is a data-quality exclusion (which
|
|
34
|
+
# files can even be evidence), not the build-time overlap-score THRESHOLD
|
|
35
|
+
# that was explicitly rejected (Section 5's resolution, Open question 4) --
|
|
36
|
+
# that rejection was about gating which edges get SHOWN based on a guessed
|
|
37
|
+
# significance cutoff; this is about which files get COUNTED at all.
|
|
38
|
+
# Confirmed live, not theorized: on OCO's own real history, sdp/orchestrator/
|
|
39
|
+
# status/*.md files are touched by 887-946 of 2,614 commits (34-36%) -- CM.md
|
|
40
|
+
# alone by 946 -- an extreme structural outlier next to any real functional
|
|
41
|
+
# file, already recognized elsewhere in this codebase's own tooling as a
|
|
42
|
+
# distinct "status file heartbeat" category (see gate-1's own check of that
|
|
43
|
+
# name). Path-based, same discipline as every other allow/exclude-list in
|
|
44
|
+
# this spec, extensible if a different project's own hub files show up.
|
|
45
|
+
_HUB_FILE_PREFIXES = ("sdp/orchestrator/status/",)
|
|
46
|
+
|
|
47
|
+
#: A file touched by more than this share of the tickets that touch anything
|
|
48
|
+
#: is bookkeeping, not subject matter, and two tickets sharing it are not
|
|
49
|
+
#: related in any sense a reader cares about.
|
|
50
|
+
#:
|
|
51
|
+
#: This is the SAME judgement as the prefix list above — which files may serve
|
|
52
|
+
#: as evidence at all — not the build-time overlap threshold Section 5 rejected.
|
|
53
|
+
#: That rejection was about gating which edges get SHOWN by a guessed
|
|
54
|
+
#: significance cutoff; this is about which files can be evidence, and the
|
|
55
|
+
#: prefix list's own comment invited it ("extensible if a different project's
|
|
56
|
+
#: own hub files show up"). They showed up, and hand-maintaining the list did
|
|
57
|
+
#: not scale: measured on OCO's devkb at 2026-08-07, four files nobody had
|
|
58
|
+
#: listed produced 161,894 of the 178,114 single-file SIMILAR_TO edges — 91% —
|
|
59
|
+
#: led by `tests/test_oco_ticket_coverage.py`, a test asserting that every
|
|
60
|
+
#: ticket has coverage and therefore touched by nearly every ticket's commits.
|
|
61
|
+
#: It alone accounted for 46% of the entire relation.
|
|
62
|
+
#:
|
|
63
|
+
#: 0.10 sits an order of magnitude below the status files the prefix list was
|
|
64
|
+
#: written for (34-36% of commits) and above the busiest genuinely functional
|
|
65
|
+
#: file measured on the same data.
|
|
66
|
+
_HUB_FILE_SHARE = 0.10
|
|
67
|
+
|
|
68
|
+
#: Below this many tickets the share above is sampling noise, not a property of
|
|
69
|
+
#: the repository — on five tickets one of them IS 20%, and on the two-ticket
|
|
70
|
+
#: case any shared file at all reads as 100% and the relation empties itself.
|
|
71
|
+
#: A statistical exclusion needs a population to be computed over; under it the
|
|
72
|
+
#: prefix list still applies and nothing else is disqualified.
|
|
73
|
+
_HUB_MIN_TICKETS = 30
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _is_hub_file(path: str) -> bool:
|
|
77
|
+
return any(path.startswith(p) for p in _HUB_FILE_PREFIXES)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def hub_files(touched: dict[str, set[str]], share: float = _HUB_FILE_SHARE) -> dict[str, float]:
|
|
81
|
+
"""Files touched by too many tickets to discriminate between any two.
|
|
82
|
+
|
|
83
|
+
Returned rather than merely applied so the build can record WHICH files
|
|
84
|
+
were disqualified: an exclusion nobody can see is one nobody can argue
|
|
85
|
+
with, and this one is load-bearing enough to be worth arguing with.
|
|
86
|
+
"""
|
|
87
|
+
if len(touched) < _HUB_MIN_TICKETS:
|
|
88
|
+
return {}
|
|
89
|
+
df: dict[str, int] = {}
|
|
90
|
+
for files in touched.values():
|
|
91
|
+
for f in files:
|
|
92
|
+
df[f] = df.get(f, 0) + 1
|
|
93
|
+
total = len(touched)
|
|
94
|
+
return {f: n / total for f, n in df.items() if n / total > share}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _touched_files(graph: DevKBGraph, ticket_key: str) -> set[str]:
|
|
98
|
+
commits = {e.src for e in graph.edges("REFERENCES") if e.dst == ticket_key}
|
|
99
|
+
if not commits:
|
|
100
|
+
return set()
|
|
101
|
+
return {e.dst for e in graph.edges("TOUCHES") if e.src in commits and not _is_hub_file(e.dst)}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
|
|
105
|
+
g = DevKBGraph()
|
|
106
|
+
tickets = [t.natural_key for t in graph.nodes("Ticket")]
|
|
107
|
+
if len(tickets) < 2:
|
|
108
|
+
return g
|
|
109
|
+
|
|
110
|
+
touched = {key: _touched_files(graph, key) for key in tickets}
|
|
111
|
+
touched = {k: v for k, v in touched.items() if v} # a ticket with no touched files can't overlap
|
|
112
|
+
|
|
113
|
+
# Second exclusion pass, computed rather than listed. Runs here and not in
|
|
114
|
+
# `_touched_files` because document frequency is a property of the whole
|
|
115
|
+
# ticket set, which that function cannot see one ticket at a time.
|
|
116
|
+
hubs = hub_files(touched)
|
|
117
|
+
if hubs:
|
|
118
|
+
touched = {k: (v - hubs.keys()) for k, v in touched.items()}
|
|
119
|
+
touched = {k: v for k, v in touched.items() if v}
|
|
120
|
+
print(f" ticket_similarity: {len(hubs)} hub file(s) excluded as evidence "
|
|
121
|
+
f"(touched by >{_HUB_FILE_SHARE:.0%} of tickets): "
|
|
122
|
+
+ ", ".join(f"{f} ({s:.0%})" for f, s in
|
|
123
|
+
sorted(hubs.items(), key=lambda kv: -kv[1])[:5])
|
|
124
|
+
+ ("..." if len(hubs) > 5 else ""))
|
|
125
|
+
if len(touched) < 2:
|
|
126
|
+
return g
|
|
127
|
+
|
|
128
|
+
# Inverse document frequency over the surviving files. A file two tickets
|
|
129
|
+
# share is only evidence in proportion to how few OTHER tickets share it;
|
|
130
|
+
# weighting by it makes the hub problem self-limiting rather than dependent
|
|
131
|
+
# on the cutoff above catching every case.
|
|
132
|
+
total_tickets = len(touched)
|
|
133
|
+
doc_freq: dict[str, int] = {}
|
|
134
|
+
for files in touched.values():
|
|
135
|
+
for f in files:
|
|
136
|
+
doc_freq[f] = doc_freq.get(f, 0) + 1
|
|
137
|
+
|
|
138
|
+
def _idf(path: str) -> float:
|
|
139
|
+
return log(total_tickets / doc_freq.get(path, 1)) + 1.0
|
|
140
|
+
|
|
141
|
+
# Inverted index (file -> tickets that touched it): only ticket pairs
|
|
142
|
+
# sharing at least one real file are ever compared. At real scale
|
|
143
|
+
# (confirmed live: 1,250 real tickets on OCO) most pairs share nothing
|
|
144
|
+
# -- a full O(n^2) pairwise scan is wasted work compared over an index
|
|
145
|
+
# built from the actual touch data.
|
|
146
|
+
file_to_tickets: dict[str, set[str]] = {}
|
|
147
|
+
for ticket, files in touched.items():
|
|
148
|
+
for f in files:
|
|
149
|
+
file_to_tickets.setdefault(f, set()).add(ticket)
|
|
150
|
+
|
|
151
|
+
shared_counts: dict[tuple[str, str], int] = {}
|
|
152
|
+
for tickets_sharing_file in file_to_tickets.values():
|
|
153
|
+
if len(tickets_sharing_file) < 2:
|
|
154
|
+
continue
|
|
155
|
+
for a, b in combinations(sorted(tickets_sharing_file), 2):
|
|
156
|
+
shared_counts[(a, b)] = shared_counts.get((a, b), 0) + 1
|
|
157
|
+
|
|
158
|
+
scored: dict[tuple[str, str], tuple[int, float, float, list[str]]] = {}
|
|
159
|
+
for (a, b), shared_count in shared_counts.items():
|
|
160
|
+
fa, fb = touched[a], touched[b]
|
|
161
|
+
overlap_score = shared_count / len(fa | fb)
|
|
162
|
+
# Rarest first: the sample a reader sees should lead with the file that
|
|
163
|
+
# actually links these two, not whichever sorts first alphabetically.
|
|
164
|
+
shared = sorted(fa & fb, key=lambda f: (-_idf(f), f))
|
|
165
|
+
weighted = sum(_idf(f) for f in shared) / sum(_idf(f) for f in (fa | fb))
|
|
166
|
+
scored[(a, b)] = (shared_count, overlap_score, weighted, shared)
|
|
167
|
+
# ODSF-314: every pair is still scored, threshold-free, but each ticket keeps only its TOP_NEIGHBOURS best: on OCO
|
|
168
|
+
# the full set was 51 neighbours per ticket (58,914 edges) while every reader used the first 8. The confidence is
|
|
169
|
+
# the weighted score itself, not a constant 1.0 for an inference.
|
|
170
|
+
best: dict[str, list[tuple[float, str]]] = {}
|
|
171
|
+
for (a, b), (_c, _o, weighted, _s) in scored.items():
|
|
172
|
+
best.setdefault(a, []).append((weighted, b))
|
|
173
|
+
best.setdefault(b, []).append((weighted, a))
|
|
174
|
+
keep = {src: {dst for _w, dst in sorted(lst, key=lambda p: (-p[0], p[1]))[:TOP_NEIGHBOURS]} for src, lst in best.items()}
|
|
175
|
+
for (a, b), (shared_count, overlap_score, weighted, shared) in scored.items():
|
|
176
|
+
for src, dst in ((a, b), (b, a)):
|
|
177
|
+
if dst not in keep.get(src, ()):
|
|
178
|
+
continue
|
|
179
|
+
g.add_edge(
|
|
180
|
+
"SIMILAR_TO", "Ticket", src, "Ticket", dst,
|
|
181
|
+
source="file-overlap-inference", confidence=round(weighted, 4),
|
|
182
|
+
evidence_ref=f"{shared_count} shared touched files ({', '.join(shared[:5])}{'...' if shared_count > 5 else ''})",
|
|
183
|
+
extra={
|
|
184
|
+
"shared_file_count": shared_count,
|
|
185
|
+
"overlap_score": round(overlap_score, 4),
|
|
186
|
+
# Ranking key. `overlap_score` stays exactly as it was so
|
|
187
|
+
# anything reading it keeps working; this is the field a
|
|
188
|
+
# consumer should sort on, because it knows the difference
|
|
189
|
+
# between sharing a Makefile and sharing one service.
|
|
190
|
+
"weighted_score": round(weighted, 4),
|
|
191
|
+
},
|
|
192
|
+
)
|
|
193
|
+
return g
|