devkb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. devkb/__init__.py +11 -0
  2. devkb/audit.py +173 -0
  3. devkb/bench/add_packs.py +52 -0
  4. devkb/bench/cli_tasks.py +105 -0
  5. devkb/bench/confirm_tasks.py +149 -0
  6. devkb/bench/module_truth.py +249 -0
  7. devkb/bench/oco_tasks.py +216 -0
  8. devkb/bench/pipeline.py +313 -0
  9. devkb/bench/process_tasks.py +135 -0
  10. devkb/bench/report.py +74 -0
  11. devkb/bench/run.py +174 -0
  12. devkb/bench/truth_check.py +148 -0
  13. devkb/bench/workflow_report.py +64 -0
  14. devkb/bench/workflow_run.py +206 -0
  15. devkb/bench/workflow_tasks.py +227 -0
  16. devkb/build.py +258 -0
  17. devkb/build_symbols.py +1394 -0
  18. devkb/call_recall.py +159 -0
  19. devkb/cli.py +53 -0
  20. devkb/coverage.py +155 -0
  21. devkb/edge_truth.py +272 -0
  22. devkb/extractors/backlog.py +113 -0
  23. devkb/extractors/component_subsystem.py +116 -0
  24. devkb/extractors/configuration.py +267 -0
  25. devkb/extractors/files_tests.py +220 -0
  26. devkb/extractors/flows.py +117 -0
  27. devkb/extractors/lifecycle.py +258 -0
  28. devkb/extractors/members.py +160 -0
  29. devkb/extractors/pull_requests.py +271 -0
  30. devkb/extractors/releases.py +49 -0
  31. devkb/extractors/source_baseline.py +66 -0
  32. devkb/extractors/spec_docs.py +220 -0
  33. devkb/extractors/symbol_tests.py +131 -0
  34. devkb/extractors/test_inventory.py +38 -0
  35. devkb/extractors/ticket_similarity.py +193 -0
  36. devkb/extractors/tickets_git.py +151 -0
  37. devkb/extractors/warnings.py +230 -0
  38. devkb/extractors/work_units.py +90 -0
  39. devkb/github.py +515 -0
  40. devkb/gitutil.py +16 -0
  41. devkb/index.py +51 -0
  42. devkb/init.py +178 -0
  43. devkb/mcp_server.py +216 -0
  44. devkb/model.py +221 -0
  45. devkb/module_file_bridge.py +57 -0
  46. devkb/ontology.md +178 -0
  47. devkb/packs.py +462 -0
  48. devkb/process_api.py +1601 -0
  49. devkb/project_context.py +191 -0
  50. devkb/query.py +251 -0
  51. devkb/service.py +324 -0
  52. devkb/startup.py +103 -0
  53. devkb/test_discovery.py +76 -0
  54. devkb/ts_symbols.py +291 -0
  55. devkb-0.2.0.dist-info/METADATA +162 -0
  56. devkb-0.2.0.dist-info/RECORD +61 -0
  57. devkb-0.2.0.dist-info/WHEEL +5 -0
  58. devkb-0.2.0.dist-info/entry_points.txt +2 -0
  59. devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
  60. devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
  61. devkb-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,220 @@
1
+ """Spec + DOCUMENTS (devkb BL-13, promoted from `devkb/ROADMAP.md`'s own
2
+ Phase 4 step 1 -- KNOWLEDGE_SPEC.md Section 4/5).
3
+
4
+ Deterministic, regex-derived, no LLM call -- the cheap half of Phase 4
5
+ (`devkb/answer.py`'s LLM-synthesis half stays out of scope, Section 9).
6
+ Closes the audit's "no node descriptions from specification documents"
7
+ finding: every ADR/design/onboarding markdown file becomes a real `Spec`
8
+ node, `DOCUMENTS` linking it to any `Ticket` it actually mentions.
9
+
10
+ Needs the FULL merged graph (real `Ticket` nodes from Phase 1) so a
11
+ `DOCUMENTS` edge only ever points at a ticket confirmed to exist -- a
12
+ stale/renamed/typo'd ticket-id mention in an old doc gets skipped, not
13
+ guessed into a dangling edge. Post-merge, same shape as BL-4/BL-6/BL-11.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import hashlib
18
+ import re
19
+
20
+ from devkb.model import DevKBGraph
21
+ from devkb.project_context import DEFAULT_SPEC_DIRS, ProjectContext
22
+
23
+ _HEADING_RE = re.compile(r"^#\s+(.+)$", re.MULTILINE)
24
+ _RELATED_LINE_RE = re.compile(r"^\*?\*?Related.*?:\*?\*?\s*(.+)$", re.MULTILINE | re.IGNORECASE)
25
+
26
+ # The two closed vocabularies ADR-SDP-SPEC-001 Section 3 defines. Closed on
27
+ # purpose: an open one is how the sibling OCO checkout ended up carrying
28
+ # `Accepted`, `Proposed`, `PROPOSED`, `DRAFT` and nine lines of prose in the
29
+ # same field. A declared value outside these sets is not honoured -- it falls
30
+ # back to the path rule below rather than inventing a fifth state.
31
+ _DOC_TYPES = frozenset({"specification", "adr", "design", "onboarding", "index"})
32
+ _STATUSES = frozenset({"draft", "review", "accepted", "superseded"})
33
+
34
+ # The front-matter block, only when it opens the file. The schema is
35
+ # deliberately flat -- scalars, `null`, and inline lists -- so this is parsed
36
+ # here rather than with PyYAML: yaml is present in this dev environment but
37
+ # absent from `pms-backend/requirements.txt`, and `scripts/deployment.sh`
38
+ # installs a fresh clone from that file. An extractor that works on this
39
+ # machine and dies on a clean checkout is worse than one with no front-matter.
40
+ _FRONT_MATTER_RE = re.compile(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*\r?\n", re.DOTALL)
41
+ _FM_KEY_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*):[ \t]*(.*)$")
42
+ # A `#` opens a comment only at the start of the value or after whitespace,
43
+ # so a title containing one (`# 3 of 9 journeys`) survives.
44
+ _FM_COMMENT_RE = re.compile(r"(?:^|\s)#.*$")
45
+
46
+
47
+ def _scalar(raw: str) -> str | list[str] | None:
48
+ value = raw.strip()
49
+ # A quoted scalar is taken whole: inside quotes a `#` is literal, not a
50
+ # comment. Stripping comments first truncated any title that contained one
51
+ # ("Only 1 of 9 journeys reach #done" became "Only 1 of 9 journeys reach").
52
+ if value[:1] in ("'", '"'):
53
+ closing = value.find(value[0], 1)
54
+ if closing != -1:
55
+ return value[1:closing]
56
+ value = _FM_COMMENT_RE.sub("", value).strip()
57
+ if value in ("", "null", "~"):
58
+ return None
59
+ if value.startswith("[") and value.endswith("]"):
60
+ inner = value[1:-1].strip()
61
+ return [p.strip().strip("'\"") for p in inner.split(",") if p.strip()] if inner else []
62
+ return value.strip("'\"")
63
+
64
+
65
+ def parse_front_matter(text: str) -> tuple[dict[str, str | list[str] | None], str]:
66
+ """Split a document into its declared front-matter and its body.
67
+
68
+ Returns `({}, text)` unchanged when there is no block, which is the common
69
+ case and must stay the cheap one: most of the corpus will carry no
70
+ front-matter for a long time, and such a document has to remain a
71
+ first-class member rather than quietly dropping out of the index.
72
+
73
+ Only top-level `key: value` lines are read. Anything indented belongs to a
74
+ nested structure this schema does not have, and is skipped rather than
75
+ guessed at.
76
+ """
77
+ match = _FRONT_MATTER_RE.match(text)
78
+ if not match:
79
+ return {}, text
80
+ fields: dict[str, str | list[str] | None] = {}
81
+ for line in match.group(1).splitlines():
82
+ if not line.strip() or line[:1] in (" ", "\t", "#", "-"):
83
+ continue
84
+ key_match = _FM_KEY_RE.match(line)
85
+ if key_match:
86
+ fields[key_match.group(1).strip().lower()] = _scalar(key_match.group(2))
87
+ return fields, text[match.end():]
88
+
89
+
90
+ def _declared(fields: dict[str, str | list[str] | None], key: str, allowed: frozenset[str]) -> str:
91
+ value = fields.get(key)
92
+ if isinstance(value, str) and value.strip().lower() in allowed:
93
+ return value.strip().lower()
94
+ return ""
95
+
96
+
97
+ def _doc_type(rel_path: str, fields: dict[str, str | list[str] | None] | None = None) -> str:
98
+ """What a document *is*, preferring its own declaration over its location.
99
+
100
+ ODSF-128: the path rule below is a fallthrough, not a classification --
101
+ `design` is what anything outside `docs/` and `.onboarding/` gets by
102
+ default. That made `backlogs/PMS-NEXT-ERA-SPECIFICATION.md`, this project's
103
+ actual product specification, sort last in the BA's injection ranking and
104
+ be the first thing dropped at the cap. A document can now say what it is.
105
+
106
+ The fallthrough is kept exactly as it was for every document that declares
107
+ nothing, so this change cannot shrink an existing corpus.
108
+ """
109
+ declared = _declared(fields or {}, "type", _DOC_TYPES)
110
+ if declared:
111
+ return declared
112
+ if rel_path.startswith("docs/adr/"):
113
+ return "adr"
114
+ if rel_path.startswith(".onboarding/"):
115
+ return "onboarding"
116
+ if rel_path.startswith("docs/"):
117
+ # A top-level `docs/*.md` is the project's read-facing specification
118
+ # set -- requirement/technical/test documents and the cross-cutting
119
+ # references beside them. Typed apart from `design` because they are a
120
+ # different altitude: `design/` is per-screen and per-sprint working
121
+ # material, these consolidate it. Both are Spec nodes; only the
122
+ # `doc_type` property distinguishes them, so this adds no node type and
123
+ # no edge verb (ADR-SDP-DEVKB-001's schema gate does not apply).
124
+ return "specification"
125
+ return "design"
126
+
127
+
128
+ def _spec_id(rel_path: str, fields: dict[str, str | list[str] | None]) -> str:
129
+ """The id a ticket's `## Trace` `Specification` row names this document by
130
+ (ODSF-230, ADR-SDP-DEVKB-002 §3 `IMPLEMENTS`).
131
+
132
+ The front-matter `id` when the document declares one (`SPEC-ODSF-003`),
133
+ else the filename stem before its first `_` -- the ADR-SDP-SPEC-001 naming
134
+ convention (`SPEC-ODSF-003_<slug>.md`, `ADR-SDP-DEVKB-002_<slug>.md`). A
135
+ property, not a key: the natural key stays the path, so no existing uid moves.
136
+ """
137
+ declared = fields.get("id")
138
+ if isinstance(declared, str) and declared.strip():
139
+ return declared.strip()
140
+ return rel_path.rsplit("/", 1)[-1].removesuffix(".md").split("_", 1)[0]
141
+
142
+
143
+ def _title(text: str) -> str | None:
144
+ m = _HEADING_RE.search(text)
145
+ return m.group(1).strip() if m else None
146
+
147
+
148
+ def _spec_files(ctx: ProjectContext) -> list[str]:
149
+ """The markdown this project calls its specifications.
150
+
151
+ ODSF-102: the roots come from the project rather than from this file. They
152
+ used to be three hardcoded globs, which meant a project keeping its specs
153
+ anywhere else was invisible to its own BA -- ODSF's whole specification set
154
+ is under `backlogs/`, so its graph held 2 Spec nodes while OCO, whose layout
155
+ happens to match those globs, held 187.
156
+
157
+ A root ending in `/**` is scanned recursively; without it, top level only.
158
+ That distinction is not decoration: `design` was recursive and `docs/adr`
159
+ deliberately was not (its siblings under `docs/` are per-owner working
160
+ areas, not specifications of the system), and a project that declares
161
+ nothing must behave exactly as it did before.
162
+
163
+ A declared root that does not exist is skipped, so a config naming a
164
+ directory the project has not created yet does not break the build.
165
+ """
166
+ out: set[str] = set()
167
+ for root in (ctx.spec_dirs or DEFAULT_SPEC_DIRS):
168
+ recursive = root.endswith("/**")
169
+ base = ctx.repo_root / (root[:-3] if recursive else root)
170
+ if not base.is_dir():
171
+ continue
172
+ for path in (base.rglob("*.md") if recursive else base.glob("*.md")):
173
+ if path.is_file():
174
+ out.add(str(path.relative_to(ctx.repo_root)))
175
+ return sorted(out)
176
+
177
+
178
+ def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
179
+ g = DevKBGraph()
180
+ real_tickets = {t.natural_key for t in graph.nodes("Ticket")}
181
+ ticket_re = re.compile(rf"{re.escape(ctx.ticket_prefix)}-\d+")
182
+
183
+ for rel_path in _spec_files(ctx):
184
+ try:
185
+ text = (ctx.repo_root / rel_path).read_text(errors="replace")
186
+ except OSError:
187
+ continue
188
+
189
+ fields, body = parse_front_matter(text)
190
+ title = _title(text) or rel_path
191
+ # Hashed over the BODY, not the whole file, so that editing front-matter
192
+ # -- appending a ticket id to `tickets:`, bumping `updated:` -- is not
193
+ # reported as the document's content having drifted. Those edits are
194
+ # bookkeeping about the document; only the body is the document.
195
+ content_sha = hashlib.sha256(body.encode("utf-8", "replace")).hexdigest()[:16]
196
+ g.add_node(
197
+ "Spec", rel_path, source="spec-scan", title=title,
198
+ doc_type=_doc_type(rel_path, fields),
199
+ # "" means the document declares no status, which is not the same
200
+ # as declaring `draft`. Only an explicit `superseded` withdraws a
201
+ # document from the BA's corpus, so silence never excludes anything.
202
+ status=_declared(fields, "status", _STATUSES),
203
+ content_sha=content_sha,
204
+ spec_id=_spec_id(rel_path, fields),
205
+ )
206
+
207
+ mentioned: set[str] = set()
208
+ for related_line in _RELATED_LINE_RE.findall(text):
209
+ mentioned.update(ticket_re.findall(related_line))
210
+ mentioned.update(ticket_re.findall(text))
211
+
212
+ for ticket_id in mentioned:
213
+ if ticket_id not in real_tickets:
214
+ continue # a stale/typo'd/renamed mention -- skip, don't guess
215
+ g.add_edge(
216
+ "DOCUMENTS", "Spec", rel_path, "Ticket", ticket_id,
217
+ source="spec-scan", confidence=0.85,
218
+ evidence_ref=f"{rel_path} mentions {ticket_id}",
219
+ )
220
+ return g
@@ -0,0 +1,131 @@
1
+ """Content-based test linkage (devkb BL-4, body/legs model).
2
+
3
+ Phase 2's TESTED_BY needs a ticket ID literally mentioned near the test --
4
+ doubly ticket-dependent (a ticket to find the commit, a ticket mention in
5
+ the test). This resolves real evidence instead: a test function's own
6
+ imports matched against Symbol natural keys (BL-3), scoped to calls that
7
+ function's own body actually makes -- not "the file imports it somewhere,"
8
+ which would just move Phase 2's coarseness from ticket-level to file-level
9
+ instead of removing it. Additive alongside Phase 2's existing
10
+ 0.7-confidence ticket-mediated edges, never a replacement.
11
+
12
+ Needs BL-3's Symbol nodes already in `graph` (build.py merges the symbols
13
+ cache before calling this) -- an empty/missing Symbol set just means zero
14
+ edges here, not an error, same "can't check yet" framing used elsewhere in
15
+ devkb for a missing input.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import ast
20
+
21
+ from devkb.build_symbols import (
22
+ EXPR, _calls_in_body, _import_descriptors, _import_targets, _imports, _module_paths, _py_literals,
23
+ _list_python_files, _resolve_cross_file_symbol, _resolve_via_literals, collect_reexports, register_reexports,
24
+ )
25
+ from devkb.model import DevKBGraph
26
+ from devkb.project_context import ProjectContext
27
+ from devkb.test_discovery import list_test_files
28
+
29
+
30
+ def _test_functions(tree: ast.AST) -> list[tuple[str, int, set[str], set[tuple[str | None, str]]]]:
31
+ """[(name, lineno, {short names this function's own body calls}), ...]
32
+ -- reuses BL-3's own body-scoped call scanner (build_symbols.py) so a
33
+ nested helper's calls don't get misattributed to the outer test. Only
34
+ the short callee name is used here (not _calls_in_body's receiver,
35
+ which BL-3b's own cross-file CALLS resolution needs but this doesn't --
36
+ a test's own import-name match is already the precise signal)."""
37
+ out: list[tuple[str, int, set[str], set[tuple[str | None, str]]]] = []
38
+ for node in ast.walk(tree):
39
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"):
40
+ pairs = set(_calls_in_body(node))
41
+ out.append((node.name, node.lineno, {short for _receiver, short in pairs}, pairs))
42
+ return out
43
+
44
+
45
+ def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
46
+ g = DevKBGraph()
47
+ by_short_name: dict[str, list[str]] = {}
48
+ for s in graph.nodes("Symbol"):
49
+ short = s.natural_key.rsplit("::", 1)[-1].rsplit(".", 1)[-1]
50
+ by_short_name.setdefault(short, []).append(s.natural_key)
51
+ if not by_short_name:
52
+ return g
53
+ # path -> [{"qualified_name": ...}], the shape _resolve_cross_file_symbol reads (ODSF-305).
54
+ symbols_by_file: dict[str, list[dict]] = {}
55
+ for s in graph.nodes("Symbol"):
56
+ path, _, qualified = s.natural_key.partition("::")
57
+ symbols_by_file.setdefault(path, []).append({"qualified_name": qualified})
58
+ # `from pkg import f` where `pkg/__init__.py` re-exports f from a submodule
59
+ register_reexports(symbols_by_file, collect_reexports(ctx.repo_root, _list_python_files(ctx.repo_root)))
60
+
61
+ # ODSF-304: every tracked python test file, wherever it lives.
62
+ for py in (ctx.repo_root / rel for rel in list_test_files(ctx.repo_root, ctx.tests_dir) if rel.endswith(".py")):
63
+ try:
64
+ text = py.read_text(errors="replace")
65
+ tree = ast.parse(text, filename=str(py))
66
+ except (SyntaxError, OSError, ValueError):
67
+ continue
68
+ rel = str(py.relative_to(ctx.repo_root))
69
+ imported = _imports(tree)
70
+ tests = _test_functions(tree)
71
+ if not tests:
72
+ continue
73
+
74
+ receivers, bare = _import_targets(_import_descriptors(tree), rel)
75
+ literals = _py_literals(tree)
76
+ for name, lineno, calls, pairs in tests:
77
+ resolved: set[str] = set()
78
+ # ODSF-305: `module.func()` and aliased or relative imports.
79
+ for receiver, short in pairs:
80
+ if receiver and "." in receiver and receiver != EXPR: # `pkg.mod.f()` after `import pkg.mod`
81
+ root, rest = receiver.split(".", 1)
82
+ modules = [f"{m}.{rest}" for m in receivers.get(root, [])]
83
+ hit = next((h for h in (_resolve_cross_file_symbol(m, rel, short, symbols_by_file) for m in modules) if h), None)
84
+ if hit:
85
+ resolved.add(hit)
86
+ continue
87
+ if receiver:
88
+ modules = receivers.get(receiver) or ([receiver] if receiver.isidentifier() else [])
89
+ hit = None
90
+ for module in modules:
91
+ hit = _resolve_cross_file_symbol(module, rel, short, symbols_by_file)
92
+ if hit:
93
+ break
94
+ if hit is None and receiver not in receivers:
95
+ hit = _resolve_via_literals(literals, short, symbols_by_file)
96
+ if hit:
97
+ resolved.add(hit)
98
+ elif short in bare:
99
+ module, original = bare[short]
100
+ hit = _resolve_cross_file_symbol(module, rel, original, symbols_by_file)
101
+ if hit:
102
+ resolved.add(hit)
103
+ for local_name, dotted_module in imported:
104
+ if local_name not in calls:
105
+ continue
106
+ candidates = by_short_name.get(local_name, [])
107
+ if not candidates:
108
+ continue
109
+ target = None
110
+ if dotted_module:
111
+ for guess in _module_paths(dotted_module):
112
+ hit = next((c for c in candidates if c.startswith(f"{guess}::")), None)
113
+ if hit:
114
+ target = hit
115
+ break
116
+ if target is None and len(candidates) == 1:
117
+ target = candidates[0]
118
+ if target:
119
+ resolved.add(target)
120
+
121
+ if not resolved:
122
+ continue
123
+ test_key = f"{rel}:{lineno}"
124
+ g.add_node("TestCase", test_key, source="ast-import-resolution", file=rel, line=lineno, function=name)
125
+ for symbol_key in resolved:
126
+ g.add_edge(
127
+ "TESTED_BY", "Symbol", symbol_key, "TestCase", test_key,
128
+ source="ast-import-resolution", confidence=0.95,
129
+ evidence_ref=f"{name} (line {lineno}) imports and calls a name resolving to {symbol_key}",
130
+ )
131
+ return g
@@ -0,0 +1,38 @@
1
+ """TestCase for every test function, not only those that name a ticket (ODSF-304).
2
+
3
+ `files_tests.py` creates a TestCase only where a ticket id sits near a test, so
4
+ a project whose tests never mention tickets (ODSF's own, and nearly all of
5
+ another's) had almost no TestCase nodes at all. This inventories every test
6
+ function that `devkb.test_discovery` finds and links it to its file with
7
+ DEFINED_IN (ADR-SDP-DEVKB-003).
8
+
9
+ Node identity is `path:line`, the same key `files_tests.py` uses, so a test
10
+ that is both inventoried and ticket-marked is one node carrying both sets of
11
+ properties (DevKBGraph merges props on a repeated key).
12
+ """
13
+ from __future__ import annotations
14
+
15
+ from devkb.model import DevKBGraph
16
+ from devkb.project_context import ProjectContext
17
+ from devkb.test_discovery import list_test_files, tests_in_file
18
+
19
+
20
+ def extract(ctx: ProjectContext) -> DevKBGraph:
21
+ g = DevKBGraph()
22
+ for rel in list_test_files(ctx.repo_root, ctx.tests_dir):
23
+ framework = "pytest" if rel.endswith(".py") else "js"
24
+ cases = tests_in_file(ctx.repo_root, rel)
25
+ if not cases:
26
+ continue
27
+ g.add_node("File", rel, source="test-inventory")
28
+ for name, line in cases:
29
+ key = f"{rel}:{line}"
30
+ g.add_node(
31
+ "TestCase", key, source="test-inventory",
32
+ file=rel, line=line, function=name, framework=framework,
33
+ )
34
+ g.add_edge(
35
+ "DEFINED_IN", "TestCase", key, "File", rel,
36
+ source="test-inventory", confidence=1.0, evidence_ref=f"{rel}:{line}",
37
+ )
38
+ return g
@@ -0,0 +1,193 @@
1
+ """Threshold-free ticket similarity (devkb BL-11, KNOWLEDGE_SPEC.md Section
2
+ 5, resolution of Open question 4).
3
+
4
+ The original draft gated SIMILAR_TO behind a build-time overlap threshold
5
+ ("what fraction counts as similar enough to surface") -- dropped per
6
+ direct pushback: a guessed constant baked into the graph build is the
7
+ wrong place to make that call. This creates the edge for ANY nonzero
8
+ touched-file overlap between two tickets' own `REFERENCES`-linked commits'
9
+ `TOUCHES`-linked files, and stores the real overlap
10
+ (`shared_file_count`/`overlap_score`) as edge data -- filtering ("show me
11
+ only the top 3" / "only above 0.3") moves to query time, the same shape
12
+ `list_nodes()`'s own `q` search already uses (rank at the edge, don't
13
+ gate at the build).
14
+
15
+ Needs the FULL merged graph (Ticket/Commit/File nodes, REFERENCES/TOUCHES
16
+ edges from Phase 1/BL-1, already shipped) -- a post-merge derivation pass,
17
+ not a new source, so it runs from `graph` itself like BL-4/BL-6, not as a
18
+ `graphs[]` entry in build.py.
19
+ """
20
+ from __future__ import annotations
21
+
22
+ from itertools import combinations
23
+ from math import log
24
+
25
+ from devkb.model import DevKBGraph
26
+
27
+ #: ODSF-314: neighbours kept per ticket; process_api's index keeps 8, so more is never read.
28
+ TOP_NEIGHBOURS = 8
29
+ from devkb.project_context import ProjectContext
30
+
31
+ # A heartbeat/status file touched by nearly every commit regardless of what
32
+ # the commit is actually about carries ZERO discriminating signal for "are
33
+ # these two tickets related" -- this is a data-quality exclusion (which
34
+ # files can even be evidence), not the build-time overlap-score THRESHOLD
35
+ # that was explicitly rejected (Section 5's resolution, Open question 4) --
36
+ # that rejection was about gating which edges get SHOWN based on a guessed
37
+ # significance cutoff; this is about which files get COUNTED at all.
38
+ # Confirmed live, not theorized: on OCO's own real history, sdp/orchestrator/
39
+ # status/*.md files are touched by 887-946 of 2,614 commits (34-36%) -- CM.md
40
+ # alone by 946 -- an extreme structural outlier next to any real functional
41
+ # file, already recognized elsewhere in this codebase's own tooling as a
42
+ # distinct "status file heartbeat" category (see gate-1's own check of that
43
+ # name). Path-based, same discipline as every other allow/exclude-list in
44
+ # this spec, extensible if a different project's own hub files show up.
45
+ _HUB_FILE_PREFIXES = ("sdp/orchestrator/status/",)
46
+
47
+ #: A file touched by more than this share of the tickets that touch anything
48
+ #: is bookkeeping, not subject matter, and two tickets sharing it are not
49
+ #: related in any sense a reader cares about.
50
+ #:
51
+ #: This is the SAME judgement as the prefix list above — which files may serve
52
+ #: as evidence at all — not the build-time overlap threshold Section 5 rejected.
53
+ #: That rejection was about gating which edges get SHOWN by a guessed
54
+ #: significance cutoff; this is about which files can be evidence, and the
55
+ #: prefix list's own comment invited it ("extensible if a different project's
56
+ #: own hub files show up"). They showed up, and hand-maintaining the list did
57
+ #: not scale: measured on OCO's devkb at 2026-08-07, four files nobody had
58
+ #: listed produced 161,894 of the 178,114 single-file SIMILAR_TO edges — 91% —
59
+ #: led by `tests/test_oco_ticket_coverage.py`, a test asserting that every
60
+ #: ticket has coverage and therefore touched by nearly every ticket's commits.
61
+ #: It alone accounted for 46% of the entire relation.
62
+ #:
63
+ #: 0.10 sits an order of magnitude below the status files the prefix list was
64
+ #: written for (34-36% of commits) and above the busiest genuinely functional
65
+ #: file measured on the same data.
66
+ _HUB_FILE_SHARE = 0.10
67
+
68
+ #: Below this many tickets the share above is sampling noise, not a property of
69
+ #: the repository — on five tickets one of them IS 20%, and on the two-ticket
70
+ #: case any shared file at all reads as 100% and the relation empties itself.
71
+ #: A statistical exclusion needs a population to be computed over; under it the
72
+ #: prefix list still applies and nothing else is disqualified.
73
+ _HUB_MIN_TICKETS = 30
74
+
75
+
76
+ def _is_hub_file(path: str) -> bool:
77
+ return any(path.startswith(p) for p in _HUB_FILE_PREFIXES)
78
+
79
+
80
+ def hub_files(touched: dict[str, set[str]], share: float = _HUB_FILE_SHARE) -> dict[str, float]:
81
+ """Files touched by too many tickets to discriminate between any two.
82
+
83
+ Returned rather than merely applied so the build can record WHICH files
84
+ were disqualified: an exclusion nobody can see is one nobody can argue
85
+ with, and this one is load-bearing enough to be worth arguing with.
86
+ """
87
+ if len(touched) < _HUB_MIN_TICKETS:
88
+ return {}
89
+ df: dict[str, int] = {}
90
+ for files in touched.values():
91
+ for f in files:
92
+ df[f] = df.get(f, 0) + 1
93
+ total = len(touched)
94
+ return {f: n / total for f, n in df.items() if n / total > share}
95
+
96
+
97
+ def _touched_files(graph: DevKBGraph, ticket_key: str) -> set[str]:
98
+ commits = {e.src for e in graph.edges("REFERENCES") if e.dst == ticket_key}
99
+ if not commits:
100
+ return set()
101
+ return {e.dst for e in graph.edges("TOUCHES") if e.src in commits and not _is_hub_file(e.dst)}
102
+
103
+
104
+ def extract(ctx: ProjectContext, graph: DevKBGraph) -> DevKBGraph:
105
+ g = DevKBGraph()
106
+ tickets = [t.natural_key for t in graph.nodes("Ticket")]
107
+ if len(tickets) < 2:
108
+ return g
109
+
110
+ touched = {key: _touched_files(graph, key) for key in tickets}
111
+ touched = {k: v for k, v in touched.items() if v} # a ticket with no touched files can't overlap
112
+
113
+ # Second exclusion pass, computed rather than listed. Runs here and not in
114
+ # `_touched_files` because document frequency is a property of the whole
115
+ # ticket set, which that function cannot see one ticket at a time.
116
+ hubs = hub_files(touched)
117
+ if hubs:
118
+ touched = {k: (v - hubs.keys()) for k, v in touched.items()}
119
+ touched = {k: v for k, v in touched.items() if v}
120
+ print(f" ticket_similarity: {len(hubs)} hub file(s) excluded as evidence "
121
+ f"(touched by >{_HUB_FILE_SHARE:.0%} of tickets): "
122
+ + ", ".join(f"{f} ({s:.0%})" for f, s in
123
+ sorted(hubs.items(), key=lambda kv: -kv[1])[:5])
124
+ + ("..." if len(hubs) > 5 else ""))
125
+ if len(touched) < 2:
126
+ return g
127
+
128
+ # Inverse document frequency over the surviving files. A file two tickets
129
+ # share is only evidence in proportion to how few OTHER tickets share it;
130
+ # weighting by it makes the hub problem self-limiting rather than dependent
131
+ # on the cutoff above catching every case.
132
+ total_tickets = len(touched)
133
+ doc_freq: dict[str, int] = {}
134
+ for files in touched.values():
135
+ for f in files:
136
+ doc_freq[f] = doc_freq.get(f, 0) + 1
137
+
138
+ def _idf(path: str) -> float:
139
+ return log(total_tickets / doc_freq.get(path, 1)) + 1.0
140
+
141
+ # Inverted index (file -> tickets that touched it): only ticket pairs
142
+ # sharing at least one real file are ever compared. At real scale
143
+ # (confirmed live: 1,250 real tickets on OCO) most pairs share nothing
144
+ # -- a full O(n^2) pairwise scan is wasted work compared over an index
145
+ # built from the actual touch data.
146
+ file_to_tickets: dict[str, set[str]] = {}
147
+ for ticket, files in touched.items():
148
+ for f in files:
149
+ file_to_tickets.setdefault(f, set()).add(ticket)
150
+
151
+ shared_counts: dict[tuple[str, str], int] = {}
152
+ for tickets_sharing_file in file_to_tickets.values():
153
+ if len(tickets_sharing_file) < 2:
154
+ continue
155
+ for a, b in combinations(sorted(tickets_sharing_file), 2):
156
+ shared_counts[(a, b)] = shared_counts.get((a, b), 0) + 1
157
+
158
+ scored: dict[tuple[str, str], tuple[int, float, float, list[str]]] = {}
159
+ for (a, b), shared_count in shared_counts.items():
160
+ fa, fb = touched[a], touched[b]
161
+ overlap_score = shared_count / len(fa | fb)
162
+ # Rarest first: the sample a reader sees should lead with the file that
163
+ # actually links these two, not whichever sorts first alphabetically.
164
+ shared = sorted(fa & fb, key=lambda f: (-_idf(f), f))
165
+ weighted = sum(_idf(f) for f in shared) / sum(_idf(f) for f in (fa | fb))
166
+ scored[(a, b)] = (shared_count, overlap_score, weighted, shared)
167
+ # ODSF-314: every pair is still scored, threshold-free, but each ticket keeps only its TOP_NEIGHBOURS best: on OCO
168
+ # the full set was 51 neighbours per ticket (58,914 edges) while every reader used the first 8. The confidence is
169
+ # the weighted score itself, not a constant 1.0 for an inference.
170
+ best: dict[str, list[tuple[float, str]]] = {}
171
+ for (a, b), (_c, _o, weighted, _s) in scored.items():
172
+ best.setdefault(a, []).append((weighted, b))
173
+ best.setdefault(b, []).append((weighted, a))
174
+ keep = {src: {dst for _w, dst in sorted(lst, key=lambda p: (-p[0], p[1]))[:TOP_NEIGHBOURS]} for src, lst in best.items()}
175
+ for (a, b), (shared_count, overlap_score, weighted, shared) in scored.items():
176
+ for src, dst in ((a, b), (b, a)):
177
+ if dst not in keep.get(src, ()):
178
+ continue
179
+ g.add_edge(
180
+ "SIMILAR_TO", "Ticket", src, "Ticket", dst,
181
+ source="file-overlap-inference", confidence=round(weighted, 4),
182
+ evidence_ref=f"{shared_count} shared touched files ({', '.join(shared[:5])}{'...' if shared_count > 5 else ''})",
183
+ extra={
184
+ "shared_file_count": shared_count,
185
+ "overlap_score": round(overlap_score, 4),
186
+ # Ranking key. `overlap_score` stays exactly as it was so
187
+ # anything reading it keeps working; this is the field a
188
+ # consumer should sort on, because it knows the difference
189
+ # between sharing a Makefile and sharing one service.
190
+ "weighted_score": round(weighted, 4),
191
+ },
192
+ )
193
+ return g