devkb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. devkb/__init__.py +11 -0
  2. devkb/audit.py +173 -0
  3. devkb/bench/add_packs.py +52 -0
  4. devkb/bench/cli_tasks.py +105 -0
  5. devkb/bench/confirm_tasks.py +149 -0
  6. devkb/bench/module_truth.py +249 -0
  7. devkb/bench/oco_tasks.py +216 -0
  8. devkb/bench/pipeline.py +313 -0
  9. devkb/bench/process_tasks.py +135 -0
  10. devkb/bench/report.py +74 -0
  11. devkb/bench/run.py +174 -0
  12. devkb/bench/truth_check.py +148 -0
  13. devkb/bench/workflow_report.py +64 -0
  14. devkb/bench/workflow_run.py +206 -0
  15. devkb/bench/workflow_tasks.py +227 -0
  16. devkb/build.py +258 -0
  17. devkb/build_symbols.py +1394 -0
  18. devkb/call_recall.py +159 -0
  19. devkb/cli.py +53 -0
  20. devkb/coverage.py +155 -0
  21. devkb/edge_truth.py +272 -0
  22. devkb/extractors/backlog.py +113 -0
  23. devkb/extractors/component_subsystem.py +116 -0
  24. devkb/extractors/configuration.py +267 -0
  25. devkb/extractors/files_tests.py +220 -0
  26. devkb/extractors/flows.py +117 -0
  27. devkb/extractors/lifecycle.py +258 -0
  28. devkb/extractors/members.py +160 -0
  29. devkb/extractors/pull_requests.py +271 -0
  30. devkb/extractors/releases.py +49 -0
  31. devkb/extractors/source_baseline.py +66 -0
  32. devkb/extractors/spec_docs.py +220 -0
  33. devkb/extractors/symbol_tests.py +131 -0
  34. devkb/extractors/test_inventory.py +38 -0
  35. devkb/extractors/ticket_similarity.py +193 -0
  36. devkb/extractors/tickets_git.py +151 -0
  37. devkb/extractors/warnings.py +230 -0
  38. devkb/extractors/work_units.py +90 -0
  39. devkb/github.py +515 -0
  40. devkb/gitutil.py +16 -0
  41. devkb/index.py +51 -0
  42. devkb/init.py +178 -0
  43. devkb/mcp_server.py +216 -0
  44. devkb/model.py +221 -0
  45. devkb/module_file_bridge.py +57 -0
  46. devkb/ontology.md +178 -0
  47. devkb/packs.py +462 -0
  48. devkb/process_api.py +1601 -0
  49. devkb/project_context.py +191 -0
  50. devkb/query.py +251 -0
  51. devkb/service.py +324 -0
  52. devkb/startup.py +103 -0
  53. devkb/test_discovery.py +76 -0
  54. devkb/ts_symbols.py +291 -0
  55. devkb-0.2.0.dist-info/METADATA +162 -0
  56. devkb-0.2.0.dist-info/RECORD +61 -0
  57. devkb-0.2.0.dist-info/WHEEL +5 -0
  58. devkb-0.2.0.dist-info/entry_points.txt +2 -0
  59. devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
  60. devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
  61. devkb-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,160 @@
1
+ """A product's graph spans its member repositories (ADR-SDP-DEVKB-005 §1-3, ODSF-312).
2
+
3
+ A product declares members in its root pms.config.json, `"graph": {"members": [{"code": "LIB", "role": "client"}]}`.
4
+ Each member is a registered project with its own graph, built by its own build with its own config; this module
5
+ reads that graph and never runs a member's extractors with the product's rules (the ODSF-104 lesson).
6
+
7
+ Repository one per repository, the product's own included; natural key the project code
8
+ IN_REPO File -> Repository, for every File of the product and of each member
9
+ prefixing a member node's natural key becomes `<CODE>:<key>`, except ConfigKey (the same setting name in two
10
+ repositories is one setting) and Repository; the product's own keys never change
11
+ CALLS_API member Symbol -> product route-handler Symbol, when a URL the member's code uses has the path of a
12
+ product Endpoint (confidence 0.6: a path match, the HTTP method is not known statically)
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import ast
17
+ import json
18
+ import re
19
+ import subprocess
20
+ from pathlib import Path
21
+ from urllib.parse import urlparse
22
+
23
+ from devkb.model import DevKBGraph
24
+ from devkb.project_context import load_config
25
+
26
+ SOURCE = "members"
27
+ UNPREFIXED = {"ConfigKey", "Repository"}
28
+
29
+
30
+ def declared_members(repo_root: Path) -> list[dict]:
31
+ try:
32
+ cfg = load_config(repo_root)
33
+ except (OSError, ValueError):
34
+ return []
35
+ members = ((cfg.get("graph") or {}).get("members") or []) if isinstance(cfg, dict) else []
36
+ return [m for m in members if isinstance(m, dict) and m.get("code")]
37
+
38
+
39
+ def _load(devkb_dir: Path) -> tuple[list[dict], list[dict]]:
40
+ nodes_p, edges_p = devkb_dir / "nodes.jsonl", devkb_dir / "edges.jsonl"
41
+ if not nodes_p.is_file() or not edges_p.is_file():
42
+ raise ValueError(f"declared member graph not built: {devkb_dir} has no nodes.jsonl/edges.jsonl "
43
+ f"(build the member first: make devkb-build PROJECT=<CODE>)")
44
+ nodes = [json.loads(l) for l in nodes_p.read_text().splitlines() if l.strip()]
45
+ edges = [json.loads(l) for l in edges_p.read_text().splitlines() if l.strip()]
46
+ return nodes, edges
47
+
48
+
49
+ def _prefixed(code: str, typ: str, key: str) -> str:
50
+ return key if typ in UNPREFIXED else f"{code}:{key}"
51
+
52
+
53
+ # ── URLs a member's code uses ────────────────────────────────────────────────
54
+ def _url_paths(node: ast.AST) -> list[str]:
55
+ out = []
56
+ for n in ast.walk(node):
57
+ if isinstance(n, ast.Constant) and isinstance(n.value, str) and n.value.startswith(("http://", "https://")):
58
+ path = urlparse(n.value).path
59
+ if path and path != "/":
60
+ out.append(path.rstrip("/"))
61
+ return out
62
+
63
+
64
+ def url_uses(text: str) -> list[tuple[str, str]]:
65
+ """(qualified function name, URL path) for every URL a function uses, directly or through a module constant."""
66
+ try:
67
+ tree = ast.parse(text)
68
+ except (SyntaxError, ValueError, RecursionError):
69
+ return []
70
+ constants: dict[str, list[str]] = {}
71
+ for stmt in tree.body:
72
+ if isinstance(stmt, ast.Assign):
73
+ paths = _url_paths(stmt.value)
74
+ for t in stmt.targets:
75
+ if isinstance(t, ast.Name) and paths:
76
+ constants[t.id] = paths
77
+ found: list[tuple[str, str]] = []
78
+
79
+ def walk(node, stack):
80
+ for child in ast.iter_child_nodes(node):
81
+ if isinstance(child, ast.ClassDef):
82
+ walk(child, stack + [child.name])
83
+ elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
84
+ qual = ".".join(stack + [child.name])
85
+ body_paths = set()
86
+ for n in ast.walk(child):
87
+ if isinstance(n, ast.Name) and n.id in constants:
88
+ body_paths.update(constants[n.id])
89
+ body_paths.update(_url_paths(child))
90
+ found.extend((qual, p) for p in sorted(body_paths))
91
+ walk(child, stack + [child.name])
92
+ walk(tree, [])
93
+ return found
94
+
95
+
96
+ def _route_matcher(graph: DevKBGraph) -> list[tuple[re.Pattern, str, str]]:
97
+ out = []
98
+ for ep in graph.nodes("Endpoint"):
99
+ path, owner = ep.props.get("path") or ep.natural_key.split(" ", 1)[-1], ep.props.get("owner_symbol")
100
+ if not owner or not path.startswith("/"):
101
+ continue
102
+ rx = re.compile("^" + re.sub(r"\\\{[^}]+\\\}|\\<[^>]+\\>", "[^/]+", re.escape(path.rstrip("/"))) + "$")
103
+ out.append((rx, owner, ep.natural_key))
104
+ return out
105
+
106
+
107
+ def link_repository(graph: DevKBGraph, code: str, role: str = "product", only_prefix: str | None = None) -> None:
108
+ """A Repository node, and IN_REPO from each File of a member (keys prefixed `<CODE>:`). The product's own files
109
+ get no edge (ODSF-314): an unprefixed key already says which repository it is in, and on OCO that was 23,240
110
+ edges repeating it."""
111
+ graph.add_node("Repository", code, source=SOURCE, role=role)
112
+ if only_prefix is None:
113
+ return
114
+ for key in [n.natural_key for n in graph.nodes("File")]:
115
+ if key.startswith(only_prefix):
116
+ graph.add_edge("IN_REPO", "File", key, "Repository", code, source=SOURCE)
117
+
118
+
119
+ def merge_members(ctx, graph: DevKBGraph, resolve) -> list[dict]:
120
+ """Fold every declared member's graph into `graph`. Returns one summary row per member."""
121
+ summary = []
122
+ routes = _route_matcher(graph)
123
+ for m in declared_members(ctx.repo_root):
124
+ code, role = str(m["code"]).upper(), str(m.get("role") or "member")
125
+ mctx = resolve(code)
126
+ nodes, edges = _load(mctx.devkb_dir)
127
+ by_uid: dict[str, tuple[str, str]] = {}
128
+ for n in nodes:
129
+ if n["type"] == "Repository":
130
+ continue
131
+ key = _prefixed(code, n["type"], n["natural_key"])
132
+ by_uid[n["uid"]] = (n["type"], key)
133
+ graph.add_node(n["type"], key, f"{SOURCE}:{code}:{n.get('source', '')}", **(n.get("properties") or {}))
134
+ n_edges = 0
135
+ for e in edges:
136
+ a, b = by_uid.get(e["from_uid"]), by_uid.get(e["to_uid"])
137
+ if a is None or b is None:
138
+ continue
139
+ p = dict(e.get("properties") or {})
140
+ src, conf, ev = p.pop("source", SOURCE), float(p.pop("confidence", 1.0) or 1.0), p.pop("evidence_ref", "")
141
+ graph.add_edge(e["verb"], a[0], a[1], b[0], b[1], f"{SOURCE}:{code}:{src}", conf, ev, p)
142
+ n_edges += 1
143
+ link_repository(graph, code, role=role, only_prefix=f"{code}:")
144
+ calls = 0
145
+ if routes:
146
+ listed = subprocess.run(["git", "ls-files", "--recurse-submodules", "--", "*.py"], cwd=mctx.repo_root,
147
+ capture_output=True, text=True, check=False).stdout.split()
148
+ for rel in listed:
149
+ try:
150
+ uses = url_uses((mctx.repo_root / rel).read_text(errors="replace"))
151
+ except OSError:
152
+ continue
153
+ for qual, path in uses:
154
+ for rx, owner, ep in routes:
155
+ if rx.match(path):
156
+ graph.add_edge("CALLS_API", "Symbol", f"{code}:{rel}::{qual}", "Symbol", owner, source=SOURCE,
157
+ confidence=0.6, evidence_ref=f"{code}:{rel} uses a URL with the path of {ep}")
158
+ calls += 1
159
+ summary.append({"code": code, "role": role, "nodes": len(by_uid), "edges": n_edges, "calls_api": calls})
160
+ return summary
@@ -0,0 +1,271 @@
1
+ """Pull requests and hosted issues (ADR-DEVKB-007).
2
+
3
+ Two layers feed one set of nodes:
4
+
5
+ offline always on, from one `git log` pass. A merge commit `Merge pull request #N from owner/branch` and a squash
6
+ commit whose subject ends `(#N)` name their pull request; a closing keyword in a pull request's commit
7
+ messages gives a low-confidence CLOSES whose target is marked unverified.
8
+ online only when asked for (`devkb build --github`, or devkb.json `"github": {"enabled": true}`). GitHub's own
9
+ commit lists and closing references replace the offline guesses, and hosted issues become Ticket nodes.
10
+
11
+ A repository whose `origin` is not a GitHub host gets no new nodes. Descriptions, review text and author logins are
12
+ never read from the API, and the owner in `Merge pull request #N from owner/branch` is dropped (it is the author's or
13
+ fork's login): the branch name is all that is kept. See devkb/github.py for the API side.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ import subprocess
19
+ import sys
20
+ from dataclasses import dataclass, field
21
+ from pathlib import Path
22
+ from typing import Callable
23
+
24
+ from devkb import github
25
+ from devkb.extractors.source_baseline import DEFAULT_SINCE
26
+ from devkb.model import DevKBGraph
27
+ from devkb.project_context import ProjectContext
28
+
29
+ MERGE_SOURCE, SQUASH_SOURCE = "git-merge-subject", "git-squash-subject"
30
+ OFFLINE_CONFIDENCE = 0.9
31
+ CLOSES_OFFLINE_CONFIDENCE = 0.7
32
+ MENTION_CONFIDENCE = 0.8
33
+ MAX_BROUGHT_IN = 1000 # commits one merge may bring in before the list is not trusted
34
+
35
+ _MERGE = re.compile(r"^Merge pull request #(\d+) from (\S+)")
36
+ _SQUASH = re.compile(r"\(#(\d+)\)\s*$")
37
+ _CLOSE = re.compile(r"\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\b:?[ \t]+(#\d+(?:[ \t]*(?:,|and|&)[ \t]*#\d+)*)", re.I)
38
+ _MENTION = re.compile(r"(?<![\w/&#])#(\d+)\b")
39
+ _NUM = re.compile(r"#(\d+)")
40
+
41
+
42
+ @dataclass
43
+ class GitCommit:
44
+ sha: str
45
+ parents: list[str]
46
+ date: str
47
+ committed: str
48
+ subject: str
49
+ body: str
50
+
51
+
52
+ @dataclass
53
+ class OfflinePR:
54
+ number: int
55
+ title: str = ""
56
+ head: str | None = None
57
+ merge_style: str | None = None
58
+ merged_at: str = ""
59
+ merge_commit: str = ""
60
+ #: sha -> source of the claim
61
+ commits: dict[str, str] = field(default_factory=dict)
62
+ closes: set[int] = field(default_factory=set)
63
+ too_large: bool = False # a merge that brought in more than MAX_BROUGHT_IN commits: only the merge is linked
64
+
65
+
66
+ # ── offline ──────────────────────────────────────────────────────────────────────────────
67
+ def read_log(repo_root: Path, since: str = DEFAULT_SINCE) -> list[GitCommit]:
68
+ try:
69
+ out = subprocess.run(
70
+ ["git", "log", "--all", f"--since={since}", "-z", "--date=short", "--format=%H%x1f%P%x1f%ad%x1f%cI%x1f%s%x1f%b"],
71
+ cwd=str(repo_root), capture_output=True, text=True, errors="replace", timeout=600).stdout
72
+ except (OSError, subprocess.SubprocessError):
73
+ return []
74
+ commits = []
75
+ for rec in out.split("\0"):
76
+ parts = rec.strip("\n").split("\x1f", 5)
77
+ if len(parts) == 6 and parts[0]:
78
+ commits.append(GitCommit(parts[0], parts[1].split(), parts[2], parts[3], parts[4], parts[5]))
79
+ return commits
80
+
81
+
82
+ def brought_in(repo_root: Path, merge: str) -> list[str] | None:
83
+ """The commits a merge commit brought in: `git rev-list <merge>^1..<merge>^2`. None when there are more than
84
+ MAX_BROUGHT_IN: such a list is not trusted, and the build reports the pull request instead of linking nothing
85
+ silently."""
86
+ try:
87
+ out = subprocess.run(["git", "rev-list", f"{merge}^1..{merge}^2"], cwd=str(repo_root), capture_output=True,
88
+ text=True, timeout=60).stdout.split()
89
+ except (OSError, subprocess.SubprocessError):
90
+ return []
91
+ return out if len(out) <= MAX_BROUGHT_IN else None
92
+
93
+
94
+ def closing_numbers(text: str) -> set[int]:
95
+ return {int(n) for m in _CLOSE.finditer(text) for n in _NUM.findall(m.group(1))}
96
+
97
+
98
+ def scan(commits: list[GitCommit], repo_root: Path,
99
+ brought: Callable[[Path, str], list[str]] = brought_in) -> dict[int, OfflinePR]:
100
+ by_sha = {c.sha: c for c in commits}
101
+ prs: dict[int, OfflinePR] = {}
102
+ # oldest first, so the first commit that names a pull request supplies its title and date
103
+ for c in sorted(commits, key=lambda c: c.committed):
104
+ m = _MERGE.match(c.subject)
105
+ members: list[str] = []
106
+ listed: list[str] | None = []
107
+ if m and len(c.parents) >= 2:
108
+ n, style, source = int(m.group(1)), "merge", MERGE_SOURCE
109
+ title = next((ln.strip() for ln in c.body.splitlines() if ln.strip()), "")
110
+ head = m.group(2).split("/", 1)[-1] # drop the owner: it is a login
111
+ listed = brought(repo_root, c.sha)
112
+ members = [s for s in (listed or []) if s in by_sha and s != c.sha]
113
+ else:
114
+ m = _SQUASH.search(c.subject)
115
+ if not m or len(c.parents) != 1:
116
+ continue
117
+ n, style, source = int(m.group(1)), "squash", SQUASH_SOURCE
118
+ title, head = c.subject[:m.start()].rstrip(), None
119
+ pr = prs.setdefault(n, OfflinePR(n))
120
+ if style == "merge" and listed is None:
121
+ pr.too_large = True
122
+ if not pr.merge_commit:
123
+ pr.title, pr.head, pr.merge_style = title, head, style
124
+ pr.merged_at, pr.merge_commit = c.committed, c.sha
125
+ pr.commits.setdefault(c.sha, source)
126
+ for s in members:
127
+ pr.commits.setdefault(s, source)
128
+ for sha in [c.sha, *members]:
129
+ pr.closes |= closing_numbers(by_sha[sha].subject + "\n" + by_sha[sha].body)
130
+ for n, pr in prs.items():
131
+ pr.closes.discard(n)
132
+ return prs
133
+
134
+
135
+ # ── building the graph ───────────────────────────────────────────────────────────────────
136
+ def _gh(n: int) -> str:
137
+ return f"GH-{n}"
138
+
139
+
140
+ def _pr(n: int) -> str:
141
+ return f"PR-{n}"
142
+
143
+
144
+ def build_graph(remote: github.Remote, commits: list[GitCommit], offline: dict[int, OfflinePR],
145
+ online: github.Online | None, authors: bool = False) -> DevKBGraph:
146
+ g = DevKBGraph()
147
+ known = {c.sha: c for c in commits}
148
+ prs_online = online.prs if online else {}
149
+ issues_online = online.issues if online else {}
150
+ # One number sequence per repository: when GitHub returns N as an issue, a `(#N)` subject named that issue, not a
151
+ # pull request, so the offline guess PR-N is dropped and the issue kept.
152
+ pr_numbers = (set(offline) - set(issues_online)) | set(prs_online)
153
+
154
+ def commit_node(sha: str) -> None:
155
+ c = known[sha]
156
+ g.add_node("Commit", sha, source="git", short_hash=sha[:10], date=c.date, subject=github.public_subject(c.subject, authors))
157
+
158
+ # ── hosted issues ──
159
+ for n, rec in sorted(issues_online.items()):
160
+ if n in pr_numbers: # one sequence per repository: an issue and a pull request never share a number
161
+ continue
162
+ g.add_node("Ticket", _gh(n), source="github-issues", title=rec["title"], ticket_type=rec["ticket_type"],
163
+ status=rec["state"].upper(), state=rec["state"], state_reason=rec.get("state_reason"),
164
+ labels=rec["labels"], milestone=rec.get("milestone"), url=rec.get("url") or remote.web("issues", n),
165
+ created_at=rec.get("created_at"), closed_at=rec.get("closed_at"), author=rec.get("author"),
166
+ has_sub_issues=bool(rec.get("has_sub_issues")))
167
+ for n, rec in sorted(issues_online.items()):
168
+ parent = rec.get("parent")
169
+ if parent and parent in issues_online and n not in pr_numbers:
170
+ g.add_edge("CHILD_OF", "Ticket", _gh(n), "Ticket", _gh(parent), source="github-sub-issues", confidence=1.0,
171
+ evidence_ref=f"sub-issue of #{parent}")
172
+
173
+ # ── pull requests ──
174
+ for n in sorted(pr_numbers):
175
+ off, on = offline.get(n), prs_online.get(n)
176
+ props: dict = {"number": n, "url": remote.web("pull", n)}
177
+ if off:
178
+ props.update(title=off.title, state="merged", merged_at=off.merged_at, head=off.head, merge_style=off.merge_style,
179
+ merge_commit=off.merge_commit)
180
+ if on:
181
+ props.update(title=on["title"], state=on["state"], merged_at=on.get("merged_at") or props.get("merged_at"),
182
+ base=on.get("base"), head=on.get("head") or props.get("head"), labels=on["labels"],
183
+ url=on.get("url") or props["url"], author=on.get("author"))
184
+ if on.get("merge_commit"):
185
+ props["merge_commit"] = on["merge_commit"]
186
+ if not props.get("merge_style") and on["merge_commit"] in known and len(known[on["merge_commit"]].parents) >= 2:
187
+ props["merge_style"] = "merge"
188
+ g.add_node("PullRequest", _pr(n), source="github-pull-requests" if on else "git-log", **props)
189
+
190
+ # MERGES: the API's list when it has one that is not cut short, else what the subjects said
191
+ api: dict[str, str] = {}
192
+ complete = False
193
+ if on and on["state"] == "merged":
194
+ for sha in on["commits"]:
195
+ api[sha] = "github-pr-commits"
196
+ if on.get("merge_commit"):
197
+ api[on["merge_commit"]] = "github-pr-commits"
198
+ complete = on.get("commits_total", 0) <= len(on["commits"])
199
+ for sha, src in sorted(api.items()):
200
+ if sha in known:
201
+ commit_node(sha)
202
+ g.add_edge("MERGES", "PullRequest", _pr(n), "Commit", sha, source=src, confidence=1.0,
203
+ evidence_ref="GitHub: the pull request's commit list and merge commit")
204
+ if off:
205
+ for sha, src in sorted(off.commits.items()):
206
+ if sha in api or (on and on["state"] == "merged" and complete):
207
+ continue # confirmed above, or contradicted by the complete list
208
+ commit_node(sha)
209
+ g.add_edge("MERGES", "PullRequest", _pr(n), "Commit", sha, source=src, confidence=OFFLINE_CONFIDENCE,
210
+ evidence_ref=github.public_subject(known[sha].subject, authors)[:80] if src == SQUASH_SOURCE or sha == off.merge_commit
211
+ else f"git rev-list {off.merge_commit[:10]}^1..{off.merge_commit[:10]}^2")
212
+
213
+ # CLOSES: GitHub's own record replaces whatever the commit messages suggested
214
+ if on:
215
+ for t in on["closes"]:
216
+ if t not in issues_online: # closed by GitHub's record, but not in the issue listing (not read yet, or moved)
217
+ g.add_node("Ticket", _gh(t), source="github-closing-reference", ticket_type="ISSUE", fetched=False,
218
+ url=remote.web("issues", t))
219
+ g.add_edge("CLOSES", "PullRequest", _pr(n), "Ticket", _gh(t), source="github-closing-reference", confidence=1.0,
220
+ evidence_ref="GitHub: closing issue reference")
221
+ elif off:
222
+ for t in sorted(off.closes):
223
+ if t in pr_numbers: # `Fixes #N` where N is a pull request, not an issue
224
+ continue
225
+ verified = t in issues_online
226
+ if not verified:
227
+ g.add_node("Ticket", _gh(t), source="git-closing-keyword", ticket_type="ISSUE", unverified=True,
228
+ url=remote.web("issues", t))
229
+ g.add_edge("CLOSES", "PullRequest", _pr(n), "Ticket", _gh(t), source="git-closing-keyword",
230
+ confidence=CLOSES_OFFLINE_CONFIDENCE,
231
+ evidence_ref=f"closing keyword for #{t} in a commit message of the pull request",
232
+ extra={"unverified": not verified})
233
+
234
+ # ── commit messages that mention an issue (online only: offline, #N might be a pull request) ──
235
+ if issues_online:
236
+ for c in commits:
237
+ for n in sorted({int(x) for x in _MENTION.findall(c.subject + "\n" + c.body)}):
238
+ if n in issues_online and n not in pr_numbers:
239
+ commit_node(c.sha)
240
+ g.add_edge("REFERENCES", "Commit", c.sha, "Ticket", _gh(n), source="git-hash-mention",
241
+ confidence=MENTION_CONFIDENCE, evidence_ref=f"commit message mentions #{n}")
242
+ return g
243
+
244
+
245
+ def extract(ctx: ProjectContext, github_flag: bool | None = None, transport: github.Transport | None = None,
246
+ since: str = DEFAULT_SINCE) -> tuple[DevKBGraph, dict | None]:
247
+ """(the graph, a report for the manifest or None when there is no GitHub remote)."""
248
+ cfg = github.settings(ctx.repo_root, github_flag)
249
+ remote = github.origin_remote(ctx.repo_root, cfg.host)
250
+ if remote is None:
251
+ if cfg.enabled:
252
+ print("devkb build: github layer requested but `origin` is not a GitHub remote; offline only", file=sys.stderr)
253
+ return DevKBGraph(), {"mode": "offline", "status": "no-remote", "reason": "origin is not a GitHub remote"}
254
+ return DevKBGraph(), None
255
+ commits = read_log(ctx.repo_root, since)
256
+ offline = scan(commits, ctx.repo_root)
257
+ online: github.Online | None = None
258
+ large = sorted(n for n, pr in offline.items() if pr.too_large)
259
+ report: dict = {"remote": remote.slug, "mode": "offline", "status": "offline",
260
+ "reason": "online layer not requested", "offline_pull_requests": len(offline)}
261
+ if cfg.enabled:
262
+ online = github.fetch(remote, ctx.devkb_dir / "github", authors=cfg.authors, transport=transport)
263
+ report = {**online.report, "offline_pull_requests": len(offline)}
264
+ if report["status"] != "ok":
265
+ print(f"devkb build: github layer {report['status']} ({report.get('reason')}); "
266
+ f"not fetched: {', '.join(report.get('not_fetched') or [])}; the answers use the offline layer"
267
+ + (" and the cache" if report.get("stale_cache_used") else ""), file=sys.stderr)
268
+ if large:
269
+ report["merges_not_expanded"] = large[:50] # more than MAX_BROUGHT_IN commits: only the merge commit is linked
270
+ return build_graph(remote, commits, offline, online if online and (online.prs or online.issues) else None,
271
+ cfg.authors), report
@@ -0,0 +1,49 @@
1
+ """Release-tag extractor (devkb BL-2, body/legs model).
2
+
3
+ The one process-log sub-signal that's actually recoverable without a ticket
4
+ system: `git tag` already records "this commit was a release," for real,
5
+ cheaply, independent of tickets/commits-clustering/everything else in this
6
+ package. Real data throughout -- source="git-tag", no inference involved.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from pathlib import Path
11
+
12
+ from devkb.extractors.tickets_git import _run
13
+ from devkb.model import DevKBGraph
14
+ from devkb.project_context import ProjectContext
15
+
16
+
17
+ def _list_tags(repo_root: Path) -> list[str]:
18
+ out = _run(["git", "tag", "--list"], repo_root)
19
+ return [t for t in out.splitlines() if t.strip()]
20
+
21
+
22
+ def _tag_commit(tag: str, repo_root: Path) -> str | None:
23
+ out = _run(["git", "rev-list", "-n", "1", tag], repo_root)
24
+ sha = out.strip()
25
+ return sha or None
26
+
27
+
28
+ def _tag_date(tag: str, repo_root: Path) -> str:
29
+ # %(creatordate:short) covers both annotated and lightweight tags --
30
+ # `git log -1 <tag> --format=%ad` would report the TAGGED COMMIT's own
31
+ # date, not when the tag itself was created, which is the wrong question
32
+ # for "when was this released."
33
+ out = _run(["git", "for-each-ref", f"refs/tags/{tag}", "--format=%(creatordate:short)"], repo_root)
34
+ return out.strip()
35
+
36
+
37
+ def extract(ctx: ProjectContext) -> DevKBGraph:
38
+ g = DevKBGraph()
39
+ for tag in _list_tags(ctx.repo_root):
40
+ commit = _tag_commit(tag, ctx.repo_root)
41
+ if commit is None:
42
+ continue # a tag git itself can't resolve -- skip, don't guess
43
+ g.add_node("Release", tag, source="git-tag", tagged_at=_tag_date(tag, ctx.repo_root))
44
+ g.add_node("Commit", commit, source="git-tag")
45
+ g.add_edge(
46
+ "TAGS", "Release", tag, "Commit", commit,
47
+ source="git-tag", confidence=1.0, evidence_ref=f"git tag {tag}",
48
+ )
49
+ return g
@@ -0,0 +1,66 @@
1
+ """Ticket-agnostic Commit/File baseline extractor (devkb BL-1, body/legs model).
2
+
3
+ Phase 1's tickets_git.py only ever finds a commit via `git log --grep=<ticket>`
4
+ -- per ticket. A repo with real commits and zero tickets filed in this
5
+ project's own convention (one project: real source, real commits, real markdown
6
+ design docs, no tickets) gets nothing from that path, by construction, not
7
+ by bug -- devkb/ROADMAP.md's own "body/legs model" section works through why.
8
+
9
+ This extractor walks git history directly, independent of any ticket, and is
10
+ purely additive: DevKBGraph.add_node()/add_edge() dedupe by natural key, so a
11
+ commit found by both this and tickets_git.py's own ticket-mediated walk just
12
+ merges into one node, no conflict, no double-counting.
13
+
14
+ Bounded by --since, not unbounded `--all` -- OCO's own ticket-mediated walk
15
+ already costs ~2m10s at 2,584 commits; an unbounded walk over the same
16
+ history needs the same scale discipline.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ from pathlib import Path
21
+
22
+ from devkb import github
23
+ from devkb.extractors.files_tests import _files_touched, is_bulk
24
+ from devkb.extractors.tickets_git import _run
25
+ from devkb.model import DevKBGraph
26
+ from devkb.project_context import ProjectContext
27
+
28
+ # Wide enough to capture a project's real history without being truly
29
+ # unbounded -- every registered project's real age today fits well within
30
+ # this, while a pathological decades-old import still gets bounded.
31
+ DEFAULT_SINCE = "3 years ago"
32
+
33
+
34
+ def _all_commits(repo_root: Path, since: str) -> list[dict[str, str]]:
35
+ out = _run(
36
+ ["git", "log", "--all", f"--since={since}", "--format=%H|%ad|%an|%s", "--date=short"],
37
+ repo_root,
38
+ )
39
+ commits = []
40
+ for line in out.splitlines():
41
+ parts = line.split("|", 3)
42
+ if len(parts) == 4:
43
+ commits.append({"hash": parts[0], "date": parts[1], "author": parts[2], "subject": parts[3]})
44
+ return commits
45
+
46
+
47
+ def extract(ctx: ProjectContext, since: str = DEFAULT_SINCE) -> DevKBGraph:
48
+ g = DevKBGraph()
49
+ authors = github.settings(ctx.repo_root).authors
50
+ for c in _all_commits(ctx.repo_root, since):
51
+ g.add_node(
52
+ "Commit", c["hash"], source="git-log-baseline",
53
+ short_hash=c["hash"][:10], date=c["date"], author=c["author"], subject=github.public_subject(c["subject"], authors),
54
+ )
55
+ touched = _files_touched(c["hash"], ctx.repo_root)
56
+ if is_bulk(touched): # ODSF-314: a bulk change keeps its size, not one edge per file
57
+ g.add_node("Commit", c["hash"], source="git-log-baseline", files_changed=len(touched), bulk=True)
58
+ continue
59
+ for path in touched:
60
+ g.add_node("File", path, source="git-log-baseline")
61
+ g.add_edge(
62
+ "TOUCHES", "Commit", c["hash"], "File", path,
63
+ source="git-log-baseline", confidence=1.0,
64
+ evidence_ref=f"git diff-tree --name-only {c['hash'][:10]}",
65
+ )
66
+ return g