devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""A product's graph spans its member repositories (ADR-SDP-DEVKB-005 §1-3, ODSF-312).
|
|
2
|
+
|
|
3
|
+
A product declares members in its root pms.config.json, `"graph": {"members": [{"code": "LIB", "role": "client"}]}`.
|
|
4
|
+
Each member is a registered project with its own graph, built by its own build with its own config; this module
|
|
5
|
+
reads that graph and never runs a member's extractors with the product's rules (the ODSF-104 lesson).
|
|
6
|
+
|
|
7
|
+
Repository one per repository, the product's own included; natural key the project code
|
|
8
|
+
IN_REPO File -> Repository, for every File of the product and of each member
|
|
9
|
+
prefixing a member node's natural key becomes `<CODE>:<key>`, except ConfigKey (the same setting name in two
|
|
10
|
+
repositories is one setting) and Repository; the product's own keys never change
|
|
11
|
+
CALLS_API member Symbol -> product route-handler Symbol, when a URL the member's code uses has the path of a
|
|
12
|
+
product Endpoint (confidence 0.6: a path match, the HTTP method is not known statically)
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import ast
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
import subprocess
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from urllib.parse import urlparse
|
|
22
|
+
|
|
23
|
+
from devkb.model import DevKBGraph
|
|
24
|
+
from devkb.project_context import load_config
|
|
25
|
+
|
|
26
|
+
SOURCE = "members"
|
|
27
|
+
UNPREFIXED = {"ConfigKey", "Repository"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def declared_members(repo_root: Path) -> list[dict]:
|
|
31
|
+
try:
|
|
32
|
+
cfg = load_config(repo_root)
|
|
33
|
+
except (OSError, ValueError):
|
|
34
|
+
return []
|
|
35
|
+
members = ((cfg.get("graph") or {}).get("members") or []) if isinstance(cfg, dict) else []
|
|
36
|
+
return [m for m in members if isinstance(m, dict) and m.get("code")]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _load(devkb_dir: Path) -> tuple[list[dict], list[dict]]:
|
|
40
|
+
nodes_p, edges_p = devkb_dir / "nodes.jsonl", devkb_dir / "edges.jsonl"
|
|
41
|
+
if not nodes_p.is_file() or not edges_p.is_file():
|
|
42
|
+
raise ValueError(f"declared member graph not built: {devkb_dir} has no nodes.jsonl/edges.jsonl "
|
|
43
|
+
f"(build the member first: make devkb-build PROJECT=<CODE>)")
|
|
44
|
+
nodes = [json.loads(l) for l in nodes_p.read_text().splitlines() if l.strip()]
|
|
45
|
+
edges = [json.loads(l) for l in edges_p.read_text().splitlines() if l.strip()]
|
|
46
|
+
return nodes, edges
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _prefixed(code: str, typ: str, key: str) -> str:
|
|
50
|
+
return key if typ in UNPREFIXED else f"{code}:{key}"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# ── URLs a member's code uses ────────────────────────────────────────────────
|
|
54
|
+
def _url_paths(node: ast.AST) -> list[str]:
|
|
55
|
+
out = []
|
|
56
|
+
for n in ast.walk(node):
|
|
57
|
+
if isinstance(n, ast.Constant) and isinstance(n.value, str) and n.value.startswith(("http://", "https://")):
|
|
58
|
+
path = urlparse(n.value).path
|
|
59
|
+
if path and path != "/":
|
|
60
|
+
out.append(path.rstrip("/"))
|
|
61
|
+
return out
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def url_uses(text: str) -> list[tuple[str, str]]:
|
|
65
|
+
"""(qualified function name, URL path) for every URL a function uses, directly or through a module constant."""
|
|
66
|
+
try:
|
|
67
|
+
tree = ast.parse(text)
|
|
68
|
+
except (SyntaxError, ValueError, RecursionError):
|
|
69
|
+
return []
|
|
70
|
+
constants: dict[str, list[str]] = {}
|
|
71
|
+
for stmt in tree.body:
|
|
72
|
+
if isinstance(stmt, ast.Assign):
|
|
73
|
+
paths = _url_paths(stmt.value)
|
|
74
|
+
for t in stmt.targets:
|
|
75
|
+
if isinstance(t, ast.Name) and paths:
|
|
76
|
+
constants[t.id] = paths
|
|
77
|
+
found: list[tuple[str, str]] = []
|
|
78
|
+
|
|
79
|
+
def walk(node, stack):
|
|
80
|
+
for child in ast.iter_child_nodes(node):
|
|
81
|
+
if isinstance(child, ast.ClassDef):
|
|
82
|
+
walk(child, stack + [child.name])
|
|
83
|
+
elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
84
|
+
qual = ".".join(stack + [child.name])
|
|
85
|
+
body_paths = set()
|
|
86
|
+
for n in ast.walk(child):
|
|
87
|
+
if isinstance(n, ast.Name) and n.id in constants:
|
|
88
|
+
body_paths.update(constants[n.id])
|
|
89
|
+
body_paths.update(_url_paths(child))
|
|
90
|
+
found.extend((qual, p) for p in sorted(body_paths))
|
|
91
|
+
walk(child, stack + [child.name])
|
|
92
|
+
walk(tree, [])
|
|
93
|
+
return found
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _route_matcher(graph: DevKBGraph) -> list[tuple[re.Pattern, str, str]]:
|
|
97
|
+
out = []
|
|
98
|
+
for ep in graph.nodes("Endpoint"):
|
|
99
|
+
path, owner = ep.props.get("path") or ep.natural_key.split(" ", 1)[-1], ep.props.get("owner_symbol")
|
|
100
|
+
if not owner or not path.startswith("/"):
|
|
101
|
+
continue
|
|
102
|
+
rx = re.compile("^" + re.sub(r"\\\{[^}]+\\\}|\\<[^>]+\\>", "[^/]+", re.escape(path.rstrip("/"))) + "$")
|
|
103
|
+
out.append((rx, owner, ep.natural_key))
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def link_repository(graph: DevKBGraph, code: str, role: str = "product", only_prefix: str | None = None) -> None:
|
|
108
|
+
"""A Repository node, and IN_REPO from each File of a member (keys prefixed `<CODE>:`). The product's own files
|
|
109
|
+
get no edge (ODSF-314): an unprefixed key already says which repository it is in, and on OCO that was 23,240
|
|
110
|
+
edges repeating it."""
|
|
111
|
+
graph.add_node("Repository", code, source=SOURCE, role=role)
|
|
112
|
+
if only_prefix is None:
|
|
113
|
+
return
|
|
114
|
+
for key in [n.natural_key for n in graph.nodes("File")]:
|
|
115
|
+
if key.startswith(only_prefix):
|
|
116
|
+
graph.add_edge("IN_REPO", "File", key, "Repository", code, source=SOURCE)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def merge_members(ctx, graph: DevKBGraph, resolve) -> list[dict]:
|
|
120
|
+
"""Fold every declared member's graph into `graph`. Returns one summary row per member."""
|
|
121
|
+
summary = []
|
|
122
|
+
routes = _route_matcher(graph)
|
|
123
|
+
for m in declared_members(ctx.repo_root):
|
|
124
|
+
code, role = str(m["code"]).upper(), str(m.get("role") or "member")
|
|
125
|
+
mctx = resolve(code)
|
|
126
|
+
nodes, edges = _load(mctx.devkb_dir)
|
|
127
|
+
by_uid: dict[str, tuple[str, str]] = {}
|
|
128
|
+
for n in nodes:
|
|
129
|
+
if n["type"] == "Repository":
|
|
130
|
+
continue
|
|
131
|
+
key = _prefixed(code, n["type"], n["natural_key"])
|
|
132
|
+
by_uid[n["uid"]] = (n["type"], key)
|
|
133
|
+
graph.add_node(n["type"], key, f"{SOURCE}:{code}:{n.get('source', '')}", **(n.get("properties") or {}))
|
|
134
|
+
n_edges = 0
|
|
135
|
+
for e in edges:
|
|
136
|
+
a, b = by_uid.get(e["from_uid"]), by_uid.get(e["to_uid"])
|
|
137
|
+
if a is None or b is None:
|
|
138
|
+
continue
|
|
139
|
+
p = dict(e.get("properties") or {})
|
|
140
|
+
src, conf, ev = p.pop("source", SOURCE), float(p.pop("confidence", 1.0) or 1.0), p.pop("evidence_ref", "")
|
|
141
|
+
graph.add_edge(e["verb"], a[0], a[1], b[0], b[1], f"{SOURCE}:{code}:{src}", conf, ev, p)
|
|
142
|
+
n_edges += 1
|
|
143
|
+
link_repository(graph, code, role=role, only_prefix=f"{code}:")
|
|
144
|
+
calls = 0
|
|
145
|
+
if routes:
|
|
146
|
+
listed = subprocess.run(["git", "ls-files", "--recurse-submodules", "--", "*.py"], cwd=mctx.repo_root,
|
|
147
|
+
capture_output=True, text=True, check=False).stdout.split()
|
|
148
|
+
for rel in listed:
|
|
149
|
+
try:
|
|
150
|
+
uses = url_uses((mctx.repo_root / rel).read_text(errors="replace"))
|
|
151
|
+
except OSError:
|
|
152
|
+
continue
|
|
153
|
+
for qual, path in uses:
|
|
154
|
+
for rx, owner, ep in routes:
|
|
155
|
+
if rx.match(path):
|
|
156
|
+
graph.add_edge("CALLS_API", "Symbol", f"{code}:{rel}::{qual}", "Symbol", owner, source=SOURCE,
|
|
157
|
+
confidence=0.6, evidence_ref=f"{code}:{rel} uses a URL with the path of {ep}")
|
|
158
|
+
calls += 1
|
|
159
|
+
summary.append({"code": code, "role": role, "nodes": len(by_uid), "edges": n_edges, "calls_api": calls})
|
|
160
|
+
return summary
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Pull requests and hosted issues (ADR-DEVKB-007).
|
|
2
|
+
|
|
3
|
+
Two layers feed one set of nodes:
|
|
4
|
+
|
|
5
|
+
offline always on, from one `git log` pass. A merge commit `Merge pull request #N from owner/branch` and a squash
|
|
6
|
+
commit whose subject ends `(#N)` name their pull request; a closing keyword in a pull request's commit
|
|
7
|
+
messages gives a low-confidence CLOSES whose target is marked unverified.
|
|
8
|
+
online only when asked for (`devkb build --github`, or devkb.json `"github": {"enabled": true}`). GitHub's own
|
|
9
|
+
commit lists and closing references replace the offline guesses, and hosted issues become Ticket nodes.
|
|
10
|
+
|
|
11
|
+
A repository whose `origin` is not a GitHub host gets no new nodes. Descriptions, review text and author logins are
|
|
12
|
+
never read from the API, and the owner in `Merge pull request #N from owner/branch` is dropped (it is the author's or
|
|
13
|
+
fork's login): the branch name is all that is kept. See devkb/github.py for the API side.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
import subprocess
|
|
19
|
+
import sys
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Callable
|
|
23
|
+
|
|
24
|
+
from devkb import github
|
|
25
|
+
from devkb.extractors.source_baseline import DEFAULT_SINCE
|
|
26
|
+
from devkb.model import DevKBGraph
|
|
27
|
+
from devkb.project_context import ProjectContext
|
|
28
|
+
|
|
29
|
+
MERGE_SOURCE, SQUASH_SOURCE = "git-merge-subject", "git-squash-subject"
|
|
30
|
+
OFFLINE_CONFIDENCE = 0.9
|
|
31
|
+
CLOSES_OFFLINE_CONFIDENCE = 0.7
|
|
32
|
+
MENTION_CONFIDENCE = 0.8
|
|
33
|
+
MAX_BROUGHT_IN = 1000 # commits one merge may bring in before the list is not trusted
|
|
34
|
+
|
|
35
|
+
_MERGE = re.compile(r"^Merge pull request #(\d+) from (\S+)")
|
|
36
|
+
_SQUASH = re.compile(r"\(#(\d+)\)\s*$")
|
|
37
|
+
_CLOSE = re.compile(r"\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\b:?[ \t]+(#\d+(?:[ \t]*(?:,|and|&)[ \t]*#\d+)*)", re.I)
|
|
38
|
+
_MENTION = re.compile(r"(?<![\w/&#])#(\d+)\b")
|
|
39
|
+
_NUM = re.compile(r"#(\d+)")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class GitCommit:
|
|
44
|
+
sha: str
|
|
45
|
+
parents: list[str]
|
|
46
|
+
date: str
|
|
47
|
+
committed: str
|
|
48
|
+
subject: str
|
|
49
|
+
body: str
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class OfflinePR:
|
|
54
|
+
number: int
|
|
55
|
+
title: str = ""
|
|
56
|
+
head: str | None = None
|
|
57
|
+
merge_style: str | None = None
|
|
58
|
+
merged_at: str = ""
|
|
59
|
+
merge_commit: str = ""
|
|
60
|
+
#: sha -> source of the claim
|
|
61
|
+
commits: dict[str, str] = field(default_factory=dict)
|
|
62
|
+
closes: set[int] = field(default_factory=set)
|
|
63
|
+
too_large: bool = False # a merge that brought in more than MAX_BROUGHT_IN commits: only the merge is linked
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# ── offline ──────────────────────────────────────────────────────────────────────────────
|
|
67
|
+
def read_log(repo_root: Path, since: str = DEFAULT_SINCE) -> list[GitCommit]:
|
|
68
|
+
try:
|
|
69
|
+
out = subprocess.run(
|
|
70
|
+
["git", "log", "--all", f"--since={since}", "-z", "--date=short", "--format=%H%x1f%P%x1f%ad%x1f%cI%x1f%s%x1f%b"],
|
|
71
|
+
cwd=str(repo_root), capture_output=True, text=True, errors="replace", timeout=600).stdout
|
|
72
|
+
except (OSError, subprocess.SubprocessError):
|
|
73
|
+
return []
|
|
74
|
+
commits = []
|
|
75
|
+
for rec in out.split("\0"):
|
|
76
|
+
parts = rec.strip("\n").split("\x1f", 5)
|
|
77
|
+
if len(parts) == 6 and parts[0]:
|
|
78
|
+
commits.append(GitCommit(parts[0], parts[1].split(), parts[2], parts[3], parts[4], parts[5]))
|
|
79
|
+
return commits
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def brought_in(repo_root: Path, merge: str) -> list[str] | None:
|
|
83
|
+
"""The commits a merge commit brought in: `git rev-list <merge>^1..<merge>^2`. None when there are more than
|
|
84
|
+
MAX_BROUGHT_IN: such a list is not trusted, and the build reports the pull request instead of linking nothing
|
|
85
|
+
silently."""
|
|
86
|
+
try:
|
|
87
|
+
out = subprocess.run(["git", "rev-list", f"{merge}^1..{merge}^2"], cwd=str(repo_root), capture_output=True,
|
|
88
|
+
text=True, timeout=60).stdout.split()
|
|
89
|
+
except (OSError, subprocess.SubprocessError):
|
|
90
|
+
return []
|
|
91
|
+
return out if len(out) <= MAX_BROUGHT_IN else None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def closing_numbers(text: str) -> set[int]:
|
|
95
|
+
return {int(n) for m in _CLOSE.finditer(text) for n in _NUM.findall(m.group(1))}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def scan(commits: list[GitCommit], repo_root: Path,
|
|
99
|
+
brought: Callable[[Path, str], list[str]] = brought_in) -> dict[int, OfflinePR]:
|
|
100
|
+
by_sha = {c.sha: c for c in commits}
|
|
101
|
+
prs: dict[int, OfflinePR] = {}
|
|
102
|
+
# oldest first, so the first commit that names a pull request supplies its title and date
|
|
103
|
+
for c in sorted(commits, key=lambda c: c.committed):
|
|
104
|
+
m = _MERGE.match(c.subject)
|
|
105
|
+
members: list[str] = []
|
|
106
|
+
listed: list[str] | None = []
|
|
107
|
+
if m and len(c.parents) >= 2:
|
|
108
|
+
n, style, source = int(m.group(1)), "merge", MERGE_SOURCE
|
|
109
|
+
title = next((ln.strip() for ln in c.body.splitlines() if ln.strip()), "")
|
|
110
|
+
head = m.group(2).split("/", 1)[-1] # drop the owner: it is a login
|
|
111
|
+
listed = brought(repo_root, c.sha)
|
|
112
|
+
members = [s for s in (listed or []) if s in by_sha and s != c.sha]
|
|
113
|
+
else:
|
|
114
|
+
m = _SQUASH.search(c.subject)
|
|
115
|
+
if not m or len(c.parents) != 1:
|
|
116
|
+
continue
|
|
117
|
+
n, style, source = int(m.group(1)), "squash", SQUASH_SOURCE
|
|
118
|
+
title, head = c.subject[:m.start()].rstrip(), None
|
|
119
|
+
pr = prs.setdefault(n, OfflinePR(n))
|
|
120
|
+
if style == "merge" and listed is None:
|
|
121
|
+
pr.too_large = True
|
|
122
|
+
if not pr.merge_commit:
|
|
123
|
+
pr.title, pr.head, pr.merge_style = title, head, style
|
|
124
|
+
pr.merged_at, pr.merge_commit = c.committed, c.sha
|
|
125
|
+
pr.commits.setdefault(c.sha, source)
|
|
126
|
+
for s in members:
|
|
127
|
+
pr.commits.setdefault(s, source)
|
|
128
|
+
for sha in [c.sha, *members]:
|
|
129
|
+
pr.closes |= closing_numbers(by_sha[sha].subject + "\n" + by_sha[sha].body)
|
|
130
|
+
for n, pr in prs.items():
|
|
131
|
+
pr.closes.discard(n)
|
|
132
|
+
return prs
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# ── building the graph ───────────────────────────────────────────────────────────────────
|
|
136
|
+
def _gh(n: int) -> str:
|
|
137
|
+
return f"GH-{n}"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _pr(n: int) -> str:
|
|
141
|
+
return f"PR-{n}"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def build_graph(remote: github.Remote, commits: list[GitCommit], offline: dict[int, OfflinePR],
|
|
145
|
+
online: github.Online | None, authors: bool = False) -> DevKBGraph:
|
|
146
|
+
g = DevKBGraph()
|
|
147
|
+
known = {c.sha: c for c in commits}
|
|
148
|
+
prs_online = online.prs if online else {}
|
|
149
|
+
issues_online = online.issues if online else {}
|
|
150
|
+
# One number sequence per repository: when GitHub returns N as an issue, a `(#N)` subject named that issue, not a
|
|
151
|
+
# pull request, so the offline guess PR-N is dropped and the issue kept.
|
|
152
|
+
pr_numbers = (set(offline) - set(issues_online)) | set(prs_online)
|
|
153
|
+
|
|
154
|
+
def commit_node(sha: str) -> None:
|
|
155
|
+
c = known[sha]
|
|
156
|
+
g.add_node("Commit", sha, source="git", short_hash=sha[:10], date=c.date, subject=github.public_subject(c.subject, authors))
|
|
157
|
+
|
|
158
|
+
# ── hosted issues ──
|
|
159
|
+
for n, rec in sorted(issues_online.items()):
|
|
160
|
+
if n in pr_numbers: # one sequence per repository: an issue and a pull request never share a number
|
|
161
|
+
continue
|
|
162
|
+
g.add_node("Ticket", _gh(n), source="github-issues", title=rec["title"], ticket_type=rec["ticket_type"],
|
|
163
|
+
status=rec["state"].upper(), state=rec["state"], state_reason=rec.get("state_reason"),
|
|
164
|
+
labels=rec["labels"], milestone=rec.get("milestone"), url=rec.get("url") or remote.web("issues", n),
|
|
165
|
+
created_at=rec.get("created_at"), closed_at=rec.get("closed_at"), author=rec.get("author"),
|
|
166
|
+
has_sub_issues=bool(rec.get("has_sub_issues")))
|
|
167
|
+
for n, rec in sorted(issues_online.items()):
|
|
168
|
+
parent = rec.get("parent")
|
|
169
|
+
if parent and parent in issues_online and n not in pr_numbers:
|
|
170
|
+
g.add_edge("CHILD_OF", "Ticket", _gh(n), "Ticket", _gh(parent), source="github-sub-issues", confidence=1.0,
|
|
171
|
+
evidence_ref=f"sub-issue of #{parent}")
|
|
172
|
+
|
|
173
|
+
# ── pull requests ──
|
|
174
|
+
for n in sorted(pr_numbers):
|
|
175
|
+
off, on = offline.get(n), prs_online.get(n)
|
|
176
|
+
props: dict = {"number": n, "url": remote.web("pull", n)}
|
|
177
|
+
if off:
|
|
178
|
+
props.update(title=off.title, state="merged", merged_at=off.merged_at, head=off.head, merge_style=off.merge_style,
|
|
179
|
+
merge_commit=off.merge_commit)
|
|
180
|
+
if on:
|
|
181
|
+
props.update(title=on["title"], state=on["state"], merged_at=on.get("merged_at") or props.get("merged_at"),
|
|
182
|
+
base=on.get("base"), head=on.get("head") or props.get("head"), labels=on["labels"],
|
|
183
|
+
url=on.get("url") or props["url"], author=on.get("author"))
|
|
184
|
+
if on.get("merge_commit"):
|
|
185
|
+
props["merge_commit"] = on["merge_commit"]
|
|
186
|
+
if not props.get("merge_style") and on["merge_commit"] in known and len(known[on["merge_commit"]].parents) >= 2:
|
|
187
|
+
props["merge_style"] = "merge"
|
|
188
|
+
g.add_node("PullRequest", _pr(n), source="github-pull-requests" if on else "git-log", **props)
|
|
189
|
+
|
|
190
|
+
# MERGES: the API's list when it has one that is not cut short, else what the subjects said
|
|
191
|
+
api: dict[str, str] = {}
|
|
192
|
+
complete = False
|
|
193
|
+
if on and on["state"] == "merged":
|
|
194
|
+
for sha in on["commits"]:
|
|
195
|
+
api[sha] = "github-pr-commits"
|
|
196
|
+
if on.get("merge_commit"):
|
|
197
|
+
api[on["merge_commit"]] = "github-pr-commits"
|
|
198
|
+
complete = on.get("commits_total", 0) <= len(on["commits"])
|
|
199
|
+
for sha, src in sorted(api.items()):
|
|
200
|
+
if sha in known:
|
|
201
|
+
commit_node(sha)
|
|
202
|
+
g.add_edge("MERGES", "PullRequest", _pr(n), "Commit", sha, source=src, confidence=1.0,
|
|
203
|
+
evidence_ref="GitHub: the pull request's commit list and merge commit")
|
|
204
|
+
if off:
|
|
205
|
+
for sha, src in sorted(off.commits.items()):
|
|
206
|
+
if sha in api or (on and on["state"] == "merged" and complete):
|
|
207
|
+
continue # confirmed above, or contradicted by the complete list
|
|
208
|
+
commit_node(sha)
|
|
209
|
+
g.add_edge("MERGES", "PullRequest", _pr(n), "Commit", sha, source=src, confidence=OFFLINE_CONFIDENCE,
|
|
210
|
+
evidence_ref=github.public_subject(known[sha].subject, authors)[:80] if src == SQUASH_SOURCE or sha == off.merge_commit
|
|
211
|
+
else f"git rev-list {off.merge_commit[:10]}^1..{off.merge_commit[:10]}^2")
|
|
212
|
+
|
|
213
|
+
# CLOSES: GitHub's own record replaces whatever the commit messages suggested
|
|
214
|
+
if on:
|
|
215
|
+
for t in on["closes"]:
|
|
216
|
+
if t not in issues_online: # closed by GitHub's record, but not in the issue listing (not read yet, or moved)
|
|
217
|
+
g.add_node("Ticket", _gh(t), source="github-closing-reference", ticket_type="ISSUE", fetched=False,
|
|
218
|
+
url=remote.web("issues", t))
|
|
219
|
+
g.add_edge("CLOSES", "PullRequest", _pr(n), "Ticket", _gh(t), source="github-closing-reference", confidence=1.0,
|
|
220
|
+
evidence_ref="GitHub: closing issue reference")
|
|
221
|
+
elif off:
|
|
222
|
+
for t in sorted(off.closes):
|
|
223
|
+
if t in pr_numbers: # `Fixes #N` where N is a pull request, not an issue
|
|
224
|
+
continue
|
|
225
|
+
verified = t in issues_online
|
|
226
|
+
if not verified:
|
|
227
|
+
g.add_node("Ticket", _gh(t), source="git-closing-keyword", ticket_type="ISSUE", unverified=True,
|
|
228
|
+
url=remote.web("issues", t))
|
|
229
|
+
g.add_edge("CLOSES", "PullRequest", _pr(n), "Ticket", _gh(t), source="git-closing-keyword",
|
|
230
|
+
confidence=CLOSES_OFFLINE_CONFIDENCE,
|
|
231
|
+
evidence_ref=f"closing keyword for #{t} in a commit message of the pull request",
|
|
232
|
+
extra={"unverified": not verified})
|
|
233
|
+
|
|
234
|
+
# ── commit messages that mention an issue (online only: offline, #N might be a pull request) ──
|
|
235
|
+
if issues_online:
|
|
236
|
+
for c in commits:
|
|
237
|
+
for n in sorted({int(x) for x in _MENTION.findall(c.subject + "\n" + c.body)}):
|
|
238
|
+
if n in issues_online and n not in pr_numbers:
|
|
239
|
+
commit_node(c.sha)
|
|
240
|
+
g.add_edge("REFERENCES", "Commit", c.sha, "Ticket", _gh(n), source="git-hash-mention",
|
|
241
|
+
confidence=MENTION_CONFIDENCE, evidence_ref=f"commit message mentions #{n}")
|
|
242
|
+
return g
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def extract(ctx: ProjectContext, github_flag: bool | None = None, transport: github.Transport | None = None,
|
|
246
|
+
since: str = DEFAULT_SINCE) -> tuple[DevKBGraph, dict | None]:
|
|
247
|
+
"""(the graph, a report for the manifest or None when there is no GitHub remote)."""
|
|
248
|
+
cfg = github.settings(ctx.repo_root, github_flag)
|
|
249
|
+
remote = github.origin_remote(ctx.repo_root, cfg.host)
|
|
250
|
+
if remote is None:
|
|
251
|
+
if cfg.enabled:
|
|
252
|
+
print("devkb build: github layer requested but `origin` is not a GitHub remote; offline only", file=sys.stderr)
|
|
253
|
+
return DevKBGraph(), {"mode": "offline", "status": "no-remote", "reason": "origin is not a GitHub remote"}
|
|
254
|
+
return DevKBGraph(), None
|
|
255
|
+
commits = read_log(ctx.repo_root, since)
|
|
256
|
+
offline = scan(commits, ctx.repo_root)
|
|
257
|
+
online: github.Online | None = None
|
|
258
|
+
large = sorted(n for n, pr in offline.items() if pr.too_large)
|
|
259
|
+
report: dict = {"remote": remote.slug, "mode": "offline", "status": "offline",
|
|
260
|
+
"reason": "online layer not requested", "offline_pull_requests": len(offline)}
|
|
261
|
+
if cfg.enabled:
|
|
262
|
+
online = github.fetch(remote, ctx.devkb_dir / "github", authors=cfg.authors, transport=transport)
|
|
263
|
+
report = {**online.report, "offline_pull_requests": len(offline)}
|
|
264
|
+
if report["status"] != "ok":
|
|
265
|
+
print(f"devkb build: github layer {report['status']} ({report.get('reason')}); "
|
|
266
|
+
f"not fetched: {', '.join(report.get('not_fetched') or [])}; the answers use the offline layer"
|
|
267
|
+
+ (" and the cache" if report.get("stale_cache_used") else ""), file=sys.stderr)
|
|
268
|
+
if large:
|
|
269
|
+
report["merges_not_expanded"] = large[:50] # more than MAX_BROUGHT_IN commits: only the merge commit is linked
|
|
270
|
+
return build_graph(remote, commits, offline, online if online and (online.prs or online.issues) else None,
|
|
271
|
+
cfg.authors), report
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Release-tag extractor (devkb BL-2, body/legs model).
|
|
2
|
+
|
|
3
|
+
The one process-log sub-signal that's actually recoverable without a ticket
|
|
4
|
+
system: `git tag` already records "this commit was a release," for real,
|
|
5
|
+
cheaply, independent of tickets/commits-clustering/everything else in this
|
|
6
|
+
package. Real data throughout -- source="git-tag", no inference involved.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from devkb.extractors.tickets_git import _run
|
|
13
|
+
from devkb.model import DevKBGraph
|
|
14
|
+
from devkb.project_context import ProjectContext
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _list_tags(repo_root: Path) -> list[str]:
|
|
18
|
+
out = _run(["git", "tag", "--list"], repo_root)
|
|
19
|
+
return [t for t in out.splitlines() if t.strip()]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _tag_commit(tag: str, repo_root: Path) -> str | None:
|
|
23
|
+
out = _run(["git", "rev-list", "-n", "1", tag], repo_root)
|
|
24
|
+
sha = out.strip()
|
|
25
|
+
return sha or None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _tag_date(tag: str, repo_root: Path) -> str:
|
|
29
|
+
# %(creatordate:short) covers both annotated and lightweight tags --
|
|
30
|
+
# `git log -1 <tag> --format=%ad` would report the TAGGED COMMIT's own
|
|
31
|
+
# date, not when the tag itself was created, which is the wrong question
|
|
32
|
+
# for "when was this released."
|
|
33
|
+
out = _run(["git", "for-each-ref", f"refs/tags/{tag}", "--format=%(creatordate:short)"], repo_root)
|
|
34
|
+
return out.strip()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def extract(ctx: ProjectContext) -> DevKBGraph:
|
|
38
|
+
g = DevKBGraph()
|
|
39
|
+
for tag in _list_tags(ctx.repo_root):
|
|
40
|
+
commit = _tag_commit(tag, ctx.repo_root)
|
|
41
|
+
if commit is None:
|
|
42
|
+
continue # a tag git itself can't resolve -- skip, don't guess
|
|
43
|
+
g.add_node("Release", tag, source="git-tag", tagged_at=_tag_date(tag, ctx.repo_root))
|
|
44
|
+
g.add_node("Commit", commit, source="git-tag")
|
|
45
|
+
g.add_edge(
|
|
46
|
+
"TAGS", "Release", tag, "Commit", commit,
|
|
47
|
+
source="git-tag", confidence=1.0, evidence_ref=f"git tag {tag}",
|
|
48
|
+
)
|
|
49
|
+
return g
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Ticket-agnostic Commit/File baseline extractor (devkb BL-1, body/legs model).
|
|
2
|
+
|
|
3
|
+
Phase 1's tickets_git.py only ever finds a commit via `git log --grep=<ticket>`
|
|
4
|
+
-- per ticket. A repo with real commits and zero tickets filed in this
|
|
5
|
+
project's own convention (one project: real source, real commits, real markdown
|
|
6
|
+
design docs, no tickets) gets nothing from that path, by construction, not
|
|
7
|
+
by bug -- devkb/ROADMAP.md's own "body/legs model" section works through why.
|
|
8
|
+
|
|
9
|
+
This extractor walks git history directly, independent of any ticket, and is
|
|
10
|
+
purely additive: DevKBGraph.add_node()/add_edge() dedupe by natural key, so a
|
|
11
|
+
commit found by both this and tickets_git.py's own ticket-mediated walk just
|
|
12
|
+
merges into one node, no conflict, no double-counting.
|
|
13
|
+
|
|
14
|
+
Bounded by --since, not unbounded `--all` -- OCO's own ticket-mediated walk
|
|
15
|
+
already costs ~2m10s at 2,584 commits; an unbounded walk over the same
|
|
16
|
+
history needs the same scale discipline.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from devkb import github
|
|
23
|
+
from devkb.extractors.files_tests import _files_touched, is_bulk
|
|
24
|
+
from devkb.extractors.tickets_git import _run
|
|
25
|
+
from devkb.model import DevKBGraph
|
|
26
|
+
from devkb.project_context import ProjectContext
|
|
27
|
+
|
|
28
|
+
# Wide enough to capture a project's real history without being truly
|
|
29
|
+
# unbounded -- every registered project's real age today fits well within
|
|
30
|
+
# this, while a pathological decades-old import still gets bounded.
|
|
31
|
+
DEFAULT_SINCE = "3 years ago"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _all_commits(repo_root: Path, since: str) -> list[dict[str, str]]:
|
|
35
|
+
out = _run(
|
|
36
|
+
["git", "log", "--all", f"--since={since}", "--format=%H|%ad|%an|%s", "--date=short"],
|
|
37
|
+
repo_root,
|
|
38
|
+
)
|
|
39
|
+
commits = []
|
|
40
|
+
for line in out.splitlines():
|
|
41
|
+
parts = line.split("|", 3)
|
|
42
|
+
if len(parts) == 4:
|
|
43
|
+
commits.append({"hash": parts[0], "date": parts[1], "author": parts[2], "subject": parts[3]})
|
|
44
|
+
return commits
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def extract(ctx: ProjectContext, since: str = DEFAULT_SINCE) -> DevKBGraph:
|
|
48
|
+
g = DevKBGraph()
|
|
49
|
+
authors = github.settings(ctx.repo_root).authors
|
|
50
|
+
for c in _all_commits(ctx.repo_root, since):
|
|
51
|
+
g.add_node(
|
|
52
|
+
"Commit", c["hash"], source="git-log-baseline",
|
|
53
|
+
short_hash=c["hash"][:10], date=c["date"], author=c["author"], subject=github.public_subject(c["subject"], authors),
|
|
54
|
+
)
|
|
55
|
+
touched = _files_touched(c["hash"], ctx.repo_root)
|
|
56
|
+
if is_bulk(touched): # ODSF-314: a bulk change keeps its size, not one edge per file
|
|
57
|
+
g.add_node("Commit", c["hash"], source="git-log-baseline", files_changed=len(touched), bulk=True)
|
|
58
|
+
continue
|
|
59
|
+
for path in touched:
|
|
60
|
+
g.add_node("File", path, source="git-log-baseline")
|
|
61
|
+
g.add_edge(
|
|
62
|
+
"TOUCHES", "Commit", c["hash"], "File", path,
|
|
63
|
+
source="git-log-baseline", confidence=1.0,
|
|
64
|
+
evidence_ref=f"git diff-tree --name-only {c['hash'][:10]}",
|
|
65
|
+
)
|
|
66
|
+
return g
|