devkb 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. devkb/__init__.py +11 -0
  2. devkb/audit.py +173 -0
  3. devkb/bench/add_packs.py +52 -0
  4. devkb/bench/cli_tasks.py +105 -0
  5. devkb/bench/confirm_tasks.py +149 -0
  6. devkb/bench/module_truth.py +249 -0
  7. devkb/bench/oco_tasks.py +216 -0
  8. devkb/bench/pipeline.py +313 -0
  9. devkb/bench/process_tasks.py +135 -0
  10. devkb/bench/report.py +74 -0
  11. devkb/bench/run.py +174 -0
  12. devkb/bench/truth_check.py +148 -0
  13. devkb/bench/workflow_report.py +64 -0
  14. devkb/bench/workflow_run.py +206 -0
  15. devkb/bench/workflow_tasks.py +227 -0
  16. devkb/build.py +258 -0
  17. devkb/build_symbols.py +1394 -0
  18. devkb/call_recall.py +159 -0
  19. devkb/cli.py +53 -0
  20. devkb/coverage.py +155 -0
  21. devkb/edge_truth.py +272 -0
  22. devkb/extractors/backlog.py +113 -0
  23. devkb/extractors/component_subsystem.py +116 -0
  24. devkb/extractors/configuration.py +267 -0
  25. devkb/extractors/files_tests.py +220 -0
  26. devkb/extractors/flows.py +117 -0
  27. devkb/extractors/lifecycle.py +258 -0
  28. devkb/extractors/members.py +160 -0
  29. devkb/extractors/pull_requests.py +271 -0
  30. devkb/extractors/releases.py +49 -0
  31. devkb/extractors/source_baseline.py +66 -0
  32. devkb/extractors/spec_docs.py +220 -0
  33. devkb/extractors/symbol_tests.py +131 -0
  34. devkb/extractors/test_inventory.py +38 -0
  35. devkb/extractors/ticket_similarity.py +193 -0
  36. devkb/extractors/tickets_git.py +151 -0
  37. devkb/extractors/warnings.py +230 -0
  38. devkb/extractors/work_units.py +90 -0
  39. devkb/github.py +515 -0
  40. devkb/gitutil.py +16 -0
  41. devkb/index.py +51 -0
  42. devkb/init.py +178 -0
  43. devkb/mcp_server.py +216 -0
  44. devkb/model.py +221 -0
  45. devkb/module_file_bridge.py +57 -0
  46. devkb/ontology.md +178 -0
  47. devkb/packs.py +462 -0
  48. devkb/process_api.py +1601 -0
  49. devkb/project_context.py +191 -0
  50. devkb/query.py +251 -0
  51. devkb/service.py +324 -0
  52. devkb/startup.py +103 -0
  53. devkb/test_discovery.py +76 -0
  54. devkb/ts_symbols.py +291 -0
  55. devkb-0.2.0.dist-info/METADATA +162 -0
  56. devkb-0.2.0.dist-info/RECORD +61 -0
  57. devkb-0.2.0.dist-info/WHEEL +5 -0
  58. devkb-0.2.0.dist-info/entry_points.txt +2 -0
  59. devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
  60. devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
  61. devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/github.py ADDED
@@ -0,0 +1,515 @@
1
+ """The opt-in online layer of ADR-DEVKB-007: pull requests and issues from GitHub, cached on this machine.
2
+
3
+ Nothing here runs unless the caller asked for it (`devkb build --github`, or `{"github": {"enabled": true}}` in
4
+ devkb.json). A build without that never imports a client, never reads a token and never opens a socket.
5
+
6
+ settings() what the repository's devkb.json and the command line ask for
7
+ origin_remote() host, owner and repository from the `origin` remote, credentials dropped
8
+ fetch() pull requests and issues, as plain dicts holding only the fields devkb stores
9
+ cache `<devkb dir>/github/{pull_requests,issues}.json`, keyed by each item's `updated_at`
10
+
11
+ What is requested, and so what can be stored: numbers, titles, states, labels, branch names, merge commit, commit ids,
12
+ the issues a pull request closes, issue type, milestone and sub-issue parent. Never a description, a comment, a review
13
+ or an email. An author login is requested only when `"github": {"authors": true}`.
14
+
15
+ Failure is a result, not an exception: an unreachable or rate-limited API leaves the build with the offline layer and
16
+ whatever the cache already held, and `fetch()` says which (`report`), so every affected answer can state its coverage.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ import os
22
+ import re
23
+ import subprocess
24
+ import urllib.error
25
+ import urllib.request
26
+ from dataclasses import dataclass, field
27
+ from datetime import datetime, timezone
28
+ from pathlib import Path
29
+ from typing import Callable
30
+
31
+ from devkb.project_context import load_config
32
+
33
+ #: transport(query, variables) -> the decoded JSON response `{"data": ..., "errors": ...}`. Raises GitHubError.
34
+ Transport = Callable[[str, dict], dict]
35
+
36
+ PAGE = 100
37
+ MAX_COMMIT_PAGES = 5 # the API lists at most 250 commits of a pull request anyway
38
+ CACHE_VERSION = 2 # 2: a listing records whether it finished, and where to resume
39
+
40
+
41
+ class GitHubError(Exception):
42
+ """kind: unreachable, rate-limited, unauthorized, not-found, schema or graphql."""
43
+
44
+ def __init__(self, kind: str, message: str) -> None:
45
+ super().__init__(message)
46
+ self.kind, self.message = kind, message
47
+
48
+
49
+ # ── settings and remote ──────────────────────────────────────────────────────────────────
50
+ @dataclass(frozen=True)
51
+ class Settings:
52
+ enabled: bool = False
53
+ authors: bool = False
54
+ host: str | None = None
55
+
56
+
57
+ def settings(repo_root: Path, flag: bool | None = None) -> Settings:
58
+ """`flag` is the command line: True for --github, False for --no-github, None to follow devkb.json."""
59
+ raw = load_config(repo_root).get("github")
60
+ cfg = raw if isinstance(raw, dict) else {}
61
+ enabled = bool(cfg.get("enabled")) if flag is None else flag
62
+ host = cfg.get("host") if isinstance(cfg.get("host"), str) and cfg.get("host") else None
63
+ return Settings(enabled=enabled, authors=cfg.get("authors") is True, host=host)
64
+
65
+
66
+ _MERGE_FROM = re.compile(r"^(Merge pull request #\d+ from )[^/\s]+/(\S.*)$")
67
+
68
+
69
+ def public_subject(subject: str, authors: bool = False) -> str:
70
+ """A commit subject as devkb stores it. `Merge pull request #N from owner/branch` names the head repository's
71
+ owner, which is the author's (or their fork's) login, so unless `"github": {"authors": true}` it is stored as
72
+ `Merge pull request #N from branch` (ADR-DEVKB-007 item 1). Git itself still holds the original."""
73
+ if authors:
74
+ return subject
75
+ m = _MERGE_FROM.match(subject)
76
+ return m.group(1) + m.group(2) if m else subject
77
+
78
+
79
+ @dataclass(frozen=True)
80
+ class Remote:
81
+ host: str
82
+ owner: str
83
+ repo: str
84
+
85
+ @property
86
+ def slug(self) -> str:
87
+ return f"{self.host}/{self.owner}/{self.repo}"
88
+
89
+ def web(self, kind: str, number: int) -> str:
90
+ return f"https://{self.host}/{self.owner}/{self.repo}/{kind}/{number}"
91
+
92
+
93
+ _REMOTE = re.compile(r"^(?:[A-Za-z][\w+.-]*://)?(?:[^@/]+@)?([^/:@]+)(?::\d+)?[:/](.+?)(?:\.git)?/?$")
94
+
95
+
96
+ def parse_remote(url: str) -> Remote | None:
97
+ """host/owner/repo from any of the forms git accepts; a user name or token in the URL is discarded."""
98
+ m = _REMOTE.match((url or "").strip())
99
+ if not m:
100
+ return None
101
+ parts = [p for p in m.group(2).split("/") if p]
102
+ if len(parts) != 2:
103
+ return None
104
+ return Remote(m.group(1).lower(), parts[0], parts[1])
105
+
106
+
107
+ def is_hosting(host: str, extra: str | None = None) -> bool:
108
+ return "github" in host or (extra is not None and host == extra.lower())
109
+
110
+
111
+ def origin_remote(repo_root: Path, extra_host: str | None = None) -> Remote | None:
112
+ """The `origin` remote when it is a GitHub host, else None: a repository without one gets no pull-request nodes."""
113
+ try:
114
+ out = subprocess.run(["git", "remote", "get-url", "origin"], cwd=str(repo_root), capture_output=True,
115
+ text=True, timeout=10).stdout
116
+ except (OSError, subprocess.SubprocessError):
117
+ return None
118
+ remote = parse_remote(out)
119
+ return remote if remote and is_hosting(remote.host, extra_host) else None
120
+
121
+
122
+ # ── authentication and the real transport ─────────────────────────────────────────────────
123
+ def resolve_token(host: str) -> str | None:
124
+ """GITHUB_TOKEN (github.com) or the enterprise variables, else the `gh` login for that host. A token is only ever
125
+ sent to the host it was found for."""
126
+ names: tuple[str, ...] = ("GITHUB_TOKEN", "GH_TOKEN") if host == "github.com" else ("GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN")
127
+ if host != "github.com" and (os.environ.get("GH_HOST") == host or host in (os.environ.get("GITHUB_SERVER_URL") or "")):
128
+ names += ("GITHUB_TOKEN", "GH_TOKEN") # CI on that enterprise host (GitHub Actions sets GITHUB_SERVER_URL)
129
+ for name in names:
130
+ if os.environ.get(name):
131
+ return os.environ[name].strip()
132
+ try:
133
+ r = subprocess.run(["gh", "auth", "token", "--hostname", host], capture_output=True, text=True, timeout=15)
134
+ except (OSError, subprocess.SubprocessError):
135
+ return None
136
+ return r.stdout.strip() or None if r.returncode == 0 else None
137
+
138
+
139
+ def api_url(host: str) -> str:
140
+ return "https://api.github.com/graphql" if host == "github.com" else f"https://{host}/api/graphql"
141
+
142
+
143
+ def http_transport(host: str, token: str, timeout: float = 60.0) -> Transport:
144
+ url = api_url(host)
145
+
146
+ def send(query: str, variables: dict) -> dict:
147
+ req = urllib.request.Request(
148
+ url, data=json.dumps({"query": query, "variables": variables}).encode(), method="POST",
149
+ headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json", "User-Agent": "devkb",
150
+ "GraphQL-Features": "issue_types,sub_issues"})
151
+ try:
152
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
153
+ return json.loads(resp.read().decode())
154
+ except urllib.error.HTTPError as exc:
155
+ body = ""
156
+ try:
157
+ body = exc.read().decode(errors="replace")[:300].lower()
158
+ except OSError:
159
+ pass
160
+ if exc.code == 401:
161
+ raise GitHubError("unauthorized", "the token was rejected (401)") from None
162
+ if exc.code == 429 or (exc.code == 403 and ("rate limit" in body or exc.headers.get("x-ratelimit-remaining") == "0"
163
+ or exc.headers.get("retry-after"))):
164
+ raise GitHubError("rate-limited", f"rate limited (HTTP {exc.code})") from None
165
+ raise GitHubError("unreachable", f"HTTP {exc.code}") from None
166
+ except (urllib.error.URLError, OSError, TimeoutError, ValueError) as exc:
167
+ raise GitHubError("unreachable", f"{type(exc).__name__}: {getattr(exc, 'reason', exc)}") from None
168
+ return send
169
+
170
+
171
+ # ── queries ───────────────────────────────────────────────────────────────────────────────
172
+ _PR_COMMITS = "commits(first: 100%s) { totalCount pageInfo { hasNextPage endCursor } nodes { commit { oid } } }"
173
+ _PR_QUERY = """
174
+ query($owner: String!, $name: String!, $after: String) {
175
+ repository(owner: $owner, name: $name) {
176
+ pullRequests(first: %(page)d, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}) {
177
+ pageInfo { hasNextPage endCursor }
178
+ nodes {
179
+ number title state mergedAt updatedAt baseRefName headRefName url
180
+ mergeCommit { oid }
181
+ labels(first: 30) { nodes { name } }
182
+ closingIssuesReferences(first: 50) { nodes { number repository { nameWithOwner } } }
183
+ %(commits)s
184
+ %(author)s
185
+ }
186
+ }
187
+ }
188
+ }"""
189
+ _PR_MORE_COMMITS = """
190
+ query($owner: String!, $name: String!, $number: Int!, $after: String) {
191
+ repository(owner: $owner, name: $name) {
192
+ pullRequest(number: $number) { %s }
193
+ }
194
+ }""" % (_PR_COMMITS % ", after: $after")
195
+ _ISSUE_QUERY = """
196
+ query($owner: String!, $name: String!, $after: String) {
197
+ repository(owner: $owner, name: $name) {
198
+ issues(first: %(page)d, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}) {
199
+ pageInfo { hasNextPage endCursor }
200
+ nodes {
201
+ number title state createdAt closedAt updatedAt url
202
+ labels(first: 30) { nodes { name } }
203
+ milestone { title }
204
+ %(extra)s
205
+ %(author)s
206
+ }
207
+ }
208
+ }
209
+ }"""
210
+ _ISSUE_EXTRA_FULL = "stateReason issueType { name } parent { number repository { nameWithOwner } } subIssues(first: 1) { totalCount }"
211
+ _ISSUE_EXTRA_BASIC = "stateReason"
212
+ _ISSUE_EXTRA_MIN = ""
213
+
214
+
215
+ def _author(authors: bool) -> str:
216
+ return "author { login }" if authors else ""
217
+
218
+
219
+ def pr_query(authors: bool) -> str:
220
+ return _PR_QUERY % {"page": PAGE, "commits": _PR_COMMITS % "", "author": _author(authors)}
221
+
222
+
223
+ def issue_query(authors: bool, extra: str) -> str:
224
+ return _ISSUE_QUERY % {"page": PAGE, "extra": extra, "author": _author(authors)}
225
+
226
+
227
+ # ── normalising what comes back: the whitelist is the privacy rule ─────────────────────────
228
+ def _labels(node: dict) -> list[str]:
229
+ return sorted({n.get("name") for n in ((node.get("labels") or {}).get("nodes") or []) if n and n.get("name")})
230
+
231
+
232
+ def _same_repo(ref: dict, repo_slug: str) -> bool:
233
+ name = (ref.get("repository") or {}).get("nameWithOwner")
234
+ return not name or not repo_slug or name.lower() == repo_slug.lower()
235
+
236
+
237
+ def _pr_record(node: dict, authors: bool, repo_slug: str = "") -> dict:
238
+ state = str(node.get("state") or "").lower()
239
+ rec = {
240
+ "number": int(node["number"]), "title": node.get("title") or "", "state": state if state in ("open", "merged", "closed") else "closed",
241
+ "merged_at": node.get("mergedAt"), "updated_at": node.get("updatedAt") or "", "base": node.get("baseRefName"),
242
+ "head": node.get("headRefName"), "url": node.get("url"), "labels": _labels(node),
243
+ "merge_commit": ((node.get("mergeCommit") or {}).get("oid")),
244
+ # only this repository's issues: a reference to another repository's #N is not this repository's GH-N
245
+ "closes": sorted({int(n["number"]) for n in ((node.get("closingIssuesReferences") or {}).get("nodes") or [])
246
+ if n and _same_repo(n, repo_slug)}),
247
+ "commits": [c["commit"]["oid"] for c in ((node.get("commits") or {}).get("nodes") or []) if c and c.get("commit")],
248
+ "commits_total": int((node.get("commits") or {}).get("totalCount") or 0),
249
+ }
250
+ if authors and (node.get("author") or {}).get("login"):
251
+ rec["author"] = node["author"]["login"]
252
+ return rec
253
+
254
+
255
+ def issue_type_from(issue_type: str | None, labels: list[str], has_children: bool) -> str:
256
+ """ADR-DEVKB-007 item 2: the issue's type, else a label (bug, epic, enhancement/feature), else EPIC when it has
257
+ sub-issues, else ISSUE. A typed issue keeps its type when it has sub-issues (a Bug with sub-issues stays a Bug);
258
+ `has_sub_issues` is recorded beside it and the `epics` alias reads both."""
259
+ if issue_type:
260
+ return re.sub(r"[^A-Z0-9]+", "_", issue_type.upper()).strip("_") or "ISSUE"
261
+ lower = {l.lower() for l in labels}
262
+ if "epic" in lower:
263
+ return "EPIC"
264
+ if "bug" in lower:
265
+ return "BUG"
266
+ if lower & {"enhancement", "feature"}:
267
+ return "FEATURE"
268
+ return "EPIC" if has_children else "ISSUE"
269
+
270
+
271
+ def _issue_record(node: dict, authors: bool, repo_slug: str) -> dict:
272
+ state = str(node.get("state") or "").lower()
273
+ labels = _labels(node)
274
+ parent = node.get("parent") or {}
275
+ same_repo = (parent.get("repository") or {}).get("nameWithOwner", "").lower() in ("", repo_slug.lower())
276
+ rec = {
277
+ "number": int(node["number"]), "title": node.get("title") or "", "state": state if state in ("open", "closed") else "closed",
278
+ "state_reason": (str(node["stateReason"]).lower() if node.get("stateReason") else None),
279
+ "labels": labels, "milestone": (node.get("milestone") or {}).get("title"), "url": node.get("url"),
280
+ "created_at": node.get("createdAt"), "closed_at": node.get("closedAt"), "updated_at": node.get("updatedAt") or "",
281
+ "ticket_type": issue_type_from((node.get("issueType") or {}).get("name"), labels,
282
+ int((node.get("subIssues") or {}).get("totalCount") or 0) > 0),
283
+ "parent": int(parent["number"]) if parent.get("number") and same_repo else None,
284
+ "has_sub_issues": int((node.get("subIssues") or {}).get("totalCount") or 0) > 0,
285
+ }
286
+ if authors and (node.get("author") or {}).get("login"):
287
+ rec["author"] = node["author"]["login"]
288
+ return rec
289
+
290
+
291
+ # ── the cache ────────────────────────────────────────────────────────────────────────────
292
+ class Cache:
293
+ """One JSON file per kind. Holds only the normalised records above, so it cannot hold a description or review."""
294
+
295
+ def __init__(self, directory: Path, remote: Remote, authors: bool) -> None:
296
+ self.dir, self.remote, self.authors = directory, remote, authors
297
+
298
+ def _path(self, kind: str) -> Path:
299
+ return self.dir / f"{kind}.json"
300
+
301
+ def load(self, kind: str) -> tuple[dict[int, dict], str, dict]:
302
+ """(records by number, when they were fetched, listing state {"complete", "cursor"}). Empty when the cache is
303
+ for another repository, another schema, or was written without the author logins now asked for."""
304
+ try:
305
+ data = json.loads(self._path(kind).read_text())
306
+ except (OSError, ValueError):
307
+ return {}, "", {}
308
+ if not isinstance(data, dict) or data.get("version") != CACHE_VERSION or data.get("repo") != self.remote.slug:
309
+ return {}, "", {}
310
+ if self.authors and not data.get("authors"):
311
+ return {}, "", {}
312
+ items = {int(k): v for k, v in (data.get("items") or {}).items() if isinstance(v, dict)}
313
+ if not self.authors:
314
+ for v in items.values():
315
+ v.pop("author", None)
316
+ return items, str(data.get("fetched_at") or ""), {"complete": bool(data.get("complete")), "cursor": data.get("cursor")}
317
+
318
+ def save(self, kind: str, items: dict[int, dict], complete: bool = True, cursor: str | None = None) -> None:
319
+ clean = {str(n): ({k: v for k, v in rec.items() if self.authors or k != "author"}) for n, rec in sorted(items.items())}
320
+ try:
321
+ self.dir.mkdir(parents=True, exist_ok=True)
322
+ self._path(kind).write_text(json.dumps({
323
+ "version": CACHE_VERSION, "repo": self.remote.slug, "authors": self.authors, "complete": complete, "cursor": cursor,
324
+ "fetched_at": datetime.now(timezone.utc).isoformat(), "items": clean}, indent=0))
325
+ except OSError:
326
+ pass
327
+
328
+
329
+ # ── fetching ─────────────────────────────────────────────────────────────────────────────
330
+ @dataclass
331
+ class Online:
332
+ """What the online layer knows. `prs` and `issues` may hold cached records from an earlier build when a refresh
333
+ failed; `report` says so."""
334
+ prs: dict[int, dict] = field(default_factory=dict)
335
+ issues: dict[int, dict] = field(default_factory=dict)
336
+ report: dict = field(default_factory=dict)
337
+
338
+
339
+ def _gql(transport: Transport, query: str, variables: dict) -> dict:
340
+ resp = transport(query, variables)
341
+ errors = resp.get("errors")
342
+ if errors or not isinstance(resp.get("data"), dict):
343
+ text = "; ".join(str(e.get("message", e)) if isinstance(e, dict) else str(e) for e in (errors or []))[:300]
344
+ kinds = " ".join(str(e.get("type", "")) for e in (errors or []) if isinstance(e, dict))
345
+ if "RATE_LIMITED" in kinds or "rate limit" in text.lower():
346
+ raise GitHubError("rate-limited", text or "rate limited")
347
+ if "NOT_FOUND" in kinds:
348
+ raise GitHubError("not-found", text or "repository not found or not visible to this token")
349
+ if re.search(r"doesn't exist on type|Unknown type|Field '.*' doesn't exist", text):
350
+ raise GitHubError("schema", text)
351
+ if "bad credentials" in text.lower():
352
+ raise GitHubError("unauthorized", text)
353
+ raise GitHubError("graphql", text or "empty response")
354
+ return resp["data"]
355
+
356
+
357
+ def _walk(transport: Transport, query: str, remote: Remote, connection: str, keep: Callable[[dict], dict],
358
+ items: dict[int, dict], state: dict, save: Callable[[dict[int, dict], bool, str | None], None],
359
+ finish: Callable[[dict], None] | None = None) -> tuple[dict[int, dict], int]:
360
+ """Read one listing, newest-updated first, into `items`. Returns (records read, requests made); a GitHubError
361
+ propagates after what was read is kept.
362
+
363
+ A finished listing is refreshed incrementally: paging stops at records older than anything cached, and the cache
364
+ is written only when that pass ends, because a partial pass would raise the "newest cached" mark over records not
365
+ yet read. An unfinished listing (a first build, or one cut short) is resumed from its saved cursor and written
366
+ after every page with `complete` false until the last page, so a repository too large for one rate-limit window
367
+ is read across several builds instead of starting over each time."""
368
+ resume = not state.get("complete")
369
+ after = state.get("cursor") if resume else None
370
+ newest = "" if resume else max((r.get("updated_at") or "" for r in items.values()), default="")
371
+ fresh: dict[int, dict] = {}
372
+ calls = 0
373
+ while True:
374
+ data = _gql(transport, query, {"owner": remote.owner, "name": remote.repo, "after": after})
375
+ calls += 1
376
+ conn = ((data.get("repository") or {}).get(connection)) or {}
377
+ page: dict[int, dict] = {}
378
+ stop = False
379
+ for node in conn.get("nodes") or []:
380
+ if not node:
381
+ continue
382
+ rec = keep(node)
383
+ if newest and rec["updated_at"] and rec["updated_at"] < newest:
384
+ stop = True
385
+ continue
386
+ page[rec["number"]] = rec
387
+ for rec in page.values():
388
+ if finish:
389
+ finish(rec)
390
+ fresh.update(page)
391
+ info = conn.get("pageInfo") or {}
392
+ more = bool(info.get("hasNextPage")) and not stop
393
+ if resume:
394
+ items.update(page)
395
+ save(items, not more, info.get("endCursor") if more else None)
396
+ if not more:
397
+ if not resume:
398
+ items.update(fresh)
399
+ save(items, True, None)
400
+ return fresh, calls
401
+ after = info.get("endCursor")
402
+
403
+
404
+ def _more_commits(transport: Transport, remote: Remote, rec: dict) -> None:
405
+ """A pull request with more than 100 commits: read the rest, up to the API's own cap. Raises GitHubError; the
406
+ caller keeps the commits read so far, and `commits_total` then says the list is cut short."""
407
+ after, pages = rec.pop("_after", None), 1
408
+ while after and pages < MAX_COMMIT_PAGES:
409
+ data = _gql(transport, _PR_MORE_COMMITS, {"owner": remote.owner, "name": remote.repo, "number": rec["number"], "after": after})
410
+ conn = ((data.get("repository") or {}).get("pullRequest") or {}).get("commits") or {}
411
+ rec["commits"] += [c["commit"]["oid"] for c in conn.get("nodes") or [] if c and c.get("commit")]
412
+ info = conn.get("pageInfo") or {}
413
+ after = info.get("endCursor") if info.get("hasNextPage") else None
414
+ pages += 1
415
+
416
+
417
+ def fetch(remote: Remote, cache_dir: Path, authors: bool = False, transport: Transport | None = None) -> Online:
418
+ """Fetch (or refresh from the cache) the repository's pull requests and issues. Never raises: the outcome is in
419
+ `report`: status `ok`, `partial` (one kind failed), or `failed` (neither could be fetched), with the reason."""
420
+ cache = Cache(cache_dir, remote, authors)
421
+ out = Online()
422
+ out.prs, pr_at, pr_state = cache.load("pull_requests")
423
+ out.issues, is_at, is_state = cache.load("issues")
424
+ report: dict = {"remote": remote.slug, "mode": "online", "authors": authors, "status": "ok", "reason": None,
425
+ "not_fetched": [], "requests": 0}
426
+ if transport is None:
427
+ token = resolve_token(remote.host)
428
+ if not token:
429
+ report.update(status="failed", reason="no GITHUB_TOKEN and no `gh` login for " + remote.host,
430
+ not_fetched=["pull_requests", "issues"], cached_from=_cached_note(pr_at, is_at))
431
+ _describe(report, out, {"pull_requests": False, "issues": False})
432
+ return out
433
+ transport = http_transport(remote.host, token)
434
+
435
+ ok = {"pull_requests": False, "issues": False}
436
+ reasons: list[str] = []
437
+ stale_cache = {k: bool(v) for k, v in (("pull_requests", out.prs), ("issues", out.issues))}
438
+
439
+ slug = f"{remote.owner}/{remote.repo}"
440
+ commits_cut: list[int] = []
441
+
442
+ def keep_pr(node: dict) -> dict:
443
+ rec = _pr_record(node, authors, slug)
444
+ info = (node.get("commits") or {}).get("pageInfo") or {}
445
+ if info.get("hasNextPage"):
446
+ rec["_after"] = info.get("endCursor")
447
+ return rec
448
+
449
+ def finish_pr(rec: dict) -> None:
450
+ """The rest of a long commit list; a failure here costs that one list, never the page."""
451
+ try:
452
+ _more_commits(transport, remote, rec)
453
+ except GitHubError:
454
+ commits_cut.append(rec["number"])
455
+ rec.pop("_after", None)
456
+
457
+ try:
458
+ fresh, calls = _walk(transport, pr_query(authors), remote, "pullRequests", keep_pr, out.prs, pr_state,
459
+ lambda items, done, cursor: cache.save("pull_requests", items, done, cursor), finish_pr)
460
+ report["requests"] += calls
461
+ ok["pull_requests"] = True
462
+ report["pull_requests_fetched"] = len(fresh)
463
+ except GitHubError as exc:
464
+ reasons.append(f"pull requests: {exc.kind}: {exc.message}")
465
+ report["kind"] = exc.kind
466
+
467
+ if commits_cut:
468
+ report["commit_lists_not_fetched"] = sorted(commits_cut)[:50]
469
+ last_error: GitHubError | None = None
470
+ for extra in (_ISSUE_EXTRA_FULL, _ISSUE_EXTRA_BASIC, _ISSUE_EXTRA_MIN):
471
+ try:
472
+ fresh, calls = _walk(transport, issue_query(authors, extra), remote, "issues",
473
+ lambda n: _issue_record(n, authors, slug), out.issues, is_state,
474
+ lambda items, done, cursor: cache.save("issues", items, done, cursor))
475
+ report["requests"] += calls
476
+ ok["issues"] = True
477
+ report["issues_fetched"] = len(fresh)
478
+ report["issue_fields"] = "full" if extra == _ISSUE_EXTRA_FULL else "basic"
479
+ last_error = None
480
+ break
481
+ except GitHubError as exc: # a host without issue types or sub-issues: ask for less, not for nothing
482
+ last_error = exc
483
+ if exc.kind != "schema":
484
+ break
485
+ if last_error is not None:
486
+ reasons.append(f"issues: {last_error.kind}: {last_error.message}")
487
+ report.setdefault("kind", last_error.kind)
488
+
489
+ report["listing_incomplete"] = [k for k in ("pull_requests", "issues")
490
+ if not Cache(cache_dir, remote, authors).load(k)[2].get("complete")]
491
+ report["not_fetched"] = [k for k, v in ok.items() if not v]
492
+ if not report["not_fetched"]:
493
+ report["status"] = "ok"
494
+ else:
495
+ report["status"] = "partial" if any(ok.values()) else "failed"
496
+ report["reason"] = "; ".join(reasons)
497
+ report["stale_cache_used"] = [k for k in report["not_fetched"] if stale_cache[k]]
498
+ report["cached_from"] = _cached_note(pr_at, is_at)
499
+ _describe(report, out, ok)
500
+ out.report = report
501
+ return out
502
+
503
+
504
+ def _cached_note(pr_at: str, is_at: str) -> dict:
505
+ return {k: v for k, v in (("pull_requests", pr_at), ("issues", is_at)) if v}
506
+
507
+
508
+ def _describe(report: dict, out: Online, ok: dict[str, bool]) -> None:
509
+ report["pull_requests"] = {"count": len(out.prs), "fresh": ok["pull_requests"]}
510
+ report["issues"] = {"count": len(out.issues), "fresh": ok["issues"]}
511
+ truncated = sorted(n for n, r in out.prs.items() if r.get("commits_total", 0) > len(r.get("commits", [])))
512
+ if truncated:
513
+ report["commit_lists_truncated"] = truncated[:50]
514
+ # callers read the report from the returned object; keep it on `out` even on the early return
515
+ out.report = report
devkb/gitutil.py ADDED
@@ -0,0 +1,16 @@
1
+ """Tiny git helpers shared across devkb/build.py and devkb/build_symbols.py --
2
+ split out specifically to avoid a circular import between them (build.py
3
+ needs build_symbols.load_cache(), build_symbols.py needs this)."""
4
+ from __future__ import annotations
5
+
6
+ import subprocess
7
+ from pathlib import Path
8
+
9
+
10
+ def git_head(repo_root: Path) -> str:
11
+ try:
12
+ r = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True,
13
+ cwd=str(repo_root), timeout=10)
14
+ return r.stdout.strip() or "unknown"
15
+ except (OSError, subprocess.SubprocessError):
16
+ return "unknown"
devkb/index.py ADDED
@@ -0,0 +1,51 @@
1
+ """Load-time index over a project's .devkb/ store.
2
+
3
+ Mirrors scripts/traverse_graph.py's GraphIndex pattern (build by_uid/out_edges/
4
+ in_edges once at load, O(1) lookups from there) rather than GraphService's
5
+ per-call O(N) scan (webapp/backend/app/services/graph_service.py:171) --
6
+ applying that lesson from the start instead of retrofitting it after a
7
+ measured stall, per ADR-SDP-DEVKB-001.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ from collections import defaultdict
13
+ from dataclasses import dataclass, field
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+
18
+ @dataclass
19
+ class DevKBIndex:
20
+ by_uid: dict[str, dict[str, Any]] = field(default_factory=dict)
21
+ by_type: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
22
+ out_edges: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
23
+ in_edges: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
24
+ by_natural_key: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict)
25
+
26
+ def node(self, node_type: str, natural_key: str) -> dict[str, Any] | None:
27
+ return self.by_natural_key.get((node_type, natural_key))
28
+
29
+
30
+ def _read_jsonl(path: Path) -> list[dict[str, Any]]:
31
+ if not path.exists():
32
+ return []
33
+ out = []
34
+ for line in path.read_text().splitlines():
35
+ line = line.strip()
36
+ if not line:
37
+ continue
38
+ out.append(json.loads(line))
39
+ return out
40
+
41
+
42
+ def load(devkb_dir: Path) -> DevKBIndex:
43
+ idx = DevKBIndex()
44
+ for node in _read_jsonl(devkb_dir / "nodes.jsonl"):
45
+ idx.by_uid[node["uid"]] = node
46
+ idx.by_type[node["type"]].append(node)
47
+ idx.by_natural_key[(node["type"], node["natural_key"])] = node
48
+ for edge in _read_jsonl(devkb_dir / "edges.jsonl"):
49
+ idx.out_edges[edge["from_uid"]].append(edge)
50
+ idx.in_edges[edge["to_uid"]].append(edge)
51
+ return idx