devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/github.py
ADDED
|
@@ -0,0 +1,515 @@
|
|
|
1
|
+
"""The opt-in online layer of ADR-DEVKB-007: pull requests and issues from GitHub, cached on this machine.
|
|
2
|
+
|
|
3
|
+
Nothing here runs unless the caller asked for it (`devkb build --github`, or `{"github": {"enabled": true}}` in
|
|
4
|
+
devkb.json). A build without that never imports a client, never reads a token and never opens a socket.
|
|
5
|
+
|
|
6
|
+
settings() what the repository's devkb.json and the command line ask for
|
|
7
|
+
origin_remote() host, owner and repository from the `origin` remote, credentials dropped
|
|
8
|
+
fetch() pull requests and issues, as plain dicts holding only the fields devkb stores
|
|
9
|
+
cache `<devkb dir>/github/{pull_requests,issues}.json`, keyed by each item's `updated_at`
|
|
10
|
+
|
|
11
|
+
What is requested, and so what can be stored: numbers, titles, states, labels, branch names, merge commit, commit ids,
|
|
12
|
+
the issues a pull request closes, issue type, milestone and sub-issue parent. Never a description, a comment, a review
|
|
13
|
+
or an email. An author login is requested only when `"github": {"authors": true}`.
|
|
14
|
+
|
|
15
|
+
Failure is a result, not an exception: an unreachable or rate-limited API leaves the build with the offline layer and
|
|
16
|
+
whatever the cache already held, and `fetch()` says which (`report`), so every affected answer can state its coverage.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import subprocess
|
|
24
|
+
import urllib.error
|
|
25
|
+
import urllib.request
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Callable
|
|
30
|
+
|
|
31
|
+
from devkb.project_context import load_config
|
|
32
|
+
|
|
33
|
+
#: transport(query, variables) -> the decoded JSON response `{"data": ..., "errors": ...}`. Raises GitHubError.
|
|
34
|
+
Transport = Callable[[str, dict], dict]
|
|
35
|
+
|
|
36
|
+
PAGE = 100
|
|
37
|
+
MAX_COMMIT_PAGES = 5 # the API lists at most 250 commits of a pull request anyway
|
|
38
|
+
CACHE_VERSION = 2 # 2: a listing records whether it finished, and where to resume
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class GitHubError(Exception):
|
|
42
|
+
"""kind: unreachable, rate-limited, unauthorized, not-found, schema or graphql."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, kind: str, message: str) -> None:
|
|
45
|
+
super().__init__(message)
|
|
46
|
+
self.kind, self.message = kind, message
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ── settings and remote ──────────────────────────────────────────────────────────────────
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class Settings:
|
|
52
|
+
enabled: bool = False
|
|
53
|
+
authors: bool = False
|
|
54
|
+
host: str | None = None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def settings(repo_root: Path, flag: bool | None = None) -> Settings:
|
|
58
|
+
"""`flag` is the command line: True for --github, False for --no-github, None to follow devkb.json."""
|
|
59
|
+
raw = load_config(repo_root).get("github")
|
|
60
|
+
cfg = raw if isinstance(raw, dict) else {}
|
|
61
|
+
enabled = bool(cfg.get("enabled")) if flag is None else flag
|
|
62
|
+
host = cfg.get("host") if isinstance(cfg.get("host"), str) and cfg.get("host") else None
|
|
63
|
+
return Settings(enabled=enabled, authors=cfg.get("authors") is True, host=host)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
_MERGE_FROM = re.compile(r"^(Merge pull request #\d+ from )[^/\s]+/(\S.*)$")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def public_subject(subject: str, authors: bool = False) -> str:
|
|
70
|
+
"""A commit subject as devkb stores it. `Merge pull request #N from owner/branch` names the head repository's
|
|
71
|
+
owner, which is the author's (or their fork's) login, so unless `"github": {"authors": true}` it is stored as
|
|
72
|
+
`Merge pull request #N from branch` (ADR-DEVKB-007 item 1). Git itself still holds the original."""
|
|
73
|
+
if authors:
|
|
74
|
+
return subject
|
|
75
|
+
m = _MERGE_FROM.match(subject)
|
|
76
|
+
return m.group(1) + m.group(2) if m else subject
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True)
|
|
80
|
+
class Remote:
|
|
81
|
+
host: str
|
|
82
|
+
owner: str
|
|
83
|
+
repo: str
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def slug(self) -> str:
|
|
87
|
+
return f"{self.host}/{self.owner}/{self.repo}"
|
|
88
|
+
|
|
89
|
+
def web(self, kind: str, number: int) -> str:
|
|
90
|
+
return f"https://{self.host}/{self.owner}/{self.repo}/{kind}/{number}"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
_REMOTE = re.compile(r"^(?:[A-Za-z][\w+.-]*://)?(?:[^@/]+@)?([^/:@]+)(?::\d+)?[:/](.+?)(?:\.git)?/?$")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def parse_remote(url: str) -> Remote | None:
|
|
97
|
+
"""host/owner/repo from any of the forms git accepts; a user name or token in the URL is discarded."""
|
|
98
|
+
m = _REMOTE.match((url or "").strip())
|
|
99
|
+
if not m:
|
|
100
|
+
return None
|
|
101
|
+
parts = [p for p in m.group(2).split("/") if p]
|
|
102
|
+
if len(parts) != 2:
|
|
103
|
+
return None
|
|
104
|
+
return Remote(m.group(1).lower(), parts[0], parts[1])
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def is_hosting(host: str, extra: str | None = None) -> bool:
|
|
108
|
+
return "github" in host or (extra is not None and host == extra.lower())
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def origin_remote(repo_root: Path, extra_host: str | None = None) -> Remote | None:
|
|
112
|
+
"""The `origin` remote when it is a GitHub host, else None: a repository without one gets no pull-request nodes."""
|
|
113
|
+
try:
|
|
114
|
+
out = subprocess.run(["git", "remote", "get-url", "origin"], cwd=str(repo_root), capture_output=True,
|
|
115
|
+
text=True, timeout=10).stdout
|
|
116
|
+
except (OSError, subprocess.SubprocessError):
|
|
117
|
+
return None
|
|
118
|
+
remote = parse_remote(out)
|
|
119
|
+
return remote if remote and is_hosting(remote.host, extra_host) else None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ── authentication and the real transport ─────────────────────────────────────────────────
|
|
123
|
+
def resolve_token(host: str) -> str | None:
|
|
124
|
+
"""GITHUB_TOKEN (github.com) or the enterprise variables, else the `gh` login for that host. A token is only ever
|
|
125
|
+
sent to the host it was found for."""
|
|
126
|
+
names: tuple[str, ...] = ("GITHUB_TOKEN", "GH_TOKEN") if host == "github.com" else ("GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN")
|
|
127
|
+
if host != "github.com" and (os.environ.get("GH_HOST") == host or host in (os.environ.get("GITHUB_SERVER_URL") or "")):
|
|
128
|
+
names += ("GITHUB_TOKEN", "GH_TOKEN") # CI on that enterprise host (GitHub Actions sets GITHUB_SERVER_URL)
|
|
129
|
+
for name in names:
|
|
130
|
+
if os.environ.get(name):
|
|
131
|
+
return os.environ[name].strip()
|
|
132
|
+
try:
|
|
133
|
+
r = subprocess.run(["gh", "auth", "token", "--hostname", host], capture_output=True, text=True, timeout=15)
|
|
134
|
+
except (OSError, subprocess.SubprocessError):
|
|
135
|
+
return None
|
|
136
|
+
return r.stdout.strip() or None if r.returncode == 0 else None
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def api_url(host: str) -> str:
|
|
140
|
+
return "https://api.github.com/graphql" if host == "github.com" else f"https://{host}/api/graphql"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def http_transport(host: str, token: str, timeout: float = 60.0) -> Transport:
|
|
144
|
+
url = api_url(host)
|
|
145
|
+
|
|
146
|
+
def send(query: str, variables: dict) -> dict:
|
|
147
|
+
req = urllib.request.Request(
|
|
148
|
+
url, data=json.dumps({"query": query, "variables": variables}).encode(), method="POST",
|
|
149
|
+
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json", "User-Agent": "devkb",
|
|
150
|
+
"GraphQL-Features": "issue_types,sub_issues"})
|
|
151
|
+
try:
|
|
152
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
153
|
+
return json.loads(resp.read().decode())
|
|
154
|
+
except urllib.error.HTTPError as exc:
|
|
155
|
+
body = ""
|
|
156
|
+
try:
|
|
157
|
+
body = exc.read().decode(errors="replace")[:300].lower()
|
|
158
|
+
except OSError:
|
|
159
|
+
pass
|
|
160
|
+
if exc.code == 401:
|
|
161
|
+
raise GitHubError("unauthorized", "the token was rejected (401)") from None
|
|
162
|
+
if exc.code == 429 or (exc.code == 403 and ("rate limit" in body or exc.headers.get("x-ratelimit-remaining") == "0"
|
|
163
|
+
or exc.headers.get("retry-after"))):
|
|
164
|
+
raise GitHubError("rate-limited", f"rate limited (HTTP {exc.code})") from None
|
|
165
|
+
raise GitHubError("unreachable", f"HTTP {exc.code}") from None
|
|
166
|
+
except (urllib.error.URLError, OSError, TimeoutError, ValueError) as exc:
|
|
167
|
+
raise GitHubError("unreachable", f"{type(exc).__name__}: {getattr(exc, 'reason', exc)}") from None
|
|
168
|
+
return send
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# ── queries ───────────────────────────────────────────────────────────────────────────────
|
|
172
|
+
_PR_COMMITS = "commits(first: 100%s) { totalCount pageInfo { hasNextPage endCursor } nodes { commit { oid } } }"
|
|
173
|
+
_PR_QUERY = """
|
|
174
|
+
query($owner: String!, $name: String!, $after: String) {
|
|
175
|
+
repository(owner: $owner, name: $name) {
|
|
176
|
+
pullRequests(first: %(page)d, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}) {
|
|
177
|
+
pageInfo { hasNextPage endCursor }
|
|
178
|
+
nodes {
|
|
179
|
+
number title state mergedAt updatedAt baseRefName headRefName url
|
|
180
|
+
mergeCommit { oid }
|
|
181
|
+
labels(first: 30) { nodes { name } }
|
|
182
|
+
closingIssuesReferences(first: 50) { nodes { number repository { nameWithOwner } } }
|
|
183
|
+
%(commits)s
|
|
184
|
+
%(author)s
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}"""
|
|
189
|
+
_PR_MORE_COMMITS = """
|
|
190
|
+
query($owner: String!, $name: String!, $number: Int!, $after: String) {
|
|
191
|
+
repository(owner: $owner, name: $name) {
|
|
192
|
+
pullRequest(number: $number) { %s }
|
|
193
|
+
}
|
|
194
|
+
}""" % (_PR_COMMITS % ", after: $after")
|
|
195
|
+
_ISSUE_QUERY = """
|
|
196
|
+
query($owner: String!, $name: String!, $after: String) {
|
|
197
|
+
repository(owner: $owner, name: $name) {
|
|
198
|
+
issues(first: %(page)d, after: $after, orderBy: {field: UPDATED_AT, direction: DESC}) {
|
|
199
|
+
pageInfo { hasNextPage endCursor }
|
|
200
|
+
nodes {
|
|
201
|
+
number title state createdAt closedAt updatedAt url
|
|
202
|
+
labels(first: 30) { nodes { name } }
|
|
203
|
+
milestone { title }
|
|
204
|
+
%(extra)s
|
|
205
|
+
%(author)s
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}"""
|
|
210
|
+
_ISSUE_EXTRA_FULL = "stateReason issueType { name } parent { number repository { nameWithOwner } } subIssues(first: 1) { totalCount }"
|
|
211
|
+
_ISSUE_EXTRA_BASIC = "stateReason"
|
|
212
|
+
_ISSUE_EXTRA_MIN = ""
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _author(authors: bool) -> str:
|
|
216
|
+
return "author { login }" if authors else ""
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def pr_query(authors: bool) -> str:
|
|
220
|
+
return _PR_QUERY % {"page": PAGE, "commits": _PR_COMMITS % "", "author": _author(authors)}
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def issue_query(authors: bool, extra: str) -> str:
|
|
224
|
+
return _ISSUE_QUERY % {"page": PAGE, "extra": extra, "author": _author(authors)}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
# ── normalising what comes back: the whitelist is the privacy rule ─────────────────────────
|
|
228
|
+
def _labels(node: dict) -> list[str]:
|
|
229
|
+
return sorted({n.get("name") for n in ((node.get("labels") or {}).get("nodes") or []) if n and n.get("name")})
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _same_repo(ref: dict, repo_slug: str) -> bool:
|
|
233
|
+
name = (ref.get("repository") or {}).get("nameWithOwner")
|
|
234
|
+
return not name or not repo_slug or name.lower() == repo_slug.lower()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _pr_record(node: dict, authors: bool, repo_slug: str = "") -> dict:
|
|
238
|
+
state = str(node.get("state") or "").lower()
|
|
239
|
+
rec = {
|
|
240
|
+
"number": int(node["number"]), "title": node.get("title") or "", "state": state if state in ("open", "merged", "closed") else "closed",
|
|
241
|
+
"merged_at": node.get("mergedAt"), "updated_at": node.get("updatedAt") or "", "base": node.get("baseRefName"),
|
|
242
|
+
"head": node.get("headRefName"), "url": node.get("url"), "labels": _labels(node),
|
|
243
|
+
"merge_commit": ((node.get("mergeCommit") or {}).get("oid")),
|
|
244
|
+
# only this repository's issues: a reference to another repository's #N is not this repository's GH-N
|
|
245
|
+
"closes": sorted({int(n["number"]) for n in ((node.get("closingIssuesReferences") or {}).get("nodes") or [])
|
|
246
|
+
if n and _same_repo(n, repo_slug)}),
|
|
247
|
+
"commits": [c["commit"]["oid"] for c in ((node.get("commits") or {}).get("nodes") or []) if c and c.get("commit")],
|
|
248
|
+
"commits_total": int((node.get("commits") or {}).get("totalCount") or 0),
|
|
249
|
+
}
|
|
250
|
+
if authors and (node.get("author") or {}).get("login"):
|
|
251
|
+
rec["author"] = node["author"]["login"]
|
|
252
|
+
return rec
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def issue_type_from(issue_type: str | None, labels: list[str], has_children: bool) -> str:
|
|
256
|
+
"""ADR-DEVKB-007 item 2: the issue's type, else a label (bug, epic, enhancement/feature), else EPIC when it has
|
|
257
|
+
sub-issues, else ISSUE. A typed issue keeps its type when it has sub-issues (a Bug with sub-issues stays a Bug);
|
|
258
|
+
`has_sub_issues` is recorded beside it and the `epics` alias reads both."""
|
|
259
|
+
if issue_type:
|
|
260
|
+
return re.sub(r"[^A-Z0-9]+", "_", issue_type.upper()).strip("_") or "ISSUE"
|
|
261
|
+
lower = {l.lower() for l in labels}
|
|
262
|
+
if "epic" in lower:
|
|
263
|
+
return "EPIC"
|
|
264
|
+
if "bug" in lower:
|
|
265
|
+
return "BUG"
|
|
266
|
+
if lower & {"enhancement", "feature"}:
|
|
267
|
+
return "FEATURE"
|
|
268
|
+
return "EPIC" if has_children else "ISSUE"
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _issue_record(node: dict, authors: bool, repo_slug: str) -> dict:
|
|
272
|
+
state = str(node.get("state") or "").lower()
|
|
273
|
+
labels = _labels(node)
|
|
274
|
+
parent = node.get("parent") or {}
|
|
275
|
+
same_repo = (parent.get("repository") or {}).get("nameWithOwner", "").lower() in ("", repo_slug.lower())
|
|
276
|
+
rec = {
|
|
277
|
+
"number": int(node["number"]), "title": node.get("title") or "", "state": state if state in ("open", "closed") else "closed",
|
|
278
|
+
"state_reason": (str(node["stateReason"]).lower() if node.get("stateReason") else None),
|
|
279
|
+
"labels": labels, "milestone": (node.get("milestone") or {}).get("title"), "url": node.get("url"),
|
|
280
|
+
"created_at": node.get("createdAt"), "closed_at": node.get("closedAt"), "updated_at": node.get("updatedAt") or "",
|
|
281
|
+
"ticket_type": issue_type_from((node.get("issueType") or {}).get("name"), labels,
|
|
282
|
+
int((node.get("subIssues") or {}).get("totalCount") or 0) > 0),
|
|
283
|
+
"parent": int(parent["number"]) if parent.get("number") and same_repo else None,
|
|
284
|
+
"has_sub_issues": int((node.get("subIssues") or {}).get("totalCount") or 0) > 0,
|
|
285
|
+
}
|
|
286
|
+
if authors and (node.get("author") or {}).get("login"):
|
|
287
|
+
rec["author"] = node["author"]["login"]
|
|
288
|
+
return rec
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
# ── the cache ────────────────────────────────────────────────────────────────────────────
|
|
292
|
+
class Cache:
|
|
293
|
+
"""One JSON file per kind. Holds only the normalised records above, so it cannot hold a description or review."""
|
|
294
|
+
|
|
295
|
+
def __init__(self, directory: Path, remote: Remote, authors: bool) -> None:
|
|
296
|
+
self.dir, self.remote, self.authors = directory, remote, authors
|
|
297
|
+
|
|
298
|
+
def _path(self, kind: str) -> Path:
|
|
299
|
+
return self.dir / f"{kind}.json"
|
|
300
|
+
|
|
301
|
+
def load(self, kind: str) -> tuple[dict[int, dict], str, dict]:
|
|
302
|
+
"""(records by number, when they were fetched, listing state {"complete", "cursor"}). Empty when the cache is
|
|
303
|
+
for another repository, another schema, or was written without the author logins now asked for."""
|
|
304
|
+
try:
|
|
305
|
+
data = json.loads(self._path(kind).read_text())
|
|
306
|
+
except (OSError, ValueError):
|
|
307
|
+
return {}, "", {}
|
|
308
|
+
if not isinstance(data, dict) or data.get("version") != CACHE_VERSION or data.get("repo") != self.remote.slug:
|
|
309
|
+
return {}, "", {}
|
|
310
|
+
if self.authors and not data.get("authors"):
|
|
311
|
+
return {}, "", {}
|
|
312
|
+
items = {int(k): v for k, v in (data.get("items") or {}).items() if isinstance(v, dict)}
|
|
313
|
+
if not self.authors:
|
|
314
|
+
for v in items.values():
|
|
315
|
+
v.pop("author", None)
|
|
316
|
+
return items, str(data.get("fetched_at") or ""), {"complete": bool(data.get("complete")), "cursor": data.get("cursor")}
|
|
317
|
+
|
|
318
|
+
def save(self, kind: str, items: dict[int, dict], complete: bool = True, cursor: str | None = None) -> None:
|
|
319
|
+
clean = {str(n): ({k: v for k, v in rec.items() if self.authors or k != "author"}) for n, rec in sorted(items.items())}
|
|
320
|
+
try:
|
|
321
|
+
self.dir.mkdir(parents=True, exist_ok=True)
|
|
322
|
+
self._path(kind).write_text(json.dumps({
|
|
323
|
+
"version": CACHE_VERSION, "repo": self.remote.slug, "authors": self.authors, "complete": complete, "cursor": cursor,
|
|
324
|
+
"fetched_at": datetime.now(timezone.utc).isoformat(), "items": clean}, indent=0))
|
|
325
|
+
except OSError:
|
|
326
|
+
pass
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
# ── fetching ─────────────────────────────────────────────────────────────────────────────
|
|
330
|
+
@dataclass
|
|
331
|
+
class Online:
|
|
332
|
+
"""What the online layer knows. `prs` and `issues` may hold cached records from an earlier build when a refresh
|
|
333
|
+
failed; `report` says so."""
|
|
334
|
+
prs: dict[int, dict] = field(default_factory=dict)
|
|
335
|
+
issues: dict[int, dict] = field(default_factory=dict)
|
|
336
|
+
report: dict = field(default_factory=dict)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _gql(transport: Transport, query: str, variables: dict) -> dict:
|
|
340
|
+
resp = transport(query, variables)
|
|
341
|
+
errors = resp.get("errors")
|
|
342
|
+
if errors or not isinstance(resp.get("data"), dict):
|
|
343
|
+
text = "; ".join(str(e.get("message", e)) if isinstance(e, dict) else str(e) for e in (errors or []))[:300]
|
|
344
|
+
kinds = " ".join(str(e.get("type", "")) for e in (errors or []) if isinstance(e, dict))
|
|
345
|
+
if "RATE_LIMITED" in kinds or "rate limit" in text.lower():
|
|
346
|
+
raise GitHubError("rate-limited", text or "rate limited")
|
|
347
|
+
if "NOT_FOUND" in kinds:
|
|
348
|
+
raise GitHubError("not-found", text or "repository not found or not visible to this token")
|
|
349
|
+
if re.search(r"doesn't exist on type|Unknown type|Field '.*' doesn't exist", text):
|
|
350
|
+
raise GitHubError("schema", text)
|
|
351
|
+
if "bad credentials" in text.lower():
|
|
352
|
+
raise GitHubError("unauthorized", text)
|
|
353
|
+
raise GitHubError("graphql", text or "empty response")
|
|
354
|
+
return resp["data"]
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _walk(transport: Transport, query: str, remote: Remote, connection: str, keep: Callable[[dict], dict],
|
|
358
|
+
items: dict[int, dict], state: dict, save: Callable[[dict[int, dict], bool, str | None], None],
|
|
359
|
+
finish: Callable[[dict], None] | None = None) -> tuple[dict[int, dict], int]:
|
|
360
|
+
"""Read one listing, newest-updated first, into `items`. Returns (records read, requests made); a GitHubError
|
|
361
|
+
propagates after what was read is kept.
|
|
362
|
+
|
|
363
|
+
A finished listing is refreshed incrementally: paging stops at records older than anything cached, and the cache
|
|
364
|
+
is written only when that pass ends, because a partial pass would raise the "newest cached" mark over records not
|
|
365
|
+
yet read. An unfinished listing (a first build, or one cut short) is resumed from its saved cursor and written
|
|
366
|
+
after every page with `complete` false until the last page, so a repository too large for one rate-limit window
|
|
367
|
+
is read across several builds instead of starting over each time."""
|
|
368
|
+
resume = not state.get("complete")
|
|
369
|
+
after = state.get("cursor") if resume else None
|
|
370
|
+
newest = "" if resume else max((r.get("updated_at") or "" for r in items.values()), default="")
|
|
371
|
+
fresh: dict[int, dict] = {}
|
|
372
|
+
calls = 0
|
|
373
|
+
while True:
|
|
374
|
+
data = _gql(transport, query, {"owner": remote.owner, "name": remote.repo, "after": after})
|
|
375
|
+
calls += 1
|
|
376
|
+
conn = ((data.get("repository") or {}).get(connection)) or {}
|
|
377
|
+
page: dict[int, dict] = {}
|
|
378
|
+
stop = False
|
|
379
|
+
for node in conn.get("nodes") or []:
|
|
380
|
+
if not node:
|
|
381
|
+
continue
|
|
382
|
+
rec = keep(node)
|
|
383
|
+
if newest and rec["updated_at"] and rec["updated_at"] < newest:
|
|
384
|
+
stop = True
|
|
385
|
+
continue
|
|
386
|
+
page[rec["number"]] = rec
|
|
387
|
+
for rec in page.values():
|
|
388
|
+
if finish:
|
|
389
|
+
finish(rec)
|
|
390
|
+
fresh.update(page)
|
|
391
|
+
info = conn.get("pageInfo") or {}
|
|
392
|
+
more = bool(info.get("hasNextPage")) and not stop
|
|
393
|
+
if resume:
|
|
394
|
+
items.update(page)
|
|
395
|
+
save(items, not more, info.get("endCursor") if more else None)
|
|
396
|
+
if not more:
|
|
397
|
+
if not resume:
|
|
398
|
+
items.update(fresh)
|
|
399
|
+
save(items, True, None)
|
|
400
|
+
return fresh, calls
|
|
401
|
+
after = info.get("endCursor")
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _more_commits(transport: Transport, remote: Remote, rec: dict) -> None:
|
|
405
|
+
"""A pull request with more than 100 commits: read the rest, up to the API's own cap. Raises GitHubError; the
|
|
406
|
+
caller keeps the commits read so far, and `commits_total` then says the list is cut short."""
|
|
407
|
+
after, pages = rec.pop("_after", None), 1
|
|
408
|
+
while after and pages < MAX_COMMIT_PAGES:
|
|
409
|
+
data = _gql(transport, _PR_MORE_COMMITS, {"owner": remote.owner, "name": remote.repo, "number": rec["number"], "after": after})
|
|
410
|
+
conn = ((data.get("repository") or {}).get("pullRequest") or {}).get("commits") or {}
|
|
411
|
+
rec["commits"] += [c["commit"]["oid"] for c in conn.get("nodes") or [] if c and c.get("commit")]
|
|
412
|
+
info = conn.get("pageInfo") or {}
|
|
413
|
+
after = info.get("endCursor") if info.get("hasNextPage") else None
|
|
414
|
+
pages += 1
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def fetch(remote: Remote, cache_dir: Path, authors: bool = False, transport: Transport | None = None) -> Online:
|
|
418
|
+
"""Fetch (or refresh from the cache) the repository's pull requests and issues. Never raises: the outcome is in
|
|
419
|
+
`report`: status `ok`, `partial` (one kind failed), or `failed` (neither could be fetched), with the reason."""
|
|
420
|
+
cache = Cache(cache_dir, remote, authors)
|
|
421
|
+
out = Online()
|
|
422
|
+
out.prs, pr_at, pr_state = cache.load("pull_requests")
|
|
423
|
+
out.issues, is_at, is_state = cache.load("issues")
|
|
424
|
+
report: dict = {"remote": remote.slug, "mode": "online", "authors": authors, "status": "ok", "reason": None,
|
|
425
|
+
"not_fetched": [], "requests": 0}
|
|
426
|
+
if transport is None:
|
|
427
|
+
token = resolve_token(remote.host)
|
|
428
|
+
if not token:
|
|
429
|
+
report.update(status="failed", reason="no GITHUB_TOKEN and no `gh` login for " + remote.host,
|
|
430
|
+
not_fetched=["pull_requests", "issues"], cached_from=_cached_note(pr_at, is_at))
|
|
431
|
+
_describe(report, out, {"pull_requests": False, "issues": False})
|
|
432
|
+
return out
|
|
433
|
+
transport = http_transport(remote.host, token)
|
|
434
|
+
|
|
435
|
+
ok = {"pull_requests": False, "issues": False}
|
|
436
|
+
reasons: list[str] = []
|
|
437
|
+
stale_cache = {k: bool(v) for k, v in (("pull_requests", out.prs), ("issues", out.issues))}
|
|
438
|
+
|
|
439
|
+
slug = f"{remote.owner}/{remote.repo}"
|
|
440
|
+
commits_cut: list[int] = []
|
|
441
|
+
|
|
442
|
+
def keep_pr(node: dict) -> dict:
|
|
443
|
+
rec = _pr_record(node, authors, slug)
|
|
444
|
+
info = (node.get("commits") or {}).get("pageInfo") or {}
|
|
445
|
+
if info.get("hasNextPage"):
|
|
446
|
+
rec["_after"] = info.get("endCursor")
|
|
447
|
+
return rec
|
|
448
|
+
|
|
449
|
+
def finish_pr(rec: dict) -> None:
|
|
450
|
+
"""The rest of a long commit list; a failure here costs that one list, never the page."""
|
|
451
|
+
try:
|
|
452
|
+
_more_commits(transport, remote, rec)
|
|
453
|
+
except GitHubError:
|
|
454
|
+
commits_cut.append(rec["number"])
|
|
455
|
+
rec.pop("_after", None)
|
|
456
|
+
|
|
457
|
+
try:
|
|
458
|
+
fresh, calls = _walk(transport, pr_query(authors), remote, "pullRequests", keep_pr, out.prs, pr_state,
|
|
459
|
+
lambda items, done, cursor: cache.save("pull_requests", items, done, cursor), finish_pr)
|
|
460
|
+
report["requests"] += calls
|
|
461
|
+
ok["pull_requests"] = True
|
|
462
|
+
report["pull_requests_fetched"] = len(fresh)
|
|
463
|
+
except GitHubError as exc:
|
|
464
|
+
reasons.append(f"pull requests: {exc.kind}: {exc.message}")
|
|
465
|
+
report["kind"] = exc.kind
|
|
466
|
+
|
|
467
|
+
if commits_cut:
|
|
468
|
+
report["commit_lists_not_fetched"] = sorted(commits_cut)[:50]
|
|
469
|
+
last_error: GitHubError | None = None
|
|
470
|
+
for extra in (_ISSUE_EXTRA_FULL, _ISSUE_EXTRA_BASIC, _ISSUE_EXTRA_MIN):
|
|
471
|
+
try:
|
|
472
|
+
fresh, calls = _walk(transport, issue_query(authors, extra), remote, "issues",
|
|
473
|
+
lambda n: _issue_record(n, authors, slug), out.issues, is_state,
|
|
474
|
+
lambda items, done, cursor: cache.save("issues", items, done, cursor))
|
|
475
|
+
report["requests"] += calls
|
|
476
|
+
ok["issues"] = True
|
|
477
|
+
report["issues_fetched"] = len(fresh)
|
|
478
|
+
report["issue_fields"] = "full" if extra == _ISSUE_EXTRA_FULL else "basic"
|
|
479
|
+
last_error = None
|
|
480
|
+
break
|
|
481
|
+
except GitHubError as exc: # a host without issue types or sub-issues: ask for less, not for nothing
|
|
482
|
+
last_error = exc
|
|
483
|
+
if exc.kind != "schema":
|
|
484
|
+
break
|
|
485
|
+
if last_error is not None:
|
|
486
|
+
reasons.append(f"issues: {last_error.kind}: {last_error.message}")
|
|
487
|
+
report.setdefault("kind", last_error.kind)
|
|
488
|
+
|
|
489
|
+
report["listing_incomplete"] = [k for k in ("pull_requests", "issues")
|
|
490
|
+
if not Cache(cache_dir, remote, authors).load(k)[2].get("complete")]
|
|
491
|
+
report["not_fetched"] = [k for k, v in ok.items() if not v]
|
|
492
|
+
if not report["not_fetched"]:
|
|
493
|
+
report["status"] = "ok"
|
|
494
|
+
else:
|
|
495
|
+
report["status"] = "partial" if any(ok.values()) else "failed"
|
|
496
|
+
report["reason"] = "; ".join(reasons)
|
|
497
|
+
report["stale_cache_used"] = [k for k in report["not_fetched"] if stale_cache[k]]
|
|
498
|
+
report["cached_from"] = _cached_note(pr_at, is_at)
|
|
499
|
+
_describe(report, out, ok)
|
|
500
|
+
out.report = report
|
|
501
|
+
return out
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _cached_note(pr_at: str, is_at: str) -> dict:
|
|
505
|
+
return {k: v for k, v in (("pull_requests", pr_at), ("issues", is_at)) if v}
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _describe(report: dict, out: Online, ok: dict[str, bool]) -> None:
|
|
509
|
+
report["pull_requests"] = {"count": len(out.prs), "fresh": ok["pull_requests"]}
|
|
510
|
+
report["issues"] = {"count": len(out.issues), "fresh": ok["issues"]}
|
|
511
|
+
truncated = sorted(n for n, r in out.prs.items() if r.get("commits_total", 0) > len(r.get("commits", [])))
|
|
512
|
+
if truncated:
|
|
513
|
+
report["commit_lists_truncated"] = truncated[:50]
|
|
514
|
+
# callers read the report from the returned object; keep it on `out` even on the early return
|
|
515
|
+
out.report = report
|
devkb/gitutil.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Tiny git helpers shared across devkb/build.py and devkb/build_symbols.py --
|
|
2
|
+
split out specifically to avoid a circular import between them (build.py
|
|
3
|
+
needs build_symbols.load_cache(), build_symbols.py needs this)."""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import subprocess
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def git_head(repo_root: Path) -> str:
|
|
11
|
+
try:
|
|
12
|
+
r = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True,
|
|
13
|
+
cwd=str(repo_root), timeout=10)
|
|
14
|
+
return r.stdout.strip() or "unknown"
|
|
15
|
+
except (OSError, subprocess.SubprocessError):
|
|
16
|
+
return "unknown"
|
devkb/index.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Load-time index over a project's .devkb/ store.
|
|
2
|
+
|
|
3
|
+
Mirrors scripts/traverse_graph.py's GraphIndex pattern (build by_uid/out_edges/
|
|
4
|
+
in_edges once at load, O(1) lookups from there) rather than GraphService's
|
|
5
|
+
per-call O(N) scan (webapp/backend/app/services/graph_service.py:171) --
|
|
6
|
+
applying that lesson from the start instead of retrofitting it after a
|
|
7
|
+
measured stall, per ADR-SDP-DEVKB-001.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class DevKBIndex:
|
|
20
|
+
by_uid: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
21
|
+
by_type: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
|
|
22
|
+
out_edges: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
|
|
23
|
+
in_edges: dict[str, list[dict[str, Any]]] = field(default_factory=lambda: defaultdict(list))
|
|
24
|
+
by_natural_key: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict)
|
|
25
|
+
|
|
26
|
+
def node(self, node_type: str, natural_key: str) -> dict[str, Any] | None:
|
|
27
|
+
return self.by_natural_key.get((node_type, natural_key))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
31
|
+
if not path.exists():
|
|
32
|
+
return []
|
|
33
|
+
out = []
|
|
34
|
+
for line in path.read_text().splitlines():
|
|
35
|
+
line = line.strip()
|
|
36
|
+
if not line:
|
|
37
|
+
continue
|
|
38
|
+
out.append(json.loads(line))
|
|
39
|
+
return out
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def load(devkb_dir: Path) -> DevKBIndex:
|
|
43
|
+
idx = DevKBIndex()
|
|
44
|
+
for node in _read_jsonl(devkb_dir / "nodes.jsonl"):
|
|
45
|
+
idx.by_uid[node["uid"]] = node
|
|
46
|
+
idx.by_type[node["type"]].append(node)
|
|
47
|
+
idx.by_natural_key[(node["type"], node["natural_key"])] = node
|
|
48
|
+
for edge in _read_jsonl(devkb_dir / "edges.jsonl"):
|
|
49
|
+
idx.out_edges[edge["from_uid"]].append(edge)
|
|
50
|
+
idx.in_edges[edge["to_uid"]].append(edge)
|
|
51
|
+
return idx
|