devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/service.py
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""A long-lived, compact index over a project's `.devkb/` store (ODSF-306).
|
|
2
|
+
|
|
3
|
+
`devkb/index.py` re-reads every JSONL line on each CLI call, which is 98 MB for
|
|
4
|
+
OCO and was never measured. This keeps one integer-addressed index in memory,
|
|
5
|
+
persists it next to the graph, and reloads only when the graph changes. It
|
|
6
|
+
answers the process API (`devkb/process_api.py`) and the MCP server
|
|
7
|
+
(`devkb/mcp_server.py`); the generic `index.py` is left alone for its other
|
|
8
|
+
callers.
|
|
9
|
+
|
|
10
|
+
Staleness is explicit. Every reply says which commit the graph was built at and
|
|
11
|
+
whether the repository has moved on, so an agent knows when to trust it.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import pickle
|
|
17
|
+
import re
|
|
18
|
+
import subprocess
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
from collections import defaultdict
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
class GraphNotBuilt(FileNotFoundError):
|
|
26
|
+
"""The project has no graph yet; the message says which command builds it."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
CACHE_VERSION = 11 # re-exports followed; module_callers, unresolved_file_count kept
|
|
30
|
+
CACHE_NAME = ".process-index.pkl"
|
|
31
|
+
TELEMETRY_NAME = "process-api-telemetry.jsonl"
|
|
32
|
+
|
|
33
|
+
# Only the properties the process API reads, so a reply is short and the cache is small.
|
|
34
|
+
KEEP_PROPS = {
|
|
35
|
+
"Ticket": ("title", "ticket_type", "priority", "status", "component", "assignee", "issue_path",
|
|
36
|
+
"state", "url", "unverified"), # ADR-DEVKB-007: hosted issues (GH-<n>)
|
|
37
|
+
"PullRequest": ("number", "title", "state", "merged_at", "base", "head", "url", "merge_style", "merge_commit",
|
|
38
|
+
"labels"), # ADR-DEVKB-007
|
|
39
|
+
"Spec": ("title", "doc_type", "status", "spec_id"),
|
|
40
|
+
"Symbol": ("kind", "line_start", "line_end", "complexity", "is_public", "docstring_summary",
|
|
41
|
+
"unresolved_calls", "unresolved_files", "unresolved_file_count", # ODSF-313: what a list answer cannot see
|
|
42
|
+
"module_callers"), # calls made outside any function (import time, class bodies)
|
|
43
|
+
"ErrorSignature": ("kind", "label", "message_template", "status_code"),
|
|
44
|
+
"TestCase": ("function", "file", "line"),
|
|
45
|
+
"Commit": ("short_hash", "subject", "date"),
|
|
46
|
+
"Endpoint": ("method", "path", "owner_symbol"),
|
|
47
|
+
"Flow": ("touch_point_kind", "step_count"),
|
|
48
|
+
"Warning": ("rule", "severity", "detail"),
|
|
49
|
+
"WorkPackage": ("title", "status"),
|
|
50
|
+
"ConfigKey": ("kind",), # ADR-SDP-DEVKB-005 (ODSF-312)
|
|
51
|
+
"Component": ("kind", "label", "file", "image", "ports", "exec", "path"),
|
|
52
|
+
"Repository": ("role",),
|
|
53
|
+
}
|
|
54
|
+
_WORD = re.compile(r"[a-z0-9]{3,}")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def stem(w: str) -> str:
|
|
58
|
+
"""Strip the common English endings so "saving records" finds `save_record`. Deliberately crude and applied the
|
|
59
|
+
same way to the index and to the query: a stem need not be a word, only the same on both sides."""
|
|
60
|
+
for suffix in ("ing", "ed", "es", "s"):
|
|
61
|
+
if w.endswith(suffix) and len(w) - len(suffix) >= 3 and not (suffix == "s" and w.endswith("ss")):
|
|
62
|
+
w = w[: -len(suffix)]
|
|
63
|
+
break
|
|
64
|
+
return w[:-1] if w.endswith("e") and len(w) > 3 else w
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def words(text: str) -> list[str]:
|
|
68
|
+
return [stem(w) for w in _WORD.findall(text.lower().replace("_", " ").replace("-", " ").replace("/", " ").replace(".", " "))]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class CompactIndex:
|
|
73
|
+
code: str
|
|
74
|
+
repo_root: str
|
|
75
|
+
built_at: str = ""
|
|
76
|
+
git_head: str = ""
|
|
77
|
+
types: list[str] = field(default_factory=list)
|
|
78
|
+
keys: list[str] = field(default_factory=list)
|
|
79
|
+
props: list[dict] = field(default_factory=list)
|
|
80
|
+
by_key: dict[tuple[str, str], int] = field(default_factory=dict)
|
|
81
|
+
short: dict[str, list[int]] = field(default_factory=lambda: defaultdict(list))
|
|
82
|
+
sym_by_file: dict[str, list[int]] = field(default_factory=lambda: defaultdict(list))
|
|
83
|
+
out: dict[str, dict[int, list[int]]] = field(default_factory=lambda: defaultdict(lambda: defaultdict(list)))
|
|
84
|
+
inn: dict[str, dict[int, list[int]]] = field(default_factory=lambda: defaultdict(lambda: defaultdict(list)))
|
|
85
|
+
sim: dict[int, list[tuple[int, float]]] = field(default_factory=lambda: defaultdict(list))
|
|
86
|
+
ep_by_owner: dict[str, list[int]] = field(default_factory=lambda: defaultdict(list))
|
|
87
|
+
spec_terms: dict[int, tuple[list[str], list[str], list[str]]] = field(default_factory=dict)
|
|
88
|
+
spec_df: dict[str, int] = field(default_factory=dict)
|
|
89
|
+
spec_inv: dict[str, list[int]] = field(default_factory=dict) # token -> specs containing it (ODSF-309)
|
|
90
|
+
flow_steps: dict[int, list[tuple[int, int]]] = field(default_factory=dict) # flow -> [(sequence, step)] (ODSF-311)
|
|
91
|
+
spec_heads: dict[int, list[str]] = field(default_factory=dict) # spec -> its first headings (ODSF-311)
|
|
92
|
+
spec_by_id: dict[str, int] = field(default_factory=dict) # lower-case spec id -> spec
|
|
93
|
+
type_count: dict[str, int] = field(default_factory=dict) # node type -> count, computed once
|
|
94
|
+
code_inv: dict[str, list[tuple[int, int]]] = field(default_factory=dict) # word -> (symbol, weight): name 3, path 2, docstring 1
|
|
95
|
+
err_inv: dict[str, list[int]] = field(default_factory=dict) # word of an error message -> ErrorSignature nodes
|
|
96
|
+
ticket_inv: dict[str, list[int]] = field(default_factory=dict) # word of a ticket title -> tickets
|
|
97
|
+
n_symbols: int = 0
|
|
98
|
+
n_edges: int = 0
|
|
99
|
+
# ADR-DEVKB-007: what the pull-request layer saw (manifest `github`), and each MERGES/CLOSES edge's confidence and
|
|
100
|
+
# whether its target is unverified, which the integer adjacency above does not keep.
|
|
101
|
+
github: dict = field(default_factory=dict)
|
|
102
|
+
link_meta: dict[tuple[str, int, int], tuple[float, bool]] = field(default_factory=dict)
|
|
103
|
+
|
|
104
|
+
# ── lookups ────────────────────────────────────────────────────────────
|
|
105
|
+
def node(self, node_type: str, key: str) -> int | None:
|
|
106
|
+
return self.by_key.get((node_type, key))
|
|
107
|
+
|
|
108
|
+
def targets(self, verb: str, src: int) -> list[int]:
|
|
109
|
+
return self.out[verb].get(src, []) if verb in self.out else []
|
|
110
|
+
|
|
111
|
+
def sources(self, verb: str, dst: int) -> list[int]:
|
|
112
|
+
return self.inn[verb].get(dst, []) if verb in self.inn else []
|
|
113
|
+
|
|
114
|
+
def type_counts(self) -> dict[str, int]:
|
|
115
|
+
return self.type_count
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _read_manifest(devkb_dir: Path) -> dict:
|
|
119
|
+
try:
|
|
120
|
+
return json.loads((devkb_dir / "manifest.json").read_text())
|
|
121
|
+
except (OSError, ValueError):
|
|
122
|
+
return {}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _signature(devkb_dir: Path) -> list:
|
|
126
|
+
sig: list = [CACHE_VERSION]
|
|
127
|
+
for name in ("nodes.jsonl", "edges.jsonl", "manifest.json"):
|
|
128
|
+
try:
|
|
129
|
+
st = (devkb_dir / name).stat()
|
|
130
|
+
sig += [name, st.st_size, st.st_mtime_ns]
|
|
131
|
+
except OSError:
|
|
132
|
+
sig += [name, None, None]
|
|
133
|
+
return sig
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _build(code: str, repo_root: Path, devkb_dir: Path) -> CompactIndex:
|
|
137
|
+
manifest = _read_manifest(devkb_dir)
|
|
138
|
+
idx = CompactIndex(code=code, repo_root=str(repo_root), built_at=manifest.get("built_at", ""),
|
|
139
|
+
git_head=manifest.get("git_head", ""),
|
|
140
|
+
github=manifest.get("github") if isinstance(manifest.get("github"), dict) else {})
|
|
141
|
+
uid_to_i: dict[str, int] = {}
|
|
142
|
+
for line in (devkb_dir / "nodes.jsonl").read_text().splitlines():
|
|
143
|
+
if not line:
|
|
144
|
+
continue
|
|
145
|
+
n = json.loads(line)
|
|
146
|
+
i = len(idx.types)
|
|
147
|
+
t, key = n["type"], n["natural_key"]
|
|
148
|
+
uid_to_i[n["uid"]] = i
|
|
149
|
+
idx.types.append(t)
|
|
150
|
+
idx.keys.append(key)
|
|
151
|
+
wanted = KEEP_PROPS.get(t)
|
|
152
|
+
idx.props.append({k: n["properties"][k] for k in wanted if k in n["properties"]} if wanted else {})
|
|
153
|
+
idx.by_key[(t, key)] = i
|
|
154
|
+
if t == "Symbol":
|
|
155
|
+
path, _, qual = key.partition("::")
|
|
156
|
+
idx.short[qual.rsplit(".", 1)[-1]].append(i)
|
|
157
|
+
idx.sym_by_file[path].append(i)
|
|
158
|
+
elif t == "Endpoint" and n["properties"].get("owner_symbol"):
|
|
159
|
+
idx.ep_by_owner[n["properties"]["owner_symbol"]].append(i)
|
|
160
|
+
for line in (devkb_dir / "edges.jsonl").read_text().splitlines():
|
|
161
|
+
if not line:
|
|
162
|
+
continue
|
|
163
|
+
e = json.loads(line)
|
|
164
|
+
a, b = uid_to_i.get(e["from_uid"]), uid_to_i.get(e["to_uid"])
|
|
165
|
+
if a is None or b is None:
|
|
166
|
+
continue
|
|
167
|
+
verb = e["verb"]
|
|
168
|
+
idx.out[verb][a].append(b)
|
|
169
|
+
idx.inn[verb][b].append(a)
|
|
170
|
+
idx.n_edges += 1
|
|
171
|
+
if verb in ("MERGES", "CLOSES"):
|
|
172
|
+
props = e["properties"]
|
|
173
|
+
idx.link_meta[(verb, a, b)] = (float(props.get("confidence") or 0.0), bool(props.get("unverified")))
|
|
174
|
+
if verb == "SIMILAR_TO":
|
|
175
|
+
idx.sim[a].append((b, float(e["properties"].get("weighted_score") or 0.0)))
|
|
176
|
+
elif verb == "STEP":
|
|
177
|
+
idx.flow_steps.setdefault(a, []).append((int(e["properties"].get("sequence") or 0), b))
|
|
178
|
+
for lst in idx.sim.values():
|
|
179
|
+
lst.sort(key=lambda p: -p[1])
|
|
180
|
+
del lst[8:]
|
|
181
|
+
for steps in idx.flow_steps.values():
|
|
182
|
+
steps.sort()
|
|
183
|
+
# Spec search terms: title, path and the first headings of each document, read once.
|
|
184
|
+
df: dict[str, int] = defaultdict(int)
|
|
185
|
+
inv: dict[str, list[int]] = {}
|
|
186
|
+
for i, t in enumerate(idx.types):
|
|
187
|
+
if t != "Spec":
|
|
188
|
+
continue
|
|
189
|
+
path = idx.keys[i]
|
|
190
|
+
heads: list[str] = []
|
|
191
|
+
try:
|
|
192
|
+
with open(repo_root / path, errors="replace") as fh:
|
|
193
|
+
for _ in range(120):
|
|
194
|
+
ln = fh.readline()
|
|
195
|
+
if not ln:
|
|
196
|
+
break
|
|
197
|
+
if ln.startswith("#"):
|
|
198
|
+
heads.append(ln.lstrip("# ").strip())
|
|
199
|
+
except OSError:
|
|
200
|
+
pass
|
|
201
|
+
title_w = words(idx.props[i].get("title", "") + " " + str(idx.props[i].get("spec_id") or ""))
|
|
202
|
+
path_w = words(path)
|
|
203
|
+
head_w = words(" ".join(heads))
|
|
204
|
+
idx.spec_terms[i] = (title_w, path_w, head_w)
|
|
205
|
+
idx.spec_heads[i] = heads[:12]
|
|
206
|
+
for w in set(title_w) | set(path_w) | set(head_w):
|
|
207
|
+
df[w] += 1
|
|
208
|
+
inv.setdefault(w, []).append(i)
|
|
209
|
+
sid = str(idx.props[i].get("spec_id") or "").strip().lower()
|
|
210
|
+
if sid:
|
|
211
|
+
idx.spec_by_id[sid] = i
|
|
212
|
+
idx.spec_df = dict(df)
|
|
213
|
+
idx.spec_inv = inv
|
|
214
|
+
counts: dict[str, int] = defaultdict(int)
|
|
215
|
+
for t in idx.types:
|
|
216
|
+
counts[t] += 1
|
|
217
|
+
idx.type_count = dict(counts)
|
|
218
|
+
idx.n_symbols = counts.get("Symbol", 0)
|
|
219
|
+
# Word indexes for the developer-task tools (ODSF-310): find code by what it is called and says, an error by its
|
|
220
|
+
# message, a past ticket by its title. Built once with the graph, so a query is a few dictionary lookups.
|
|
221
|
+
for i, t in enumerate(idx.types):
|
|
222
|
+
if t == "Symbol":
|
|
223
|
+
path, _, qual = idx.keys[i].partition("::")
|
|
224
|
+
weight: dict[str, int] = {}
|
|
225
|
+
for w in words(qual):
|
|
226
|
+
weight[w] = 3
|
|
227
|
+
for w in words(path):
|
|
228
|
+
weight[w] = max(weight.get(w, 0), 2)
|
|
229
|
+
for w in words(idx.props[i].get("docstring_summary") or ""):
|
|
230
|
+
weight[w] = max(weight.get(w, 0), 1)
|
|
231
|
+
for w, wt in weight.items():
|
|
232
|
+
idx.code_inv.setdefault(w, []).append((i, wt))
|
|
233
|
+
elif t == "ErrorSignature":
|
|
234
|
+
p = idx.props[i]
|
|
235
|
+
for w in set(words((p.get("message_template") or "") + " " + (p.get("label") or ""))):
|
|
236
|
+
idx.err_inv.setdefault(w, []).append(i)
|
|
237
|
+
elif t == "Ticket":
|
|
238
|
+
for w in set(words(idx.props[i].get("title") or "")):
|
|
239
|
+
idx.ticket_inv.setdefault(w, []).append(i)
|
|
240
|
+
# Plain dicts only: a defaultdict built from a lambda cannot be pickled.
|
|
241
|
+
idx.short, idx.sym_by_file = dict(idx.short), dict(idx.sym_by_file)
|
|
242
|
+
idx.ep_by_owner, idx.sim = dict(idx.ep_by_owner), dict(idx.sim)
|
|
243
|
+
idx.out = {v: dict(m) for v, m in idx.out.items()}
|
|
244
|
+
idx.inn = {v: dict(m) for v, m in idx.inn.items()}
|
|
245
|
+
return idx
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def load_index(code: str, repo_root: Path, devkb_dir: Path, use_cache: bool = True) -> CompactIndex:
|
|
249
|
+
"""Read the cache when it still matches the graph files, else rebuild and save it."""
|
|
250
|
+
cache = devkb_dir / CACHE_NAME
|
|
251
|
+
sig = _signature(devkb_dir)
|
|
252
|
+
if use_cache and cache.exists():
|
|
253
|
+
try:
|
|
254
|
+
with open(cache, "rb") as fh:
|
|
255
|
+
saved_sig, idx = pickle.load(fh)
|
|
256
|
+
if saved_sig == sig:
|
|
257
|
+
return idx
|
|
258
|
+
except Exception: # a stale or corrupt cache is rebuilt, never trusted
|
|
259
|
+
pass
|
|
260
|
+
if not (Path(devkb_dir) / "nodes.jsonl").exists():
|
|
261
|
+
raise GraphNotBuilt(f"no devkb graph at {devkb_dir}: run `devkb build --project {repo_root}` first")
|
|
262
|
+
idx = _build(code, repo_root, devkb_dir)
|
|
263
|
+
try:
|
|
264
|
+
with open(cache, "wb") as fh:
|
|
265
|
+
pickle.dump((sig, idx), fh, protocol=pickle.HIGHEST_PROTOCOL)
|
|
266
|
+
except OSError:
|
|
267
|
+
pass
|
|
268
|
+
return idx
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class Staleness:
|
|
272
|
+
"""Whether the repository has moved past the commit the graph was built at.
|
|
273
|
+
|
|
274
|
+
The first call answers synchronously. After that the answer is refreshed in a background thread when it is
|
|
275
|
+
older than `ttl` seconds, so no tool call ever waits on `git` for it (ODSF-309: a synchronous refresh every
|
|
276
|
+
5 seconds put a 5 to 30 ms spike on whichever call happened to land in the window)."""
|
|
277
|
+
|
|
278
|
+
def __init__(self, repo_root: Path, built_head: str, ttl: float = 5.0) -> None:
|
|
279
|
+
self.repo_root, self.built_head, self.ttl = repo_root, built_head, ttl
|
|
280
|
+
self._at = 0.0
|
|
281
|
+
self._value = (False, "", 0)
|
|
282
|
+
self._busy = False
|
|
283
|
+
|
|
284
|
+
def _refresh(self) -> None:
|
|
285
|
+
head, behind = "", 0
|
|
286
|
+
try:
|
|
287
|
+
head = subprocess.run(["git", "rev-parse", "HEAD"], cwd=self.repo_root, capture_output=True,
|
|
288
|
+
text=True, timeout=5).stdout.strip()
|
|
289
|
+
if head and self.built_head and head != self.built_head:
|
|
290
|
+
out = subprocess.run(["git", "rev-list", "--count", f"{self.built_head}..HEAD"], cwd=self.repo_root,
|
|
291
|
+
capture_output=True, text=True, timeout=5).stdout.strip()
|
|
292
|
+
behind = int(out) if out.isdigit() else 0
|
|
293
|
+
except (OSError, subprocess.SubprocessError, ValueError):
|
|
294
|
+
pass
|
|
295
|
+
self._value = (bool(head and self.built_head and head != self.built_head), head, behind)
|
|
296
|
+
self._at = time.monotonic()
|
|
297
|
+
self._busy = False
|
|
298
|
+
|
|
299
|
+
def check(self) -> tuple[bool, str, int]:
|
|
300
|
+
if self._at == 0.0:
|
|
301
|
+
self._refresh()
|
|
302
|
+
elif time.monotonic() - self._at >= self.ttl and not self._busy:
|
|
303
|
+
self._busy = True
|
|
304
|
+
threading.Thread(target=self._refresh, daemon=True).start()
|
|
305
|
+
return self._value
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
class Telemetry:
|
|
309
|
+
"""One JSON line per call: tool, milliseconds, reply bytes. This is how the
|
|
310
|
+
API is dogfooded and how the benchmark proves what the graph returned."""
|
|
311
|
+
|
|
312
|
+
def __init__(self, devkb_dir: Path, enabled: bool = True) -> None:
|
|
313
|
+
self.path = devkb_dir / TELEMETRY_NAME
|
|
314
|
+
self.enabled = enabled
|
|
315
|
+
|
|
316
|
+
def log(self, tool: str, ms: float, size: int, ok: bool, **extra) -> None:
|
|
317
|
+
if not self.enabled:
|
|
318
|
+
return
|
|
319
|
+
try:
|
|
320
|
+
with open(self.path, "a") as fh:
|
|
321
|
+
fh.write(json.dumps({"t": round(time.time(), 3), "tool": tool, "ms": round(ms, 2),
|
|
322
|
+
"bytes": size, "ok": ok, **extra}) + "\n")
|
|
323
|
+
except OSError:
|
|
324
|
+
pass
|
devkb/startup.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Write a role's session-start brief from the graph (ODSF-311).
|
|
3
|
+
|
|
4
|
+
An agent starting a session as a tester, a developer, a DevOps engineer, a data engineer, an architect or a tech lead
|
|
5
|
+
needs different things first. This renders the process API's `role_brief` as a short markdown file, so the facts are
|
|
6
|
+
in the context from the first turn without a single tool call.
|
|
7
|
+
|
|
8
|
+
python3 devkb/startup.py --project ODSF --role QA # .devkb/startup/QA.md
|
|
9
|
+
python3 devkb/startup.py --project ODSF --all # one file per declared role
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
18
|
+
|
|
19
|
+
from devkb.process_api import ProcessAPI # noqa: E402
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _paths(paths: list[str], cap: int = 12) -> str:
|
|
23
|
+
if not paths:
|
|
24
|
+
return "nothing declared"
|
|
25
|
+
shown = ", ".join(f"`{p}`" for p in paths[:cap])
|
|
26
|
+
return shown + (f", and {len(paths) - cap} more in the profile" if len(paths) > cap else "")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def render(brief: dict) -> str:
|
|
30
|
+
meta = brief.get("_meta", {})
|
|
31
|
+
p = brief.get("profile", {})
|
|
32
|
+
lines = [f"# Session start: {brief['role']}", "",
|
|
33
|
+
f"Generated from the dev-process graph built {meta.get('built', '?')} at {meta.get('head', '?')}"
|
|
34
|
+
+ (" (the repository has moved on: re-check anything that decides a change)." if meta.get("stale") else "."),
|
|
35
|
+
f"Your role profile is `{p.get('file', '?')}`; read it when a rule below is not enough.", ""]
|
|
36
|
+
for key, label in (("mission", "Mission"), ("decides", "You decide"), ("not_decide", "You do not decide")):
|
|
37
|
+
if p.get(key):
|
|
38
|
+
lines.append(f"- **{label}**: {p[key]}")
|
|
39
|
+
t = brief.get("territory", {})
|
|
40
|
+
lines += ["", "## Territory", f"- **You own**: {_paths(t.get('own', []))}", f"- **Never touch**: {_paths(t.get('forbidden', []))}"]
|
|
41
|
+
w = brief.get("work", {})
|
|
42
|
+
lines += ["", "## Work waiting", "Tickets by status: " + (", ".join(f"{k} {v}" for k, v in sorted(w.get("counts", {}).items())) or "none") + "."]
|
|
43
|
+
for it in w.get("open", []):
|
|
44
|
+
lines.append(f"- `{it['t']}` {it['p']} {it['s']}: {it['title']}")
|
|
45
|
+
a = brief.get("area", {})
|
|
46
|
+
if a.get("files"):
|
|
47
|
+
lines += ["", "## Your code", f"{a['files']} files, {a.get('symbols', 0)} functions and classes. Most depended on:"]
|
|
48
|
+
lines += [f"- `{s['k']}` ({s['prod_callers']} callers outside tests)" for s in a.get("key_symbols", [])]
|
|
49
|
+
else:
|
|
50
|
+
lines += ["", "## Your code", "Your territory holds documents and configuration, not indexed code."]
|
|
51
|
+
if a.get("endpoints"):
|
|
52
|
+
lines.append("- Endpoints: " + ", ".join(f"`{e}`" for e in a["endpoints"]))
|
|
53
|
+
if a.get("run"):
|
|
54
|
+
lines += ["", "Run your tests with:", "", f" {a['run']}"]
|
|
55
|
+
if brief.get("docs"):
|
|
56
|
+
lines += ["", "## Read first"] + [f"- `{d['p']}`: {d['t']}" for d in brief["docs"]]
|
|
57
|
+
extras = brief.get("extras") or {}
|
|
58
|
+
if extras:
|
|
59
|
+
lines += ["", "## For this role"]
|
|
60
|
+
for k, v in extras.items():
|
|
61
|
+
if isinstance(v, list):
|
|
62
|
+
lines.append(f"- **{k.replace('_', ' ')}**: " + ("; ".join(str(x) for x in v) if v else "none"))
|
|
63
|
+
elif isinstance(v, dict):
|
|
64
|
+
lines.append(f"- **{k.replace('_', ' ')}**: " + (", ".join(f"{a2} {b2}" for a2, b2 in v.items()) or "none"))
|
|
65
|
+
elif v is not None:
|
|
66
|
+
lines.append(f"- **{k.replace('_', ' ')}**: {v}")
|
|
67
|
+
lines += ["", "## Ask the graph",
|
|
68
|
+
"Before reading the repository, use the devkb-sdlc tools through these skills: "
|
|
69
|
+
+ ", ".join(f"`{s}`" for s in brief.get("ask", [])) + ". Check `_meta.stale` on every answer.", ""]
|
|
70
|
+
return "\n".join(lines)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def write(api: ProcessAPI, role: str, out_dir: Path) -> Path | None:
|
|
74
|
+
brief = api.call("role_brief", role=role)
|
|
75
|
+
if "error" in brief:
|
|
76
|
+
print(brief["error"], file=sys.stderr)
|
|
77
|
+
return None
|
|
78
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
79
|
+
path = out_dir / f"{brief['role']}.md"
|
|
80
|
+
path.write_text(render(brief))
|
|
81
|
+
return path
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def main() -> int:
|
|
85
|
+
ap = argparse.ArgumentParser(description="write role session-start briefs from the dev-process graph")
|
|
86
|
+
ap.add_argument("--project", required=True)
|
|
87
|
+
ap.add_argument("--role")
|
|
88
|
+
ap.add_argument("--all", action="store_true")
|
|
89
|
+
ns = ap.parse_args()
|
|
90
|
+
api = ProcessAPI(ns.project)
|
|
91
|
+
out_dir = api.ctx.devkb_dir / "startup"
|
|
92
|
+
roles = sorted(api._role_table()) if ns.all else [ns.role] if ns.role else []
|
|
93
|
+
if not roles:
|
|
94
|
+
ap.error("--role CODE or --all")
|
|
95
|
+
for role in roles:
|
|
96
|
+
path = write(api, role, out_dir)
|
|
97
|
+
if path:
|
|
98
|
+
print(f"{path} {len(path.read_text().split())} words")
|
|
99
|
+
return 0
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
if __name__ == "__main__":
|
|
103
|
+
sys.exit(main())
|
devkb/test_discovery.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Where a project's tests are, and which test functions they hold (ODSF-304).
|
|
2
|
+
|
|
3
|
+
devkb used to look for tests under `<repo>/tests` only. ODSF keeps its tests in
|
|
4
|
+
`pms-backend/tests/` and `sdp/devsecops/automation/`, and another keeps them in
|
|
5
|
+
`tests/`, `ai-gateway/tests/` and `kb-service/tests/`, so the graph reported
|
|
6
|
+
0 of about 2,070 test functions for ODSF and 157 of about 2,777 for another. A
|
|
7
|
+
graph that answers "what tests this?" from a fraction of the tests is worse
|
|
8
|
+
than no graph, because it answers confidently.
|
|
9
|
+
|
|
10
|
+
Discovery is by file name over `git ls-files` (so `.gitignore` is honoured and
|
|
11
|
+
there is no exclusion list to keep in sync), not by directory, so a project
|
|
12
|
+
cannot hide tests by putting them somewhere new.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import ast
|
|
17
|
+
import re
|
|
18
|
+
import subprocess
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
_PY_TEST = re.compile(r"(^|/)(test_[^/]+|[^/]+_test)\.py$")
|
|
22
|
+
_JS_TEST = re.compile(r"(^|/)[^/]+\.(test|spec)\.[cm]?[jt]sx?$")
|
|
23
|
+
# A path segment that means "not this project's own tests".
|
|
24
|
+
_SKIP = re.compile(r"(^|/)(node_modules|\.impl-worktrees|\.claude|vendor|output)/")
|
|
25
|
+
_JS_CASE = re.compile(
|
|
26
|
+
r"^\s*(?:it|test)(?:\.(?:only|skip|todo))?\s*\(\s*(['\"`])(.+?)\1", re.MULTILINE,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def list_test_files(repo_root: Path, tests_dir: Path | None = None) -> list[str]:
|
|
31
|
+
"""Repo-relative paths of every tracked test file, sorted.
|
|
32
|
+
|
|
33
|
+
`tests_dir`, when given and present, is walked too (every `.py` under it),
|
|
34
|
+
which keeps the pre-ODSF-304 behaviour for a test file that exists on disk
|
|
35
|
+
but is not yet committed: a developer's new test is still seen."""
|
|
36
|
+
proc = subprocess.run(
|
|
37
|
+
["git", "ls-files", "--recurse-submodules"], cwd=repo_root, capture_output=True, text=True, check=False,
|
|
38
|
+
)
|
|
39
|
+
found = []
|
|
40
|
+
for rel in proc.stdout.splitlines():
|
|
41
|
+
if not rel or _SKIP.search(rel):
|
|
42
|
+
continue
|
|
43
|
+
if _PY_TEST.search(rel) or _JS_TEST.search(rel):
|
|
44
|
+
found.append(rel)
|
|
45
|
+
if tests_dir is not None and tests_dir.is_dir():
|
|
46
|
+
for path in tests_dir.rglob("*.py"):
|
|
47
|
+
rel = path.relative_to(repo_root).as_posix()
|
|
48
|
+
if not _SKIP.search(rel):
|
|
49
|
+
found.append(rel)
|
|
50
|
+
return sorted(set(found))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def tests_in_file(repo_root: Path, rel: str) -> list[tuple[str, int]]:
|
|
54
|
+
"""[(test name, 1-based line), ...] for one test file. Unreadable or
|
|
55
|
+
unparseable files yield nothing rather than raising: one broken file must
|
|
56
|
+
not hide the rest of a project's tests."""
|
|
57
|
+
path = repo_root / rel
|
|
58
|
+
try:
|
|
59
|
+
text = path.read_text(errors="replace")
|
|
60
|
+
except OSError:
|
|
61
|
+
return []
|
|
62
|
+
if rel.endswith(".py"):
|
|
63
|
+
try:
|
|
64
|
+
tree = ast.parse(text, filename=rel)
|
|
65
|
+
except (SyntaxError, ValueError):
|
|
66
|
+
return []
|
|
67
|
+
return sorted(
|
|
68
|
+
(node.name, node.lineno)
|
|
69
|
+
for node in ast.walk(tree)
|
|
70
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
|
71
|
+
and node.name.startswith("test_")
|
|
72
|
+
)
|
|
73
|
+
out = []
|
|
74
|
+
for match in _JS_CASE.finditer(text):
|
|
75
|
+
out.append((match.group(2), text.count("\n", 0, match.start()) + 1))
|
|
76
|
+
return out
|