devkb 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- devkb/__init__.py +11 -0
- devkb/audit.py +173 -0
- devkb/bench/add_packs.py +52 -0
- devkb/bench/cli_tasks.py +105 -0
- devkb/bench/confirm_tasks.py +149 -0
- devkb/bench/module_truth.py +249 -0
- devkb/bench/oco_tasks.py +216 -0
- devkb/bench/pipeline.py +313 -0
- devkb/bench/process_tasks.py +135 -0
- devkb/bench/report.py +74 -0
- devkb/bench/run.py +174 -0
- devkb/bench/truth_check.py +148 -0
- devkb/bench/workflow_report.py +64 -0
- devkb/bench/workflow_run.py +206 -0
- devkb/bench/workflow_tasks.py +227 -0
- devkb/build.py +258 -0
- devkb/build_symbols.py +1394 -0
- devkb/call_recall.py +159 -0
- devkb/cli.py +53 -0
- devkb/coverage.py +155 -0
- devkb/edge_truth.py +272 -0
- devkb/extractors/backlog.py +113 -0
- devkb/extractors/component_subsystem.py +116 -0
- devkb/extractors/configuration.py +267 -0
- devkb/extractors/files_tests.py +220 -0
- devkb/extractors/flows.py +117 -0
- devkb/extractors/lifecycle.py +258 -0
- devkb/extractors/members.py +160 -0
- devkb/extractors/pull_requests.py +271 -0
- devkb/extractors/releases.py +49 -0
- devkb/extractors/source_baseline.py +66 -0
- devkb/extractors/spec_docs.py +220 -0
- devkb/extractors/symbol_tests.py +131 -0
- devkb/extractors/test_inventory.py +38 -0
- devkb/extractors/ticket_similarity.py +193 -0
- devkb/extractors/tickets_git.py +151 -0
- devkb/extractors/warnings.py +230 -0
- devkb/extractors/work_units.py +90 -0
- devkb/github.py +515 -0
- devkb/gitutil.py +16 -0
- devkb/index.py +51 -0
- devkb/init.py +178 -0
- devkb/mcp_server.py +216 -0
- devkb/model.py +221 -0
- devkb/module_file_bridge.py +57 -0
- devkb/ontology.md +178 -0
- devkb/packs.py +462 -0
- devkb/process_api.py +1601 -0
- devkb/project_context.py +191 -0
- devkb/query.py +251 -0
- devkb/service.py +324 -0
- devkb/startup.py +103 -0
- devkb/test_discovery.py +76 -0
- devkb/ts_symbols.py +291 -0
- devkb-0.2.0.dist-info/METADATA +162 -0
- devkb-0.2.0.dist-info/RECORD +61 -0
- devkb-0.2.0.dist-info/WHEEL +5 -0
- devkb-0.2.0.dist-info/entry_points.txt +2 -0
- devkb-0.2.0.dist-info/licenses/LICENSE +202 -0
- devkb-0.2.0.dist-info/licenses/NOTICE +11 -0
- devkb-0.2.0.dist-info/top_level.txt +1 -0
devkb/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""devkb — cross-project dev-process knowledge graph (ADR-SDP-DEVKB-001).
|
|
2
|
+
|
|
3
|
+
Tickets, Bug Lineage, git commit log, work packages/milestones (Phase 1);
|
|
4
|
+
source files/tests, compliance warnings, specs (later phases). One small,
|
|
5
|
+
governed graph per registered project, persisted at
|
|
6
|
+
<project-repo>/.devkb/{nodes,edges,manifest}.jsonl — not a fork of OCO's
|
|
7
|
+
cyber-threat-intel graph (ADR-SDP-FACTORY-EXTRACTION-001 judged that one
|
|
8
|
+
domain-coupled and deliberately non-reusable) and not an extension of
|
|
9
|
+
discovery/'s component-architecture graph (discovery/ontology.md scopes
|
|
10
|
+
itself to source structure only, by design).
|
|
11
|
+
"""
|
devkb/audit.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""What a devkb graph is made of, and what it costs (ODSF-314).
|
|
3
|
+
|
|
4
|
+
Per verb: count, share of all edges, fan-out per source node (p50/p99/max), source mix, confidence spread, dangling
|
|
5
|
+
edges, the share that touches a generated path, and whether any process-API tool reads it. Per node type: count,
|
|
6
|
+
orphans, generated-path share. Size: JSONL and cache bytes, cold load and warm query time.
|
|
7
|
+
|
|
8
|
+
python3 devkb/audit.py --project OCO # the project's live .devkb
|
|
9
|
+
python3 devkb/audit.py --project OCO --dir <snapshot> # any built graph directory
|
|
10
|
+
python3 devkb/audit.py --project OCO --json OUT.json
|
|
11
|
+
python3 devkb/audit.py --project OCO --check # exit 1 when a floor in FLOORS fails
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import collections
|
|
17
|
+
import json
|
|
18
|
+
import re
|
|
19
|
+
import statistics
|
|
20
|
+
import sys
|
|
21
|
+
import time
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
25
|
+
|
|
26
|
+
from devkb.model import VERBS # noqa: E402
|
|
27
|
+
|
|
28
|
+
HERE = Path(__file__).resolve().parent
|
|
29
|
+
GENERATED = re.compile(r"(^|/)(package-lock\.json|yarn\.lock|pnpm-lock\.yaml|poetry\.lock|uv\.lock|Cargo\.lock)$|"
|
|
30
|
+
r"\.min\.(js|css)$|(^|/)(dist|build|node_modules|__pycache__|\.next[^/]*)/")
|
|
31
|
+
# Floors a healthy graph keeps; set from the measured values in docs/graph-audit-odsf314.md.
|
|
32
|
+
FLOORS = {"max_low_value_share": 0.60, "max_generated_edge_share": 0.02, "max_dangling_share": 0.001} # OCO after ODSF-314: 0.56, 0.000, 0.00005
|
|
33
|
+
LOW_VALUE = ("TOUCHES", "SIMILAR_TO", "IN_REPO", "GROUPS")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _pct(xs: list[int], q: float) -> int:
|
|
37
|
+
if not xs:
|
|
38
|
+
return 0
|
|
39
|
+
xs = sorted(xs)
|
|
40
|
+
return xs[min(len(xs) - 1, int(q * (len(xs) - 1)))]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def tool_reads() -> dict[str, int]:
|
|
44
|
+
text = "".join((HERE / f).read_text() for f in ("process_api.py", "packs.py", "service.py") if (HERE / f).exists())
|
|
45
|
+
return {v: len(re.findall(r'"' + v + r'"', text)) for v in VERBS}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def audit(devkb_dir: Path) -> dict:
|
|
49
|
+
nodes: dict[str, tuple[str, str]] = {}
|
|
50
|
+
for line in (devkb_dir / "nodes.jsonl").read_text().splitlines():
|
|
51
|
+
if line:
|
|
52
|
+
n = json.loads(line)
|
|
53
|
+
nodes[n["uid"]] = (n["type"], n["natural_key"])
|
|
54
|
+
degree: collections.Counter = collections.Counter()
|
|
55
|
+
verb = collections.Counter()
|
|
56
|
+
fan: dict[str, collections.Counter] = collections.defaultdict(collections.Counter)
|
|
57
|
+
src: dict[str, collections.Counter] = collections.defaultdict(collections.Counter)
|
|
58
|
+
conf: dict[str, list[float]] = collections.defaultdict(list)
|
|
59
|
+
pairs: dict[str, collections.Counter] = collections.defaultdict(collections.Counter)
|
|
60
|
+
dangling, generated = collections.Counter(), collections.Counter()
|
|
61
|
+
for line in (devkb_dir / "edges.jsonl").read_text().splitlines():
|
|
62
|
+
if not line:
|
|
63
|
+
continue
|
|
64
|
+
e = json.loads(line)
|
|
65
|
+
v, a, b = e["verb"], nodes.get(e["from_uid"]), nodes.get(e["to_uid"])
|
|
66
|
+
verb[v] += 1
|
|
67
|
+
p = e.get("properties") or {}
|
|
68
|
+
src[v][str(p.get("source", "?")).split(":")[-1]] += 1
|
|
69
|
+
conf[v].append(float(p.get("confidence") or 0))
|
|
70
|
+
if a is None or b is None:
|
|
71
|
+
dangling[v] += 1
|
|
72
|
+
continue
|
|
73
|
+
degree[e["from_uid"]] += 1
|
|
74
|
+
degree[e["to_uid"]] += 1
|
|
75
|
+
fan[v][e["from_uid"]] += 1
|
|
76
|
+
pairs[v][f"{a[0]}->{b[0]}"] += 1
|
|
77
|
+
if GENERATED.search(a[1].split("::")[0]) or GENERATED.search(b[1].split("::")[0]):
|
|
78
|
+
generated[v] += 1
|
|
79
|
+
total = sum(verb.values()) or 1
|
|
80
|
+
reads = tool_reads()
|
|
81
|
+
verbs = {}
|
|
82
|
+
for v in sorted(verb, key=lambda x: -verb[x]):
|
|
83
|
+
fo = list(fan[v].values())
|
|
84
|
+
cs = conf[v]
|
|
85
|
+
verbs[v] = {"n": verb[v], "share": round(verb[v] / total, 4), "pairs": dict(pairs[v].most_common(3)),
|
|
86
|
+
"fanout_p50": _pct(fo, 0.5), "fanout_p99": _pct(fo, 0.99), "fanout_max": max(fo, default=0),
|
|
87
|
+
"sources": dict(src[v].most_common(4)), "conf_min": round(min(cs), 2), "conf_max": round(max(cs), 2),
|
|
88
|
+
"conf_distinct": len({round(c, 2) for c in cs}), "dangling": dangling[v], "generated": generated[v],
|
|
89
|
+
"tool_reads": reads.get(v, 0)}
|
|
90
|
+
types = collections.Counter(t for t, _ in nodes.values())
|
|
91
|
+
orphans = collections.Counter(nodes[u][0] for u in nodes if not degree[u])
|
|
92
|
+
gen_nodes = collections.Counter(t for u, (t, k) in nodes.items() if GENERATED.search(k.split("::")[0]))
|
|
93
|
+
size = {name: (devkb_dir / name).stat().st_size for name in ("nodes.jsonl", "edges.jsonl", ".process-index.pkl")
|
|
94
|
+
if (devkb_dir / name).exists()}
|
|
95
|
+
return {"dir": str(devkb_dir), "nodes": len(nodes), "edges": total, "verbs": verbs,
|
|
96
|
+
"types": {t: {"n": n, "orphans": orphans[t], "generated": gen_nodes[t]} for t, n in types.most_common()},
|
|
97
|
+
"unused_verbs": [v for v in VERBS if v not in verb], "bytes": size,
|
|
98
|
+
"low_value_share": round(sum(verb[v] for v in LOW_VALUE) / total, 4),
|
|
99
|
+
"generated_edge_share": round(sum(generated.values()) / total, 4),
|
|
100
|
+
"dangling_share": round(sum(dangling.values()) / total, 5)}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def timing(code: str, devkb_dir: Path) -> dict:
|
|
104
|
+
from devkb import service # noqa: PLC0415
|
|
105
|
+
from devkb.process_api import ProcessAPI # noqa: PLC0415
|
|
106
|
+
cache = devkb_dir / service.CACHE_NAME
|
|
107
|
+
if cache.exists():
|
|
108
|
+
cache.unlink()
|
|
109
|
+
t0 = time.perf_counter()
|
|
110
|
+
api = ProcessAPI(code, devkb_dir=devkb_dir, telemetry=False)
|
|
111
|
+
cold = time.perf_counter() - t0
|
|
112
|
+
t0 = time.perf_counter()
|
|
113
|
+
ProcessAPI(code, devkb_dir=devkb_dir, telemetry=False)
|
|
114
|
+
warm_load = time.perf_counter() - t0
|
|
115
|
+
sym = max((v[0] for v in api.idx.short.values() if len(v) == 1), key=lambda s: len(api.idx.sources("CALLS", s)), default=None)
|
|
116
|
+
samples = []
|
|
117
|
+
for _ in range(20):
|
|
118
|
+
t0 = time.perf_counter()
|
|
119
|
+
api.call("callers", symbol=api.idx.keys[sym], depth=2)
|
|
120
|
+
samples.append((time.perf_counter() - t0) * 1000)
|
|
121
|
+
return {"cold_build_s": round(cold, 2), "cached_load_s": round(warm_load, 2),
|
|
122
|
+
"callers_p50_ms": round(statistics.median(samples), 2), "cache_bytes": cache.stat().st_size if cache.exists() else 0}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def render(r: dict) -> str:
|
|
126
|
+
lines = [f"Graph `{r['dir']}`: {r['nodes']:,} nodes, {r['edges']:,} edges; low-value verbs {r['low_value_share']:.0%} of edges, "
|
|
127
|
+
f"generated paths {r['generated_edge_share']:.1%}, dangling {r['dangling_share']:.3%}.", "",
|
|
128
|
+
"| Verb | Edges | Share | Fan-out p50/p99/max | Confidence (distinct) | Generated | Tool reads | Main source |",
|
|
129
|
+
"|---|---|---|---|---|---|---|---|"]
|
|
130
|
+
for v, s in r["verbs"].items():
|
|
131
|
+
lines.append(f"| `{v}` | {s['n']:,} | {s['share']:.1%} | {s['fanout_p50']}/{s['fanout_p99']}/{s['fanout_max']} | "
|
|
132
|
+
f"{s['conf_min']}–{s['conf_max']} ({s['conf_distinct']}) | {s['generated']:,} | {s['tool_reads']} | "
|
|
133
|
+
f"{next(iter(s['sources']), '')} |")
|
|
134
|
+
lines += ["", "Verbs with no edges: " + (", ".join(f"`{v}`" for v in r["unused_verbs"]) or "none"), "",
|
|
135
|
+
"| Node type | Count | Orphans | Generated |", "|---|---|---|---|"]
|
|
136
|
+
lines += [f"| `{t}` | {s['n']:,} | {s['orphans']:,} | {s['generated']:,} |" for t, s in r["types"].items()]
|
|
137
|
+
if "timing" in r:
|
|
138
|
+
t = r["timing"]
|
|
139
|
+
lines += ["", f"Cold index build {t['cold_build_s']} s, cached load {t['cached_load_s']} s, `callers` p50 {t['callers_p50_ms']} ms, "
|
|
140
|
+
f"cache {t['cache_bytes'] / 1e6:.1f} MB; JSONL {sum(v for k, v in r['bytes'].items() if k.endswith('.jsonl')) / 1e6:.1f} MB."]
|
|
141
|
+
return "\n".join(lines)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def main() -> int:
|
|
145
|
+
ap = argparse.ArgumentParser(description="audit a devkb graph's content and cost")
|
|
146
|
+
ap.add_argument("--project", required=True)
|
|
147
|
+
ap.add_argument("--dir", help="a built graph directory (default: the project's .devkb)")
|
|
148
|
+
ap.add_argument("--json")
|
|
149
|
+
ap.add_argument("--timing", action="store_true", help="also measure index build and query time")
|
|
150
|
+
ap.add_argument("--check", action="store_true")
|
|
151
|
+
ns = ap.parse_args()
|
|
152
|
+
if ns.dir:
|
|
153
|
+
d = Path(ns.dir)
|
|
154
|
+
else:
|
|
155
|
+
from devkb.project_context import resolve # noqa: PLC0415
|
|
156
|
+
d = resolve(ns.project).devkb_dir
|
|
157
|
+
r = audit(d)
|
|
158
|
+
if ns.timing:
|
|
159
|
+
r["timing"] = timing(ns.project, d)
|
|
160
|
+
print(render(r))
|
|
161
|
+
if ns.json:
|
|
162
|
+
Path(ns.json).write_text(json.dumps(r, indent=1))
|
|
163
|
+
if ns.check:
|
|
164
|
+
bad = [k for k, v in (("max_low_value_share", r["low_value_share"]), ("max_generated_edge_share", r["generated_edge_share"]),
|
|
165
|
+
("max_dangling_share", r["dangling_share"])) if v > FLOORS[k]]
|
|
166
|
+
if bad:
|
|
167
|
+
print("FLOOR FAILED: " + ", ".join(bad), file=sys.stderr)
|
|
168
|
+
return 1
|
|
169
|
+
return 0
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
if __name__ == "__main__":
|
|
173
|
+
sys.exit(main())
|
devkb/bench/add_packs.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Attach a predefined graph pack to each workflow benchmark task (ODSF-313).
|
|
3
|
+
|
|
4
|
+
The pack is built from what the task's prompt states and nothing else (the error message; the function and the file
|
|
5
|
+
it is defined in; the role), the same input a build loop has from a ticket. The truth is never read here.
|
|
6
|
+
|
|
7
|
+
DEVKB_DIR=<snapshot> DEVKB_REPO=<checkout> python3 devkb/bench/add_packs.py OCO TASKS.json OUT.json
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
17
|
+
|
|
18
|
+
from devkb.packs import Packer # noqa: E402
|
|
19
|
+
from devkb.process_api import ProcessAPI # noqa: E402
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def inputs(task: dict) -> tuple[str, dict]:
|
|
23
|
+
prompt = task["prompt"]
|
|
24
|
+
if task["kind"] == "ticket_code": # ADDENDUM-1 S2
|
|
25
|
+
return "ticket", {"ticket_id": re.search(r"Ticket ([A-Z]+-\d+)", prompt).group(1)}
|
|
26
|
+
if task["kind"] == "commit_trace":
|
|
27
|
+
return "commit_trace", {"commit": re.search(r"Commit ([0-9a-f]{7,40})", prompt).group(1)}
|
|
28
|
+
if task["kind"] in ("err_to_handler", "rca"):
|
|
29
|
+
return "error", {"message": re.search(r'"(.+?)"\. ', prompt).group(1)}
|
|
30
|
+
name = re.search(r"`([^`]+)`", prompt).group(1)
|
|
31
|
+
path = re.search(r"\(defined in ([^)]+)\)", prompt).group(1)
|
|
32
|
+
if task["kind"] == "impact":
|
|
33
|
+
return "change", {"name": name, "path": path}
|
|
34
|
+
m = re.search(r"as role (\w+)", prompt)
|
|
35
|
+
return "refactor", {"name": name, "role": m.group(1) if m else "", "path": path}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main() -> int:
|
|
39
|
+
code, src, dst = sys.argv[1:4]
|
|
40
|
+
packer = Packer(ProcessAPI(code, telemetry=False))
|
|
41
|
+
tasks = json.load(open(src))
|
|
42
|
+
for t in tasks:
|
|
43
|
+
kind, kw = inputs(t)
|
|
44
|
+
t["pack"] = getattr(packer, kind)(**kw)
|
|
45
|
+
t["pack_inputs"] = {"kind": kind, **kw}
|
|
46
|
+
print(t["id"], kind, len(t["pack"].split()), "words")
|
|
47
|
+
json.dump(tasks, open(dst, "w"), indent=1)
|
|
48
|
+
return 0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
if __name__ == "__main__":
|
|
52
|
+
sys.exit(main())
|
devkb/bench/cli_tasks.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Benchmark tasks for a repository without HTTP route handlers (a command-line tool, a library).
|
|
3
|
+
|
|
4
|
+
python3 devkb/bench/cli_tasks.py CHECKOUT OUT.json --seed 2029 --per-kind 8 --prefix L-
|
|
5
|
+
|
|
6
|
+
confirm_tasks.py asks which route handlers reach a function, which has no answer in a repository whose only entry
|
|
7
|
+
point is `main`. These two kinds keep its truth (import-path resolution, module_truth.py, never the graph) and ask
|
|
8
|
+
what a developer of such a repository asks before a change:
|
|
9
|
+
|
|
10
|
+
rca an error message (four or more words, raised once, a literal) in a module-level function: the raiser,
|
|
11
|
+
every production function that reaches it through calls at any depth (2 to 15, a clean closure: every
|
|
12
|
+
function on the way is a module-level function the truth sees), and every test file that calls the
|
|
13
|
+
raiser or one of those functions
|
|
14
|
+
refactor a module-level function called from 4 to 25 files, tests among them but not all: the files with a
|
|
15
|
+
direct caller, and the test files among them
|
|
16
|
+
|
|
17
|
+
Selection is shuffled with the seed; up to --per-kind tasks of each kind, one per function.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import argparse
|
|
22
|
+
import collections
|
|
23
|
+
import json
|
|
24
|
+
import random
|
|
25
|
+
import sys
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
29
|
+
import module_truth as mt # noqa: E402
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def build(root: Path, seed: int, per_kind: int) -> list[dict]:
|
|
33
|
+
T = mt.Truth(root)
|
|
34
|
+
callers = T.callers()
|
|
35
|
+
rnd = random.Random(seed)
|
|
36
|
+
used: set[str] = set()
|
|
37
|
+
tasks: list[dict] = []
|
|
38
|
+
|
|
39
|
+
counts = collections.Counter(msg for f in T.files.values() for _q, msg in f.raises)
|
|
40
|
+
errs = []
|
|
41
|
+
for rel, f in T.files.items():
|
|
42
|
+
if mt.is_test(rel):
|
|
43
|
+
continue
|
|
44
|
+
for qual, msg in f.raises:
|
|
45
|
+
key = (rel, qual)
|
|
46
|
+
if counts[msg] != 1 or len(msg.split()) < 4 or "{" in msg or not T.is_function(key):
|
|
47
|
+
continue
|
|
48
|
+
prod, tests, clean = T.up(key)
|
|
49
|
+
test_files = sorted({k[0] for k in tests})
|
|
50
|
+
if clean and 2 <= len(prod) <= 15 and test_files:
|
|
51
|
+
errs.append((key, msg, sorted(mt.short(k) for k in prod), test_files))
|
|
52
|
+
rnd.shuffle(errs)
|
|
53
|
+
for key, msg, prod, test_files in errs:
|
|
54
|
+
if len([t for t in tasks if t["kind"] == "rca"]) == per_kind:
|
|
55
|
+
break
|
|
56
|
+
if key[1] in used:
|
|
57
|
+
continue
|
|
58
|
+
used.add(key[1])
|
|
59
|
+
tasks.append({"id": f"ERR-{key[1]}", "kind": "rca", "skill": "devkb-rca",
|
|
60
|
+
"prompt": (f'A user of this tool reports an error whose message is: "{msg}". Trace it: name the function that '
|
|
61
|
+
"raises it, every function outside the tests that can reach that function through calls at any depth, "
|
|
62
|
+
"and every test file that calls the raising function or any of those functions. Answer with one JSON "
|
|
63
|
+
'object: {"raiser": "path::function", "callers": ["path::function", ...], "test_files": ["path", ...]}.'),
|
|
64
|
+
"truth": {"raiser": mt.short(key), "callers": prod, "test_files": test_files}})
|
|
65
|
+
|
|
66
|
+
wide = []
|
|
67
|
+
for rel, f in T.files.items():
|
|
68
|
+
if mt.is_test(rel):
|
|
69
|
+
continue
|
|
70
|
+
for name in sorted(f.functions):
|
|
71
|
+
key = (rel, name)
|
|
72
|
+
if name in used or len(name) < 6:
|
|
73
|
+
continue
|
|
74
|
+
files = sorted({c[0] for c in callers.get(key, ())})
|
|
75
|
+
if 4 <= len(files) <= 25 and 0 < sum(mt.is_test(x) for x in files) < len(files):
|
|
76
|
+
wide.append((key, files))
|
|
77
|
+
rnd.shuffle(wide)
|
|
78
|
+
for key, files in wide[:per_kind]:
|
|
79
|
+
used.add(key[1])
|
|
80
|
+
tasks.append({"id": f"WIDE-{key[1]}", "kind": "refactor", "skill": "devkb-refactor",
|
|
81
|
+
"prompt": (f"You will change the signature of `{key[1]}` (defined in {key[0]}). Before editing, scope the change: "
|
|
82
|
+
"list every file that contains a direct caller (tests included), and the test files among them. "
|
|
83
|
+
'Answer with one JSON object: {"caller_files": [...], "test_files": [...]}.'),
|
|
84
|
+
"truth": {"caller_files": files, "test_files": [x for x in files if mt.is_test(x)]}})
|
|
85
|
+
return tasks
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def main() -> int:
|
|
89
|
+
ap = argparse.ArgumentParser()
|
|
90
|
+
ap.add_argument("checkout")
|
|
91
|
+
ap.add_argument("out")
|
|
92
|
+
ap.add_argument("--seed", type=int, required=True)
|
|
93
|
+
ap.add_argument("--per-kind", type=int, default=8)
|
|
94
|
+
ap.add_argument("--prefix", default="")
|
|
95
|
+
a = ap.parse_args()
|
|
96
|
+
tasks = build(Path(a.checkout), a.seed, a.per_kind)
|
|
97
|
+
for t in tasks:
|
|
98
|
+
t["id"] = a.prefix + t["id"]
|
|
99
|
+
print(t["id"], {k: (len(v) if isinstance(v, list) else v) for k, v in t["truth"].items()})
|
|
100
|
+
json.dump(tasks, open(a.out, "w"), indent=1)
|
|
101
|
+
return 0
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
if __name__ == "__main__":
|
|
105
|
+
sys.exit(main())
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Confirmatory task set (ODSF-315, PREREGISTRATION.md): the three exploratory task kinds on import-path truth.
|
|
3
|
+
|
|
4
|
+
python3 devkb/bench/confirm_tasks.py CHECKOUT OUT.json --seed 2027 --per-kind 4 --exclude names.json --prefix C-
|
|
5
|
+
|
|
6
|
+
Selection, fixed before any confirmatory run:
|
|
7
|
+
err_to_handler a literal message of four or more words raised once, in a module-level non-route function; clean
|
|
8
|
+
closure; one to twelve route handlers reach it; at least one test file on the paths; deepest first;
|
|
9
|
+
one task per raising function
|
|
10
|
+
impact a module-level non-test, non-route function; clean closure; three to fifteen handlers; a test file;
|
|
11
|
+
four or more production callers; shuffled with the seed
|
|
12
|
+
refactor a module-level function called from eight to twenty-five files, tests among them but not all;
|
|
13
|
+
shuffled with the seed; the role-boundary question only where the repository declares roles
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import collections
|
|
19
|
+
import json
|
|
20
|
+
import random
|
|
21
|
+
import sys
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
25
|
+
import module_truth as mt # noqa: E402
|
|
26
|
+
import workflow_tasks as wt # noqa: E402
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def build(root: Path, seed: int, per_kind: int, exclude: set[str]) -> list[dict]:
|
|
30
|
+
T = mt.Truth(root)
|
|
31
|
+
callers = T.callers()
|
|
32
|
+
rnd = random.Random(seed)
|
|
33
|
+
used = set(exclude)
|
|
34
|
+
tasks: list[dict] = []
|
|
35
|
+
|
|
36
|
+
def reach(key):
|
|
37
|
+
prod, tests, clean = T.up(key)
|
|
38
|
+
handlers = sorted(mt.short(k) for k in prod if T.is_route(k))
|
|
39
|
+
test_files = sorted({k[0] for k in tests})
|
|
40
|
+
return prod, handlers, test_files, clean
|
|
41
|
+
|
|
42
|
+
counts = collections.Counter(msg for f in T.files.values() for _q, msg in f.raises)
|
|
43
|
+
errs = []
|
|
44
|
+
for rel, f in T.files.items():
|
|
45
|
+
if mt.is_test(rel):
|
|
46
|
+
continue
|
|
47
|
+
for qual, msg in f.raises:
|
|
48
|
+
key = (rel, qual)
|
|
49
|
+
if (qual in used or counts[msg] != 1 or len(msg.split()) < 4 or "{" in msg or not T.is_function(key)
|
|
50
|
+
or T.is_route(key)):
|
|
51
|
+
continue
|
|
52
|
+
prod, hs, tests, clean = reach(key)
|
|
53
|
+
if clean and 1 <= len(hs) <= 12 and tests and len(prod) >= 1:
|
|
54
|
+
errs.append((key, msg, hs, tests, len(prod)))
|
|
55
|
+
errs.sort(key=lambda e: (-e[4], -len(e[3]), e[0]))
|
|
56
|
+
chosen = 0
|
|
57
|
+
for key, msg, hs, tests, n in errs:
|
|
58
|
+
if chosen == per_kind:
|
|
59
|
+
break
|
|
60
|
+
if key[1] in used:
|
|
61
|
+
continue # one task per raising function, even when it raises several messages
|
|
62
|
+
chosen += 1
|
|
63
|
+
used.add(key[1])
|
|
64
|
+
tasks.append({"id": f"ERR-{key[1]}", "kind": "err_to_handler", "skill": "devkb-rca",
|
|
65
|
+
"prompt": (f'Users of the web API get an error whose message is: "{msg}". Trace it: name the function that raises it, '
|
|
66
|
+
"every HTTP route handler function (a function decorated as a FastAPI route) that can reach that function "
|
|
67
|
+
"through calls at any depth, and every test file that calls the raising function or any function on a path "
|
|
68
|
+
"between it and those handlers. Answer with one JSON object: "
|
|
69
|
+
'{"raiser": "path::function", "handlers": ["path::function", ...], "test_files": ["path", ...]}.'),
|
|
70
|
+
"truth": {"raiser": mt.short(key), "handlers": hs, "test_files": tests, "closure_size": n}})
|
|
71
|
+
|
|
72
|
+
imp = []
|
|
73
|
+
for rel, f in T.files.items():
|
|
74
|
+
if mt.is_test(rel):
|
|
75
|
+
continue
|
|
76
|
+
for name in sorted(f.functions):
|
|
77
|
+
key = (rel, name)
|
|
78
|
+
if name in used or len(name) < 8 or T.is_route(key):
|
|
79
|
+
continue
|
|
80
|
+
prod, hs, tests, clean = reach(key)
|
|
81
|
+
if clean and 3 <= len(hs) <= 15 and tests and len(prod) >= 4:
|
|
82
|
+
imp.append((key, hs, tests, len(prod)))
|
|
83
|
+
rnd.shuffle(imp)
|
|
84
|
+
for key, hs, tests, n in imp[:per_kind]:
|
|
85
|
+
used.add(key[1])
|
|
86
|
+
tasks.append({"id": f"IMP-{key[1]}", "kind": "impact", "skill": "devkb-feature",
|
|
87
|
+
"prompt": (f"You are about to change the behaviour of `{key[1]}` (defined in {key[0]}). Before editing, find which HTTP route "
|
|
88
|
+
"handler functions (functions decorated as FastAPI routes) can reach it through calls at any depth, and which "
|
|
89
|
+
"test files you must run: every test file that calls it or any function on a path between it and those handlers. "
|
|
90
|
+
'Answer with one JSON object: {"handlers": ["path::function", ...], "test_files": ["path", ...]}.'),
|
|
91
|
+
"truth": {"handlers": hs, "test_files": tests, "closure_size": n}})
|
|
92
|
+
|
|
93
|
+
roles = wt.dnt(root)
|
|
94
|
+
wide = []
|
|
95
|
+
for rel, f in T.files.items():
|
|
96
|
+
if mt.is_test(rel):
|
|
97
|
+
continue
|
|
98
|
+
for name in sorted(f.functions):
|
|
99
|
+
key = (rel, name)
|
|
100
|
+
if name in used or len(name) < 8:
|
|
101
|
+
continue
|
|
102
|
+
files = sorted({c[0] for c in callers.get(key, ())})
|
|
103
|
+
if not (8 <= len(files) <= 25 and 0 < sum(mt.is_test(x) for x in files) < len(files)):
|
|
104
|
+
continue
|
|
105
|
+
role, blocked = "", None
|
|
106
|
+
for r, prefixes in sorted(roles.items()):
|
|
107
|
+
b = sorted(x for x in files if any(x.startswith(p) for p in prefixes))
|
|
108
|
+
if b and len(b) < len(files) and not any(rel.startswith(p) for p in prefixes):
|
|
109
|
+
role, blocked = r, b
|
|
110
|
+
break
|
|
111
|
+
wide.append((key, files, role, blocked))
|
|
112
|
+
rnd.shuffle(wide)
|
|
113
|
+
for key, files, role, blocked in wide[:per_kind]:
|
|
114
|
+
tests = [x for x in files if mt.is_test(x)]
|
|
115
|
+
if role:
|
|
116
|
+
prompt = (f"You work as role {role} and will change the signature of `{key[1]}` (defined in {key[0]}). Before editing, scope "
|
|
117
|
+
"the change: list every file that contains a direct caller (tests included), the test files among them, and which "
|
|
118
|
+
f"of those files role {role} must not edit according to the DO NOT TOUCH block of its profile in sdp/process/roles. "
|
|
119
|
+
'Answer with one JSON object: {"caller_files": [...], "test_files": [...], "blocked_files": [...]}.')
|
|
120
|
+
truth = {"caller_files": files, "test_files": tests, "blocked_files": blocked}
|
|
121
|
+
else:
|
|
122
|
+
prompt = (f"You will change the signature of `{key[1]}` (defined in {key[0]}). Before editing, scope the change: "
|
|
123
|
+
"list every file that contains a direct caller (tests included), and the test files among them. "
|
|
124
|
+
'Answer with one JSON object: {"caller_files": [...], "test_files": [...]}.')
|
|
125
|
+
truth = {"caller_files": files, "test_files": tests}
|
|
126
|
+
tasks.append({"id": f"WIDE-{key[1]}", "kind": "refactor", "skill": "devkb-refactor", "prompt": prompt, "truth": truth})
|
|
127
|
+
return tasks
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def main() -> int:
|
|
131
|
+
ap = argparse.ArgumentParser()
|
|
132
|
+
ap.add_argument("checkout")
|
|
133
|
+
ap.add_argument("out")
|
|
134
|
+
ap.add_argument("--seed", type=int, required=True)
|
|
135
|
+
ap.add_argument("--per-kind", type=int, default=4)
|
|
136
|
+
ap.add_argument("--exclude")
|
|
137
|
+
ap.add_argument("--prefix", default="")
|
|
138
|
+
a = ap.parse_args()
|
|
139
|
+
tasks = build(Path(a.checkout), a.seed, a.per_kind, set(json.load(open(a.exclude))) if a.exclude else set())
|
|
140
|
+
for t in tasks:
|
|
141
|
+
t["id"] = a.prefix + t["id"]
|
|
142
|
+
json.dump(tasks, open(a.out, "w"), indent=1)
|
|
143
|
+
for t in tasks:
|
|
144
|
+
print(t["id"], {k: (len(v) if isinstance(v, (list, dict)) else v) for k, v in t["truth"].items()})
|
|
145
|
+
return 0
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
if __name__ == "__main__":
|
|
149
|
+
sys.exit(main())
|