witnis 1.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. witnis/__init__.py +3 -0
  2. witnis/__main__.py +3 -0
  3. witnis/bench.py +367 -0
  4. witnis/cli.py +1228 -0
  5. witnis/compare_baselines.py +275 -0
  6. witnis/core/__init__.py +1 -0
  7. witnis/core/ab_experiment.py +171 -0
  8. witnis/core/bench_report.py +142 -0
  9. witnis/core/calls_python.py +204 -0
  10. witnis/core/discover.py +62 -0
  11. witnis/core/envelope.py +34 -0
  12. witnis/core/gitinfo.py +359 -0
  13. witnis/core/ground_truth.py +284 -0
  14. witnis/core/gt_filters.py +40 -0
  15. witnis/core/index.py +252 -0
  16. witnis/core/loop_init.py +206 -0
  17. witnis/core/lsp_python.py +302 -0
  18. witnis/core/models.py +23 -0
  19. witnis/core/native.py +79 -0
  20. witnis/core/parse_go.py +39 -0
  21. witnis/core/parse_java.py +48 -0
  22. witnis/core/parse_js.py +108 -0
  23. witnis/core/parse_python.py +58 -0
  24. witnis/core/pr_gate.py +584 -0
  25. witnis/core/references_python.py +54 -0
  26. witnis/core/relations_go.py +98 -0
  27. witnis/core/relations_java.py +370 -0
  28. witnis/core/relations_js.py +126 -0
  29. witnis/core/relations_python.py +177 -0
  30. witnis/core/runtime.py +50 -0
  31. witnis/core/scene.py +214 -0
  32. witnis/core/security.py +117 -0
  33. witnis/core/store.py +785 -0
  34. witnis/core/tools.py +2625 -0
  35. witnis/core/verify.py +605 -0
  36. witnis/core/verify_isolate.py +275 -0
  37. witnis/mcp_server.py +476 -0
  38. witnis/sdk.py +83 -0
  39. witnis/studio/__init__.py +5 -0
  40. witnis/studio/assets/blast-radius.svg +79 -0
  41. witnis/studio/assets/caught-lying.svg +23 -0
  42. witnis/studio/assets/trusted-agent-loop.svg +34 -0
  43. witnis/studio/assets/witnis-masthead.svg +13 -0
  44. witnis/studio/home.html +144 -0
  45. witnis/studio/render.py +212 -0
  46. witnis/studio/scene_template.html +301 -0
  47. witnis/studio/template.html +350 -0
  48. witnis-1.11.0.dist-info/METADATA +91 -0
  49. witnis-1.11.0.dist-info/RECORD +53 -0
  50. witnis-1.11.0.dist-info/WHEEL +5 -0
  51. witnis-1.11.0.dist-info/entry_points.txt +2 -0
  52. witnis-1.11.0.dist-info/licenses/LICENSE +190 -0
  53. witnis-1.11.0.dist-info/top_level.txt +1 -0
witnis/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """Witnis — engineering witness for AI coding agents."""
2
+
3
+ __version__ = "1.11.0"
witnis/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from witnis.cli import app
2
+
3
+ app()
witnis/bench.py ADDED
@@ -0,0 +1,367 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from dataclasses import dataclass
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from witnis.core.index import index_repository
9
+ from witnis.core.tools import (
10
+ analyze_impact,
11
+ check_import_graph,
12
+ check_index_health,
13
+ explain_code,
14
+ export_evidence_bundle,
15
+ find_related_tests,
16
+ get_evidence,
17
+ export_relation_graph,
18
+ list_capabilities,
19
+ list_index_snapshots,
20
+ ingest_runtime_traces,
21
+ query_lsp,
22
+ refresh_evidence,
23
+ render_evidence_report,
24
+ repo_overview,
25
+ repo_search,
26
+ repo_status,
27
+ trace_symbol,
28
+ )
29
+ from witnis.compare_baselines import run_baseline_compare
30
+ from witnis.core.verify import create_verification_plan, run_verification
31
+ from witnis.studio import render_studio
32
+
33
+
34
+ @dataclass
35
+ class TaskResult:
36
+ id: str
37
+ passed: bool
38
+ detail: str
39
+
40
+
41
+ def _load_tasks(path: Path) -> dict[str, Any]:
42
+ return json.loads(path.read_text(encoding="utf-8"))
43
+
44
+
45
+ def _check_expect(kind: str, payload: dict[str, Any], expect: dict[str, Any]) -> tuple[bool, str]:
46
+ if kind == "search":
47
+ names = {m.get("name") for m in payload.get("result", {}).get("matches", [])}
48
+ wanted = set(expect.get("any_symbol_names") or [])
49
+ ok = bool(names & wanted)
50
+ return ok, f"names={sorted(names)}"
51
+
52
+ if kind == "trace":
53
+ called_by = payload.get("result", {}).get("called_by") or []
54
+ suffixes = expect.get("called_by_source_suffix") or []
55
+ deco_targets = expect.get("decorated_by_target") or []
56
+ if deco_targets:
57
+ deco = payload.get("result", {}).get("decorated_by") or []
58
+ ok = any(
59
+ any(str(edge.get("target")) == t for t in deco_targets) for edge in deco
60
+ )
61
+ return ok, f"decorated_by={deco}"
62
+ ok = any(
63
+ any(str(edge.get("source", "")).endswith(suf) for suf in suffixes)
64
+ for edge in called_by
65
+ )
66
+ return ok, f"called_by={called_by}"
67
+
68
+ if kind == "impact":
69
+ direct = payload.get("result", {}).get("direct") or []
70
+ suffixes = expect.get("direct_source_suffix") or []
71
+ ok = any(
72
+ any(str(edge.get("source", "")).endswith(suf) for suf in suffixes)
73
+ for edge in direct
74
+ )
75
+ return ok, f"direct={direct}"
76
+
77
+ if kind == "related_tests":
78
+ tests = payload.get("result", {}).get("tests") or []
79
+ needles = expect.get("test_file_contains") or []
80
+ ok = any(
81
+ any(n in str(t.get("file_path")) for n in needles) for t in tests
82
+ )
83
+ return ok, f"tests={tests}"
84
+
85
+ if kind == "evidence":
86
+ count = len(payload.get("evidence") or [])
87
+ need = int(expect.get("min_evidence") or 1)
88
+ return count >= need, f"evidence_count={count}"
89
+
90
+ if kind == "overview":
91
+ result = payload.get("result") or {}
92
+ ok = (
93
+ int(result.get("symbols") or 0) >= int(expect.get("min_symbols") or 0)
94
+ and int(result.get("files") or 0) >= int(expect.get("min_files") or 0)
95
+ )
96
+ return ok, f"files={result.get('files')} symbols={result.get('symbols')}"
97
+
98
+ if kind == "overview_relations":
99
+ sample = payload.get("result", {}).get("relation_sample") or []
100
+ needles = expect.get("relation_target_contains") or []
101
+ ok = any(any(n in str(e.get("target")) for n in needles) for e in sample)
102
+ return ok, f"relations={sample}"
103
+
104
+ if kind == "plan_verify":
105
+ steps = payload.get("result", {}).get("steps") or []
106
+ kinds = {s.get("kind") for s in steps}
107
+ wanted = set(expect.get("step_kinds") or [])
108
+ ok = wanted.issubset(kinds)
109
+ return ok, f"step_kinds={sorted(kinds)}"
110
+
111
+ if kind == "verify_blocked":
112
+ state = payload.get("result", {}).get("state")
113
+ ok = state == expect.get("state")
114
+ return ok, f"state={state}"
115
+
116
+ if kind == "explain_code":
117
+ snippet = payload.get("result", {}).get("snippet") or []
118
+ need = int(expect.get("min_snippet_lines") or 1)
119
+ ok = len(snippet) >= need
120
+ return ok, f"snippet_lines={len(snippet)}"
121
+
122
+ if kind == "mcp_tools":
123
+ names = set(payload.get("result", {}).get("tools") or [])
124
+ wanted = set(expect.get("tool_names") or [])
125
+ ok = wanted.issubset(names)
126
+ return ok, f"tools={sorted(names)}"
127
+
128
+ if kind == "import_graph":
129
+ broken = int(payload.get("result", {}).get("broken_count") or 0)
130
+ max_broken = int(expect.get("max_broken") or 0)
131
+ ok = broken <= max_broken
132
+ return ok, f"broken_count={broken}"
133
+
134
+ if kind == "export_bundle":
135
+ bundle = payload.get("result", {}).get("bundle") or {}
136
+ wanted = set(expect.get("bundle_keys") or [])
137
+ ok = wanted.issubset(set(bundle.keys()))
138
+ return ok, f"bundle_keys={sorted(bundle.keys())}"
139
+
140
+ if kind == "index_health":
141
+ issues = payload.get("result", {}).get("issues") or []
142
+ max_issues = int(expect.get("max_issues") or 0)
143
+ ok = len(issues) <= max_issues
144
+ return ok, f"issues={issues}"
145
+
146
+ if kind == "report":
147
+ md = payload.get("result", {}).get("markdown") or ""
148
+ needles = expect.get("markdown_contains") or []
149
+ ok = all(n in md for n in needles)
150
+ return ok, f"markdown_len={len(md)}"
151
+
152
+ if kind == "status":
153
+ result = payload.get("result") or {}
154
+ wanted = set(expect.get("has_keys") or [])
155
+ ok = wanted.issubset(set(result.keys()))
156
+ return ok, f"keys={sorted(result.keys())}"
157
+
158
+ if kind == "snapshots":
159
+ count = int(payload.get("result", {}).get("count") or 0)
160
+ need = int(expect.get("min_count") or 1)
161
+ ok = count >= need
162
+ return ok, f"count={count}"
163
+
164
+ if kind == "graph":
165
+ graph = payload.get("result", {}).get("graph") or ""
166
+ needles = expect.get("graph_contains") or []
167
+ min_edges = int(expect.get("min_edges") or 0)
168
+ edges = int(payload.get("result", {}).get("edge_count") or 0)
169
+ ok = all(n in graph for n in needles) and edges >= min_edges
170
+ return ok, f"edges={edges}"
171
+
172
+ if kind == "capabilities":
173
+ tools = payload.get("result", {}).get("mcp_tools") or []
174
+ experimental = payload.get("result", {}).get("mcp_experimental") or []
175
+ need = int(expect.get("min_mcp_tools") or 1)
176
+ ok = len(tools) >= need
177
+ exact = expect.get("exact_mcp_tools")
178
+ if exact:
179
+ ok = ok and set(tools) == set(exact)
180
+ min_exp = expect.get("min_mcp_experimental")
181
+ if min_exp is not None:
182
+ ok = ok and len(experimental) >= int(min_exp)
183
+ return ok, f"mcp_tools={len(tools)} experimental={len(experimental)}"
184
+
185
+ if kind == "studio":
186
+ result = payload.get("result") or {}
187
+ ok = True
188
+ if expect.get("has_graph") is True:
189
+ ok = ok and bool(result.get("has_graph"))
190
+ if expect.get("has_report") is True:
191
+ ok = ok and bool(result.get("has_report"))
192
+ return ok, f"studio={result}"
193
+
194
+ if kind == "refresh_evidence":
195
+ result = payload.get("result") or {}
196
+ checked = int(result.get("checked") or 0)
197
+ need = int(expect.get("min_checked") or 0)
198
+ ok = payload.get("status") in {"success", "partial"} and checked >= need
199
+ return ok, f"checked={checked} stale={result.get('marked_stale')}"
200
+
201
+ if kind == "ingest_traces":
202
+ result = payload.get("result") or {}
203
+ accepted = int(result.get("accepted") or 0)
204
+ need = int(expect.get("min_accepted") or 1)
205
+ ok = payload.get("status") == "success" and accepted >= need
206
+ return ok, f"accepted={accepted} added={result.get('relations_added')}"
207
+
208
+ if kind == "baselines":
209
+ result = payload.get("result") or {}
210
+ deltas = result.get("deltas") or {}
211
+ security = result.get("security") or {}
212
+ ok = (
213
+ int(deltas.get("typed_symbols_over_grep") or 0)
214
+ >= int(expect.get("min_typed_symbols") or 1)
215
+ and int(deltas.get("relation_edges_over_grep") or 0)
216
+ >= int(expect.get("min_relation_edges") or 1)
217
+ )
218
+ if expect.get("verify_blocked") is True:
219
+ ok = ok and bool(security.get("verify_blocked_without_auth"))
220
+ return ok, f"deltas={deltas} security={security}"
221
+
222
+ if kind in {"go_search", "fixture_search", "js_search", "java_search"}:
223
+ names = {m.get("name") for m in payload.get("result", {}).get("matches", [])}
224
+ wanted = set(expect.get("any_symbol_names") or [])
225
+ ok = bool(names & wanted)
226
+ return ok, f"names={sorted(names)}"
227
+
228
+ if kind == "lsp":
229
+ result = payload.get("result") or {}
230
+ if expect.get("allow_unavailable") and not result.get("available"):
231
+ return True, "lsp_unavailable_allowed"
232
+ ok = bool(result.get("definitions") or result.get("references"))
233
+ return ok, f"lsp={result}"
234
+
235
+ if kind == "overview_min":
236
+ result = payload.get("result") or {}
237
+ ok = (
238
+ int(result.get("files") or 0) >= int(expect.get("min_files") or 0)
239
+ and int(result.get("symbols") or 0) >= int(expect.get("min_symbols") or 0)
240
+ )
241
+ return ok, f"files={result.get('files')} symbols={result.get('symbols')}"
242
+
243
+ return False, f"unknown kind={kind}"
244
+
245
+
246
+ def run_benchmark(
247
+ repo_root: Path,
248
+ tasks_path: Path,
249
+ db_path: Path,
250
+ ) -> dict[str, Any]:
251
+ spec = _load_tasks(tasks_path)
252
+ fixture_rel = spec.get("fixture") or "fixtures/mini_py"
253
+ fixture = (repo_root / fixture_rel).resolve()
254
+ index_repository(fixture, db_path, incremental=False)
255
+
256
+ results: list[TaskResult] = []
257
+ for task in spec.get("tasks") or []:
258
+ kind = task["kind"]
259
+ query = task.get("query") or ""
260
+ expect = task.get("expect") or {}
261
+ if kind == "search":
262
+ payload = repo_search(db_path, query)
263
+ elif kind == "trace":
264
+ payload = trace_symbol(db_path, query)
265
+ elif kind == "impact":
266
+ payload = analyze_impact(db_path, query)
267
+ elif kind == "related_tests":
268
+ payload = find_related_tests(db_path, query)
269
+ elif kind == "evidence":
270
+ payload = get_evidence(db_path, query)
271
+ elif kind in {"overview", "overview_relations"}:
272
+ payload = repo_overview(db_path)
273
+ elif kind == "plan_verify":
274
+ payload = create_verification_plan(db_path, target=query or None)
275
+ elif kind == "verify_blocked":
276
+ payload = run_verification(db_path, execute=False, target=query or None)
277
+ elif kind == "explain_code":
278
+ payload = explain_code(db_path, query)
279
+ elif kind == "mcp_tools":
280
+ from witnis.mcp_server import build_server
281
+
282
+ payload = {
283
+ "result": {
284
+ "tools": [t.name for t in build_server()._tool_manager.list_tools()]
285
+ }
286
+ }
287
+ elif kind == "import_graph":
288
+ payload = check_import_graph(db_path)
289
+ elif kind == "export_bundle":
290
+ payload = export_evidence_bundle(db_path, target=query or None)
291
+ elif kind == "index_health":
292
+ payload = check_index_health(db_path)
293
+ elif kind == "report":
294
+ payload = render_evidence_report(db_path, target=query or None)
295
+ elif kind == "status":
296
+ payload = repo_status(db_path)
297
+ elif kind == "snapshots":
298
+ payload = list_index_snapshots(db_path)
299
+ elif kind == "graph":
300
+ payload = export_relation_graph(db_path, query or "login", fmt="mermaid")
301
+ elif kind == "capabilities":
302
+ payload = list_capabilities()
303
+ elif kind == "studio":
304
+ out = db_path.parent / "bench-studio.html"
305
+ payload = render_studio(db_path, out=out, target=query or "login")
306
+ elif kind == "refresh_evidence":
307
+ payload = refresh_evidence(db_path)
308
+ elif kind == "ingest_traces":
309
+ rel = query or "fixtures/mini_py/sample_trace.jsonl"
310
+ payload = ingest_runtime_traces(db_path, (repo_root / rel).resolve())
311
+ elif kind == "baselines":
312
+ payload = run_baseline_compare(
313
+ (repo_root / (spec.get("fixture") or "fixtures/mini_py")).resolve(),
314
+ db_path,
315
+ query=query or "login",
316
+ )
317
+ elif kind == "go_search":
318
+ go_db = db_path.parent / "bench-go.sqlite"
319
+ index_repository(
320
+ (repo_root / "fixtures" / "mini_go").resolve(),
321
+ go_db,
322
+ incremental=False,
323
+ )
324
+ payload = repo_search(go_db, query or "Login")
325
+ elif kind in {"fixture_search", "js_search", "java_search"}:
326
+ fixture_rel = (
327
+ task.get("fixture")
328
+ or expect.get("fixture")
329
+ or {
330
+ "js_search": "fixtures/mini_js",
331
+ "java_search": "fixtures/mini_java",
332
+ "fixture_search": "fixtures/public_svc",
333
+ }.get(kind, "fixtures/public_svc")
334
+ )
335
+ alt_db = db_path.parent / f"bench-{kind}.sqlite"
336
+ index_repository(
337
+ (repo_root / fixture_rel).resolve(),
338
+ alt_db,
339
+ incremental=False,
340
+ )
341
+ payload = repo_search(alt_db, query)
342
+ elif kind == "overview_min":
343
+ fixture_rel = task.get("fixture") or "fixtures/public_svc"
344
+ alt_db = db_path.parent / "bench-overview.sqlite"
345
+ index_repository(
346
+ (repo_root / fixture_rel).resolve(),
347
+ alt_db,
348
+ incremental=False,
349
+ )
350
+ payload = repo_overview(alt_db)
351
+ elif kind == "lsp":
352
+ payload = query_lsp(db_path, query or "AuthService")
353
+ else:
354
+ results.append(TaskResult(task["id"], False, f"unsupported kind {kind}"))
355
+ continue
356
+ ok, detail = _check_expect(kind, payload, expect)
357
+ results.append(TaskResult(task["id"], ok, detail))
358
+
359
+ passed = sum(1 for r in results if r.passed)
360
+ total = len(results)
361
+ return {
362
+ "fixture": str(fixture),
363
+ "passed": passed,
364
+ "total": total,
365
+ "score": (passed / total) if total else 0.0,
366
+ "results": [{"id": r.id, "passed": r.passed, "detail": r.detail} for r in results],
367
+ }