wikifier 4.6.3__tar.gz → 4.6.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wikifier-4.6.3/wikifier.egg-info → wikifier-4.6.4}/PKG-INFO +4 -3
- {wikifier-4.6.3 → wikifier-4.6.4}/README.md +3 -2
- {wikifier-4.6.3 → wikifier-4.6.4}/skills/run.md +2 -1
- wikifier-4.6.4/tests/test_cache_store.py +193 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/__init__.py +1 -1
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/agent_loop.py +38 -3
- wikifier-4.6.4/wikifier/cache_store.py +348 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/cli.py +141 -55
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/import_cache.py +110 -38
- wikifier-4.6.4/wikifier/parsers/go_lang.py +164 -0
- {wikifier-4.6.3 → wikifier-4.6.4/wikifier.egg-info}/PKG-INFO +4 -3
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier.egg-info/SOURCES.txt +3 -0
- wikifier-4.6.3/wikifier/parsers/go_lang.py +0 -87
- {wikifier-4.6.3 → wikifier-4.6.4}/CONTRIBUTING.md +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/LICENSE +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/MANIFEST.in +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/diagnostics.html +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/docs/spec.md +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/index.html +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/pyproject.toml +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/setup.cfg +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_agent_loop.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_agent_scale.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_barrel_invalidation.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_gap_closure.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_health.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_import_cache.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_multi_lang_parsers.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_parsers.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/tests/test_selftest_wrappers.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/__main__.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/contracts.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/daemon.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/diagnostics.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/health.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/index.html +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/library.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/locking.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/mcp/__init__.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/mcp/server.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/__init__.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/_edge.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/bree.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/c_cpp.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/cdia.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/csharp.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/java.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/javascript.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/python.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/parsers/rust.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/project_root.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/resolution.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/scripts/wikifier.bat +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/scripts/wikifier.ps1 +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/scripts/wikifier.sh +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier/serve.py +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier.egg-info/dependency_links.txt +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier.egg-info/entry_points.txt +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier.egg-info/requires.txt +0 -0
- {wikifier-4.6.3 → wikifier-4.6.4}/wikifier.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wikifier
|
|
3
|
-
Version: 4.6.
|
|
3
|
+
Version: 4.6.4
|
|
4
4
|
Summary: Zero-dependency agent-to-agent codebase wiki for LLMs and AI agents. Autonomously maintained via record-change and mark-green for token-efficient lookup of files, dependencies, health, and summaries — across tiny scripts to 50k+ monorepos. Optional MCP server with rich tools for agents.
|
|
5
5
|
Author-email: Aron Amos <aron@example.com>
|
|
6
6
|
Maintainer: Aron Amos
|
|
@@ -109,8 +109,9 @@ Advanced intel as needed: `get_dependencies`, `get_dependents`, `get_cycles`, ba
|
|
|
109
109
|
|
|
110
110
|
- **Import analysis** — Python, JS/TS (ESM/CJS, barrels), Rust (`use`/`mod` + best-effort `crate::` paths), Go, C/C++ includes, C# usings; per-edge confidence; barrel expansion for TS/JS
|
|
111
111
|
- **Incremental pipeline** — pure-Python `update-maps`: dirty parse → import cache → reverse deps → cycles → `library.md`
|
|
112
|
-
- **Warm agent maps (4.6.3+)** — zero-dirty fast path skips full graph/`library.md` rewrite
|
|
113
|
-
- **
|
|
112
|
+
- **Warm agent maps (4.6.3+ / 4.6.4+)** — zero-dirty fast path skips full graph/`library.md` rewrite; **stdlib SQLite** import cache (dual-read legacy JSON) so warm dirty scans use mtime/hash index not multi‑MB pair JSON; content-hash dirty; `--directory=` scopes walks
|
|
113
|
+
- **Partial-map honesty** — `map_coverage` on `update_maps` / bootstrap (`complete`, `files_remaining_dirty`, `acs_version`, `cache_backend`); budgeted runs are not “done”
|
|
114
|
+
- **Selective agent work** — health + suggest bias to 🔴/actionable 🟡 only; **ACS v1.3** `reason_code` / `agent_signal`; prefer `actionable_low_conf_edges` + reason codes — **never** raw `low_conf_edges` alone
|
|
114
115
|
- **Scale** — reverse index + barrel invalidation so one edit doesn’t re-scan the monorepo
|
|
115
116
|
- **MCP tools** — optional server for Claude, Cursor, Cline, and other MCP clients
|
|
116
117
|
- **Zero core dependencies** — stdlib only; forks can add their own stack on top
|
|
@@ -77,8 +77,9 @@ Advanced intel as needed: `get_dependencies`, `get_dependents`, `get_cycles`, ba
|
|
|
77
77
|
|
|
78
78
|
- **Import analysis** — Python, JS/TS (ESM/CJS, barrels), Rust (`use`/`mod` + best-effort `crate::` paths), Go, C/C++ includes, C# usings; per-edge confidence; barrel expansion for TS/JS
|
|
79
79
|
- **Incremental pipeline** — pure-Python `update-maps`: dirty parse → import cache → reverse deps → cycles → `library.md`
|
|
80
|
-
- **Warm agent maps (4.6.3+)** — zero-dirty fast path skips full graph/`library.md` rewrite
|
|
81
|
-
- **
|
|
80
|
+
- **Warm agent maps (4.6.3+ / 4.6.4+)** — zero-dirty fast path skips full graph/`library.md` rewrite; **stdlib SQLite** import cache (dual-read legacy JSON) so warm dirty scans use mtime/hash index not multi‑MB pair JSON; content-hash dirty; `--directory=` scopes walks
|
|
81
|
+
- **Partial-map honesty** — `map_coverage` on `update_maps` / bootstrap (`complete`, `files_remaining_dirty`, `acs_version`, `cache_backend`); budgeted runs are not “done”
|
|
82
|
+
- **Selective agent work** — health + suggest bias to 🔴/actionable 🟡 only; **ACS v1.3** `reason_code` / `agent_signal`; prefer `actionable_low_conf_edges` + reason codes — **never** raw `low_conf_edges` alone
|
|
82
83
|
- **Scale** — reverse index + barrel invalidation so one edit doesn’t re-scan the monorepo
|
|
83
84
|
- **MCP tools** — optional server for Claude, Cursor, Cline, and other MCP clients
|
|
84
85
|
- **Zero core dependencies** — stdlib only; forks can add their own stack on top
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
- File Tree + `wikifier serve`; MCP status/attention use library + emoji (not `[GREEN]` tags).
|
|
12
12
|
- **First-run:** `init` → `update-maps` → `health --summary` → `suggest_next_actions`. Map is automatic; wiki *prose* is agent-filled over time.
|
|
13
13
|
- **Map-first ≠ wiki-done:** `health_score` **Map Ready** + many 🟡 *Initial stubs* means structural coverage only. Do **not** bulk-wiki stubs. Wiki a file when you edit it, then `mark-green`. Prefer `actionable_yellow` / 🔴 over raw yellow counts.
|
|
14
|
-
- **Steady-state selective work:** only 🔴 and *actionable* 🟡 (never re-wiki 🟢; never re-wiki the whole stub set). ACS
|
|
14
|
+
- **Steady-state selective work:** only 🔴 and *actionable* 🟡 (never re-wiki 🟢; never re-wiki the whole stub set). **ACS v1.3:** use `actionable_low_conf_edges` + `reason_code_counts` / `agent_signal` (`skip`|`investigate`). Do **not** thrash on raw `low_conf_edges` (includes external/bare scores ~0.48). Prefer `map_coverage.complete` / `files_remaining_dirty` after budgeted `update-maps` — `success: true` ≠ map complete.
|
|
15
|
+
- **Warm maps:** import cache prefers stdlib SQLite (`.wikifier_staging/import_cache.sqlite`); zero-dirty path uses mtime index + meta, not full pair payload.
|
|
15
16
|
- **Dual scope:** `monitored_paths` = check-changes/health scope (keep lean). `update-maps --directory=` / `--max-files=` = map walk. Never set `project_root` to a multi-repo parent (e.g. `cloned_sample_projects`).
|
|
16
17
|
- **CLI target:** `wikifier --target /abs/project health --summary` (flags stripped before the command). Or `WIKIFIER_PROJECT_ROOT=…`.
|
|
17
18
|
- **Long-horizon:** `wikifier autonomous-status` (or `readiness`) before unattended runs; daemon writes `.wikifier_staging/daemon_heartbeat.json`. Soak ≥72h is still evidence work, not automatic.
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
"""SQLite cache store + map_coverage + warm path (shipped APIs, zero-dep)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
import unittest
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from tests._base import TempProjectTestCase
|
|
11
|
+
|
|
12
|
+
from wikifier import cli
|
|
13
|
+
from wikifier import import_cache as ic
|
|
14
|
+
from wikifier import cache_store as cs
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TestSqliteCacheRoundTrip(TempProjectTestCase):
|
|
18
|
+
def test_save_load_sqlite_primary(self):
|
|
19
|
+
self.write("a.py", "import os\n")
|
|
20
|
+
r = cli.update_maps(project_root=self.root, full=True)
|
|
21
|
+
self.assertTrue(r.get("success"), r)
|
|
22
|
+
self.assertTrue(cs.has_sqlite(self.root), "sqlite should exist after save")
|
|
23
|
+
cache = ic.load_cache(self.root)
|
|
24
|
+
self.assertIn("a.py", cache)
|
|
25
|
+
self.assertIsInstance(cache["a.py"].get("mtime"), int)
|
|
26
|
+
# light index works without requiring pairs field presence
|
|
27
|
+
idx = ic.load_mtime_index(self.root)
|
|
28
|
+
self.assertIn("a.py", idx)
|
|
29
|
+
self.assertEqual(idx["a.py"]["mtime"], cache["a.py"]["mtime"])
|
|
30
|
+
|
|
31
|
+
def test_legacy_json_dual_read_then_migrate(self):
|
|
32
|
+
# Plant legacy JSON only (no sqlite)
|
|
33
|
+
staging = self.root / ".wikifier_staging"
|
|
34
|
+
staging.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
import json
|
|
36
|
+
legacy = {
|
|
37
|
+
"x.py": {
|
|
38
|
+
"mtime": 1,
|
|
39
|
+
"content_hash": "sha256:abc",
|
|
40
|
+
"imports": ["os"],
|
|
41
|
+
"resolved": [],
|
|
42
|
+
"resolved_pairs": [{"raw": "os", "resolved": "", "confidence_score": 0.48}],
|
|
43
|
+
},
|
|
44
|
+
"_acs_summary": {
|
|
45
|
+
"acs_version": "1.2",
|
|
46
|
+
"total_scored_edges": 1,
|
|
47
|
+
"actionable_low_conf_edges": 0,
|
|
48
|
+
"low_conf_edges": 1,
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
(staging / "import_cache.json").write_text(
|
|
52
|
+
json.dumps(legacy), encoding="utf-8"
|
|
53
|
+
)
|
|
54
|
+
loaded = ic.load_cache(self.root)
|
|
55
|
+
self.assertIn("x.py", loaded)
|
|
56
|
+
# update_maps migrates
|
|
57
|
+
self.write("x.py", "import os\n")
|
|
58
|
+
r = cli.update_maps(project_root=self.root, full=False)
|
|
59
|
+
self.assertTrue(r.get("success"), r)
|
|
60
|
+
self.assertTrue(cs.has_sqlite(self.root))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class TestWarmPathNoFullJsonTax(TempProjectTestCase):
|
|
64
|
+
def test_zero_dirty_reports_sqlite_backend_and_coverage(self):
|
|
65
|
+
self.write("m.py", "print(1)\n")
|
|
66
|
+
r1 = cli.update_maps(project_root=self.root, full=True)
|
|
67
|
+
self.assertTrue(r1.get("success"), r1)
|
|
68
|
+
r2 = cli.update_maps(project_root=self.root, full=False)
|
|
69
|
+
self.assertTrue(r2.get("success"), r2)
|
|
70
|
+
self.assertTrue(r2.get("zero_dirty_fast_path"), r2)
|
|
71
|
+
self.assertIn(r2.get("cache_backend"), ("sqlite", "sqlite+json", "json"))
|
|
72
|
+
cov = r2.get("map_coverage") or {}
|
|
73
|
+
self.assertTrue(cov.get("complete"), cov)
|
|
74
|
+
self.assertIn("files_remaining_dirty", cov)
|
|
75
|
+
self.assertEqual(int(cov["files_remaining_dirty"]), 0)
|
|
76
|
+
|
|
77
|
+
def test_zero_dirty_skips_full_payload_load_cache_dict(self):
|
|
78
|
+
"""Shipped warm path must not deserialize full file payloads on 0-dirty.
|
|
79
|
+
|
|
80
|
+
Criterion: after SQLite is warm, zero-dirty update_maps must not call
|
|
81
|
+
cache_store.load_cache_dict (full pairs) or ic.load_cache (legacy full).
|
|
82
|
+
Light index + meta only.
|
|
83
|
+
"""
|
|
84
|
+
self.write("warm.py", "print(0)\n")
|
|
85
|
+
r1 = cli.update_maps(project_root=self.root, full=True)
|
|
86
|
+
self.assertTrue(r1.get("success"), r1)
|
|
87
|
+
self.assertTrue(cs.has_sqlite(self.root))
|
|
88
|
+
# Second run seeds zero-dirty + sqlite
|
|
89
|
+
r_seed = cli.update_maps(project_root=self.root, full=False)
|
|
90
|
+
self.assertTrue(r_seed.get("zero_dirty_fast_path"), r_seed)
|
|
91
|
+
|
|
92
|
+
full_loads = {"load_cache_dict": 0, "load_cache": 0}
|
|
93
|
+
real_lcd = cs.load_cache_dict
|
|
94
|
+
real_lc = ic.load_cache
|
|
95
|
+
|
|
96
|
+
def counting_lcd(root):
|
|
97
|
+
full_loads["load_cache_dict"] += 1
|
|
98
|
+
return real_lcd(root)
|
|
99
|
+
|
|
100
|
+
def counting_lc(root):
|
|
101
|
+
full_loads["load_cache"] += 1
|
|
102
|
+
return real_lc(root)
|
|
103
|
+
|
|
104
|
+
# Patch at the modules the shipped path imports from
|
|
105
|
+
import wikifier.cache_store as cs_mod
|
|
106
|
+
import wikifier.import_cache as ic_mod
|
|
107
|
+
orig_cs = cs_mod.load_cache_dict
|
|
108
|
+
orig_ic = ic_mod.load_cache
|
|
109
|
+
cs_mod.load_cache_dict = counting_lcd # type: ignore
|
|
110
|
+
ic_mod.load_cache = counting_lc # type: ignore
|
|
111
|
+
try:
|
|
112
|
+
r2 = cli.update_maps(project_root=self.root, full=False)
|
|
113
|
+
finally:
|
|
114
|
+
cs_mod.load_cache_dict = orig_cs # type: ignore
|
|
115
|
+
ic_mod.load_cache = orig_ic # type: ignore
|
|
116
|
+
|
|
117
|
+
self.assertTrue(r2.get("success"), r2)
|
|
118
|
+
self.assertTrue(r2.get("zero_dirty_fast_path"), r2)
|
|
119
|
+
self.assertEqual(
|
|
120
|
+
full_loads["load_cache_dict"],
|
|
121
|
+
0,
|
|
122
|
+
f"warm 0-dirty must not call load_cache_dict; counts={full_loads}",
|
|
123
|
+
)
|
|
124
|
+
self.assertEqual(
|
|
125
|
+
full_loads["load_cache"],
|
|
126
|
+
0,
|
|
127
|
+
f"warm 0-dirty must not call load_cache; counts={full_loads}",
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
def test_max_files_coverage_incomplete(self):
|
|
131
|
+
for i in range(6):
|
|
132
|
+
self.write(f"f{i}.py", f"v={i}\n")
|
|
133
|
+
r = cli.update_maps(project_root=self.root, full=True, max_files=2)
|
|
134
|
+
self.assertTrue(r.get("success"), r)
|
|
135
|
+
cov = r.get("map_coverage") or {}
|
|
136
|
+
self.assertFalse(cov.get("complete"), cov)
|
|
137
|
+
self.assertEqual(int(cov.get("files_skipped") or 0), 4)
|
|
138
|
+
self.assertGreaterEqual(int(cov.get("files_remaining_dirty") or 0), 4)
|
|
139
|
+
self.assertIn("agent_note", cov)
|
|
140
|
+
|
|
141
|
+
def test_mtime_index_skips_content_stable(self):
|
|
142
|
+
p = self.write("s.py", "x=1\n")
|
|
143
|
+
cli.update_maps(project_root=self.root, full=True)
|
|
144
|
+
cache = ic.load_cache(self.root)
|
|
145
|
+
ent = cache["s.py"]
|
|
146
|
+
ent["mtime"] = 1
|
|
147
|
+
ic.save_cache(self.root, cache)
|
|
148
|
+
time.sleep(0.05)
|
|
149
|
+
os.utime(p, None)
|
|
150
|
+
updates = []
|
|
151
|
+
need = ic.compute_files_needing_reparse(
|
|
152
|
+
self.root, [p], content_stable_mtime_updates=updates
|
|
153
|
+
)
|
|
154
|
+
# content_hash should make this stable once hash stored
|
|
155
|
+
if ent.get("content_hash"):
|
|
156
|
+
self.assertEqual(need, [])
|
|
157
|
+
self.assertGreaterEqual(len(updates), 1)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class TestBootstrapCoverageAndAcsGuidance(TempProjectTestCase):
|
|
161
|
+
def test_bootstrap_surfaces_acs_guidance(self):
|
|
162
|
+
self.write("h.py", "import os\n")
|
|
163
|
+
cli.update_maps(project_root=self.root, full=True)
|
|
164
|
+
boot = cli.session_bootstrap(project_root=self.root)
|
|
165
|
+
self.assertTrue(boot.get("success"), boot)
|
|
166
|
+
self.assertIn("acs_guidance", boot)
|
|
167
|
+
self.assertIn("actionable_low_conf_edges", boot["acs_guidance"])
|
|
168
|
+
acs = boot.get("acs") or {}
|
|
169
|
+
self.assertIn("acs_version", acs)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
class TestGoModResolve(TempProjectTestCase):
|
|
173
|
+
def test_same_module_import_resolves(self):
|
|
174
|
+
from wikifier.parsers import go_lang as go
|
|
175
|
+
|
|
176
|
+
self.write(
|
|
177
|
+
"go.mod",
|
|
178
|
+
"module example.com/demo\n\ngo 1.21\n",
|
|
179
|
+
)
|
|
180
|
+
self.write("pkg/util/util.go", "package util\nfunc X() {}\n")
|
|
181
|
+
main = self.write(
|
|
182
|
+
"cmd/main.go",
|
|
183
|
+
'package main\nimport "example.com/demo/pkg/util"\n',
|
|
184
|
+
)
|
|
185
|
+
edges = go.parse_go_imports(str(main))
|
|
186
|
+
self.assertTrue(edges)
|
|
187
|
+
resolved = [e for e in edges if e.get("resolved_path")]
|
|
188
|
+
self.assertTrue(resolved, msg=f"edges={edges}")
|
|
189
|
+
self.assertIn("util", resolved[0]["resolved_path"])
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
if __name__ == "__main__":
|
|
193
|
+
unittest.main()
|
|
@@ -548,8 +548,33 @@ def session_bootstrap(
|
|
|
548
548
|
acs_actionable = 0
|
|
549
549
|
try:
|
|
550
550
|
from . import import_cache as ic
|
|
551
|
-
|
|
552
|
-
|
|
551
|
+
try:
|
|
552
|
+
from . import cache_store as cs
|
|
553
|
+
except Exception:
|
|
554
|
+
cs = None # type: ignore
|
|
555
|
+
# Prefer light meta read (SQLite) over full multi-MB pair deserialize
|
|
556
|
+
acs: Dict[str, Any] = {}
|
|
557
|
+
cyc: Dict[str, Any] = {}
|
|
558
|
+
if cs is not None:
|
|
559
|
+
meta = cs.load_meta(root, keys=("_acs_summary", "_cycles", "_map_coverage"))
|
|
560
|
+
acs = meta.get("_acs_summary") if isinstance(meta.get("_acs_summary"), dict) else {}
|
|
561
|
+
cyc = meta.get("_cycles") if isinstance(meta.get("_cycles"), dict) else {}
|
|
562
|
+
cov = meta.get("_map_coverage") if isinstance(meta.get("_map_coverage"), dict) else {}
|
|
563
|
+
if cov:
|
|
564
|
+
out["map_coverage"] = cov
|
|
565
|
+
out["cache_backend"] = cs.backend_name(root)
|
|
566
|
+
needs_full = (
|
|
567
|
+
not acs
|
|
568
|
+
or str(acs.get("acs_version") or "") < "1.3"
|
|
569
|
+
or "reason_code_counts" not in acs
|
|
570
|
+
or "actionable_low_conf_edges" not in acs
|
|
571
|
+
)
|
|
572
|
+
if needs_full:
|
|
573
|
+
cache = ic.load_cache(root) or {}
|
|
574
|
+
acs = ic.ensure_acs_summary_persisted(cache, root) or {}
|
|
575
|
+
cyc = cache.get("_cycles") or {}
|
|
576
|
+
if cs is not None and not out.get("cache_backend"):
|
|
577
|
+
out["cache_backend"] = cs.backend_name(root)
|
|
553
578
|
acs_actionable = int(acs.get("actionable_low_conf_edges") or 0)
|
|
554
579
|
out["acs"] = {
|
|
555
580
|
"acs_version": acs.get("acs_version"),
|
|
@@ -557,12 +582,22 @@ def session_bootstrap(
|
|
|
557
582
|
"low_conf_edges": acs.get("low_conf_edges"),
|
|
558
583
|
"dynamic_literal_noise_edges": acs.get("dynamic_literal_noise_edges"),
|
|
559
584
|
"avg_confidence": acs.get("avg_confidence"),
|
|
585
|
+
"reason_code_counts": acs.get("reason_code_counts"),
|
|
586
|
+
"agent_signal_counts": acs.get("agent_signal_counts"),
|
|
560
587
|
}
|
|
561
|
-
|
|
588
|
+
# Agents must not thrash on raw low_conf_edges (ACS v1.3 scores more edges)
|
|
589
|
+
out["acs_guidance"] = (
|
|
590
|
+
"Prefer actionable_low_conf_edges + reason_code_counts "
|
|
591
|
+
"(skip external_or_bare/dynamic_literal); do not use low_conf_edges alone as a work queue."
|
|
592
|
+
)
|
|
562
593
|
out["cycles"] = {
|
|
563
594
|
"cyclic_scc_count": (cyc.get("stats") or {}).get("cyclic_scc_count"),
|
|
564
595
|
"all_cycle_files_sample": (cyc.get("all_cycle_files") or [])[:10],
|
|
565
596
|
}
|
|
597
|
+
if "map_coverage" not in out and cs is not None:
|
|
598
|
+
snap = cs.get_map_coverage_from_meta(root)
|
|
599
|
+
if snap.get("map_coverage"):
|
|
600
|
+
out["map_coverage"] = snap["map_coverage"]
|
|
566
601
|
except Exception as e:
|
|
567
602
|
out["acs_error"] = str(e)
|
|
568
603
|
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
"""Stdlib SQLite import-cache store (agent warm-path; zero new deps).
|
|
2
|
+
|
|
3
|
+
AGENT MAP:
|
|
4
|
+
load_cache_dict / save_cache_dict — full cache dict (file entries + _meta keys)
|
|
5
|
+
load_mtime_index — light {rel: mtime, content_hash} for dirty
|
|
6
|
+
load_meta / save_meta_key — reserved _keys without loading pairs
|
|
7
|
+
update_file_index_rows — mtime/hash refresh without full rewrite
|
|
8
|
+
backend_name / has_sqlite
|
|
9
|
+
|
|
10
|
+
Dual-read: if ``import_cache.sqlite`` missing but legacy ``import_cache.json``
|
|
11
|
+
exists, load JSON and migrate on next save. Warm dirty detection should use
|
|
12
|
+
``load_mtime_index`` so agents avoid deserializing multi‑MB pair payloads.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import sqlite3
|
|
20
|
+
from contextlib import contextmanager
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple
|
|
23
|
+
|
|
24
|
+
STAGING = ".wikifier_staging"
|
|
25
|
+
SQLITE_NAME = "import_cache.sqlite"
|
|
26
|
+
JSON_NAME = "import_cache.json"
|
|
27
|
+
|
|
28
|
+
_SCHEMA = """
|
|
29
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
30
|
+
rel_path TEXT PRIMARY KEY NOT NULL,
|
|
31
|
+
mtime INTEGER NOT NULL DEFAULT 0,
|
|
32
|
+
content_hash TEXT,
|
|
33
|
+
payload TEXT NOT NULL DEFAULT '{}'
|
|
34
|
+
);
|
|
35
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
36
|
+
key TEXT PRIMARY KEY NOT NULL,
|
|
37
|
+
value TEXT NOT NULL
|
|
38
|
+
);
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def sqlite_path(root: Path) -> Path:
|
|
43
|
+
return Path(root) / STAGING / SQLITE_NAME
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def json_path(root: Path) -> Path:
|
|
47
|
+
return Path(root) / STAGING / JSON_NAME
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def has_sqlite(root: Path) -> bool:
|
|
51
|
+
try:
|
|
52
|
+
return sqlite_path(root).is_file() and sqlite_path(root).stat().st_size > 0
|
|
53
|
+
except Exception:
|
|
54
|
+
return False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def backend_name(root: Path) -> str:
|
|
58
|
+
if has_sqlite(root):
|
|
59
|
+
return "sqlite"
|
|
60
|
+
if json_path(root).is_file():
|
|
61
|
+
return "json"
|
|
62
|
+
return "none"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _connect(root: Path) -> sqlite3.Connection:
|
|
66
|
+
path = sqlite_path(root)
|
|
67
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
68
|
+
conn = sqlite3.connect(str(path), timeout=60.0)
|
|
69
|
+
try:
|
|
70
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
71
|
+
conn.execute("PRAGMA synchronous=NORMAL")
|
|
72
|
+
conn.executescript(_SCHEMA)
|
|
73
|
+
except Exception:
|
|
74
|
+
conn.close()
|
|
75
|
+
raise
|
|
76
|
+
return conn
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@contextmanager
|
|
80
|
+
def _db(root: Path) -> Iterator[sqlite3.Connection]:
|
|
81
|
+
conn = _connect(root)
|
|
82
|
+
try:
|
|
83
|
+
yield conn
|
|
84
|
+
conn.commit()
|
|
85
|
+
finally:
|
|
86
|
+
conn.close()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def load_mtime_index(root: Path) -> Dict[str, Dict[str, Any]]:
|
|
90
|
+
"""Light index for dirty detection: rel → {mtime, content_hash}.
|
|
91
|
+
|
|
92
|
+
Prefers SQLite. Falls back to scanning legacy JSON (expensive once).
|
|
93
|
+
"""
|
|
94
|
+
root = Path(root)
|
|
95
|
+
if has_sqlite(root):
|
|
96
|
+
out: Dict[str, Dict[str, Any]] = {}
|
|
97
|
+
try:
|
|
98
|
+
with _db(root) as conn:
|
|
99
|
+
for rel, mtime, chash in conn.execute(
|
|
100
|
+
"SELECT rel_path, mtime, content_hash FROM files"
|
|
101
|
+
):
|
|
102
|
+
out[str(rel)] = {
|
|
103
|
+
"mtime": int(mtime or 0),
|
|
104
|
+
"content_hash": chash,
|
|
105
|
+
}
|
|
106
|
+
return out
|
|
107
|
+
except Exception:
|
|
108
|
+
pass
|
|
109
|
+
# Legacy JSON fallback
|
|
110
|
+
jp = json_path(root)
|
|
111
|
+
if not jp.is_file():
|
|
112
|
+
return {}
|
|
113
|
+
try:
|
|
114
|
+
with open(jp, "r", encoding="utf-8") as f:
|
|
115
|
+
data = json.load(f)
|
|
116
|
+
except Exception:
|
|
117
|
+
return {}
|
|
118
|
+
out = {}
|
|
119
|
+
if not isinstance(data, dict):
|
|
120
|
+
return out
|
|
121
|
+
for k, v in data.items():
|
|
122
|
+
if not isinstance(k, str) or k.startswith("_") or not isinstance(v, dict):
|
|
123
|
+
continue
|
|
124
|
+
out[k] = {
|
|
125
|
+
"mtime": int(v.get("mtime", 0) or 0),
|
|
126
|
+
"content_hash": v.get("content_hash"),
|
|
127
|
+
}
|
|
128
|
+
return out
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def load_meta(root: Path, keys: Optional[Iterable[str]] = None) -> Dict[str, Any]:
|
|
132
|
+
"""Load reserved meta keys (``_acs_summary``, ``_cycles``, …) without file payloads."""
|
|
133
|
+
root = Path(root)
|
|
134
|
+
want = set(keys) if keys is not None else None
|
|
135
|
+
if has_sqlite(root):
|
|
136
|
+
try:
|
|
137
|
+
with _db(root) as conn:
|
|
138
|
+
rows = conn.execute("SELECT key, value FROM meta").fetchall()
|
|
139
|
+
out: Dict[str, Any] = {}
|
|
140
|
+
for k, raw in rows:
|
|
141
|
+
if want is not None and k not in want:
|
|
142
|
+
continue
|
|
143
|
+
try:
|
|
144
|
+
out[str(k)] = json.loads(raw)
|
|
145
|
+
except Exception:
|
|
146
|
+
out[str(k)] = raw
|
|
147
|
+
return out
|
|
148
|
+
except Exception:
|
|
149
|
+
pass
|
|
150
|
+
# JSON fallback — still full parse (legacy only)
|
|
151
|
+
jp = json_path(root)
|
|
152
|
+
if not jp.is_file():
|
|
153
|
+
return {}
|
|
154
|
+
try:
|
|
155
|
+
with open(jp, "r", encoding="utf-8") as f:
|
|
156
|
+
data = json.load(f)
|
|
157
|
+
except Exception:
|
|
158
|
+
return {}
|
|
159
|
+
if not isinstance(data, dict):
|
|
160
|
+
return {}
|
|
161
|
+
out = {}
|
|
162
|
+
for k, v in data.items():
|
|
163
|
+
if not isinstance(k, str) or not k.startswith("_"):
|
|
164
|
+
continue
|
|
165
|
+
if want is not None and k not in want:
|
|
166
|
+
continue
|
|
167
|
+
out[k] = v
|
|
168
|
+
return out
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def load_cache_dict(root: Path) -> Dict[str, Any]:
|
|
172
|
+
"""Full cache dict: file entries + reserved keys (same shape as historical JSON)."""
|
|
173
|
+
root = Path(root)
|
|
174
|
+
if has_sqlite(root):
|
|
175
|
+
try:
|
|
176
|
+
cache: Dict[str, Any] = {}
|
|
177
|
+
with _db(root) as conn:
|
|
178
|
+
for rel, mtime, chash, payload in conn.execute(
|
|
179
|
+
"SELECT rel_path, mtime, content_hash, payload FROM files"
|
|
180
|
+
):
|
|
181
|
+
try:
|
|
182
|
+
ent = json.loads(payload) if payload else {}
|
|
183
|
+
except Exception:
|
|
184
|
+
ent = {}
|
|
185
|
+
if not isinstance(ent, dict):
|
|
186
|
+
ent = {}
|
|
187
|
+
ent["mtime"] = int(mtime or 0)
|
|
188
|
+
if chash:
|
|
189
|
+
ent["content_hash"] = chash
|
|
190
|
+
cache[str(rel)] = ent
|
|
191
|
+
for k, raw in conn.execute("SELECT key, value FROM meta"):
|
|
192
|
+
try:
|
|
193
|
+
cache[str(k)] = json.loads(raw)
|
|
194
|
+
except Exception:
|
|
195
|
+
cache[str(k)] = raw
|
|
196
|
+
return cache
|
|
197
|
+
except Exception:
|
|
198
|
+
pass
|
|
199
|
+
jp = json_path(root)
|
|
200
|
+
if not jp.is_file():
|
|
201
|
+
return {}
|
|
202
|
+
try:
|
|
203
|
+
with open(jp, "r", encoding="utf-8") as f:
|
|
204
|
+
data = json.load(f)
|
|
205
|
+
return data if isinstance(data, dict) else {}
|
|
206
|
+
except Exception:
|
|
207
|
+
return {}
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def save_cache_dict(root: Path, cache: Dict[str, Any], write_json: Optional[bool] = None) -> str:
|
|
211
|
+
"""Persist cache to SQLite (primary). Optionally dual-write compact JSON.
|
|
212
|
+
|
|
213
|
+
``write_json``: None → dual-write if env WIKIFIER_CACHE_JSON=1 or file count ≤ 400
|
|
214
|
+
(keeps small projects greppable; avoids rewriting 20MB+ JSON on large monorepos).
|
|
215
|
+
|
|
216
|
+
Returns backend name written: ``sqlite`` or ``sqlite+json``.
|
|
217
|
+
"""
|
|
218
|
+
root = Path(root)
|
|
219
|
+
if not isinstance(cache, dict):
|
|
220
|
+
return "none"
|
|
221
|
+
# Barrel preservation: merge missing barrel keys from existing store
|
|
222
|
+
_BARREL = ("_barrel_resolutions", "_barrel_file_index")
|
|
223
|
+
if any(k not in cache for k in _BARREL):
|
|
224
|
+
try:
|
|
225
|
+
prev = load_cache_dict(root)
|
|
226
|
+
for k in _BARREL:
|
|
227
|
+
if k not in cache and prev.get(k):
|
|
228
|
+
cache[k] = prev[k]
|
|
229
|
+
except Exception:
|
|
230
|
+
pass
|
|
231
|
+
|
|
232
|
+
file_n = sum(
|
|
233
|
+
1
|
|
234
|
+
for k, v in cache.items()
|
|
235
|
+
if isinstance(k, str) and not k.startswith("_") and isinstance(v, dict)
|
|
236
|
+
)
|
|
237
|
+
if write_json is None:
|
|
238
|
+
env = os.environ.get("WIKIFIER_CACHE_JSON", "").strip().lower()
|
|
239
|
+
if env in ("1", "true", "yes", "always"):
|
|
240
|
+
write_json = True
|
|
241
|
+
elif env in ("0", "false", "no", "never"):
|
|
242
|
+
write_json = False
|
|
243
|
+
else:
|
|
244
|
+
write_json = file_n <= 400
|
|
245
|
+
|
|
246
|
+
with _db(root) as conn:
|
|
247
|
+
conn.execute("DELETE FROM files")
|
|
248
|
+
conn.execute("DELETE FROM meta")
|
|
249
|
+
for k, v in cache.items():
|
|
250
|
+
if not isinstance(k, str):
|
|
251
|
+
continue
|
|
252
|
+
if k.startswith("_"):
|
|
253
|
+
try:
|
|
254
|
+
raw = json.dumps(v, ensure_ascii=False, separators=(",", ":"))
|
|
255
|
+
except Exception:
|
|
256
|
+
raw = json.dumps(str(v))
|
|
257
|
+
conn.execute(
|
|
258
|
+
"INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
|
|
259
|
+
(k, raw),
|
|
260
|
+
)
|
|
261
|
+
elif isinstance(v, dict):
|
|
262
|
+
mtime = int(v.get("mtime", 0) or 0)
|
|
263
|
+
chash = v.get("content_hash")
|
|
264
|
+
try:
|
|
265
|
+
payload = json.dumps(v, ensure_ascii=False, separators=(",", ":"))
|
|
266
|
+
except Exception:
|
|
267
|
+
payload = "{}"
|
|
268
|
+
conn.execute(
|
|
269
|
+
"INSERT OR REPLACE INTO files(rel_path, mtime, content_hash, payload) "
|
|
270
|
+
"VALUES (?, ?, ?, ?)",
|
|
271
|
+
(k, mtime, chash, payload),
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
if write_json:
|
|
275
|
+
jp = json_path(root)
|
|
276
|
+
jp.parent.mkdir(parents=True, exist_ok=True)
|
|
277
|
+
with open(jp, "w", encoding="utf-8") as f:
|
|
278
|
+
json.dump(cache, f, ensure_ascii=False, separators=(",", ":"))
|
|
279
|
+
return "sqlite+json"
|
|
280
|
+
# If dual-write disabled, leave legacy json in place for dual-read until removed;
|
|
281
|
+
# do not delete automatically (safe migration).
|
|
282
|
+
return "sqlite"
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def update_file_index_rows(
|
|
286
|
+
root: Path,
|
|
287
|
+
rows: List[Tuple[str, int, Optional[str]]],
|
|
288
|
+
) -> int:
|
|
289
|
+
"""Update mtime/content_hash for existing file rows without rewriting payloads.
|
|
290
|
+
|
|
291
|
+
rows: (rel_path, mtime, content_hash|None)
|
|
292
|
+
Returns number of rows updated (SQLite only; 0 if no db).
|
|
293
|
+
"""
|
|
294
|
+
root = Path(root)
|
|
295
|
+
if not has_sqlite(root) or not rows:
|
|
296
|
+
return 0
|
|
297
|
+
n = 0
|
|
298
|
+
with _db(root) as conn:
|
|
299
|
+
for rel, mtime, chash in rows:
|
|
300
|
+
if chash:
|
|
301
|
+
cur = conn.execute(
|
|
302
|
+
"UPDATE files SET mtime = ?, content_hash = ? WHERE rel_path = ?",
|
|
303
|
+
(int(mtime), chash, rel),
|
|
304
|
+
)
|
|
305
|
+
else:
|
|
306
|
+
cur = conn.execute(
|
|
307
|
+
"UPDATE files SET mtime = ? WHERE rel_path = ?",
|
|
308
|
+
(int(mtime), rel),
|
|
309
|
+
)
|
|
310
|
+
n += int(cur.rowcount or 0)
|
|
311
|
+
return n
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def save_meta_key(root: Path, key: str, value: Any) -> bool:
|
|
315
|
+
"""Write a single meta key to SQLite (warm ACS upgrade without full rewrite)."""
|
|
316
|
+
root = Path(root)
|
|
317
|
+
if not key.startswith("_"):
|
|
318
|
+
return False
|
|
319
|
+
try:
|
|
320
|
+
raw = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
321
|
+
except Exception:
|
|
322
|
+
return False
|
|
323
|
+
if not has_sqlite(root):
|
|
324
|
+
# Fall back: load full, set, save
|
|
325
|
+
cache = load_cache_dict(root)
|
|
326
|
+
cache[key] = value
|
|
327
|
+
save_cache_dict(root, cache)
|
|
328
|
+
return True
|
|
329
|
+
with _db(root) as conn:
|
|
330
|
+
conn.execute(
|
|
331
|
+
"INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
|
|
332
|
+
(key, raw),
|
|
333
|
+
)
|
|
334
|
+
return True
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def get_map_coverage_from_meta(root: Path) -> Dict[str, Any]:
|
|
338
|
+
"""Read last map_coverage snapshot from meta if present."""
|
|
339
|
+
meta = load_meta(root, keys=("_map_coverage", "_acs_summary"))
|
|
340
|
+
cov = meta.get("_map_coverage") if isinstance(meta.get("_map_coverage"), dict) else {}
|
|
341
|
+
acs = meta.get("_acs_summary") if isinstance(meta.get("_acs_summary"), dict) else {}
|
|
342
|
+
return {
|
|
343
|
+
"map_coverage": cov,
|
|
344
|
+
"acs_version": acs.get("acs_version"),
|
|
345
|
+
"cache_backend": backend_name(root),
|
|
346
|
+
"actionable_low_conf_edges": acs.get("actionable_low_conf_edges"),
|
|
347
|
+
"reason_code_counts": acs.get("reason_code_counts"),
|
|
348
|
+
}
|