wikifier 4.2.0__tar.gz → 4.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wikifier-4.2.0/wikifier.egg-info → wikifier-4.3.0}/PKG-INFO +11 -11
- {wikifier-4.2.0 → wikifier-4.3.0}/README.md +10 -10
- {wikifier-4.2.0 → wikifier-4.3.0}/skills/run.md +3 -3
- {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_parsers.py +58 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/__init__.py +1 -1
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/cli.py +7 -2
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/import_cache.py +12 -1
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/bree.py +82 -8
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/javascript.py +316 -23
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/scripts/wikifier.bat +2 -2
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/scripts/wikifier.ps1 +2 -2
- wikifier-4.3.0/wikifier/scripts/wikifier.sh +822 -0
- {wikifier-4.2.0 → wikifier-4.3.0/wikifier.egg-info}/PKG-INFO +11 -11
- wikifier-4.2.0/wikifier/scripts/wikifier.sh +0 -2906
- {wikifier-4.2.0 → wikifier-4.3.0}/CONTRIBUTING.md +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/LICENSE +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/MANIFEST.in +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/diagnostics.html +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/docs/spec.md +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/index.html +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/pyproject.toml +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/setup.cfg +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_barrel_invalidation.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_health.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_import_cache.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/__main__.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/contracts.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/daemon.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/diagnostics.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/health.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/index.html +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/library.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/locking.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/mcp/__init__.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/mcp/server.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/__init__.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/cdia.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/python.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/resolution.py +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/SOURCES.txt +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/dependency_links.txt +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/entry_points.txt +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/requires.txt +0 -0
- {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wikifier
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.3.0
|
|
4
4
|
Summary: Zero-dependency agent-to-agent codebase wiki for LLMs and AI agents. Autonomously maintained via record-change and mark-green for token-efficient lookup of files, dependencies, health, and summaries — across tiny scripts to 50k+ monorepos. Optional MCP server with rich tools for agents.
|
|
5
5
|
Author-email: Aron Amos <aron@example.com>
|
|
6
6
|
Maintainer: Aron Amos
|
|
@@ -92,7 +92,7 @@ For external projects or monorepos, always pass an explicit root: `WIKIFIER_PROJ
|
|
|
92
92
|
|
|
93
93
|
## What You Get
|
|
94
94
|
|
|
95
|
-
- **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces),
|
|
95
|
+
- **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces), per-edge confidence scoring with actionable explanations, and **name-routed barrel expansion**: an import through an `export *` barrel resolves to the leaves that actually define the imported symbols, not hundreds of false edges
|
|
96
96
|
- **A pure-Python update pipeline** — `update-maps` parses every changed file in-process, persists a canonical import cache, computes reverse dependencies, Tarjan-based circular dependency detection, and regenerates `library.md` atomically
|
|
97
97
|
- **Incremental everything** — mtime-based dirty detection plus a barrel-aware reverse index means editing one file re-analyzes only its true consumers, even in barrel-heavy monorepos
|
|
98
98
|
- **Autonomous maintenance** — agents log intent with `record-change`, refresh summaries, and `mark-green`; a background `monitor`/daemon keeps the matrix fresh
|
|
@@ -107,7 +107,7 @@ Validated by dogfooding on Wikifier itself plus large open-source codebases —
|
|
|
107
107
|
| Codebase | Scale | Full `update-maps` |
|
|
108
108
|
|----------|-------|--------------------|
|
|
109
109
|
| llama_index | 3,837 Python files | **~8.5s** (17k import edges, full map) |
|
|
110
|
-
| Babylon.js | 3,905 TypeScript files, barrel-heavy |
|
|
110
|
+
| Babylon.js | 3,905 TypeScript files, barrel-heavy | **~4.5min** full build (44k precise edges, 5,389-node map); scoped re-runs ~80s |
|
|
111
111
|
| Linux kernel / LLVM / Rust | 37k–54k file trees | candidate scan in 3–8s |
|
|
112
112
|
|
|
113
113
|
JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the first build take seconds. Verified by a stdlib-only test suite (`python -m unittest discover tests`).
|
|
@@ -120,7 +120,7 @@ JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the
|
|
|
120
120
|
| `wikifier check-changes` | Incremental change scan + health/pending update |
|
|
121
121
|
| `wikifier record-change <file> "reason"` | Log the *why* (required after edits) |
|
|
122
122
|
| `wikifier mark-green <file>` | Mark the wiki entry current |
|
|
123
|
-
| `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (pure-Python pipeline;
|
|
123
|
+
| `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (single pure-Python pipeline; the shell launcher delegates here) |
|
|
124
124
|
| `wikifier health [--summary\|--json]` | Health matrix — compact formats for agents |
|
|
125
125
|
| `wikifier cycles` | Circular dependency report with break recommendations |
|
|
126
126
|
| `wikifier monitor &` / `wikifier daemon start` | Background heartbeat / managed maintenance |
|
|
@@ -143,16 +143,16 @@ Works with Claude Desktop, Claude Code, Cursor, Cline, and any Model Context Pro
|
|
|
143
143
|
|
|
144
144
|
Wikifier is a token-saving wiki layer *for agents*, and deliberately nothing more: not a general documentation generator, not an IDE plugin, not a code search engine for humans. Agents consult the map instead of re-reading sources, keep it current as they work, and create new wiki entries on the fly. The human dashboard is a window into that, not the product.
|
|
145
145
|
|
|
146
|
-
## What's New in v4.
|
|
146
|
+
## What's New in v4.3.0
|
|
147
147
|
|
|
148
|
-
|
|
148
|
+
Precision and performance, validated on real monorepos:
|
|
149
149
|
|
|
150
|
-
-
|
|
151
|
-
- **
|
|
152
|
-
-
|
|
153
|
-
-
|
|
150
|
+
- **Name-routed barrel expansion** — an import through an `export *` barrel resolves to the leaves that define the imported symbols (on Babylon.js, one 778-edge import became 3 precise edges; repo-wide edge count dropped 89%). Imports with no routable names are capped with the truncation *reported* on the edge — never silent
|
|
151
|
+
- **Thin shell** — the launcher now delegates `update-maps` to the Python pipeline (2,910 → 785 lines; 3m39s → 2s on a small project; a hang-prone legacy code path is gone entirely)
|
|
152
|
+
- **Bounded barrel-cache persistence + memoized root discovery** — Babylon.js's import cache shrank 274MB → 101MB, and a pathological scoped re-run went from 75 minutes to ~80 seconds
|
|
153
|
+
- `imported_names` is now populated on JS/TS edges, and files outside the configured project root resolve correctly (containment rule)
|
|
154
154
|
|
|
155
|
-
Full
|
|
155
|
+
v4.2.0 (same day) made `update-maps` a real full pipeline, fixed the lock deadlock behind historical MCP timeouts, and added the test suite. Full history: [`CHANGELOG.md`](CHANGELOG.md) and `Findings/`.
|
|
156
156
|
|
|
157
157
|
## Links
|
|
158
158
|
|
|
@@ -60,7 +60,7 @@ For external projects or monorepos, always pass an explicit root: `WIKIFIER_PROJ
|
|
|
60
60
|
|
|
61
61
|
## What You Get
|
|
62
62
|
|
|
63
|
-
- **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces),
|
|
63
|
+
- **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces), per-edge confidence scoring with actionable explanations, and **name-routed barrel expansion**: an import through an `export *` barrel resolves to the leaves that actually define the imported symbols, not hundreds of false edges
|
|
64
64
|
- **A pure-Python update pipeline** — `update-maps` parses every changed file in-process, persists a canonical import cache, computes reverse dependencies, Tarjan-based circular dependency detection, and regenerates `library.md` atomically
|
|
65
65
|
- **Incremental everything** — mtime-based dirty detection plus a barrel-aware reverse index means editing one file re-analyzes only its true consumers, even in barrel-heavy monorepos
|
|
66
66
|
- **Autonomous maintenance** — agents log intent with `record-change`, refresh summaries, and `mark-green`; a background `monitor`/daemon keeps the matrix fresh
|
|
@@ -75,7 +75,7 @@ Validated by dogfooding on Wikifier itself plus large open-source codebases —
|
|
|
75
75
|
| Codebase | Scale | Full `update-maps` |
|
|
76
76
|
|----------|-------|--------------------|
|
|
77
77
|
| llama_index | 3,837 Python files | **~8.5s** (17k import edges, full map) |
|
|
78
|
-
| Babylon.js | 3,905 TypeScript files, barrel-heavy |
|
|
78
|
+
| Babylon.js | 3,905 TypeScript files, barrel-heavy | **~4.5min** full build (44k precise edges, 5,389-node map); scoped re-runs ~80s |
|
|
79
79
|
| Linux kernel / LLVM / Rust | 37k–54k file trees | candidate scan in 3–8s |
|
|
80
80
|
|
|
81
81
|
JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the first build take seconds. Verified by a stdlib-only test suite (`python -m unittest discover tests`).
|
|
@@ -88,7 +88,7 @@ JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the
|
|
|
88
88
|
| `wikifier check-changes` | Incremental change scan + health/pending update |
|
|
89
89
|
| `wikifier record-change <file> "reason"` | Log the *why* (required after edits) |
|
|
90
90
|
| `wikifier mark-green <file>` | Mark the wiki entry current |
|
|
91
|
-
| `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (pure-Python pipeline;
|
|
91
|
+
| `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (single pure-Python pipeline; the shell launcher delegates here) |
|
|
92
92
|
| `wikifier health [--summary\|--json]` | Health matrix — compact formats for agents |
|
|
93
93
|
| `wikifier cycles` | Circular dependency report with break recommendations |
|
|
94
94
|
| `wikifier monitor &` / `wikifier daemon start` | Background heartbeat / managed maintenance |
|
|
@@ -111,16 +111,16 @@ Works with Claude Desktop, Claude Code, Cursor, Cline, and any Model Context Pro
|
|
|
111
111
|
|
|
112
112
|
Wikifier is a token-saving wiki layer *for agents*, and deliberately nothing more: not a general documentation generator, not an IDE plugin, not a code search engine for humans. Agents consult the map instead of re-reading sources, keep it current as they work, and create new wiki entries on the fly. The human dashboard is a window into that, not the product.
|
|
113
113
|
|
|
114
|
-
## What's New in v4.
|
|
114
|
+
## What's New in v4.3.0
|
|
115
115
|
|
|
116
|
-
|
|
116
|
+
Precision and performance, validated on real monorepos:
|
|
117
117
|
|
|
118
|
-
-
|
|
119
|
-
- **
|
|
120
|
-
-
|
|
121
|
-
-
|
|
118
|
+
- **Name-routed barrel expansion** — an import through an `export *` barrel resolves to the leaves that define the imported symbols (on Babylon.js, one 778-edge import became 3 precise edges; repo-wide edge count dropped 89%). Imports with no routable names are capped with the truncation *reported* on the edge — never silent
|
|
119
|
+
- **Thin shell** — the launcher now delegates `update-maps` to the Python pipeline (2,910 → 785 lines; 3m39s → 2s on a small project; a hang-prone legacy code path is gone entirely)
|
|
120
|
+
- **Bounded barrel-cache persistence + memoized root discovery** — Babylon.js's import cache shrank 274MB → 101MB, and a pathological scoped re-run went from 75 minutes to ~80 seconds
|
|
121
|
+
- `imported_names` is now populated on JS/TS edges, and files outside the configured project root resolve correctly (containment rule)
|
|
122
122
|
|
|
123
|
-
Full
|
|
123
|
+
v4.2.0 (same day) made `update-maps` a real full pipeline, fixed the lock deadlock behind historical MCP timeouts, and added the test suite. Full history: [`CHANGELOG.md`](CHANGELOG.md) and `Findings/`.
|
|
124
124
|
|
|
125
125
|
## Links
|
|
126
126
|
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
**Status**: Active. Supersedes v0.5.
|
|
8
8
|
**See also**: `README.md` (Intended Use section: strictly agent-to-agent wiki for token saving), M5-Dogfood-Assessment-Report.md, Findings/2026-06-10-Dogfood-Refactor-Validation.md, and the library in wikifier/.
|
|
9
9
|
|
|
10
|
-
**v0.6 migration notes (
|
|
11
|
-
- `update-maps` (CLI) now runs the **pure-Python full pipeline
|
|
10
|
+
**v0.6 migration notes (packages v4.2.0–v4.3.0)** — changes are additive or strictly-better defaults; v0.5 agent behavior keeps working:
|
|
11
|
+
- `update-maps` (CLI) now runs the **pure-Python full pipeline**: every dirty file is parsed in-process, the canonical per-file cache is persisted, reverse deps + cycles + ACS are computed, and `library.md` is regenerated atomically. `--python-primary` is accepted but redundant; the in-shell first-pass was retired entirely (wikifier.sh's update-maps delegates to this pipeline; `--sh`/`--legacy-sh` are deprecated no-ops). Scoping is explicit via `--directory=`/`--max-files=` and reported in the result (`files_skipped`) — there are no silent caps.
|
|
12
12
|
- `run_full_update`/`update_maps` returns gained additive fields: `files_parsed` (actual), `parseable_files`, `edges_persisted`, `files_skipped`, `cycles`, `library`. Tolerate additive fields as always.
|
|
13
13
|
- `wikifier health --summary|--json|--format=...` now work from the CLI (previously library-only).
|
|
14
14
|
- A POSIX lock self-deadlock in library `record_change`/`mark_green`/`check_changes` was fixed (locking is now re-entrant per process). MCP timeouts on those tools should no longer occur; the CLI fallback guidance below remains valid as defense in depth.
|
|
@@ -158,7 +158,7 @@ See the full design, mandatory workflow example, and M2 exit criteria in the pla
|
|
|
158
158
|
| `wikifier prepare-edit` | `<file>` | Stage current mtime before you start editing (for future diffing). |
|
|
159
159
|
| `wikifier mark-green` | `<file> [reason]` | Flip file status to Green after you have written/updated its wiki summary. |
|
|
160
160
|
| `wikifier monitor` | — | Start background 30s heartbeat (run with `&` or in separate terminal). |
|
|
161
|
-
| `wikifier update-maps` | `[--full] [--directory=...] [--max-files=N]
|
|
161
|
+
| `wikifier update-maps` | `[--full] [--directory=...] [--max-files=N]` | Rebuild `library.md` + import cache (single pure-Python pipeline; the shell launcher delegates here). |
|
|
162
162
|
| `wikifier validate` | — | Ensure every file in monitored_paths has at least a health row. |
|
|
163
163
|
| `wikifier journal` | `[YYYY-MM-DD]` | Read the journal for a day (default = today). |
|
|
164
164
|
| `wikifier issues` | `[simple|moderate|high|critical]` | List logged issues by severity. |
|
|
@@ -153,5 +153,63 @@ class TestJavaScriptParser(TempProjectTestCase):
|
|
|
153
153
|
self.assertEqual(edge.get("resolution_confidence"), "high")
|
|
154
154
|
|
|
155
155
|
|
|
156
|
+
class TestBarrelLeafExplosionPolicy(TempProjectTestCase):
|
|
157
|
+
"""Leaf-explosion policy: name routing + reported cap.
|
|
158
|
+
|
|
159
|
+
A named import through an `export *` barrel must emit the entry edge plus
|
|
160
|
+
only the leaves that export the requested names (precision); imports with
|
|
161
|
+
no usable names (side-effect/namespace/dynamic) fall back to a cap whose
|
|
162
|
+
truncation is reported via barrel_leaf_selection — never silent.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
LEAVES = 30
|
|
166
|
+
|
|
167
|
+
def setUp(self):
|
|
168
|
+
super().setUp()
|
|
169
|
+
lines = []
|
|
170
|
+
for i in range(self.LEAVES):
|
|
171
|
+
self.write(f"big/leaf{i}.js", f"export const thing{i} = {i};\n")
|
|
172
|
+
lines.append(f"export * from './leaf{i}.js';")
|
|
173
|
+
self.write("big/index.js", "\n".join(lines) + "\n")
|
|
174
|
+
self.write("named.js", "import { thing7 } from './big';\n")
|
|
175
|
+
self.write("nameless.js", "import * as big from './big';\n")
|
|
176
|
+
|
|
177
|
+
def _parse(self, rel):
|
|
178
|
+
self.reset_js_parser_state()
|
|
179
|
+
from wikifier.parsers.javascript import parse_javascript_imports
|
|
180
|
+
return parse_javascript_imports(str(self.root / rel))
|
|
181
|
+
|
|
182
|
+
def test_named_import_routes_to_defining_leaf(self):
|
|
183
|
+
edges = self._parse("named.js")
|
|
184
|
+
paths = [e.get("resolved_path") for e in edges]
|
|
185
|
+
self.assertIn("big/index.js", paths, "entry-barrel edge must be kept")
|
|
186
|
+
self.assertIn("big/leaf7.js", paths, "the leaf defining thing7 must be kept")
|
|
187
|
+
self.assertLessEqual(
|
|
188
|
+
len(edges), 4,
|
|
189
|
+
f"name routing should drop non-matching leaves, got {len(edges)}: {paths}",
|
|
190
|
+
)
|
|
191
|
+
edge = find_edge(edges, raw_module="./big")
|
|
192
|
+
self.assertIn("thing7", edge.get("imported_names") or [])
|
|
193
|
+
|
|
194
|
+
def test_nameless_import_is_capped_with_reported_selection(self):
|
|
195
|
+
import os
|
|
196
|
+
old = os.environ.get("WIKIFIER_BARREL_LEAF_CAP")
|
|
197
|
+
os.environ["WIKIFIER_BARREL_LEAF_CAP"] = "5"
|
|
198
|
+
try:
|
|
199
|
+
edges = self._parse("nameless.js")
|
|
200
|
+
finally:
|
|
201
|
+
if old is None:
|
|
202
|
+
os.environ.pop("WIKIFIER_BARREL_LEAF_CAP", None)
|
|
203
|
+
else:
|
|
204
|
+
os.environ["WIKIFIER_BARREL_LEAF_CAP"] = old
|
|
205
|
+
barrel_edges = [e for e in edges if e.get("via_barrel")]
|
|
206
|
+
self.assertLessEqual(len(barrel_edges), 6, "cap must bound emission (entry + 5 leaves)")
|
|
207
|
+
sels = [e.get("barrel_leaf_selection") for e in edges if e.get("barrel_leaf_selection")]
|
|
208
|
+
self.assertTrue(sels, "truncation must be reported via barrel_leaf_selection")
|
|
209
|
+
self.assertTrue(sels[0].get("truncated"))
|
|
210
|
+
self.assertEqual(sels[0].get("leaves_emitted"), 5)
|
|
211
|
+
self.assertEqual(sels[0].get("leaves_total"), self.LEAVES)
|
|
212
|
+
|
|
213
|
+
|
|
156
214
|
if __name__ == "__main__":
|
|
157
215
|
unittest.main(verbosity=2)
|
|
@@ -381,6 +381,7 @@ def _pair_from_parser_edge(edge: Dict[str, Any], root: Path) -> Optional[Dict[st
|
|
|
381
381
|
"confidence_score", "confidence_reasons", "confidence_explanation",
|
|
382
382
|
"barrel_v2", "resolution_metadata", "strategy", "cdia_v1",
|
|
383
383
|
"conditional_analysis", "dynamic_analysis", "diagnostic",
|
|
384
|
+
"imported_names", "barrel_leaf_selection",
|
|
384
385
|
):
|
|
385
386
|
v = edge.get(k)
|
|
386
387
|
if v not in (None, "", [], {}):
|
|
@@ -732,8 +733,12 @@ def main():
|
|
|
732
733
|
python_primary_requested = True
|
|
733
734
|
continue # consume, do not forward to sh
|
|
734
735
|
if a in ("--sh", "--legacy-sh", "--no-python-primary"):
|
|
735
|
-
|
|
736
|
-
|
|
736
|
+
# The in-shell update-maps implementation was retired (2026-06-10
|
|
737
|
+
# thin-shell rework); wikifier.sh itself now delegates here.
|
|
738
|
+
print("[wikifier] note: --sh is a deprecated no-op — the legacy shell "
|
|
739
|
+
"update-maps path was retired; the Python pipeline always runs.",
|
|
740
|
+
file=sys.stderr)
|
|
741
|
+
continue # consume; stay on the Python pipeline
|
|
737
742
|
if a in ("update-maps", "update_maps"):
|
|
738
743
|
is_update_maps_cmd = True
|
|
739
744
|
stripped_filtered.append(a)
|
|
@@ -68,7 +68,15 @@ def save_cache(root: Path, cache: Dict[str, Any]) -> None:
|
|
|
68
68
|
|
|
69
69
|
Uses file locking (M2-Rem-07) to prevent corruption when multiple
|
|
70
70
|
agents are running update-maps or health operations concurrently.
|
|
71
|
+
|
|
72
|
+
Set WIKIFIER_DEBUG_SAVES=1 to print each save's call site to stderr —
|
|
73
|
+
the diagnostic for "who keeps rewriting the cache mid-run".
|
|
71
74
|
"""
|
|
75
|
+
if os.environ.get("WIKIFIER_DEBUG_SAVES"):
|
|
76
|
+
import sys as _sys
|
|
77
|
+
import traceback
|
|
78
|
+
frames = "".join(traceback.format_stack()[-4:-1])
|
|
79
|
+
print(f"[save_cache] root={root}\n{frames}", file=_sys.stderr)
|
|
72
80
|
if locking:
|
|
73
81
|
with locking.file_lock(root):
|
|
74
82
|
_do_save_cache(root, cache)
|
|
@@ -100,8 +108,11 @@ def _do_save_cache(root: Path, cache: Dict[str, Any]) -> None:
|
|
|
100
108
|
cache[k] = prev[k]
|
|
101
109
|
except Exception:
|
|
102
110
|
pass # never let preservation break the save itself
|
|
111
|
+
# Compact separators: the cache is machine-consumed only, and indent=2
|
|
112
|
+
# made barrel-heavy caches ~2-3x larger and proportionally slower to
|
|
113
|
+
# write (observed: 274MB on Babylon.js).
|
|
103
114
|
with open(cache_path, "w", encoding="utf-8") as f:
|
|
104
|
-
json.dump(cache, f,
|
|
115
|
+
json.dump(cache, f, ensure_ascii=False, separators=(",", ":"))
|
|
105
116
|
|
|
106
117
|
|
|
107
118
|
def get_file_data(cache: Dict[str, Any], rel_path: str) -> Optional[Dict[str, Any]]:
|
|
@@ -342,6 +342,10 @@ class BarrelResolutionCache:
|
|
|
342
342
|
def __post_init__(self) -> None:
|
|
343
343
|
self.resolutions = dict(self.resolutions or {})
|
|
344
344
|
self.file_index = dict(self.file_index or {})
|
|
345
|
+
# Transient set views over persisted membership lists (never serialized;
|
|
346
|
+
# see _membership). Without these, store() membership tests were linear
|
|
347
|
+
# scans that went quadratic across a run on barrels with many importers.
|
|
348
|
+
self._fast_sets: Dict[str, set] = {}
|
|
345
349
|
|
|
346
350
|
@classmethod
|
|
347
351
|
def from_cache(cls, cache: Dict[str, Any]) -> "BarrelResolutionCache":
|
|
@@ -358,6 +362,16 @@ class BarrelResolutionCache:
|
|
|
358
362
|
state); an explicit empty dict therefore remains the way to express
|
|
359
363
|
intentional clearing (prune-to-zero, clear()).
|
|
360
364
|
"""
|
|
365
|
+
# Stable output: importer/chain lists are append-ordered in memory for
|
|
366
|
+
# speed; sort once here so the persisted form is deterministic.
|
|
367
|
+
for entry in self.resolutions.values():
|
|
368
|
+
if isinstance(entry.get("importers"), list):
|
|
369
|
+
entry["importers"] = sorted(set(entry["importers"]))
|
|
370
|
+
for idxe in self.file_index.values():
|
|
371
|
+
if isinstance(idxe.get("importers"), list):
|
|
372
|
+
idxe["importers"] = sorted(set(idxe["importers"]))
|
|
373
|
+
if isinstance(idxe.get("chain_ids"), list):
|
|
374
|
+
idxe["chain_ids"] = sorted(set(idxe["chain_ids"]))
|
|
361
375
|
cache["_barrel_resolutions"] = self.resolutions or {}
|
|
362
376
|
cache["_barrel_file_index"] = self.file_index or {}
|
|
363
377
|
|
|
@@ -372,6 +386,58 @@ class BarrelResolutionCache:
|
|
|
372
386
|
def get(self, chain_id: str) -> Optional[Dict[str, Any]]:
|
|
373
387
|
return self.resolutions.get(chain_id)
|
|
374
388
|
|
|
389
|
+
@staticmethod
|
|
390
|
+
def _lean_results(results: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
|
|
391
|
+
"""Compress chain results for persistence.
|
|
392
|
+
|
|
393
|
+
Stored results exist so a cache hit can replay the expansion (and so
|
|
394
|
+
name routing on hits still sees the full leaf set) — they do NOT need
|
|
395
|
+
the heavy per-leaf payloads (resolution_metadata, barrel_v2 hop blobs,
|
|
396
|
+
analysis fields), which the emission layer rebuilds or defaults.
|
|
397
|
+
Those payloads were the dominant weight of barrel-heavy caches
|
|
398
|
+
(274MB import_cache.json on Babylon.js). Leaves are also deduped by
|
|
399
|
+
resolved_path (BREE can reach the same leaf via multiple hop paths).
|
|
400
|
+
"""
|
|
401
|
+
lean: List[Dict[str, Any]] = []
|
|
402
|
+
seen: set = set()
|
|
403
|
+
for r in results or []:
|
|
404
|
+
if not isinstance(r, dict):
|
|
405
|
+
continue
|
|
406
|
+
rp = r.get("resolved_path")
|
|
407
|
+
if rp:
|
|
408
|
+
if rp in seen:
|
|
409
|
+
continue
|
|
410
|
+
seen.add(rp)
|
|
411
|
+
slim = {}
|
|
412
|
+
for k in (
|
|
413
|
+
"module", "resolved_path", "via_barrel", "barrel_chain",
|
|
414
|
+
"barrel_depth", "is_conditional", "conditional_context",
|
|
415
|
+
"barrel_detector",
|
|
416
|
+
):
|
|
417
|
+
v = r.get(k)
|
|
418
|
+
if v is not None:
|
|
419
|
+
slim[k] = v
|
|
420
|
+
lean.append(slim)
|
|
421
|
+
return lean
|
|
422
|
+
|
|
423
|
+
@staticmethod
|
|
424
|
+
def _lean_hops(hops: Optional[List[Any]]) -> List[Dict[str, Any]]:
|
|
425
|
+
"""Keep only primitive hop fields (drop per-hop metadata dicts)."""
|
|
426
|
+
out: List[Dict[str, Any]] = []
|
|
427
|
+
for h in hops or []:
|
|
428
|
+
if isinstance(h, dict):
|
|
429
|
+
out.append({k: v for k, v in h.items() if not isinstance(v, (dict, list))})
|
|
430
|
+
return out
|
|
431
|
+
|
|
432
|
+
def _membership(self, key: str, current: List[str]) -> set:
|
|
433
|
+
"""Transient set view over a persisted list (membership tests during
|
|
434
|
+
store() were linear scans — quadratic across a run on popular barrels)."""
|
|
435
|
+
s = self._fast_sets.get(key)
|
|
436
|
+
if s is None:
|
|
437
|
+
s = set(current)
|
|
438
|
+
self._fast_sets[key] = s
|
|
439
|
+
return s
|
|
440
|
+
|
|
375
441
|
def store(
|
|
376
442
|
self,
|
|
377
443
|
*,
|
|
@@ -433,15 +499,19 @@ class BarrelResolutionCache:
|
|
|
433
499
|
snap[ck or str(k)] = 0.0
|
|
434
500
|
cid = chain_id or self._make_chain_id(bc, start_specifier)
|
|
435
501
|
entry = self.resolutions.get(cid, {})
|
|
436
|
-
# merge importers
|
|
437
|
-
|
|
438
|
-
|
|
502
|
+
# merge importers (set-backed membership; lists stay the persisted form)
|
|
503
|
+
imp_list = entry.get("importers", [])
|
|
504
|
+
imp_set = self._membership(f"res:{cid}", imp_list)
|
|
505
|
+
for imp in imps:
|
|
506
|
+
if imp and imp not in imp_set:
|
|
507
|
+
imp_set.add(imp)
|
|
508
|
+
imp_list.append(imp)
|
|
439
509
|
entry.update({
|
|
440
510
|
"chain_id": cid,
|
|
441
|
-
"importers":
|
|
511
|
+
"importers": imp_list,
|
|
442
512
|
"barrel_chain": bc or entry.get("barrel_chain", []),
|
|
443
|
-
"hops": hops
|
|
444
|
-
"results": results
|
|
513
|
+
"hops": self._lean_hops(hops) if hops else entry.get("hops", []),
|
|
514
|
+
"results": self._lean_results(results) if results else entry.get("results", []),
|
|
445
515
|
"start_specifier": start_specifier or entry.get("start_specifier", ""),
|
|
446
516
|
"detector_used": detector_used or entry.get("detector_used", "unknown"),
|
|
447
517
|
"is_partial": is_partial or entry.get("is_partial", False),
|
|
@@ -461,10 +531,14 @@ class BarrelResolutionCache:
|
|
|
461
531
|
if fkey not in self.file_index:
|
|
462
532
|
self.file_index[fkey] = {"chain_ids": [], "importers": []}
|
|
463
533
|
idxe = self.file_index[fkey]
|
|
464
|
-
|
|
534
|
+
cid_set = self._membership(f"idx_c:{fkey}", idxe["chain_ids"])
|
|
535
|
+
if cid not in cid_set:
|
|
536
|
+
cid_set.add(cid)
|
|
465
537
|
idxe["chain_ids"].append(cid)
|
|
538
|
+
imp_idx_set = self._membership(f"idx_i:{fkey}", idxe["importers"])
|
|
466
539
|
for imp in imps:
|
|
467
|
-
if imp and imp not in
|
|
540
|
+
if imp and imp not in imp_idx_set:
|
|
541
|
+
imp_idx_set.add(imp)
|
|
468
542
|
idxe["importers"].append(imp)
|
|
469
543
|
|
|
470
544
|
return cid
|