wikifier 4.2.0__tar.gz → 4.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {wikifier-4.2.0/wikifier.egg-info → wikifier-4.3.0}/PKG-INFO +11 -11
  2. {wikifier-4.2.0 → wikifier-4.3.0}/README.md +10 -10
  3. {wikifier-4.2.0 → wikifier-4.3.0}/skills/run.md +3 -3
  4. {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_parsers.py +58 -0
  5. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/__init__.py +1 -1
  6. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/cli.py +7 -2
  7. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/import_cache.py +12 -1
  8. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/bree.py +82 -8
  9. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/javascript.py +316 -23
  10. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/scripts/wikifier.bat +2 -2
  11. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/scripts/wikifier.ps1 +2 -2
  12. wikifier-4.3.0/wikifier/scripts/wikifier.sh +822 -0
  13. {wikifier-4.2.0 → wikifier-4.3.0/wikifier.egg-info}/PKG-INFO +11 -11
  14. wikifier-4.2.0/wikifier/scripts/wikifier.sh +0 -2906
  15. {wikifier-4.2.0 → wikifier-4.3.0}/CONTRIBUTING.md +0 -0
  16. {wikifier-4.2.0 → wikifier-4.3.0}/LICENSE +0 -0
  17. {wikifier-4.2.0 → wikifier-4.3.0}/MANIFEST.in +0 -0
  18. {wikifier-4.2.0 → wikifier-4.3.0}/diagnostics.html +0 -0
  19. {wikifier-4.2.0 → wikifier-4.3.0}/docs/spec.md +0 -0
  20. {wikifier-4.2.0 → wikifier-4.3.0}/index.html +0 -0
  21. {wikifier-4.2.0 → wikifier-4.3.0}/pyproject.toml +0 -0
  22. {wikifier-4.2.0 → wikifier-4.3.0}/setup.cfg +0 -0
  23. {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_barrel_invalidation.py +0 -0
  24. {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_health.py +0 -0
  25. {wikifier-4.2.0 → wikifier-4.3.0}/tests/test_import_cache.py +0 -0
  26. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/__main__.py +0 -0
  27. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/contracts.py +0 -0
  28. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/daemon.py +0 -0
  29. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/diagnostics.py +0 -0
  30. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/health.py +0 -0
  31. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/index.html +0 -0
  32. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/library.py +0 -0
  33. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/locking.py +0 -0
  34. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/mcp/__init__.py +0 -0
  35. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/mcp/server.py +0 -0
  36. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/__init__.py +0 -0
  37. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/cdia.py +0 -0
  38. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/parsers/python.py +0 -0
  39. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier/resolution.py +0 -0
  40. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/SOURCES.txt +0 -0
  41. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/dependency_links.txt +0 -0
  42. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/entry_points.txt +0 -0
  43. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/requires.txt +0 -0
  44. {wikifier-4.2.0 → wikifier-4.3.0}/wikifier.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: wikifier
3
- Version: 4.2.0
3
+ Version: 4.3.0
4
4
  Summary: Zero-dependency agent-to-agent codebase wiki for LLMs and AI agents. Autonomously maintained via record-change and mark-green for token-efficient lookup of files, dependencies, health, and summaries — across tiny scripts to 50k+ monorepos. Optional MCP server with rich tools for agents.
5
5
  Author-email: Aron Amos <aron@example.com>
6
6
  Maintainer: Aron Amos
@@ -92,7 +92,7 @@ For external projects or monorepos, always pass an explicit root: `WIKIFIER_PROJ
92
92
 
93
93
  ## What You Get
94
94
 
95
- - **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces), barrel/re-export chain expansion, and per-edge confidence scoring with actionable explanations
95
+ - **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces), per-edge confidence scoring with actionable explanations, and **name-routed barrel expansion**: an import through an `export *` barrel resolves to the leaves that actually define the imported symbols, not hundreds of false edges
96
96
  - **A pure-Python update pipeline** — `update-maps` parses every changed file in-process, persists a canonical import cache, computes reverse dependencies, Tarjan-based circular dependency detection, and regenerates `library.md` atomically
97
97
  - **Incremental everything** — mtime-based dirty detection plus a barrel-aware reverse index means editing one file re-analyzes only its true consumers, even in barrel-heavy monorepos
98
98
  - **Autonomous maintenance** — agents log intent with `record-change`, refresh summaries, and `mark-green`; a background `monitor`/daemon keeps the matrix fresh
@@ -107,7 +107,7 @@ Validated by dogfooding on Wikifier itself plus large open-source codebases —
107
107
  | Codebase | Scale | Full `update-maps` |
108
108
  |----------|-------|--------------------|
109
109
  | llama_index | 3,837 Python files | **~8.5s** (17k import edges, full map) |
110
- | Babylon.js | 3,905 TypeScript files, barrel-heavy | completes with full 5,389-node map |
110
+ | Babylon.js | 3,905 TypeScript files, barrel-heavy | **~4.5min** full build (44k precise edges, 5,389-node map); scoped re-runs ~80s |
111
111
  | Linux kernel / LLVM / Rust | 37k–54k file trees | candidate scan in 3–8s |
112
112
 
113
113
  JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the first build take seconds. Verified by a stdlib-only test suite (`python -m unittest discover tests`).
@@ -120,7 +120,7 @@ JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the
120
120
  | `wikifier check-changes` | Incremental change scan + health/pending update |
121
121
  | `wikifier record-change <file> "reason"` | Log the *why* (required after edits) |
122
122
  | `wikifier mark-green <file>` | Mark the wiki entry current |
123
- | `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (pure-Python pipeline; `--sh` for the legacy shell path) |
123
+ | `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (single pure-Python pipeline; the shell launcher delegates here) |
124
124
  | `wikifier health [--summary\|--json]` | Health matrix — compact formats for agents |
125
125
  | `wikifier cycles` | Circular dependency report with break recommendations |
126
126
  | `wikifier monitor &` / `wikifier daemon start` | Background heartbeat / managed maintenance |
@@ -143,16 +143,16 @@ Works with Claude Desktop, Claude Code, Cursor, Cline, and any Model Context Pro
143
143
 
144
144
  Wikifier is a token-saving wiki layer *for agents*, and deliberately nothing more: not a general documentation generator, not an IDE plugin, not a code search engine for humans. Agents consult the map instead of re-reading sources, keep it current as they work, and create new wiki entries on the fly. The human dashboard is a window into that, not the product.
145
145
 
146
- ## What's New in v4.2.0
146
+ ## What's New in v4.3.0
147
147
 
148
- A full real-world dogfood sweep (this repo + RecipeLab + 8 large OSS projects) followed by a fix pass for everything it surfaced:
148
+ Precision and performance, validated on real monorepos:
149
149
 
150
- - `update-maps` now defaults to the **pure-Python pipeline**: every changed file actually parsed, canonical cache schema, cycles/ACS/reverse-deps computed, `library.md` regenerated atomically
151
- - **43× faster JS/TS parsing** (barrel-cache persistence batched per run instead of per chain)
152
- - Fixed a POSIX lock self-deadlock in the library workflow calls (the likely cause of historical MCP timeouts), barrel-churn invalidation, static-import flag pollution, exclude-pattern globs, `health --summary/--json`, and several shell-path crashes one of which could destroy an existing `library.md`
153
- - **New stdlib-only test suite** (28 tests) the project's first formal regression net
150
+ - **Name-routed barrel expansion** — an import through an `export *` barrel resolves to the leaves that define the imported symbols (on Babylon.js, one 778-edge import became 3 precise edges; repo-wide edge count dropped 89%). Imports with no routable names are capped with the truncation *reported* on the edge — never silent
151
+ - **Thin shell** the launcher now delegates `update-maps` to the Python pipeline (2,910 785 lines; 3m39s → 2s on a small project; a hang-prone legacy code path is gone entirely)
152
+ - **Bounded barrel-cache persistence + memoized root discovery** Babylon.js's import cache shrank 274MB 101MB, and a pathological scoped re-run went from 75 minutes to ~80 seconds
153
+ - `imported_names` is now populated on JS/TS edges, and files outside the configured project root resolve correctly (containment rule)
154
154
 
155
- Full details: [`CHANGELOG.md`](CHANGELOG.md) and `Findings/2026-06-10-Dogfood-Refactor-Validation.md`.
155
+ v4.2.0 (same day) made `update-maps` a real full pipeline, fixed the lock deadlock behind historical MCP timeouts, and added the test suite. Full history: [`CHANGELOG.md`](CHANGELOG.md) and `Findings/`.
156
156
 
157
157
  ## Links
158
158
 
@@ -60,7 +60,7 @@ For external projects or monorepos, always pass an explicit root: `WIKIFIER_PROJ
60
60
 
61
61
  ## What You Get
62
62
 
63
- - **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces), barrel/re-export chain expansion, and per-edge confidence scoring with actionable explanations
63
+ - **Fast, real dependency analysis** — Python and JavaScript/TypeScript import parsing (ES modules, CommonJS, dynamic imports, TypeScript path aliases, package.json exports, workspaces), per-edge confidence scoring with actionable explanations, and **name-routed barrel expansion**: an import through an `export *` barrel resolves to the leaves that actually define the imported symbols, not hundreds of false edges
64
64
  - **A pure-Python update pipeline** — `update-maps` parses every changed file in-process, persists a canonical import cache, computes reverse dependencies, Tarjan-based circular dependency detection, and regenerates `library.md` atomically
65
65
  - **Incremental everything** — mtime-based dirty detection plus a barrel-aware reverse index means editing one file re-analyzes only its true consumers, even in barrel-heavy monorepos
66
66
  - **Autonomous maintenance** — agents log intent with `record-change`, refresh summaries, and `mark-green`; a background `monitor`/daemon keeps the matrix fresh
@@ -75,7 +75,7 @@ Validated by dogfooding on Wikifier itself plus large open-source codebases —
75
75
  | Codebase | Scale | Full `update-maps` |
76
76
  |----------|-------|--------------------|
77
77
  | llama_index | 3,837 Python files | **~8.5s** (17k import edges, full map) |
78
- | Babylon.js | 3,905 TypeScript files, barrel-heavy | completes with full 5,389-node map |
78
+ | Babylon.js | 3,905 TypeScript files, barrel-heavy | **~4.5min** full build (44k precise edges, 5,389-node map); scoped re-runs ~80s |
79
79
  | Linux kernel / LLVM / Rust | 37k–54k file trees | candidate scan in 3–8s |
80
80
 
81
81
  JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the first build take seconds. Verified by a stdlib-only test suite (`python -m unittest discover tests`).
@@ -88,7 +88,7 @@ JS/TS parsing runs at ~22ms per file; Python at ~1ms. Incremental runs after the
88
88
  | `wikifier check-changes` | Incremental change scan + health/pending update |
89
89
  | `wikifier record-change <file> "reason"` | Log the *why* (required after edits) |
90
90
  | `wikifier mark-green <file>` | Mark the wiki entry current |
91
- | `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (pure-Python pipeline; `--sh` for the legacy shell path) |
91
+ | `wikifier update-maps [--full] [--directory=src/]` | Rebuild dependency graph + library.md (single pure-Python pipeline; the shell launcher delegates here) |
92
92
  | `wikifier health [--summary\|--json]` | Health matrix — compact formats for agents |
93
93
  | `wikifier cycles` | Circular dependency report with break recommendations |
94
94
  | `wikifier monitor &` / `wikifier daemon start` | Background heartbeat / managed maintenance |
@@ -111,16 +111,16 @@ Works with Claude Desktop, Claude Code, Cursor, Cline, and any Model Context Pro
111
111
 
112
112
  Wikifier is a token-saving wiki layer *for agents*, and deliberately nothing more: not a general documentation generator, not an IDE plugin, not a code search engine for humans. Agents consult the map instead of re-reading sources, keep it current as they work, and create new wiki entries on the fly. The human dashboard is a window into that, not the product.
113
113
 
114
- ## What's New in v4.2.0
114
+ ## What's New in v4.3.0
115
115
 
116
- A full real-world dogfood sweep (this repo + RecipeLab + 8 large OSS projects) followed by a fix pass for everything it surfaced:
116
+ Precision and performance, validated on real monorepos:
117
117
 
118
- - `update-maps` now defaults to the **pure-Python pipeline**: every changed file actually parsed, canonical cache schema, cycles/ACS/reverse-deps computed, `library.md` regenerated atomically
119
- - **43× faster JS/TS parsing** (barrel-cache persistence batched per run instead of per chain)
120
- - Fixed a POSIX lock self-deadlock in the library workflow calls (the likely cause of historical MCP timeouts), barrel-churn invalidation, static-import flag pollution, exclude-pattern globs, `health --summary/--json`, and several shell-path crashes one of which could destroy an existing `library.md`
121
- - **New stdlib-only test suite** (28 tests) the project's first formal regression net
118
+ - **Name-routed barrel expansion** — an import through an `export *` barrel resolves to the leaves that define the imported symbols (on Babylon.js, one 778-edge import became 3 precise edges; repo-wide edge count dropped 89%). Imports with no routable names are capped with the truncation *reported* on the edge — never silent
119
+ - **Thin shell** the launcher now delegates `update-maps` to the Python pipeline (2,910 785 lines; 3m39s → 2s on a small project; a hang-prone legacy code path is gone entirely)
120
+ - **Bounded barrel-cache persistence + memoized root discovery** Babylon.js's import cache shrank 274MB 101MB, and a pathological scoped re-run went from 75 minutes to ~80 seconds
121
+ - `imported_names` is now populated on JS/TS edges, and files outside the configured project root resolve correctly (containment rule)
122
122
 
123
- Full details: [`CHANGELOG.md`](CHANGELOG.md) and `Findings/2026-06-10-Dogfood-Refactor-Validation.md`.
123
+ v4.2.0 (same day) made `update-maps` a real full pipeline, fixed the lock deadlock behind historical MCP timeouts, and added the test suite. Full history: [`CHANGELOG.md`](CHANGELOG.md) and `Findings/`.
124
124
 
125
125
  ## Links
126
126
 
@@ -7,8 +7,8 @@
7
7
  **Status**: Active. Supersedes v0.5.
8
8
  **See also**: `README.md` (Intended Use section: strictly agent-to-agent wiki for token saving), M5-Dogfood-Assessment-Report.md, Findings/2026-06-10-Dogfood-Refactor-Validation.md, and the library in wikifier/.
9
9
 
10
- **v0.6 migration notes (package v4.2.0)** — changes are additive or strictly-better defaults; v0.5 agent behavior keeps working:
11
- - `update-maps` (CLI) now runs the **pure-Python full pipeline by default**: every dirty file is parsed in-process, the canonical per-file cache is persisted, reverse deps + cycles + ACS are computed, and `library.md` is regenerated atomically. `--python-primary` is accepted but redundant; `--sh`/`--legacy-sh` selects the legacy shell path (slow fallback). Scoping is explicit via `--directory=`/`--max-files=` and reported in the result (`files_skipped`) — there are no silent caps.
10
+ **v0.6 migration notes (packages v4.2.0–v4.3.0)** — changes are additive or strictly-better defaults; v0.5 agent behavior keeps working:
11
+ - `update-maps` (CLI) now runs the **pure-Python full pipeline**: every dirty file is parsed in-process, the canonical per-file cache is persisted, reverse deps + cycles + ACS are computed, and `library.md` is regenerated atomically. `--python-primary` is accepted but redundant; the in-shell first-pass was retired entirely (wikifier.sh's update-maps delegates to this pipeline; `--sh`/`--legacy-sh` are deprecated no-ops). Scoping is explicit via `--directory=`/`--max-files=` and reported in the result (`files_skipped`) — there are no silent caps.
12
12
  - `run_full_update`/`update_maps` returns gained additive fields: `files_parsed` (actual), `parseable_files`, `edges_persisted`, `files_skipped`, `cycles`, `library`. Tolerate additive fields as always.
13
13
  - `wikifier health --summary|--json|--format=...` now work from the CLI (previously library-only).
14
14
  - A POSIX lock self-deadlock in library `record_change`/`mark_green`/`check_changes` was fixed (locking is now re-entrant per process). MCP timeouts on those tools should no longer occur; the CLI fallback guidance below remains valid as defense in depth.
@@ -158,7 +158,7 @@ See the full design, mandatory workflow example, and M2 exit criteria in the pla
158
158
  | `wikifier prepare-edit` | `<file>` | Stage current mtime before you start editing (for future diffing). |
159
159
  | `wikifier mark-green` | `<file> [reason]` | Flip file status to Green after you have written/updated its wiki summary. |
160
160
  | `wikifier monitor` | — | Start background 30s heartbeat (run with `&` or in separate terminal). |
161
- | `wikifier update-maps` | `[--full] [--directory=...] [--max-files=N] [--sh]` | Rebuild `library.md` + import cache (pure-Python full pipeline by default; `--sh` = legacy shell fallback). |
161
+ | `wikifier update-maps` | `[--full] [--directory=...] [--max-files=N]` | Rebuild `library.md` + import cache (single pure-Python pipeline; the shell launcher delegates here). |
162
162
  | `wikifier validate` | — | Ensure every file in monitored_paths has at least a health row. |
163
163
  | `wikifier journal` | `[YYYY-MM-DD]` | Read the journal for a day (default = today). |
164
164
  | `wikifier issues` | `[simple|moderate|high|critical]` | List logged issues by severity. |
@@ -153,5 +153,63 @@ class TestJavaScriptParser(TempProjectTestCase):
153
153
  self.assertEqual(edge.get("resolution_confidence"), "high")
154
154
 
155
155
 
156
+ class TestBarrelLeafExplosionPolicy(TempProjectTestCase):
157
+ """Leaf-explosion policy: name routing + reported cap.
158
+
159
+ A named import through an `export *` barrel must emit the entry edge plus
160
+ only the leaves that export the requested names (precision); imports with
161
+ no usable names (side-effect/namespace/dynamic) fall back to a cap whose
162
+ truncation is reported via barrel_leaf_selection — never silent.
163
+ """
164
+
165
+ LEAVES = 30
166
+
167
+ def setUp(self):
168
+ super().setUp()
169
+ lines = []
170
+ for i in range(self.LEAVES):
171
+ self.write(f"big/leaf{i}.js", f"export const thing{i} = {i};\n")
172
+ lines.append(f"export * from './leaf{i}.js';")
173
+ self.write("big/index.js", "\n".join(lines) + "\n")
174
+ self.write("named.js", "import { thing7 } from './big';\n")
175
+ self.write("nameless.js", "import * as big from './big';\n")
176
+
177
+ def _parse(self, rel):
178
+ self.reset_js_parser_state()
179
+ from wikifier.parsers.javascript import parse_javascript_imports
180
+ return parse_javascript_imports(str(self.root / rel))
181
+
182
+ def test_named_import_routes_to_defining_leaf(self):
183
+ edges = self._parse("named.js")
184
+ paths = [e.get("resolved_path") for e in edges]
185
+ self.assertIn("big/index.js", paths, "entry-barrel edge must be kept")
186
+ self.assertIn("big/leaf7.js", paths, "the leaf defining thing7 must be kept")
187
+ self.assertLessEqual(
188
+ len(edges), 4,
189
+ f"name routing should drop non-matching leaves, got {len(edges)}: {paths}",
190
+ )
191
+ edge = find_edge(edges, raw_module="./big")
192
+ self.assertIn("thing7", edge.get("imported_names") or [])
193
+
194
+ def test_nameless_import_is_capped_with_reported_selection(self):
195
+ import os
196
+ old = os.environ.get("WIKIFIER_BARREL_LEAF_CAP")
197
+ os.environ["WIKIFIER_BARREL_LEAF_CAP"] = "5"
198
+ try:
199
+ edges = self._parse("nameless.js")
200
+ finally:
201
+ if old is None:
202
+ os.environ.pop("WIKIFIER_BARREL_LEAF_CAP", None)
203
+ else:
204
+ os.environ["WIKIFIER_BARREL_LEAF_CAP"] = old
205
+ barrel_edges = [e for e in edges if e.get("via_barrel")]
206
+ self.assertLessEqual(len(barrel_edges), 6, "cap must bound emission (entry + 5 leaves)")
207
+ sels = [e.get("barrel_leaf_selection") for e in edges if e.get("barrel_leaf_selection")]
208
+ self.assertTrue(sels, "truncation must be reported via barrel_leaf_selection")
209
+ self.assertTrue(sels[0].get("truncated"))
210
+ self.assertEqual(sels[0].get("leaves_emitted"), 5)
211
+ self.assertEqual(sels[0].get("leaves_total"), self.LEAVES)
212
+
213
+
156
214
  if __name__ == "__main__":
157
215
  unittest.main(verbosity=2)
@@ -61,4 +61,4 @@ from .contracts import (
61
61
  compute_acs_confidence,
62
62
  )
63
63
 
64
- __version__ = "4.2.0"
64
+ __version__ = "4.3.0"
@@ -381,6 +381,7 @@ def _pair_from_parser_edge(edge: Dict[str, Any], root: Path) -> Optional[Dict[st
381
381
  "confidence_score", "confidence_reasons", "confidence_explanation",
382
382
  "barrel_v2", "resolution_metadata", "strategy", "cdia_v1",
383
383
  "conditional_analysis", "dynamic_analysis", "diagnostic",
384
+ "imported_names", "barrel_leaf_selection",
384
385
  ):
385
386
  v = edge.get(k)
386
387
  if v not in (None, "", [], {}):
@@ -732,8 +733,12 @@ def main():
732
733
  python_primary_requested = True
733
734
  continue # consume, do not forward to sh
734
735
  if a in ("--sh", "--legacy-sh", "--no-python-primary"):
735
- python_primary_requested = False
736
- continue # consume, do not forward to sh
736
+ # The in-shell update-maps implementation was retired (2026-06-10
737
+ # thin-shell rework); wikifier.sh itself now delegates here.
738
+ print("[wikifier] note: --sh is a deprecated no-op — the legacy shell "
739
+ "update-maps path was retired; the Python pipeline always runs.",
740
+ file=sys.stderr)
741
+ continue # consume; stay on the Python pipeline
737
742
  if a in ("update-maps", "update_maps"):
738
743
  is_update_maps_cmd = True
739
744
  stripped_filtered.append(a)
@@ -68,7 +68,15 @@ def save_cache(root: Path, cache: Dict[str, Any]) -> None:
68
68
 
69
69
  Uses file locking (M2-Rem-07) to prevent corruption when multiple
70
70
  agents are running update-maps or health operations concurrently.
71
+
72
+ Set WIKIFIER_DEBUG_SAVES=1 to print each save's call site to stderr —
73
+ the diagnostic for "who keeps rewriting the cache mid-run".
71
74
  """
75
+ if os.environ.get("WIKIFIER_DEBUG_SAVES"):
76
+ import sys as _sys
77
+ import traceback
78
+ frames = "".join(traceback.format_stack()[-4:-1])
79
+ print(f"[save_cache] root={root}\n{frames}", file=_sys.stderr)
72
80
  if locking:
73
81
  with locking.file_lock(root):
74
82
  _do_save_cache(root, cache)
@@ -100,8 +108,11 @@ def _do_save_cache(root: Path, cache: Dict[str, Any]) -> None:
100
108
  cache[k] = prev[k]
101
109
  except Exception:
102
110
  pass # never let preservation break the save itself
111
+ # Compact separators: the cache is machine-consumed only, and indent=2
112
+ # made barrel-heavy caches ~2-3x larger and proportionally slower to
113
+ # write (observed: 274MB on Babylon.js).
103
114
  with open(cache_path, "w", encoding="utf-8") as f:
104
- json.dump(cache, f, indent=2, ensure_ascii=False)
115
+ json.dump(cache, f, ensure_ascii=False, separators=(",", ":"))
105
116
 
106
117
 
107
118
  def get_file_data(cache: Dict[str, Any], rel_path: str) -> Optional[Dict[str, Any]]:
@@ -342,6 +342,10 @@ class BarrelResolutionCache:
342
342
  def __post_init__(self) -> None:
343
343
  self.resolutions = dict(self.resolutions or {})
344
344
  self.file_index = dict(self.file_index or {})
345
+ # Transient set views over persisted membership lists (never serialized;
346
+ # see _membership). Without these, store() membership tests were linear
347
+ # scans that went quadratic across a run on barrels with many importers.
348
+ self._fast_sets: Dict[str, set] = {}
345
349
 
346
350
  @classmethod
347
351
  def from_cache(cls, cache: Dict[str, Any]) -> "BarrelResolutionCache":
@@ -358,6 +362,16 @@ class BarrelResolutionCache:
358
362
  state); an explicit empty dict therefore remains the way to express
359
363
  intentional clearing (prune-to-zero, clear()).
360
364
  """
365
+ # Stable output: importer/chain lists are append-ordered in memory for
366
+ # speed; sort once here so the persisted form is deterministic.
367
+ for entry in self.resolutions.values():
368
+ if isinstance(entry.get("importers"), list):
369
+ entry["importers"] = sorted(set(entry["importers"]))
370
+ for idxe in self.file_index.values():
371
+ if isinstance(idxe.get("importers"), list):
372
+ idxe["importers"] = sorted(set(idxe["importers"]))
373
+ if isinstance(idxe.get("chain_ids"), list):
374
+ idxe["chain_ids"] = sorted(set(idxe["chain_ids"]))
361
375
  cache["_barrel_resolutions"] = self.resolutions or {}
362
376
  cache["_barrel_file_index"] = self.file_index or {}
363
377
 
@@ -372,6 +386,58 @@ class BarrelResolutionCache:
372
386
  def get(self, chain_id: str) -> Optional[Dict[str, Any]]:
373
387
  return self.resolutions.get(chain_id)
374
388
 
389
+ @staticmethod
390
+ def _lean_results(results: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
391
+ """Compress chain results for persistence.
392
+
393
+ Stored results exist so a cache hit can replay the expansion (and so
394
+ name routing on hits still sees the full leaf set) — they do NOT need
395
+ the heavy per-leaf payloads (resolution_metadata, barrel_v2 hop blobs,
396
+ analysis fields), which the emission layer rebuilds or defaults.
397
+ Those payloads were the dominant weight of barrel-heavy caches
398
+ (274MB import_cache.json on Babylon.js). Leaves are also deduped by
399
+ resolved_path (BREE can reach the same leaf via multiple hop paths).
400
+ """
401
+ lean: List[Dict[str, Any]] = []
402
+ seen: set = set()
403
+ for r in results or []:
404
+ if not isinstance(r, dict):
405
+ continue
406
+ rp = r.get("resolved_path")
407
+ if rp:
408
+ if rp in seen:
409
+ continue
410
+ seen.add(rp)
411
+ slim = {}
412
+ for k in (
413
+ "module", "resolved_path", "via_barrel", "barrel_chain",
414
+ "barrel_depth", "is_conditional", "conditional_context",
415
+ "barrel_detector",
416
+ ):
417
+ v = r.get(k)
418
+ if v is not None:
419
+ slim[k] = v
420
+ lean.append(slim)
421
+ return lean
422
+
423
+ @staticmethod
424
+ def _lean_hops(hops: Optional[List[Any]]) -> List[Dict[str, Any]]:
425
+ """Keep only primitive hop fields (drop per-hop metadata dicts)."""
426
+ out: List[Dict[str, Any]] = []
427
+ for h in hops or []:
428
+ if isinstance(h, dict):
429
+ out.append({k: v for k, v in h.items() if not isinstance(v, (dict, list))})
430
+ return out
431
+
432
+ def _membership(self, key: str, current: List[str]) -> set:
433
+ """Transient set view over a persisted list (membership tests during
434
+ store() were linear scans — quadratic across a run on popular barrels)."""
435
+ s = self._fast_sets.get(key)
436
+ if s is None:
437
+ s = set(current)
438
+ self._fast_sets[key] = s
439
+ return s
440
+
375
441
  def store(
376
442
  self,
377
443
  *,
@@ -433,15 +499,19 @@ class BarrelResolutionCache:
433
499
  snap[ck or str(k)] = 0.0
434
500
  cid = chain_id or self._make_chain_id(bc, start_specifier)
435
501
  entry = self.resolutions.get(cid, {})
436
- # merge importers
437
- existing_imps = set(entry.get("importers", []))
438
- existing_imps.update(imps)
502
+ # merge importers (set-backed membership; lists stay the persisted form)
503
+ imp_list = entry.get("importers", [])
504
+ imp_set = self._membership(f"res:{cid}", imp_list)
505
+ for imp in imps:
506
+ if imp and imp not in imp_set:
507
+ imp_set.add(imp)
508
+ imp_list.append(imp)
439
509
  entry.update({
440
510
  "chain_id": cid,
441
- "importers": sorted(existing_imps),
511
+ "importers": imp_list,
442
512
  "barrel_chain": bc or entry.get("barrel_chain", []),
443
- "hops": hops or entry.get("hops", []),
444
- "results": results or entry.get("results", []),
513
+ "hops": self._lean_hops(hops) if hops else entry.get("hops", []),
514
+ "results": self._lean_results(results) if results else entry.get("results", []),
445
515
  "start_specifier": start_specifier or entry.get("start_specifier", ""),
446
516
  "detector_used": detector_used or entry.get("detector_used", "unknown"),
447
517
  "is_partial": is_partial or entry.get("is_partial", False),
@@ -461,10 +531,14 @@ class BarrelResolutionCache:
461
531
  if fkey not in self.file_index:
462
532
  self.file_index[fkey] = {"chain_ids": [], "importers": []}
463
533
  idxe = self.file_index[fkey]
464
- if cid not in idxe["chain_ids"]:
534
+ cid_set = self._membership(f"idx_c:{fkey}", idxe["chain_ids"])
535
+ if cid not in cid_set:
536
+ cid_set.add(cid)
465
537
  idxe["chain_ids"].append(cid)
538
+ imp_idx_set = self._membership(f"idx_i:{fkey}", idxe["importers"])
466
539
  for imp in imps:
467
- if imp and imp not in idxe["importers"]:
540
+ if imp and imp not in imp_idx_set:
541
+ imp_idx_set.add(imp)
468
542
  idxe["importers"].append(imp)
469
543
 
470
544
  return cid