wikifier 4.5.3__tar.gz → 4.5.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wikifier-4.5.3/wikifier.egg-info → wikifier-4.5.5}/PKG-INFO +15 -3
- {wikifier-4.5.3 → wikifier-4.5.5}/README.md +14 -2
- {wikifier-4.5.3 → wikifier-4.5.5}/skills/run.md +2 -1
- {wikifier-4.5.3 → wikifier-4.5.5}/tests/test_gap_closure.py +66 -0
- wikifier-4.5.5/tests/test_multi_lang_parsers.py +104 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/__init__.py +1 -1
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/cli.py +151 -19
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/health.py +128 -77
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/mcp/server.py +14 -2
- wikifier-4.5.5/wikifier/parsers/__init__.py +23 -0
- wikifier-4.5.5/wikifier/parsers/_edge.py +71 -0
- wikifier-4.5.5/wikifier/parsers/c_cpp.py +83 -0
- wikifier-4.5.5/wikifier/parsers/csharp.py +64 -0
- wikifier-4.5.5/wikifier/parsers/go_lang.py +87 -0
- wikifier-4.5.5/wikifier/parsers/java.py +64 -0
- wikifier-4.5.5/wikifier/parsers/rust.py +133 -0
- {wikifier-4.5.3 → wikifier-4.5.5/wikifier.egg-info}/PKG-INFO +15 -3
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier.egg-info/SOURCES.txt +13 -0
- wikifier-4.5.3/wikifier/parsers/__init__.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/CONTRIBUTING.md +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/LICENSE +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/MANIFEST.in +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/diagnostics.html +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/docs/spec.md +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/index.html +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/pyproject.toml +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/setup.cfg +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/tests/test_barrel_invalidation.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/tests/test_health.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/tests/test_import_cache.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/tests/test_parsers.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/tests/test_selftest_wrappers.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/__main__.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/contracts.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/daemon.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/diagnostics.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/import_cache.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/index.html +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/library.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/locking.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/mcp/__init__.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/parsers/bree.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/parsers/cdia.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/parsers/javascript.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/parsers/python.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/resolution.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/scripts/wikifier.bat +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/scripts/wikifier.ps1 +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/scripts/wikifier.sh +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier/serve.py +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier.egg-info/dependency_links.txt +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier.egg-info/entry_points.txt +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier.egg-info/requires.txt +0 -0
- {wikifier-4.5.3 → wikifier-4.5.5}/wikifier.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wikifier
|
|
3
|
-
Version: 4.5.
|
|
3
|
+
Version: 4.5.5
|
|
4
4
|
Summary: Zero-dependency agent-to-agent codebase wiki for LLMs and AI agents. Autonomously maintained via record-change and mark-green for token-efficient lookup of files, dependencies, health, and summaries — across tiny scripts to 50k+ monorepos. Optional MCP server with rich tools for agents.
|
|
5
5
|
Author-email: Aron Amos <aron@example.com>
|
|
6
6
|
Maintainer: Aron Amos
|
|
@@ -40,7 +40,19 @@ Dynamic: license-file
|
|
|
40
40
|
|
|
41
41
|
Wikifier is an **agent-to-agent** tool: it builds a living map of a project (health matrix, dependency graph, short file summaries) and agents keep that map current as they work. Humans can peek via a small dashboard; the product is the agent loop, not a general docs site or IDE.
|
|
42
42
|
|
|
43
|
-
Works from small scripts to large monorepos. **Deep import maps**
|
|
43
|
+
Works from small scripts to large monorepos. **Deep import/include maps** (zero-dep regex parsers):
|
|
44
|
+
|
|
45
|
+
| Language | Extensions | Notes |
|
|
46
|
+
|----------|------------|--------|
|
|
47
|
+
| Python | `.py` | full ACS/CDIA path |
|
|
48
|
+
| JavaScript / TypeScript | `.js` `.ts` `.jsx` `.tsx` | barrels (BREE), dynamic/CDIA |
|
|
49
|
+
| Rust | `.rs` | `use` / `mod` / `extern crate` |
|
|
50
|
+
| Go | `.go` | `import` / import blocks |
|
|
51
|
+
| C / C++ | `.c` `.h` `.cpp` `.cc` `.cxx` `.hpp` `.hh` | `#include` (local + system) |
|
|
52
|
+
| C# | `.cs` | `using` namespaces |
|
|
53
|
+
| Java | `.java` | `import` / `import static` |
|
|
54
|
+
|
|
55
|
+
Health/journal still work for any monitored path. Parsers are **pragmatic regex** (not full cargo/`go.mod`/classpath/`-I` resolution). Prefer **lean `monitored_paths.txt`** on huge monorepos; raise dirty cap with `WIKIFIER_CHECK_CHANGES_MAX` (default 2000) only when needed.
|
|
44
56
|
|
|
45
57
|
## Why
|
|
46
58
|
|
|
@@ -90,7 +102,7 @@ MCP **Core 6** (start every session): `get_project_status`, `check_changes`, `ge
|
|
|
90
102
|
|
|
91
103
|
## What you get
|
|
92
104
|
|
|
93
|
-
- **Import analysis** — Python
|
|
105
|
+
- **Import analysis** — Python, JS/TS (ESM/CJS, barrels), Rust, Go, C/C++ includes, C# usings; per-edge confidence; name-routed barrel expansion for TS/JS
|
|
94
106
|
- **Incremental pipeline** — pure-Python `update-maps`: dirty parse → import cache → reverse deps → cycles → `library.md`
|
|
95
107
|
- **Selective agent work** — health + suggest bias to 🔴/🟡 only; ACS *actionable* low-conf excludes stdlib/external noise
|
|
96
108
|
- **Scale** — reverse index + barrel invalidation so one edit doesn’t re-scan the monorepo
|
|
@@ -8,7 +8,19 @@
|
|
|
8
8
|
|
|
9
9
|
Wikifier is an **agent-to-agent** tool: it builds a living map of a project (health matrix, dependency graph, short file summaries) and agents keep that map current as they work. Humans can peek via a small dashboard; the product is the agent loop, not a general docs site or IDE.
|
|
10
10
|
|
|
11
|
-
Works from small scripts to large monorepos. **Deep import maps**
|
|
11
|
+
Works from small scripts to large monorepos. **Deep import/include maps** (zero-dep regex parsers):
|
|
12
|
+
|
|
13
|
+
| Language | Extensions | Notes |
|
|
14
|
+
|----------|------------|--------|
|
|
15
|
+
| Python | `.py` | full ACS/CDIA path |
|
|
16
|
+
| JavaScript / TypeScript | `.js` `.ts` `.jsx` `.tsx` | barrels (BREE), dynamic/CDIA |
|
|
17
|
+
| Rust | `.rs` | `use` / `mod` / `extern crate` |
|
|
18
|
+
| Go | `.go` | `import` / import blocks |
|
|
19
|
+
| C / C++ | `.c` `.h` `.cpp` `.cc` `.cxx` `.hpp` `.hh` | `#include` (local + system) |
|
|
20
|
+
| C# | `.cs` | `using` namespaces |
|
|
21
|
+
| Java | `.java` | `import` / `import static` |
|
|
22
|
+
|
|
23
|
+
Health/journal still work for any monitored path. Parsers are **pragmatic regex** (not full cargo/`go.mod`/classpath/`-I` resolution). Prefer **lean `monitored_paths.txt`** on huge monorepos; raise dirty cap with `WIKIFIER_CHECK_CHANGES_MAX` (default 2000) only when needed.
|
|
12
24
|
|
|
13
25
|
## Why
|
|
14
26
|
|
|
@@ -58,7 +70,7 @@ MCP **Core 6** (start every session): `get_project_status`, `check_changes`, `ge
|
|
|
58
70
|
|
|
59
71
|
## What you get
|
|
60
72
|
|
|
61
|
-
- **Import analysis** — Python
|
|
73
|
+
- **Import analysis** — Python, JS/TS (ESM/CJS, barrels), Rust, Go, C/C++ includes, C# usings; per-edge confidence; name-routed barrel expansion for TS/JS
|
|
62
74
|
- **Incremental pipeline** — pure-Python `update-maps`: dirty parse → import cache → reverse deps → cycles → `library.md`
|
|
63
75
|
- **Selective agent work** — health + suggest bias to 🔴/🟡 only; ACS *actionable* low-conf excludes stdlib/external noise
|
|
64
76
|
- **Scale** — reverse index + barrel invalidation so one edit doesn’t re-scan the monorepo
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
- **First-run:** `init` → `update-maps` → `health --summary` → `suggest_next_actions`. Map is automatic; wiki *prose* is agent-filled over time.
|
|
13
13
|
- **Steady-state selective work:** only 🔴/🟡 (never re-wiki 🟢 unless required). ACS suggestions use `actionable_low_conf_edges` (stdlib/external bare demoted).
|
|
14
14
|
- **Deletions:** `record-deletion` + check-changes ghost detection (missing paths → Red DELETED).
|
|
15
|
-
- **Languages:** deep import maps = Python
|
|
15
|
+
- **Languages:** deep import maps = Python, JS/TS, Rust, Go, C/C++, C#, Java (regex; not full toolchain resolution). journal/pending = audit queue, not Jira.
|
|
16
|
+
- **Huge monorepos:** lean `monitored_paths.txt` (package roots, not bare `.`). Optional `WIKIFIER_CHECK_CHANGES_MAX` (default 2000). `update-maps` seeds Yellow health stubs for newly parsed files (map-first).
|
|
16
17
|
- **Monitor:** optional background `monitor`/`daemon` for mtime heartbeat; agents still own wiki/mark-green.
|
|
17
18
|
- Health module: use `importlib.import_module("wikifier.health")` or `wikifier.health_module` — `from wikifier import health` is the *function*.
|
|
18
19
|
|
|
@@ -123,6 +123,72 @@ class TestGhostAndDeletionG7(TempProjectTestCase):
|
|
|
123
123
|
self.assertIn("🔴", entries["gone.py"]["status"])
|
|
124
124
|
self.assertIn("DELETED", entries["gone.py"]["reason"])
|
|
125
125
|
|
|
126
|
+
def test_record_deletion_rejects_flag_paths(self):
|
|
127
|
+
res = cli.record_deletion("--help", "oops", project_root=self.root)
|
|
128
|
+
self.assertFalse(res.get("success"))
|
|
129
|
+
self.assertIn("flag", (res.get("error") or "").lower())
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class TestPendingAndPollutionHygiene(TempProjectTestCase):
|
|
133
|
+
def test_pending_empty_marker_not_counted_and_no_dual_state(self):
|
|
134
|
+
health_mod = importlib.import_module("wikifier.health")
|
|
135
|
+
# Dual-state pollution (historical bug): empty marker + real items
|
|
136
|
+
p = self.root / "pending_updates.md"
|
|
137
|
+
p.write_text(
|
|
138
|
+
"# Pending Updates\n\n(no active items)\n- src/a.py: review\n",
|
|
139
|
+
encoding="utf-8",
|
|
140
|
+
)
|
|
141
|
+
self.assertEqual(health_mod.count_pending(self.root), 1)
|
|
142
|
+
# Normalize via remove of the only item → empty marker only
|
|
143
|
+
health_mod.remove_from_pending(self.root, "src/a.py")
|
|
144
|
+
text = p.read_text(encoding="utf-8")
|
|
145
|
+
self.assertIn("no pending", text.lower())
|
|
146
|
+
self.assertNotIn("- src/a.py", text)
|
|
147
|
+
self.assertEqual(health_mod.count_pending(self.root), 0)
|
|
148
|
+
# Add clears empty marker
|
|
149
|
+
health_mod.add_to_pending(self.root, "src/b.py", "need wiki")
|
|
150
|
+
text2 = p.read_text(encoding="utf-8")
|
|
151
|
+
self.assertIn("- src/b.py:", text2)
|
|
152
|
+
self.assertNotIn("no pending", text2.lower())
|
|
153
|
+
self.assertNotIn("no active", text2.lower())
|
|
154
|
+
self.assertEqual(health_mod.count_pending(self.root), 1)
|
|
155
|
+
summary = health_mod.get_summary(self.root)
|
|
156
|
+
self.assertEqual(summary["pending_updates"], 1)
|
|
157
|
+
|
|
158
|
+
def test_superseded_and_flag_deleted_keys_pruned(self):
|
|
159
|
+
health_mod = importlib.import_module("wikifier.health")
|
|
160
|
+
# Inject pollution keys directly then save
|
|
161
|
+
data = {
|
|
162
|
+
"version": 2,
|
|
163
|
+
"last_updated": "2026-01-01",
|
|
164
|
+
"entries": {
|
|
165
|
+
"M5.3 Cycle1 evidence append: 3 subs spawned": {
|
|
166
|
+
"status": "🔴 Red",
|
|
167
|
+
"reason": "DELETED — Historical early M5.3 launch note",
|
|
168
|
+
"last_updated": "2026-01-01",
|
|
169
|
+
},
|
|
170
|
+
"--help": {
|
|
171
|
+
"status": "🔴 Red",
|
|
172
|
+
"reason": "DELETED — removed",
|
|
173
|
+
"last_updated": "2026-01-01",
|
|
174
|
+
},
|
|
175
|
+
"real.py": {
|
|
176
|
+
"status": "🟢 Green",
|
|
177
|
+
"reason": "ok",
|
|
178
|
+
"last_updated": "2026-01-01",
|
|
179
|
+
},
|
|
180
|
+
},
|
|
181
|
+
}
|
|
182
|
+
health_mod.save_health(self.root, data)
|
|
183
|
+
entries = health_mod.load_health(self.root)["entries"]
|
|
184
|
+
self.assertNotIn("M5.3 Cycle1 evidence append: 3 subs spawned", entries)
|
|
185
|
+
self.assertNotIn("--help", entries)
|
|
186
|
+
self.assertIn("real.py", entries)
|
|
187
|
+
# Real path DELETED audit is kept
|
|
188
|
+
health_mod.upsert_entry(self.root, "gone_real.py", "🔴 Red", "DELETED — intentional")
|
|
189
|
+
entries2 = health_mod.load_health(self.root)["entries"]
|
|
190
|
+
self.assertIn("gone_real.py", entries2)
|
|
191
|
+
|
|
126
192
|
|
|
127
193
|
if __name__ == "__main__":
|
|
128
194
|
unittest.main()
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Multi-language parser smoke tests (Rust, Go, C/C++, C#)."""
|
|
2
|
+
|
|
3
|
+
import unittest
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from tests._base import TempProjectTestCase
|
|
7
|
+
|
|
8
|
+
from wikifier.parsers import rust, go_lang, c_cpp, csharp, java
|
|
9
|
+
from wikifier.cli import run_full_update, check_changes
|
|
10
|
+
import importlib
|
|
11
|
+
import os
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TestRustParser(TempProjectTestCase):
|
|
15
|
+
def test_use_and_mod(self):
|
|
16
|
+
self.write("lib.rs", "mod foo;\nuse crate::foo::Bar;\nuse std::io;\n")
|
|
17
|
+
self.write("foo.rs", "pub struct Bar;\n")
|
|
18
|
+
edges = rust.parse_rust_imports(str(self.root / "lib.rs"))
|
|
19
|
+
kinds = {e.get("statement_type") for e in edges}
|
|
20
|
+
self.assertIn("mod", kinds)
|
|
21
|
+
self.assertIn("use", kinds)
|
|
22
|
+
mod_e = next(e for e in edges if e.get("statement_type") == "mod")
|
|
23
|
+
self.assertTrue(mod_e.get("resolved_path") and mod_e["resolved_path"].endswith("foo.rs"))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TestGoParser(TempProjectTestCase):
|
|
27
|
+
def test_import_block(self):
|
|
28
|
+
self.write("main.go", 'package main\nimport (\n\t"fmt"\n\t"os"\n)\n')
|
|
29
|
+
edges = go_lang.parse_go_imports(str(self.root / "main.go"))
|
|
30
|
+
mods = {e["raw_module"] for e in edges}
|
|
31
|
+
self.assertIn("fmt", mods)
|
|
32
|
+
self.assertIn("os", mods)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TestCParser(TempProjectTestCase):
|
|
36
|
+
def test_local_and_system_include(self):
|
|
37
|
+
self.write("a.c", '#include "b.h"\n#include <stdio.h>\n')
|
|
38
|
+
self.write("b.h", "int x;\n")
|
|
39
|
+
edges = c_cpp.parse_c_cpp_imports(str(self.root / "a.c"))
|
|
40
|
+
self.assertGreaterEqual(len(edges), 2)
|
|
41
|
+
local = next(e for e in edges if e["raw_module"] == "b.h")
|
|
42
|
+
self.assertTrue(local.get("resolved_path"))
|
|
43
|
+
sys_e = next(e for e in edges if e["raw_module"] == "stdio.h")
|
|
44
|
+
self.assertEqual((sys_e.get("diagnostic") or {}).get("category"), "external_or_bare")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TestCsharpParser(TempProjectTestCase):
|
|
48
|
+
def test_using(self):
|
|
49
|
+
self.write("A.cs", "using System;\nusing System.IO;\nusing MyApp.Core;\n")
|
|
50
|
+
edges = csharp.parse_csharp_imports(str(self.root / "A.cs"))
|
|
51
|
+
mods = {e["raw_module"] for e in edges}
|
|
52
|
+
self.assertIn("System", mods)
|
|
53
|
+
self.assertIn("MyApp.Core", mods)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class TestJavaParser(TempProjectTestCase):
|
|
57
|
+
def test_import(self):
|
|
58
|
+
self.write("A.java", "import java.util.List;\nimport com.foo.Bar;\n")
|
|
59
|
+
edges = java.parse_java_imports(str(self.root / "A.java"))
|
|
60
|
+
mods = {e["raw_module"] for e in edges}
|
|
61
|
+
self.assertIn("java.util.List", mods)
|
|
62
|
+
self.assertIn("com.foo.Bar", mods)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class TestPipelineMultiLang(TempProjectTestCase):
|
|
66
|
+
def test_run_full_update_parses_all_langs(self):
|
|
67
|
+
self.write("a.py", "import os\n")
|
|
68
|
+
self.write("b.rs", "mod x;\n")
|
|
69
|
+
self.write("x.rs", "pub fn f() {}\n")
|
|
70
|
+
self.write("c.go", 'package p\nimport "fmt"\n')
|
|
71
|
+
self.write("d.c", '#include "e.h"\n')
|
|
72
|
+
self.write("e.h", "int y;\n")
|
|
73
|
+
self.write("f.cs", "using System;\n")
|
|
74
|
+
self.write("G.java", "import java.io.File;\n")
|
|
75
|
+
res = run_full_update(root=self.root, force_full=True, verbose=False)
|
|
76
|
+
self.assertTrue(res.get("success"))
|
|
77
|
+
self.assertGreaterEqual(res.get("files_parsed", 0), 5)
|
|
78
|
+
self.assertIn("languages_parsed", res)
|
|
79
|
+
self.assertGreaterEqual(res.get("health_stubs_seeded", 0), 1)
|
|
80
|
+
health_mod = importlib.import_module("wikifier.health")
|
|
81
|
+
entries = health_mod.load_health(self.root).get("entries") or {}
|
|
82
|
+
self.assertTrue(any(k.endswith((".py", ".rs", ".java", ".cs", ".go", ".c")) for k in entries))
|
|
83
|
+
|
|
84
|
+
def test_check_changes_cap_env(self):
|
|
85
|
+
self.write("monitored_paths.txt", ".\n")
|
|
86
|
+
for i in range(5):
|
|
87
|
+
self.write(f"f{i}.py", f"x={i}\n")
|
|
88
|
+
run_full_update(root=self.root, force_full=True, verbose=False)
|
|
89
|
+
import time
|
|
90
|
+
time.sleep(0.05)
|
|
91
|
+
for i in range(5):
|
|
92
|
+
(self.root / f"f{i}.py").write_text(f"x={i + 1}\n")
|
|
93
|
+
os.environ["WIKIFIER_CHECK_CHANGES_MAX"] = "2"
|
|
94
|
+
try:
|
|
95
|
+
r = check_changes(project_root=self.root)
|
|
96
|
+
self.assertTrue(r.get("success"))
|
|
97
|
+
self.assertLessEqual(r.get("changes_detected", 0), 2)
|
|
98
|
+
self.assertTrue(r.get("dirty_truncated") or (r.get("dirty_total") or 0) <= 2)
|
|
99
|
+
finally:
|
|
100
|
+
os.environ.pop("WIKIFIER_CHECK_CHANGES_MAX", None)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
if __name__ == "__main__":
|
|
104
|
+
unittest.main()
|
|
@@ -174,7 +174,13 @@ def _collect_candidate_source_files(root: Path) -> List[Path]:
|
|
|
174
174
|
".pnpm", ".yarn", ".store", "store", "virtual-store", ".pnp",
|
|
175
175
|
".pnp.cjs", "node_modules/.pnpm", "node_modules/.yarn"
|
|
176
176
|
}
|
|
177
|
-
|
|
177
|
+
# Deep-map languages (agent multi-lang dogfood)
|
|
178
|
+
exts = (
|
|
179
|
+
".py", ".js", ".ts", ".jsx", ".tsx",
|
|
180
|
+
".rs", ".go",
|
|
181
|
+
".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".hh",
|
|
182
|
+
".cs", ".java",
|
|
183
|
+
)
|
|
178
184
|
try:
|
|
179
185
|
root = Path(root).resolve()
|
|
180
186
|
except Exception:
|
|
@@ -499,6 +505,26 @@ def run_full_update(
|
|
|
499
505
|
# === 3. Parse every dirty file (in-process, BREE batched) ===
|
|
500
506
|
from .parsers import javascript as js_parser
|
|
501
507
|
from .parsers import python as py_parser
|
|
508
|
+
try:
|
|
509
|
+
from .parsers import rust as rust_parser
|
|
510
|
+
except Exception:
|
|
511
|
+
rust_parser = None
|
|
512
|
+
try:
|
|
513
|
+
from .parsers import go_lang as go_parser
|
|
514
|
+
except Exception:
|
|
515
|
+
go_parser = None
|
|
516
|
+
try:
|
|
517
|
+
from .parsers import c_cpp as c_cpp_parser
|
|
518
|
+
except Exception:
|
|
519
|
+
c_cpp_parser = None
|
|
520
|
+
try:
|
|
521
|
+
from .parsers import csharp as csharp_parser
|
|
522
|
+
except Exception:
|
|
523
|
+
csharp_parser = None
|
|
524
|
+
try:
|
|
525
|
+
from .parsers import java as java_parser
|
|
526
|
+
except Exception:
|
|
527
|
+
java_parser = None
|
|
502
528
|
try:
|
|
503
529
|
from .parsers import bree as bree_mod
|
|
504
530
|
except Exception:
|
|
@@ -521,10 +547,28 @@ def run_full_update(
|
|
|
521
547
|
except Exception:
|
|
522
548
|
return None
|
|
523
549
|
|
|
550
|
+
def _parse_file(fstr: str, low: str):
|
|
551
|
+
if low.endswith((".js", ".ts", ".jsx", ".tsx")):
|
|
552
|
+
return js_parser.parse_javascript_imports(fstr) or []
|
|
553
|
+
if low.endswith(".py"):
|
|
554
|
+
return py_parser.parse_python_imports(fstr) or []
|
|
555
|
+
if low.endswith(".rs") and rust_parser is not None:
|
|
556
|
+
return rust_parser.parse_rust_imports(fstr) or []
|
|
557
|
+
if low.endswith(".go") and go_parser is not None:
|
|
558
|
+
return go_parser.parse_go_imports(fstr) or []
|
|
559
|
+
if low.endswith((".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".hh")) and c_cpp_parser is not None:
|
|
560
|
+
return c_cpp_parser.parse_c_cpp_imports(fstr) or []
|
|
561
|
+
if low.endswith(".cs") and csharp_parser is not None:
|
|
562
|
+
return csharp_parser.parse_csharp_imports(fstr) or []
|
|
563
|
+
if low.endswith(".java") and java_parser is not None:
|
|
564
|
+
return java_parser.parse_java_imports(fstr) or []
|
|
565
|
+
return None
|
|
566
|
+
|
|
524
567
|
new_entries: Dict[str, Dict[str, Any]] = {}
|
|
525
568
|
edges_total = 0
|
|
526
569
|
parsed_count = 0
|
|
527
570
|
parse_errors: List[Dict[str, str]] = []
|
|
571
|
+
lang_counts: Dict[str, int] = {}
|
|
528
572
|
|
|
529
573
|
if bree_mod is not None:
|
|
530
574
|
try:
|
|
@@ -536,11 +580,8 @@ def run_full_update(
|
|
|
536
580
|
fstr = str(f)
|
|
537
581
|
low = fstr.lower()
|
|
538
582
|
try:
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
elif low.endswith(".py"):
|
|
542
|
-
edges = py_parser.parse_python_imports(fstr) or []
|
|
543
|
-
else:
|
|
583
|
+
edges = _parse_file(fstr, low)
|
|
584
|
+
if edges is None:
|
|
544
585
|
continue
|
|
545
586
|
except Exception as pe:
|
|
546
587
|
parse_errors.append({"file": fstr, "error": f"{type(pe).__name__}: {pe}"})
|
|
@@ -557,6 +598,8 @@ def run_full_update(
|
|
|
557
598
|
}
|
|
558
599
|
parsed_count += 1
|
|
559
600
|
edges_total += len(pairs)
|
|
601
|
+
ext = Path(low).suffix.lower() or "unknown"
|
|
602
|
+
lang_counts[ext] = lang_counts.get(ext, 0) + 1
|
|
560
603
|
if verbose and parsed_count % 200 == 0:
|
|
561
604
|
print(f"[run_full_update] parsed {parsed_count}/{len(dirty)}")
|
|
562
605
|
finally:
|
|
@@ -568,10 +611,28 @@ def run_full_update(
|
|
|
568
611
|
|
|
569
612
|
result["files_parsed"] = parsed_count
|
|
570
613
|
result["edges_persisted"] = edges_total
|
|
614
|
+
result["languages_parsed"] = lang_counts
|
|
571
615
|
if parse_errors:
|
|
572
616
|
result["parse_errors"] = parse_errors[:10]
|
|
573
617
|
result["parse_error_count"] = len(parse_errors)
|
|
574
618
|
|
|
619
|
+
# Seed health stubs for newly parsed files (map-first wiki: agents fill prose later)
|
|
620
|
+
health_seeded = 0
|
|
621
|
+
if _health_mod is not None and new_entries:
|
|
622
|
+
try:
|
|
623
|
+
existing = (_health_mod.load_health(root) or {}).get("entries") or {}
|
|
624
|
+
for rel in new_entries:
|
|
625
|
+
if rel in existing:
|
|
626
|
+
continue
|
|
627
|
+
_health_mod.upsert_entry(
|
|
628
|
+
root, rel, "🟡 Yellow",
|
|
629
|
+
"Initial stub — parsed into map; agent should wiki + mark-green when editing"
|
|
630
|
+
)
|
|
631
|
+
health_seeded += 1
|
|
632
|
+
except Exception:
|
|
633
|
+
health_seeded = 0
|
|
634
|
+
result["health_stubs_seeded"] = health_seeded
|
|
635
|
+
|
|
575
636
|
# === 4. Persist (single save; reload first to pick up the barrel flush) ===
|
|
576
637
|
cache = ic.load_cache(root) or {}
|
|
577
638
|
cache.update(new_entries)
|
|
@@ -744,16 +805,40 @@ def main():
|
|
|
744
805
|
if res.get("message"):
|
|
745
806
|
print(res["message"])
|
|
746
807
|
return 0 if res.get("success", True) else 1
|
|
747
|
-
if _cmd0 == "record-change"
|
|
808
|
+
if _cmd0 == "record-change":
|
|
809
|
+
if not _args or _args[0] in ("--help", "-h", "help"):
|
|
810
|
+
print('Usage: wikifier record-change <file> "<reason>"')
|
|
811
|
+
return 0
|
|
812
|
+
if _args[0].startswith("-") or len(_args) < 2:
|
|
813
|
+
print('Usage: wikifier record-change <file> "<reason>"', file=sys.stderr)
|
|
814
|
+
return 1
|
|
748
815
|
res = record_change(_args[0], " ".join(_args[1:]), project_root=project_root)
|
|
749
816
|
print(res.get("message") or res)
|
|
750
817
|
return 0 if res.get("success") else 1
|
|
751
|
-
if _cmd0 == "mark-green"
|
|
818
|
+
if _cmd0 == "mark-green":
|
|
819
|
+
if not _args or _args[0] in ("--help", "-h", "help"):
|
|
820
|
+
print("Usage: wikifier mark-green <file> [reason]")
|
|
821
|
+
return 0
|
|
822
|
+
if _args[0].startswith("-"):
|
|
823
|
+
print("Usage: wikifier mark-green <file> [reason]", file=sys.stderr)
|
|
824
|
+
return 1
|
|
752
825
|
reason = " ".join(_args[1:]) if len(_args) > 1 else ""
|
|
753
826
|
res = mark_green(_args[0], reason, project_root=project_root)
|
|
754
827
|
print(res.get("message") or res)
|
|
755
828
|
return 0 if res.get("success") else 1
|
|
756
|
-
if _cmd0 == "record-deletion"
|
|
829
|
+
if _cmd0 == "record-deletion":
|
|
830
|
+
# Guard: `record-deletion --help` must not treat `--help` as a file path
|
|
831
|
+
# (that polluted health with a 🔴 DELETED "--help" key).
|
|
832
|
+
if not _args or _args[0] in ("--help", "-h", "help"):
|
|
833
|
+
print('Usage: wikifier record-deletion <file> "<reason>"')
|
|
834
|
+
return 0
|
|
835
|
+
if _args[0].startswith("-"):
|
|
836
|
+
print(
|
|
837
|
+
'Usage: wikifier record-deletion <file> "<reason>"\n'
|
|
838
|
+
" <file> must be a project path, not a flag.",
|
|
839
|
+
file=sys.stderr,
|
|
840
|
+
)
|
|
841
|
+
return 1
|
|
757
842
|
reason = " ".join(_args[1:]) if len(_args) > 1 else "removed"
|
|
758
843
|
res = record_deletion(_args[0], reason, project_root=project_root)
|
|
759
844
|
print(res.get("message") or res)
|
|
@@ -978,8 +1063,14 @@ def _ensure_journal_entry(root: Path, action: str, file: str, reason: str) -> No
|
|
|
978
1063
|
|
|
979
1064
|
|
|
980
1065
|
def _add_to_pending(root: Path, file: str, msg: str) -> None:
|
|
981
|
-
"""
|
|
1066
|
+
"""Append to pending_updates.md via health helpers (normalized empty/items)."""
|
|
982
1067
|
try:
|
|
1068
|
+
if _health_mod is not None and hasattr(_health_mod, "add_to_pending"):
|
|
1069
|
+
# Caller may already hold project lock (re-entrant).
|
|
1070
|
+
_health_mod._do_add_to_pending(root, file, msg) if hasattr(
|
|
1071
|
+
_health_mod, "_do_add_to_pending"
|
|
1072
|
+
) else _health_mod.add_to_pending(root, file, msg)
|
|
1073
|
+
return
|
|
983
1074
|
p = root / "pending_updates.md"
|
|
984
1075
|
line = f"- {file}: {msg}\n"
|
|
985
1076
|
with open(p, "a", encoding="utf-8") as f:
|
|
@@ -989,8 +1080,14 @@ def _add_to_pending(root: Path, file: str, msg: str) -> None:
|
|
|
989
1080
|
|
|
990
1081
|
|
|
991
1082
|
def _remove_from_pending(root: Path, file: str) -> None:
|
|
992
|
-
"""Best-effort removal (used by mark_green
|
|
1083
|
+
"""Best-effort removal (used by mark_green). Prefers health normalizer."""
|
|
993
1084
|
try:
|
|
1085
|
+
if _health_mod is not None and hasattr(_health_mod, "_do_remove_from_pending"):
|
|
1086
|
+
_health_mod._do_remove_from_pending(root, file)
|
|
1087
|
+
return
|
|
1088
|
+
if _health_mod is not None and hasattr(_health_mod, "remove_from_pending"):
|
|
1089
|
+
_health_mod.remove_from_pending(root, file)
|
|
1090
|
+
return
|
|
994
1091
|
p = root / "pending_updates.md"
|
|
995
1092
|
if p.exists():
|
|
996
1093
|
lines = [ln for ln in p.read_text(encoding="utf-8").splitlines() if file not in ln]
|
|
@@ -1079,11 +1176,28 @@ def check_changes(project_root: Optional[Union[str, Path]] = None) -> Dict[str,
|
|
|
1079
1176
|
except Exception:
|
|
1080
1177
|
pass
|
|
1081
1178
|
|
|
1179
|
+
# Cap is configurable: WIKIFIER_CHECK_CHANGES_MAX (default 2000; was hard 200).
|
|
1180
|
+
# Huge monorepos with monitored_paths=. can still thrash — prefer lean monitored paths.
|
|
1181
|
+
try:
|
|
1182
|
+
max_dirty = int(os.environ.get("WIKIFIER_CHECK_CHANGES_MAX", "2000") or "2000")
|
|
1183
|
+
except ValueError:
|
|
1184
|
+
max_dirty = 2000
|
|
1185
|
+
max_dirty = max(1, min(max_dirty, 50000))
|
|
1186
|
+
try:
|
|
1187
|
+
max_ghosts = int(os.environ.get("WIKIFIER_CHECK_CHANGES_GHOST_MAX", "200") or "200")
|
|
1188
|
+
except ValueError:
|
|
1189
|
+
max_ghosts = 200
|
|
1190
|
+
max_ghosts = max(1, min(max_ghosts, 10000))
|
|
1191
|
+
|
|
1192
|
+
dirty_list = list(dirty or [])
|
|
1193
|
+
dirty_truncated = len(dirty_list) > max_dirty
|
|
1194
|
+
dirty_batch = dirty_list[:max_dirty]
|
|
1195
|
+
|
|
1082
1196
|
changed_count = 0
|
|
1083
1197
|
ghosts_marked = 0
|
|
1084
1198
|
if _health_mod is not None:
|
|
1085
1199
|
root_res = root.resolve()
|
|
1086
|
-
for p in
|
|
1200
|
+
for p in dirty_batch:
|
|
1087
1201
|
try:
|
|
1088
1202
|
pr = Path(p).resolve()
|
|
1089
1203
|
if not str(pr).startswith(str(root_res)):
|
|
@@ -1102,7 +1216,8 @@ def check_changes(project_root: Optional[Union[str, Path]] = None) -> Dict[str,
|
|
|
1102
1216
|
# G7: surface ghost health entries (tracked path missing on disk, not already DELETED)
|
|
1103
1217
|
try:
|
|
1104
1218
|
if hasattr(_health_mod, "find_ghost_entries"):
|
|
1105
|
-
|
|
1219
|
+
ghosts_all = _health_mod.find_ghost_entries(root) or []
|
|
1220
|
+
for g in ghosts_all[:max_ghosts]:
|
|
1106
1221
|
_health_mod.upsert_entry(
|
|
1107
1222
|
root, g, "🔴 Red",
|
|
1108
1223
|
"DELETED — path missing on disk (check_changes ghost detection)"
|
|
@@ -1115,15 +1230,24 @@ def check_changes(project_root: Optional[Union[str, Path]] = None) -> Dict[str,
|
|
|
1115
1230
|
except Exception:
|
|
1116
1231
|
pass
|
|
1117
1232
|
|
|
1233
|
+
msg = (
|
|
1234
|
+
f"Python-primary check_changes complete: {changed_count} files marked/updated"
|
|
1235
|
+
+ (f", {ghosts_marked} ghost(s) marked Red" if ghosts_marked else "")
|
|
1236
|
+
+ ". Health + pending + journal touched."
|
|
1237
|
+
)
|
|
1238
|
+
if dirty_truncated:
|
|
1239
|
+
msg += (
|
|
1240
|
+
f" Note: dirty set truncated to {max_dirty} of {len(dirty_list)} "
|
|
1241
|
+
f"(set WIKIFIER_CHECK_CHANGES_MAX or lean monitored_paths.txt)."
|
|
1242
|
+
)
|
|
1118
1243
|
result.update({
|
|
1119
1244
|
"success": True,
|
|
1120
1245
|
"changes_detected": changed_count,
|
|
1246
|
+
"dirty_total": len(dirty_list),
|
|
1247
|
+
"dirty_truncated": dirty_truncated,
|
|
1248
|
+
"max_dirty": max_dirty,
|
|
1121
1249
|
"ghosts_marked": ghosts_marked,
|
|
1122
|
-
"message":
|
|
1123
|
-
f"Python-primary check_changes complete: {changed_count} files marked/updated"
|
|
1124
|
-
+ (f", {ghosts_marked} ghost(s) marked Red" if ghosts_marked else "")
|
|
1125
|
-
+ ". Health + pending + journal touched."
|
|
1126
|
-
),
|
|
1250
|
+
"message": msg,
|
|
1127
1251
|
})
|
|
1128
1252
|
except Exception as e:
|
|
1129
1253
|
result["error"] = str(e)
|
|
@@ -1177,9 +1301,13 @@ def record_deletion(file: str, reason: str, project_root: Optional[Union[str, Pa
|
|
|
1177
1301
|
|
|
1178
1302
|
G7: marks 🔴 DELETED, pending + journal, and best-effort prunes barrel cache
|
|
1179
1303
|
references so deleted paths do not keep invalidating importers forever.
|
|
1304
|
+
Rejects flag-like paths (`--help`) so CLI misuse cannot pollute health.
|
|
1180
1305
|
"""
|
|
1181
1306
|
root = _get_effective_root(project_root)
|
|
1182
1307
|
result: Dict[str, Any] = {"success": False, "file": file, "project_root": str(root), "action": "deletion"}
|
|
1308
|
+
if not file or str(file).startswith("-") or str(file) in ("--help", "-h", "help"):
|
|
1309
|
+
result["error"] = "file must be a project path, not a flag/empty string"
|
|
1310
|
+
return result
|
|
1183
1311
|
try:
|
|
1184
1312
|
lock_ctx = (locking.file_lock(root) if locking is not None else _nullcontext())
|
|
1185
1313
|
with lock_ctx:
|
|
@@ -1191,7 +1319,11 @@ def record_deletion(file: str, reason: str, project_root: Optional[Union[str, Pa
|
|
|
1191
1319
|
except Exception:
|
|
1192
1320
|
rel = file
|
|
1193
1321
|
if _health_mod is not None:
|
|
1194
|
-
|
|
1322
|
+
# Prefer unlocked upsert when we already hold the project lock.
|
|
1323
|
+
if hasattr(_health_mod, "_do_upsert_entry"):
|
|
1324
|
+
_health_mod._do_upsert_entry(root, rel, "🔴 Red", f"DELETED — {reason}")
|
|
1325
|
+
else:
|
|
1326
|
+
_health_mod.upsert_entry(root, rel, "🔴 Red", f"DELETED — {reason}")
|
|
1195
1327
|
_add_to_pending(root, rel, f"File was deleted. Consider wiki archival. {reason}")
|
|
1196
1328
|
_ensure_journal_entry(root, "record-deletion", rel, reason or "No reason provided.")
|
|
1197
1329
|
prune_stats: Dict[str, Any] = {}
|