witan-code 0.8.0__tar.gz → 0.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {witan_code-0.8.0 → witan_code-0.8.1}/CHANGELOG.md +51 -0
  2. {witan_code-0.8.0 → witan_code-0.8.1}/PKG-INFO +30 -1
  3. {witan_code-0.8.0 → witan_code-0.8.1}/README.md +29 -0
  4. {witan_code-0.8.0 → witan_code-0.8.1}/pyproject.toml +2 -2
  5. witan_code-0.8.1/tests/test_indexer_scope.py +296 -0
  6. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/bridge.py +19 -9
  7. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/cli.py +3 -0
  8. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/indexer.py +146 -19
  9. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/server.py +1 -0
  10. {witan_code-0.8.0 → witan_code-0.8.1}/.gitignore +0 -0
  11. {witan_code-0.8.0 → witan_code-0.8.1}/config/claude.json +0 -0
  12. {witan_code-0.8.0 → witan_code-0.8.1}/config/copilot.json +0 -0
  13. {witan_code-0.8.0 → witan_code-0.8.1}/config/pi.json +0 -0
  14. {witan_code-0.8.0 → witan_code-0.8.1}/docs/BRANCH_INDEXING.md +0 -0
  15. {witan_code-0.8.0 → witan_code-0.8.1}/docs/CLI_REFERENCE.md +0 -0
  16. {witan_code-0.8.0 → witan_code-0.8.1}/docs/EDGE_PRECISION_TIERS.md +0 -0
  17. {witan_code-0.8.0 → witan_code-0.8.1}/docs/PACKAGE_MAP.md +0 -0
  18. {witan_code-0.8.0 → witan_code-0.8.1}/docs/STAGE2_STITCHING.md +0 -0
  19. {witan_code-0.8.0 → witan_code-0.8.1}/docs/SYMBOL_FORMAT.md +0 -0
  20. {witan_code-0.8.0 → witan_code-0.8.1}/docs/SYMBOL_TABLE.md +0 -0
  21. {witan_code-0.8.0 → witan_code-0.8.1}/docs/USER_GUIDE.md +0 -0
  22. {witan_code-0.8.0 → witan_code-0.8.1}/install.sh +0 -0
  23. {witan_code-0.8.0 → witan_code-0.8.1}/tests/__init__.py +0 -0
  24. {witan_code-0.8.0 → witan_code-0.8.1}/tests/conftest.py +0 -0
  25. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_branches.py +0 -0
  26. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_bridge.py +0 -0
  27. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_cli_dispatch.py +0 -0
  28. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_cli_hooks.py +0 -0
  29. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_cli_output_format.py +0 -0
  30. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_config.py +0 -0
  31. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_config_targets.py +0 -0
  32. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_context.py +0 -0
  33. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_crossrepo.py +0 -0
  34. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_edges.py +0 -0
  35. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_elicit.py +0 -0
  36. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_enrichment.py +0 -0
  37. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_graph.py +0 -0
  38. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_hooks.py +0 -0
  39. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_indexer.py +0 -0
  40. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_languages.py +0 -0
  41. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_maintenance.py +0 -0
  42. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_package_map.py +0 -0
  43. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_remote_proxy.py +0 -0
  44. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_repo.py +0 -0
  45. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_server.py +0 -0
  46. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_setup.py +0 -0
  47. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_stitch.py +0 -0
  48. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_store.py +0 -0
  49. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_symbol_table.py +0 -0
  50. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_tasks.py +0 -0
  51. {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_visualize.py +0 -0
  52. {witan_code-0.8.0 → witan_code-0.8.1}/uv.lock +0 -0
  53. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/__init__.py +0 -0
  54. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/__main__.py +0 -0
  55. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/bridge_extractors.py +0 -0
  56. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/config.py +0 -0
  57. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/context.py +0 -0
  58. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/edges.py +0 -0
  59. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/elicit.py +0 -0
  60. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/extensions/pi/codegraph.ts +0 -0
  61. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/graph.py +0 -0
  62. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/hooks.py +0 -0
  63. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/maintenance.py +0 -0
  64. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/output.py +0 -0
  65. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/package_map.py +0 -0
  66. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/bridge.gq +0 -0
  67. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/code_mutations.gq +0 -0
  68. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/code_read.gq +0 -0
  69. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/delete.gq +0 -0
  70. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/bash.scm +0 -0
  71. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/hcl.scm +0 -0
  72. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/python.scm +0 -0
  73. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/sql.scm +0 -0
  74. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/typescript.scm +0 -0
  75. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/yaml.scm +0 -0
  76. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/remote/__init__.py +0 -0
  77. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/remote/oidc.py +0 -0
  78. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/remote/proxy.py +0 -0
  79. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/repo.py +0 -0
  80. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/schema/bridge-schema.pg +0 -0
  81. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/schema/code-schema.pg +0 -0
  82. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/setup.py +0 -0
  83. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/skills/witan-code/SKILL.md +0 -0
  84. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/stitch.py +0 -0
  85. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/store.py +0 -0
  86. {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/visualize.py +0 -0
@@ -6,6 +6,57 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/) (pre-1.0:
7
7
  a MINOR bump may include breaking changes).
8
8
 
9
+ ## [0.8.1] - 2026-07-31
10
+
11
+ ### Fixed
12
+
13
+ - **Nested checkouts are no longer indexed as part of the parent repo.** The
14
+ walk skipped `.git` *directories*, but a linked worktree or submodule marks
15
+ itself with a `.git` **file**, so `.claude/worktrees/<name>/` was descended
16
+ into and every file there was attributed to the parent repo — at a different
17
+ path, at whatever revision that worktree happened to be on. In this repo's
18
+ own store that was **732 of 990 indexed files (74%)**, spread across every
19
+ branch view. The practical damage was to `code_search_symbol` and friends:
20
+ a query returned the same function many times over with *different, older
21
+ signatures*, so reading the first hit gave a confidently wrong answer, and
22
+ `code_impact` inflated blast radius with callers that only exist on an
23
+ abandoned branch. Any subdirectory holding a `.git` entry is now skipped.
24
+ Indexing from *inside* a worktree is unaffected — only descending into one
25
+ from the parent is refused — so the hooks keep working on a branch.
26
+ - **A full-repo index now purges files the repo no longer has**, reported as
27
+ `purged=N`. Nothing removed a `CodeFile`/`Symbol` once written unless that
28
+ same file was re-indexed, so deletions leaked forever: this repo's store
29
+ still held `_detach.py` and its tests, moved out to `witan-core` several
30
+ releases ago. This is also what clears the worktree rows above — excluding a
31
+ path stops adding rows but cannot retract the ones already written, so
32
+ without it the fix would only have stopped the bleeding.
33
+
34
+ Purging never runs against a shared cluster graph (`is_remote`): there the
35
+ default branch is indexed by CI and everyone else gets a read-only view, so
36
+ one developer's working tree must not reconcile it for everybody. Inert
37
+ today — code stores are still local — and load-bearing once witan-code
38
+ addresses `--server`/`--graph`.
39
+
40
+ Membership is decided by the set of files just collected, **not** by whether
41
+ the file still exists on disk: a linked worktree's files are on disk and
42
+ still must go. Purging requires a confirmed git root and a full-repo target;
43
+ a subpath, a single file (the reindex hook), or a non-git directory never
44
+ purges — without a git root `base` falls back to the target directory, which
45
+ would make every stored path look stale.
46
+
47
+ ### Changed
48
+
49
+ - An unreadable directory now suppresses the purge for that run (and is
50
+ reported on stderr, counting toward `errors`). `os.walk` hands such a
51
+ directory to `onerror` and otherwise continues silently, so a subtree the
52
+ walk could not enter is indistinguishable from one that was deleted — which
53
+ would have taken its still-present files' rows with it. Indexing still
54
+ proceeds with whatever was readable; only the destructive half backs off.
55
+ - The file walk prunes as it goes (`os.walk`) instead of walking everything and
56
+ filtering afterwards, so `node_modules`/`.venv` subtrees are no longer
57
+ traversed in full. Collection order is now sorted, making a run reproducible.
58
+ - `code_reindex` returns a `purged` count alongside the existing counters.
59
+
9
60
  ## [0.8.0] - 2026-07-31
10
61
 
11
62
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: witan-code
3
- Version: 0.8.0
3
+ Version: 0.8.1
4
4
  Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
5
5
  Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
6
6
  Project-URL: Repository, https://github.com/mitodl/agent-kit
@@ -573,9 +573,38 @@ Manual install: register the bare commands directly under the matching event
573
573
  in `settings.json` (see the linked README for the exact JSON) — there are no
574
574
  scripts to symlink.
575
575
 
576
+ ## What gets indexed
577
+
578
+ The walk skips the usual noise directories (`.git`, `node_modules`, `.venv`,
579
+ `__pycache__`, `dist`, `build`, the various caches) and — importantly — does
580
+ not descend into a **nested checkout**: any subdirectory containing a `.git`
581
+ entry belongs to a different repository. That covers linked worktrees
582
+ (`.claude/worktrees/<name>/`, where `.git` is a *file*, not a directory),
583
+ submodules, and plain clones dropped inside the tree. Their files are that
584
+ repo's; indexing them here would attribute them to this one and leave the
585
+ store serving stale copies of itself under a second set of paths.
586
+
587
+ Indexing *from inside* a worktree still works normally — only descending into
588
+ one from the parent is refused — so the hooks keep indexing while an agent
589
+ works on a branch.
590
+
576
591
  ## Incremental indexing
577
592
 
578
593
  Each `CodeFile` stores a sha256 `contentHash`. On reindex, if the hash is
579
594
  unchanged the file is skipped. Otherwise its `Symbol`s and `CodeFile` are
580
595
  deleted (as separate `delete.gq` calls — Omnigraph cannot mix deletes with
581
596
  inserts in one query) and then re-parsed and re-inserted.
597
+
598
+ A **full-repo** index additionally purges rows for files the repo no longer
599
+ has — deleted, or newly excluded by the rules above. Membership is decided by
600
+ the set of files just collected, not by whether the file still exists on disk:
601
+ a linked worktree's files are very much on disk, they simply aren't this
602
+ repo's. The count is reported as `purged=N` when non-zero. Purging requires a
603
+ confirmed git root; indexing a subpath, a single file (the reindex hook), a
604
+ directory that isn't a git checkout, or a directory the walk could not fully
605
+ read never purges.
606
+
607
+ Nor does a **shared cluster graph** (`is_remote`). There the default branch is
608
+ indexed by CI and everyone else reads it — reconciling it against one
609
+ developer's working tree (a sparse checkout, a stale one, uncommitted
610
+ deletions) would purge files for every other user of that graph.
@@ -541,9 +541,38 @@ Manual install: register the bare commands directly under the matching event
541
541
  in `settings.json` (see the linked README for the exact JSON) — there are no
542
542
  scripts to symlink.
543
543
 
544
+ ## What gets indexed
545
+
546
+ The walk skips the usual noise directories (`.git`, `node_modules`, `.venv`,
547
+ `__pycache__`, `dist`, `build`, the various caches) and — importantly — does
548
+ not descend into a **nested checkout**: any subdirectory containing a `.git`
549
+ entry belongs to a different repository. That covers linked worktrees
550
+ (`.claude/worktrees/<name>/`, where `.git` is a *file*, not a directory),
551
+ submodules, and plain clones dropped inside the tree. Their files are that
552
+ repo's; indexing them here would attribute them to this one and leave the
553
+ store serving stale copies of itself under a second set of paths.
554
+
555
+ Indexing *from inside* a worktree still works normally — only descending into
556
+ one from the parent is refused — so the hooks keep indexing while an agent
557
+ works on a branch.
558
+
544
559
  ## Incremental indexing
545
560
 
546
561
  Each `CodeFile` stores a sha256 `contentHash`. On reindex, if the hash is
547
562
  unchanged the file is skipped. Otherwise its `Symbol`s and `CodeFile` are
548
563
  deleted (as separate `delete.gq` calls — Omnigraph cannot mix deletes with
549
564
  inserts in one query) and then re-parsed and re-inserted.
565
+
566
+ A **full-repo** index additionally purges rows for files the repo no longer
567
+ has — deleted, or newly excluded by the rules above. Membership is decided by
568
+ the set of files just collected, not by whether the file still exists on disk:
569
+ a linked worktree's files are very much on disk, they simply aren't this
570
+ repo's. The count is reported as `purged=N` when non-zero. Purging requires a
571
+ confirmed git root; indexing a subpath, a single file (the reindex hook), a
572
+ directory that isn't a git checkout, or a directory the walk could not fully
573
+ read never purges.
574
+
575
+ Nor does a **shared cluster graph** (`is_remote`). There the default branch is
576
+ indexed by CI and everyone else reads it — reconciling it against one
577
+ developer's working tree (a sparse checkout, a stale one, uncommitted
578
+ deletions) would purge files for every other user of that graph.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "witan-code"
3
- version = "0.8.0"
3
+ version = "0.8.1"
4
4
  description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
5
5
  readme = "README.md"
6
6
  license = "BSD-3-Clause"
@@ -90,7 +90,7 @@ testpaths = ["tests"]
90
90
  packages = ["witan_code"]
91
91
 
92
92
  [tool.bumpversion]
93
- current_version = "0.8.0"
93
+ current_version = "0.8.1"
94
94
  allow_dirty = true
95
95
 
96
96
  [[tool.bumpversion.files]]
@@ -0,0 +1,296 @@
1
+ """What counts as "this repo's files", and what happens to rows that stop counting.
2
+
3
+ A linked worktree or submodule under the repo is a *different* checkout, so its
4
+ files must not be attributed to this repo — and once excluded, the rows already
5
+ written for them have to go, or the store keeps serving stale copies of the
6
+ repo to itself (which is exactly how one ended up 74% duplicates).
7
+ """
8
+
9
+ import subprocess
10
+ from types import SimpleNamespace
11
+ from pathlib import Path
12
+
13
+ from witan_code import indexer
14
+
15
+ from .conftest import requires_stack
16
+
17
+
18
+ def _repo(tmp_path: Path, monkeypatch, name: str = "cg") -> Path:
19
+ """A real git checkout — purging is gated on a confirmed git root, so a
20
+ bare directory would (correctly) never purge and the tests would pass
21
+ vacuously."""
22
+ monkeypatch.setenv("WITAN_REPO", f"https://github.com/test/{name}")
23
+ monkeypatch.setenv("WITAN_CODE_DIR", str(tmp_path / "code"))
24
+ src = tmp_path / "repo"
25
+ src.mkdir()
26
+ subprocess.run(["git", "init", "-q", "-b", "main"], cwd=src, check=True)
27
+ (src / "a.py").write_text("def a():\n return 1\n")
28
+ return src
29
+
30
+
31
+ def _files(store, cfg) -> set[str]:
32
+ """Repo-relative paths currently indexed in ``store``."""
33
+ from witan_code.graph import OmnigraphClient
34
+
35
+ rows = OmnigraphClient(str(store), cfg.queries_dir).read(
36
+ "code_read.gq", "all_file_hashes", {}
37
+ )
38
+ return {r["slug"].split("#", 1)[1] for r in rows}
39
+
40
+
41
+ # ── _collect_files: what the walk descends into ──────────────────────────────
42
+
43
+
44
+ def test_nested_worktree_is_not_collected(tmp_path):
45
+ """A linked worktree's `.git` is a FILE, not a directory — the `.git` entry
46
+ in _SKIP_DIRS never matched it, which is how these got indexed."""
47
+ (tmp_path / "a.py").write_text("x = 1\n")
48
+ wt = tmp_path / ".claude" / "worktrees" / "feature-x"
49
+ wt.mkdir(parents=True)
50
+ (wt / ".git").write_text("gitdir: /somewhere/.git/worktrees/feature-x\n")
51
+ (wt / "copy.py").write_text("x = 1\n")
52
+
53
+ collected = indexer._collect_files(tmp_path)
54
+
55
+ assert tmp_path / "a.py" in collected
56
+ assert not any("worktrees" in p.as_posix() for p in collected)
57
+
58
+
59
+ def test_nested_clone_with_a_git_directory_is_not_collected(tmp_path):
60
+ (tmp_path / "a.py").write_text("x = 1\n")
61
+ vendored = tmp_path / "vendored"
62
+ (vendored / ".git").mkdir(parents=True)
63
+ (vendored / "dep.py").write_text("x = 1\n")
64
+
65
+ collected = indexer._collect_files(tmp_path)
66
+
67
+ assert collected == [tmp_path / "a.py"]
68
+
69
+
70
+ def test_indexing_from_inside_a_worktree_still_works(tmp_path):
71
+ """The target root is never a pruning candidate — only descending into a
72
+ nested checkout from outside is refused. The hooks index from inside a
73
+ worktree whenever an agent works there, so this must keep collecting."""
74
+ wt = tmp_path / "worktrees" / "feature-x"
75
+ wt.mkdir(parents=True)
76
+ (wt / ".git").write_text("gitdir: /somewhere\n")
77
+ (wt / "own.py").write_text("x = 1\n")
78
+
79
+ assert indexer._collect_files(wt) == [wt / "own.py"]
80
+
81
+
82
+ def test_skip_dirs_are_pruned_not_merely_filtered(tmp_path):
83
+ (tmp_path / "a.py").write_text("x = 1\n")
84
+ junk = tmp_path / "node_modules" / "pkg"
85
+ junk.mkdir(parents=True)
86
+ (junk / "index.js").write_text("var x = 1;\n")
87
+
88
+ assert indexer._collect_files(tmp_path) == [tmp_path / "a.py"]
89
+
90
+
91
+ def test_collection_order_is_deterministic(tmp_path):
92
+ for name in ("c.py", "a.py", "b.py"):
93
+ (tmp_path / name).write_text("x = 1\n")
94
+ (tmp_path / "sub").mkdir()
95
+ (tmp_path / "sub" / "d.py").write_text("x = 1\n")
96
+
97
+ assert indexer._collect_files(tmp_path) == sorted(indexer._collect_files(tmp_path))
98
+
99
+
100
+ # ── Purging rows that are no longer this repo's ──────────────────────────────
101
+
102
+
103
+ @requires_stack
104
+ def test_full_index_purges_a_file_deleted_from_disk(tmp_path, monkeypatch):
105
+ from witan_code import config as cfg_mod
106
+ from witan_code import store as store_mod
107
+
108
+ src = _repo(tmp_path, monkeypatch)
109
+ (src / "gone.py").write_text("def gone():\n return 1\n")
110
+ cfg = cfg_mod.load()
111
+ indexer.index_path(src, config=cfg)
112
+ store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
113
+ assert _files(store, cfg) == {"a.py", "gone.py"}
114
+
115
+ (src / "gone.py").unlink()
116
+ stats = indexer.index_path(src, config=cfg)
117
+
118
+ assert stats.purged == 1
119
+ assert _files(store, cfg) == {"a.py"}
120
+
121
+
122
+ @requires_stack
123
+ def test_full_index_purges_files_that_became_a_nested_checkout(tmp_path, monkeypatch):
124
+ """The case an on-disk existence check cannot catch: the files are still
125
+ there, they just stopped being this repo's."""
126
+ from witan_code import config as cfg_mod
127
+ from witan_code import store as store_mod
128
+
129
+ src = _repo(tmp_path, monkeypatch)
130
+ nested = src / "sub"
131
+ nested.mkdir()
132
+ (nested / "b.py").write_text("def b():\n return 1\n")
133
+ cfg = cfg_mod.load()
134
+ indexer.index_path(src, config=cfg)
135
+ store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
136
+ assert _files(store, cfg) == {"a.py", "sub/b.py"}
137
+
138
+ # `sub/` becomes a linked worktree; its file is untouched on disk.
139
+ (nested / ".git").write_text("gitdir: /somewhere\n")
140
+ stats = indexer.index_path(src, config=cfg)
141
+
142
+ assert (nested / "b.py").exists()
143
+ assert stats.purged == 1
144
+ assert _files(store, cfg) == {"a.py"}
145
+
146
+
147
+ @requires_stack
148
+ def test_indexing_a_subpath_purges_nothing(tmp_path, monkeypatch):
149
+ """Everything outside the subpath is legitimately uncollected — treating
150
+ that as stale would empty the store on every PostToolUse single-file run."""
151
+ from witan_code import config as cfg_mod
152
+ from witan_code import store as store_mod
153
+
154
+ src = _repo(tmp_path, monkeypatch)
155
+ sub = src / "pkg"
156
+ sub.mkdir()
157
+ (sub / "b.py").write_text("def b():\n return 1\n")
158
+ cfg = cfg_mod.load()
159
+ indexer.index_path(src, config=cfg)
160
+ store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
161
+
162
+ stats = indexer.index_path(sub, config=cfg)
163
+
164
+ assert stats.purged == 0
165
+ assert _files(store, cfg) == {"a.py", "pkg/b.py"}
166
+
167
+
168
+ @requires_stack
169
+ def test_purge_survives_a_forced_reindex(tmp_path, monkeypatch):
170
+ """`force` skips the hash comparison but must still read the stored file
171
+ set — otherwise a --force run silently stops purging."""
172
+ from witan_code import config as cfg_mod
173
+ from witan_code import store as store_mod
174
+
175
+ src = _repo(tmp_path, monkeypatch)
176
+ (src / "gone.py").write_text("def gone():\n return 1\n")
177
+ cfg = cfg_mod.load()
178
+ indexer.index_path(src, config=cfg)
179
+ store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
180
+
181
+ (src / "gone.py").unlink()
182
+ stats = indexer.index_path(src, force=True, config=cfg)
183
+
184
+ assert stats.purged == 1
185
+ assert _files(store, cfg) == {"a.py"}
186
+
187
+
188
+ @requires_stack
189
+ def test_no_purge_without_a_confirmed_git_root(tmp_path, monkeypatch):
190
+ """The guard on the destructive path.
191
+
192
+ Without git, `base` falls back to the target directory, so `full_repo` is
193
+ true for ANY directory indexed. Combined with a WITAN_REPO override (one
194
+ slug, two different bases) an unguarded purge would delete every row whose
195
+ path was stored relative to the real root — i.e. empty the store.
196
+ """
197
+ from witan_code import config as cfg_mod
198
+ from witan_code import store as store_mod
199
+
200
+ monkeypatch.setenv("WITAN_REPO", "https://github.com/test/nogit")
201
+ monkeypatch.setenv("WITAN_CODE_DIR", str(tmp_path / "code"))
202
+ src = tmp_path / "plain" # deliberately NOT a git checkout
203
+ (src / "pkg").mkdir(parents=True)
204
+ (src / "a.py").write_text("def a():\n return 1\n")
205
+ (src / "pkg" / "b.py").write_text("def b():\n return 1\n")
206
+
207
+ cfg = cfg_mod.load()
208
+ indexer.index_path(src, config=cfg)
209
+ store = store_mod.store_for_repo("https://github.com/test/nogit", cfg)
210
+ assert _files(store, cfg) == {"a.py", "pkg/b.py"}
211
+
212
+ stats = indexer.index_path(src / "pkg", config=cfg)
213
+
214
+ assert stats.purged == 0
215
+ # Both original rows survive — nothing was deleted, which is the point.
216
+ # This run also ADDS a duplicate `b.py` (paths relative to the subdirectory
217
+ # base rather than the real root): a pre-existing quirk of overriding
218
+ # WITAN_REPO without a git root, unrelated to purging. Asserted as a subset
219
+ # so this test pins the guard and not that quirk.
220
+ assert {"a.py", "pkg/b.py"} <= _files(store, cfg)
221
+
222
+
223
+ @requires_stack
224
+ def test_unreadable_directory_suppresses_the_purge(tmp_path, monkeypatch):
225
+ """A subtree the walk cannot read looks exactly like a deleted one.
226
+
227
+ os.walk reports such a directory to `onerror` and otherwise carries on
228
+ silently, so without this guard an unreadable subtree would take its
229
+ still-present files' rows with it — a permission blip turning into data
230
+ loss. Indexing still proceeds with whatever was readable.
231
+ """
232
+ from witan_code import config as cfg_mod
233
+ from witan_code import store as store_mod
234
+
235
+ src = _repo(tmp_path, monkeypatch)
236
+ (src / "sub").mkdir()
237
+ (src / "sub" / "b.py").write_text("def b():\n return 1\n")
238
+ cfg = cfg_mod.load()
239
+ indexer.index_path(src, config=cfg)
240
+ store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
241
+ assert _files(store, cfg) == {"a.py", "sub/b.py"}
242
+
243
+ # Simulate `sub/` becoming unreadable: os.walk yields the root only and
244
+ # hands the failure to onerror, exactly as a PermissionError does.
245
+ real_walk = indexer.os.walk
246
+
247
+ def _walk(top, *args, onerror=None, **kwargs):
248
+ for entry in real_walk(top, *args, **kwargs):
249
+ root, dirs, files = entry
250
+ if Path(root) == src:
251
+ dirs[:] = [d for d in dirs if d != "sub"]
252
+ if onerror is not None:
253
+ onerror(PermissionError(13, "Permission denied", str(src / "sub")))
254
+ yield entry
255
+
256
+ monkeypatch.setattr(indexer.os, "walk", _walk)
257
+ stats = indexer.index_path(src, config=cfg)
258
+
259
+ assert stats.errors >= 1 # the unreadable directory is reported, not hidden
260
+ assert stats.purged == 0
261
+ assert _files(store, cfg) == {"a.py", "sub/b.py"}
262
+
263
+
264
+ # ── _may_purge: the gate on the one destructive step ─────────────────────────
265
+
266
+
267
+ def _gate(**overrides):
268
+ """Call `_may_purge` with an authoritative baseline, overriding one clause."""
269
+ kwargs = {
270
+ "full_repo": True,
271
+ "repo_root": Path("/repo"),
272
+ "walk_errors": [],
273
+ "client": SimpleNamespace(is_remote=False),
274
+ }
275
+ kwargs.update(overrides)
276
+ return indexer._may_purge(**kwargs)
277
+
278
+
279
+ def test_purge_allowed_only_when_this_machine_is_authoritative():
280
+ assert _gate() is True
281
+
282
+ # A subpath run collects a fraction of the repo by design.
283
+ assert _gate(full_repo=False) is False
284
+ # No git root => `base` is the target dir, so `full_repo` means nothing.
285
+ assert _gate(repo_root=None) is False
286
+ # An unreadable subtree looks exactly like a deleted one.
287
+ assert _gate(walk_errors=[PermissionError(13, "denied", "/repo/sub")]) is False
288
+
289
+
290
+ def test_purge_refused_against_a_shared_cluster_graph():
291
+ """A remote graph is shared: the default branch is indexed by CI and
292
+ everyone else reads it. Reconciling it against one developer's working
293
+ tree — sparse checkout, stale checkout, uncommitted deletions — would
294
+ purge files for every other user of that graph.
295
+ """
296
+ assert _gate(client=SimpleNamespace(is_remote=True)) is False
@@ -32,16 +32,22 @@ def write_bindings(
32
32
  identity: package_map.PackageIdentity | None = None,
33
33
  base: Path | None = None,
34
34
  branch: str | None = None,
35
+ indexed_files: frozenset[str] | None = None,
35
36
  ) -> int:
36
37
  """Purge stale bindings and merge in fresh ones for ``repo``.
37
38
 
38
39
  Purging is always per-file: the files (re)parsed this run, the files the
39
40
  fresh batch carries bindings for (covers Tier B sources like OpenAPI
40
- specs, which aren't in ``touched_files``), and — on a full-repo run with
41
- ``base`` — files whose stored bindings no longer exist on disk. An
42
- incremental full-repo index skips unchanged files, so a repo-wide purge
43
- here would drop those files' bindings with nothing in the batch to
44
- restore them.
41
+ specs, which aren't in ``touched_files``), and — on a full-repo run — the
42
+ files that are no longer part of the repo. An incremental full-repo index
43
+ skips unchanged files, so a repo-wide purge here would drop those files'
44
+ bindings with nothing in the batch to restore them.
45
+
46
+ ``indexed_files`` is the indexer's collected set (repo-relative paths) and
47
+ is the accurate membership test for that last group: a file can still be
48
+ on disk yet no longer belong to this repo — a linked worktree's files are
49
+ the case that motivated it. Callers that don't have the set fall back to
50
+ an on-disk existence check against ``base``, which catches deletions only.
45
51
 
46
52
  Store-level confidence adjustments (self_provided_key, known_provider_package)
47
53
  are applied here before writing. The cross-repo half of each signal is
@@ -90,13 +96,17 @@ def write_bindings(
90
96
  repo_symbol_rows = _read_repo_symbols(client) if has_endpoint_consumers else []
91
97
 
92
98
  purge_files = set(touched_files) | {b.file for b in bindings}
93
- if full_repo and base is not None:
99
+ if full_repo and (indexed_files is not None or base is not None):
100
+
101
+ def _gone(rel: str) -> bool:
102
+ if indexed_files is not None:
103
+ return rel not in indexed_files
104
+ return not (base / rel).exists()
105
+
94
106
  purge_files |= {
95
107
  r["file"]
96
108
  for r in all_rows
97
- if r.get("repo") == repo
98
- and r.get("file")
99
- and not (base / r["file"]).exists()
109
+ if r.get("repo") == repo and r.get("file") and _gone(r["file"])
100
110
  }
101
111
  for rel in sorted(purge_files):
102
112
  client.change(
@@ -591,6 +591,9 @@ def _print_summary(action: str, path: Path, stats: indexer.IndexStats) -> None:
591
591
  f"scanned={stats.scanned} indexed={stats.indexed} "
592
592
  f"skipped={stats.skipped} symbols={stats.symbols} "
593
593
  f"edges={stats.edges} bindings={stats.bindings} errors={stats.errors}"
594
+ # Only when it happened: a purge is newsworthy (rows were deleted),
595
+ # but printing purged=0 on every routine index is noise.
596
+ + (f" purged={stats.purged}" if stats.purged else "")
594
597
  )
595
598
 
596
599
 
@@ -12,7 +12,9 @@ and will miss dynamic dispatch and produce occasional false links.
12
12
  import functools
13
13
  import hashlib
14
14
  import importlib
15
+ import os
15
16
  import sys
17
+ from collections.abc import Callable
16
18
  from dataclasses import dataclass, field
17
19
  from pathlib import Path
18
20
 
@@ -241,6 +243,11 @@ class IndexStats:
241
243
  edges: int = 0
242
244
  bindings: int = 0
243
245
  errors: int = 0
246
+ purged: int = 0
247
+ """Files dropped from the store because they are no longer part of the
248
+ repo — deleted, or newly excluded (a nested checkout, a skipped
249
+ directory). Full-repo runs only; reported so a mass cleanup is visible
250
+ rather than silent."""
244
251
 
245
252
 
246
253
  # ── Public entry points ───────────────────────────────────────────
@@ -279,11 +286,12 @@ def index_path(
279
286
  client.ensure_branch()
280
287
 
281
288
  # One query for all existing file hashes → the incremental skip check is
282
- # in-memory, not a query per file.
289
+ # in-memory, not a query per file. Read even under `force` (where the
290
+ # hashes go unused): a full-repo run also needs the stored file set to
291
+ # find rows for files that are no longer part of the repo.
283
292
  existing: dict[str, str] = {}
284
- if not force:
285
- for row in client.read("code_read.gq", "all_file_hashes", {}):
286
- existing[row["slug"]] = row.get("content_hash")
293
+ for row in client.read("code_read.gq", "all_file_hashes", {}):
294
+ existing[row["slug"]] = row.get("content_hash")
287
295
 
288
296
  stats = IndexStats()
289
297
  records: list[dict] = []
@@ -291,7 +299,21 @@ def index_path(
291
299
  bindings: list[ParsedBinding] = []
292
300
  touched_files: list[str] = []
293
301
 
294
- for path in _collect_files(target):
302
+ # A directory the walk could not read makes `collected` incomplete in a way
303
+ # that is invisible from the result alone — and the purge below reads
304
+ # "missing from the collected set" as "no longer part of the repo".
305
+ walk_errors: list[OSError] = []
306
+ collected = _collect_files(target, on_error=walk_errors.append)
307
+ for exc in walk_errors:
308
+ stats.errors += 1
309
+ print(f"codegraph: could not read {exc.filename}: {exc}", file=sys.stderr)
310
+
311
+ # Every file the repo should have indexed, changed or not — the authority
312
+ # for the purge below, which `touched_files` is not (an incremental run
313
+ # touches only what changed).
314
+ indexed_rel = {_relative_path(p, base) for p in collected}
315
+
316
+ for path in collected:
295
317
  stats.scanned += 1
296
318
  try:
297
319
  result = _parse_for_index(path, base, slug, existing, force=force)
@@ -310,10 +332,35 @@ def index_path(
310
332
  touched_files.append(parsed.path)
311
333
  stats.indexed += 1
312
334
 
335
+ full_repo = target.is_dir() and target.resolve() == base.resolve()
336
+ can_purge = _may_purge(
337
+ full_repo=full_repo,
338
+ repo_root=repo_root,
339
+ walk_errors=walk_errors,
340
+ client=client,
341
+ )
342
+
313
343
  # Drop stale data for changed files (new files have nothing to delete), then
314
344
  # bulk-load every node and edge in a single omnigraph call.
315
345
  for file_id in reindexed_file_ids:
316
346
  _delete_file_data(file_id, client)
347
+
348
+ # A full-repo run also drops files the repo no longer has. Membership is
349
+ # decided by the collected set, NOT by whether the file still exists on
350
+ # disk: a linked worktree's files are very much on disk, they simply
351
+ # aren't this repo's to index (see `_is_nested_checkout`). Without this,
352
+ # excluding a path stops adding rows but leaves every row already written
353
+ # — which is how a store ends up majority stale copies of itself.
354
+ # Subpath runs are exempt: everything outside the subpath is legitimately
355
+ # absent from `indexed_rel` and must not be treated as stale.
356
+ if can_purge:
357
+ prefix = f"{slug}#"
358
+ for file_id in sorted(existing):
359
+ rel = file_id[len(prefix) :] if file_id.startswith(prefix) else None
360
+ if rel is not None and rel not in indexed_rel:
361
+ _delete_file_data(file_id, client)
362
+ stats.purged += 1
363
+
317
364
  client.load(_dedupe(records), mode="merge")
318
365
 
319
366
  # Cross-repo bridge — a SEPARATE phase after the per-repo store write, so the
@@ -326,7 +373,6 @@ def index_path(
326
373
  # (docs/BRANCH_INDEXING.md § Bridge store) rather than skipping the bridge
327
374
  # entirely: the shared main view still never sees in-flight bindings, but
328
375
  # they're no longer dropped on the floor either.
329
- full_repo = target.is_dir() and target.resolve() == base.resolve()
330
376
  if full_repo:
331
377
  bindings.extend(bridge_extractors.extract_repo_bindings(base, slug))
332
378
  try:
@@ -339,6 +385,7 @@ def index_path(
339
385
  identity=package_map.load(base, slug),
340
386
  base=base,
341
387
  branch=branch,
388
+ indexed_files=frozenset(indexed_rel) if can_purge else None,
342
389
  )
343
390
  except Exception as exc: # noqa: BLE001 — bridge is best-effort, never fatal
344
391
  print(f"codegraph: bridge update failed: {exc}", file=sys.stderr)
@@ -371,19 +418,102 @@ def _dedupe(records: list[dict]) -> list[dict]:
371
418
  return out
372
419
 
373
420
 
374
- def _collect_files(target: Path) -> list[Path]:
421
+ def _may_purge(
422
+ *,
423
+ full_repo: bool,
424
+ repo_root: Path | None,
425
+ walk_errors: list[OSError],
426
+ client: OmnigraphClient,
427
+ ) -> bool:
428
+ """Whether this run is entitled to delete rows for files it did not collect.
429
+
430
+ Purging is the one destructive thing an index does, so it asks for more
431
+ than the run being a full-repo one. Every clause answers the same question
432
+ — is this machine's file listing authoritative for the view being written?
433
+
434
+ - ``full_repo``: a subpath run legitimately collects a fraction of the repo.
435
+ - ``repo_root``: without a git root, ``base`` falls back to the target
436
+ directory, making ``full_repo`` true for ANY directory — index a
437
+ subdirectory then and every path stored relative to the real root looks
438
+ stale.
439
+ - ``walk_errors``: a directory the walk could not read is indistinguishable
440
+ from one that was deleted, so its files would go while sitting on disk.
441
+ - ``client.is_remote``: a shared cluster graph is not this machine's to
442
+ reconcile. One developer's working tree — sparse checkout, stale
443
+ checkout, local deletions — would otherwise purge files for everyone on
444
+ the shared ``main`` view. There the default branch is indexed by CI and
445
+ everyone else reads it; a designated writer will need an explicit opt-in
446
+ rather than inheriting the right by being remote.
447
+ """
448
+ return (
449
+ full_repo and repo_root is not None and not walk_errors and not client.is_remote
450
+ )
451
+
452
+
453
+ def _relative_path(path: Path, base: Path) -> str:
454
+ """A file's repo-relative path — the second half of its ``repo#rel`` id.
455
+
456
+ Shared by the parse path and the stale-file purge so the two agree on what
457
+ identifies a file; a mismatch there would purge rows that were just written.
458
+ """
459
+ try:
460
+ return path.resolve().relative_to(base.resolve()).as_posix()
461
+ except ValueError:
462
+ return path.name
463
+
464
+
465
+ def _is_nested_checkout(path: Path) -> bool:
466
+ """Whether ``path`` is the root of a *different* git checkout.
467
+
468
+ A `.git` entry inside a subdirectory means that subtree belongs to another
469
+ repository — a linked worktree (`.claude/worktrees/<name>/`, where `.git`
470
+ is a *file* pointing back at the parent's `.git/worktrees/`), a submodule
471
+ (also a `.git` file), or a plain clone someone dropped in. Its files are
472
+ that repo's, and indexing them here attributes them to this one.
473
+ """
474
+ return (path / ".git").exists()
475
+
476
+
477
+ def _collect_files(
478
+ target: Path, *, on_error: Callable[[OSError], None] | None = None
479
+ ) -> list[Path]:
480
+ """Every indexable file under ``target``, sorted.
481
+
482
+ ``on_error`` is handed any directory that could not be read. ``os.walk``
483
+ swallows those silently by default, which is fine for indexing (skip what
484
+ you can't read) but NOT for the purge that consumes this list: an
485
+ unreadable subtree would look like a set of deleted files and take their
486
+ rows with it. ``index_path`` passes a collector and declines to purge when
487
+ anything fired.
488
+ """
375
489
  if target.is_file():
376
490
  return [target] if target.suffix in _EXT_TO_SPEC else []
377
491
 
378
492
  out: list[Path] = []
379
- for path in target.rglob("*"):
380
- if path.is_dir():
381
- continue
382
- if any(part in _SKIP_DIRS for part in path.parts):
383
- continue
384
- if path.suffix in _EXT_TO_SPEC:
385
- out.append(path)
386
- return out
493
+ for root, dirs, files in os.walk(target, onerror=on_error):
494
+ root_path = Path(root)
495
+ # Prune in place, so a skipped directory is never descended into at
496
+ # all. The previous rglob("*") walked every file under node_modules/
497
+ # .venv/.git and discarded them afterwards; this also makes the
498
+ # nested-checkout test cheap, since it runs once per surviving
499
+ # directory rather than once per file.
500
+ #
501
+ # `target` itself is never a pruning candidate (only entries in
502
+ # `dirs` are), so indexing a repo root — or a path *inside* a
503
+ # worktree, which is how the hooks index while an agent works there —
504
+ # still works. Only descending into one from outside is refused.
505
+ dirs[:] = [
506
+ d
507
+ for d in dirs
508
+ if d not in _SKIP_DIRS and not _is_nested_checkout(root_path / d)
509
+ ]
510
+ for name in files:
511
+ path = root_path / name
512
+ if path.suffix in _EXT_TO_SPEC:
513
+ out.append(path)
514
+ # os.walk's directory order is arbitrary; sort so a run is reproducible
515
+ # and `_dedupe`'s first-wins tie-break is stable across runs.
516
+ return sorted(out)
387
517
 
388
518
 
389
519
  # ── Per-file indexing ─────────────────────────────────────────────
@@ -402,10 +532,7 @@ def _parse_for_index(
402
532
  raw = path.read_bytes()
403
533
  content_hash = hashlib.sha256(raw).hexdigest()
404
534
 
405
- try:
406
- rel = path.resolve().relative_to(base.resolve()).as_posix()
407
- except ValueError:
408
- rel = path.name
535
+ rel = _relative_path(path, base)
409
536
  file_id = f"{slug}#{rel}"
410
537
 
411
538
  if not force and existing.get(file_id) == content_hash:
@@ -1024,4 +1024,5 @@ async def code_reindex(path: str | None = None, force: bool = False) -> dict:
1024
1024
  "edges": stats.edges,
1025
1025
  "bindings": stats.bindings,
1026
1026
  "errors": stats.errors,
1027
+ "purged": stats.purged,
1027
1028
  }
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes