witan-code 0.8.0__tar.gz → 0.8.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {witan_code-0.8.0 → witan_code-0.8.1}/CHANGELOG.md +51 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/PKG-INFO +30 -1
- {witan_code-0.8.0 → witan_code-0.8.1}/README.md +29 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/pyproject.toml +2 -2
- witan_code-0.8.1/tests/test_indexer_scope.py +296 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/bridge.py +19 -9
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/cli.py +3 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/indexer.py +146 -19
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/server.py +1 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/.gitignore +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/config/claude.json +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/config/copilot.json +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/config/pi.json +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/BRANCH_INDEXING.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/CLI_REFERENCE.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/EDGE_PRECISION_TIERS.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/PACKAGE_MAP.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/STAGE2_STITCHING.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/SYMBOL_FORMAT.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/SYMBOL_TABLE.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/docs/USER_GUIDE.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/install.sh +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/__init__.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/conftest.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_branches.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_bridge.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_cli_dispatch.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_cli_hooks.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_cli_output_format.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_config.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_config_targets.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_context.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_crossrepo.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_edges.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_elicit.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_enrichment.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_graph.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_hooks.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_indexer.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_languages.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_maintenance.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_package_map.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_remote_proxy.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_repo.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_server.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_setup.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_stitch.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_store.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_symbol_table.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_tasks.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/tests/test_visualize.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/uv.lock +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/__init__.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/__main__.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/bridge_extractors.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/config.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/context.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/edges.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/elicit.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/extensions/pi/codegraph.ts +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/graph.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/hooks.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/maintenance.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/output.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/package_map.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/bridge.gq +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/code_mutations.gq +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/code_read.gq +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries/delete.gq +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/bash.scm +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/hcl.scm +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/python.scm +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/sql.scm +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/typescript.scm +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/queries_ts/yaml.scm +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/remote/__init__.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/remote/oidc.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/remote/proxy.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/repo.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/schema/bridge-schema.pg +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/schema/code-schema.pg +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/setup.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/skills/witan-code/SKILL.md +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/stitch.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/store.py +0 -0
- {witan_code-0.8.0 → witan_code-0.8.1}/witan_code/visualize.py +0 -0
|
@@ -6,6 +6,57 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/) (pre-1.0:
|
|
7
7
|
a MINOR bump may include breaking changes).
|
|
8
8
|
|
|
9
|
+
## [0.8.1] - 2026-07-31
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- **Nested checkouts are no longer indexed as part of the parent repo.** The
|
|
14
|
+
walk skipped `.git` *directories*, but a linked worktree or submodule marks
|
|
15
|
+
itself with a `.git` **file**, so `.claude/worktrees/<name>/` was descended
|
|
16
|
+
into and every file there was attributed to the parent repo — at a different
|
|
17
|
+
path, at whatever revision that worktree happened to be on. In this repo's
|
|
18
|
+
own store that was **732 of 990 indexed files (74%)**, spread across every
|
|
19
|
+
branch view. The practical damage was to `code_search_symbol` and friends:
|
|
20
|
+
a query returned the same function many times over with *different, older
|
|
21
|
+
signatures*, so reading the first hit gave a confidently wrong answer, and
|
|
22
|
+
`code_impact` inflated blast radius with callers that only exist on an
|
|
23
|
+
abandoned branch. Any subdirectory holding a `.git` entry is now skipped.
|
|
24
|
+
Indexing from *inside* a worktree is unaffected — only descending into one
|
|
25
|
+
from the parent is refused — so the hooks keep working on a branch.
|
|
26
|
+
- **A full-repo index now purges files the repo no longer has**, reported as
|
|
27
|
+
`purged=N`. Nothing removed a `CodeFile`/`Symbol` once written unless that
|
|
28
|
+
same file was re-indexed, so deletions leaked forever: this repo's store
|
|
29
|
+
still held `_detach.py` and its tests, moved out to `witan-core` several
|
|
30
|
+
releases ago. This is also what clears the worktree rows above — excluding a
|
|
31
|
+
path stops adding rows but cannot retract the ones already written, so
|
|
32
|
+
without it the fix would only have stopped the bleeding.
|
|
33
|
+
|
|
34
|
+
Purging never runs against a shared cluster graph (`is_remote`): there the
|
|
35
|
+
default branch is indexed by CI and everyone else gets a read-only view, so
|
|
36
|
+
one developer's working tree must not reconcile it for everybody. Inert
|
|
37
|
+
today — code stores are still local — and load-bearing once witan-code
|
|
38
|
+
addresses `--server`/`--graph`.
|
|
39
|
+
|
|
40
|
+
Membership is decided by the set of files just collected, **not** by whether
|
|
41
|
+
the file still exists on disk: a linked worktree's files are on disk and
|
|
42
|
+
still must go. Purging requires a confirmed git root and a full-repo target;
|
|
43
|
+
a subpath, a single file (the reindex hook), or a non-git directory never
|
|
44
|
+
purges — without a git root `base` falls back to the target directory, which
|
|
45
|
+
would make every stored path look stale.
|
|
46
|
+
|
|
47
|
+
### Changed
|
|
48
|
+
|
|
49
|
+
- An unreadable directory now suppresses the purge for that run (and is
|
|
50
|
+
reported on stderr, counting toward `errors`). `os.walk` hands such a
|
|
51
|
+
directory to `onerror` and otherwise continues silently, so a subtree the
|
|
52
|
+
walk could not enter is indistinguishable from one that was deleted — which
|
|
53
|
+
would have taken its still-present files' rows with it. Indexing still
|
|
54
|
+
proceeds with whatever was readable; only the destructive half backs off.
|
|
55
|
+
- The file walk prunes as it goes (`os.walk`) instead of walking everything and
|
|
56
|
+
filtering afterwards, so `node_modules`/`.venv` subtrees are no longer
|
|
57
|
+
traversed in full. Collection order is now sorted, making a run reproducible.
|
|
58
|
+
- `code_reindex` returns a `purged` count alongside the existing counters.
|
|
59
|
+
|
|
9
60
|
## [0.8.0] - 2026-07-31
|
|
10
61
|
|
|
11
62
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: witan-code
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.1
|
|
4
4
|
Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
|
|
5
5
|
Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
|
|
6
6
|
Project-URL: Repository, https://github.com/mitodl/agent-kit
|
|
@@ -573,9 +573,38 @@ Manual install: register the bare commands directly under the matching event
|
|
|
573
573
|
in `settings.json` (see the linked README for the exact JSON) — there are no
|
|
574
574
|
scripts to symlink.
|
|
575
575
|
|
|
576
|
+
## What gets indexed
|
|
577
|
+
|
|
578
|
+
The walk skips the usual noise directories (`.git`, `node_modules`, `.venv`,
|
|
579
|
+
`__pycache__`, `dist`, `build`, the various caches) and — importantly — does
|
|
580
|
+
not descend into a **nested checkout**: any subdirectory containing a `.git`
|
|
581
|
+
entry belongs to a different repository. That covers linked worktrees
|
|
582
|
+
(`.claude/worktrees/<name>/`, where `.git` is a *file*, not a directory),
|
|
583
|
+
submodules, and plain clones dropped inside the tree. Their files are that
|
|
584
|
+
repo's; indexing them here would attribute them to this one and leave the
|
|
585
|
+
store serving stale copies of itself under a second set of paths.
|
|
586
|
+
|
|
587
|
+
Indexing *from inside* a worktree still works normally — only descending into
|
|
588
|
+
one from the parent is refused — so the hooks keep indexing while an agent
|
|
589
|
+
works on a branch.
|
|
590
|
+
|
|
576
591
|
## Incremental indexing
|
|
577
592
|
|
|
578
593
|
Each `CodeFile` stores a sha256 `contentHash`. On reindex, if the hash is
|
|
579
594
|
unchanged the file is skipped. Otherwise its `Symbol`s and `CodeFile` are
|
|
580
595
|
deleted (as separate `delete.gq` calls — Omnigraph cannot mix deletes with
|
|
581
596
|
inserts in one query) and then re-parsed and re-inserted.
|
|
597
|
+
|
|
598
|
+
A **full-repo** index additionally purges rows for files the repo no longer
|
|
599
|
+
has — deleted, or newly excluded by the rules above. Membership is decided by
|
|
600
|
+
the set of files just collected, not by whether the file still exists on disk:
|
|
601
|
+
a linked worktree's files are very much on disk, they simply aren't this
|
|
602
|
+
repo's. The count is reported as `purged=N` when non-zero. Purging requires a
|
|
603
|
+
confirmed git root; indexing a subpath, a single file (the reindex hook), a
|
|
604
|
+
directory that isn't a git checkout, or a directory the walk could not fully
|
|
605
|
+
read never purges.
|
|
606
|
+
|
|
607
|
+
Nor does a **shared cluster graph** (`is_remote`). There the default branch is
|
|
608
|
+
indexed by CI and everyone else reads it — reconciling it against one
|
|
609
|
+
developer's working tree (a sparse checkout, a stale one, uncommitted
|
|
610
|
+
deletions) would purge files for every other user of that graph.
|
|
@@ -541,9 +541,38 @@ Manual install: register the bare commands directly under the matching event
|
|
|
541
541
|
in `settings.json` (see the linked README for the exact JSON) — there are no
|
|
542
542
|
scripts to symlink.
|
|
543
543
|
|
|
544
|
+
## What gets indexed
|
|
545
|
+
|
|
546
|
+
The walk skips the usual noise directories (`.git`, `node_modules`, `.venv`,
|
|
547
|
+
`__pycache__`, `dist`, `build`, the various caches) and — importantly — does
|
|
548
|
+
not descend into a **nested checkout**: any subdirectory containing a `.git`
|
|
549
|
+
entry belongs to a different repository. That covers linked worktrees
|
|
550
|
+
(`.claude/worktrees/<name>/`, where `.git` is a *file*, not a directory),
|
|
551
|
+
submodules, and plain clones dropped inside the tree. Their files are that
|
|
552
|
+
repo's; indexing them here would attribute them to this one and leave the
|
|
553
|
+
store serving stale copies of itself under a second set of paths.
|
|
554
|
+
|
|
555
|
+
Indexing *from inside* a worktree still works normally — only descending into
|
|
556
|
+
one from the parent is refused — so the hooks keep indexing while an agent
|
|
557
|
+
works on a branch.
|
|
558
|
+
|
|
544
559
|
## Incremental indexing
|
|
545
560
|
|
|
546
561
|
Each `CodeFile` stores a sha256 `contentHash`. On reindex, if the hash is
|
|
547
562
|
unchanged the file is skipped. Otherwise its `Symbol`s and `CodeFile` are
|
|
548
563
|
deleted (as separate `delete.gq` calls — Omnigraph cannot mix deletes with
|
|
549
564
|
inserts in one query) and then re-parsed and re-inserted.
|
|
565
|
+
|
|
566
|
+
A **full-repo** index additionally purges rows for files the repo no longer
|
|
567
|
+
has — deleted, or newly excluded by the rules above. Membership is decided by
|
|
568
|
+
the set of files just collected, not by whether the file still exists on disk:
|
|
569
|
+
a linked worktree's files are very much on disk, they simply aren't this
|
|
570
|
+
repo's. The count is reported as `purged=N` when non-zero. Purging requires a
|
|
571
|
+
confirmed git root; indexing a subpath, a single file (the reindex hook), a
|
|
572
|
+
directory that isn't a git checkout, or a directory the walk could not fully
|
|
573
|
+
read never purges.
|
|
574
|
+
|
|
575
|
+
Nor does a **shared cluster graph** (`is_remote`). There the default branch is
|
|
576
|
+
indexed by CI and everyone else reads it — reconciling it against one
|
|
577
|
+
developer's working tree (a sparse checkout, a stale one, uncommitted
|
|
578
|
+
deletions) would purge files for every other user of that graph.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "witan-code"
|
|
3
|
-
version = "0.8.
|
|
3
|
+
version = "0.8.1"
|
|
4
4
|
description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "BSD-3-Clause"
|
|
@@ -90,7 +90,7 @@ testpaths = ["tests"]
|
|
|
90
90
|
packages = ["witan_code"]
|
|
91
91
|
|
|
92
92
|
[tool.bumpversion]
|
|
93
|
-
current_version = "0.8.
|
|
93
|
+
current_version = "0.8.1"
|
|
94
94
|
allow_dirty = true
|
|
95
95
|
|
|
96
96
|
[[tool.bumpversion.files]]
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""What counts as "this repo's files", and what happens to rows that stop counting.
|
|
2
|
+
|
|
3
|
+
A linked worktree or submodule under the repo is a *different* checkout, so its
|
|
4
|
+
files must not be attributed to this repo — and once excluded, the rows already
|
|
5
|
+
written for them have to go, or the store keeps serving stale copies of the
|
|
6
|
+
repo to itself (which is exactly how one ended up 74% duplicates).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import subprocess
|
|
10
|
+
from types import SimpleNamespace
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from witan_code import indexer
|
|
14
|
+
|
|
15
|
+
from .conftest import requires_stack
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _repo(tmp_path: Path, monkeypatch, name: str = "cg") -> Path:
|
|
19
|
+
"""A real git checkout — purging is gated on a confirmed git root, so a
|
|
20
|
+
bare directory would (correctly) never purge and the tests would pass
|
|
21
|
+
vacuously."""
|
|
22
|
+
monkeypatch.setenv("WITAN_REPO", f"https://github.com/test/{name}")
|
|
23
|
+
monkeypatch.setenv("WITAN_CODE_DIR", str(tmp_path / "code"))
|
|
24
|
+
src = tmp_path / "repo"
|
|
25
|
+
src.mkdir()
|
|
26
|
+
subprocess.run(["git", "init", "-q", "-b", "main"], cwd=src, check=True)
|
|
27
|
+
(src / "a.py").write_text("def a():\n return 1\n")
|
|
28
|
+
return src
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _files(store, cfg) -> set[str]:
|
|
32
|
+
"""Repo-relative paths currently indexed in ``store``."""
|
|
33
|
+
from witan_code.graph import OmnigraphClient
|
|
34
|
+
|
|
35
|
+
rows = OmnigraphClient(str(store), cfg.queries_dir).read(
|
|
36
|
+
"code_read.gq", "all_file_hashes", {}
|
|
37
|
+
)
|
|
38
|
+
return {r["slug"].split("#", 1)[1] for r in rows}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# ── _collect_files: what the walk descends into ──────────────────────────────
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_nested_worktree_is_not_collected(tmp_path):
|
|
45
|
+
"""A linked worktree's `.git` is a FILE, not a directory — the `.git` entry
|
|
46
|
+
in _SKIP_DIRS never matched it, which is how these got indexed."""
|
|
47
|
+
(tmp_path / "a.py").write_text("x = 1\n")
|
|
48
|
+
wt = tmp_path / ".claude" / "worktrees" / "feature-x"
|
|
49
|
+
wt.mkdir(parents=True)
|
|
50
|
+
(wt / ".git").write_text("gitdir: /somewhere/.git/worktrees/feature-x\n")
|
|
51
|
+
(wt / "copy.py").write_text("x = 1\n")
|
|
52
|
+
|
|
53
|
+
collected = indexer._collect_files(tmp_path)
|
|
54
|
+
|
|
55
|
+
assert tmp_path / "a.py" in collected
|
|
56
|
+
assert not any("worktrees" in p.as_posix() for p in collected)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_nested_clone_with_a_git_directory_is_not_collected(tmp_path):
|
|
60
|
+
(tmp_path / "a.py").write_text("x = 1\n")
|
|
61
|
+
vendored = tmp_path / "vendored"
|
|
62
|
+
(vendored / ".git").mkdir(parents=True)
|
|
63
|
+
(vendored / "dep.py").write_text("x = 1\n")
|
|
64
|
+
|
|
65
|
+
collected = indexer._collect_files(tmp_path)
|
|
66
|
+
|
|
67
|
+
assert collected == [tmp_path / "a.py"]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_indexing_from_inside_a_worktree_still_works(tmp_path):
|
|
71
|
+
"""The target root is never a pruning candidate — only descending into a
|
|
72
|
+
nested checkout from outside is refused. The hooks index from inside a
|
|
73
|
+
worktree whenever an agent works there, so this must keep collecting."""
|
|
74
|
+
wt = tmp_path / "worktrees" / "feature-x"
|
|
75
|
+
wt.mkdir(parents=True)
|
|
76
|
+
(wt / ".git").write_text("gitdir: /somewhere\n")
|
|
77
|
+
(wt / "own.py").write_text("x = 1\n")
|
|
78
|
+
|
|
79
|
+
assert indexer._collect_files(wt) == [wt / "own.py"]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_skip_dirs_are_pruned_not_merely_filtered(tmp_path):
|
|
83
|
+
(tmp_path / "a.py").write_text("x = 1\n")
|
|
84
|
+
junk = tmp_path / "node_modules" / "pkg"
|
|
85
|
+
junk.mkdir(parents=True)
|
|
86
|
+
(junk / "index.js").write_text("var x = 1;\n")
|
|
87
|
+
|
|
88
|
+
assert indexer._collect_files(tmp_path) == [tmp_path / "a.py"]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_collection_order_is_deterministic(tmp_path):
|
|
92
|
+
for name in ("c.py", "a.py", "b.py"):
|
|
93
|
+
(tmp_path / name).write_text("x = 1\n")
|
|
94
|
+
(tmp_path / "sub").mkdir()
|
|
95
|
+
(tmp_path / "sub" / "d.py").write_text("x = 1\n")
|
|
96
|
+
|
|
97
|
+
assert indexer._collect_files(tmp_path) == sorted(indexer._collect_files(tmp_path))
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ── Purging rows that are no longer this repo's ──────────────────────────────
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@requires_stack
|
|
104
|
+
def test_full_index_purges_a_file_deleted_from_disk(tmp_path, monkeypatch):
|
|
105
|
+
from witan_code import config as cfg_mod
|
|
106
|
+
from witan_code import store as store_mod
|
|
107
|
+
|
|
108
|
+
src = _repo(tmp_path, monkeypatch)
|
|
109
|
+
(src / "gone.py").write_text("def gone():\n return 1\n")
|
|
110
|
+
cfg = cfg_mod.load()
|
|
111
|
+
indexer.index_path(src, config=cfg)
|
|
112
|
+
store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
|
|
113
|
+
assert _files(store, cfg) == {"a.py", "gone.py"}
|
|
114
|
+
|
|
115
|
+
(src / "gone.py").unlink()
|
|
116
|
+
stats = indexer.index_path(src, config=cfg)
|
|
117
|
+
|
|
118
|
+
assert stats.purged == 1
|
|
119
|
+
assert _files(store, cfg) == {"a.py"}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@requires_stack
|
|
123
|
+
def test_full_index_purges_files_that_became_a_nested_checkout(tmp_path, monkeypatch):
|
|
124
|
+
"""The case an on-disk existence check cannot catch: the files are still
|
|
125
|
+
there, they just stopped being this repo's."""
|
|
126
|
+
from witan_code import config as cfg_mod
|
|
127
|
+
from witan_code import store as store_mod
|
|
128
|
+
|
|
129
|
+
src = _repo(tmp_path, monkeypatch)
|
|
130
|
+
nested = src / "sub"
|
|
131
|
+
nested.mkdir()
|
|
132
|
+
(nested / "b.py").write_text("def b():\n return 1\n")
|
|
133
|
+
cfg = cfg_mod.load()
|
|
134
|
+
indexer.index_path(src, config=cfg)
|
|
135
|
+
store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
|
|
136
|
+
assert _files(store, cfg) == {"a.py", "sub/b.py"}
|
|
137
|
+
|
|
138
|
+
# `sub/` becomes a linked worktree; its file is untouched on disk.
|
|
139
|
+
(nested / ".git").write_text("gitdir: /somewhere\n")
|
|
140
|
+
stats = indexer.index_path(src, config=cfg)
|
|
141
|
+
|
|
142
|
+
assert (nested / "b.py").exists()
|
|
143
|
+
assert stats.purged == 1
|
|
144
|
+
assert _files(store, cfg) == {"a.py"}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@requires_stack
|
|
148
|
+
def test_indexing_a_subpath_purges_nothing(tmp_path, monkeypatch):
|
|
149
|
+
"""Everything outside the subpath is legitimately uncollected — treating
|
|
150
|
+
that as stale would empty the store on every PostToolUse single-file run."""
|
|
151
|
+
from witan_code import config as cfg_mod
|
|
152
|
+
from witan_code import store as store_mod
|
|
153
|
+
|
|
154
|
+
src = _repo(tmp_path, monkeypatch)
|
|
155
|
+
sub = src / "pkg"
|
|
156
|
+
sub.mkdir()
|
|
157
|
+
(sub / "b.py").write_text("def b():\n return 1\n")
|
|
158
|
+
cfg = cfg_mod.load()
|
|
159
|
+
indexer.index_path(src, config=cfg)
|
|
160
|
+
store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
|
|
161
|
+
|
|
162
|
+
stats = indexer.index_path(sub, config=cfg)
|
|
163
|
+
|
|
164
|
+
assert stats.purged == 0
|
|
165
|
+
assert _files(store, cfg) == {"a.py", "pkg/b.py"}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@requires_stack
|
|
169
|
+
def test_purge_survives_a_forced_reindex(tmp_path, monkeypatch):
|
|
170
|
+
"""`force` skips the hash comparison but must still read the stored file
|
|
171
|
+
set — otherwise a --force run silently stops purging."""
|
|
172
|
+
from witan_code import config as cfg_mod
|
|
173
|
+
from witan_code import store as store_mod
|
|
174
|
+
|
|
175
|
+
src = _repo(tmp_path, monkeypatch)
|
|
176
|
+
(src / "gone.py").write_text("def gone():\n return 1\n")
|
|
177
|
+
cfg = cfg_mod.load()
|
|
178
|
+
indexer.index_path(src, config=cfg)
|
|
179
|
+
store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
|
|
180
|
+
|
|
181
|
+
(src / "gone.py").unlink()
|
|
182
|
+
stats = indexer.index_path(src, force=True, config=cfg)
|
|
183
|
+
|
|
184
|
+
assert stats.purged == 1
|
|
185
|
+
assert _files(store, cfg) == {"a.py"}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@requires_stack
|
|
189
|
+
def test_no_purge_without_a_confirmed_git_root(tmp_path, monkeypatch):
|
|
190
|
+
"""The guard on the destructive path.
|
|
191
|
+
|
|
192
|
+
Without git, `base` falls back to the target directory, so `full_repo` is
|
|
193
|
+
true for ANY directory indexed. Combined with a WITAN_REPO override (one
|
|
194
|
+
slug, two different bases) an unguarded purge would delete every row whose
|
|
195
|
+
path was stored relative to the real root — i.e. empty the store.
|
|
196
|
+
"""
|
|
197
|
+
from witan_code import config as cfg_mod
|
|
198
|
+
from witan_code import store as store_mod
|
|
199
|
+
|
|
200
|
+
monkeypatch.setenv("WITAN_REPO", "https://github.com/test/nogit")
|
|
201
|
+
monkeypatch.setenv("WITAN_CODE_DIR", str(tmp_path / "code"))
|
|
202
|
+
src = tmp_path / "plain" # deliberately NOT a git checkout
|
|
203
|
+
(src / "pkg").mkdir(parents=True)
|
|
204
|
+
(src / "a.py").write_text("def a():\n return 1\n")
|
|
205
|
+
(src / "pkg" / "b.py").write_text("def b():\n return 1\n")
|
|
206
|
+
|
|
207
|
+
cfg = cfg_mod.load()
|
|
208
|
+
indexer.index_path(src, config=cfg)
|
|
209
|
+
store = store_mod.store_for_repo("https://github.com/test/nogit", cfg)
|
|
210
|
+
assert _files(store, cfg) == {"a.py", "pkg/b.py"}
|
|
211
|
+
|
|
212
|
+
stats = indexer.index_path(src / "pkg", config=cfg)
|
|
213
|
+
|
|
214
|
+
assert stats.purged == 0
|
|
215
|
+
# Both original rows survive — nothing was deleted, which is the point.
|
|
216
|
+
# This run also ADDS a duplicate `b.py` (paths relative to the subdirectory
|
|
217
|
+
# base rather than the real root): a pre-existing quirk of overriding
|
|
218
|
+
# WITAN_REPO without a git root, unrelated to purging. Asserted as a subset
|
|
219
|
+
# so this test pins the guard and not that quirk.
|
|
220
|
+
assert {"a.py", "pkg/b.py"} <= _files(store, cfg)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@requires_stack
|
|
224
|
+
def test_unreadable_directory_suppresses_the_purge(tmp_path, monkeypatch):
|
|
225
|
+
"""A subtree the walk cannot read looks exactly like a deleted one.
|
|
226
|
+
|
|
227
|
+
os.walk reports such a directory to `onerror` and otherwise carries on
|
|
228
|
+
silently, so without this guard an unreadable subtree would take its
|
|
229
|
+
still-present files' rows with it — a permission blip turning into data
|
|
230
|
+
loss. Indexing still proceeds with whatever was readable.
|
|
231
|
+
"""
|
|
232
|
+
from witan_code import config as cfg_mod
|
|
233
|
+
from witan_code import store as store_mod
|
|
234
|
+
|
|
235
|
+
src = _repo(tmp_path, monkeypatch)
|
|
236
|
+
(src / "sub").mkdir()
|
|
237
|
+
(src / "sub" / "b.py").write_text("def b():\n return 1\n")
|
|
238
|
+
cfg = cfg_mod.load()
|
|
239
|
+
indexer.index_path(src, config=cfg)
|
|
240
|
+
store = store_mod.store_for_repo("https://github.com/test/cg", cfg)
|
|
241
|
+
assert _files(store, cfg) == {"a.py", "sub/b.py"}
|
|
242
|
+
|
|
243
|
+
# Simulate `sub/` becoming unreadable: os.walk yields the root only and
|
|
244
|
+
# hands the failure to onerror, exactly as a PermissionError does.
|
|
245
|
+
real_walk = indexer.os.walk
|
|
246
|
+
|
|
247
|
+
def _walk(top, *args, onerror=None, **kwargs):
|
|
248
|
+
for entry in real_walk(top, *args, **kwargs):
|
|
249
|
+
root, dirs, files = entry
|
|
250
|
+
if Path(root) == src:
|
|
251
|
+
dirs[:] = [d for d in dirs if d != "sub"]
|
|
252
|
+
if onerror is not None:
|
|
253
|
+
onerror(PermissionError(13, "Permission denied", str(src / "sub")))
|
|
254
|
+
yield entry
|
|
255
|
+
|
|
256
|
+
monkeypatch.setattr(indexer.os, "walk", _walk)
|
|
257
|
+
stats = indexer.index_path(src, config=cfg)
|
|
258
|
+
|
|
259
|
+
assert stats.errors >= 1 # the unreadable directory is reported, not hidden
|
|
260
|
+
assert stats.purged == 0
|
|
261
|
+
assert _files(store, cfg) == {"a.py", "sub/b.py"}
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
# ── _may_purge: the gate on the one destructive step ─────────────────────────
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _gate(**overrides):
|
|
268
|
+
"""Call `_may_purge` with an authoritative baseline, overriding one clause."""
|
|
269
|
+
kwargs = {
|
|
270
|
+
"full_repo": True,
|
|
271
|
+
"repo_root": Path("/repo"),
|
|
272
|
+
"walk_errors": [],
|
|
273
|
+
"client": SimpleNamespace(is_remote=False),
|
|
274
|
+
}
|
|
275
|
+
kwargs.update(overrides)
|
|
276
|
+
return indexer._may_purge(**kwargs)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def test_purge_allowed_only_when_this_machine_is_authoritative():
|
|
280
|
+
assert _gate() is True
|
|
281
|
+
|
|
282
|
+
# A subpath run collects a fraction of the repo by design.
|
|
283
|
+
assert _gate(full_repo=False) is False
|
|
284
|
+
# No git root => `base` is the target dir, so `full_repo` means nothing.
|
|
285
|
+
assert _gate(repo_root=None) is False
|
|
286
|
+
# An unreadable subtree looks exactly like a deleted one.
|
|
287
|
+
assert _gate(walk_errors=[PermissionError(13, "denied", "/repo/sub")]) is False
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def test_purge_refused_against_a_shared_cluster_graph():
|
|
291
|
+
"""A remote graph is shared: the default branch is indexed by CI and
|
|
292
|
+
everyone else reads it. Reconciling it against one developer's working
|
|
293
|
+
tree — sparse checkout, stale checkout, uncommitted deletions — would
|
|
294
|
+
purge files for every other user of that graph.
|
|
295
|
+
"""
|
|
296
|
+
assert _gate(client=SimpleNamespace(is_remote=True)) is False
|
|
@@ -32,16 +32,22 @@ def write_bindings(
|
|
|
32
32
|
identity: package_map.PackageIdentity | None = None,
|
|
33
33
|
base: Path | None = None,
|
|
34
34
|
branch: str | None = None,
|
|
35
|
+
indexed_files: frozenset[str] | None = None,
|
|
35
36
|
) -> int:
|
|
36
37
|
"""Purge stale bindings and merge in fresh ones for ``repo``.
|
|
37
38
|
|
|
38
39
|
Purging is always per-file: the files (re)parsed this run, the files the
|
|
39
40
|
fresh batch carries bindings for (covers Tier B sources like OpenAPI
|
|
40
|
-
specs, which aren't in ``touched_files``), and — on a full-repo run
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
specs, which aren't in ``touched_files``), and — on a full-repo run — the
|
|
42
|
+
files that are no longer part of the repo. An incremental full-repo index
|
|
43
|
+
skips unchanged files, so a repo-wide purge here would drop those files'
|
|
44
|
+
bindings with nothing in the batch to restore them.
|
|
45
|
+
|
|
46
|
+
``indexed_files`` is the indexer's collected set (repo-relative paths) and
|
|
47
|
+
is the accurate membership test for that last group: a file can still be
|
|
48
|
+
on disk yet no longer belong to this repo — a linked worktree's files are
|
|
49
|
+
the case that motivated it. Callers that don't have the set fall back to
|
|
50
|
+
an on-disk existence check against ``base``, which catches deletions only.
|
|
45
51
|
|
|
46
52
|
Store-level confidence adjustments (self_provided_key, known_provider_package)
|
|
47
53
|
are applied here before writing. The cross-repo half of each signal is
|
|
@@ -90,13 +96,17 @@ def write_bindings(
|
|
|
90
96
|
repo_symbol_rows = _read_repo_symbols(client) if has_endpoint_consumers else []
|
|
91
97
|
|
|
92
98
|
purge_files = set(touched_files) | {b.file for b in bindings}
|
|
93
|
-
if full_repo and base is not None:
|
|
99
|
+
if full_repo and (indexed_files is not None or base is not None):
|
|
100
|
+
|
|
101
|
+
def _gone(rel: str) -> bool:
|
|
102
|
+
if indexed_files is not None:
|
|
103
|
+
return rel not in indexed_files
|
|
104
|
+
return not (base / rel).exists()
|
|
105
|
+
|
|
94
106
|
purge_files |= {
|
|
95
107
|
r["file"]
|
|
96
108
|
for r in all_rows
|
|
97
|
-
if r.get("repo") == repo
|
|
98
|
-
and r.get("file")
|
|
99
|
-
and not (base / r["file"]).exists()
|
|
109
|
+
if r.get("repo") == repo and r.get("file") and _gone(r["file"])
|
|
100
110
|
}
|
|
101
111
|
for rel in sorted(purge_files):
|
|
102
112
|
client.change(
|
|
@@ -591,6 +591,9 @@ def _print_summary(action: str, path: Path, stats: indexer.IndexStats) -> None:
|
|
|
591
591
|
f"scanned={stats.scanned} indexed={stats.indexed} "
|
|
592
592
|
f"skipped={stats.skipped} symbols={stats.symbols} "
|
|
593
593
|
f"edges={stats.edges} bindings={stats.bindings} errors={stats.errors}"
|
|
594
|
+
# Only when it happened: a purge is newsworthy (rows were deleted),
|
|
595
|
+
# but printing purged=0 on every routine index is noise.
|
|
596
|
+
+ (f" purged={stats.purged}" if stats.purged else "")
|
|
594
597
|
)
|
|
595
598
|
|
|
596
599
|
|
|
@@ -12,7 +12,9 @@ and will miss dynamic dispatch and produce occasional false links.
|
|
|
12
12
|
import functools
|
|
13
13
|
import hashlib
|
|
14
14
|
import importlib
|
|
15
|
+
import os
|
|
15
16
|
import sys
|
|
17
|
+
from collections.abc import Callable
|
|
16
18
|
from dataclasses import dataclass, field
|
|
17
19
|
from pathlib import Path
|
|
18
20
|
|
|
@@ -241,6 +243,11 @@ class IndexStats:
|
|
|
241
243
|
edges: int = 0
|
|
242
244
|
bindings: int = 0
|
|
243
245
|
errors: int = 0
|
|
246
|
+
purged: int = 0
|
|
247
|
+
"""Files dropped from the store because they are no longer part of the
|
|
248
|
+
repo — deleted, or newly excluded (a nested checkout, a skipped
|
|
249
|
+
directory). Full-repo runs only; reported so a mass cleanup is visible
|
|
250
|
+
rather than silent."""
|
|
244
251
|
|
|
245
252
|
|
|
246
253
|
# ── Public entry points ───────────────────────────────────────────
|
|
@@ -279,11 +286,12 @@ def index_path(
|
|
|
279
286
|
client.ensure_branch()
|
|
280
287
|
|
|
281
288
|
# One query for all existing file hashes → the incremental skip check is
|
|
282
|
-
# in-memory, not a query per file.
|
|
289
|
+
# in-memory, not a query per file. Read even under `force` (where the
|
|
290
|
+
# hashes go unused): a full-repo run also needs the stored file set to
|
|
291
|
+
# find rows for files that are no longer part of the repo.
|
|
283
292
|
existing: dict[str, str] = {}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
existing[row["slug"]] = row.get("content_hash")
|
|
293
|
+
for row in client.read("code_read.gq", "all_file_hashes", {}):
|
|
294
|
+
existing[row["slug"]] = row.get("content_hash")
|
|
287
295
|
|
|
288
296
|
stats = IndexStats()
|
|
289
297
|
records: list[dict] = []
|
|
@@ -291,7 +299,21 @@ def index_path(
|
|
|
291
299
|
bindings: list[ParsedBinding] = []
|
|
292
300
|
touched_files: list[str] = []
|
|
293
301
|
|
|
294
|
-
|
|
302
|
+
# A directory the walk could not read makes `collected` incomplete in a way
|
|
303
|
+
# that is invisible from the result alone — and the purge below reads
|
|
304
|
+
# "missing from the collected set" as "no longer part of the repo".
|
|
305
|
+
walk_errors: list[OSError] = []
|
|
306
|
+
collected = _collect_files(target, on_error=walk_errors.append)
|
|
307
|
+
for exc in walk_errors:
|
|
308
|
+
stats.errors += 1
|
|
309
|
+
print(f"codegraph: could not read {exc.filename}: {exc}", file=sys.stderr)
|
|
310
|
+
|
|
311
|
+
# Every file the repo should have indexed, changed or not — the authority
|
|
312
|
+
# for the purge below, which `touched_files` is not (an incremental run
|
|
313
|
+
# touches only what changed).
|
|
314
|
+
indexed_rel = {_relative_path(p, base) for p in collected}
|
|
315
|
+
|
|
316
|
+
for path in collected:
|
|
295
317
|
stats.scanned += 1
|
|
296
318
|
try:
|
|
297
319
|
result = _parse_for_index(path, base, slug, existing, force=force)
|
|
@@ -310,10 +332,35 @@ def index_path(
|
|
|
310
332
|
touched_files.append(parsed.path)
|
|
311
333
|
stats.indexed += 1
|
|
312
334
|
|
|
335
|
+
full_repo = target.is_dir() and target.resolve() == base.resolve()
|
|
336
|
+
can_purge = _may_purge(
|
|
337
|
+
full_repo=full_repo,
|
|
338
|
+
repo_root=repo_root,
|
|
339
|
+
walk_errors=walk_errors,
|
|
340
|
+
client=client,
|
|
341
|
+
)
|
|
342
|
+
|
|
313
343
|
# Drop stale data for changed files (new files have nothing to delete), then
|
|
314
344
|
# bulk-load every node and edge in a single omnigraph call.
|
|
315
345
|
for file_id in reindexed_file_ids:
|
|
316
346
|
_delete_file_data(file_id, client)
|
|
347
|
+
|
|
348
|
+
# A full-repo run also drops files the repo no longer has. Membership is
|
|
349
|
+
# decided by the collected set, NOT by whether the file still exists on
|
|
350
|
+
# disk: a linked worktree's files are very much on disk, they simply
|
|
351
|
+
# aren't this repo's to index (see `_is_nested_checkout`). Without this,
|
|
352
|
+
# excluding a path stops adding rows but leaves every row already written
|
|
353
|
+
# — which is how a store ends up majority stale copies of itself.
|
|
354
|
+
# Subpath runs are exempt: everything outside the subpath is legitimately
|
|
355
|
+
# absent from `indexed_rel` and must not be treated as stale.
|
|
356
|
+
if can_purge:
|
|
357
|
+
prefix = f"{slug}#"
|
|
358
|
+
for file_id in sorted(existing):
|
|
359
|
+
rel = file_id[len(prefix) :] if file_id.startswith(prefix) else None
|
|
360
|
+
if rel is not None and rel not in indexed_rel:
|
|
361
|
+
_delete_file_data(file_id, client)
|
|
362
|
+
stats.purged += 1
|
|
363
|
+
|
|
317
364
|
client.load(_dedupe(records), mode="merge")
|
|
318
365
|
|
|
319
366
|
# Cross-repo bridge — a SEPARATE phase after the per-repo store write, so the
|
|
@@ -326,7 +373,6 @@ def index_path(
|
|
|
326
373
|
# (docs/BRANCH_INDEXING.md § Bridge store) rather than skipping the bridge
|
|
327
374
|
# entirely: the shared main view still never sees in-flight bindings, but
|
|
328
375
|
# they're no longer dropped on the floor either.
|
|
329
|
-
full_repo = target.is_dir() and target.resolve() == base.resolve()
|
|
330
376
|
if full_repo:
|
|
331
377
|
bindings.extend(bridge_extractors.extract_repo_bindings(base, slug))
|
|
332
378
|
try:
|
|
@@ -339,6 +385,7 @@ def index_path(
|
|
|
339
385
|
identity=package_map.load(base, slug),
|
|
340
386
|
base=base,
|
|
341
387
|
branch=branch,
|
|
388
|
+
indexed_files=frozenset(indexed_rel) if can_purge else None,
|
|
342
389
|
)
|
|
343
390
|
except Exception as exc: # noqa: BLE001 — bridge is best-effort, never fatal
|
|
344
391
|
print(f"codegraph: bridge update failed: {exc}", file=sys.stderr)
|
|
@@ -371,19 +418,102 @@ def _dedupe(records: list[dict]) -> list[dict]:
|
|
|
371
418
|
return out
|
|
372
419
|
|
|
373
420
|
|
|
374
|
-
def
|
|
421
|
+
def _may_purge(
|
|
422
|
+
*,
|
|
423
|
+
full_repo: bool,
|
|
424
|
+
repo_root: Path | None,
|
|
425
|
+
walk_errors: list[OSError],
|
|
426
|
+
client: OmnigraphClient,
|
|
427
|
+
) -> bool:
|
|
428
|
+
"""Whether this run is entitled to delete rows for files it did not collect.
|
|
429
|
+
|
|
430
|
+
Purging is the one destructive thing an index does, so it asks for more
|
|
431
|
+
than the run being a full-repo one. Every clause answers the same question
|
|
432
|
+
— is this machine's file listing authoritative for the view being written?
|
|
433
|
+
|
|
434
|
+
- ``full_repo``: a subpath run legitimately collects a fraction of the repo.
|
|
435
|
+
- ``repo_root``: without a git root, ``base`` falls back to the target
|
|
436
|
+
directory, making ``full_repo`` true for ANY directory — index a
|
|
437
|
+
subdirectory then and every path stored relative to the real root looks
|
|
438
|
+
stale.
|
|
439
|
+
- ``walk_errors``: a directory the walk could not read is indistinguishable
|
|
440
|
+
from one that was deleted, so its files would go while sitting on disk.
|
|
441
|
+
- ``client.is_remote``: a shared cluster graph is not this machine's to
|
|
442
|
+
reconcile. One developer's working tree — sparse checkout, stale
|
|
443
|
+
checkout, local deletions — would otherwise purge files for everyone on
|
|
444
|
+
the shared ``main`` view. There the default branch is indexed by CI and
|
|
445
|
+
everyone else reads it; a designated writer will need an explicit opt-in
|
|
446
|
+
rather than inheriting the right by being remote.
|
|
447
|
+
"""
|
|
448
|
+
return (
|
|
449
|
+
full_repo and repo_root is not None and not walk_errors and not client.is_remote
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _relative_path(path: Path, base: Path) -> str:
|
|
454
|
+
"""A file's repo-relative path — the second half of its ``repo#rel`` id.
|
|
455
|
+
|
|
456
|
+
Shared by the parse path and the stale-file purge so the two agree on what
|
|
457
|
+
identifies a file; a mismatch there would purge rows that were just written.
|
|
458
|
+
"""
|
|
459
|
+
try:
|
|
460
|
+
return path.resolve().relative_to(base.resolve()).as_posix()
|
|
461
|
+
except ValueError:
|
|
462
|
+
return path.name
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _is_nested_checkout(path: Path) -> bool:
|
|
466
|
+
"""Whether ``path`` is the root of a *different* git checkout.
|
|
467
|
+
|
|
468
|
+
A `.git` entry inside a subdirectory means that subtree belongs to another
|
|
469
|
+
repository — a linked worktree (`.claude/worktrees/<name>/`, where `.git`
|
|
470
|
+
is a *file* pointing back at the parent's `.git/worktrees/`), a submodule
|
|
471
|
+
(also a `.git` file), or a plain clone someone dropped in. Its files are
|
|
472
|
+
that repo's, and indexing them here attributes them to this one.
|
|
473
|
+
"""
|
|
474
|
+
return (path / ".git").exists()
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _collect_files(
|
|
478
|
+
target: Path, *, on_error: Callable[[OSError], None] | None = None
|
|
479
|
+
) -> list[Path]:
|
|
480
|
+
"""Every indexable file under ``target``, sorted.
|
|
481
|
+
|
|
482
|
+
``on_error`` is handed any directory that could not be read. ``os.walk``
|
|
483
|
+
swallows those silently by default, which is fine for indexing (skip what
|
|
484
|
+
you can't read) but NOT for the purge that consumes this list: an
|
|
485
|
+
unreadable subtree would look like a set of deleted files and take their
|
|
486
|
+
rows with it. ``index_path`` passes a collector and declines to purge when
|
|
487
|
+
anything fired.
|
|
488
|
+
"""
|
|
375
489
|
if target.is_file():
|
|
376
490
|
return [target] if target.suffix in _EXT_TO_SPEC else []
|
|
377
491
|
|
|
378
492
|
out: list[Path] = []
|
|
379
|
-
for
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
493
|
+
for root, dirs, files in os.walk(target, onerror=on_error):
|
|
494
|
+
root_path = Path(root)
|
|
495
|
+
# Prune in place, so a skipped directory is never descended into at
|
|
496
|
+
# all. The previous rglob("*") walked every file under node_modules/
|
|
497
|
+
# .venv/.git and discarded them afterwards; this also makes the
|
|
498
|
+
# nested-checkout test cheap, since it runs once per surviving
|
|
499
|
+
# directory rather than once per file.
|
|
500
|
+
#
|
|
501
|
+
# `target` itself is never a pruning candidate (only entries in
|
|
502
|
+
# `dirs` are), so indexing a repo root — or a path *inside* a
|
|
503
|
+
# worktree, which is how the hooks index while an agent works there —
|
|
504
|
+
# still works. Only descending into one from outside is refused.
|
|
505
|
+
dirs[:] = [
|
|
506
|
+
d
|
|
507
|
+
for d in dirs
|
|
508
|
+
if d not in _SKIP_DIRS and not _is_nested_checkout(root_path / d)
|
|
509
|
+
]
|
|
510
|
+
for name in files:
|
|
511
|
+
path = root_path / name
|
|
512
|
+
if path.suffix in _EXT_TO_SPEC:
|
|
513
|
+
out.append(path)
|
|
514
|
+
# os.walk's directory order is arbitrary; sort so a run is reproducible
|
|
515
|
+
# and `_dedupe`'s first-wins tie-break is stable across runs.
|
|
516
|
+
return sorted(out)
|
|
387
517
|
|
|
388
518
|
|
|
389
519
|
# ── Per-file indexing ─────────────────────────────────────────────
|
|
@@ -402,10 +532,7 @@ def _parse_for_index(
|
|
|
402
532
|
raw = path.read_bytes()
|
|
403
533
|
content_hash = hashlib.sha256(raw).hexdigest()
|
|
404
534
|
|
|
405
|
-
|
|
406
|
-
rel = path.resolve().relative_to(base.resolve()).as_posix()
|
|
407
|
-
except ValueError:
|
|
408
|
-
rel = path.name
|
|
535
|
+
rel = _relative_path(path, base)
|
|
409
536
|
file_id = f"{slug}#{rel}"
|
|
410
537
|
|
|
411
538
|
if not force and existing.get(file_id) == content_hash:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|