witan-code 0.10.0__tar.gz → 0.11.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {witan_code-0.10.0 → witan_code-0.11.1}/PKG-INFO +3 -2
- {witan_code-0.10.0 → witan_code-0.11.1}/pyproject.toml +16 -4
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/conftest.py +19 -0
- witan_code-0.11.1/tests/test_batched_deletes.py +134 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_branches.py +30 -0
- witan_code-0.11.1/tests/test_chunked_load.py +190 -0
- witan_code-0.11.1/tests/test_github_app.py +310 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_indexer_scope.py +14 -4
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_ingest.py +6 -2
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_store.py +168 -15
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/bridge.py +15 -4
- witan_code-0.11.1/witan_code/chunking.py +80 -0
- witan_code-0.11.1/witan_code/github_app.py +296 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/graph.py +47 -5
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/indexer.py +73 -8
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/remote/store.py +59 -7
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/server.py +7 -4
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/store.py +134 -77
- {witan_code-0.10.0 → witan_code-0.11.1}/.gitignore +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/CHANGELOG.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/README.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/config/claude.json +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/config/copilot.json +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/config/pi.json +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/BRANCH_INDEXING.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/CLI_REFERENCE.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/EDGE_PRECISION_TIERS.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/PACKAGE_MAP.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/STAGE2_STITCHING.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/SYMBOL_FORMAT.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/SYMBOL_TABLE.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/docs/USER_GUIDE.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/install.sh +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/__init__.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_bridge.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_cli_dispatch.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_cli_hooks.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_cli_output_format.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_config.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_config_targets.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_context.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_crossrepo.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_edges.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_elicit.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_enrichment.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_graph.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_hooks.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_identity.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_indexer.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_languages.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_maintenance.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_package_map.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_reaper.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_remote_proxy.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_remote_store.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_repo.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_server.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_setup.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_stitch.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_symbol_table.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_tasks.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_views.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/tests/test_visualize.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/uv.lock +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/__init__.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/__main__.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/bridge_extractors.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/cli.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/config.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/context.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/edges.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/elicit.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/extensions/pi/codegraph.ts +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/hooks.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/identity.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/ingest.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/maintenance.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/output.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/package_map.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries/bridge.gq +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries/code_mutations.gq +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries/code_read.gq +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries/delete.gq +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries_ts/bash.scm +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries_ts/hcl.scm +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries_ts/python.scm +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries_ts/sql.scm +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries_ts/typescript.scm +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/queries_ts/yaml.scm +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/reaper.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/remote/__init__.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/remote/oidc.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/remote/proxy.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/repo.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/schema/bridge-schema.pg +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/schema/code-schema.pg +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/setup.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/skills/witan-code/SKILL.md +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/stitch.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/views.py +0 -0
- {witan_code-0.10.0 → witan_code-0.11.1}/witan_code/visualize.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: witan-code
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.1
|
|
4
4
|
Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
|
|
5
5
|
Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
|
|
6
6
|
Project-URL: Repository, https://github.com/mitodl/agent-kit
|
|
@@ -13,6 +13,7 @@ Requires-Python: >=3.11
|
|
|
13
13
|
Requires-Dist: agent-config-kit<1,>=0.4
|
|
14
14
|
Requires-Dist: cyclopts<5,>=4
|
|
15
15
|
Requires-Dist: fastmcp<5,>=3.4.2
|
|
16
|
+
Requires-Dist: joserfc<2,>=1.0
|
|
16
17
|
Requires-Dist: pydantic<3,>=2
|
|
17
18
|
Requires-Dist: pyyaml<7,>=6
|
|
18
19
|
Requires-Dist: rich>=13
|
|
@@ -25,7 +26,7 @@ Requires-Dist: tree-sitter-sql<0.4,>=0.3
|
|
|
25
26
|
Requires-Dist: tree-sitter-typescript<0.24,>=0.23
|
|
26
27
|
Requires-Dist: tree-sitter-yaml<0.8,>=0.7
|
|
27
28
|
Requires-Dist: tree-sitter<0.26,>=0.25
|
|
28
|
-
Requires-Dist: witan-core[cli,remote]<1,>=0.
|
|
29
|
+
Requires-Dist: witan-core[cli,remote]<1,>=0.8
|
|
29
30
|
Provides-Extra: tasks
|
|
30
31
|
Requires-Dist: fastmcp-tasks<5,>=4.0.0b1; extra == 'tasks'
|
|
31
32
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "witan-code"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.11.1"
|
|
4
4
|
description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "BSD-3-Clause"
|
|
@@ -15,12 +15,21 @@ dependencies = [
|
|
|
15
15
|
# proxy) behind `witan-code login` and the read commands' remote mode.
|
|
16
16
|
# Costs nothing extra in practice — fastmcp is already required here and
|
|
17
17
|
# itself depends on httpx2, the extra's only other dependency.
|
|
18
|
-
|
|
18
|
+
# >=0.8: the batched reindex deletes call OmnigraphClient.change_many with
|
|
19
|
+
# chunk_size, neither of which 0.7 has. A looser floor lets an external
|
|
20
|
+
# install resolve a witan-core that fails on the first reindex.
|
|
21
|
+
"witan-core[cli,remote]>=0.8,<1",
|
|
19
22
|
"fastmcp>=3.4.2,<5",
|
|
20
23
|
"cyclopts>=4,<5",
|
|
21
24
|
# [targets.<name>] override models (config.py) — also transitively pulled
|
|
22
25
|
# in by fastmcp, declared explicitly since config.py imports it directly.
|
|
23
26
|
"pydantic>=2,<3",
|
|
27
|
+
# RS256 signing for the GitHub App JWT the CI indexer exchanges for a
|
|
28
|
+
# clone credential (github_app.py). Reaches the environment anyway via
|
|
29
|
+
# fastmcp -> authlib -> joserfc, but that chain is nobody's contract: a
|
|
30
|
+
# fastmcp release that drops authlib would break private-repo cloning at
|
|
31
|
+
# runtime, in a CronJob, rather than at resolve time here.
|
|
32
|
+
"joserfc>=1.0,<2",
|
|
24
33
|
# Pinned to 0.25.x: tree-sitter 0.26.0 has a use-after-free in its pyo3 Node
|
|
25
34
|
# binding that segfaults (exit 139) under the heavy .parent()/.children()
|
|
26
35
|
# churn _walk_defs does on real-world-sized files — deterministically on the
|
|
@@ -80,7 +89,10 @@ witan-core = { workspace = true }
|
|
|
80
89
|
# fastmcp-tasks: the `tasks` extra. Without it CI would only ever exercise the
|
|
81
90
|
# extension-absent branch, and `code_reindex`'s task declaration would go
|
|
82
91
|
# untested — the failure mode there is the server refusing to start at all.
|
|
83
|
-
|
|
92
|
+
# cryptography: test_github_app generates a throwaway RSA key so the JWT path
|
|
93
|
+
# is exercised for real rather than mocked. joserfc requires it at runtime, so
|
|
94
|
+
# this only makes an existing import explicit.
|
|
95
|
+
test = ["pytest>=8", "fastmcp-tasks>=4.0.0b1,<5", "cryptography"]
|
|
84
96
|
|
|
85
97
|
[tool.pytest.ini_options]
|
|
86
98
|
pythonpath = ["."]
|
|
@@ -90,7 +102,7 @@ testpaths = ["tests"]
|
|
|
90
102
|
packages = ["witan_code"]
|
|
91
103
|
|
|
92
104
|
[tool.bumpversion]
|
|
93
|
-
current_version = "0.
|
|
105
|
+
current_version = "0.11.1"
|
|
94
106
|
allow_dirty = true
|
|
95
107
|
|
|
96
108
|
[[tool.bumpversion.files]]
|
|
@@ -56,6 +56,25 @@ def _fresh_identity():
|
|
|
56
56
|
identity.reset_cache()
|
|
57
57
|
|
|
58
58
|
|
|
59
|
+
@pytest.fixture(autouse=True)
|
|
60
|
+
def _fresh_git_context():
|
|
61
|
+
"""Forget the memoized git context between tests.
|
|
62
|
+
|
|
63
|
+
``server._git_context`` caches ``detect()``/``store_branch()`` for
|
|
64
|
+
``_GIT_CONTEXT_TTL`` (2s) to amortize git subprocesses across one agent
|
|
65
|
+
turn. Across tests that TTL is a *race*: a test that sets ``WITAN_REPO``
|
|
66
|
+
within 2s of an earlier one gets the earlier test's repo, and its own
|
|
67
|
+
``monkeypatch.setenv`` is silently ignored. Whether that lands depends on
|
|
68
|
+
how fast the preceding tests ran, which is why it reproduced locally and
|
|
69
|
+
not in CI.
|
|
70
|
+
"""
|
|
71
|
+
from witan_code import server
|
|
72
|
+
|
|
73
|
+
server._git_context.clear()
|
|
74
|
+
yield
|
|
75
|
+
server._git_context.clear()
|
|
76
|
+
|
|
77
|
+
|
|
59
78
|
@pytest.fixture
|
|
60
79
|
def sample_repo(tmp_path, monkeypatch):
|
|
61
80
|
"""A tiny source tree plus a configured, isolated code store."""
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""A reindex must clear stale rows in ONE commit, not two per file.
|
|
2
|
+
|
|
3
|
+
The insert side has always been bulk (`client.load`). The DELETE side was not:
|
|
4
|
+
`_delete_file_data` issued two `mutate` calls per changed-or-purged file, so a
|
|
5
|
+
200-file reindex cost 400 Lance versions — which made a reindex the largest
|
|
6
|
+
fragmentation source in the store the bulk load exists to protect.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import subprocess
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
from .conftest import requires_stack
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture
|
|
18
|
+
def git_repo(tmp_path, monkeypatch) -> Path:
|
|
19
|
+
"""A real git checkout — purging is gated on a confirmed git root, so a
|
|
20
|
+
bare directory would (correctly) never purge and the test would pass
|
|
21
|
+
vacuously."""
|
|
22
|
+
monkeypatch.setenv("WITAN_REPO", "https://github.com/test/cgdel")
|
|
23
|
+
monkeypatch.setenv("WITAN_CODE_DIR", str(tmp_path / "code"))
|
|
24
|
+
src = tmp_path / "repo"
|
|
25
|
+
src.mkdir()
|
|
26
|
+
subprocess.run(["git", "init", "-q", "-b", "main"], cwd=src, check=True)
|
|
27
|
+
(src / "keep.py").write_text("def keep():\n return 1\n")
|
|
28
|
+
return src
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.fixture
|
|
32
|
+
def mutates(monkeypatch):
|
|
33
|
+
"""Count `omnigraph mutate` subprocess invocations while in scope."""
|
|
34
|
+
real_run = subprocess.run
|
|
35
|
+
calls: list[list[str]] = []
|
|
36
|
+
|
|
37
|
+
def counting_run(cmd, *args, **kwargs):
|
|
38
|
+
if isinstance(cmd, list) and "mutate" in cmd:
|
|
39
|
+
calls.append(cmd)
|
|
40
|
+
return real_run(cmd, *args, **kwargs)
|
|
41
|
+
|
|
42
|
+
monkeypatch.setattr(subprocess, "run", counting_run)
|
|
43
|
+
|
|
44
|
+
class Counter:
|
|
45
|
+
def reset(self):
|
|
46
|
+
calls.clear()
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def count(self):
|
|
50
|
+
return len(calls)
|
|
51
|
+
|
|
52
|
+
return Counter()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@requires_stack
|
|
56
|
+
def test_reindexing_many_changed_files_is_one_delete_commit(
|
|
57
|
+
sample_repo, mutates, tmp_path
|
|
58
|
+
):
|
|
59
|
+
from witan_code import config as cfg_mod
|
|
60
|
+
from witan_code import indexer
|
|
61
|
+
|
|
62
|
+
cfg = cfg_mod.load()
|
|
63
|
+
for i in range(5):
|
|
64
|
+
(sample_repo / f"mod{i}.py").write_text(f"def f{i}():\n return {i}\n")
|
|
65
|
+
first = indexer.index_path(sample_repo, config=cfg)
|
|
66
|
+
assert first.indexed >= 6
|
|
67
|
+
|
|
68
|
+
# Change every file so all of them are REINDEXED (new files delete nothing).
|
|
69
|
+
for i in range(5):
|
|
70
|
+
(sample_repo / f"mod{i}.py").write_text(f"def f{i}():\n return {i} + 1\n")
|
|
71
|
+
mutates.reset()
|
|
72
|
+
second = indexer.index_path(sample_repo, config=cfg)
|
|
73
|
+
assert second.indexed == 5
|
|
74
|
+
# 5 files x (delete symbols + delete file) = 10 deletes, previously 10
|
|
75
|
+
# commits. Now one, and nothing else on this path mutates.
|
|
76
|
+
assert mutates.count == 1
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@requires_stack
|
|
80
|
+
def test_purged_files_share_the_same_delete_commit(git_repo, mutates):
|
|
81
|
+
from witan_code import config as cfg_mod
|
|
82
|
+
from witan_code import indexer
|
|
83
|
+
|
|
84
|
+
cfg = cfg_mod.load()
|
|
85
|
+
for i in range(4):
|
|
86
|
+
(git_repo / f"gone{i}.py").write_text(f"def g{i}():\n return {i}\n")
|
|
87
|
+
indexer.index_path(git_repo, config=cfg)
|
|
88
|
+
|
|
89
|
+
for i in range(4):
|
|
90
|
+
(git_repo / f"gone{i}.py").unlink()
|
|
91
|
+
mutates.reset()
|
|
92
|
+
stats = indexer.index_path(git_repo, config=cfg)
|
|
93
|
+
# purged files feed the SAME delete_steps list as reindexed ones, so the
|
|
94
|
+
# whole purge is one commit rather than two per file
|
|
95
|
+
assert stats.purged == 4
|
|
96
|
+
assert mutates.count == 1
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@requires_stack
|
|
100
|
+
def test_a_clean_reindex_writes_no_delete_commit(sample_repo, mutates):
|
|
101
|
+
from witan_code import config as cfg_mod
|
|
102
|
+
from witan_code import indexer
|
|
103
|
+
|
|
104
|
+
cfg = cfg_mod.load()
|
|
105
|
+
indexer.index_path(sample_repo, config=cfg)
|
|
106
|
+
mutates.reset()
|
|
107
|
+
stats = indexer.index_path(sample_repo, config=cfg)
|
|
108
|
+
# nothing changed: no deletes to issue, so no mutate at all
|
|
109
|
+
assert stats.indexed == 0
|
|
110
|
+
assert mutates.count == 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@requires_stack
|
|
114
|
+
def test_deletes_are_chunked_so_the_composed_query_stays_bounded(
|
|
115
|
+
sample_repo, mutates, monkeypatch
|
|
116
|
+
):
|
|
117
|
+
from witan_code import config as cfg_mod
|
|
118
|
+
from witan_code import indexer
|
|
119
|
+
|
|
120
|
+
# 2 delete statements per file; a cap of 4 means 2 files per commit.
|
|
121
|
+
monkeypatch.setattr(indexer, "_DELETE_BATCH_SIZE", 4)
|
|
122
|
+
cfg = cfg_mod.load()
|
|
123
|
+
for i in range(5):
|
|
124
|
+
(sample_repo / f"chunk{i}.py").write_text(f"def c{i}():\n return {i}\n")
|
|
125
|
+
indexer.index_path(sample_repo, config=cfg)
|
|
126
|
+
|
|
127
|
+
for i in range(5):
|
|
128
|
+
(sample_repo / f"chunk{i}.py").write_text(f"def c{i}():\n return {i} + 1\n")
|
|
129
|
+
mutates.reset()
|
|
130
|
+
stats = indexer.index_path(sample_repo, config=cfg)
|
|
131
|
+
assert stats.indexed == 5
|
|
132
|
+
# 5 changed files + svc.py is unchanged -> 10 delete statements at 4 per
|
|
133
|
+
# commit = 3 commits (4 + 4 + 2)
|
|
134
|
+
assert mutates.count == 3
|
|
@@ -9,6 +9,7 @@ from witan_code import repo as repo_module
|
|
|
9
9
|
from .conftest import SAMPLE, requires_stack
|
|
10
10
|
|
|
11
11
|
REPO = "https://github.com/test/cg"
|
|
12
|
+
LEAKED_REPO = "https://github.com/test/leaked-git-context"
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
def _git(base, *args):
|
|
@@ -294,6 +295,35 @@ def test_cached_git_refreshes_after_ttl(monkeypatch):
|
|
|
294
295
|
assert srv._cached_detect() == "b"
|
|
295
296
|
|
|
296
297
|
|
|
298
|
+
# These two run as a pair, in order: the first leaves residue the second must
|
|
299
|
+
# not see. Without the autouse ``_fresh_git_context`` fixture the second fails,
|
|
300
|
+
# because the 2s TTL has not expired between two adjacent tests. That is the
|
|
301
|
+
# exact mechanism that made test_crossrepo's fan-out collapse to one repo.
|
|
302
|
+
def test_git_context_survives_within_a_test(monkeypatch):
|
|
303
|
+
# Seed from a known-clean state rather than inheriting whatever the
|
|
304
|
+
# preceding test memoized: test_cached_git_refreshes_after_ttl leaves a
|
|
305
|
+
# fresh "b" behind, which within the 2s TTL would make _cached_detect()
|
|
306
|
+
# below return "b" and fail this test for the wrong reason. That matters
|
|
307
|
+
# precisely when the autouse fixture is toggled off to check that the pair
|
|
308
|
+
# still detects its removal — the residue the next test sees must
|
|
309
|
+
# provably be LEAKED_REPO, set here, and not an unrelated predecessor's.
|
|
310
|
+
from witan_code import server as srv
|
|
311
|
+
|
|
312
|
+
srv._git_context.clear()
|
|
313
|
+
monkeypatch.setattr(srv.repo_module, "detect", lambda: LEAKED_REPO)
|
|
314
|
+
assert srv._cached_detect() == LEAKED_REPO
|
|
315
|
+
assert srv._git_context["detect"][1] == LEAKED_REPO
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def test_git_context_does_not_leak_into_the_next_test():
|
|
319
|
+
"""The previous test's memoized repo must not be visible here."""
|
|
320
|
+
from witan_code import server as srv
|
|
321
|
+
|
|
322
|
+
assert srv._git_context == {}, (
|
|
323
|
+
"a previous test's git context leaked; _fresh_git_context must clear it"
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
297
327
|
@requires_stack
|
|
298
328
|
def test_bridge_client_follows_current_checkout_branch(tmp_path, monkeypatch):
|
|
299
329
|
"""code_interface_providers auto-detects the cwd's repo+branch: sees the
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""A repo-scale load must survive the server's request-body ceiling.
|
|
2
|
+
|
|
3
|
+
`omnigraph load` POSTs its data file as one body, so a large repo got back
|
|
4
|
+
`413 Payload Too Large` against the deployed server while working fine against
|
|
5
|
+
a local store. Splitting it is not just slicing: an edge whose endpoint is in
|
|
6
|
+
neither the store nor its own batch fails the WHOLE load, and the indexer's
|
|
7
|
+
record list routinely has cross-file edges ahead of their target nodes.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from witan_code import indexer
|
|
16
|
+
from witan_code.chunking import LOAD_MAX_BYTES, chunk_records
|
|
17
|
+
|
|
18
|
+
from .conftest import requires_stack
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _node(slug, type_="Symbol", **extra):
|
|
22
|
+
return {"type": type_, "data": {"slug": slug, **extra}}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _edge(kind, src, dst):
|
|
26
|
+
return {"edge": kind, "from": src, "to": dst}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# ── chunk_records ─────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_empty_yields_nothing():
|
|
33
|
+
assert list(chunk_records([])) == []
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_max_bytes_must_be_positive():
|
|
37
|
+
with pytest.raises(ValueError, match="max_bytes must be >= 1"):
|
|
38
|
+
list(chunk_records([_node("a")], 0))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_every_node_precedes_every_edge():
|
|
42
|
+
"""The property the whole design rests on.
|
|
43
|
+
|
|
44
|
+
Input deliberately interleaves them the way `_file_records` does — each
|
|
45
|
+
file's nodes then its edges — with an edge pointing at a node defined in a
|
|
46
|
+
LATER file. Slicing by index would put that edge in an earlier batch than
|
|
47
|
+
its target and fail the load.
|
|
48
|
+
"""
|
|
49
|
+
records = [
|
|
50
|
+
_node("f1::a"),
|
|
51
|
+
_edge("Calls", "f1::a", "f2::b"), # target defined below
|
|
52
|
+
_node("f2::b"),
|
|
53
|
+
_edge("Calls", "f2::b", "f1::a"),
|
|
54
|
+
]
|
|
55
|
+
batches = list(chunk_records(records, LOAD_MAX_BYTES))
|
|
56
|
+
flat = [record for batch in batches for record in batch]
|
|
57
|
+
last_node = max(i for i, r in enumerate(flat) if "type" in r)
|
|
58
|
+
first_edge = min(i for i, r in enumerate(flat) if "type" not in r)
|
|
59
|
+
assert last_node < first_edge
|
|
60
|
+
assert len(flat) == len(records), "no record may be dropped"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_nothing_is_dropped_or_duplicated():
|
|
64
|
+
records = [_node(f"s{i}") for i in range(50)]
|
|
65
|
+
records += [_edge("Calls", f"s{i}", f"s{i + 1}") for i in range(49)]
|
|
66
|
+
flat = [r for batch in chunk_records(records, 200) for r in batch]
|
|
67
|
+
assert flat == [r for r in records if "type" in r] + [
|
|
68
|
+
r for r in records if "type" not in r
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_batches_respect_the_byte_budget():
|
|
73
|
+
records = [_node(f"s{i}", padding="x" * 100) for i in range(40)]
|
|
74
|
+
budget = 500
|
|
75
|
+
batches = list(chunk_records(records, budget))
|
|
76
|
+
assert len(batches) > 1, "the budget must actually force a split"
|
|
77
|
+
for batch in batches:
|
|
78
|
+
if len(batch) == 1:
|
|
79
|
+
continue # an unsplittable single record is allowed to exceed
|
|
80
|
+
size = sum(len(json.dumps(r).encode()) + 1 for r in batch)
|
|
81
|
+
assert size <= budget
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_an_oversized_record_is_yielded_alone_not_dropped():
|
|
85
|
+
big = _node("huge", padding="x" * 5000)
|
|
86
|
+
records = [_node("small"), big, _node("small2")]
|
|
87
|
+
batches = list(chunk_records(records, 100))
|
|
88
|
+
flat = [r for batch in batches for r in batch]
|
|
89
|
+
assert big in flat
|
|
90
|
+
assert [b for b in batches if b == [big]], "oversized record needs its own batch"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ── _defer_content_hashes ─────────────────────────────────────────
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_defer_swaps_in_a_sentinel_and_returns_the_real_rows():
|
|
97
|
+
real_hash = hashlib.sha256(b"x").hexdigest()
|
|
98
|
+
records = [
|
|
99
|
+
{"type": "CodeFile", "data": {"slug": "r#a.py", "content_hash": real_hash}},
|
|
100
|
+
_node("r#a.py::foo"),
|
|
101
|
+
_edge("Defines", "r#a.py", "r#a.py::foo"),
|
|
102
|
+
]
|
|
103
|
+
returned = indexer._defer_content_hashes(records)
|
|
104
|
+
|
|
105
|
+
assert records[0]["data"]["content_hash"] == indexer._PENDING_CONTENT_HASH
|
|
106
|
+
assert returned == [
|
|
107
|
+
{"type": "CodeFile", "data": {"slug": "r#a.py", "content_hash": real_hash}}
|
|
108
|
+
]
|
|
109
|
+
assert records[1:] == [
|
|
110
|
+
_node("r#a.py::foo"),
|
|
111
|
+
_edge("Defines", "r#a.py", "r#a.py::foo"),
|
|
112
|
+
]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_sentinel_can_never_collide_with_a_real_hash():
|
|
116
|
+
"""The skip check is `stored == sha256(file)`; the sentinel must never match."""
|
|
117
|
+
digest = hashlib.sha256(b"anything").hexdigest()
|
|
118
|
+
assert indexer._PENDING_CONTENT_HASH != digest
|
|
119
|
+
assert len(indexer._PENDING_CONTENT_HASH) != len(digest)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ── Against a real store ──────────────────────────────────────────
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@requires_stack
|
|
126
|
+
def test_chunked_load_builds_the_same_graph_as_one_call(sample_repo, monkeypatch):
|
|
127
|
+
"""Force many tiny batches and prove cross-batch edges still resolve.
|
|
128
|
+
|
|
129
|
+
This is the test that would have caught naive slicing: with a 1-record
|
|
130
|
+
budget, every edge is in its own batch and can only resolve against nodes
|
|
131
|
+
already persisted by an earlier one.
|
|
132
|
+
"""
|
|
133
|
+
from witan_code import config as cfg_mod
|
|
134
|
+
from witan_code import graph as graph_mod
|
|
135
|
+
|
|
136
|
+
real_load = graph_mod.OmnigraphClient.load
|
|
137
|
+
|
|
138
|
+
def tiny_batches(self, records, mode="merge", **kwargs):
|
|
139
|
+
return real_load(self, records, mode, max_bytes=1)
|
|
140
|
+
|
|
141
|
+
monkeypatch.setattr(graph_mod.OmnigraphClient, "load", tiny_batches)
|
|
142
|
+
stats = indexer.index_path(sample_repo, config=cfg_mod.load())
|
|
143
|
+
assert stats.indexed >= 1
|
|
144
|
+
assert stats.edges >= 1
|
|
145
|
+
|
|
146
|
+
from witan_code import repo as repo_mod
|
|
147
|
+
from witan_code import store as store_mod
|
|
148
|
+
|
|
149
|
+
cfg = cfg_mod.load()
|
|
150
|
+
client = store_mod.ensure_store(repo_mod.detect(start=sample_repo), cfg).client(cfg)
|
|
151
|
+
hashes = {
|
|
152
|
+
row["slug"]: row["content_hash"]
|
|
153
|
+
for row in client.read("code_read.gq", "all_file_hashes", {})
|
|
154
|
+
}
|
|
155
|
+
assert hashes, "the repo should have indexed files"
|
|
156
|
+
assert indexer._PENDING_CONTENT_HASH not in hashes.values(), (
|
|
157
|
+
"every hash must be committed once the load completed"
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@requires_stack
|
|
162
|
+
def test_a_failed_load_leaves_the_file_reindexable(sample_repo, monkeypatch):
|
|
163
|
+
"""The property the sentinel exists to preserve.
|
|
164
|
+
|
|
165
|
+
If the hash-commit load never happens, the next run must re-index the file
|
|
166
|
+
rather than skip it. Without the sentinel the file would carry its real
|
|
167
|
+
hash from the first load and be skipped forever.
|
|
168
|
+
"""
|
|
169
|
+
from witan_code import config as cfg_mod
|
|
170
|
+
from witan_code import graph as graph_mod
|
|
171
|
+
|
|
172
|
+
real_load = graph_mod.OmnigraphClient.load
|
|
173
|
+
calls = {"n": 0}
|
|
174
|
+
|
|
175
|
+
def fail_the_hash_commit(self, records, mode="merge", **kwargs):
|
|
176
|
+
calls["n"] += 1
|
|
177
|
+
if calls["n"] == 2: # the hash-commit load
|
|
178
|
+
msg = "simulated mid-load failure"
|
|
179
|
+
raise RuntimeError(msg)
|
|
180
|
+
return real_load(self, records, mode, **kwargs)
|
|
181
|
+
|
|
182
|
+
monkeypatch.setattr(graph_mod.OmnigraphClient, "load", fail_the_hash_commit)
|
|
183
|
+
cfg = cfg_mod.load()
|
|
184
|
+
with pytest.raises(RuntimeError, match="simulated mid-load failure"):
|
|
185
|
+
indexer.index_path(sample_repo, config=cfg)
|
|
186
|
+
|
|
187
|
+
monkeypatch.setattr(graph_mod.OmnigraphClient, "load", real_load)
|
|
188
|
+
stats = indexer.index_path(sample_repo, config=cfg_mod.load())
|
|
189
|
+
assert stats.indexed >= 1, "the file must be re-indexed, not skipped"
|
|
190
|
+
assert stats.skipped == 0
|