witan-code 0.11.0__tar.gz → 0.11.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {witan_code-0.11.0 → witan_code-0.11.1}/PKG-INFO +1 -1
- {witan_code-0.11.0 → witan_code-0.11.1}/pyproject.toml +2 -2
- witan_code-0.11.1/tests/test_chunked_load.py +190 -0
- witan_code-0.11.1/witan_code/chunking.py +80 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/graph.py +39 -4
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/indexer.py +47 -1
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/store.py +32 -7
- {witan_code-0.11.0 → witan_code-0.11.1}/.gitignore +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/CHANGELOG.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/README.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/config/claude.json +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/config/copilot.json +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/config/pi.json +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/BRANCH_INDEXING.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/CLI_REFERENCE.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/EDGE_PRECISION_TIERS.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/PACKAGE_MAP.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/STAGE2_STITCHING.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/SYMBOL_FORMAT.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/SYMBOL_TABLE.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/docs/USER_GUIDE.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/install.sh +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/__init__.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/conftest.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_batched_deletes.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_branches.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_bridge.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_cli_dispatch.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_cli_hooks.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_cli_output_format.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_config.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_config_targets.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_context.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_crossrepo.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_edges.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_elicit.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_enrichment.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_github_app.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_graph.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_hooks.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_identity.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_indexer.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_indexer_scope.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_ingest.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_languages.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_maintenance.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_package_map.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_reaper.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_remote_proxy.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_remote_store.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_repo.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_server.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_setup.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_stitch.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_store.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_symbol_table.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_tasks.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_views.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_visualize.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/uv.lock +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/__init__.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/__main__.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/bridge.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/bridge_extractors.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/cli.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/config.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/context.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/edges.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/elicit.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/extensions/pi/codegraph.ts +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/github_app.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/hooks.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/identity.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/ingest.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/maintenance.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/output.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/package_map.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/bridge.gq +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/code_mutations.gq +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/code_read.gq +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/delete.gq +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/bash.scm +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/hcl.scm +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/python.scm +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/sql.scm +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/typescript.scm +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/yaml.scm +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/reaper.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/__init__.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/oidc.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/proxy.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/repo.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/schema/bridge-schema.pg +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/schema/code-schema.pg +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/server.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/setup.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/skills/witan-code/SKILL.md +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/stitch.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/store.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/views.py +0 -0
- {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/visualize.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: witan-code
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.1
|
|
4
4
|
Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
|
|
5
5
|
Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
|
|
6
6
|
Project-URL: Repository, https://github.com/mitodl/agent-kit
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "witan-code"
|
|
3
|
-
version = "0.11.
|
|
3
|
+
version = "0.11.1"
|
|
4
4
|
description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "BSD-3-Clause"
|
|
@@ -102,7 +102,7 @@ testpaths = ["tests"]
|
|
|
102
102
|
packages = ["witan_code"]
|
|
103
103
|
|
|
104
104
|
[tool.bumpversion]
|
|
105
|
-
current_version = "0.11.
|
|
105
|
+
current_version = "0.11.1"
|
|
106
106
|
allow_dirty = true
|
|
107
107
|
|
|
108
108
|
[[tool.bumpversion.files]]
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""A repo-scale load must survive the server's request-body ceiling.
|
|
2
|
+
|
|
3
|
+
`omnigraph load` POSTs its data file as one body, so a large repo got back
|
|
4
|
+
`413 Payload Too Large` against the deployed server while working fine against
|
|
5
|
+
a local store. Splitting it is not just slicing: an edge whose endpoint is in
|
|
6
|
+
neither the store nor its own batch fails the WHOLE load, and the indexer's
|
|
7
|
+
record list routinely has cross-file edges ahead of their target nodes.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from witan_code import indexer
|
|
16
|
+
from witan_code.chunking import LOAD_MAX_BYTES, chunk_records
|
|
17
|
+
|
|
18
|
+
from .conftest import requires_stack
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _node(slug, type_="Symbol", **extra):
|
|
22
|
+
return {"type": type_, "data": {"slug": slug, **extra}}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _edge(kind, src, dst):
|
|
26
|
+
return {"edge": kind, "from": src, "to": dst}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# ── chunk_records ─────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_empty_yields_nothing():
|
|
33
|
+
assert list(chunk_records([])) == []
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_max_bytes_must_be_positive():
|
|
37
|
+
with pytest.raises(ValueError, match="max_bytes must be >= 1"):
|
|
38
|
+
list(chunk_records([_node("a")], 0))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_every_node_precedes_every_edge():
|
|
42
|
+
"""The property the whole design rests on.
|
|
43
|
+
|
|
44
|
+
Input deliberately interleaves them the way `_file_records` does — each
|
|
45
|
+
file's nodes then its edges — with an edge pointing at a node defined in a
|
|
46
|
+
LATER file. Slicing by index would put that edge in an earlier batch than
|
|
47
|
+
its target and fail the load.
|
|
48
|
+
"""
|
|
49
|
+
records = [
|
|
50
|
+
_node("f1::a"),
|
|
51
|
+
_edge("Calls", "f1::a", "f2::b"), # target defined below
|
|
52
|
+
_node("f2::b"),
|
|
53
|
+
_edge("Calls", "f2::b", "f1::a"),
|
|
54
|
+
]
|
|
55
|
+
batches = list(chunk_records(records, LOAD_MAX_BYTES))
|
|
56
|
+
flat = [record for batch in batches for record in batch]
|
|
57
|
+
last_node = max(i for i, r in enumerate(flat) if "type" in r)
|
|
58
|
+
first_edge = min(i for i, r in enumerate(flat) if "type" not in r)
|
|
59
|
+
assert last_node < first_edge
|
|
60
|
+
assert len(flat) == len(records), "no record may be dropped"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_nothing_is_dropped_or_duplicated():
|
|
64
|
+
records = [_node(f"s{i}") for i in range(50)]
|
|
65
|
+
records += [_edge("Calls", f"s{i}", f"s{i + 1}") for i in range(49)]
|
|
66
|
+
flat = [r for batch in chunk_records(records, 200) for r in batch]
|
|
67
|
+
assert flat == [r for r in records if "type" in r] + [
|
|
68
|
+
r for r in records if "type" not in r
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_batches_respect_the_byte_budget():
|
|
73
|
+
records = [_node(f"s{i}", padding="x" * 100) for i in range(40)]
|
|
74
|
+
budget = 500
|
|
75
|
+
batches = list(chunk_records(records, budget))
|
|
76
|
+
assert len(batches) > 1, "the budget must actually force a split"
|
|
77
|
+
for batch in batches:
|
|
78
|
+
if len(batch) == 1:
|
|
79
|
+
continue # an unsplittable single record is allowed to exceed
|
|
80
|
+
size = sum(len(json.dumps(r).encode()) + 1 for r in batch)
|
|
81
|
+
assert size <= budget
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_an_oversized_record_is_yielded_alone_not_dropped():
|
|
85
|
+
big = _node("huge", padding="x" * 5000)
|
|
86
|
+
records = [_node("small"), big, _node("small2")]
|
|
87
|
+
batches = list(chunk_records(records, 100))
|
|
88
|
+
flat = [r for batch in batches for r in batch]
|
|
89
|
+
assert big in flat
|
|
90
|
+
assert [b for b in batches if b == [big]], "oversized record needs its own batch"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ── _defer_content_hashes ─────────────────────────────────────────
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_defer_swaps_in_a_sentinel_and_returns_the_real_rows():
|
|
97
|
+
real_hash = hashlib.sha256(b"x").hexdigest()
|
|
98
|
+
records = [
|
|
99
|
+
{"type": "CodeFile", "data": {"slug": "r#a.py", "content_hash": real_hash}},
|
|
100
|
+
_node("r#a.py::foo"),
|
|
101
|
+
_edge("Defines", "r#a.py", "r#a.py::foo"),
|
|
102
|
+
]
|
|
103
|
+
returned = indexer._defer_content_hashes(records)
|
|
104
|
+
|
|
105
|
+
assert records[0]["data"]["content_hash"] == indexer._PENDING_CONTENT_HASH
|
|
106
|
+
assert returned == [
|
|
107
|
+
{"type": "CodeFile", "data": {"slug": "r#a.py", "content_hash": real_hash}}
|
|
108
|
+
]
|
|
109
|
+
assert records[1:] == [
|
|
110
|
+
_node("r#a.py::foo"),
|
|
111
|
+
_edge("Defines", "r#a.py", "r#a.py::foo"),
|
|
112
|
+
]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_sentinel_can_never_collide_with_a_real_hash():
|
|
116
|
+
"""The skip check is `stored == sha256(file)`; the sentinel must never match."""
|
|
117
|
+
digest = hashlib.sha256(b"anything").hexdigest()
|
|
118
|
+
assert indexer._PENDING_CONTENT_HASH != digest
|
|
119
|
+
assert len(indexer._PENDING_CONTENT_HASH) != len(digest)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ── Against a real store ──────────────────────────────────────────
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@requires_stack
|
|
126
|
+
def test_chunked_load_builds_the_same_graph_as_one_call(sample_repo, monkeypatch):
|
|
127
|
+
"""Force many tiny batches and prove cross-batch edges still resolve.
|
|
128
|
+
|
|
129
|
+
This is the test that would have caught naive slicing: with a 1-record
|
|
130
|
+
budget, every edge is in its own batch and can only resolve against nodes
|
|
131
|
+
already persisted by an earlier one.
|
|
132
|
+
"""
|
|
133
|
+
from witan_code import config as cfg_mod
|
|
134
|
+
from witan_code import graph as graph_mod
|
|
135
|
+
|
|
136
|
+
real_load = graph_mod.OmnigraphClient.load
|
|
137
|
+
|
|
138
|
+
def tiny_batches(self, records, mode="merge", **kwargs):
|
|
139
|
+
return real_load(self, records, mode, max_bytes=1)
|
|
140
|
+
|
|
141
|
+
monkeypatch.setattr(graph_mod.OmnigraphClient, "load", tiny_batches)
|
|
142
|
+
stats = indexer.index_path(sample_repo, config=cfg_mod.load())
|
|
143
|
+
assert stats.indexed >= 1
|
|
144
|
+
assert stats.edges >= 1
|
|
145
|
+
|
|
146
|
+
from witan_code import repo as repo_mod
|
|
147
|
+
from witan_code import store as store_mod
|
|
148
|
+
|
|
149
|
+
cfg = cfg_mod.load()
|
|
150
|
+
client = store_mod.ensure_store(repo_mod.detect(start=sample_repo), cfg).client(cfg)
|
|
151
|
+
hashes = {
|
|
152
|
+
row["slug"]: row["content_hash"]
|
|
153
|
+
for row in client.read("code_read.gq", "all_file_hashes", {})
|
|
154
|
+
}
|
|
155
|
+
assert hashes, "the repo should have indexed files"
|
|
156
|
+
assert indexer._PENDING_CONTENT_HASH not in hashes.values(), (
|
|
157
|
+
"every hash must be committed once the load completed"
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@requires_stack
|
|
162
|
+
def test_a_failed_load_leaves_the_file_reindexable(sample_repo, monkeypatch):
|
|
163
|
+
"""The property the sentinel exists to preserve.
|
|
164
|
+
|
|
165
|
+
If the hash-commit load never happens, the next run must re-index the file
|
|
166
|
+
rather than skip it. Without the sentinel the file would carry its real
|
|
167
|
+
hash from the first load and be skipped forever.
|
|
168
|
+
"""
|
|
169
|
+
from witan_code import config as cfg_mod
|
|
170
|
+
from witan_code import graph as graph_mod
|
|
171
|
+
|
|
172
|
+
real_load = graph_mod.OmnigraphClient.load
|
|
173
|
+
calls = {"n": 0}
|
|
174
|
+
|
|
175
|
+
def fail_the_hash_commit(self, records, mode="merge", **kwargs):
|
|
176
|
+
calls["n"] += 1
|
|
177
|
+
if calls["n"] == 2: # the hash-commit load
|
|
178
|
+
msg = "simulated mid-load failure"
|
|
179
|
+
raise RuntimeError(msg)
|
|
180
|
+
return real_load(self, records, mode, **kwargs)
|
|
181
|
+
|
|
182
|
+
monkeypatch.setattr(graph_mod.OmnigraphClient, "load", fail_the_hash_commit)
|
|
183
|
+
cfg = cfg_mod.load()
|
|
184
|
+
with pytest.raises(RuntimeError, match="simulated mid-load failure"):
|
|
185
|
+
indexer.index_path(sample_repo, config=cfg)
|
|
186
|
+
|
|
187
|
+
monkeypatch.setattr(graph_mod.OmnigraphClient, "load", real_load)
|
|
188
|
+
stats = indexer.index_path(sample_repo, config=cfg_mod.load())
|
|
189
|
+
assert stats.indexed >= 1, "the file must be re-indexed, not skipped"
|
|
190
|
+
assert stats.skipped == 0
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Splitting a bulk load into batches omnigraph-server will actually accept.
|
|
2
|
+
|
|
3
|
+
``omnigraph load`` POSTs its whole data file as one request body, and the
|
|
4
|
+
server buffers it. Above a cap it answers ``413 Payload Too Large: Failed to
|
|
5
|
+
buffer the request body``, which is how a repo-scale index dies against a
|
|
6
|
+
deployed server while working fine against a local store.
|
|
7
|
+
|
|
8
|
+
The cap is not reachable from configuration on either side: it is an axum
|
|
9
|
+
``DefaultBodyLimit``, and neither ``omnigraph-server --help`` nor ``omnigraph
|
|
10
|
+
load --help`` exposes a knob for it. (It is NOT
|
|
11
|
+
``OMNIGRAPH_PER_ACTOR_BYTES_MAX``, which is 256 MiB in the deployed cluster —
|
|
12
|
+
a per-actor admission budget the 413 fires an order of magnitude below.) So
|
|
13
|
+
the split has to happen client-side.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from collections.abc import Iterable, Iterator
|
|
20
|
+
|
|
21
|
+
__all__ = ["LOAD_MAX_BYTES", "chunk_records"]
|
|
22
|
+
|
|
23
|
+
# Bisected against omnigraph 0.8.1 — the version deployed in the cluster:
|
|
24
|
+
# ~26 MiB of records accepted, ~54 MiB refused. The cap is only known as that
|
|
25
|
+
# range, so aim well under its low end rather than close to it.
|
|
26
|
+
LOAD_MAX_BYTES = 8 * 1024 * 1024
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def chunk_records(
|
|
30
|
+
records: Iterable[dict],
|
|
31
|
+
max_bytes: int = LOAD_MAX_BYTES,
|
|
32
|
+
) -> Iterator[list[dict]]:
|
|
33
|
+
"""Yield byte-bounded batches of load records, every node before any edge.
|
|
34
|
+
|
|
35
|
+
ORDER IS LOAD-BEARING HERE, and not for the reason ``change_many``'s is.
|
|
36
|
+
Measured against 0.8.1: an edge resolves against nodes already persisted by
|
|
37
|
+
an earlier load, OR against nodes anywhere in the same batch (position
|
|
38
|
+
within a batch does not matter) — but an endpoint in neither fails the
|
|
39
|
+
WHOLE load with ``dst '...' not found in <Node>``.
|
|
40
|
+
|
|
41
|
+
That rules out slicing the record list as it stands. ``indexer`` builds it
|
|
42
|
+
per file as ``[the file's nodes, the file's edges]`` and concatenates, while
|
|
43
|
+
Calls/References/Imports/Inherits edges routinely point at symbols defined
|
|
44
|
+
in files appearing LATER in the list. Chunking by index would put those
|
|
45
|
+
edges in a batch before their target nodes and break runs that work today —
|
|
46
|
+
unpredictably, since it depends on which repo and which file order. Emitting
|
|
47
|
+
every node first instead makes the resolvable set identical to the
|
|
48
|
+
single-call load this replaces.
|
|
49
|
+
|
|
50
|
+
A record larger than ``max_bytes`` on its own is still yielded, alone: it
|
|
51
|
+
cannot be split, and refusing it here would only trade a server-side 413 for
|
|
52
|
+
a client-side error.
|
|
53
|
+
"""
|
|
54
|
+
if max_bytes < 1:
|
|
55
|
+
msg = f"max_bytes must be >= 1, got {max_bytes}"
|
|
56
|
+
raise ValueError(msg)
|
|
57
|
+
# Partitioned in ONE pass: a repo-scale index passes hundreds of thousands
|
|
58
|
+
# of records, and materializing the input before splitting it held two full
|
|
59
|
+
# pointer lists at once for no benefit.
|
|
60
|
+
nodes: list[dict] = []
|
|
61
|
+
edges: list[dict] = []
|
|
62
|
+
for record in records:
|
|
63
|
+
(nodes if "type" in record else edges).append(record)
|
|
64
|
+
for group in (nodes, edges):
|
|
65
|
+
yield from _batches(group, max_bytes)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _batches(records: list[dict], max_bytes: int) -> Iterator[list[dict]]:
|
|
69
|
+
batch: list[dict] = []
|
|
70
|
+
size = 0
|
|
71
|
+
for record in records:
|
|
72
|
+
# +1 for the newline the JSONL writer adds after each record.
|
|
73
|
+
cost = len(json.dumps(record).encode()) + 1
|
|
74
|
+
if batch and size + cost > max_bytes:
|
|
75
|
+
yield batch
|
|
76
|
+
batch, size = [], 0
|
|
77
|
+
batch.append(record)
|
|
78
|
+
size += cost
|
|
79
|
+
if batch:
|
|
80
|
+
yield batch
|
|
@@ -15,6 +15,7 @@ from pathlib import Path
|
|
|
15
15
|
|
|
16
16
|
from witan_core.omnigraph import OmnigraphClient as _BaseOmnigraphClient
|
|
17
17
|
|
|
18
|
+
from . import chunking
|
|
18
19
|
from . import config as cfg_module
|
|
19
20
|
from . import identity as identity_module
|
|
20
21
|
from . import views
|
|
@@ -140,16 +141,50 @@ class OmnigraphClient(_BaseOmnigraphClient):
|
|
|
140
141
|
connect_retry=connect_retry,
|
|
141
142
|
)
|
|
142
143
|
|
|
143
|
-
def load(
|
|
144
|
-
|
|
144
|
+
def load(
|
|
145
|
+
self,
|
|
146
|
+
records: list[dict],
|
|
147
|
+
mode: str = "merge",
|
|
148
|
+
*,
|
|
149
|
+
max_bytes: int = chunking.LOAD_MAX_BYTES,
|
|
150
|
+
) -> None:
|
|
151
|
+
"""Bulk-load node/edge records via ``omnigraph load``.
|
|
145
152
|
|
|
146
153
|
Each record is a JSONL line: ``{"type": Node, "data": {...}}`` for a
|
|
147
154
|
node or ``{"edge": Edge, "from": key, "to": key}`` for an edge. This
|
|
148
|
-
replaces thousands of per-record ``mutate`` subprocesses
|
|
149
|
-
|
|
155
|
+
replaces thousands of per-record ``mutate`` subprocesses — essential for
|
|
156
|
+
indexing large repositories.
|
|
157
|
+
|
|
158
|
+
Against a ``--server`` the CLI POSTs the data file as ONE request body,
|
|
159
|
+
so a repo-scale load is split into batches under ``max_bytes`` — see
|
|
160
|
+
:mod:`witan_code.chunking` for the 413 this avoids, and for why every
|
|
161
|
+
node has to be written before any edge. A local store reads the file
|
|
162
|
+
directly and has no such limit, but chunking there costs only a few
|
|
163
|
+
extra Lance versions, which ``omnigraph optimize`` reclaims — not worth
|
|
164
|
+
a second code path.
|
|
165
|
+
|
|
166
|
+
★ ``overwrite`` IS DELIBERATELY NEVER CHUNKED. Measured against 0.8.1,
|
|
167
|
+
that mode TRUNCATES the node type rather than replacing matching rows:
|
|
168
|
+
loading one CodeFile row with ``--mode overwrite`` into a store already
|
|
169
|
+
holding a different one left ``rows=1``, not 2. Split it and every batch
|
|
170
|
+
would erase the one before it, so it stays a single call and an
|
|
171
|
+
oversized one keeps failing loudly at the server.
|
|
172
|
+
|
|
173
|
+
ATOMICITY IS TRADED AWAY for the chunked modes, exactly as
|
|
174
|
+
``change_many(chunk_size=...)`` does: batches commit independently and a
|
|
175
|
+
failure part-way leaves the earlier ones applied. Callers depending on
|
|
176
|
+
all-or-nothing must handle it — ``indexer.index_path`` does, by holding
|
|
177
|
+
each file's ``content_hash`` back until every batch has landed.
|
|
150
178
|
"""
|
|
151
179
|
if not records:
|
|
152
180
|
return
|
|
181
|
+
if mode == "overwrite":
|
|
182
|
+
self._load_batch(records, mode)
|
|
183
|
+
return
|
|
184
|
+
for batch in chunking.chunk_records(records, max_bytes):
|
|
185
|
+
self._load_batch(batch, mode)
|
|
186
|
+
|
|
187
|
+
def _load_batch(self, records: list[dict], mode: str) -> None:
|
|
153
188
|
fd, tmp = tempfile.mkstemp(suffix=".jsonl", prefix="codegraph-load-")
|
|
154
189
|
try:
|
|
155
190
|
with os.fdopen(fd, "w") as fh:
|
|
@@ -44,6 +44,12 @@ _QUERIES_TS_DIR = Path(__file__).parent / "queries_ts"
|
|
|
44
44
|
# this is 250 files per commit.
|
|
45
45
|
_DELETE_BATCH_SIZE = 500
|
|
46
46
|
|
|
47
|
+
# Stand-in written into CodeFile.content_hash for the duration of a load, so a
|
|
48
|
+
# run that fails part-way leaves nothing the incremental skip check will match.
|
|
49
|
+
# Any non-sha256 string does; this one is obvious in a store dump. See
|
|
50
|
+
# `_defer_content_hashes`.
|
|
51
|
+
_PENDING_CONTENT_HASH = "pending"
|
|
52
|
+
|
|
47
53
|
|
|
48
54
|
@dataclass(frozen=True)
|
|
49
55
|
class LanguageSpec:
|
|
@@ -386,7 +392,13 @@ def index_path(
|
|
|
386
392
|
stats.purged += 1
|
|
387
393
|
|
|
388
394
|
client.change_many(delete_steps, chunk_size=_DELETE_BATCH_SIZE)
|
|
389
|
-
|
|
395
|
+
# Two loads, not one: the second commits the content hashes the first held
|
|
396
|
+
# back, so a load that dies part-way is re-done rather than skipped forever.
|
|
397
|
+
# See `_defer_content_hashes`.
|
|
398
|
+
deduped = _dedupe(records)
|
|
399
|
+
real_hash_rows = _defer_content_hashes(deduped)
|
|
400
|
+
client.load(deduped, mode="merge")
|
|
401
|
+
client.load(real_hash_rows, mode="merge")
|
|
390
402
|
|
|
391
403
|
# Cross-repo bridge — a SEPARATE phase after the per-repo store write, so the
|
|
392
404
|
# two stores' write locks never nest. A full-repo index (target is the repo
|
|
@@ -419,6 +431,40 @@ def index_path(
|
|
|
419
431
|
return stats
|
|
420
432
|
|
|
421
433
|
|
|
434
|
+
def _defer_content_hashes(records: list[dict]) -> list[dict]:
|
|
435
|
+
"""Hold every CodeFile's ``content_hash`` back until the load has landed.
|
|
436
|
+
|
|
437
|
+
Mutates the CodeFile rows in ``records`` to carry ``_PENDING_CONTENT_HASH``
|
|
438
|
+
and returns copies carrying the REAL hashes, to be loaded once the main load
|
|
439
|
+
succeeds.
|
|
440
|
+
|
|
441
|
+
WHY THIS EXISTS. ``index_path`` deletes a changed file's rows and reloads
|
|
442
|
+
them. While that reload was one atomic ``load``, a failure left the file
|
|
443
|
+
with no recorded hash at all, so the next run re-indexed it — failure was
|
|
444
|
+
self-healing, for free. Chunking the load (``chunking.chunk_records``) takes
|
|
445
|
+
that away: the nodes land, hash and all, in an early batch, and if a later
|
|
446
|
+
batch fails then ``_parse_for_index``'s ``existing.get(file_id) ==
|
|
447
|
+
content_hash`` check SKIPS the file on every subsequent run. Its symbols
|
|
448
|
+
would be present and its edges permanently missing, silently, until someone
|
|
449
|
+
thought to force a full reindex. Chunking also raises the odds of a
|
|
450
|
+
part-way failure, since it turns one request into several.
|
|
451
|
+
|
|
452
|
+
The sentinel restores the old behaviour: it can never equal a real hash (a
|
|
453
|
+
64-char sha256 hexdigest), so a file whose hashes were never committed is
|
|
454
|
+
simply re-indexed. Nothing else reads ``content_hash`` — only this check and
|
|
455
|
+
the informational ``get_file`` query — so a row briefly carrying the
|
|
456
|
+
sentinel is accurate: that file does need re-indexing.
|
|
457
|
+
"""
|
|
458
|
+
real_rows: list[dict] = []
|
|
459
|
+
for record in records:
|
|
460
|
+
if record.get("type") != "CodeFile":
|
|
461
|
+
continue
|
|
462
|
+
data = record["data"]
|
|
463
|
+
real_rows.append({"type": "CodeFile", "data": dict(data)})
|
|
464
|
+
data["content_hash"] = _PENDING_CONTENT_HASH
|
|
465
|
+
return real_rows
|
|
466
|
+
|
|
467
|
+
|
|
422
468
|
def _dedupe(records: list[dict]) -> list[dict]:
|
|
423
469
|
"""Drop duplicate node slugs / edges so one collision can't fail the load.
|
|
424
470
|
|
|
@@ -33,6 +33,8 @@ from fastmcp.client.auth import BearerAuth
|
|
|
33
33
|
from fastmcp.client.transports import StreamableHttpTransport
|
|
34
34
|
from fastmcp.exceptions import ToolError
|
|
35
35
|
|
|
36
|
+
from .. import chunking
|
|
37
|
+
|
|
36
38
|
__all__ = [
|
|
37
39
|
"RemoteStoreClient",
|
|
38
40
|
"RemoteStoreUnsupported",
|
|
@@ -251,16 +253,39 @@ class RemoteStoreClient:
|
|
|
251
253
|
query_file, query_name, params, surface_conflict=surface_conflict
|
|
252
254
|
)
|
|
253
255
|
|
|
254
|
-
def load(
|
|
256
|
+
def load(
|
|
257
|
+
self,
|
|
258
|
+
records: list[dict],
|
|
259
|
+
mode: str = "merge",
|
|
260
|
+
*,
|
|
261
|
+
max_bytes: int = chunking.LOAD_MAX_BYTES,
|
|
262
|
+
) -> None:
|
|
263
|
+
"""Bulk-load records through the MCP tier's ``code_store_load``.
|
|
264
|
+
|
|
265
|
+
Chunked on the same terms as :meth:`graph.OmnigraphClient.load`, for the
|
|
266
|
+
same reason one layer down: ``code_store_load`` hands the records to an
|
|
267
|
+
``omnigraph load`` on the server side, so a repo-scale call runs into
|
|
268
|
+
the same buffered-body ceiling. This transport adds one of its own — the
|
|
269
|
+
records ride as a JSON tool parameter over the MCP session — which the
|
|
270
|
+
byte budget bounds as well.
|
|
271
|
+
|
|
272
|
+
``overwrite`` is not chunked here either; see the note there.
|
|
273
|
+
"""
|
|
255
274
|
if not records:
|
|
256
275
|
return
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
records
|
|
261
|
-
mode=mode,
|
|
262
|
-
view=self.branch,
|
|
276
|
+
batches = (
|
|
277
|
+
[records]
|
|
278
|
+
if mode == "overwrite"
|
|
279
|
+
else chunking.chunk_records(records, max_bytes)
|
|
263
280
|
)
|
|
281
|
+
for batch in batches:
|
|
282
|
+
self._session.call(
|
|
283
|
+
"code_store_load",
|
|
284
|
+
graph=self.graph,
|
|
285
|
+
records=batch,
|
|
286
|
+
mode=mode,
|
|
287
|
+
view=self.branch,
|
|
288
|
+
)
|
|
264
289
|
|
|
265
290
|
def ensure_branch(self) -> None:
|
|
266
291
|
if self.branch is None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|