witan-code 0.11.0__tar.gz → 0.11.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. {witan_code-0.11.0 → witan_code-0.11.1}/PKG-INFO +1 -1
  2. {witan_code-0.11.0 → witan_code-0.11.1}/pyproject.toml +2 -2
  3. witan_code-0.11.1/tests/test_chunked_load.py +190 -0
  4. witan_code-0.11.1/witan_code/chunking.py +80 -0
  5. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/graph.py +39 -4
  6. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/indexer.py +47 -1
  7. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/store.py +32 -7
  8. {witan_code-0.11.0 → witan_code-0.11.1}/.gitignore +0 -0
  9. {witan_code-0.11.0 → witan_code-0.11.1}/CHANGELOG.md +0 -0
  10. {witan_code-0.11.0 → witan_code-0.11.1}/README.md +0 -0
  11. {witan_code-0.11.0 → witan_code-0.11.1}/config/claude.json +0 -0
  12. {witan_code-0.11.0 → witan_code-0.11.1}/config/copilot.json +0 -0
  13. {witan_code-0.11.0 → witan_code-0.11.1}/config/pi.json +0 -0
  14. {witan_code-0.11.0 → witan_code-0.11.1}/docs/BRANCH_INDEXING.md +0 -0
  15. {witan_code-0.11.0 → witan_code-0.11.1}/docs/CLI_REFERENCE.md +0 -0
  16. {witan_code-0.11.0 → witan_code-0.11.1}/docs/EDGE_PRECISION_TIERS.md +0 -0
  17. {witan_code-0.11.0 → witan_code-0.11.1}/docs/PACKAGE_MAP.md +0 -0
  18. {witan_code-0.11.0 → witan_code-0.11.1}/docs/STAGE2_STITCHING.md +0 -0
  19. {witan_code-0.11.0 → witan_code-0.11.1}/docs/SYMBOL_FORMAT.md +0 -0
  20. {witan_code-0.11.0 → witan_code-0.11.1}/docs/SYMBOL_TABLE.md +0 -0
  21. {witan_code-0.11.0 → witan_code-0.11.1}/docs/USER_GUIDE.md +0 -0
  22. {witan_code-0.11.0 → witan_code-0.11.1}/install.sh +0 -0
  23. {witan_code-0.11.0 → witan_code-0.11.1}/tests/__init__.py +0 -0
  24. {witan_code-0.11.0 → witan_code-0.11.1}/tests/conftest.py +0 -0
  25. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_batched_deletes.py +0 -0
  26. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_branches.py +0 -0
  27. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_bridge.py +0 -0
  28. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_cli_dispatch.py +0 -0
  29. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_cli_hooks.py +0 -0
  30. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_cli_output_format.py +0 -0
  31. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_config.py +0 -0
  32. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_config_targets.py +0 -0
  33. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_context.py +0 -0
  34. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_crossrepo.py +0 -0
  35. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_edges.py +0 -0
  36. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_elicit.py +0 -0
  37. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_enrichment.py +0 -0
  38. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_github_app.py +0 -0
  39. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_graph.py +0 -0
  40. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_hooks.py +0 -0
  41. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_identity.py +0 -0
  42. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_indexer.py +0 -0
  43. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_indexer_scope.py +0 -0
  44. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_ingest.py +0 -0
  45. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_languages.py +0 -0
  46. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_maintenance.py +0 -0
  47. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_package_map.py +0 -0
  48. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_reaper.py +0 -0
  49. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_remote_proxy.py +0 -0
  50. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_remote_store.py +0 -0
  51. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_repo.py +0 -0
  52. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_server.py +0 -0
  53. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_setup.py +0 -0
  54. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_stitch.py +0 -0
  55. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_store.py +0 -0
  56. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_symbol_table.py +0 -0
  57. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_tasks.py +0 -0
  58. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_views.py +0 -0
  59. {witan_code-0.11.0 → witan_code-0.11.1}/tests/test_visualize.py +0 -0
  60. {witan_code-0.11.0 → witan_code-0.11.1}/uv.lock +0 -0
  61. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/__init__.py +0 -0
  62. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/__main__.py +0 -0
  63. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/bridge.py +0 -0
  64. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/bridge_extractors.py +0 -0
  65. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/cli.py +0 -0
  66. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/config.py +0 -0
  67. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/context.py +0 -0
  68. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/edges.py +0 -0
  69. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/elicit.py +0 -0
  70. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/extensions/pi/codegraph.ts +0 -0
  71. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/github_app.py +0 -0
  72. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/hooks.py +0 -0
  73. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/identity.py +0 -0
  74. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/ingest.py +0 -0
  75. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/maintenance.py +0 -0
  76. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/output.py +0 -0
  77. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/package_map.py +0 -0
  78. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/bridge.gq +0 -0
  79. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/code_mutations.gq +0 -0
  80. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/code_read.gq +0 -0
  81. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries/delete.gq +0 -0
  82. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/bash.scm +0 -0
  83. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/hcl.scm +0 -0
  84. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/python.scm +0 -0
  85. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/sql.scm +0 -0
  86. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/typescript.scm +0 -0
  87. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/queries_ts/yaml.scm +0 -0
  88. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/reaper.py +0 -0
  89. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/__init__.py +0 -0
  90. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/oidc.py +0 -0
  91. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/remote/proxy.py +0 -0
  92. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/repo.py +0 -0
  93. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/schema/bridge-schema.pg +0 -0
  94. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/schema/code-schema.pg +0 -0
  95. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/server.py +0 -0
  96. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/setup.py +0 -0
  97. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/skills/witan-code/SKILL.md +0 -0
  98. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/stitch.py +0 -0
  99. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/store.py +0 -0
  100. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/views.py +0 -0
  101. {witan_code-0.11.0 → witan_code-0.11.1}/witan_code/visualize.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: witan-code
3
- Version: 0.11.0
3
+ Version: 0.11.1
4
4
  Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
5
5
  Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
6
6
  Project-URL: Repository, https://github.com/mitodl/agent-kit
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "witan-code"
3
- version = "0.11.0"
3
+ version = "0.11.1"
4
4
  description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
5
5
  readme = "README.md"
6
6
  license = "BSD-3-Clause"
@@ -102,7 +102,7 @@ testpaths = ["tests"]
102
102
  packages = ["witan_code"]
103
103
 
104
104
  [tool.bumpversion]
105
- current_version = "0.11.0"
105
+ current_version = "0.11.1"
106
106
  allow_dirty = true
107
107
 
108
108
  [[tool.bumpversion.files]]
@@ -0,0 +1,190 @@
1
+ """A repo-scale load must survive the server's request-body ceiling.
2
+
3
+ `omnigraph load` POSTs its data file as one body, so a large repo got back
4
+ `413 Payload Too Large` against the deployed server while working fine against
5
+ a local store. Splitting it is not just slicing: an edge whose endpoint is in
6
+ neither the store nor its own batch fails the WHOLE load, and the indexer's
7
+ record list routinely has cross-file edges ahead of their target nodes.
8
+ """
9
+
10
+ import hashlib
11
+ import json
12
+
13
+ import pytest
14
+
15
+ from witan_code import indexer
16
+ from witan_code.chunking import LOAD_MAX_BYTES, chunk_records
17
+
18
+ from .conftest import requires_stack
19
+
20
+
21
+ def _node(slug, type_="Symbol", **extra):
22
+ return {"type": type_, "data": {"slug": slug, **extra}}
23
+
24
+
25
+ def _edge(kind, src, dst):
26
+ return {"edge": kind, "from": src, "to": dst}
27
+
28
+
29
+ # ── chunk_records ─────────────────────────────────────────────────
30
+
31
+
32
+ def test_empty_yields_nothing():
33
+ assert list(chunk_records([])) == []
34
+
35
+
36
+ def test_max_bytes_must_be_positive():
37
+ with pytest.raises(ValueError, match="max_bytes must be >= 1"):
38
+ list(chunk_records([_node("a")], 0))
39
+
40
+
41
+ def test_every_node_precedes_every_edge():
42
+ """The property the whole design rests on.
43
+
44
+ Input deliberately interleaves them the way `_file_records` does — each
45
+ file's nodes then its edges — with an edge pointing at a node defined in a
46
+ LATER file. Slicing by index would put that edge in an earlier batch than
47
+ its target and fail the load.
48
+ """
49
+ records = [
50
+ _node("f1::a"),
51
+ _edge("Calls", "f1::a", "f2::b"), # target defined below
52
+ _node("f2::b"),
53
+ _edge("Calls", "f2::b", "f1::a"),
54
+ ]
55
+ batches = list(chunk_records(records, LOAD_MAX_BYTES))
56
+ flat = [record for batch in batches for record in batch]
57
+ last_node = max(i for i, r in enumerate(flat) if "type" in r)
58
+ first_edge = min(i for i, r in enumerate(flat) if "type" not in r)
59
+ assert last_node < first_edge
60
+ assert len(flat) == len(records), "no record may be dropped"
61
+
62
+
63
+ def test_nothing_is_dropped_or_duplicated():
64
+ records = [_node(f"s{i}") for i in range(50)]
65
+ records += [_edge("Calls", f"s{i}", f"s{i + 1}") for i in range(49)]
66
+ flat = [r for batch in chunk_records(records, 200) for r in batch]
67
+ assert flat == [r for r in records if "type" in r] + [
68
+ r for r in records if "type" not in r
69
+ ]
70
+
71
+
72
+ def test_batches_respect_the_byte_budget():
73
+ records = [_node(f"s{i}", padding="x" * 100) for i in range(40)]
74
+ budget = 500
75
+ batches = list(chunk_records(records, budget))
76
+ assert len(batches) > 1, "the budget must actually force a split"
77
+ for batch in batches:
78
+ if len(batch) == 1:
79
+ continue # an unsplittable single record is allowed to exceed
80
+ size = sum(len(json.dumps(r).encode()) + 1 for r in batch)
81
+ assert size <= budget
82
+
83
+
84
+ def test_an_oversized_record_is_yielded_alone_not_dropped():
85
+ big = _node("huge", padding="x" * 5000)
86
+ records = [_node("small"), big, _node("small2")]
87
+ batches = list(chunk_records(records, 100))
88
+ flat = [r for batch in batches for r in batch]
89
+ assert big in flat
90
+ assert [b for b in batches if b == [big]], "oversized record needs its own batch"
91
+
92
+
93
+ # ── _defer_content_hashes ─────────────────────────────────────────
94
+
95
+
96
+ def test_defer_swaps_in_a_sentinel_and_returns_the_real_rows():
97
+ real_hash = hashlib.sha256(b"x").hexdigest()
98
+ records = [
99
+ {"type": "CodeFile", "data": {"slug": "r#a.py", "content_hash": real_hash}},
100
+ _node("r#a.py::foo"),
101
+ _edge("Defines", "r#a.py", "r#a.py::foo"),
102
+ ]
103
+ returned = indexer._defer_content_hashes(records)
104
+
105
+ assert records[0]["data"]["content_hash"] == indexer._PENDING_CONTENT_HASH
106
+ assert returned == [
107
+ {"type": "CodeFile", "data": {"slug": "r#a.py", "content_hash": real_hash}}
108
+ ]
109
+ assert records[1:] == [
110
+ _node("r#a.py::foo"),
111
+ _edge("Defines", "r#a.py", "r#a.py::foo"),
112
+ ]
113
+
114
+
115
+ def test_sentinel_can_never_collide_with_a_real_hash():
116
+ """The skip check is `stored == sha256(file)`; the sentinel must never match."""
117
+ digest = hashlib.sha256(b"anything").hexdigest()
118
+ assert indexer._PENDING_CONTENT_HASH != digest
119
+ assert len(indexer._PENDING_CONTENT_HASH) != len(digest)
120
+
121
+
122
+ # ── Against a real store ──────────────────────────────────────────
123
+
124
+
125
+ @requires_stack
126
+ def test_chunked_load_builds_the_same_graph_as_one_call(sample_repo, monkeypatch):
127
+ """Force many tiny batches and prove cross-batch edges still resolve.
128
+
129
+ This is the test that would have caught naive slicing: with a 1-record
130
+ budget, every edge is in its own batch and can only resolve against nodes
131
+ already persisted by an earlier one.
132
+ """
133
+ from witan_code import config as cfg_mod
134
+ from witan_code import graph as graph_mod
135
+
136
+ real_load = graph_mod.OmnigraphClient.load
137
+
138
+ def tiny_batches(self, records, mode="merge", **kwargs):
139
+ return real_load(self, records, mode, max_bytes=1)
140
+
141
+ monkeypatch.setattr(graph_mod.OmnigraphClient, "load", tiny_batches)
142
+ stats = indexer.index_path(sample_repo, config=cfg_mod.load())
143
+ assert stats.indexed >= 1
144
+ assert stats.edges >= 1
145
+
146
+ from witan_code import repo as repo_mod
147
+ from witan_code import store as store_mod
148
+
149
+ cfg = cfg_mod.load()
150
+ client = store_mod.ensure_store(repo_mod.detect(start=sample_repo), cfg).client(cfg)
151
+ hashes = {
152
+ row["slug"]: row["content_hash"]
153
+ for row in client.read("code_read.gq", "all_file_hashes", {})
154
+ }
155
+ assert hashes, "the repo should have indexed files"
156
+ assert indexer._PENDING_CONTENT_HASH not in hashes.values(), (
157
+ "every hash must be committed once the load completed"
158
+ )
159
+
160
+
161
+ @requires_stack
162
+ def test_a_failed_load_leaves_the_file_reindexable(sample_repo, monkeypatch):
163
+ """The property the sentinel exists to preserve.
164
+
165
+ If the hash-commit load never happens, the next run must re-index the file
166
+ rather than skip it. Without the sentinel the file would carry its real
167
+ hash from the first load and be skipped forever.
168
+ """
169
+ from witan_code import config as cfg_mod
170
+ from witan_code import graph as graph_mod
171
+
172
+ real_load = graph_mod.OmnigraphClient.load
173
+ calls = {"n": 0}
174
+
175
+ def fail_the_hash_commit(self, records, mode="merge", **kwargs):
176
+ calls["n"] += 1
177
+ if calls["n"] == 2: # the hash-commit load
178
+ msg = "simulated mid-load failure"
179
+ raise RuntimeError(msg)
180
+ return real_load(self, records, mode, **kwargs)
181
+
182
+ monkeypatch.setattr(graph_mod.OmnigraphClient, "load", fail_the_hash_commit)
183
+ cfg = cfg_mod.load()
184
+ with pytest.raises(RuntimeError, match="simulated mid-load failure"):
185
+ indexer.index_path(sample_repo, config=cfg)
186
+
187
+ monkeypatch.setattr(graph_mod.OmnigraphClient, "load", real_load)
188
+ stats = indexer.index_path(sample_repo, config=cfg_mod.load())
189
+ assert stats.indexed >= 1, "the file must be re-indexed, not skipped"
190
+ assert stats.skipped == 0
@@ -0,0 +1,80 @@
1
+ """Splitting a bulk load into batches omnigraph-server will actually accept.
2
+
3
+ ``omnigraph load`` POSTs its whole data file as one request body, and the
4
+ server buffers it. Above a cap it answers ``413 Payload Too Large: Failed to
5
+ buffer the request body``, which is how a repo-scale index dies against a
6
+ deployed server while working fine against a local store.
7
+
8
+ The cap is not reachable from configuration on either side: it is an axum
9
+ ``DefaultBodyLimit``, and neither ``omnigraph-server --help`` nor ``omnigraph
10
+ load --help`` exposes a knob for it. (It is NOT
11
+ ``OMNIGRAPH_PER_ACTOR_BYTES_MAX``, which is 256 MiB in the deployed cluster —
12
+ a per-actor admission budget the 413 fires an order of magnitude below.) So
13
+ the split has to happen client-side.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ from collections.abc import Iterable, Iterator
20
+
21
+ __all__ = ["LOAD_MAX_BYTES", "chunk_records"]
22
+
23
+ # Bisected against omnigraph 0.8.1 — the version deployed in the cluster:
24
+ # ~26 MiB of records accepted, ~54 MiB refused. The cap is only known as that
25
+ # range, so aim well under its low end rather than close to it.
26
+ LOAD_MAX_BYTES = 8 * 1024 * 1024
27
+
28
+
29
+ def chunk_records(
30
+ records: Iterable[dict],
31
+ max_bytes: int = LOAD_MAX_BYTES,
32
+ ) -> Iterator[list[dict]]:
33
+ """Yield byte-bounded batches of load records, every node before any edge.
34
+
35
+ ORDER IS LOAD-BEARING HERE, and not for the reason ``change_many``'s is.
36
+ Measured against 0.8.1: an edge resolves against nodes already persisted by
37
+ an earlier load, OR against nodes anywhere in the same batch (position
38
+ within a batch does not matter) — but an endpoint in neither fails the
39
+ WHOLE load with ``dst '...' not found in <Node>``.
40
+
41
+ That rules out slicing the record list as it stands. ``indexer`` builds it
42
+ per file as ``[the file's nodes, the file's edges]`` and concatenates, while
43
+ Calls/References/Imports/Inherits edges routinely point at symbols defined
44
+ in files appearing LATER in the list. Chunking by index would put those
45
+ edges in a batch before their target nodes and break runs that work today —
46
+ unpredictably, since it depends on which repo and which file order. Emitting
47
+ every node first instead makes the resolvable set identical to the
48
+ single-call load this replaces.
49
+
50
+ A record larger than ``max_bytes`` on its own is still yielded, alone: it
51
+ cannot be split, and refusing it here would only trade a server-side 413 for
52
+ a client-side error.
53
+ """
54
+ if max_bytes < 1:
55
+ msg = f"max_bytes must be >= 1, got {max_bytes}"
56
+ raise ValueError(msg)
57
+ # Partitioned in ONE pass: a repo-scale index passes hundreds of thousands
58
+ # of records, and materializing the input before splitting it held two full
59
+ # pointer lists at once for no benefit.
60
+ nodes: list[dict] = []
61
+ edges: list[dict] = []
62
+ for record in records:
63
+ (nodes if "type" in record else edges).append(record)
64
+ for group in (nodes, edges):
65
+ yield from _batches(group, max_bytes)
66
+
67
+
68
+ def _batches(records: list[dict], max_bytes: int) -> Iterator[list[dict]]:
69
+ batch: list[dict] = []
70
+ size = 0
71
+ for record in records:
72
+ # +1 for the newline the JSONL writer adds after each record.
73
+ cost = len(json.dumps(record).encode()) + 1
74
+ if batch and size + cost > max_bytes:
75
+ yield batch
76
+ batch, size = [], 0
77
+ batch.append(record)
78
+ size += cost
79
+ if batch:
80
+ yield batch
@@ -15,6 +15,7 @@ from pathlib import Path
15
15
 
16
16
  from witan_core.omnigraph import OmnigraphClient as _BaseOmnigraphClient
17
17
 
18
+ from . import chunking
18
19
  from . import config as cfg_module
19
20
  from . import identity as identity_module
20
21
  from . import views
@@ -140,16 +141,50 @@ class OmnigraphClient(_BaseOmnigraphClient):
140
141
  connect_retry=connect_retry,
141
142
  )
142
143
 
143
- def load(self, records: list[dict], mode: str = "merge") -> None:
144
- """Bulk-load node/edge records via one ``omnigraph load`` call.
144
+ def load(
145
+ self,
146
+ records: list[dict],
147
+ mode: str = "merge",
148
+ *,
149
+ max_bytes: int = chunking.LOAD_MAX_BYTES,
150
+ ) -> None:
151
+ """Bulk-load node/edge records via ``omnigraph load``.
145
152
 
146
153
  Each record is a JSONL line: ``{"type": Node, "data": {...}}`` for a
147
154
  node or ``{"edge": Edge, "from": key, "to": key}`` for an edge. This
148
- replaces thousands of per-record ``mutate`` subprocesses with a single
149
- invocation — essential for indexing large repositories.
155
+ replaces thousands of per-record ``mutate`` subprocesses — essential for
156
+ indexing large repositories.
157
+
158
+ Against a ``--server`` the CLI POSTs the data file as ONE request body,
159
+ so a repo-scale load is split into batches under ``max_bytes`` — see
160
+ :mod:`witan_code.chunking` for the 413 this avoids, and for why every
161
+ node has to be written before any edge. A local store reads the file
162
+ directly and has no such limit, but chunking there costs only a few
163
+ extra Lance versions, which ``omnigraph optimize`` reclaims — not worth
164
+ a second code path.
165
+
166
+ ★ ``overwrite`` IS DELIBERATELY NEVER CHUNKED. Measured against 0.8.1,
167
+ that mode TRUNCATES the node type rather than replacing matching rows:
168
+ loading one CodeFile row with ``--mode overwrite`` into a store already
169
+ holding a different one left ``rows=1``, not 2. Split it and every batch
170
+ would erase the one before it, so it stays a single call and an
171
+ oversized one keeps failing loudly at the server.
172
+
173
+ ATOMICITY IS TRADED AWAY for the chunked modes, exactly as
174
+ ``change_many(chunk_size=...)`` does: batches commit independently and a
175
+ failure part-way leaves the earlier ones applied. Callers depending on
176
+ all-or-nothing must handle it — ``indexer.index_path`` does, by holding
177
+ each file's ``content_hash`` back until every batch has landed.
150
178
  """
151
179
  if not records:
152
180
  return
181
+ if mode == "overwrite":
182
+ self._load_batch(records, mode)
183
+ return
184
+ for batch in chunking.chunk_records(records, max_bytes):
185
+ self._load_batch(batch, mode)
186
+
187
+ def _load_batch(self, records: list[dict], mode: str) -> None:
153
188
  fd, tmp = tempfile.mkstemp(suffix=".jsonl", prefix="codegraph-load-")
154
189
  try:
155
190
  with os.fdopen(fd, "w") as fh:
@@ -44,6 +44,12 @@ _QUERIES_TS_DIR = Path(__file__).parent / "queries_ts"
44
44
  # this is 250 files per commit.
45
45
  _DELETE_BATCH_SIZE = 500
46
46
 
47
+ # Stand-in written into CodeFile.content_hash for the duration of a load, so a
48
+ # run that fails part-way leaves nothing the incremental skip check will match.
49
+ # Any non-sha256 string does; this one is obvious in a store dump. See
50
+ # `_defer_content_hashes`.
51
+ _PENDING_CONTENT_HASH = "pending"
52
+
47
53
 
48
54
  @dataclass(frozen=True)
49
55
  class LanguageSpec:
@@ -386,7 +392,13 @@ def index_path(
386
392
  stats.purged += 1
387
393
 
388
394
  client.change_many(delete_steps, chunk_size=_DELETE_BATCH_SIZE)
389
- client.load(_dedupe(records), mode="merge")
395
+ # Two loads, not one: the second commits the content hashes the first held
396
+ # back, so a load that dies part-way is re-done rather than skipped forever.
397
+ # See `_defer_content_hashes`.
398
+ deduped = _dedupe(records)
399
+ real_hash_rows = _defer_content_hashes(deduped)
400
+ client.load(deduped, mode="merge")
401
+ client.load(real_hash_rows, mode="merge")
390
402
 
391
403
  # Cross-repo bridge — a SEPARATE phase after the per-repo store write, so the
392
404
  # two stores' write locks never nest. A full-repo index (target is the repo
@@ -419,6 +431,40 @@ def index_path(
419
431
  return stats
420
432
 
421
433
 
434
+ def _defer_content_hashes(records: list[dict]) -> list[dict]:
435
+ """Hold every CodeFile's ``content_hash`` back until the load has landed.
436
+
437
+ Mutates the CodeFile rows in ``records`` to carry ``_PENDING_CONTENT_HASH``
438
+ and returns copies carrying the REAL hashes, to be loaded once the main load
439
+ succeeds.
440
+
441
+ WHY THIS EXISTS. ``index_path`` deletes a changed file's rows and reloads
442
+ them. While that reload was one atomic ``load``, a failure left the file
443
+ with no recorded hash at all, so the next run re-indexed it — failure was
444
+ self-healing, for free. Chunking the load (``chunking.chunk_records``) takes
445
+ that away: the nodes land, hash and all, in an early batch, and if a later
446
+ batch fails then ``_parse_for_index``'s ``existing.get(file_id) ==
447
+ content_hash`` check SKIPS the file on every subsequent run. Its symbols
448
+ would be present and its edges permanently missing, silently, until someone
449
+ thought to force a full reindex. Chunking also raises the odds of a
450
+ part-way failure, since it turns one request into several.
451
+
452
+ The sentinel restores the old behaviour: it can never equal a real hash (a
453
+ 64-char sha256 hexdigest), so a file whose hashes were never committed is
454
+ simply re-indexed. Nothing else reads ``content_hash`` — only this check and
455
+ the informational ``get_file`` query — so a row briefly carrying the
456
+ sentinel is accurate: that file does need re-indexing.
457
+ """
458
+ real_rows: list[dict] = []
459
+ for record in records:
460
+ if record.get("type") != "CodeFile":
461
+ continue
462
+ data = record["data"]
463
+ real_rows.append({"type": "CodeFile", "data": dict(data)})
464
+ data["content_hash"] = _PENDING_CONTENT_HASH
465
+ return real_rows
466
+
467
+
422
468
  def _dedupe(records: list[dict]) -> list[dict]:
423
469
  """Drop duplicate node slugs / edges so one collision can't fail the load.
424
470
 
@@ -33,6 +33,8 @@ from fastmcp.client.auth import BearerAuth
33
33
  from fastmcp.client.transports import StreamableHttpTransport
34
34
  from fastmcp.exceptions import ToolError
35
35
 
36
+ from .. import chunking
37
+
36
38
  __all__ = [
37
39
  "RemoteStoreClient",
38
40
  "RemoteStoreUnsupported",
@@ -251,16 +253,39 @@ class RemoteStoreClient:
251
253
  query_file, query_name, params, surface_conflict=surface_conflict
252
254
  )
253
255
 
254
- def load(self, records: list[dict], mode: str = "merge") -> None:
256
+ def load(
257
+ self,
258
+ records: list[dict],
259
+ mode: str = "merge",
260
+ *,
261
+ max_bytes: int = chunking.LOAD_MAX_BYTES,
262
+ ) -> None:
263
+ """Bulk-load records through the MCP tier's ``code_store_load``.
264
+
265
+ Chunked on the same terms as :meth:`graph.OmnigraphClient.load`, for the
266
+ same reason one layer down: ``code_store_load`` hands the records to an
267
+ ``omnigraph load`` on the server side, so a repo-scale call runs into
268
+ the same buffered-body ceiling. This transport adds one of its own — the
269
+ records ride as a JSON tool parameter over the MCP session — which the
270
+ byte budget bounds as well.
271
+
272
+ ``overwrite`` is not chunked here either; see the note there.
273
+ """
255
274
  if not records:
256
275
  return
257
- self._session.call(
258
- "code_store_load",
259
- graph=self.graph,
260
- records=records,
261
- mode=mode,
262
- view=self.branch,
276
+ batches = (
277
+ [records]
278
+ if mode == "overwrite"
279
+ else chunking.chunk_records(records, max_bytes)
263
280
  )
281
+ for batch in batches:
282
+ self._session.call(
283
+ "code_store_load",
284
+ graph=self.graph,
285
+ records=batch,
286
+ mode=mode,
287
+ view=self.branch,
288
+ )
264
289
 
265
290
  def ensure_branch(self) -> None:
266
291
  if self.branch is None:
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes