witan-code 0.12.3__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {witan_code-0.12.3 → witan_code-0.13.0}/CHANGELOG.md +25 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/PKG-INFO +3 -3
- {witan_code-0.12.3 → witan_code-0.13.0}/pyproject.toml +11 -4
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_remote_store.py +66 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/bridge.py +21 -5
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/indexer.py +35 -4
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/remote/proxy.py +18 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/remote/store.py +63 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/.gitignore +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/README.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/config/claude.json +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/config/copilot.json +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/config/pi.json +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/BRANCH_INDEXING.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/CLI_REFERENCE.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/EDGE_PRECISION_TIERS.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/PACKAGE_MAP.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/STAGE2_STITCHING.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/SYMBOL_FORMAT.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/SYMBOL_TABLE.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/docs/USER_GUIDE.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/install.sh +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/__init__.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/conftest.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_batched_deletes.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_branches.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_bridge.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_chunked_load.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_cli_dispatch.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_cli_hooks.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_cli_output_format.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_config.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_config_targets.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_context.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_crossrepo.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_edges.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_elicit.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_enrichment.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_github_app.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_graph.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_hooks.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_identity.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_indexer.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_indexer_scope.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_ingest.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_languages.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_maintenance.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_package_map.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_reaper.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_remote_proxy.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_repo.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_server.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_setup.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_stitch.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_store.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_symbol_table.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_tasks.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_views.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/tests/test_visualize.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/__init__.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/__main__.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/bridge_extractors.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/cli.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/config.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/context.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/edges.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/elicit.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/extensions/pi/codegraph.ts +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/github_app.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/graph.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/hooks.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/identity.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/ingest.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/maintenance.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/output.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/package_map.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries/bridge.gq +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries/code_mutations.gq +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries/code_read.gq +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries/delete.gq +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries_ts/bash.scm +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries_ts/hcl.scm +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries_ts/python.scm +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries_ts/sql.scm +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries_ts/typescript.scm +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/queries_ts/yaml.scm +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/reaper.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/remote/__init__.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/remote/oidc.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/repo.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/schema/bridge-schema.pg +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/schema/code-schema.pg +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/server.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/setup.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/skills/witan-code/SKILL.md +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/stitch.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/store.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/views.py +0 -0
- {witan_code-0.12.3 → witan_code-0.13.0}/witan_code/visualize.py +0 -0
|
@@ -6,6 +6,31 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/) (pre-1.0:
|
|
7
7
|
a MINOR bump may include breaking changes).
|
|
8
8
|
|
|
9
|
+
## [0.13.0] - 2026-08-13
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- **An index write cut off by a gateway is no longer retried.**
|
|
14
|
+
`RemoteStoreClient` reconnects and retries on a transport fault, which is
|
|
15
|
+
right for a dropped connection and wrong for a 502/504: measured against the
|
|
16
|
+
deployed service, most writes cut that way had already committed — one run of
|
|
17
|
+
4-, 8- and 16-writer bursts committed all 28 of its writes, and a later
|
|
18
|
+
standalone 16-writer burst committed 14 of 16 — so the blind retry writes
|
|
19
|
+
those rows a
|
|
20
|
+
second time. `_refuse_if_indeterminate` now stops it and says the outcome is
|
|
21
|
+
unknown, matching `_refuse_if_too_large` beside it
|
|
22
|
+
([#225](https://github.com/mitodl/agent-kit/pull/225)).
|
|
23
|
+
★ This transport is separate from the CLI proxy, and had to be fixed
|
|
24
|
+
separately. `RemoteServerProxy._writes()` returns `False` because nothing
|
|
25
|
+
*that proxy* dispatches writes — `code_reindex` is refused as local-only —
|
|
26
|
+
which is easy to misread as "witan-code never writes remotely". Index writes
|
|
27
|
+
go out over `code_store_mutate` / `code_store_mutate_many` /
|
|
28
|
+
`code_store_load`, on this client, and it does its own classification.
|
|
29
|
+
- The CI indexer's per-repo delete no longer times out. The batch was sized
|
|
30
|
+
against the argv length limit while the real constraint is time: a
|
|
31
|
+
500-statement delete chunk takes ~181s against a 120s client timeout
|
|
32
|
+
([#223](https://github.com/mitodl/agent-kit/pull/223)).
|
|
33
|
+
|
|
9
34
|
## [0.12.3] - 2026-08-07
|
|
10
35
|
|
|
11
36
|
### Fixed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: witan-code
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
|
|
5
5
|
Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
|
|
6
6
|
Project-URL: Repository, https://github.com/mitodl/agent-kit
|
|
@@ -26,7 +26,7 @@ Requires-Dist: tree-sitter-sql<0.4,>=0.3
|
|
|
26
26
|
Requires-Dist: tree-sitter-typescript<0.24,>=0.23
|
|
27
27
|
Requires-Dist: tree-sitter-yaml<0.8,>=0.7
|
|
28
28
|
Requires-Dist: tree-sitter<0.26,>=0.25
|
|
29
|
-
Requires-Dist: witan-core[cli,observability,remote]<1,>=0.
|
|
29
|
+
Requires-Dist: witan-core[cli,observability,remote]<1,>=0.17
|
|
30
30
|
Provides-Extra: tasks
|
|
31
31
|
Requires-Dist: fastmcp-tasks<5,>=4.0.0b1; extra == 'tasks'
|
|
32
32
|
Description-Content-Type: text/markdown
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "witan-code"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.13.0"
|
|
4
4
|
description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "BSD-3-Clause"
|
|
@@ -22,13 +22,20 @@ dependencies = [
|
|
|
22
22
|
# and it has now been tripped THREE times. If you add a witan_core symbol in
|
|
23
23
|
# the same change as its caller, bump this line in that change: the
|
|
24
24
|
# workspace resolves witan-core by path, so nothing here can catch it.
|
|
25
|
-
# (>=0.
|
|
25
|
+
# (>=0.17 for remote.proxy.RemoteWriteIndeterminate AND gateway_failure,
|
|
26
|
+
# both imported at module scope by remote/store.py — so an older witan-core
|
|
27
|
+
# fails at IMPORT, exactly like the cases above; it does not degrade into
|
|
28
|
+
# retrying an indeterminate write, because the module holding that retry is
|
|
29
|
+
# the one that fails to load. Worth pinning precisely because store.py is
|
|
30
|
+
# the transport index writes actually go out on: the floor is what keeps a
|
|
31
|
+
# published install from being dead on arrival rather than subtly wrong;
|
|
32
|
+
# >=0.14 was required for remote.proxy.RemoteToolFailed, re-exported at
|
|
26
33
|
# module scope by witan_code/remote/proxy.py and named by cli.py's cli();
|
|
27
34
|
# >=0.11 for chunking.MCP_LOAD_MAX_BYTES, evaluated as a default argument;
|
|
28
35
|
# >=0.10 for witan_core.chunking + load_batch, both moved up from this
|
|
29
36
|
# package; >=0.9 for observability.middleware; >=0.8 for
|
|
30
37
|
# OmnigraphClient.change_many with chunk_size.)
|
|
31
|
-
"witan-core[cli,remote,observability]>=0.
|
|
38
|
+
"witan-core[cli,remote,observability]>=0.17,<1",
|
|
32
39
|
"fastmcp>=3.4.2,<5",
|
|
33
40
|
"cyclopts>=4,<5",
|
|
34
41
|
# [targets.<name>] override models (config.py) — also transitively pulled
|
|
@@ -112,7 +119,7 @@ testpaths = ["tests"]
|
|
|
112
119
|
packages = ["witan_code"]
|
|
113
120
|
|
|
114
121
|
[tool.bumpversion]
|
|
115
|
-
current_version = "0.
|
|
122
|
+
current_version = "0.13.0"
|
|
116
123
|
allow_dirty = true
|
|
117
124
|
|
|
118
125
|
[[tool.bumpversion.files]]
|
|
@@ -23,6 +23,7 @@ from witan_code import config as cfg_module
|
|
|
23
23
|
from witan_code import store as store_module
|
|
24
24
|
from witan_code.remote.store import (
|
|
25
25
|
RemotePayloadTooLarge,
|
|
26
|
+
RemoteWriteIndeterminate,
|
|
26
27
|
RemoteStoreClient,
|
|
27
28
|
RemoteStoreUnsupported,
|
|
28
29
|
StoreSession,
|
|
@@ -789,3 +790,68 @@ def test_a_reconnect_re_asks_the_surface():
|
|
|
789
790
|
client.change_many(_steps(1))
|
|
790
791
|
session.close()
|
|
791
792
|
assert [entry for entry, _ in log].count("list_tools") == 2
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _gateway(status: int):
|
|
796
|
+
import httpx2
|
|
797
|
+
|
|
798
|
+
request = httpx2.Request("POST", "http://x/mcp")
|
|
799
|
+
response = httpx2.Response(status, request=request, text="<html>502 Bad Gateway")
|
|
800
|
+
return httpx2.HTTPStatusError(str(status), request=request, response=response)
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
@pytest.mark.parametrize("status", [502, 504])
|
|
804
|
+
def test_a_write_cut_off_by_a_gateway_is_never_re_invoked(status):
|
|
805
|
+
"""★ THE RETRY IS THE BUG, and here it duplicates data rather than wasting a
|
|
806
|
+
round trip.
|
|
807
|
+
|
|
808
|
+
A 502/504 is not a ``ToolError``, so it lands in the generic arm — which
|
|
809
|
+
reconnects and sends the write AGAIN. Measured against CI, the backend
|
|
810
|
+
usually finished the write before the deadline cut the response (28 of 28
|
|
811
|
+
committed in one burst, 14 of 16 in another), so that second invocation
|
|
812
|
+
writes the rows a second time. The outcome is indeterminate, not failed, and
|
|
813
|
+
the only safe response is to stop and say so.
|
|
814
|
+
"""
|
|
815
|
+
log: list = []
|
|
816
|
+
exc = _gateway(status)
|
|
817
|
+
clients = iter([_RefusesForSize(log, exc), _RefusesForSize(log, exc)])
|
|
818
|
+
session = StoreSession("http://x/mcp", lambda: "jwt", lambda _t: next(clients))
|
|
819
|
+
with pytest.raises(RemoteWriteIndeterminate) as caught:
|
|
820
|
+
RemoteStoreClient(REPO, session).load([{"id": "n1", "type": "Symbol"}])
|
|
821
|
+
# Exactly one attempt: the reconnect-and-retry must not have happened.
|
|
822
|
+
assert [entry[0] for entry in log] == ["connect", "attempt"]
|
|
823
|
+
message = str(caught.value)
|
|
824
|
+
assert "INDETERMINATE" in message
|
|
825
|
+
assert "NOT retried" in message
|
|
826
|
+
assert f"HTTP {status}" in message
|
|
827
|
+
assert "`code_store_load`" in message
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def test_a_read_cut_off_by_a_gateway_still_reconnects():
|
|
831
|
+
"""The other half: repeating a read changes nothing, so a dropped session is
|
|
832
|
+
exactly what the reconnect is for. Refusing it would be a strict loss."""
|
|
833
|
+
log: list = []
|
|
834
|
+
|
|
835
|
+
class _FailsOnceWithGateway(_FakeClient):
|
|
836
|
+
def __init__(self, log, fail):
|
|
837
|
+
super().__init__(log)
|
|
838
|
+
self._fail = fail
|
|
839
|
+
|
|
840
|
+
async def call_tool(self, name, arguments):
|
|
841
|
+
if self._fail:
|
|
842
|
+
self._log.append(("attempt", name))
|
|
843
|
+
raise _gateway(502)
|
|
844
|
+
return await super().call_tool(name, arguments)
|
|
845
|
+
|
|
846
|
+
clients = iter(
|
|
847
|
+
[_FailsOnceWithGateway(log, True), _FailsOnceWithGateway(log, False)]
|
|
848
|
+
)
|
|
849
|
+
session = StoreSession("http://x/mcp", lambda: "jwt", lambda _t: next(clients))
|
|
850
|
+
assert RemoteStoreClient(REPO, session).list_branches() == ["main"]
|
|
851
|
+
assert [entry[0] for entry in log] == [
|
|
852
|
+
"connect",
|
|
853
|
+
"attempt",
|
|
854
|
+
"disconnect",
|
|
855
|
+
"connect",
|
|
856
|
+
"code_store_views",
|
|
857
|
+
]
|
|
@@ -22,9 +22,22 @@ from .bridge_extractors import (
|
|
|
22
22
|
from .graph import OmnigraphClient, check_writable
|
|
23
23
|
from .store import bridge_store, ensure_bridge_store
|
|
24
24
|
|
|
25
|
-
# Delete statements per commit when purging a repo's bindings.
|
|
26
|
-
#
|
|
27
|
-
|
|
25
|
+
# Delete statements per commit when purging a repo's bindings. A full-repo purge
|
|
26
|
+
# has to be capped rather than built whole.
|
|
27
|
+
#
|
|
28
|
+
# Same reasoning, and the same measurement, as `_DELETE_BATCH_SIZE` in
|
|
29
|
+
# `indexer.py` — read the comment there. Short version: the argv justification is
|
|
30
|
+
# stale (the deployed path is the HTTP transport, no argv), a matching delete
|
|
31
|
+
# costs ~362ms, and 500 of them is ~181s against a 120s client timeout. One
|
|
32
|
+
# statement per file here rather than two, so this is 128 files per commit.
|
|
33
|
+
#
|
|
34
|
+
# Lowered together with the indexer's cap deliberately. This purge has not been
|
|
35
|
+
# SEEN to time out, but it is the identical shape against the same engine, and a
|
|
36
|
+
# repo with enough changed files would trip it the same way. Fixing only the one
|
|
37
|
+
# that happened to fail first would leave the other waiting for a busy day.
|
|
38
|
+
# (The 362ms figure was measured on `Symbol` deletes; `InterfaceBinding` deletes
|
|
39
|
+
# were not timed separately, so treat this cap as the same order, not as exact.)
|
|
40
|
+
_PURGE_BATCH_SIZE = 128
|
|
28
41
|
|
|
29
42
|
|
|
30
43
|
logger = get_logger("witan.code.bridge")
|
|
@@ -139,8 +152,11 @@ def write_bindings(
|
|
|
139
152
|
}
|
|
140
153
|
# One commit for the whole purge, not one per file: this runs over every
|
|
141
154
|
# touched file on each index, and a commit apiece fragments the bridge store
|
|
142
|
-
# the same way the per-file deletes did the per-repo one. Chunked because
|
|
143
|
-
#
|
|
155
|
+
# the same way the per-file deletes did the per-repo one. Chunked because a
|
|
156
|
+
# full-repo purge is unbounded and a matching delete costs ~362ms, so an
|
|
157
|
+
# uncapped batch runs past the client's 120s timeout — see
|
|
158
|
+
# `_PURGE_BATCH_SIZE` for the measurements and for why the argv reasoning
|
|
159
|
+
# this comment used to give no longer applies.
|
|
144
160
|
client.change_many(
|
|
145
161
|
[
|
|
146
162
|
("bridge.gq", "delete_bindings_in_file", {"repo_file": f"{repo}|{rel}"})
|
|
@@ -39,10 +39,41 @@ logger = get_logger("witan.code.index")
|
|
|
39
39
|
_QUERIES_TS_DIR = Path(__file__).parent / "queries_ts"
|
|
40
40
|
|
|
41
41
|
# Delete statements per commit when clearing reindexed/purged files. A full-repo
|
|
42
|
-
# reindex can touch thousands of files,
|
|
43
|
-
#
|
|
44
|
-
#
|
|
45
|
-
|
|
42
|
+
# reindex can touch thousands of files, so the batch has to be capped rather than
|
|
43
|
+
# built whole. Two statements per file, so this is 64 files per commit.
|
|
44
|
+
#
|
|
45
|
+
# ★ THE BINDING CONSTRAINT IS TIME, NOT ARGV. The original cap was 500, justified
|
|
46
|
+
# by the composed query riding in a single argv element. That reasoning does not
|
|
47
|
+
# apply to the deployed indexer at all: it uses the pooled HTTP transport, where
|
|
48
|
+
# the query travels in a JSON body and argv is never involved. Meanwhile a
|
|
49
|
+
# MATCHING delete costs ~362ms server-side, so 500 statements is ~181 SECONDS —
|
|
50
|
+
# past `witan_core.omnigraph_http.DEFAULT_TIMEOUT_SECONDS` (120s), and the chunk
|
|
51
|
+
# fails with a bare "timed out".
|
|
52
|
+
#
|
|
53
|
+
# That is not hypothetical: it is why `witan-ci-index` failed every run from
|
|
54
|
+
# 2026-08-07, in CI and Production alike, always on ol-infrastructure —
|
|
55
|
+
# `indexer.py` change_many → chunk of 500 → timeout → "indexed 13, failed 1".
|
|
56
|
+
#
|
|
57
|
+
# Measured 2026-08-12 against the CI data tier, inserting then deleting real rows
|
|
58
|
+
# in `code-github-com-mitodl-lehrer`:
|
|
59
|
+
# 50 matching deletes 19.23s (385 ms/stmt)
|
|
60
|
+
# 100 matching deletes 36.29s (363 ms/stmt)
|
|
61
|
+
# 250 matching deletes 90.47s (362 ms/stmt)
|
|
62
|
+
# Linear, and — measured on the SMALLEST graph (1,567 rows) — essentially
|
|
63
|
+
# independent of table size, so this is not a big-repo problem. Any repo whose
|
|
64
|
+
# reindex touches ~165 files in one run would hit it; ol-infrastructure is simply
|
|
65
|
+
# the one busy enough to do so every four hours.
|
|
66
|
+
#
|
|
67
|
+
# 128 statements is ~46s, a 2.5x margin under the timeout. It does NOT make a
|
|
68
|
+
# reindex faster — the cost is linear per statement either way — it makes it
|
|
69
|
+
# COMPLETE, which for a four-hourly job is the property that matters.
|
|
70
|
+
#
|
|
71
|
+
# ★ Raising DEFAULT_TIMEOUT_SECONDS instead would be the wrong lever: that
|
|
72
|
+
# budget exists to catch a genuinely wedged server, and widening it to fit a
|
|
73
|
+
# known-slow operation blunts the one signal that says the server is stuck.
|
|
74
|
+
# The real defect is that a single keyed delete costs 362ms at all — tracked as
|
|
75
|
+
# tk-upstream-omnigraph-a-single-row-insert-costs-a-f-eeeae3.
|
|
76
|
+
_DELETE_BATCH_SIZE = 128
|
|
46
77
|
|
|
47
78
|
# Stand-in written into CodeFile.content_hash for the duration of a load, so a
|
|
48
79
|
# run that fails part-way leaves nothing the incremental skip check will match.
|
|
@@ -46,6 +46,24 @@ class RemoteServerProxy(RemoteMCPProxy):
|
|
|
46
46
|
def _is_admin_tool(self, name: str) -> bool:
|
|
47
47
|
return name in _LOCAL_ONLY
|
|
48
48
|
|
|
49
|
+
def _writes(self, name: str) -> bool:
|
|
50
|
+
# Nothing THIS PROXY dispatches remotely writes. `code_reindex` is the
|
|
51
|
+
# only tool on this surface that does, and `_LOCAL_ONLY` refuses it
|
|
52
|
+
# before it can ever reach a transport — so a gateway cut-off here is
|
|
53
|
+
# always a read that returned nothing, never a write whose fate is
|
|
54
|
+
# unknown. Stated as a rule rather than a list because it follows from
|
|
55
|
+
# `_LOCAL_ONLY` above: anything that starts writing has to be added
|
|
56
|
+
# there or here, and both are three lines apart.
|
|
57
|
+
#
|
|
58
|
+
# ★ NOT the same thing as "witan-code never writes remotely". Index
|
|
59
|
+
# writes go to the deployed store over a DIFFERENT transport —
|
|
60
|
+
# `remote.store.RemoteStoreClient`, dispatching `code_store_mutate`,
|
|
61
|
+
# `code_store_mutate_many` and `code_store_load` — which does its own
|
|
62
|
+
# gateway classification, and has to, because it reconnects and retries
|
|
63
|
+
# (`_refuse_if_indeterminate`). Reading this `False` as covering that
|
|
64
|
+
# path is the mistake this note exists to stop.
|
|
65
|
+
return False
|
|
66
|
+
|
|
49
67
|
def _unreachable_hint(self) -> str:
|
|
50
68
|
# Two ways to name the wrong setting here, and this avoids both.
|
|
51
69
|
# `remote_url` is what routes a client to this proxy — NOT
|
|
@@ -36,6 +36,8 @@ from witan_core.observability import get_logger
|
|
|
36
36
|
from witan_core.remote.proxy import (
|
|
37
37
|
RemotePayloadTooLarge,
|
|
38
38
|
RemoteToolFailed,
|
|
39
|
+
RemoteWriteIndeterminate,
|
|
40
|
+
gateway_failure,
|
|
39
41
|
payload_too_large,
|
|
40
42
|
)
|
|
41
43
|
|
|
@@ -125,6 +127,54 @@ def _load_refusal(exc: BaseException, *, batch: int, mode: str, max_bytes: int)
|
|
|
125
127
|
# asked rather than assumed — see `RemoteStoreClient.change_many`.
|
|
126
128
|
_MUTATE_MANY_TOOL = "code_store_mutate_many"
|
|
127
129
|
|
|
130
|
+
# The store tools that change nothing. Everything else on this transport is
|
|
131
|
+
# treated as a write.
|
|
132
|
+
#
|
|
133
|
+
# ★ THE READS ARE LISTED, NOT THE WRITES, so an unclassified tool defaults to
|
|
134
|
+
# "assume it wrote" — the same asymmetry `RemoteMCPProxy._writes` is built on.
|
|
135
|
+
# Calling a read a write costs one over-careful sentence; calling a write a read
|
|
136
|
+
# tells somebody their index landed when it may not have, or invites the retry
|
|
137
|
+
# that writes it twice.
|
|
138
|
+
#
|
|
139
|
+
# `code_store_open` is deliberately absent: opening can create the store, so it
|
|
140
|
+
# is not a read.
|
|
141
|
+
_READ_ONLY_STORE_TOOLS = frozenset({"code_store_read", "code_store_views"})
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _refuse_if_indeterminate(exc: BaseException, tool: str, url: str) -> None:
|
|
145
|
+
"""Re-raise a WRITE cut off by a gateway as indeterminate. Otherwise no-op.
|
|
146
|
+
|
|
147
|
+
★ THIS TRANSPORT'S RECONNECT-AND-RETRY IS WHY THIS HAS TO EXIST. A 502/504
|
|
148
|
+
is not a ``ToolError``, so it lands in the generic arm of
|
|
149
|
+
:meth:`StoreSession.call`, which reconnects and re-invokes — and for a write
|
|
150
|
+
cut off mid-flight, re-invoking is the duplicate this whole change exists to
|
|
151
|
+
prevent. Measured against CI: a burst's 502s mostly HAD committed (28 of 28
|
|
152
|
+
in one run, 14 of 16 in another), so the blind retry writes those rows a
|
|
153
|
+
second time.
|
|
154
|
+
|
|
155
|
+
Exactly the shape of ``_refuse_if_too_large`` above, and called from the
|
|
156
|
+
same two places for the same reason: retrying cannot help, and the retry
|
|
157
|
+
actively hurts.
|
|
158
|
+
|
|
159
|
+
Reads fall through untouched — repeating one changes nothing, and the
|
|
160
|
+
reconnect is the right response to a dropped session.
|
|
161
|
+
"""
|
|
162
|
+
if tool in _READ_ONLY_STORE_TOOLS:
|
|
163
|
+
return
|
|
164
|
+
cut_off = gateway_failure(exc)
|
|
165
|
+
if cut_off is None:
|
|
166
|
+
return
|
|
167
|
+
status = cut_off.response.status_code
|
|
168
|
+
msg = (
|
|
169
|
+
f"The deployed store at {url} answered HTTP {status} for `{tool}`: the "
|
|
170
|
+
f"request reached it and was cut off before a reply came back. `{tool}` "
|
|
171
|
+
"writes, so ITS OUTCOME IS INDETERMINATE — it may or may not have been "
|
|
172
|
+
"applied, and nothing in this response says which. This was NOT "
|
|
173
|
+
"retried, because retrying blind writes it twice if it did land. "
|
|
174
|
+
"Re-read the store before deciding; a reindex is safe to re-run."
|
|
175
|
+
)
|
|
176
|
+
raise RemoteWriteIndeterminate(msg) from exc
|
|
177
|
+
|
|
128
178
|
|
|
129
179
|
class _LoopThread:
|
|
130
180
|
"""A private event loop on a daemon thread, driven from synchronous code."""
|
|
@@ -189,6 +239,11 @@ class StoreSession:
|
|
|
189
239
|
raise RemoteToolFailed(str(exc)) from exc
|
|
190
240
|
except Exception as exc: # noqa: BLE001 — transport-shaped; retry once
|
|
191
241
|
_refuse_if_too_large(exc, tool, self.url)
|
|
242
|
+
# BEFORE the reconnect, and that order is the whole point: a
|
|
243
|
+
# gateway cut-off on a write must not be re-invoked, because the
|
|
244
|
+
# backend usually finished it and re-invoking writes the row
|
|
245
|
+
# twice. Only unclassified transport faults reach the retry.
|
|
246
|
+
_refuse_if_indeterminate(exc, tool, self.url)
|
|
192
247
|
# Info: one reconnect is routine (idle session dropped), but a
|
|
193
248
|
# deployment that reconnects on every call is pathological and
|
|
194
249
|
# invisible without this.
|
|
@@ -204,6 +259,14 @@ class StoreSession:
|
|
|
204
259
|
# arm exists to prevent.
|
|
205
260
|
_refuse_if_too_large(refused, tool, self.url)
|
|
206
261
|
raise RemoteToolFailed(str(refused)) from refused
|
|
262
|
+
except Exception as retried: # noqa: BLE001 — same two rules
|
|
263
|
+
# The reconnect landed and the SECOND call was cut off. The
|
|
264
|
+
# write is just as indeterminate as it would have been the
|
|
265
|
+
# first time, so it gets the same sentence rather than
|
|
266
|
+
# escaping as a traceback.
|
|
267
|
+
_refuse_if_too_large(retried, tool, self.url)
|
|
268
|
+
_refuse_if_indeterminate(retried, tool, self.url)
|
|
269
|
+
raise
|
|
207
270
|
|
|
208
271
|
def has_tool(self, tool: str) -> bool:
|
|
209
272
|
"""Whether the deployment serves ``tool``.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|