witan-code 0.24.0__tar.gz → 0.25.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {witan_code-0.24.0 → witan_code-0.25.0}/.gitignore +1 -1
- {witan_code-0.24.0 → witan_code-0.25.0}/CHANGELOG.md +26 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/PKG-INFO +5 -3
- {witan_code-0.24.0 → witan_code-0.25.0}/README.md +3 -1
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/BRANCH_INDEXING.md +2 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/pyproject.toml +9 -3
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_maintenance.py +31 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_reaper.py +78 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_store.py +15 -7
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/cli.py +41 -13
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/reaper.py +59 -5
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/store.py +1 -1
- {witan_code-0.24.0 → witan_code-0.25.0}/changelog.d/README.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/config/claude.json +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/config/copilot.json +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/config/pi.json +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/conftest.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/CLI_REFERENCE.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/EDGE_PRECISION_TIERS.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/PACKAGE_MAP.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/STAGE2_STITCHING.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/SYMBOL_FORMAT.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/SYMBOL_TABLE.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/docs/USER_GUIDE.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/install.sh +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/__init__.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/conftest.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_batched_deletes.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_branches.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_bridge.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_chunked_load.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_dispatch.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_hooks.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_output_format.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_selected_target.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_config.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_config_targets.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_context.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_crossrepo.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_edges.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_elicit.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_enrichment.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_github_app.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_graph.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_hooks.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_identity.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_index_failure_diagnostics.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_index_reports_its_store.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_index_requires_a_repo_key.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_indexer.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_indexer_scope.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_ingest.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_languages.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_package_map.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_rebuild_honours_the_repo_key.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_refusal_audit.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_remote_proxy.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_remote_store.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_repo.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_server.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_setup.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_stitch.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_symbol_table.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_tasks.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_tool_schema.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_views.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_visualize.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/__init__.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/__main__.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/bridge.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/bridge_extractors.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/config.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/context.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/edges.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/elicit.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/extensions/pi/codegraph.ts +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/github_app.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/graph.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/hooks.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/identity.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/indexer.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/ingest.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/maintenance.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/output.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/package_map.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/bridge.gq +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/code_mutations.gq +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/code_read.gq +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/delete.gq +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/bash.scm +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/hcl.scm +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/python.scm +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/sql.scm +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/typescript.scm +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/yaml.scm +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/__init__.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/oidc.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/proxy.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/store.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/repo.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/schema/bridge-schema.pg +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/schema/code-schema.pg +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/selected_target.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/server.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/setup.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/skills/witan-code/SKILL.md +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/stitch.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/views.py +0 -0
- {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/visualize.py +0 -0
|
@@ -26,7 +26,7 @@ build/
|
|
|
26
26
|
*.egg-info/
|
|
27
27
|
# omnigraph binary downloaded by hatch_build.py — not committed
|
|
28
28
|
_bin/
|
|
29
|
-
# omnigraph cluster state (populated by `omnigraph cluster
|
|
29
|
+
# omnigraph cluster state (populated by `omnigraph cluster apply`)
|
|
30
30
|
__cluster/
|
|
31
31
|
# The witan UI bundle, written by `npm run build` in mcp/servers/witan/ui/.
|
|
32
32
|
# A build output, so not committed — hatch's `artifacts` setting in that
|
|
@@ -8,6 +8,32 @@ a MINOR bump may include breaking changes).
|
|
|
8
8
|
|
|
9
9
|
<!-- scriv-insert-here -->
|
|
10
10
|
|
|
11
|
+
## [0.25.0] - 2026-10-15
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
|
|
15
|
+
- `witan-code cleanup` no longer defaults to `--keep 10`. From omnigraph 0.12
|
|
16
|
+
`--keep` counts graph commits per live branch, not versions per table, so
|
|
17
|
+
that default would cut a store to its last ten writes. With neither bound
|
|
18
|
+
it now keeps the last 30 days, as the Stop hook does. The help text
|
|
19
|
+
describes both bounds in graph commits. `--older-than` on its own
|
|
20
|
+
no longer adds an implicit `--keep 10`.
|
|
21
|
+
- The refusal for a shared cluster graph notes that from omnigraph 0.12
|
|
22
|
+
compaction is also refused while the server holds the cluster's `serve`
|
|
23
|
+
lock.
|
|
24
|
+
|
|
25
|
+
- Requires witan-core 0.41 or later, the first release on omnigraph 0.13.
|
|
26
|
+
Every deployed omnigraph server is on 0.13 and refuses a client that does
|
|
27
|
+
not send `Omnigraph-Http-Api: 0.13`.
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
|
|
31
|
+
- `reap-views` ages a graph's views concurrently, `WITAN_CODE_VIEW_SURVEY_WORKERS` at a time (default 8; `1` is serial). Ageing a view is one `commit list --branch` that returns its whole reachable history, and serially production's 309-view bridge graph took 327s on its own, pushing the reaper CronJob past its 600s deadline; 8 workers take it to 50s. Each graph now also prints a `surveying views` line before its survey, so a job killed mid-sweep shows which graph it was on.
|
|
32
|
+
|
|
33
|
+
- A cluster graph that is not served is recognised on omnigraph 0.13, whose
|
|
34
|
+
404 says `graph not found` without the quoted id. It was reported as an
|
|
35
|
+
unreachable cluster.
|
|
36
|
+
|
|
11
37
|
## [0.24.0] - 2026-10-02
|
|
12
38
|
|
|
13
39
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: witan-code
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.25.0
|
|
4
4
|
Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
|
|
5
5
|
Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
|
|
6
6
|
Project-URL: Repository, https://github.com/mitodl/agent-kit
|
|
@@ -26,7 +26,7 @@ Requires-Dist: tree-sitter-sql<0.4,>=0.3
|
|
|
26
26
|
Requires-Dist: tree-sitter-typescript<0.24,>=0.23
|
|
27
27
|
Requires-Dist: tree-sitter-yaml<0.8,>=0.7
|
|
28
28
|
Requires-Dist: tree-sitter<0.26,>=0.25
|
|
29
|
-
Requires-Dist: witan-core[cli,observability,remote,sentry]<1,>=0.
|
|
29
|
+
Requires-Dist: witan-core[cli,observability,remote,sentry]<1,>=0.41
|
|
30
30
|
Provides-Extra: tasks
|
|
31
31
|
Requires-Dist: fastmcp-tasks<5,>=4; extra == 'tasks'
|
|
32
32
|
Description-Content-Type: text/markdown
|
|
@@ -584,7 +584,9 @@ graph):
|
|
|
584
584
|
instead.
|
|
585
585
|
- `cleanup [--store PATH] [--bridge] [--keep N] [--older-than DUR] --yes` —
|
|
586
586
|
reclaim disk by GC'ing old Lance versions (**destructive**; requires
|
|
587
|
-
`--yes`).
|
|
587
|
+
`--yes`). `--keep N` retains the newest N graph commits of every live
|
|
588
|
+
branch (omnigraph 0.12+; versions per table on 0.11); with neither bound it
|
|
589
|
+
keeps the last 30 days.
|
|
588
590
|
- `checkpoint` — opportunistically compact the current repo's store and the
|
|
589
591
|
bridge store if due (see [Hooks](#hooks)); registered as the `Stop` hook,
|
|
590
592
|
not usually run by hand.
|
|
@@ -551,7 +551,9 @@ graph):
|
|
|
551
551
|
instead.
|
|
552
552
|
- `cleanup [--store PATH] [--bridge] [--keep N] [--older-than DUR] --yes` —
|
|
553
553
|
reclaim disk by GC'ing old Lance versions (**destructive**; requires
|
|
554
|
-
`--yes`).
|
|
554
|
+
`--yes`). `--keep N` retains the newest N graph commits of every live
|
|
555
|
+
branch (omnigraph 0.12+; versions per table on 0.11); with neither bound it
|
|
556
|
+
keeps the last 30 days.
|
|
555
557
|
- `checkpoint` — opportunistically compact the current repo's store and the
|
|
556
558
|
bridge store if due (see [Hooks](#hooks)); registered as the `Stop` hook,
|
|
557
559
|
not usually run by hand.
|
|
@@ -222,6 +222,8 @@ date, owner, or size. Two rules follow:
|
|
|
222
222
|
ago — reaping it would race the indexer that just created it.
|
|
223
223
|
|
|
224
224
|
The window is `WITAN_CODE_VIEW_MAX_IDLE_DAYS` (default 14; `0` disables).
|
|
225
|
+
Ageing a view is one `commit list` per view, so the sweep ages
|
|
226
|
+
`WITAN_CODE_VIEW_SURVEY_WORKERS` views at once (default 8; `1` is serial).
|
|
225
227
|
Idleness is not abandonment: a branch parked past the window and picked back up
|
|
226
228
|
loses its view, not its work, and the next index rebuilds it. Reporting is the
|
|
227
229
|
default and `--apply` deletes, because the window is the one input nothing
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "witan-code"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.25.0"
|
|
4
4
|
description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "BSD-3-Clause"
|
|
@@ -170,7 +170,13 @@ dependencies = [
|
|
|
170
170
|
# `witan_core.cli.report_install` only prints `InstallResult.prerequisites`
|
|
171
171
|
# from 0.40.0. It reads the field with getattr, so 0.39 imports and runs —
|
|
172
172
|
# and silently drops the warning.
|
|
173
|
-
|
|
173
|
+
#
|
|
174
|
+
# ★ >=0.41 IS A COMPATIBILITY FLOOR: 0.41.0 pins omnigraph 0.13 (storage
|
|
175
|
+
# format 14) and sends `Omnigraph-Http-Api: 0.13`. Every deployed server
|
|
176
|
+
# is on 0.13, which answers a request without that header with `400
|
|
177
|
+
# api_contract_mismatch`, so 0.40 imports and runs, then fails on the
|
|
178
|
+
# first read of a shared code graph.
|
|
179
|
+
"witan-core[cli,remote,observability,sentry]>=0.41,<1",
|
|
174
180
|
"fastmcp>=4,<5",
|
|
175
181
|
"cyclopts>=4,<6",
|
|
176
182
|
# [targets.<name>] override models (config.py) — also transitively pulled
|
|
@@ -253,7 +259,7 @@ testpaths = ["tests"]
|
|
|
253
259
|
packages = ["witan_code"]
|
|
254
260
|
|
|
255
261
|
[tool.bumpversion]
|
|
256
|
-
current_version = "0.
|
|
262
|
+
current_version = "0.25.0"
|
|
257
263
|
allow_dirty = true
|
|
258
264
|
|
|
259
265
|
[[tool.bumpversion.files]]
|
|
@@ -231,6 +231,37 @@ def test_cli_optimize_and_cleanup(tmp_path, capsys):
|
|
|
231
231
|
assert "Cleaned up" in capsys.readouterr().out
|
|
232
232
|
|
|
233
233
|
|
|
234
|
+
def test_cli_cleanup_with_no_bound_keeps_the_hooks_window(
|
|
235
|
+
tmp_path, capsys, monkeypatch
|
|
236
|
+
):
|
|
237
|
+
"""From omnigraph 0.12 `--keep` counts graph commits, so a default of
|
|
238
|
+
`--keep 10` would cut a store to its last ten writes. No bound means the
|
|
239
|
+
age window the Stop hook already uses."""
|
|
240
|
+
from witan_code import cli as cli_module
|
|
241
|
+
from witan_code import maintenance
|
|
242
|
+
|
|
243
|
+
store = tmp_path / "s.omni"
|
|
244
|
+
store.mkdir()
|
|
245
|
+
calls = []
|
|
246
|
+
|
|
247
|
+
class _Client:
|
|
248
|
+
def cleanup(self, **kwargs):
|
|
249
|
+
calls.append(kwargs)
|
|
250
|
+
|
|
251
|
+
monkeypatch.setattr(cli_module, "_maintenance_client", lambda _ref: _Client())
|
|
252
|
+
|
|
253
|
+
cli_module.cleanup(store=str(store))
|
|
254
|
+
assert f"newer than {maintenance.CLEANUP_OLDER_THAN}" in capsys.readouterr().out
|
|
255
|
+
assert calls == []
|
|
256
|
+
|
|
257
|
+
cli_module.cleanup(store=str(store), yes=True)
|
|
258
|
+
cli_module.cleanup(store=str(store), keep=3, yes=True)
|
|
259
|
+
assert calls == [
|
|
260
|
+
{"keep": None, "older_than": maintenance.CLEANUP_OLDER_THAN},
|
|
261
|
+
{"keep": 3, "older_than": None},
|
|
262
|
+
]
|
|
263
|
+
|
|
264
|
+
|
|
234
265
|
@requires_omnigraph
|
|
235
266
|
def test_cli_optimize_then_cleanup(tmp_path, capsys):
|
|
236
267
|
from witan_code import cli as cli_module
|
|
@@ -10,6 +10,8 @@ branch's own last write is recoverable from `commit list --branch` via
|
|
|
10
10
|
identically, so it is asserted rather than mocked.
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
13
15
|
from pathlib import Path
|
|
14
16
|
from types import SimpleNamespace
|
|
15
17
|
|
|
@@ -110,6 +112,27 @@ def test_a_malformed_window_is_an_error_not_a_default(monkeypatch):
|
|
|
110
112
|
reaper_module.max_idle_days()
|
|
111
113
|
|
|
112
114
|
|
|
115
|
+
def test_survey_workers_defaults_when_unset(monkeypatch):
|
|
116
|
+
monkeypatch.delenv(reaper_module.SURVEY_WORKERS_ENV_VAR, raising=False)
|
|
117
|
+
assert reaper_module.survey_workers() == reaper_module.DEFAULT_SURVEY_WORKERS
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_survey_workers_reads_the_env_var(monkeypatch):
|
|
121
|
+
monkeypatch.setenv(reaper_module.SURVEY_WORKERS_ENV_VAR, "3")
|
|
122
|
+
assert reaper_module.survey_workers() == 3
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_survey_workers_below_one_means_one(monkeypatch):
|
|
126
|
+
monkeypatch.setenv(reaper_module.SURVEY_WORKERS_ENV_VAR, "0")
|
|
127
|
+
assert reaper_module.survey_workers() == 1
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def test_malformed_survey_workers_is_an_error_not_a_default(monkeypatch):
|
|
131
|
+
monkeypatch.setenv(reaper_module.SURVEY_WORKERS_ENV_VAR, "lots")
|
|
132
|
+
with pytest.raises(ValueError, match="not a whole number of workers"):
|
|
133
|
+
reaper_module.survey_workers()
|
|
134
|
+
|
|
135
|
+
|
|
113
136
|
# ── Authority ────────────────────────────────────────────────────────────────
|
|
114
137
|
|
|
115
138
|
|
|
@@ -439,3 +462,58 @@ def test_survey_skips_protected_views_entirely():
|
|
|
439
462
|
)
|
|
440
463
|
reaper_module.survey(client, now=NOW, max_idle=14)
|
|
441
464
|
assert asked == ["act-alice/x"]
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def test_survey_ages_views_concurrently():
|
|
468
|
+
"""One `commit list` per view, serially, is what outgrew the CronJob's
|
|
469
|
+
600s deadline on production's 309-view bridge graph. Each call waits at a
|
|
470
|
+
barrier that opens only once `workers` calls are in flight together, so a
|
|
471
|
+
serial survey breaks the barrier instead of passing."""
|
|
472
|
+
workers = 4
|
|
473
|
+
barrier = threading.Barrier(workers)
|
|
474
|
+
|
|
475
|
+
def last_write(name):
|
|
476
|
+
barrier.wait(timeout=5)
|
|
477
|
+
return NOW
|
|
478
|
+
|
|
479
|
+
client = SimpleNamespace(
|
|
480
|
+
list_branches=lambda: [f"act-alice/v{i}" for i in range(workers)],
|
|
481
|
+
branch_last_write=last_write,
|
|
482
|
+
)
|
|
483
|
+
ages, _ = reaper_module.survey(client, now=NOW, max_idle=14, workers=workers)
|
|
484
|
+
assert len(ages) == workers
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def test_a_concurrent_survey_pairs_each_view_with_its_own_last_write():
|
|
488
|
+
"""Completion order must not leak into the result: a view aged against
|
|
489
|
+
its neighbour's timestamp is reaped, or kept, on someone else's evidence.
|
|
490
|
+
Earlier views finish last here, so an order-of-completion bug shows."""
|
|
491
|
+
names = [f"act-alice/v{i}" for i in range(6)]
|
|
492
|
+
idle_days = {name: 30.0 - i for i, name in enumerate(names)}
|
|
493
|
+
|
|
494
|
+
def last_write(name):
|
|
495
|
+
time.sleep(0.01 * (len(names) - names.index(name)))
|
|
496
|
+
return NOW - idle_days[name] * DAY
|
|
497
|
+
|
|
498
|
+
client = SimpleNamespace(list_branches=lambda: names, branch_last_write=last_write)
|
|
499
|
+
ages, _ = reaper_module.survey(client, now=NOW, max_idle=14, workers=3)
|
|
500
|
+
|
|
501
|
+
assert [a.view for a in ages] == names
|
|
502
|
+
assert {a.view: a.idle_days(NOW) for a in ages} == pytest.approx(idle_days)
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def test_a_concurrent_survey_still_raises_on_a_failed_read():
|
|
506
|
+
"""The CLI turns a RuntimeError from the survey into `FAILED to survey`
|
|
507
|
+
and a non-zero exit. Running the reads on a pool must not swallow it."""
|
|
508
|
+
|
|
509
|
+
def last_write(name):
|
|
510
|
+
if name == "act-alice/bad":
|
|
511
|
+
raise RuntimeError("omnigraph commit list failed")
|
|
512
|
+
return NOW
|
|
513
|
+
|
|
514
|
+
client = SimpleNamespace(
|
|
515
|
+
list_branches=lambda: ["act-alice/ok", "act-alice/bad", "act-bob/ok"],
|
|
516
|
+
branch_last_write=last_write,
|
|
517
|
+
)
|
|
518
|
+
with pytest.raises(RuntimeError, match="commit list failed"):
|
|
519
|
+
reaper_module.survey(client, now=NOW, max_idle=14, workers=3)
|
|
@@ -632,15 +632,21 @@ class _NotFoundConnection:
|
|
|
632
632
|
def request(self, method, path, body=None, headers=None):
|
|
633
633
|
pass
|
|
634
634
|
|
|
635
|
+
#: 0.13.0 dropped the quoted id: `{"error": "graph not found", "code":
|
|
636
|
+
#: "not_found"}`, measured the same way on 2026-10-08.
|
|
637
|
+
error = "graph 'nosuch' not found"
|
|
638
|
+
|
|
635
639
|
def getresponse(self):
|
|
640
|
+
_error = self.error
|
|
641
|
+
|
|
636
642
|
class _Response:
|
|
637
643
|
status = 404
|
|
638
644
|
will_close = False
|
|
645
|
+
# Unstamped, as a 0.11.0 server's responses are.
|
|
646
|
+
headers = http_module.http.client.HTTPMessage()
|
|
639
647
|
|
|
640
648
|
def read(self):
|
|
641
|
-
return json.dumps(
|
|
642
|
-
{"error": "graph 'nosuch' not found", "code": "not_found"}
|
|
643
|
-
).encode()
|
|
649
|
+
return json.dumps({"error": _error, "code": "not_found"}).encode()
|
|
644
650
|
|
|
645
651
|
def getheader(self, _name, default=None):
|
|
646
652
|
return default
|
|
@@ -651,18 +657,20 @@ class _NotFoundConnection:
|
|
|
651
657
|
pass
|
|
652
658
|
|
|
653
659
|
|
|
654
|
-
|
|
660
|
+
@pytest.mark.parametrize("error", ["graph 'nosuch' not found", "graph not found"])
|
|
661
|
+
def test_a_missing_cluster_graph_is_still_recognised_over_http(monkeypatch, error):
|
|
655
662
|
"""★ THE WORDING `_NOT_FOUND_RE` KEYS ON NOW ARRIVES FROM THE SERVER.
|
|
656
663
|
|
|
657
664
|
`list_branches` was unconditionally a subprocess, so `_NOT_FOUND_RE` was
|
|
658
665
|
tuned against what the CLI printed (see the comment above it). It is a
|
|
659
666
|
`branch list` statement now, so a remote client takes the pooled transport
|
|
660
667
|
and this discrimination runs on the HTTP body instead. If the server ever
|
|
661
|
-
stops quoting the id, every missing graph files as
|
|
662
|
-
rather than `ClusterGraphMissing`, which is the exact
|
|
663
|
-
types exist to prevent — and nothing else in the suite would notice,
|
|
668
|
+
stops quoting the id (0.13.0 did), every missing graph files as
|
|
669
|
+
`ClusterUnreachable` rather than `ClusterGraphMissing`, which is the exact
|
|
670
|
+
confusion those two types exist to prevent — and nothing else in the suite would notice,
|
|
664
671
|
because every other probe test fakes the client.
|
|
665
672
|
"""
|
|
673
|
+
monkeypatch.setattr(_NotFoundConnection, "error", error)
|
|
666
674
|
monkeypatch.setattr(http_module.http.client, "HTTPSConnection", _NotFoundConnection)
|
|
667
675
|
monkeypatch.setattr(http_module.http.client, "HTTPConnection", _NotFoundConnection)
|
|
668
676
|
monkeypatch.setenv("WITAN_CODE_SERVER", "https://omnigraph.test")
|
|
@@ -680,7 +680,10 @@ def _resolve_store(store: str | None, *, bridge: bool = False):
|
|
|
680
680
|
if ref.is_remote:
|
|
681
681
|
print(
|
|
682
682
|
f"{ref} is a shared cluster graph — compaction runs server-side "
|
|
683
|
-
"against the storage root, not from a client.
|
|
683
|
+
"against the storage root, not from a client. From omnigraph 0.12 "
|
|
684
|
+
"it is also refused while the server holds the cluster's `serve` "
|
|
685
|
+
"lock, so the data tier's maintenance job runs it in a stop "
|
|
686
|
+
"window. Nothing to do here."
|
|
684
687
|
)
|
|
685
688
|
return None
|
|
686
689
|
if not ref.exists():
|
|
@@ -744,18 +747,21 @@ def cleanup(
|
|
|
744
747
|
*,
|
|
745
748
|
store: str | None = None,
|
|
746
749
|
bridge: bool = False,
|
|
747
|
-
keep: int =
|
|
750
|
+
keep: int | None = None,
|
|
748
751
|
older_than: str | None = None,
|
|
749
752
|
yes: bool = False,
|
|
750
753
|
) -> None:
|
|
751
754
|
"""Remove old Lance versions from a code-graph store (**destructive**).
|
|
752
755
|
|
|
753
756
|
``optimize`` compacts fragments but leaves old versions behind; this GCs
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
+
the ones no retained graph commit pins. A commit is retained when either
|
|
758
|
+
bound keeps it. From omnigraph 0.11 it is also the only thing that
|
|
759
|
+
reclaims the storage of deleted branches (``branches --prune``,
|
|
757
760
|
``reap-views --apply``). Irreversible, so it requires ``--yes``.
|
|
758
761
|
|
|
762
|
+
With neither bound given it keeps the last 30 days, the same policy the
|
|
763
|
+
Stop hook and the deployed maintenance job use.
|
|
764
|
+
|
|
759
765
|
Parameters
|
|
760
766
|
----------
|
|
761
767
|
store
|
|
@@ -763,24 +769,36 @@ def cleanup(
|
|
|
763
769
|
bridge
|
|
764
770
|
Clean the shared cross-repo bridge store instead.
|
|
765
771
|
keep
|
|
766
|
-
|
|
772
|
+
Keep the newest N graph commits of every live branch, whatever
|
|
773
|
+
their age. (omnigraph 0.11 counted versions per table instead.)
|
|
767
774
|
older_than
|
|
768
|
-
|
|
775
|
+
Keep every graph commit newer than this Go-style duration (e.g. 7d).
|
|
769
776
|
yes
|
|
770
777
|
Confirm the destructive operation (required to actually run).
|
|
771
778
|
"""
|
|
779
|
+
from . import maintenance
|
|
780
|
+
|
|
772
781
|
ref = _resolve_store(store, bridge=bridge)
|
|
773
782
|
if ref is None:
|
|
774
783
|
return
|
|
784
|
+
if keep is None and older_than is None:
|
|
785
|
+
older_than = maintenance.CLEANUP_OLDER_THAN
|
|
786
|
+
kept = []
|
|
787
|
+
if keep is not None:
|
|
788
|
+
kept.append(
|
|
789
|
+
f"the {keep} newest graph commit(s) of every live branch "
|
|
790
|
+
"(version(s) per table on omnigraph 0.11)"
|
|
791
|
+
)
|
|
792
|
+
if older_than is not None:
|
|
793
|
+
kept.append(f"every graph commit newer than {older_than}")
|
|
794
|
+
policy = " and ".join(kept)
|
|
775
795
|
if not yes:
|
|
776
796
|
print(
|
|
777
|
-
f"cleanup is destructive — would keep
|
|
778
|
-
|
|
779
|
-
+ (f" and anything newer than {older_than}" if older_than else "")
|
|
780
|
-
+ f" in {ref}.\nRe-run with --yes to proceed."
|
|
797
|
+
f"cleanup is destructive — would keep {policy} in {ref}.\n"
|
|
798
|
+
"Re-run with --yes to proceed."
|
|
781
799
|
)
|
|
782
800
|
return
|
|
783
|
-
print(f"Cleaning up {ref} (
|
|
801
|
+
print(f"Cleaning up {ref} (keeping {policy}) …")
|
|
784
802
|
_maintenance_client(ref).cleanup(keep=keep, older_than=older_than)
|
|
785
803
|
print("Cleaned up.")
|
|
786
804
|
|
|
@@ -841,6 +859,7 @@ def reap_views(
|
|
|
841
859
|
return
|
|
842
860
|
try:
|
|
843
861
|
idle = reaper_module.max_idle_days() if max_idle_days is None else max_idle_days
|
|
862
|
+
workers = reaper_module.survey_workers()
|
|
844
863
|
except ValueError as exc:
|
|
845
864
|
print(exc)
|
|
846
865
|
raise SystemExit(1) from None
|
|
@@ -865,9 +884,18 @@ def reap_views(
|
|
|
865
884
|
|
|
866
885
|
failed_graphs = 0
|
|
867
886
|
for name, client in targets:
|
|
887
|
+
# Before the survey, not after: the report only prints once a graph is
|
|
888
|
+
# done, so a job killed mid-survey otherwise leaves no trace of which
|
|
889
|
+
# graph it was on.
|
|
890
|
+
print(f"{name}: surveying views", flush=True)
|
|
868
891
|
try:
|
|
869
892
|
report = reaper_module.reap(
|
|
870
|
-
client,
|
|
893
|
+
client,
|
|
894
|
+
graph=name,
|
|
895
|
+
max_idle=idle,
|
|
896
|
+
apply=apply,
|
|
897
|
+
cfg=cfg,
|
|
898
|
+
workers=workers,
|
|
871
899
|
)
|
|
872
900
|
except PermissionError as exc:
|
|
873
901
|
print(exc)
|
|
@@ -45,6 +45,7 @@ from __future__ import annotations
|
|
|
45
45
|
import os
|
|
46
46
|
import time
|
|
47
47
|
from collections.abc import Iterable
|
|
48
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
48
49
|
from dataclasses import dataclass, field
|
|
49
50
|
|
|
50
51
|
from . import config as cfg_module
|
|
@@ -52,13 +53,16 @@ from . import views
|
|
|
52
53
|
|
|
53
54
|
__all__ = [
|
|
54
55
|
"DEFAULT_MAX_IDLE_DAYS",
|
|
56
|
+
"DEFAULT_SURVEY_WORKERS",
|
|
55
57
|
"MAX_IDLE_ENV_VAR",
|
|
56
58
|
"PROTECTED_VIEWS",
|
|
59
|
+
"SURVEY_WORKERS_ENV_VAR",
|
|
57
60
|
"ReapReport",
|
|
58
61
|
"ViewAge",
|
|
59
62
|
"max_idle_days",
|
|
60
63
|
"reap",
|
|
61
64
|
"select_stale",
|
|
65
|
+
"survey_workers",
|
|
62
66
|
]
|
|
63
67
|
|
|
64
68
|
MAX_IDLE_ENV_VAR = "WITAN_CODE_VIEW_MAX_IDLE_DAYS"
|
|
@@ -67,6 +71,16 @@ MAX_IDLE_ENV_VAR = "WITAN_CODE_VIEW_MAX_IDLE_DAYS"
|
|
|
67
71
|
DEFAULT_MAX_IDLE_DAYS = 14.0
|
|
68
72
|
"""Long enough to cover a two-week vacation, short enough to bound sprawl."""
|
|
69
73
|
|
|
74
|
+
SURVEY_WORKERS_ENV_VAR = "WITAN_CODE_VIEW_SURVEY_WORKERS"
|
|
75
|
+
"""How many views to age at once. ``1`` ages them one after another."""
|
|
76
|
+
|
|
77
|
+
DEFAULT_SURVEY_WORKERS = 8
|
|
78
|
+
"""Measured against production's bridge graph (309 views, omnigraph 0.11):
|
|
79
|
+
one at a time took 327s, 4 workers 85s, 8 workers 50s, and right after the
|
|
80
|
+
8-worker run ``kubectl top`` read the 1-CPU omnigraph-server at 780m. Kept
|
|
81
|
+
there rather than higher so the sweep leaves that server room for its other
|
|
82
|
+
readers."""
|
|
83
|
+
|
|
70
84
|
PROTECTED_VIEWS = frozenset({"main"})
|
|
71
85
|
"""Never reaped, whatever its age.
|
|
72
86
|
|
|
@@ -89,6 +103,20 @@ def max_idle_days() -> float:
|
|
|
89
103
|
return max(value, 0.0)
|
|
90
104
|
|
|
91
105
|
|
|
106
|
+
def survey_workers() -> int:
|
|
107
|
+
"""The configured survey concurrency; anything below ``1`` means ``1``."""
|
|
108
|
+
raw = os.environ.get(SURVEY_WORKERS_ENV_VAR)
|
|
109
|
+
if raw is None or not raw.strip():
|
|
110
|
+
return DEFAULT_SURVEY_WORKERS
|
|
111
|
+
try:
|
|
112
|
+
value = int(raw)
|
|
113
|
+
except ValueError as exc:
|
|
114
|
+
raise ValueError(
|
|
115
|
+
f"{SURVEY_WORKERS_ENV_VAR}={raw!r} is not a whole number of workers"
|
|
116
|
+
) from exc
|
|
117
|
+
return max(value, 1)
|
|
118
|
+
|
|
119
|
+
|
|
92
120
|
@dataclass(frozen=True)
|
|
93
121
|
class ViewAge:
|
|
94
122
|
"""One branch view and when it was last written."""
|
|
@@ -152,21 +180,46 @@ class ReapReport:
|
|
|
152
180
|
|
|
153
181
|
|
|
154
182
|
def survey(
|
|
155
|
-
client, *, now: float, max_idle: float
|
|
183
|
+
client, *, now: float, max_idle: float, workers: int | None = None
|
|
156
184
|
) -> tuple[list[ViewAge], list[ViewAge]]:
|
|
157
185
|
"""Age every view on ``client``'s graph; return ``(all, stale)``.
|
|
158
186
|
|
|
159
187
|
Read-only — the dry-run half of :func:`reap`, and what the CLI prints when
|
|
160
188
|
it has not been told to delete anything.
|
|
161
189
|
"""
|
|
190
|
+
names = [name for name in client.list_branches() if name not in PROTECTED_VIEWS]
|
|
191
|
+
workers = survey_workers() if workers is None else workers
|
|
192
|
+
last_writes = _last_writes(client, names, workers=workers)
|
|
162
193
|
ages = [
|
|
163
|
-
ViewAge(view=name, last_write=
|
|
164
|
-
for name in
|
|
165
|
-
if name not in PROTECTED_VIEWS
|
|
194
|
+
ViewAge(view=name, last_write=last_write)
|
|
195
|
+
for name, last_write in zip(names, last_writes, strict=True)
|
|
166
196
|
]
|
|
167
197
|
return ages, select_stale(ages, now=now, max_idle=max_idle)
|
|
168
198
|
|
|
169
199
|
|
|
200
|
+
def _last_writes(client, names: list[str], *, workers: int) -> list[float | None]:
|
|
201
|
+
"""``client.branch_last_write`` for each of ``names``, in order.
|
|
202
|
+
|
|
203
|
+
Concurrent because the survey is one ``commit list --branch`` per view and
|
|
204
|
+
each returns the view's whole reachable history, ``main``'s included — so
|
|
205
|
+
the sweep costs views x commit-log length and both only grow. On omnigraph
|
|
206
|
+
0.11 ``commit list`` without ``--branch`` returns ``main``'s commits alone
|
|
207
|
+
and ``changes`` is per-branch too, so the calls can be overlapped but not
|
|
208
|
+
merged. Serially, production's bridge graph alone outgrew the reaper
|
|
209
|
+
CronJob's 600s deadline.
|
|
210
|
+
|
|
211
|
+
The first failure propagates, as it did serially, and views not yet
|
|
212
|
+
started are cancelled rather than aged for a survey that is already lost.
|
|
213
|
+
"""
|
|
214
|
+
if workers <= 1 or len(names) <= 1:
|
|
215
|
+
return [client.branch_last_write(name) for name in names]
|
|
216
|
+
pool = ThreadPoolExecutor(max_workers=min(workers, len(names)))
|
|
217
|
+
try:
|
|
218
|
+
return list(pool.map(client.branch_last_write, names))
|
|
219
|
+
finally:
|
|
220
|
+
pool.shutdown(cancel_futures=True)
|
|
221
|
+
|
|
222
|
+
|
|
170
223
|
def reap(
|
|
171
224
|
client,
|
|
172
225
|
*,
|
|
@@ -175,6 +228,7 @@ def reap(
|
|
|
175
228
|
max_idle: float | None = None,
|
|
176
229
|
apply: bool = False,
|
|
177
230
|
cfg: cfg_module.Config | None = None,
|
|
231
|
+
workers: int | None = None,
|
|
178
232
|
) -> ReapReport:
|
|
179
233
|
"""Sweep one graph's stale branch views.
|
|
180
234
|
|
|
@@ -208,7 +262,7 @@ def reap(
|
|
|
208
262
|
if max_idle <= 0:
|
|
209
263
|
return ReapReport(graph=graph, now=now)
|
|
210
264
|
|
|
211
|
-
ages, stale = survey(client, now=now, max_idle=max_idle)
|
|
265
|
+
ages, stale = survey(client, now=now, max_idle=max_idle, workers=workers)
|
|
212
266
|
report = ReapReport(graph=graph, now=now, scanned=len(ages), stale=stale)
|
|
213
267
|
if not apply:
|
|
214
268
|
return report
|
|
@@ -554,7 +554,7 @@ def discard_store(ref: StoreRef) -> int:
|
|
|
554
554
|
# these two exception types exist to prevent.
|
|
555
555
|
_NOT_SERVED = "is not served by the omnigraph-server"
|
|
556
556
|
_NOT_FOUND_RE = re.compile(
|
|
557
|
-
rf"graph '[^']*' not found|{re.escape(_NOT_SERVED)}", re.IGNORECASE
|
|
557
|
+
rf"graph (?:'[^']*' )?not found|{re.escape(_NOT_SERVED)}", re.IGNORECASE
|
|
558
558
|
)
|
|
559
559
|
|
|
560
560
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|