witan-code 0.24.0__tar.gz → 0.25.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. {witan_code-0.24.0 → witan_code-0.25.0}/.gitignore +1 -1
  2. {witan_code-0.24.0 → witan_code-0.25.0}/CHANGELOG.md +26 -0
  3. {witan_code-0.24.0 → witan_code-0.25.0}/PKG-INFO +5 -3
  4. {witan_code-0.24.0 → witan_code-0.25.0}/README.md +3 -1
  5. {witan_code-0.24.0 → witan_code-0.25.0}/docs/BRANCH_INDEXING.md +2 -0
  6. {witan_code-0.24.0 → witan_code-0.25.0}/pyproject.toml +9 -3
  7. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_maintenance.py +31 -0
  8. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_reaper.py +78 -0
  9. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_store.py +15 -7
  10. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/cli.py +41 -13
  11. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/reaper.py +59 -5
  12. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/store.py +1 -1
  13. {witan_code-0.24.0 → witan_code-0.25.0}/changelog.d/README.md +0 -0
  14. {witan_code-0.24.0 → witan_code-0.25.0}/config/claude.json +0 -0
  15. {witan_code-0.24.0 → witan_code-0.25.0}/config/copilot.json +0 -0
  16. {witan_code-0.24.0 → witan_code-0.25.0}/config/pi.json +0 -0
  17. {witan_code-0.24.0 → witan_code-0.25.0}/conftest.py +0 -0
  18. {witan_code-0.24.0 → witan_code-0.25.0}/docs/CLI_REFERENCE.md +0 -0
  19. {witan_code-0.24.0 → witan_code-0.25.0}/docs/EDGE_PRECISION_TIERS.md +0 -0
  20. {witan_code-0.24.0 → witan_code-0.25.0}/docs/PACKAGE_MAP.md +0 -0
  21. {witan_code-0.24.0 → witan_code-0.25.0}/docs/STAGE2_STITCHING.md +0 -0
  22. {witan_code-0.24.0 → witan_code-0.25.0}/docs/SYMBOL_FORMAT.md +0 -0
  23. {witan_code-0.24.0 → witan_code-0.25.0}/docs/SYMBOL_TABLE.md +0 -0
  24. {witan_code-0.24.0 → witan_code-0.25.0}/docs/USER_GUIDE.md +0 -0
  25. {witan_code-0.24.0 → witan_code-0.25.0}/install.sh +0 -0
  26. {witan_code-0.24.0 → witan_code-0.25.0}/tests/__init__.py +0 -0
  27. {witan_code-0.24.0 → witan_code-0.25.0}/tests/conftest.py +0 -0
  28. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_batched_deletes.py +0 -0
  29. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_branches.py +0 -0
  30. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_bridge.py +0 -0
  31. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_chunked_load.py +0 -0
  32. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_dispatch.py +0 -0
  33. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_hooks.py +0 -0
  34. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_output_format.py +0 -0
  35. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_cli_selected_target.py +0 -0
  36. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_config.py +0 -0
  37. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_config_targets.py +0 -0
  38. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_context.py +0 -0
  39. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_crossrepo.py +0 -0
  40. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_edges.py +0 -0
  41. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_elicit.py +0 -0
  42. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_enrichment.py +0 -0
  43. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_github_app.py +0 -0
  44. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_graph.py +0 -0
  45. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_hooks.py +0 -0
  46. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_identity.py +0 -0
  47. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_index_failure_diagnostics.py +0 -0
  48. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_index_reports_its_store.py +0 -0
  49. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_index_requires_a_repo_key.py +0 -0
  50. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_indexer.py +0 -0
  51. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_indexer_scope.py +0 -0
  52. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_ingest.py +0 -0
  53. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_languages.py +0 -0
  54. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_package_map.py +0 -0
  55. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_rebuild_honours_the_repo_key.py +0 -0
  56. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_refusal_audit.py +0 -0
  57. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_remote_proxy.py +0 -0
  58. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_remote_store.py +0 -0
  59. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_repo.py +0 -0
  60. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_server.py +0 -0
  61. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_setup.py +0 -0
  62. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_stitch.py +0 -0
  63. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_symbol_table.py +0 -0
  64. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_tasks.py +0 -0
  65. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_tool_schema.py +0 -0
  66. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_views.py +0 -0
  67. {witan_code-0.24.0 → witan_code-0.25.0}/tests/test_visualize.py +0 -0
  68. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/__init__.py +0 -0
  69. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/__main__.py +0 -0
  70. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/bridge.py +0 -0
  71. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/bridge_extractors.py +0 -0
  72. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/config.py +0 -0
  73. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/context.py +0 -0
  74. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/edges.py +0 -0
  75. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/elicit.py +0 -0
  76. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/extensions/pi/codegraph.ts +0 -0
  77. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/github_app.py +0 -0
  78. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/graph.py +0 -0
  79. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/hooks.py +0 -0
  80. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/identity.py +0 -0
  81. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/indexer.py +0 -0
  82. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/ingest.py +0 -0
  83. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/maintenance.py +0 -0
  84. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/output.py +0 -0
  85. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/package_map.py +0 -0
  86. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/bridge.gq +0 -0
  87. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/code_mutations.gq +0 -0
  88. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/code_read.gq +0 -0
  89. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries/delete.gq +0 -0
  90. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/bash.scm +0 -0
  91. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/hcl.scm +0 -0
  92. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/python.scm +0 -0
  93. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/sql.scm +0 -0
  94. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/typescript.scm +0 -0
  95. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/queries_ts/yaml.scm +0 -0
  96. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/__init__.py +0 -0
  97. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/oidc.py +0 -0
  98. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/proxy.py +0 -0
  99. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/remote/store.py +0 -0
  100. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/repo.py +0 -0
  101. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/schema/bridge-schema.pg +0 -0
  102. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/schema/code-schema.pg +0 -0
  103. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/selected_target.py +0 -0
  104. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/server.py +0 -0
  105. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/setup.py +0 -0
  106. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/skills/witan-code/SKILL.md +0 -0
  107. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/stitch.py +0 -0
  108. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/views.py +0 -0
  109. {witan_code-0.24.0 → witan_code-0.25.0}/witan_code/visualize.py +0 -0
@@ -26,7 +26,7 @@ build/
26
26
  *.egg-info/
27
27
  # omnigraph binary downloaded by hatch_build.py — not committed
28
28
  _bin/
29
- # omnigraph cluster state (populated by `omnigraph cluster import`)
29
+ # omnigraph cluster state (populated by `omnigraph cluster apply`)
30
30
  __cluster/
31
31
  # The witan UI bundle, written by `npm run build` in mcp/servers/witan/ui/.
32
32
  # A build output, so not committed — hatch's `artifacts` setting in that
@@ -8,6 +8,32 @@ a MINOR bump may include breaking changes).
8
8
 
9
9
  <!-- scriv-insert-here -->
10
10
 
11
+ ## [0.25.0] - 2026-10-15
12
+
13
+ ### Changed
14
+
15
+ - `witan-code cleanup` no longer defaults to `--keep 10`. From omnigraph 0.12
16
+ `--keep` counts graph commits per live branch, not versions per table, so
17
+ that default would cut a store to its last ten writes. With neither bound
18
+ it now keeps the last 30 days, as the Stop hook does. The help text
19
+ describes both bounds in graph commits. `--older-than` on its own
20
+ no longer adds an implicit `--keep 10`.
21
+ - The refusal for a shared cluster graph notes that from omnigraph 0.12
22
+ compaction is also refused while the server holds the cluster's `serve`
23
+ lock.
24
+
25
+ - Requires witan-core 0.41 or later, the first release on omnigraph 0.13.
26
+ Every deployed omnigraph server is on 0.13 and refuses a client that does
27
+ not send `Omnigraph-Http-Api: 0.13`.
28
+
29
+ ### Fixed
30
+
31
+ - `reap-views` ages a graph's views concurrently, `WITAN_CODE_VIEW_SURVEY_WORKERS` at a time (default 8; `1` is serial). Ageing a view is one `commit list --branch` that returns its whole reachable history, and serially production's 309-view bridge graph took 327s on its own, pushing the reaper CronJob past its 600s deadline; 8 workers take it to 50s. Each graph now also prints a `surveying views` line before its survey, so a job killed mid-sweep shows which graph it was on.
32
+
33
+ - A cluster graph that is not served is recognised on omnigraph 0.13, whose
34
+ 404 says `graph not found` without the quoted id. It was reported as an
35
+ unreachable cluster.
36
+
11
37
  ## [0.24.0] - 2026-10-02
12
38
 
13
39
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: witan-code
3
- Version: 0.24.0
3
+ Version: 0.25.0
4
4
  Summary: witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)
5
5
  Project-URL: Homepage, https://github.com/mitodl/agent-kit/tree/main/mcp/servers/witan-code
6
6
  Project-URL: Repository, https://github.com/mitodl/agent-kit
@@ -26,7 +26,7 @@ Requires-Dist: tree-sitter-sql<0.4,>=0.3
26
26
  Requires-Dist: tree-sitter-typescript<0.24,>=0.23
27
27
  Requires-Dist: tree-sitter-yaml<0.8,>=0.7
28
28
  Requires-Dist: tree-sitter<0.26,>=0.25
29
- Requires-Dist: witan-core[cli,observability,remote,sentry]<1,>=0.40
29
+ Requires-Dist: witan-core[cli,observability,remote,sentry]<1,>=0.41
30
30
  Provides-Extra: tasks
31
31
  Requires-Dist: fastmcp-tasks<5,>=4; extra == 'tasks'
32
32
  Description-Content-Type: text/markdown
@@ -584,7 +584,9 @@ graph):
584
584
  instead.
585
585
  - `cleanup [--store PATH] [--bridge] [--keep N] [--older-than DUR] --yes` —
586
586
  reclaim disk by GC'ing old Lance versions (**destructive**; requires
587
- `--yes`).
587
+ `--yes`). `--keep N` retains the newest N graph commits of every live
588
+ branch (omnigraph 0.12+; versions per table on 0.11); with neither bound it
589
+ keeps the last 30 days.
588
590
  - `checkpoint` — opportunistically compact the current repo's store and the
589
591
  bridge store if due (see [Hooks](#hooks)); registered as the `Stop` hook,
590
592
  not usually run by hand.
@@ -551,7 +551,9 @@ graph):
551
551
  instead.
552
552
  - `cleanup [--store PATH] [--bridge] [--keep N] [--older-than DUR] --yes` —
553
553
  reclaim disk by GC'ing old Lance versions (**destructive**; requires
554
- `--yes`).
554
+ `--yes`). `--keep N` retains the newest N graph commits of every live
555
+ branch (omnigraph 0.12+; versions per table on 0.11); with neither bound it
556
+ keeps the last 30 days.
555
557
  - `checkpoint` — opportunistically compact the current repo's store and the
556
558
  bridge store if due (see [Hooks](#hooks)); registered as the `Stop` hook,
557
559
  not usually run by hand.
@@ -222,6 +222,8 @@ date, owner, or size. Two rules follow:
222
222
  ago — reaping it would race the indexer that just created it.
223
223
 
224
224
  The window is `WITAN_CODE_VIEW_MAX_IDLE_DAYS` (default 14; `0` disables).
225
+ Ageing a view is one `commit list` per view, so the sweep ages
226
+ `WITAN_CODE_VIEW_SURVEY_WORKERS` views at once (default 8; `1` is serial).
225
227
  Idleness is not abandonment: a branch parked past the window and picked back up
226
228
  loses its view, not its work, and the next index rebuilds it. Reporting is the
227
229
  default and `--apply` deletes, because the window is the one input nothing
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "witan-code"
3
- version = "0.24.0"
3
+ version = "0.25.0"
4
4
  description = "witan-code — tree-sitter code graph + cross-repo bridge (mounts under `witan code`)"
5
5
  readme = "README.md"
6
6
  license = "BSD-3-Clause"
@@ -170,7 +170,13 @@ dependencies = [
170
170
  # `witan_core.cli.report_install` only prints `InstallResult.prerequisites`
171
171
  # from 0.40.0. It reads the field with getattr, so 0.39 imports and runs —
172
172
  # and silently drops the warning.
173
- "witan-core[cli,remote,observability,sentry]>=0.40,<1",
173
+ #
174
+ # ★ >=0.41 IS A COMPATIBILITY FLOOR: 0.41.0 pins omnigraph 0.13 (storage
175
+ # format 14) and sends `Omnigraph-Http-Api: 0.13`. Every deployed server
176
+ # is on 0.13, which answers a request without that header with `400
177
+ # api_contract_mismatch`, so 0.40 imports and runs, then fails on the
178
+ # first read of a shared code graph.
179
+ "witan-core[cli,remote,observability,sentry]>=0.41,<1",
174
180
  "fastmcp>=4,<5",
175
181
  "cyclopts>=4,<6",
176
182
  # [targets.<name>] override models (config.py) — also transitively pulled
@@ -253,7 +259,7 @@ testpaths = ["tests"]
253
259
  packages = ["witan_code"]
254
260
 
255
261
  [tool.bumpversion]
256
- current_version = "0.24.0"
262
+ current_version = "0.25.0"
257
263
  allow_dirty = true
258
264
 
259
265
  [[tool.bumpversion.files]]
@@ -231,6 +231,37 @@ def test_cli_optimize_and_cleanup(tmp_path, capsys):
231
231
  assert "Cleaned up" in capsys.readouterr().out
232
232
 
233
233
 
234
+ def test_cli_cleanup_with_no_bound_keeps_the_hooks_window(
235
+ tmp_path, capsys, monkeypatch
236
+ ):
237
+ """From omnigraph 0.12 `--keep` counts graph commits, so a default of
238
+ `--keep 10` would cut a store to its last ten writes. No bound means the
239
+ age window the Stop hook already uses."""
240
+ from witan_code import cli as cli_module
241
+ from witan_code import maintenance
242
+
243
+ store = tmp_path / "s.omni"
244
+ store.mkdir()
245
+ calls = []
246
+
247
+ class _Client:
248
+ def cleanup(self, **kwargs):
249
+ calls.append(kwargs)
250
+
251
+ monkeypatch.setattr(cli_module, "_maintenance_client", lambda _ref: _Client())
252
+
253
+ cli_module.cleanup(store=str(store))
254
+ assert f"newer than {maintenance.CLEANUP_OLDER_THAN}" in capsys.readouterr().out
255
+ assert calls == []
256
+
257
+ cli_module.cleanup(store=str(store), yes=True)
258
+ cli_module.cleanup(store=str(store), keep=3, yes=True)
259
+ assert calls == [
260
+ {"keep": None, "older_than": maintenance.CLEANUP_OLDER_THAN},
261
+ {"keep": 3, "older_than": None},
262
+ ]
263
+
264
+
234
265
  @requires_omnigraph
235
266
  def test_cli_optimize_then_cleanup(tmp_path, capsys):
236
267
  from witan_code import cli as cli_module
@@ -10,6 +10,8 @@ branch's own last write is recoverable from `commit list --branch` via
10
10
  identically, so it is asserted rather than mocked.
11
11
  """
12
12
 
13
+ import threading
14
+ import time
13
15
  from pathlib import Path
14
16
  from types import SimpleNamespace
15
17
 
@@ -110,6 +112,27 @@ def test_a_malformed_window_is_an_error_not_a_default(monkeypatch):
110
112
  reaper_module.max_idle_days()
111
113
 
112
114
 
115
+ def test_survey_workers_defaults_when_unset(monkeypatch):
116
+ monkeypatch.delenv(reaper_module.SURVEY_WORKERS_ENV_VAR, raising=False)
117
+ assert reaper_module.survey_workers() == reaper_module.DEFAULT_SURVEY_WORKERS
118
+
119
+
120
+ def test_survey_workers_reads_the_env_var(monkeypatch):
121
+ monkeypatch.setenv(reaper_module.SURVEY_WORKERS_ENV_VAR, "3")
122
+ assert reaper_module.survey_workers() == 3
123
+
124
+
125
+ def test_survey_workers_below_one_means_one(monkeypatch):
126
+ monkeypatch.setenv(reaper_module.SURVEY_WORKERS_ENV_VAR, "0")
127
+ assert reaper_module.survey_workers() == 1
128
+
129
+
130
+ def test_malformed_survey_workers_is_an_error_not_a_default(monkeypatch):
131
+ monkeypatch.setenv(reaper_module.SURVEY_WORKERS_ENV_VAR, "lots")
132
+ with pytest.raises(ValueError, match="not a whole number of workers"):
133
+ reaper_module.survey_workers()
134
+
135
+
113
136
  # ── Authority ────────────────────────────────────────────────────────────────
114
137
 
115
138
 
@@ -439,3 +462,58 @@ def test_survey_skips_protected_views_entirely():
439
462
  )
440
463
  reaper_module.survey(client, now=NOW, max_idle=14)
441
464
  assert asked == ["act-alice/x"]
465
+
466
+
467
+ def test_survey_ages_views_concurrently():
468
+ """One `commit list` per view, serially, is what outgrew the CronJob's
469
+ 600s deadline on production's 309-view bridge graph. Each call waits at a
470
+ barrier that opens only once `workers` calls are in flight together, so a
471
+ serial survey breaks the barrier instead of passing."""
472
+ workers = 4
473
+ barrier = threading.Barrier(workers)
474
+
475
+ def last_write(name):
476
+ barrier.wait(timeout=5)
477
+ return NOW
478
+
479
+ client = SimpleNamespace(
480
+ list_branches=lambda: [f"act-alice/v{i}" for i in range(workers)],
481
+ branch_last_write=last_write,
482
+ )
483
+ ages, _ = reaper_module.survey(client, now=NOW, max_idle=14, workers=workers)
484
+ assert len(ages) == workers
485
+
486
+
487
+ def test_a_concurrent_survey_pairs_each_view_with_its_own_last_write():
488
+ """Completion order must not leak into the result: a view aged against
489
+ its neighbour's timestamp is reaped, or kept, on someone else's evidence.
490
+ Earlier views finish last here, so an order-of-completion bug shows."""
491
+ names = [f"act-alice/v{i}" for i in range(6)]
492
+ idle_days = {name: 30.0 - i for i, name in enumerate(names)}
493
+
494
+ def last_write(name):
495
+ time.sleep(0.01 * (len(names) - names.index(name)))
496
+ return NOW - idle_days[name] * DAY
497
+
498
+ client = SimpleNamespace(list_branches=lambda: names, branch_last_write=last_write)
499
+ ages, _ = reaper_module.survey(client, now=NOW, max_idle=14, workers=3)
500
+
501
+ assert [a.view for a in ages] == names
502
+ assert {a.view: a.idle_days(NOW) for a in ages} == pytest.approx(idle_days)
503
+
504
+
505
+ def test_a_concurrent_survey_still_raises_on_a_failed_read():
506
+ """The CLI turns a RuntimeError from the survey into `FAILED to survey`
507
+ and a non-zero exit. Running the reads on a pool must not swallow it."""
508
+
509
+ def last_write(name):
510
+ if name == "act-alice/bad":
511
+ raise RuntimeError("omnigraph commit list failed")
512
+ return NOW
513
+
514
+ client = SimpleNamespace(
515
+ list_branches=lambda: ["act-alice/ok", "act-alice/bad", "act-bob/ok"],
516
+ branch_last_write=last_write,
517
+ )
518
+ with pytest.raises(RuntimeError, match="commit list failed"):
519
+ reaper_module.survey(client, now=NOW, max_idle=14, workers=3)
@@ -632,15 +632,21 @@ class _NotFoundConnection:
632
632
  def request(self, method, path, body=None, headers=None):
633
633
  pass
634
634
 
635
+ #: 0.13.0 dropped the quoted id: `{"error": "graph not found", "code":
636
+ #: "not_found"}`, measured the same way on 2026-10-08.
637
+ error = "graph 'nosuch' not found"
638
+
635
639
  def getresponse(self):
640
+ _error = self.error
641
+
636
642
  class _Response:
637
643
  status = 404
638
644
  will_close = False
645
+ # Unstamped, as a 0.11.0 server's responses are.
646
+ headers = http_module.http.client.HTTPMessage()
639
647
 
640
648
  def read(self):
641
- return json.dumps(
642
- {"error": "graph 'nosuch' not found", "code": "not_found"}
643
- ).encode()
649
+ return json.dumps({"error": _error, "code": "not_found"}).encode()
644
650
 
645
651
  def getheader(self, _name, default=None):
646
652
  return default
@@ -651,18 +657,20 @@ class _NotFoundConnection:
651
657
  pass
652
658
 
653
659
 
654
- def test_a_missing_cluster_graph_is_still_recognised_over_http(monkeypatch):
660
+ @pytest.mark.parametrize("error", ["graph 'nosuch' not found", "graph not found"])
661
+ def test_a_missing_cluster_graph_is_still_recognised_over_http(monkeypatch, error):
655
662
  """★ THE WORDING `_NOT_FOUND_RE` KEYS ON NOW ARRIVES FROM THE SERVER.
656
663
 
657
664
  `list_branches` was unconditionally a subprocess, so `_NOT_FOUND_RE` was
658
665
  tuned against what the CLI printed (see the comment above it). It is a
659
666
  `branch list` statement now, so a remote client takes the pooled transport
660
667
  and this discrimination runs on the HTTP body instead. If the server ever
661
- stops quoting the id, every missing graph files as `ClusterUnreachable`
662
- rather than `ClusterGraphMissing`, which is the exact confusion those two
663
- types exist to prevent — and nothing else in the suite would notice,
668
+ stops quoting the id (0.13.0 did), every missing graph files as
669
+ `ClusterUnreachable` rather than `ClusterGraphMissing`, which is the exact
670
+ confusion those two types exist to prevent — and nothing else in the suite would notice,
664
671
  because every other probe test fakes the client.
665
672
  """
673
+ monkeypatch.setattr(_NotFoundConnection, "error", error)
666
674
  monkeypatch.setattr(http_module.http.client, "HTTPSConnection", _NotFoundConnection)
667
675
  monkeypatch.setattr(http_module.http.client, "HTTPConnection", _NotFoundConnection)
668
676
  monkeypatch.setenv("WITAN_CODE_SERVER", "https://omnigraph.test")
@@ -680,7 +680,10 @@ def _resolve_store(store: str | None, *, bridge: bool = False):
680
680
  if ref.is_remote:
681
681
  print(
682
682
  f"{ref} is a shared cluster graph — compaction runs server-side "
683
- "against the storage root, not from a client. Nothing to do here."
683
+ "against the storage root, not from a client. From omnigraph 0.12 "
684
+ "it is also refused while the server holds the cluster's `serve` "
685
+ "lock, so the data tier's maintenance job runs it in a stop "
686
+ "window. Nothing to do here."
684
687
  )
685
688
  return None
686
689
  if not ref.exists():
@@ -744,18 +747,21 @@ def cleanup(
744
747
  *,
745
748
  store: str | None = None,
746
749
  bridge: bool = False,
747
- keep: int = 10,
750
+ keep: int | None = None,
748
751
  older_than: str | None = None,
749
752
  yes: bool = False,
750
753
  ) -> None:
751
754
  """Remove old Lance versions from a code-graph store (**destructive**).
752
755
 
753
756
  ``optimize`` compacts fragments but leaves old versions behind; this GCs
754
- them, keeping the most recent ``keep`` versions per table (and/or those
755
- newer than ``older_than``). From omnigraph 0.11 it is also the only thing
756
- that reclaims the storage of deleted branches (``branches --prune``,
757
+ the ones no retained graph commit pins. A commit is retained when either
758
+ bound keeps it. From omnigraph 0.11 it is also the only thing that
759
+ reclaims the storage of deleted branches (``branches --prune``,
757
760
  ``reap-views --apply``). Irreversible, so it requires ``--yes``.
758
761
 
762
+ With neither bound given it keeps the last 30 days, the same policy the
763
+ Stop hook and the deployed maintenance job use.
764
+
759
765
  Parameters
760
766
  ----------
761
767
  store
@@ -763,24 +769,36 @@ def cleanup(
763
769
  bridge
764
770
  Clean the shared cross-repo bridge store instead.
765
771
  keep
766
- Number of recent versions to keep per table.
772
+ Keep the newest N graph commits of every live branch, whatever
773
+ their age. (omnigraph 0.11 counted versions per table instead.)
767
774
  older_than
768
- Also keep versions newer than this Go-style duration (e.g. 7d).
775
+ Keep every graph commit newer than this Go-style duration (e.g. 7d).
769
776
  yes
770
777
  Confirm the destructive operation (required to actually run).
771
778
  """
779
+ from . import maintenance
780
+
772
781
  ref = _resolve_store(store, bridge=bridge)
773
782
  if ref is None:
774
783
  return
784
+ if keep is None and older_than is None:
785
+ older_than = maintenance.CLEANUP_OLDER_THAN
786
+ kept = []
787
+ if keep is not None:
788
+ kept.append(
789
+ f"the {keep} newest graph commit(s) of every live branch "
790
+ "(version(s) per table on omnigraph 0.11)"
791
+ )
792
+ if older_than is not None:
793
+ kept.append(f"every graph commit newer than {older_than}")
794
+ policy = " and ".join(kept)
775
795
  if not yes:
776
796
  print(
777
- f"cleanup is destructive — would keep the {keep} most recent "
778
- f"version(s) per table"
779
- + (f" and anything newer than {older_than}" if older_than else "")
780
- + f" in {ref}.\nRe-run with --yes to proceed."
797
+ f"cleanup is destructive — would keep {policy} in {ref}.\n"
798
+ "Re-run with --yes to proceed."
781
799
  )
782
800
  return
783
- print(f"Cleaning up {ref} (keep={keep}) …")
801
+ print(f"Cleaning up {ref} (keeping {policy}) …")
784
802
  _maintenance_client(ref).cleanup(keep=keep, older_than=older_than)
785
803
  print("Cleaned up.")
786
804
 
@@ -841,6 +859,7 @@ def reap_views(
841
859
  return
842
860
  try:
843
861
  idle = reaper_module.max_idle_days() if max_idle_days is None else max_idle_days
862
+ workers = reaper_module.survey_workers()
844
863
  except ValueError as exc:
845
864
  print(exc)
846
865
  raise SystemExit(1) from None
@@ -865,9 +884,18 @@ def reap_views(
865
884
 
866
885
  failed_graphs = 0
867
886
  for name, client in targets:
887
+ # Before the survey, not after: the report only prints once a graph is
888
+ # done, so a job killed mid-survey otherwise leaves no trace of which
889
+ # graph it was on.
890
+ print(f"{name}: surveying views", flush=True)
868
891
  try:
869
892
  report = reaper_module.reap(
870
- client, graph=name, max_idle=idle, apply=apply, cfg=cfg
893
+ client,
894
+ graph=name,
895
+ max_idle=idle,
896
+ apply=apply,
897
+ cfg=cfg,
898
+ workers=workers,
871
899
  )
872
900
  except PermissionError as exc:
873
901
  print(exc)
@@ -45,6 +45,7 @@ from __future__ import annotations
45
45
  import os
46
46
  import time
47
47
  from collections.abc import Iterable
48
+ from concurrent.futures import ThreadPoolExecutor
48
49
  from dataclasses import dataclass, field
49
50
 
50
51
  from . import config as cfg_module
@@ -52,13 +53,16 @@ from . import views
52
53
 
53
54
  __all__ = [
54
55
  "DEFAULT_MAX_IDLE_DAYS",
56
+ "DEFAULT_SURVEY_WORKERS",
55
57
  "MAX_IDLE_ENV_VAR",
56
58
  "PROTECTED_VIEWS",
59
+ "SURVEY_WORKERS_ENV_VAR",
57
60
  "ReapReport",
58
61
  "ViewAge",
59
62
  "max_idle_days",
60
63
  "reap",
61
64
  "select_stale",
65
+ "survey_workers",
62
66
  ]
63
67
 
64
68
  MAX_IDLE_ENV_VAR = "WITAN_CODE_VIEW_MAX_IDLE_DAYS"
@@ -67,6 +71,16 @@ MAX_IDLE_ENV_VAR = "WITAN_CODE_VIEW_MAX_IDLE_DAYS"
67
71
  DEFAULT_MAX_IDLE_DAYS = 14.0
68
72
  """Long enough to cover a two-week vacation, short enough to bound sprawl."""
69
73
 
74
+ SURVEY_WORKERS_ENV_VAR = "WITAN_CODE_VIEW_SURVEY_WORKERS"
75
+ """How many views to age at once. ``1`` ages them one after another."""
76
+
77
+ DEFAULT_SURVEY_WORKERS = 8
78
+ """Measured against production's bridge graph (309 views, omnigraph 0.11):
79
+ one at a time took 327s, 4 workers 85s, 8 workers 50s, and right after the
80
+ 8-worker run ``kubectl top`` read the 1-CPU omnigraph-server at 780m. Kept
81
+ there rather than higher so the sweep leaves that server room for its other
82
+ readers."""
83
+
70
84
  PROTECTED_VIEWS = frozenset({"main"})
71
85
  """Never reaped, whatever its age.
72
86
 
@@ -89,6 +103,20 @@ def max_idle_days() -> float:
89
103
  return max(value, 0.0)
90
104
 
91
105
 
106
+ def survey_workers() -> int:
107
+ """The configured survey concurrency; anything below ``1`` means ``1``."""
108
+ raw = os.environ.get(SURVEY_WORKERS_ENV_VAR)
109
+ if raw is None or not raw.strip():
110
+ return DEFAULT_SURVEY_WORKERS
111
+ try:
112
+ value = int(raw)
113
+ except ValueError as exc:
114
+ raise ValueError(
115
+ f"{SURVEY_WORKERS_ENV_VAR}={raw!r} is not a whole number of workers"
116
+ ) from exc
117
+ return max(value, 1)
118
+
119
+
92
120
  @dataclass(frozen=True)
93
121
  class ViewAge:
94
122
  """One branch view and when it was last written."""
@@ -152,21 +180,46 @@ class ReapReport:
152
180
 
153
181
 
154
182
  def survey(
155
- client, *, now: float, max_idle: float
183
+ client, *, now: float, max_idle: float, workers: int | None = None
156
184
  ) -> tuple[list[ViewAge], list[ViewAge]]:
157
185
  """Age every view on ``client``'s graph; return ``(all, stale)``.
158
186
 
159
187
  Read-only — the dry-run half of :func:`reap`, and what the CLI prints when
160
188
  it has not been told to delete anything.
161
189
  """
190
+ names = [name for name in client.list_branches() if name not in PROTECTED_VIEWS]
191
+ workers = survey_workers() if workers is None else workers
192
+ last_writes = _last_writes(client, names, workers=workers)
162
193
  ages = [
163
- ViewAge(view=name, last_write=client.branch_last_write(name))
164
- for name in client.list_branches()
165
- if name not in PROTECTED_VIEWS
194
+ ViewAge(view=name, last_write=last_write)
195
+ for name, last_write in zip(names, last_writes, strict=True)
166
196
  ]
167
197
  return ages, select_stale(ages, now=now, max_idle=max_idle)
168
198
 
169
199
 
200
+ def _last_writes(client, names: list[str], *, workers: int) -> list[float | None]:
201
+ """``client.branch_last_write`` for each of ``names``, in order.
202
+
203
+ Concurrent because the survey is one ``commit list --branch`` per view and
204
+ each returns the view's whole reachable history, ``main``'s included — so
205
+ the sweep costs views x commit-log length and both only grow. On omnigraph
206
+ 0.11 ``commit list`` without ``--branch`` returns ``main``'s commits alone
207
+ and ``changes`` is per-branch too, so the calls can be overlapped but not
208
+ merged. Serially, production's bridge graph alone outgrew the reaper
209
+ CronJob's 600s deadline.
210
+
211
+ The first failure propagates, as it did serially, and views not yet
212
+ started are cancelled rather than aged for a survey that is already lost.
213
+ """
214
+ if workers <= 1 or len(names) <= 1:
215
+ return [client.branch_last_write(name) for name in names]
216
+ pool = ThreadPoolExecutor(max_workers=min(workers, len(names)))
217
+ try:
218
+ return list(pool.map(client.branch_last_write, names))
219
+ finally:
220
+ pool.shutdown(cancel_futures=True)
221
+
222
+
170
223
  def reap(
171
224
  client,
172
225
  *,
@@ -175,6 +228,7 @@ def reap(
175
228
  max_idle: float | None = None,
176
229
  apply: bool = False,
177
230
  cfg: cfg_module.Config | None = None,
231
+ workers: int | None = None,
178
232
  ) -> ReapReport:
179
233
  """Sweep one graph's stale branch views.
180
234
 
@@ -208,7 +262,7 @@ def reap(
208
262
  if max_idle <= 0:
209
263
  return ReapReport(graph=graph, now=now)
210
264
 
211
- ages, stale = survey(client, now=now, max_idle=max_idle)
265
+ ages, stale = survey(client, now=now, max_idle=max_idle, workers=workers)
212
266
  report = ReapReport(graph=graph, now=now, scanned=len(ages), stale=stale)
213
267
  if not apply:
214
268
  return report
@@ -554,7 +554,7 @@ def discard_store(ref: StoreRef) -> int:
554
554
  # these two exception types exist to prevent.
555
555
  _NOT_SERVED = "is not served by the omnigraph-server"
556
556
  _NOT_FOUND_RE = re.compile(
557
- rf"graph '[^']*' not found|{re.escape(_NOT_SERVED)}", re.IGNORECASE
557
+ rf"graph (?:'[^']*' )?not found|{re.escape(_NOT_SERVED)}", re.IGNORECASE
558
558
  )
559
559
 
560
560
 
File without changes
File without changes
File without changes