witan-core 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- witan_core/__init__.py +29 -0
- witan_core/_detach.py +25 -0
- witan_core/cli.py +113 -0
- witan_core/elicit.py +59 -0
- witan_core/maintenance.py +90 -0
- witan_core/omnigraph.py +353 -0
- witan_core/omnigraph_install.py +145 -0
- witan_core/remote/__init__.py +18 -0
- witan_core/remote/oidc.py +317 -0
- witan_core/remote/proxy.py +134 -0
- witan_core/repo_key.py +52 -0
- witan_core/timeutil.py +10 -0
- witan_core-0.2.0.dist-info/METADATA +77 -0
- witan_core-0.2.0.dist-info/RECORD +15 -0
- witan_core-0.2.0.dist-info/WHEEL +4 -0
witan_core/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""witan-core — shared core for the witan MCP servers.
|
|
2
|
+
|
|
3
|
+
A leaf package that ``witan`` (witan-council) and ``witan_code`` both depend on,
|
|
4
|
+
extracted from the surface they were previously carrying as copy-paste-and-diverge
|
|
5
|
+
duplicates. See ``docs/design/witan-core-extraction-spec.md``.
|
|
6
|
+
|
|
7
|
+
``witan_core`` imports neither ``witan`` nor ``witan_code``: it sits below both,
|
|
8
|
+
preserving the one-directional ``witan`` → ``witan_code`` optional-mount DAG.
|
|
9
|
+
|
|
10
|
+
The root package is dependency-free (stdlib only). Heavier concerns live in
|
|
11
|
+
submodules gated behind extras and are NOT re-exported here: ``witan_core.elicit``
|
|
12
|
+
needs ``fastmcp`` (the ``mcp`` extra); the omnigraph installer imports ``rich``
|
|
13
|
+
lazily.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from ._detach import popen_detached
|
|
19
|
+
from .omnigraph_install import install_omnigraph
|
|
20
|
+
from .repo_key import find_git_config, normalise
|
|
21
|
+
from .timeutil import now_iso
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"find_git_config",
|
|
25
|
+
"install_omnigraph",
|
|
26
|
+
"normalise",
|
|
27
|
+
"now_iso",
|
|
28
|
+
"popen_detached",
|
|
29
|
+
]
|
witan_core/_detach.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Cross-platform detached-subprocess spawning.
|
|
2
|
+
|
|
3
|
+
``subprocess.Popen``'s ``start_new_session=True`` (setsid) is POSIX-only — a
|
|
4
|
+
no-op on Windows, where a background hook child can otherwise get torn down
|
|
5
|
+
with its parent's process group/console. Used by every hook that must spawn
|
|
6
|
+
work and return immediately: the witan-code SessionStart indexer and both
|
|
7
|
+
servers' throttled optimize checkpoint (``maintenance.py``), spawned from the
|
|
8
|
+
``Stop`` hook.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def popen_detached(args: list[str], **kwargs: object) -> subprocess.Popen:
|
|
18
|
+
"""Spawn ``args`` fully detached from the current process, cross-platform."""
|
|
19
|
+
if sys.platform == "win32":
|
|
20
|
+
kwargs["creationflags"] = (
|
|
21
|
+
subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS
|
|
22
|
+
)
|
|
23
|
+
else:
|
|
24
|
+
kwargs["start_new_session"] = True
|
|
25
|
+
return subprocess.Popen(args, **kwargs) # noqa: S603 — caller controls argv
|
witan_core/cli.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Shared CLI scaffolding for the witan servers' ``setup`` commands.
|
|
2
|
+
|
|
3
|
+
Uses the ``cli`` extra: ``make_app`` needs cyclopts, and ``report_install``
|
|
4
|
+
uses agent-config-kit's ``InstallResult`` type. This module never imports rich
|
|
5
|
+
itself — the extra bundles it only for ``report_install``'s styled branch,
|
|
6
|
+
which runs on a rich ``Console`` the *caller* passes in (without one it plain
|
|
7
|
+
``print``s), so importing/using ``witan_core.cli`` never requires rich.
|
|
8
|
+
Following the package convention (see ``__init__``), nothing here is re-exported
|
|
9
|
+
from the root package — import from ``witan_core.cli`` directly.
|
|
10
|
+
|
|
11
|
+
The pieces both ``witan`` (witan-council) and ``witan_code`` carried as
|
|
12
|
+
copy-paste duplicates: the supported-agent constants, the ``--version`` app
|
|
13
|
+
factory, ``--author`` resolution, and the install-result printer. Everything
|
|
14
|
+
else in each server's ``setup`` command is package-local (different bundles,
|
|
15
|
+
different post-install healing) and stays put.
|
|
16
|
+
|
|
17
|
+
``report_install`` deliberately renders plain text without a ``console`` so
|
|
18
|
+
witan-code's setup can call it without importing rich just for that line.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import os
|
|
24
|
+
import subprocess
|
|
25
|
+
from typing import TYPE_CHECKING, Literal
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
import cyclopts
|
|
29
|
+
from agent_config_kit import InstallResult
|
|
30
|
+
from rich.console import Console
|
|
31
|
+
|
|
32
|
+
# The coding agents witan supports installing into. Both servers' `setup`
|
|
33
|
+
# commands mirror this literal for their `--agent` argument; "all" fans out
|
|
34
|
+
# across every detected platform.
|
|
35
|
+
AgentName = Literal["claude", "pi", "copilot", "opencode", "all"]
|
|
36
|
+
|
|
37
|
+
# Human-readable labels keyed by the AgentName literals (minus "all"), used in
|
|
38
|
+
# install reports. `.get(name, name)` falls back to the raw key.
|
|
39
|
+
AGENT_NAMES: dict[str, str] = {
|
|
40
|
+
"claude": "Claude Code",
|
|
41
|
+
"pi": "Pi",
|
|
42
|
+
"copilot": "GitHub Copilot",
|
|
43
|
+
"opencode": "OpenCode",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def make_app(*, name: str, help_text: str, version_dist: str) -> cyclopts.App:
|
|
48
|
+
"""Build the top-level cyclopts ``App`` for a witan CLI.
|
|
49
|
+
|
|
50
|
+
``version_dist`` is the installed distribution name whose version
|
|
51
|
+
``--version`` reports (e.g. ``"witan-council"`` or ``"witan-code"``).
|
|
52
|
+
"""
|
|
53
|
+
import cyclopts
|
|
54
|
+
from agent_config_kit.version import resolve_version
|
|
55
|
+
|
|
56
|
+
return cyclopts.App(
|
|
57
|
+
name=name,
|
|
58
|
+
help=help_text,
|
|
59
|
+
version=lambda: resolve_version(version_dist),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def resolve_author(author: str | None) -> str:
|
|
64
|
+
"""Resolve the graph-attribution author for a ``setup`` run.
|
|
65
|
+
|
|
66
|
+
An explicit value wins; otherwise fall back to ``git config user.name``,
|
|
67
|
+
then ``$USER``, then ``"unknown"``.
|
|
68
|
+
"""
|
|
69
|
+
if author is not None:
|
|
70
|
+
return author
|
|
71
|
+
try:
|
|
72
|
+
resolved = subprocess.check_output(
|
|
73
|
+
["git", "config", "user.name"],
|
|
74
|
+
text=True,
|
|
75
|
+
stderr=subprocess.DEVNULL,
|
|
76
|
+
).strip()
|
|
77
|
+
except (subprocess.CalledProcessError, OSError):
|
|
78
|
+
# OSError covers a missing git binary (FileNotFoundError) plus any other
|
|
79
|
+
# OS-level failure to spawn it (e.g. PermissionError); degrade to the
|
|
80
|
+
# $USER/"unknown" fallback rather than letting `setup` crash.
|
|
81
|
+
resolved = ""
|
|
82
|
+
return resolved or os.environ.get("USER", "unknown")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def report_install(
|
|
86
|
+
name: str,
|
|
87
|
+
result: InstallResult,
|
|
88
|
+
*,
|
|
89
|
+
dry_run: bool,
|
|
90
|
+
console: Console | None = None,
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Print an agent-config ``InstallResult``.
|
|
93
|
+
|
|
94
|
+
With a rich ``console`` the output is styled (witan-council's look); without
|
|
95
|
+
one it is plain ``print`` (witan-code's look). Keeping both branches here
|
|
96
|
+
lets the two servers share the printer without witan-code taking on rich for
|
|
97
|
+
it.
|
|
98
|
+
"""
|
|
99
|
+
label = AGENT_NAMES.get(name, name)
|
|
100
|
+
if console is not None:
|
|
101
|
+
console.print(f"\n[bold]{label}[/bold]")
|
|
102
|
+
for path in result.planned:
|
|
103
|
+
tag = " [dim](dry-run)[/dim]" if dry_run else ""
|
|
104
|
+
console.print(f" [green]→[/green] {path}{tag}")
|
|
105
|
+
for path, reason in result.skipped:
|
|
106
|
+
console.print(f" [yellow]skip[/yellow] {path} — {reason}")
|
|
107
|
+
else:
|
|
108
|
+
print(f"\n{label}")
|
|
109
|
+
for path in result.planned:
|
|
110
|
+
tag = " (dry-run)" if dry_run else ""
|
|
111
|
+
print(f" -> {path}{tag}")
|
|
112
|
+
for path, reason in result.skipped:
|
|
113
|
+
print(f" skip {path} — {reason}")
|
witan_core/elicit.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Additive MCP elicitation primitives, shared by both witan servers.
|
|
2
|
+
|
|
3
|
+
FastMCP supports ``ctx.elicit``, but the connected client may not, and many
|
|
4
|
+
tools also run under headless automation (context/checkpoint hooks, background
|
|
5
|
+
indexers). These helpers keep elicitation strictly *additive*: when the client
|
|
6
|
+
can't elicit — or ``ctx`` is absent — they return the caller's ``default`` so
|
|
7
|
+
behavior is exactly what it was before elicitation existed. Only an *explicit*
|
|
8
|
+
user decline changes the outcome.
|
|
9
|
+
|
|
10
|
+
This module is NOT imported by ``witan_core/__init__`` — it depends on
|
|
11
|
+
``fastmcp`` (the ``mcp`` extra), so importing the base package stays
|
|
12
|
+
dependency-free. Each server composes its own repo-elicitation helpers
|
|
13
|
+
(``repo_or_detect`` / ``choose_repo``) on top of these primitives.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import TYPE_CHECKING
|
|
19
|
+
|
|
20
|
+
from fastmcp.server.elicitation import AcceptedElicitation
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from fastmcp import Context
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
async def confirm(
|
|
27
|
+
ctx: Context | None, message: str, *, default_when_unsupported: bool
|
|
28
|
+
) -> bool:
|
|
29
|
+
"""Ask a yes/no question.
|
|
30
|
+
|
|
31
|
+
``accept`` → the chosen bool; ``decline``/``cancel`` → ``False``; and when
|
|
32
|
+
elicitation is unsupported or errors (headless client, no ``ctx``) →
|
|
33
|
+
``default_when_unsupported`` — pick that so the non-interactive path keeps
|
|
34
|
+
today's behavior (e.g. ``False`` for "don't act", ``True`` for "proceed").
|
|
35
|
+
"""
|
|
36
|
+
if ctx is None:
|
|
37
|
+
return default_when_unsupported
|
|
38
|
+
try:
|
|
39
|
+
result = await ctx.elicit(message, response_type=bool)
|
|
40
|
+
except Exception: # noqa: BLE001 — any elicit failure means "can't ask"
|
|
41
|
+
return default_when_unsupported
|
|
42
|
+
if isinstance(result, AcceptedElicitation):
|
|
43
|
+
return bool(result.data)
|
|
44
|
+
return False
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def text(ctx: Context | None, message: str, *, default: str) -> str:
|
|
48
|
+
"""Ask for a line of text. A non-empty accepted value is returned; a
|
|
49
|
+
decline/cancel, an empty value, an unsupported client, or no ``ctx`` all
|
|
50
|
+
fall back to ``default``."""
|
|
51
|
+
if ctx is None:
|
|
52
|
+
return default
|
|
53
|
+
try:
|
|
54
|
+
result = await ctx.elicit(message, response_type=str)
|
|
55
|
+
except Exception: # noqa: BLE001
|
|
56
|
+
return default
|
|
57
|
+
if isinstance(result, AcceptedElicitation) and (result.data or "").strip():
|
|
58
|
+
return result.data.strip()
|
|
59
|
+
return default
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Throttle + atomic-stamp mechanics for opportunistic store compaction.
|
|
2
|
+
|
|
3
|
+
Both servers run ``omnigraph optimize`` opportunistically from their ``Stop``
|
|
4
|
+
hook, at most once per interval per store, never blocking the agent. The tricky,
|
|
5
|
+
error-prone parts are shared here; each server wraps them with its own store-key
|
|
6
|
+
type, env var, stamp-file location, and detached command (see
|
|
7
|
+
``witan.maintenance`` / ``witan_code.maintenance``):
|
|
8
|
+
|
|
9
|
+
- ``resolve_interval`` — parse the throttle window from an env var.
|
|
10
|
+
- ``mark_run`` — write the last-run stamp *atomically* (temp file + ``os.replace``)
|
|
11
|
+
so a concurrent/interrupted write can't leave a half-written stamp that reads
|
|
12
|
+
as "never run" and defeats the throttle.
|
|
13
|
+
- ``last_run`` — read it back, treating missing/corrupt as "never run".
|
|
14
|
+
- ``is_due`` — the disabled / remote-store / missing-store / window-elapsed logic.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
# Remote stores (omnigraph-server) are compacted server-side, not by a client
|
|
24
|
+
# hook, so opportunistic optimize never fires for them.
|
|
25
|
+
REMOTE_PREFIXES = ("http://", "https://", "s3://")
|
|
26
|
+
|
|
27
|
+
# Optimize takes the store's write lock and is ~tens of seconds on a bloated
|
|
28
|
+
# store, so daily is a safe default throttle window.
|
|
29
|
+
DEFAULT_OPTIMIZE_INTERVAL = 24 * 3600.0
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def resolve_interval(env_var: str, default: float = DEFAULT_OPTIMIZE_INTERVAL) -> float:
|
|
33
|
+
"""Throttle window in seconds from ``env_var``; ``0``/negative disables."""
|
|
34
|
+
raw = os.environ.get(env_var)
|
|
35
|
+
if raw is None:
|
|
36
|
+
return default
|
|
37
|
+
try:
|
|
38
|
+
return max(0.0, float(raw))
|
|
39
|
+
except ValueError:
|
|
40
|
+
return default
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def last_run(stamp_file: Path) -> float:
|
|
44
|
+
"""Last-optimize timestamp from ``stamp_file``; ``0.0`` if missing/corrupt."""
|
|
45
|
+
try:
|
|
46
|
+
return float(json.loads(stamp_file.read_text()).get("stamp", 0.0))
|
|
47
|
+
except Exception: # noqa: BLE001 — missing/corrupt stamp → treat as never run
|
|
48
|
+
return 0.0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def mark_run(stamp_file: Path, when: float) -> None:
|
|
52
|
+
"""Record the last-optimize time atomically.
|
|
53
|
+
|
|
54
|
+
Concurrent Stop hooks (or an interrupted write) could otherwise leave a
|
|
55
|
+
half-written stamp that ``last_run`` reads as "never run", defeating the
|
|
56
|
+
throttle and letting optimize spawn repeatedly. Write a process-unique temp
|
|
57
|
+
file and ``os.replace`` it in, so a reader always sees a complete file.
|
|
58
|
+
"""
|
|
59
|
+
tmp = stamp_file.with_name(f"{stamp_file.name}.{os.getpid()}.tmp")
|
|
60
|
+
try:
|
|
61
|
+
tmp.write_text(json.dumps({"stamp": when}))
|
|
62
|
+
os.replace(tmp, stamp_file)
|
|
63
|
+
except OSError:
|
|
64
|
+
try:
|
|
65
|
+
tmp.unlink(missing_ok=True)
|
|
66
|
+
except OSError:
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def is_due(
|
|
71
|
+
*,
|
|
72
|
+
store: str | Path,
|
|
73
|
+
stamp_file: Path,
|
|
74
|
+
interval: float,
|
|
75
|
+
now: float,
|
|
76
|
+
require_exists: bool,
|
|
77
|
+
) -> bool:
|
|
78
|
+
"""Whether an opportunistic optimize is due for ``store``.
|
|
79
|
+
|
|
80
|
+
False when auto-optimize is disabled (``interval<=0``), the store is remote
|
|
81
|
+
(maintained server-side), ``require_exists`` and the store doesn't exist
|
|
82
|
+
yet, or the throttle window hasn't elapsed since ``last_run``.
|
|
83
|
+
"""
|
|
84
|
+
if interval <= 0:
|
|
85
|
+
return False
|
|
86
|
+
if str(store).startswith(REMOTE_PREFIXES):
|
|
87
|
+
return False
|
|
88
|
+
if require_exists and not Path(store).exists():
|
|
89
|
+
return False
|
|
90
|
+
return now - last_run(stamp_file) >= interval
|
witan_core/omnigraph.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
"""Shared base for the omnigraph CLI subprocess wrapper.
|
|
2
|
+
|
|
3
|
+
Both servers drive the ``omnigraph`` binary the same way: a per-store advisory
|
|
4
|
+
write lock for local stores, a retry/repair loop for optimistic-concurrency
|
|
5
|
+
drift, a self-backoff for the deployed omnigraph-server's per-actor admission
|
|
6
|
+
cap, and named read/mutate queries. That LOCAL-vs-REMOTE-generic surface lives
|
|
7
|
+
here; each server subclasses to add its own tail:
|
|
8
|
+
|
|
9
|
+
- ``witan`` adds a write ``guard`` + ``surface_conflict`` (CAS task claims),
|
|
10
|
+
``apply_schema``, and the storage-version friendly-error hint.
|
|
11
|
+
- ``witan_code`` adds omnigraph *branch* support (``_extra_args`` override,
|
|
12
|
+
``ensure_branch``/…) and bulk ``load``.
|
|
13
|
+
|
|
14
|
+
Subclass knobs:
|
|
15
|
+
- ``_SETUP_HINT`` — the install command named in the "binary not found" error.
|
|
16
|
+
- ``_STORAGE_MISMATCH_HINT`` — remediation text for a storage-version mismatch;
|
|
17
|
+
``None`` (the default) leaves that error on the generic failure path.
|
|
18
|
+
- ``_extra_args(subcommand)`` — extra CLI args injected into every ``_run`` (e.g.
|
|
19
|
+
``--branch``); returns ``[]`` by default.
|
|
20
|
+
|
|
21
|
+
This module is stdlib-only but imports ``fcntl`` (POSIX advisory locks), so it is
|
|
22
|
+
NOT re-exported from ``witan_core``'s package root — import it as
|
|
23
|
+
``witan_core.omnigraph``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import fcntl
|
|
29
|
+
import json
|
|
30
|
+
import os
|
|
31
|
+
import random
|
|
32
|
+
import re
|
|
33
|
+
import shutil
|
|
34
|
+
import subprocess
|
|
35
|
+
import time
|
|
36
|
+
from collections.abc import Callable
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
|
|
39
|
+
# omnigraph local stores use optimistic concurrency (Lance manifest versions) and
|
|
40
|
+
# are NOT safe for concurrent writers. We serialize writes with a per-store
|
|
41
|
+
# advisory lock (prevention) and, as a safety net, retry transient "stale view"
|
|
42
|
+
# conflicts and `omnigraph repair` stores already in the drifted state.
|
|
43
|
+
_WRITE_SUBCOMMANDS = {"mutate", "load", "optimize", "cleanup"}
|
|
44
|
+
_RETRYABLE = ("stale view", "manifest table version", "refresh and retry")
|
|
45
|
+
_NEEDS_REPAIR = ("ahead of manifest", "omnigraph repair")
|
|
46
|
+
_MAX_ATTEMPTS = 8
|
|
47
|
+
|
|
48
|
+
# omnigraph uses strict single-version storage: a release that bumps the
|
|
49
|
+
# internal schema version refuses to open graphs an older binary wrote,
|
|
50
|
+
# raising exactly this pair of substrings wrapped in an unhelpful Rust panic +
|
|
51
|
+
# backtrace. Not retryable/repairable.
|
|
52
|
+
_STORAGE_VERSION_MISMATCH_MARKERS = ("stamped at internal schema", "reads only")
|
|
53
|
+
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
|
|
54
|
+
|
|
55
|
+
# omnigraph-server (remote http(s)/s3 stores) enforces a hard per-actor
|
|
56
|
+
# admission cap — both an in-flight *count* (default 16) and a concurrent
|
|
57
|
+
# *byte* budget — and rejects excess concurrent writes outright (HTTP 429)
|
|
58
|
+
# rather than queuing them. The CLI's error path only surfaces the JSON body's
|
|
59
|
+
# message text, not the `Retry-After: 60` header, so this can't read it and
|
|
60
|
+
# backs off on its own, much shorter, schedule. Independent of _MAX_ATTEMPTS
|
|
61
|
+
# and of surface_conflict (it isn't a compare-and-swap race, just admission
|
|
62
|
+
# control). Lives in the shared base because witan-code also writes to the
|
|
63
|
+
# deployed omnigraph-server.
|
|
64
|
+
_ADMISSION_CAP_MARKERS = ("in-flight count cap", "byte budget exceeded")
|
|
65
|
+
_ADMISSION_CAP_MAX_ATTEMPTS = 6
|
|
66
|
+
_ADMISSION_CAP_BASE_DELAY = 0.25
|
|
67
|
+
_ADMISSION_CAP_MAX_DELAY = 4.0
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _admission_cap_backoff(attempt: int) -> float:
|
|
71
|
+
"""Exponential backoff with jitter — plain exponential backoff makes
|
|
72
|
+
concurrent retries from the same burst (the actual trigger case here)
|
|
73
|
+
retry in lockstep and re-collide on the cap; jitter breaks that up."""
|
|
74
|
+
delay = _ADMISSION_CAP_BASE_DELAY * (2 ** (attempt - 1))
|
|
75
|
+
jitter = random.uniform(0, 0.1 * delay)
|
|
76
|
+
return min(delay + jitter, _ADMISSION_CAP_MAX_DELAY)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class OmnigraphConflict(RuntimeError):
|
|
80
|
+
"""An optimistic-concurrency (Lance manifest version) write conflict that the
|
|
81
|
+
caller asked to surface instead of retry.
|
|
82
|
+
|
|
83
|
+
omnigraph serializes local writes with our advisory flock, but on shared
|
|
84
|
+
http(s)/s3 stores that lock is skipped and two writers racing the same
|
|
85
|
+
manifest version make one commit fail with a "stale view" / "manifest table
|
|
86
|
+
version" error. ``_execute`` normally masks that by re-running the same
|
|
87
|
+
mutation — fine for an idempotent upsert, WRONG for a compare-and-swap claim
|
|
88
|
+
where the retry would blindly re-apply the claim over whoever won the race.
|
|
89
|
+
A caller passing ``surface_conflict=True`` gets this exception instead so it
|
|
90
|
+
can re-read and decide (see ``task_claim``)."""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _is_storage_version_mismatch(msg: str) -> bool:
|
|
94
|
+
lowered = msg.lower()
|
|
95
|
+
return all(marker in lowered for marker in _STORAGE_VERSION_MISMATCH_MARKERS)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _friendly_storage_error(raw: str, hint: str) -> str:
|
|
99
|
+
"""Strip the Rust panic's ANSI codes and backtrace boilerplate ("Location:",
|
|
100
|
+
"Backtrace omitted...") from a storage-version-mismatch error, keeping just
|
|
101
|
+
omnigraph's own message, and append the caller's remediation ``hint``."""
|
|
102
|
+
cleaned = _ANSI_RE.sub("", raw).split("Location:")[0].strip()
|
|
103
|
+
return f"{cleaned}\n\n{hint}"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class OmnigraphClient:
|
|
107
|
+
"""Subprocess wrapper for the omnigraph CLI (shared base)."""
|
|
108
|
+
|
|
109
|
+
#: Install command named in the "binary not found" error. Override per server.
|
|
110
|
+
_SETUP_HINT = "the omnigraph installer (`witan setup` / `witan-code setup`)"
|
|
111
|
+
#: Remediation text appended to a storage-version-mismatch error. ``None``
|
|
112
|
+
#: leaves that error on the generic failure path (no friendly rewrite).
|
|
113
|
+
_STORAGE_MISMATCH_HINT: str | None = None
|
|
114
|
+
|
|
115
|
+
def __init__(
|
|
116
|
+
self,
|
|
117
|
+
graph_uri: str,
|
|
118
|
+
queries_dir: Path,
|
|
119
|
+
token: str | None = None,
|
|
120
|
+
guard: Callable[[str, dict], dict] | None = None,
|
|
121
|
+
) -> None:
|
|
122
|
+
self.graph_uri = graph_uri
|
|
123
|
+
self.queries_dir = queries_dir
|
|
124
|
+
self.token = token
|
|
125
|
+
self.guard = guard
|
|
126
|
+
self._binary = self._find_binary()
|
|
127
|
+
|
|
128
|
+
# ── Public API ────────────────────────────────────────────────
|
|
129
|
+
|
|
130
|
+
def read(
|
|
131
|
+
self,
|
|
132
|
+
query_file: str,
|
|
133
|
+
query_name: str,
|
|
134
|
+
params: dict,
|
|
135
|
+
) -> list[dict]:
|
|
136
|
+
"""Run a named read query. Returns a list of result rows."""
|
|
137
|
+
result = self._run(
|
|
138
|
+
"query",
|
|
139
|
+
"--query",
|
|
140
|
+
str(self.queries_dir / query_file),
|
|
141
|
+
query_name,
|
|
142
|
+
"--params",
|
|
143
|
+
json.dumps(params),
|
|
144
|
+
"--format",
|
|
145
|
+
"json",
|
|
146
|
+
)
|
|
147
|
+
if not result.strip():
|
|
148
|
+
return []
|
|
149
|
+
try:
|
|
150
|
+
parsed = json.loads(result)
|
|
151
|
+
except json.JSONDecodeError as exc:
|
|
152
|
+
raise RuntimeError(f"omnigraph returned non-JSON: {result!r}") from exc
|
|
153
|
+
# v0.7.0 wraps results in {rows: [...], columns: [...], ...}
|
|
154
|
+
rows = parsed.get("rows", parsed) if isinstance(parsed, dict) else parsed
|
|
155
|
+
# strip alias prefixes: "p.slug" → "slug"
|
|
156
|
+
return [{k.split(".", 1)[-1]: v for k, v in row.items()} for row in rows]
|
|
157
|
+
|
|
158
|
+
def change(
|
|
159
|
+
self,
|
|
160
|
+
query_file: str,
|
|
161
|
+
query_name: str,
|
|
162
|
+
params: dict,
|
|
163
|
+
*,
|
|
164
|
+
surface_conflict: bool = False,
|
|
165
|
+
) -> None:
|
|
166
|
+
"""Run a named mutation query.
|
|
167
|
+
|
|
168
|
+
The optional ``guard`` runs first: it may raise to reject the write, or
|
|
169
|
+
return rewritten params (e.g. with secrets redacted) that are what
|
|
170
|
+
actually get persisted.
|
|
171
|
+
|
|
172
|
+
``surface_conflict``: raise :class:`OmnigraphConflict` on an
|
|
173
|
+
optimistic-concurrency conflict instead of transparently retrying the
|
|
174
|
+
write. Callers implementing a compare-and-swap (e.g. ``task_claim``) need
|
|
175
|
+
this so a lost race is detectable rather than silently clobbered.
|
|
176
|
+
"""
|
|
177
|
+
if self.guard is not None:
|
|
178
|
+
params = self.guard(query_name, params)
|
|
179
|
+
self._run(
|
|
180
|
+
"mutate",
|
|
181
|
+
"--query",
|
|
182
|
+
str(self.queries_dir / query_file),
|
|
183
|
+
query_name,
|
|
184
|
+
"--params",
|
|
185
|
+
json.dumps(params),
|
|
186
|
+
surface_conflict=surface_conflict,
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
def optimize(self) -> str:
|
|
190
|
+
"""Compact small Lance fragments across every table (non-destructive).
|
|
191
|
+
|
|
192
|
+
Every write appends a new tiny Lance fragment + manifest version; an
|
|
193
|
+
un-compacted store bloats until *opening* it dominates query latency.
|
|
194
|
+
``omnigraph optimize`` collapses the fragments. It takes the store's
|
|
195
|
+
write lock and can run for tens of seconds, so callers throttle it and
|
|
196
|
+
keep it off the hot path (see each server's ``maintenance``).
|
|
197
|
+
"""
|
|
198
|
+
return self._run("optimize")
|
|
199
|
+
|
|
200
|
+
def cleanup(self, *, keep: int | None = None, older_than: str | None = None) -> str:
|
|
201
|
+
"""Reclaim disk by removing old Lance versions (**destructive**).
|
|
202
|
+
|
|
203
|
+
``optimize`` compacts fragments but leaves old versions behind, so disk
|
|
204
|
+
stays large until they are GC'd. ``cleanup`` removes them, keeping the
|
|
205
|
+
most recent ``keep`` versions per table and/or those newer than
|
|
206
|
+
``older_than`` (a Go-style duration like ``7d``). At least one bound must
|
|
207
|
+
be given (omnigraph requires it). ``--confirm`` is passed so it runs.
|
|
208
|
+
"""
|
|
209
|
+
if keep is None and older_than is None:
|
|
210
|
+
raise ValueError("cleanup requires keep and/or older_than")
|
|
211
|
+
args = ["--confirm"]
|
|
212
|
+
if keep is not None:
|
|
213
|
+
args += ["--keep", str(keep)]
|
|
214
|
+
if older_than is not None:
|
|
215
|
+
args += ["--older-than", older_than]
|
|
216
|
+
return self._run("cleanup", *args)
|
|
217
|
+
|
|
218
|
+
# ── Internals ─────────────────────────────────────────────────
|
|
219
|
+
|
|
220
|
+
def _extra_args(self, subcommand: str) -> list[str]:
|
|
221
|
+
"""Extra CLI args injected into every ``_run`` command (after the store
|
|
222
|
+
flag, before the caller's args). Empty by default; subclasses override
|
|
223
|
+
(e.g. witan-code injects ``--branch``)."""
|
|
224
|
+
return []
|
|
225
|
+
|
|
226
|
+
def _run(self, subcommand: str, *args: str, surface_conflict: bool = False) -> str:
|
|
227
|
+
quiet = ["--quiet"] if subcommand in _WRITE_SUBCOMMANDS else []
|
|
228
|
+
cmd = [
|
|
229
|
+
self._binary,
|
|
230
|
+
subcommand,
|
|
231
|
+
"--store",
|
|
232
|
+
self.graph_uri,
|
|
233
|
+
*quiet,
|
|
234
|
+
*self._extra_args(subcommand),
|
|
235
|
+
*args,
|
|
236
|
+
]
|
|
237
|
+
return self._execute(
|
|
238
|
+
cmd,
|
|
239
|
+
subcommand,
|
|
240
|
+
is_write=subcommand in _WRITE_SUBCOMMANDS,
|
|
241
|
+
surface_conflict=surface_conflict,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
def _execute(
|
|
245
|
+
self,
|
|
246
|
+
cmd: list[str],
|
|
247
|
+
label: str,
|
|
248
|
+
*,
|
|
249
|
+
is_write: bool,
|
|
250
|
+
surface_conflict: bool = False,
|
|
251
|
+
) -> str:
|
|
252
|
+
"""Run an omnigraph CLI command under the write lock (for writes) with the
|
|
253
|
+
retry/repair loop for optimistic-concurrency conflicts."""
|
|
254
|
+
env = dict(os.environ)
|
|
255
|
+
if self.token:
|
|
256
|
+
env["OMNIGRAPH_SERVER_BEARER_TOKEN"] = self.token
|
|
257
|
+
|
|
258
|
+
lock_fh = self._acquire_write_lock(is_write)
|
|
259
|
+
try:
|
|
260
|
+
attempt = 0
|
|
261
|
+
admission_cap_attempt = 0
|
|
262
|
+
while True:
|
|
263
|
+
try:
|
|
264
|
+
result = subprocess.run(
|
|
265
|
+
cmd, capture_output=True, text=True, env=env
|
|
266
|
+
)
|
|
267
|
+
except OSError as exc:
|
|
268
|
+
raise RuntimeError(
|
|
269
|
+
f"omnigraph {label} could not run: {exc}"
|
|
270
|
+
) from exc
|
|
271
|
+
if result.returncode == 0:
|
|
272
|
+
return result.stdout
|
|
273
|
+
err = result.stderr
|
|
274
|
+
err_lower = err.lower()
|
|
275
|
+
if self._STORAGE_MISMATCH_HINT and _is_storage_version_mismatch(err):
|
|
276
|
+
raise RuntimeError(
|
|
277
|
+
_friendly_storage_error(err, self._STORAGE_MISMATCH_HINT)
|
|
278
|
+
) from None
|
|
279
|
+
if any(m in err_lower for m in _ADMISSION_CAP_MARKERS):
|
|
280
|
+
# Independent budget/backoff from the drift retries below —
|
|
281
|
+
# doesn't consume _MAX_ATTEMPTS and ignores surface_conflict.
|
|
282
|
+
admission_cap_attempt += 1
|
|
283
|
+
if admission_cap_attempt < _ADMISSION_CAP_MAX_ATTEMPTS:
|
|
284
|
+
time.sleep(_admission_cap_backoff(admission_cap_attempt))
|
|
285
|
+
continue
|
|
286
|
+
raise RuntimeError(
|
|
287
|
+
f"omnigraph {label} failed after "
|
|
288
|
+
f"{_ADMISSION_CAP_MAX_ATTEMPTS} attempts (actor "
|
|
289
|
+
f"admission cap exceeded):\n{err.strip()}"
|
|
290
|
+
)
|
|
291
|
+
if surface_conflict and any(m in err_lower for m in _RETRYABLE):
|
|
292
|
+
# A compare-and-swap caller wants to lose the race, not
|
|
293
|
+
# re-apply its write over the winner. Surface immediately.
|
|
294
|
+
raise OmnigraphConflict(err.strip()) from None
|
|
295
|
+
attempt += 1
|
|
296
|
+
if attempt < _MAX_ATTEMPTS:
|
|
297
|
+
if any(m in err_lower for m in _NEEDS_REPAIR):
|
|
298
|
+
self._repair(env)
|
|
299
|
+
continue
|
|
300
|
+
if any(m in err_lower for m in _RETRYABLE):
|
|
301
|
+
time.sleep(0.05 * attempt)
|
|
302
|
+
continue
|
|
303
|
+
raise RuntimeError(
|
|
304
|
+
f"omnigraph {label} failed (exit {result.returncode}):\n"
|
|
305
|
+
f"{err.strip()}"
|
|
306
|
+
)
|
|
307
|
+
finally:
|
|
308
|
+
if lock_fh is not None:
|
|
309
|
+
fcntl.flock(lock_fh.fileno(), fcntl.LOCK_UN)
|
|
310
|
+
lock_fh.close()
|
|
311
|
+
|
|
312
|
+
def _acquire_write_lock(self, is_write: bool):
|
|
313
|
+
"""Hold a per-store exclusive lock for writes (local stores)."""
|
|
314
|
+
if not is_write or self.graph_uri.startswith(("http://", "https://", "s3://")):
|
|
315
|
+
return None
|
|
316
|
+
lock_path = Path(f"{self.graph_uri}.lock")
|
|
317
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
318
|
+
fh = open(lock_path, "w") # noqa: SIM115 — released in _execute's finally
|
|
319
|
+
fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
|
|
320
|
+
return fh
|
|
321
|
+
|
|
322
|
+
def _repair(self, env: dict) -> None:
|
|
323
|
+
"""Reconcile manifest/HEAD drift so the retried write can proceed."""
|
|
324
|
+
subprocess.run(
|
|
325
|
+
[
|
|
326
|
+
self._binary,
|
|
327
|
+
"repair",
|
|
328
|
+
"--store",
|
|
329
|
+
self.graph_uri,
|
|
330
|
+
"--confirm",
|
|
331
|
+
"--force",
|
|
332
|
+
"--quiet",
|
|
333
|
+
],
|
|
334
|
+
capture_output=True,
|
|
335
|
+
text=True,
|
|
336
|
+
env=env,
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
@classmethod
|
|
340
|
+
def _find_binary(cls) -> str:
|
|
341
|
+
binary = shutil.which("omnigraph")
|
|
342
|
+
if binary is not None:
|
|
343
|
+
return binary
|
|
344
|
+
# MCP servers are often launched by a desktop app or IDE extension
|
|
345
|
+
# whose process doesn't inherit a shell PATH — the `setup` command
|
|
346
|
+
# always installs to this fixed location, so check it directly rather
|
|
347
|
+
# than relying on PATH alone.
|
|
348
|
+
fallback = Path.home() / ".local" / "bin" / "omnigraph"
|
|
349
|
+
if fallback.exists():
|
|
350
|
+
return str(fallback)
|
|
351
|
+
raise RuntimeError(
|
|
352
|
+
f"omnigraph binary not found. Install via: {cls._SETUP_HINT}"
|
|
353
|
+
)
|