wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/project.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Projects: a directory with a ``wspx.toml``, shipped to the lab as a snapshot.
|
|
2
|
+
|
|
3
|
+
A session started inside a project runs with that project's settings, test
|
|
4
|
+
suites and data, wherever its lab runs. The CLI packs the project into a
|
|
5
|
+
reproducible archive (the same files always give the same bytes, so an
|
|
6
|
+
unchanged project is stored once), the control plane keeps it in the object
|
|
7
|
+
store, and the lab unpacks it as its working directory.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import gzip
|
|
13
|
+
import io
|
|
14
|
+
import os
|
|
15
|
+
import tarfile
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import Path, PurePosixPath
|
|
18
|
+
|
|
19
|
+
from wspx.config import WspxConfig, find_config
|
|
20
|
+
from wspx.errors import ConfigError, WspxError
|
|
21
|
+
|
|
22
|
+
DEFAULT_EXCLUDE = (
|
|
23
|
+
"**/.git",
|
|
24
|
+
"**/.hg",
|
|
25
|
+
"**/.svn",
|
|
26
|
+
"**/.venv",
|
|
27
|
+
"**/venv",
|
|
28
|
+
"**/node_modules",
|
|
29
|
+
"**/__pycache__",
|
|
30
|
+
"**/*.pyc",
|
|
31
|
+
"**/.mypy_cache",
|
|
32
|
+
"**/.pytest_cache",
|
|
33
|
+
"**/.ruff_cache",
|
|
34
|
+
"**/.DS_Store",
|
|
35
|
+
"**/wspx-store",
|
|
36
|
+
"**/*.safetensors",
|
|
37
|
+
"**/*.bin",
|
|
38
|
+
"**/*.gguf",
|
|
39
|
+
"**/*.pt",
|
|
40
|
+
"**/*.pth",
|
|
41
|
+
"**/*.ckpt",
|
|
42
|
+
)
|
|
43
|
+
"""Never uploaded: version control, environments, caches, stores and weights."""
|
|
44
|
+
|
|
45
|
+
MAX_BYTES = 100 * 1024 * 1024
|
|
46
|
+
"""Uncompressed size limit for a project snapshot."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class Project:
|
|
51
|
+
root: Path
|
|
52
|
+
config: WspxConfig
|
|
53
|
+
|
|
54
|
+
@classmethod
|
|
55
|
+
def find(cls, start: str | Path | None = None) -> Project | None:
|
|
56
|
+
"""The project containing ``start`` (default: the working directory), if any."""
|
|
57
|
+
path = find_config(start)
|
|
58
|
+
return cls(path.parent, WspxConfig.load(path)) if path else None
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def name(self) -> str:
|
|
62
|
+
return self.config.project.name or self.root.name
|
|
63
|
+
|
|
64
|
+
def files(self) -> list[PurePosixPath]:
|
|
65
|
+
"""Project files to upload, relative to the root, in a stable order."""
|
|
66
|
+
from modal import FilePatternMatcher
|
|
67
|
+
|
|
68
|
+
ignored = FilePatternMatcher(*DEFAULT_EXCLUDE, *self.config.project.exclude)
|
|
69
|
+
found: list[PurePosixPath] = []
|
|
70
|
+
for directory, dirnames, filenames in os.walk(self.root):
|
|
71
|
+
here = Path(directory).relative_to(self.root)
|
|
72
|
+
dirnames[:] = sorted(d for d in dirnames if not ignored(here / d))
|
|
73
|
+
for name in filenames:
|
|
74
|
+
rel = here / name
|
|
75
|
+
path = self.root / rel
|
|
76
|
+
# Links are skipped: they can point outside the project.
|
|
77
|
+
if not ignored(rel) and path.is_file() and not path.is_symlink():
|
|
78
|
+
found.append(PurePosixPath(rel.as_posix()))
|
|
79
|
+
return sorted(found)
|
|
80
|
+
|
|
81
|
+
def pack(self) -> bytes:
|
|
82
|
+
"""A gzipped tar of :meth:`files`, byte-for-byte reproducible."""
|
|
83
|
+
files = self.files()
|
|
84
|
+
sizes = {rel: (self.root / rel).stat().st_size for rel in files}
|
|
85
|
+
total = sum(sizes.values())
|
|
86
|
+
if total > MAX_BYTES:
|
|
87
|
+
largest = sorted(sizes, key=lambda r: -sizes[r])[:5]
|
|
88
|
+
listing = ", ".join(f"{r} ({sizes[r] / 1e6:.0f} MB)" for r in largest)
|
|
89
|
+
raise ConfigError(
|
|
90
|
+
f"project {self.name} is {total / 1e6:.0f} MB, over the "
|
|
91
|
+
f"{MAX_BYTES // 2**20} MB limit (largest: {listing}); "
|
|
92
|
+
"add paths to [project] exclude in wspx.toml"
|
|
93
|
+
)
|
|
94
|
+
buf = io.BytesIO()
|
|
95
|
+
with (
|
|
96
|
+
gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz,
|
|
97
|
+
tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as tar,
|
|
98
|
+
):
|
|
99
|
+
for rel in files:
|
|
100
|
+
path = self.root / rel
|
|
101
|
+
info = tarfile.TarInfo(str(rel))
|
|
102
|
+
info.size = sizes[rel]
|
|
103
|
+
info.mode = 0o755 if os.access(path, os.X_OK) else 0o644
|
|
104
|
+
with path.open("rb") as fh:
|
|
105
|
+
tar.addfile(info, fh)
|
|
106
|
+
return buf.getvalue()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def members(data: bytes) -> list[tarfile.TarInfo]:
|
|
110
|
+
"""Validate a snapshot without unpacking it: regular files with safe relative paths."""
|
|
111
|
+
try:
|
|
112
|
+
with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
|
|
113
|
+
infos = tar.getmembers()
|
|
114
|
+
except (tarfile.TarError, OSError, EOFError) as exc:
|
|
115
|
+
raise WspxError(f"not a project snapshot: {exc}") from exc
|
|
116
|
+
total = 0
|
|
117
|
+
for info in infos:
|
|
118
|
+
rel = PurePosixPath(info.name)
|
|
119
|
+
if not info.isfile() or rel.is_absolute() or ".." in rel.parts:
|
|
120
|
+
raise WspxError(f"project snapshot entry {info.name!r} is not a plain relative file")
|
|
121
|
+
total += info.size
|
|
122
|
+
if total > MAX_BYTES:
|
|
123
|
+
raise WspxError(f"project snapshot unpacks to {total / 1e6:.0f} MB, over the limit")
|
|
124
|
+
return infos
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def extract(data: bytes, dest: str | Path) -> Path:
|
|
128
|
+
"""Unpack a snapshot into ``dest`` (created if needed); returns ``dest``."""
|
|
129
|
+
root = Path(dest).resolve()
|
|
130
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
131
|
+
infos = members(data)
|
|
132
|
+
with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
|
|
133
|
+
for info in infos:
|
|
134
|
+
target = (root / info.name).resolve()
|
|
135
|
+
if not target.is_relative_to(root):
|
|
136
|
+
raise WspxError(f"project snapshot entry {info.name!r} escapes the workspace")
|
|
137
|
+
source = tar.extractfile(info)
|
|
138
|
+
if source is None:
|
|
139
|
+
raise WspxError(f"project snapshot entry {info.name!r} has no data")
|
|
140
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
141
|
+
target.write_bytes(source.read())
|
|
142
|
+
target.chmod(0o755 if info.mode & 0o111 else 0o644)
|
|
143
|
+
return root
|
wspx/py.typed
ADDED
|
File without changes
|
wspx/safety/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Idle compute is turned off, in independent layers (§5.14.1).
|
|
2
|
+
|
|
3
|
+
1. Session lifecycle (hot → grace → cold), in the control plane.
|
|
4
|
+
2. Leases: a lab runs only while it holds one; it exits when renewals stop.
|
|
5
|
+
3. Modal backstops: sandbox ``idle_timeout`` and ``timeout``.
|
|
6
|
+
4. The sweeper: every 5 minutes, terminate any wspx sandbox without a valid lease.
|
|
7
|
+
5. Optional Modal budgets, set by the user on the ``wspx`` environment.
|
|
8
|
+
|
|
9
|
+
``wspx kill --hard`` needs only the user's Modal token, so it works with the
|
|
10
|
+
control plane gone.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
TAG_KEY = "wspx"
|
|
14
|
+
TAG_VALUE = "lab"
|
|
15
|
+
LEASE_DICT = "wspx-leases"
|
wspx/safety/kill.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""``wspx kill --hard``: stop everything with nothing but the user's Modal token."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
|
|
9
|
+
from wspx.safety import TAG_KEY, TAG_VALUE
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class KillReport:
|
|
14
|
+
terminated: list[str] = field(default_factory=list)
|
|
15
|
+
app_stopped: bool = False
|
|
16
|
+
errors: list[str] = field(default_factory=list)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def hard_kill(
|
|
20
|
+
*, app_name: str = "wspx", environment: str = "wspx"
|
|
21
|
+
) -> KillReport: # pragma: no cover - needs Modal
|
|
22
|
+
"""Terminate every wspx-tagged sandbox in the environment and stop the app."""
|
|
23
|
+
import modal
|
|
24
|
+
|
|
25
|
+
report = KillReport()
|
|
26
|
+
try:
|
|
27
|
+
app = modal.App.lookup(app_name, environment_name=environment)
|
|
28
|
+
sandboxes = list(modal.Sandbox.list(app_id=app.app_id, tags={TAG_KEY: TAG_VALUE}))
|
|
29
|
+
except Exception as exc:
|
|
30
|
+
report.errors.append(f"listing sandboxes: {exc}")
|
|
31
|
+
sandboxes = []
|
|
32
|
+
for sandbox in sandboxes:
|
|
33
|
+
try:
|
|
34
|
+
sandbox.terminate()
|
|
35
|
+
report.terminated.append(sandbox.object_id)
|
|
36
|
+
except Exception as exc:
|
|
37
|
+
report.errors.append(f"{sandbox.object_id}: {exc}")
|
|
38
|
+
proc = subprocess.run(
|
|
39
|
+
[sys.executable, "-m", "modal", "app", "stop", app_name, "--env", environment],
|
|
40
|
+
capture_output=True,
|
|
41
|
+
text=True,
|
|
42
|
+
check=False,
|
|
43
|
+
)
|
|
44
|
+
report.app_stopped = proc.returncode == 0
|
|
45
|
+
if proc.returncode != 0:
|
|
46
|
+
report.errors.append(proc.stderr.strip()[-300:])
|
|
47
|
+
return report
|
wspx/safety/sweeper.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""The sweeper: terminates wspx sandboxes that hold no valid lease."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Callable, Iterable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Any, Protocol
|
|
9
|
+
|
|
10
|
+
from wspx.safety import TAG_KEY, TAG_VALUE
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SandboxLike(Protocol):
|
|
14
|
+
object_id: str
|
|
15
|
+
|
|
16
|
+
def get_tags(self) -> dict[str, str]: ...
|
|
17
|
+
|
|
18
|
+
def terminate(self) -> Any: ...
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class SweepReport:
|
|
23
|
+
checked: int
|
|
24
|
+
terminated: list[str]
|
|
25
|
+
kept: list[str]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def sweep(
|
|
29
|
+
sandboxes: Iterable[SandboxLike],
|
|
30
|
+
lease_of: Callable[[str], float | None],
|
|
31
|
+
*,
|
|
32
|
+
now: float | None = None,
|
|
33
|
+
grace: float = 30.0,
|
|
34
|
+
) -> SweepReport:
|
|
35
|
+
"""Terminate every wspx sandbox whose lease is missing or expired (plus a little grace)."""
|
|
36
|
+
now = now or time.time()
|
|
37
|
+
terminated, kept = [], []
|
|
38
|
+
checked = 0
|
|
39
|
+
for sandbox in sandboxes:
|
|
40
|
+
tags = sandbox.get_tags() or {}
|
|
41
|
+
if tags.get(TAG_KEY) != TAG_VALUE:
|
|
42
|
+
continue
|
|
43
|
+
checked += 1
|
|
44
|
+
expires = lease_of(sandbox.object_id)
|
|
45
|
+
if expires is None or expires + grace < now:
|
|
46
|
+
sandbox.terminate()
|
|
47
|
+
terminated.append(sandbox.object_id)
|
|
48
|
+
else:
|
|
49
|
+
kept.append(sandbox.object_id)
|
|
50
|
+
return SweepReport(checked, terminated, kept)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def modal_sweep(
|
|
54
|
+
*, app_name: str = "wspx", environment: str = "wspx"
|
|
55
|
+
) -> SweepReport: # pragma: no cover - needs Modal
|
|
56
|
+
import modal
|
|
57
|
+
|
|
58
|
+
from wspx.safety import LEASE_DICT
|
|
59
|
+
|
|
60
|
+
app = modal.App.lookup(app_name, environment_name=environment)
|
|
61
|
+
leases = modal.Dict.from_name(LEASE_DICT, environment_name=environment, create_if_missing=True)
|
|
62
|
+
|
|
63
|
+
def lease_of(sandbox_id: str) -> float | None:
|
|
64
|
+
value = leases.get(sandbox_id)
|
|
65
|
+
return float(value) if value is not None else None
|
|
66
|
+
|
|
67
|
+
boxes: list[Any] = list(modal.Sandbox.list(app_id=app.app_id, tags={TAG_KEY: TAG_VALUE}))
|
|
68
|
+
return sweep(boxes, lease_of)
|
wspx/store/__init__.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""The wspx store: content-addressed objects plus refs, on any filesystem.
|
|
2
|
+
|
|
3
|
+
Layout (design §7)::
|
|
4
|
+
|
|
5
|
+
<root>/
|
|
6
|
+
objects/<aa>/<bb…> patch blobs, commit JSON, check reports
|
|
7
|
+
refs/heads/<branch> branch → commit
|
|
8
|
+
refs/sessions/<id>/HEAD per-session HEAD (usually symbolic)
|
|
9
|
+
models/<fingerprint>/ Model IR and index
|
|
10
|
+
results/ runs/ traces/ sessions/ result sets, eval cache, traces, session state
|
|
11
|
+
|
|
12
|
+
On Modal this root is the ``wspx-store`` Volume, written only by the control
|
|
13
|
+
plane. Locally it is any directory.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from wspx.store.objects import ObjectStore, atomic_write, canonical_json, is_oid, oid_of, short
|
|
21
|
+
from wspx.store.refs import ANY, RefStore
|
|
22
|
+
|
|
23
|
+
__all__ = ["ANY", "ObjectStore", "RefStore", "Store", "canonical_json", "is_oid", "oid_of", "short"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class Store:
|
|
27
|
+
def __init__(self, root: str | Path) -> None:
|
|
28
|
+
self.root = Path(root)
|
|
29
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
self.objects = ObjectStore(self.root / "objects")
|
|
31
|
+
self.refs = RefStore(self.root / "refs")
|
|
32
|
+
|
|
33
|
+
def path(self, *parts: str) -> Path:
|
|
34
|
+
"""A path under the store root for non-object data (IR, results, runs, traces)."""
|
|
35
|
+
target = self.root.joinpath(*parts).resolve()
|
|
36
|
+
if not target.is_relative_to(self.root.resolve()):
|
|
37
|
+
raise ValueError(f"path escapes the store: {'/'.join(parts)}")
|
|
38
|
+
return target
|
|
39
|
+
|
|
40
|
+
def model_dir(self, fingerprint: str) -> Path:
|
|
41
|
+
return self.path("models", fingerprint.replace(":", "_"))
|
|
42
|
+
|
|
43
|
+
# Non-object data goes through these, so a read-only lab can journal its writes.
|
|
44
|
+
|
|
45
|
+
def write_file(self, rel: str, data: bytes) -> None:
|
|
46
|
+
atomic_write(self.path(rel), data)
|
|
47
|
+
|
|
48
|
+
def read_file(self, rel: str) -> bytes | None:
|
|
49
|
+
target = self.path(rel)
|
|
50
|
+
return target.read_bytes() if target.is_file() else None
|
|
51
|
+
|
|
52
|
+
def exists(self, rel: str) -> bool:
|
|
53
|
+
return self.path(rel).is_file()
|
|
54
|
+
|
|
55
|
+
def glob(self, rel: str, pattern: str) -> list[str]:
|
|
56
|
+
"""Relative paths of files under ``rel`` matching ``pattern``."""
|
|
57
|
+
base = self.path(rel)
|
|
58
|
+
if not base.is_dir():
|
|
59
|
+
return []
|
|
60
|
+
root = self.root.resolve()
|
|
61
|
+
return sorted(p.relative_to(root).as_posix() for p in base.glob(pattern) if p.is_file())
|
|
62
|
+
|
|
63
|
+
def __repr__(self) -> str:
|
|
64
|
+
return f"Store({str(self.root)!r})"
|
wspx/store/objects.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Content-addressed object storage.
|
|
2
|
+
|
|
3
|
+
Objects are immutable and keyed by ``sha256:<hex>`` of their bytes. Reads
|
|
4
|
+
re-hash by default, so silent corruption surfaces as :class:`IntegrityError`
|
|
5
|
+
instead of a wrong tensor. Writes are atomic (temp file + rename), which keeps
|
|
6
|
+
the store consistent across crashes and on network volumes.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import tempfile
|
|
16
|
+
from collections.abc import Iterator
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from wspx.errors import IntegrityError, StoreError
|
|
21
|
+
|
|
22
|
+
OID_PREFIX = "sha256:"
|
|
23
|
+
_OID_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
24
|
+
_HEX_PREFIX_RE = re.compile(r"^[0-9a-f]{4,64}$")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def oid_of(data: bytes) -> str:
|
|
28
|
+
return OID_PREFIX + hashlib.sha256(data).hexdigest()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def is_oid(text: str) -> bool:
|
|
32
|
+
return bool(_OID_RE.match(text))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def short(oid: str, length: int = 7) -> str:
|
|
36
|
+
"""``sha256:5be01c7…`` → ``5be01c7``."""
|
|
37
|
+
return oid.removeprefix(OID_PREFIX)[:length]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def canonical_json(obj: Any) -> bytes:
|
|
41
|
+
"""Deterministic JSON: sorted keys, no whitespace, no NaN. Hashes depend on it."""
|
|
42
|
+
return json.dumps(
|
|
43
|
+
obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
|
|
44
|
+
).encode("utf-8")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def atomic_write(path: Path, data: bytes) -> None:
|
|
48
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
|
|
50
|
+
try:
|
|
51
|
+
with os.fdopen(fd, "wb") as fh:
|
|
52
|
+
fh.write(data)
|
|
53
|
+
fh.flush()
|
|
54
|
+
os.fsync(fh.fileno())
|
|
55
|
+
os.replace(tmp, path)
|
|
56
|
+
except BaseException:
|
|
57
|
+
Path(tmp).unlink(missing_ok=True)
|
|
58
|
+
raise
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class ObjectStore:
|
|
62
|
+
def __init__(self, root: str | Path) -> None:
|
|
63
|
+
self.root = Path(root)
|
|
64
|
+
|
|
65
|
+
def _path(self, oid: str) -> Path:
|
|
66
|
+
if not is_oid(oid):
|
|
67
|
+
raise StoreError(f"not an object id: {oid!r}")
|
|
68
|
+
hexd = oid[len(OID_PREFIX) :]
|
|
69
|
+
return self.root / hexd[:2] / hexd[2:]
|
|
70
|
+
|
|
71
|
+
def put(self, data: bytes) -> str:
|
|
72
|
+
oid = oid_of(data)
|
|
73
|
+
path = self._path(oid)
|
|
74
|
+
if not path.exists():
|
|
75
|
+
atomic_write(path, data)
|
|
76
|
+
return oid
|
|
77
|
+
|
|
78
|
+
def get(self, oid: str, *, verify: bool = True) -> bytes:
|
|
79
|
+
path = self._path(oid)
|
|
80
|
+
try:
|
|
81
|
+
data = path.read_bytes()
|
|
82
|
+
except FileNotFoundError:
|
|
83
|
+
raise StoreError(f"object {short(oid, 12)} not found") from None
|
|
84
|
+
if verify and oid_of(data) != oid:
|
|
85
|
+
raise IntegrityError(f"object {short(oid, 12)} is corrupt: content hash mismatch")
|
|
86
|
+
return data
|
|
87
|
+
|
|
88
|
+
def has(self, oid: str) -> bool:
|
|
89
|
+
return self._path(oid).exists()
|
|
90
|
+
|
|
91
|
+
def size(self, oid: str) -> int:
|
|
92
|
+
return self._path(oid).stat().st_size
|
|
93
|
+
|
|
94
|
+
def put_json(self, obj: Any) -> str:
|
|
95
|
+
return self.put(canonical_json(obj))
|
|
96
|
+
|
|
97
|
+
def get_json(self, oid: str) -> Any:
|
|
98
|
+
return json.loads(self.get(oid))
|
|
99
|
+
|
|
100
|
+
def __iter__(self) -> Iterator[str]:
|
|
101
|
+
if not self.root.exists():
|
|
102
|
+
return
|
|
103
|
+
for fan in sorted(self.root.iterdir()):
|
|
104
|
+
if fan.is_dir() and len(fan.name) == 2:
|
|
105
|
+
for obj in sorted(fan.iterdir()):
|
|
106
|
+
if not obj.name.startswith("."):
|
|
107
|
+
yield OID_PREFIX + fan.name + obj.name
|
|
108
|
+
|
|
109
|
+
def resolve_prefix(self, text: str) -> str:
|
|
110
|
+
"""Expand ``5be01c7`` or ``sha256:5be0…`` to a full object id."""
|
|
111
|
+
prefix = text.removeprefix(OID_PREFIX).lower()
|
|
112
|
+
if is_oid(OID_PREFIX + prefix) and self.has(OID_PREFIX + prefix):
|
|
113
|
+
return OID_PREFIX + prefix
|
|
114
|
+
if not _HEX_PREFIX_RE.match(prefix):
|
|
115
|
+
raise StoreError(f"{text!r} is not an object id or prefix")
|
|
116
|
+
fan = self.root / prefix[:2]
|
|
117
|
+
matches = (
|
|
118
|
+
[
|
|
119
|
+
OID_PREFIX + prefix[:2] + p.name
|
|
120
|
+
for p in fan.iterdir()
|
|
121
|
+
if p.name.startswith(prefix[2:])
|
|
122
|
+
]
|
|
123
|
+
if fan.is_dir()
|
|
124
|
+
else []
|
|
125
|
+
)
|
|
126
|
+
if not matches:
|
|
127
|
+
raise StoreError(f"no object matches {text!r}")
|
|
128
|
+
if len(matches) > 1:
|
|
129
|
+
raise StoreError(f"{text!r} is ambiguous ({len(matches)} objects); use more digits")
|
|
130
|
+
return matches[0]
|
wspx/store/overlay.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""A store for a lab that may not write the store volume (§3, §6).
|
|
2
|
+
|
|
3
|
+
Reads come from the read-only volume (and anything this lab wrote since);
|
|
4
|
+
writes land in a local overlay and are recorded in a journal. After each tool
|
|
5
|
+
call the control plane (the store's single writer) takes the journal and
|
|
6
|
+
applies it, refs with compare-and-swap. Untrusted code in the lab therefore
|
|
7
|
+
never holds a writable handle on shared state.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import base64
|
|
13
|
+
import threading
|
|
14
|
+
from collections.abc import Iterator
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from wspx.errors import StoreError
|
|
19
|
+
from wspx.store import Store
|
|
20
|
+
from wspx.store.objects import OID_PREFIX, ObjectStore, atomic_write, oid_of
|
|
21
|
+
from wspx.store.refs import ANY, SYMBOLIC_PREFIX, RefStore, _Any
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Journal:
|
|
25
|
+
def __init__(self) -> None:
|
|
26
|
+
self._lock = threading.Lock()
|
|
27
|
+
self.objects: dict[str, bytes] = {}
|
|
28
|
+
self.files: dict[str, bytes] = {}
|
|
29
|
+
self.refs: list[dict[str, Any]] = []
|
|
30
|
+
|
|
31
|
+
def drain(self) -> dict[str, Any]:
|
|
32
|
+
"""The pending writes as JSON-safe data; clears the journal."""
|
|
33
|
+
with self._lock:
|
|
34
|
+
out = {
|
|
35
|
+
"objects": {k: base64.b64encode(v).decode() for k, v in self.objects.items()},
|
|
36
|
+
"files": {k: base64.b64encode(v).decode() for k, v in self.files.items()},
|
|
37
|
+
"refs": list(self.refs),
|
|
38
|
+
}
|
|
39
|
+
self.objects, self.files, self.refs = {}, {}, []
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
def __bool__(self) -> bool:
|
|
43
|
+
return bool(self.objects or self.files or self.refs)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class _OverlayObjects(ObjectStore):
|
|
47
|
+
def __init__(self, base: ObjectStore, root: Path, journal: Journal) -> None:
|
|
48
|
+
super().__init__(root)
|
|
49
|
+
self.base = base
|
|
50
|
+
self.journal = journal
|
|
51
|
+
|
|
52
|
+
def put(self, data: bytes) -> str:
|
|
53
|
+
oid = oid_of(data)
|
|
54
|
+
if self.base.has(oid) or super().has(oid):
|
|
55
|
+
return oid
|
|
56
|
+
super().put(data)
|
|
57
|
+
with self.journal._lock:
|
|
58
|
+
self.journal.objects[oid] = data
|
|
59
|
+
return oid
|
|
60
|
+
|
|
61
|
+
def get(self, oid: str, *, verify: bool = True) -> bytes:
|
|
62
|
+
if super().has(oid):
|
|
63
|
+
return super().get(oid, verify=verify)
|
|
64
|
+
return self.base.get(oid, verify=verify)
|
|
65
|
+
|
|
66
|
+
def has(self, oid: str) -> bool:
|
|
67
|
+
return super().has(oid) or self.base.has(oid)
|
|
68
|
+
|
|
69
|
+
def size(self, oid: str) -> int:
|
|
70
|
+
return super().size(oid) if super().has(oid) else self.base.size(oid)
|
|
71
|
+
|
|
72
|
+
def __iter__(self) -> Iterator[str]:
|
|
73
|
+
seen = set(super().__iter__())
|
|
74
|
+
yield from sorted(seen | set(self.base))
|
|
75
|
+
|
|
76
|
+
def resolve_prefix(self, text: str) -> str:
|
|
77
|
+
found = set()
|
|
78
|
+
for resolve in (lambda t: ObjectStore.resolve_prefix(self, t), self.base.resolve_prefix):
|
|
79
|
+
try:
|
|
80
|
+
found.add(resolve(text))
|
|
81
|
+
except StoreError as exc:
|
|
82
|
+
if "ambiguous" in str(exc):
|
|
83
|
+
raise
|
|
84
|
+
if len(found) == 1:
|
|
85
|
+
return found.pop()
|
|
86
|
+
if not found:
|
|
87
|
+
raise StoreError(f"no object matches {text!r}")
|
|
88
|
+
raise StoreError(f"{text!r} is ambiguous; use more digits")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class _OverlayRefs(RefStore):
|
|
92
|
+
def __init__(self, base: RefStore, root: Path, journal: Journal) -> None:
|
|
93
|
+
super().__init__(root)
|
|
94
|
+
self.base = base
|
|
95
|
+
self.journal = journal
|
|
96
|
+
|
|
97
|
+
def exists(self, name: str) -> bool:
|
|
98
|
+
return super().exists(name) or self.base.exists(name)
|
|
99
|
+
|
|
100
|
+
def read_raw(self, name: str) -> str | None:
|
|
101
|
+
if super().exists(name):
|
|
102
|
+
return super().read_raw(name)
|
|
103
|
+
return self.base.read_raw(name)
|
|
104
|
+
|
|
105
|
+
def list(self, prefix: str = "heads/") -> dict[str, str]:
|
|
106
|
+
out = dict(self.base.list(prefix))
|
|
107
|
+
for name in super().list(prefix):
|
|
108
|
+
value = self.read(name)
|
|
109
|
+
if value is None:
|
|
110
|
+
out.pop(name, None)
|
|
111
|
+
else:
|
|
112
|
+
out[name] = value
|
|
113
|
+
return out
|
|
114
|
+
|
|
115
|
+
def update(self, name: str, new: str, expected: str | _Any | None = ANY) -> None:
|
|
116
|
+
target = self._resolve_symbolic_chain(name)
|
|
117
|
+
with self._locked(target):
|
|
118
|
+
current = self.read_raw(target)
|
|
119
|
+
if expected is not ANY and current != expected:
|
|
120
|
+
from wspx.errors import RefConflict
|
|
121
|
+
|
|
122
|
+
raise RefConflict(f"ref {target!r} moved: expected {expected}, found {current}")
|
|
123
|
+
atomic_write(self._path(target), f"{new}\n".encode())
|
|
124
|
+
self._record({"op": "update", "name": target, "value": new, "expected": current})
|
|
125
|
+
|
|
126
|
+
def set_symbolic(self, name: str, target: str) -> None:
|
|
127
|
+
super().set_symbolic(name, target)
|
|
128
|
+
self._record({"op": "symbolic", "name": name, "value": target})
|
|
129
|
+
|
|
130
|
+
def delete(self, name: str, expected: str | _Any | None = ANY) -> None:
|
|
131
|
+
with self._locked(name):
|
|
132
|
+
current = self.read_raw(name)
|
|
133
|
+
if expected is not ANY and current != expected:
|
|
134
|
+
from wspx.errors import RefConflict
|
|
135
|
+
|
|
136
|
+
raise RefConflict(f"ref {name!r} moved; refusing to delete")
|
|
137
|
+
atomic_write(self._path(name), b"") # tombstone over the base
|
|
138
|
+
self._record({"op": "delete", "name": name, "expected": current})
|
|
139
|
+
|
|
140
|
+
def _record(self, entry: dict[str, Any]) -> None:
|
|
141
|
+
with self.journal._lock:
|
|
142
|
+
self.journal.refs.append(entry)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class OverlayStore(Store):
|
|
146
|
+
def __init__(self, base: str | Path, overlay: str | Path) -> None:
|
|
147
|
+
self.base = Store.__new__(Store)
|
|
148
|
+
self.base.root = Path(base)
|
|
149
|
+
self.base.objects = ObjectStore(self.base.root / "objects")
|
|
150
|
+
self.base.refs = RefStore(self.base.root / "refs")
|
|
151
|
+
self.root = Path(overlay)
|
|
152
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
153
|
+
self.journal = Journal()
|
|
154
|
+
self.objects = _OverlayObjects(self.base.objects, self.root / "objects", self.journal)
|
|
155
|
+
self.refs = _OverlayRefs(self.base.refs, self.root / "refs", self.journal)
|
|
156
|
+
|
|
157
|
+
def path(self, *parts: str) -> Path:
|
|
158
|
+
mine = super().path(*parts)
|
|
159
|
+
if mine.exists():
|
|
160
|
+
return mine
|
|
161
|
+
return self.base.path(*parts)
|
|
162
|
+
|
|
163
|
+
def write_file(self, rel: str, data: bytes) -> None:
|
|
164
|
+
atomic_write(Store.path(self, rel), data)
|
|
165
|
+
with self.journal._lock:
|
|
166
|
+
self.journal.files[rel] = data
|
|
167
|
+
|
|
168
|
+
def read_file(self, rel: str) -> bytes | None:
|
|
169
|
+
mine = Store.path(self, rel)
|
|
170
|
+
if mine.is_file():
|
|
171
|
+
return mine.read_bytes()
|
|
172
|
+
return self.base.read_file(rel)
|
|
173
|
+
|
|
174
|
+
def exists(self, rel: str) -> bool:
|
|
175
|
+
return Store.path(self, rel).is_file() or self.base.exists(rel)
|
|
176
|
+
|
|
177
|
+
def glob(self, rel: str, pattern: str) -> list[str]:
|
|
178
|
+
return sorted(set(Store.glob(self, rel, pattern)) | set(self.base.glob(rel, pattern)))
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def apply_journal(store: Store, journal: dict[str, Any]) -> dict[str, int]:
|
|
182
|
+
"""Persist a lab's journal into the real store (the control plane is the only writer)."""
|
|
183
|
+
for oid, b64 in journal.get("objects", {}).items():
|
|
184
|
+
data = base64.b64decode(b64)
|
|
185
|
+
if oid_of(data) != oid:
|
|
186
|
+
raise StoreError(f"journal object {oid[:19]} does not match its content")
|
|
187
|
+
store.objects.put(data)
|
|
188
|
+
for rel, b64 in journal.get("files", {}).items():
|
|
189
|
+
if rel.startswith(("objects/", "refs/")) or ".." in rel.split("/"):
|
|
190
|
+
raise StoreError(f"journal may not write {rel!r}")
|
|
191
|
+
store.write_file(rel, base64.b64decode(b64))
|
|
192
|
+
for entry in journal.get("refs", []):
|
|
193
|
+
name = entry["name"]
|
|
194
|
+
if entry["op"] == "symbolic":
|
|
195
|
+
store.refs.set_symbolic(name, entry["value"])
|
|
196
|
+
elif entry["op"] == "delete":
|
|
197
|
+
store.refs.delete(name, expected=entry.get("expected"))
|
|
198
|
+
else:
|
|
199
|
+
value = entry["value"]
|
|
200
|
+
if value.startswith(SYMBOLIC_PREFIX):
|
|
201
|
+
raise StoreError("journal ref updates must hold object ids")
|
|
202
|
+
if value and not value.startswith(OID_PREFIX):
|
|
203
|
+
raise StoreError(f"journal ref value {value!r} is not an object id")
|
|
204
|
+
store.refs.update(name, value, expected=entry.get("expected"))
|
|
205
|
+
return {
|
|
206
|
+
"objects": len(journal.get("objects", {})),
|
|
207
|
+
"files": len(journal.get("files", {})),
|
|
208
|
+
"refs": len(journal.get("refs", [])),
|
|
209
|
+
}
|