wspx 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. wspx/__init__.py +24 -0
  2. wspx/_version.py +1 -0
  3. wspx/addressing.py +415 -0
  4. wspx/auth.py +77 -0
  5. wspx/cli/__init__.py +1 -0
  6. wspx/cli/client.py +124 -0
  7. wspx/cli/config.py +89 -0
  8. wspx/cli/connect.py +88 -0
  9. wspx/cli/main.py +848 -0
  10. wspx/cloud/__init__.py +20 -0
  11. wspx/cloud/app.py +191 -0
  12. wspx/cloud/backend.py +141 -0
  13. wspx/cloud/compose.py +85 -0
  14. wspx/cloud/deploy.py +102 -0
  15. wspx/config.py +189 -0
  16. wspx/control/__init__.py +5 -0
  17. wspx/control/app.py +488 -0
  18. wspx/control/backend.py +244 -0
  19. wspx/control/gateway_client.py +79 -0
  20. wspx/control/mcp_server.py +71 -0
  21. wspx/control/sessions.py +424 -0
  22. wspx/control/tracedb.py +183 -0
  23. wspx/control/ui/assets/index-BJFweZAn.css +1 -0
  24. wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
  25. wspx/control/ui/index.html +13 -0
  26. wspx/control/views.py +180 -0
  27. wspx/diagnostics.py +146 -0
  28. wspx/driver/__init__.py +7 -0
  29. wspx/driver/loop.py +255 -0
  30. wspx/driver/messages.py +62 -0
  31. wspx/driver/providers/__init__.py +84 -0
  32. wspx/driver/providers/anthropic.py +149 -0
  33. wspx/driver/providers/compat.py +115 -0
  34. wspx/driver/providers/google.py +138 -0
  35. wspx/driver/providers/openai.py +125 -0
  36. wspx/driver/run.py +134 -0
  37. wspx/errors.py +69 -0
  38. wspx/gateway/__init__.py +8 -0
  39. wspx/gateway/app.py +626 -0
  40. wspx/gateway/ledger.py +536 -0
  41. wspx/gateway/prices.py +75 -0
  42. wspx/gateway/usage.py +284 -0
  43. wspx/handles.py +117 -0
  44. wspx/integrations/AGENTS.snippet.md +8 -0
  45. wspx/integrations/SKILL.md +52 -0
  46. wspx/integrations/__init__.py +17 -0
  47. wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
  48. wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
  49. wspx/integrations/codex-hooks.toml +9 -0
  50. wspx/lab/__init__.py +5 -0
  51. wspx/lab/analysis.py +485 -0
  52. wspx/lab/check.py +489 -0
  53. wspx/lab/evaluate.py +108 -0
  54. wspx/lab/export.py +275 -0
  55. wspx/lab/grep.py +754 -0
  56. wspx/lab/index.py +239 -0
  57. wspx/lab/ir/__init__.py +1 -0
  58. wspx/lab/ir/capture.py +299 -0
  59. wspx/lab/ir/graph.py +148 -0
  60. wspx/lab/ir/model_ir.py +522 -0
  61. wspx/lab/ir/semantics.py +1255 -0
  62. wspx/lab/ir/tape.py +853 -0
  63. wspx/lab/loader.py +204 -0
  64. wspx/lab/merge.py +200 -0
  65. wspx/lab/ops/__init__.py +76 -0
  66. wspx/lab/ops/builtin.py +379 -0
  67. wspx/lab/patch.py +320 -0
  68. wspx/lab/probe.py +206 -0
  69. wspx/lab/results.py +114 -0
  70. wspx/lab/sdk.py +120 -0
  71. wspx/lab/server.py +236 -0
  72. wspx/lab/session.py +1574 -0
  73. wspx/lab/tensors.py +119 -0
  74. wspx/lab/text.py +69 -0
  75. wspx/lab/tree.py +302 -0
  76. wspx/loops.py +127 -0
  77. wspx/project.py +143 -0
  78. wspx/py.typed +0 -0
  79. wspx/safety/__init__.py +15 -0
  80. wspx/safety/kill.py +47 -0
  81. wspx/safety/sweeper.py +68 -0
  82. wspx/store/__init__.py +64 -0
  83. wspx/store/objects.py +130 -0
  84. wspx/store/overlay.py +209 -0
  85. wspx/store/refs.py +150 -0
  86. wspx/testing/__init__.py +21 -0
  87. wspx/testing/api.py +155 -0
  88. wspx/testing/builtin.py +139 -0
  89. wspx/testing/runner.py +257 -0
  90. wspx/tools/__init__.py +17 -0
  91. wspx/tools/dispatch.py +129 -0
  92. wspx/tools/schemas.py +313 -0
  93. wspx/vcs/__init__.py +17 -0
  94. wspx/vcs/repo.py +431 -0
  95. wspx-0.1.0.dev0.dist-info/METADATA +278 -0
  96. wspx-0.1.0.dev0.dist-info/RECORD +99 -0
  97. wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
  98. wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
  99. wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/project.py ADDED
@@ -0,0 +1,143 @@
1
+ """Projects: a directory with a ``wspx.toml``, shipped to the lab as a snapshot.
2
+
3
+ A session started inside a project runs with that project's settings, test
4
+ suites and data, wherever its lab runs. The CLI packs the project into a
5
+ reproducible archive (the same files always give the same bytes, so an
6
+ unchanged project is stored once), the control plane keeps it in the object
7
+ store, and the lab unpacks it as its working directory.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import gzip
13
+ import io
14
+ import os
15
+ import tarfile
16
+ from dataclasses import dataclass
17
+ from pathlib import Path, PurePosixPath
18
+
19
+ from wspx.config import WspxConfig, find_config
20
+ from wspx.errors import ConfigError, WspxError
21
+
22
+ DEFAULT_EXCLUDE = (
23
+ "**/.git",
24
+ "**/.hg",
25
+ "**/.svn",
26
+ "**/.venv",
27
+ "**/venv",
28
+ "**/node_modules",
29
+ "**/__pycache__",
30
+ "**/*.pyc",
31
+ "**/.mypy_cache",
32
+ "**/.pytest_cache",
33
+ "**/.ruff_cache",
34
+ "**/.DS_Store",
35
+ "**/wspx-store",
36
+ "**/*.safetensors",
37
+ "**/*.bin",
38
+ "**/*.gguf",
39
+ "**/*.pt",
40
+ "**/*.pth",
41
+ "**/*.ckpt",
42
+ )
43
+ """Never uploaded: version control, environments, caches, stores and weights."""
44
+
45
+ MAX_BYTES = 100 * 1024 * 1024
46
+ """Uncompressed size limit for a project snapshot."""
47
+
48
+
49
+ @dataclass(frozen=True)
50
+ class Project:
51
+ root: Path
52
+ config: WspxConfig
53
+
54
+ @classmethod
55
+ def find(cls, start: str | Path | None = None) -> Project | None:
56
+ """The project containing ``start`` (default: the working directory), if any."""
57
+ path = find_config(start)
58
+ return cls(path.parent, WspxConfig.load(path)) if path else None
59
+
60
+ @property
61
+ def name(self) -> str:
62
+ return self.config.project.name or self.root.name
63
+
64
+ def files(self) -> list[PurePosixPath]:
65
+ """Project files to upload, relative to the root, in a stable order."""
66
+ from modal import FilePatternMatcher
67
+
68
+ ignored = FilePatternMatcher(*DEFAULT_EXCLUDE, *self.config.project.exclude)
69
+ found: list[PurePosixPath] = []
70
+ for directory, dirnames, filenames in os.walk(self.root):
71
+ here = Path(directory).relative_to(self.root)
72
+ dirnames[:] = sorted(d for d in dirnames if not ignored(here / d))
73
+ for name in filenames:
74
+ rel = here / name
75
+ path = self.root / rel
76
+ # Links are skipped: they can point outside the project.
77
+ if not ignored(rel) and path.is_file() and not path.is_symlink():
78
+ found.append(PurePosixPath(rel.as_posix()))
79
+ return sorted(found)
80
+
81
+ def pack(self) -> bytes:
82
+ """A gzipped tar of :meth:`files`, byte-for-byte reproducible."""
83
+ files = self.files()
84
+ sizes = {rel: (self.root / rel).stat().st_size for rel in files}
85
+ total = sum(sizes.values())
86
+ if total > MAX_BYTES:
87
+ largest = sorted(sizes, key=lambda r: -sizes[r])[:5]
88
+ listing = ", ".join(f"{r} ({sizes[r] / 1e6:.0f} MB)" for r in largest)
89
+ raise ConfigError(
90
+ f"project {self.name} is {total / 1e6:.0f} MB, over the "
91
+ f"{MAX_BYTES // 2**20} MB limit (largest: {listing}); "
92
+ "add paths to [project] exclude in wspx.toml"
93
+ )
94
+ buf = io.BytesIO()
95
+ with (
96
+ gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz,
97
+ tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as tar,
98
+ ):
99
+ for rel in files:
100
+ path = self.root / rel
101
+ info = tarfile.TarInfo(str(rel))
102
+ info.size = sizes[rel]
103
+ info.mode = 0o755 if os.access(path, os.X_OK) else 0o644
104
+ with path.open("rb") as fh:
105
+ tar.addfile(info, fh)
106
+ return buf.getvalue()
107
+
108
+
109
+ def members(data: bytes) -> list[tarfile.TarInfo]:
110
+ """Validate a snapshot without unpacking it: regular files with safe relative paths."""
111
+ try:
112
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
113
+ infos = tar.getmembers()
114
+ except (tarfile.TarError, OSError, EOFError) as exc:
115
+ raise WspxError(f"not a project snapshot: {exc}") from exc
116
+ total = 0
117
+ for info in infos:
118
+ rel = PurePosixPath(info.name)
119
+ if not info.isfile() or rel.is_absolute() or ".." in rel.parts:
120
+ raise WspxError(f"project snapshot entry {info.name!r} is not a plain relative file")
121
+ total += info.size
122
+ if total > MAX_BYTES:
123
+ raise WspxError(f"project snapshot unpacks to {total / 1e6:.0f} MB, over the limit")
124
+ return infos
125
+
126
+
127
+ def extract(data: bytes, dest: str | Path) -> Path:
128
+ """Unpack a snapshot into ``dest`` (created if needed); returns ``dest``."""
129
+ root = Path(dest).resolve()
130
+ root.mkdir(parents=True, exist_ok=True)
131
+ infos = members(data)
132
+ with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar:
133
+ for info in infos:
134
+ target = (root / info.name).resolve()
135
+ if not target.is_relative_to(root):
136
+ raise WspxError(f"project snapshot entry {info.name!r} escapes the workspace")
137
+ source = tar.extractfile(info)
138
+ if source is None:
139
+ raise WspxError(f"project snapshot entry {info.name!r} has no data")
140
+ target.parent.mkdir(parents=True, exist_ok=True)
141
+ target.write_bytes(source.read())
142
+ target.chmod(0o755 if info.mode & 0o111 else 0o644)
143
+ return root
wspx/py.typed ADDED
File without changes
@@ -0,0 +1,15 @@
1
+ """Idle compute is turned off, in independent layers (§5.14.1).
2
+
3
+ 1. Session lifecycle (hot → grace → cold), in the control plane.
4
+ 2. Leases: a lab runs only while it holds one; it exits when renewals stop.
5
+ 3. Modal backstops: sandbox ``idle_timeout`` and ``timeout``.
6
+ 4. The sweeper: every 5 minutes, terminate any wspx sandbox without a valid lease.
7
+ 5. Optional Modal budgets, set by the user on the ``wspx`` environment.
8
+
9
+ ``wspx kill --hard`` needs only the user's Modal token, so it works with the
10
+ control plane gone.
11
+ """
12
+
13
+ TAG_KEY = "wspx"
14
+ TAG_VALUE = "lab"
15
+ LEASE_DICT = "wspx-leases"
wspx/safety/kill.py ADDED
@@ -0,0 +1,47 @@
1
+ """``wspx kill --hard``: stop everything with nothing but the user's Modal token."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import subprocess
6
+ import sys
7
+ from dataclasses import dataclass, field
8
+
9
+ from wspx.safety import TAG_KEY, TAG_VALUE
10
+
11
+
12
+ @dataclass
13
+ class KillReport:
14
+ terminated: list[str] = field(default_factory=list)
15
+ app_stopped: bool = False
16
+ errors: list[str] = field(default_factory=list)
17
+
18
+
19
+ def hard_kill(
20
+ *, app_name: str = "wspx", environment: str = "wspx"
21
+ ) -> KillReport: # pragma: no cover - needs Modal
22
+ """Terminate every wspx-tagged sandbox in the environment and stop the app."""
23
+ import modal
24
+
25
+ report = KillReport()
26
+ try:
27
+ app = modal.App.lookup(app_name, environment_name=environment)
28
+ sandboxes = list(modal.Sandbox.list(app_id=app.app_id, tags={TAG_KEY: TAG_VALUE}))
29
+ except Exception as exc:
30
+ report.errors.append(f"listing sandboxes: {exc}")
31
+ sandboxes = []
32
+ for sandbox in sandboxes:
33
+ try:
34
+ sandbox.terminate()
35
+ report.terminated.append(sandbox.object_id)
36
+ except Exception as exc:
37
+ report.errors.append(f"{sandbox.object_id}: {exc}")
38
+ proc = subprocess.run(
39
+ [sys.executable, "-m", "modal", "app", "stop", app_name, "--env", environment],
40
+ capture_output=True,
41
+ text=True,
42
+ check=False,
43
+ )
44
+ report.app_stopped = proc.returncode == 0
45
+ if proc.returncode != 0:
46
+ report.errors.append(proc.stderr.strip()[-300:])
47
+ return report
wspx/safety/sweeper.py ADDED
@@ -0,0 +1,68 @@
1
+ """The sweeper: terminates wspx sandboxes that hold no valid lease."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import time
6
+ from collections.abc import Callable, Iterable
7
+ from dataclasses import dataclass
8
+ from typing import Any, Protocol
9
+
10
+ from wspx.safety import TAG_KEY, TAG_VALUE
11
+
12
+
13
+ class SandboxLike(Protocol):
14
+ object_id: str
15
+
16
+ def get_tags(self) -> dict[str, str]: ...
17
+
18
+ def terminate(self) -> Any: ...
19
+
20
+
21
+ @dataclass
22
+ class SweepReport:
23
+ checked: int
24
+ terminated: list[str]
25
+ kept: list[str]
26
+
27
+
28
+ def sweep(
29
+ sandboxes: Iterable[SandboxLike],
30
+ lease_of: Callable[[str], float | None],
31
+ *,
32
+ now: float | None = None,
33
+ grace: float = 30.0,
34
+ ) -> SweepReport:
35
+ """Terminate every wspx sandbox whose lease is missing or expired (plus a little grace)."""
36
+ now = now or time.time()
37
+ terminated, kept = [], []
38
+ checked = 0
39
+ for sandbox in sandboxes:
40
+ tags = sandbox.get_tags() or {}
41
+ if tags.get(TAG_KEY) != TAG_VALUE:
42
+ continue
43
+ checked += 1
44
+ expires = lease_of(sandbox.object_id)
45
+ if expires is None or expires + grace < now:
46
+ sandbox.terminate()
47
+ terminated.append(sandbox.object_id)
48
+ else:
49
+ kept.append(sandbox.object_id)
50
+ return SweepReport(checked, terminated, kept)
51
+
52
+
53
+ def modal_sweep(
54
+ *, app_name: str = "wspx", environment: str = "wspx"
55
+ ) -> SweepReport: # pragma: no cover - needs Modal
56
+ import modal
57
+
58
+ from wspx.safety import LEASE_DICT
59
+
60
+ app = modal.App.lookup(app_name, environment_name=environment)
61
+ leases = modal.Dict.from_name(LEASE_DICT, environment_name=environment, create_if_missing=True)
62
+
63
+ def lease_of(sandbox_id: str) -> float | None:
64
+ value = leases.get(sandbox_id)
65
+ return float(value) if value is not None else None
66
+
67
+ boxes: list[Any] = list(modal.Sandbox.list(app_id=app.app_id, tags={TAG_KEY: TAG_VALUE}))
68
+ return sweep(boxes, lease_of)
wspx/store/__init__.py ADDED
@@ -0,0 +1,64 @@
1
+ """The wspx store: content-addressed objects plus refs, on any filesystem.
2
+
3
+ Layout (design §7)::
4
+
5
+ <root>/
6
+ objects/<aa>/<bb…> patch blobs, commit JSON, check reports
7
+ refs/heads/<branch> branch → commit
8
+ refs/sessions/<id>/HEAD per-session HEAD (usually symbolic)
9
+ models/<fingerprint>/ Model IR and index
10
+ results/ runs/ traces/ sessions/ result sets, eval cache, traces, session state
11
+
12
+ On Modal this root is the ``wspx-store`` Volume, written only by the control
13
+ plane. Locally it is any directory.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ from pathlib import Path
19
+
20
+ from wspx.store.objects import ObjectStore, atomic_write, canonical_json, is_oid, oid_of, short
21
+ from wspx.store.refs import ANY, RefStore
22
+
23
+ __all__ = ["ANY", "ObjectStore", "RefStore", "Store", "canonical_json", "is_oid", "oid_of", "short"]
24
+
25
+
26
+ class Store:
27
+ def __init__(self, root: str | Path) -> None:
28
+ self.root = Path(root)
29
+ self.root.mkdir(parents=True, exist_ok=True)
30
+ self.objects = ObjectStore(self.root / "objects")
31
+ self.refs = RefStore(self.root / "refs")
32
+
33
+ def path(self, *parts: str) -> Path:
34
+ """A path under the store root for non-object data (IR, results, runs, traces)."""
35
+ target = self.root.joinpath(*parts).resolve()
36
+ if not target.is_relative_to(self.root.resolve()):
37
+ raise ValueError(f"path escapes the store: {'/'.join(parts)}")
38
+ return target
39
+
40
+ def model_dir(self, fingerprint: str) -> Path:
41
+ return self.path("models", fingerprint.replace(":", "_"))
42
+
43
+ # Non-object data goes through these, so a read-only lab can journal its writes.
44
+
45
+ def write_file(self, rel: str, data: bytes) -> None:
46
+ atomic_write(self.path(rel), data)
47
+
48
+ def read_file(self, rel: str) -> bytes | None:
49
+ target = self.path(rel)
50
+ return target.read_bytes() if target.is_file() else None
51
+
52
+ def exists(self, rel: str) -> bool:
53
+ return self.path(rel).is_file()
54
+
55
+ def glob(self, rel: str, pattern: str) -> list[str]:
56
+ """Relative paths of files under ``rel`` matching ``pattern``."""
57
+ base = self.path(rel)
58
+ if not base.is_dir():
59
+ return []
60
+ root = self.root.resolve()
61
+ return sorted(p.relative_to(root).as_posix() for p in base.glob(pattern) if p.is_file())
62
+
63
+ def __repr__(self) -> str:
64
+ return f"Store({str(self.root)!r})"
wspx/store/objects.py ADDED
@@ -0,0 +1,130 @@
1
+ """Content-addressed object storage.
2
+
3
+ Objects are immutable and keyed by ``sha256:<hex>`` of their bytes. Reads
4
+ re-hash by default, so silent corruption surfaces as :class:`IntegrityError`
5
+ instead of a wrong tensor. Writes are atomic (temp file + rename), which keeps
6
+ the store consistent across crashes and on network volumes.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import hashlib
12
+ import json
13
+ import os
14
+ import re
15
+ import tempfile
16
+ from collections.abc import Iterator
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ from wspx.errors import IntegrityError, StoreError
21
+
22
+ OID_PREFIX = "sha256:"
23
+ _OID_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
24
+ _HEX_PREFIX_RE = re.compile(r"^[0-9a-f]{4,64}$")
25
+
26
+
27
+ def oid_of(data: bytes) -> str:
28
+ return OID_PREFIX + hashlib.sha256(data).hexdigest()
29
+
30
+
31
+ def is_oid(text: str) -> bool:
32
+ return bool(_OID_RE.match(text))
33
+
34
+
35
+ def short(oid: str, length: int = 7) -> str:
36
+ """``sha256:5be01c7…`` → ``5be01c7``."""
37
+ return oid.removeprefix(OID_PREFIX)[:length]
38
+
39
+
40
+ def canonical_json(obj: Any) -> bytes:
41
+ """Deterministic JSON: sorted keys, no whitespace, no NaN. Hashes depend on it."""
42
+ return json.dumps(
43
+ obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
44
+ ).encode("utf-8")
45
+
46
+
47
+ def atomic_write(path: Path, data: bytes) -> None:
48
+ path.parent.mkdir(parents=True, exist_ok=True)
49
+ fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
50
+ try:
51
+ with os.fdopen(fd, "wb") as fh:
52
+ fh.write(data)
53
+ fh.flush()
54
+ os.fsync(fh.fileno())
55
+ os.replace(tmp, path)
56
+ except BaseException:
57
+ Path(tmp).unlink(missing_ok=True)
58
+ raise
59
+
60
+
61
+ class ObjectStore:
62
+ def __init__(self, root: str | Path) -> None:
63
+ self.root = Path(root)
64
+
65
+ def _path(self, oid: str) -> Path:
66
+ if not is_oid(oid):
67
+ raise StoreError(f"not an object id: {oid!r}")
68
+ hexd = oid[len(OID_PREFIX) :]
69
+ return self.root / hexd[:2] / hexd[2:]
70
+
71
+ def put(self, data: bytes) -> str:
72
+ oid = oid_of(data)
73
+ path = self._path(oid)
74
+ if not path.exists():
75
+ atomic_write(path, data)
76
+ return oid
77
+
78
+ def get(self, oid: str, *, verify: bool = True) -> bytes:
79
+ path = self._path(oid)
80
+ try:
81
+ data = path.read_bytes()
82
+ except FileNotFoundError:
83
+ raise StoreError(f"object {short(oid, 12)} not found") from None
84
+ if verify and oid_of(data) != oid:
85
+ raise IntegrityError(f"object {short(oid, 12)} is corrupt: content hash mismatch")
86
+ return data
87
+
88
+ def has(self, oid: str) -> bool:
89
+ return self._path(oid).exists()
90
+
91
+ def size(self, oid: str) -> int:
92
+ return self._path(oid).stat().st_size
93
+
94
+ def put_json(self, obj: Any) -> str:
95
+ return self.put(canonical_json(obj))
96
+
97
+ def get_json(self, oid: str) -> Any:
98
+ return json.loads(self.get(oid))
99
+
100
+ def __iter__(self) -> Iterator[str]:
101
+ if not self.root.exists():
102
+ return
103
+ for fan in sorted(self.root.iterdir()):
104
+ if fan.is_dir() and len(fan.name) == 2:
105
+ for obj in sorted(fan.iterdir()):
106
+ if not obj.name.startswith("."):
107
+ yield OID_PREFIX + fan.name + obj.name
108
+
109
+ def resolve_prefix(self, text: str) -> str:
110
+ """Expand ``5be01c7`` or ``sha256:5be0…`` to a full object id."""
111
+ prefix = text.removeprefix(OID_PREFIX).lower()
112
+ if is_oid(OID_PREFIX + prefix) and self.has(OID_PREFIX + prefix):
113
+ return OID_PREFIX + prefix
114
+ if not _HEX_PREFIX_RE.match(prefix):
115
+ raise StoreError(f"{text!r} is not an object id or prefix")
116
+ fan = self.root / prefix[:2]
117
+ matches = (
118
+ [
119
+ OID_PREFIX + prefix[:2] + p.name
120
+ for p in fan.iterdir()
121
+ if p.name.startswith(prefix[2:])
122
+ ]
123
+ if fan.is_dir()
124
+ else []
125
+ )
126
+ if not matches:
127
+ raise StoreError(f"no object matches {text!r}")
128
+ if len(matches) > 1:
129
+ raise StoreError(f"{text!r} is ambiguous ({len(matches)} objects); use more digits")
130
+ return matches[0]
wspx/store/overlay.py ADDED
@@ -0,0 +1,209 @@
1
+ """A store for a lab that may not write the store volume (§3, §6).
2
+
3
+ Reads come from the read-only volume (and anything this lab wrote since);
4
+ writes land in a local overlay and are recorded in a journal. After each tool
5
+ call the control plane (the store's single writer) takes the journal and
6
+ applies it, refs with compare-and-swap. Untrusted code in the lab therefore
7
+ never holds a writable handle on shared state.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import base64
13
+ import threading
14
+ from collections.abc import Iterator
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ from wspx.errors import StoreError
19
+ from wspx.store import Store
20
+ from wspx.store.objects import OID_PREFIX, ObjectStore, atomic_write, oid_of
21
+ from wspx.store.refs import ANY, SYMBOLIC_PREFIX, RefStore, _Any
22
+
23
+
24
+ class Journal:
25
+ def __init__(self) -> None:
26
+ self._lock = threading.Lock()
27
+ self.objects: dict[str, bytes] = {}
28
+ self.files: dict[str, bytes] = {}
29
+ self.refs: list[dict[str, Any]] = []
30
+
31
+ def drain(self) -> dict[str, Any]:
32
+ """The pending writes as JSON-safe data; clears the journal."""
33
+ with self._lock:
34
+ out = {
35
+ "objects": {k: base64.b64encode(v).decode() for k, v in self.objects.items()},
36
+ "files": {k: base64.b64encode(v).decode() for k, v in self.files.items()},
37
+ "refs": list(self.refs),
38
+ }
39
+ self.objects, self.files, self.refs = {}, {}, []
40
+ return out
41
+
42
+ def __bool__(self) -> bool:
43
+ return bool(self.objects or self.files or self.refs)
44
+
45
+
46
+ class _OverlayObjects(ObjectStore):
47
+ def __init__(self, base: ObjectStore, root: Path, journal: Journal) -> None:
48
+ super().__init__(root)
49
+ self.base = base
50
+ self.journal = journal
51
+
52
+ def put(self, data: bytes) -> str:
53
+ oid = oid_of(data)
54
+ if self.base.has(oid) or super().has(oid):
55
+ return oid
56
+ super().put(data)
57
+ with self.journal._lock:
58
+ self.journal.objects[oid] = data
59
+ return oid
60
+
61
+ def get(self, oid: str, *, verify: bool = True) -> bytes:
62
+ if super().has(oid):
63
+ return super().get(oid, verify=verify)
64
+ return self.base.get(oid, verify=verify)
65
+
66
+ def has(self, oid: str) -> bool:
67
+ return super().has(oid) or self.base.has(oid)
68
+
69
+ def size(self, oid: str) -> int:
70
+ return super().size(oid) if super().has(oid) else self.base.size(oid)
71
+
72
+ def __iter__(self) -> Iterator[str]:
73
+ seen = set(super().__iter__())
74
+ yield from sorted(seen | set(self.base))
75
+
76
+ def resolve_prefix(self, text: str) -> str:
77
+ found = set()
78
+ for resolve in (lambda t: ObjectStore.resolve_prefix(self, t), self.base.resolve_prefix):
79
+ try:
80
+ found.add(resolve(text))
81
+ except StoreError as exc:
82
+ if "ambiguous" in str(exc):
83
+ raise
84
+ if len(found) == 1:
85
+ return found.pop()
86
+ if not found:
87
+ raise StoreError(f"no object matches {text!r}")
88
+ raise StoreError(f"{text!r} is ambiguous; use more digits")
89
+
90
+
91
+ class _OverlayRefs(RefStore):
92
+ def __init__(self, base: RefStore, root: Path, journal: Journal) -> None:
93
+ super().__init__(root)
94
+ self.base = base
95
+ self.journal = journal
96
+
97
+ def exists(self, name: str) -> bool:
98
+ return super().exists(name) or self.base.exists(name)
99
+
100
+ def read_raw(self, name: str) -> str | None:
101
+ if super().exists(name):
102
+ return super().read_raw(name)
103
+ return self.base.read_raw(name)
104
+
105
+ def list(self, prefix: str = "heads/") -> dict[str, str]:
106
+ out = dict(self.base.list(prefix))
107
+ for name in super().list(prefix):
108
+ value = self.read(name)
109
+ if value is None:
110
+ out.pop(name, None)
111
+ else:
112
+ out[name] = value
113
+ return out
114
+
115
+ def update(self, name: str, new: str, expected: str | _Any | None = ANY) -> None:
116
+ target = self._resolve_symbolic_chain(name)
117
+ with self._locked(target):
118
+ current = self.read_raw(target)
119
+ if expected is not ANY and current != expected:
120
+ from wspx.errors import RefConflict
121
+
122
+ raise RefConflict(f"ref {target!r} moved: expected {expected}, found {current}")
123
+ atomic_write(self._path(target), f"{new}\n".encode())
124
+ self._record({"op": "update", "name": target, "value": new, "expected": current})
125
+
126
+ def set_symbolic(self, name: str, target: str) -> None:
127
+ super().set_symbolic(name, target)
128
+ self._record({"op": "symbolic", "name": name, "value": target})
129
+
130
+ def delete(self, name: str, expected: str | _Any | None = ANY) -> None:
131
+ with self._locked(name):
132
+ current = self.read_raw(name)
133
+ if expected is not ANY and current != expected:
134
+ from wspx.errors import RefConflict
135
+
136
+ raise RefConflict(f"ref {name!r} moved; refusing to delete")
137
+ atomic_write(self._path(name), b"") # tombstone over the base
138
+ self._record({"op": "delete", "name": name, "expected": current})
139
+
140
+ def _record(self, entry: dict[str, Any]) -> None:
141
+ with self.journal._lock:
142
+ self.journal.refs.append(entry)
143
+
144
+
145
+ class OverlayStore(Store):
146
+ def __init__(self, base: str | Path, overlay: str | Path) -> None:
147
+ self.base = Store.__new__(Store)
148
+ self.base.root = Path(base)
149
+ self.base.objects = ObjectStore(self.base.root / "objects")
150
+ self.base.refs = RefStore(self.base.root / "refs")
151
+ self.root = Path(overlay)
152
+ self.root.mkdir(parents=True, exist_ok=True)
153
+ self.journal = Journal()
154
+ self.objects = _OverlayObjects(self.base.objects, self.root / "objects", self.journal)
155
+ self.refs = _OverlayRefs(self.base.refs, self.root / "refs", self.journal)
156
+
157
+ def path(self, *parts: str) -> Path:
158
+ mine = super().path(*parts)
159
+ if mine.exists():
160
+ return mine
161
+ return self.base.path(*parts)
162
+
163
+ def write_file(self, rel: str, data: bytes) -> None:
164
+ atomic_write(Store.path(self, rel), data)
165
+ with self.journal._lock:
166
+ self.journal.files[rel] = data
167
+
168
+ def read_file(self, rel: str) -> bytes | None:
169
+ mine = Store.path(self, rel)
170
+ if mine.is_file():
171
+ return mine.read_bytes()
172
+ return self.base.read_file(rel)
173
+
174
+ def exists(self, rel: str) -> bool:
175
+ return Store.path(self, rel).is_file() or self.base.exists(rel)
176
+
177
+ def glob(self, rel: str, pattern: str) -> list[str]:
178
+ return sorted(set(Store.glob(self, rel, pattern)) | set(self.base.glob(rel, pattern)))
179
+
180
+
181
+ def apply_journal(store: Store, journal: dict[str, Any]) -> dict[str, int]:
182
+ """Persist a lab's journal into the real store (the control plane is the only writer)."""
183
+ for oid, b64 in journal.get("objects", {}).items():
184
+ data = base64.b64decode(b64)
185
+ if oid_of(data) != oid:
186
+ raise StoreError(f"journal object {oid[:19]} does not match its content")
187
+ store.objects.put(data)
188
+ for rel, b64 in journal.get("files", {}).items():
189
+ if rel.startswith(("objects/", "refs/")) or ".." in rel.split("/"):
190
+ raise StoreError(f"journal may not write {rel!r}")
191
+ store.write_file(rel, base64.b64decode(b64))
192
+ for entry in journal.get("refs", []):
193
+ name = entry["name"]
194
+ if entry["op"] == "symbolic":
195
+ store.refs.set_symbolic(name, entry["value"])
196
+ elif entry["op"] == "delete":
197
+ store.refs.delete(name, expected=entry.get("expected"))
198
+ else:
199
+ value = entry["value"]
200
+ if value.startswith(SYMBOLIC_PREFIX):
201
+ raise StoreError("journal ref updates must hold object ids")
202
+ if value and not value.startswith(OID_PREFIX):
203
+ raise StoreError(f"journal ref value {value!r} is not an object id")
204
+ store.refs.update(name, value, expected=entry.get("expected"))
205
+ return {
206
+ "objects": len(journal.get("objects", {})),
207
+ "files": len(journal.get("files", {})),
208
+ "refs": len(journal.get("refs", [])),
209
+ }