mindslice 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mindslice/__init__.py +58 -0
- mindslice/__main__.py +5 -0
- mindslice/baking/__init__.py +1 -0
- mindslice/baking/archive.py +66 -0
- mindslice/baking/bake.py +299 -0
- mindslice/baking/bake_batch.py +537 -0
- mindslice/baking/slice_transfer.py +265 -0
- mindslice/errors.py +104 -0
- mindslice/interfaces/__init__.py +1 -0
- mindslice/interfaces/bake_review.py +230 -0
- mindslice/interfaces/cli.py +392 -0
- mindslice/interfaces/cli_addressing.py +52 -0
- mindslice/interfaces/doctor.py +184 -0
- mindslice/interfaces/mcp_server.py +536 -0
- mindslice/interfaces/render.py +346 -0
- mindslice/interfaces/staging_review.py +92 -0
- mindslice/model/__init__.py +1 -0
- mindslice/model/addressing.py +251 -0
- mindslice/model/card.py +641 -0
- mindslice/model/lineage.py +228 -0
- mindslice/model/text.py +19 -0
- mindslice/model/tomlfmt.py +129 -0
- mindslice/operations/__init__.py +1 -0
- mindslice/operations/audit.py +195 -0
- mindslice/operations/boot.py +183 -0
- mindslice/operations/impact.py +98 -0
- mindslice/operations/recovery.py +113 -0
- mindslice/operations/staging.py +641 -0
- mindslice/operations/status.py +137 -0
- mindslice/operations/validation.py +250 -0
- mindslice/operations/writer.py +481 -0
- mindslice/retrieval/__init__.py +1 -0
- mindslice/retrieval/index.py +634 -0
- mindslice/retrieval/lexical.py +215 -0
- mindslice/retrieval/query.py +593 -0
- mindslice/storage/__init__.py +1 -0
- mindslice/storage/card_io.py +14 -0
- mindslice/storage/config.py +192 -0
- mindslice/storage/fsutil.py +141 -0
- mindslice/storage/history.py +182 -0
- mindslice/storage/locking.py +125 -0
- mindslice/storage/snapshots.py +262 -0
- mindslice/workspace.py +385 -0
- mindslice-0.1.0.dist-info/METADATA +98 -0
- mindslice-0.1.0.dist-info/RECORD +50 -0
- mindslice-0.1.0.dist-info/WHEEL +5 -0
- mindslice-0.1.0.dist-info/entry_points.txt +2 -0
- mindslice-0.1.0.dist-info/licenses/LICENSE +202 -0
- mindslice-0.1.0.dist-info/licenses/NOTICE +22 -0
- mindslice-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""``mounts.toml`` (workspace mount table) and ``slice.toml`` (slice manifest) handling."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import tomllib
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from ..model import tomlfmt
|
|
12
|
+
from ..model.addressing import normalize_alias
|
|
13
|
+
from ..errors import MountError
|
|
14
|
+
|
|
15
|
+
MOUNTS_FILE = "mounts.toml"
|
|
16
|
+
SLICE_FILE = "slice.toml"
|
|
17
|
+
OVERVIEW_PATH = "overview"
|
|
18
|
+
DESCRIPTION_CHARS = 160
|
|
19
|
+
|
|
20
|
+
DEFAULT_OVERLAY = {"path": "./overlay", "alias": "@overlay", "precedence": 100, "role": "runtime"}
|
|
21
|
+
|
|
22
|
+
_MOUNTS_HEADER = (
|
|
23
|
+
"MindSlice mount table.\n"
|
|
24
|
+
"precedence only decides which mount wins for unqualified paths (and breaks exact score ties);\n"
|
|
25
|
+
"role is opaque provenance metadata that callers may use as a trust policy (follow_supersedes)."
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Mount:
|
|
31
|
+
alias: str
|
|
32
|
+
path: str # as written in mounts.toml
|
|
33
|
+
precedence: int
|
|
34
|
+
role: str
|
|
35
|
+
kind: str # "overlay" | "slice"
|
|
36
|
+
root: Path # absolute, resolved directory
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def writable(self) -> bool:
|
|
40
|
+
return self.kind == "overlay"
|
|
41
|
+
|
|
42
|
+
def to_toml(self) -> dict[str, Any]:
|
|
43
|
+
return {"path": self.path, "alias": self.alias, "precedence": self.precedence, "role": self.role}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class MountTable:
|
|
48
|
+
overlay: Mount
|
|
49
|
+
mounts: list[Mount] = field(default_factory=list)
|
|
50
|
+
|
|
51
|
+
def all(self) -> list[Mount]:
|
|
52
|
+
"""All mounts, overlay included, ordered by precedence (desc) then alias."""
|
|
53
|
+
return sorted([self.overlay, *self.mounts], key=lambda m: (-m.precedence, m.alias))
|
|
54
|
+
|
|
55
|
+
def by_alias(self) -> dict[str, Mount]:
|
|
56
|
+
return {m.alias: m for m in self.all()}
|
|
57
|
+
|
|
58
|
+
def get(self, alias: str) -> Mount | None:
|
|
59
|
+
return self.by_alias().get(alias)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _resolve_root(workspace: Path, path: str) -> Path:
|
|
63
|
+
p = Path(os.path.expanduser(path))
|
|
64
|
+
if not p.is_absolute():
|
|
65
|
+
p = workspace / p
|
|
66
|
+
return Path(os.path.abspath(p))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _int(value: Any, what: str) -> int:
|
|
70
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
71
|
+
raise MountError(f"{what}: precedence must be an integer (got {value!r})")
|
|
72
|
+
return value
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _mount_from(entry: dict[str, Any], workspace: Path, kind: str, what: str) -> Mount:
|
|
76
|
+
if not isinstance(entry, dict):
|
|
77
|
+
raise MountError(f"{what}: expected a table")
|
|
78
|
+
path = entry.get("path")
|
|
79
|
+
if not isinstance(path, str) or not path.strip():
|
|
80
|
+
raise MountError(f"{what}: 'path' is required")
|
|
81
|
+
alias = normalize_alias(str(entry.get("alias") or ""))
|
|
82
|
+
role = entry.get("role", "runtime" if kind == "overlay" else "baseline")
|
|
83
|
+
if not isinstance(role, str):
|
|
84
|
+
raise MountError(f"{what}: role must be a string")
|
|
85
|
+
precedence = _int(entry.get("precedence", 100 if kind == "overlay" else 50), what)
|
|
86
|
+
return Mount(alias, path, precedence, role, kind, _resolve_root(workspace, path))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def load_mounts(workspace: Path) -> MountTable:
|
|
90
|
+
path = workspace / MOUNTS_FILE
|
|
91
|
+
data: dict[str, Any] = {}
|
|
92
|
+
if path.exists():
|
|
93
|
+
try:
|
|
94
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
95
|
+
except tomllib.TOMLDecodeError as exc:
|
|
96
|
+
raise MountError(f"{path}: invalid TOML: {exc}") from exc
|
|
97
|
+
overlay_entry = dict(DEFAULT_OVERLAY)
|
|
98
|
+
overlay_entry.update(data.get("overlay") or {})
|
|
99
|
+
overlay = _mount_from(overlay_entry, workspace, "overlay", "[overlay]")
|
|
100
|
+
mounts: list[Mount] = []
|
|
101
|
+
seen = {overlay.alias}
|
|
102
|
+
raw_mounts = data.get("mounts") or []
|
|
103
|
+
if not isinstance(raw_mounts, list):
|
|
104
|
+
raise MountError("mounts.toml: 'mounts' must be an array of tables ([[mounts]])")
|
|
105
|
+
for i, entry in enumerate(raw_mounts):
|
|
106
|
+
m = _mount_from(entry, workspace, "slice", f"[[mounts]] #{i + 1}")
|
|
107
|
+
if m.alias in seen:
|
|
108
|
+
raise MountError(f"mounts.toml: duplicate alias {m.alias}")
|
|
109
|
+
seen.add(m.alias)
|
|
110
|
+
mounts.append(m)
|
|
111
|
+
return MountTable(overlay, mounts)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def dump_mounts(table: MountTable) -> str:
|
|
115
|
+
return tomlfmt.dumps_document(
|
|
116
|
+
tables={"overlay": table.overlay.to_toml()},
|
|
117
|
+
array_tables={"mounts": [m.to_toml() for m in table.mounts]},
|
|
118
|
+
header=_MOUNTS_HEADER,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def save_mounts(workspace: Path, table: MountTable) -> str:
|
|
123
|
+
from .fsutil import atomic_write_text
|
|
124
|
+
|
|
125
|
+
text = dump_mounts(table)
|
|
126
|
+
atomic_write_text(workspace / MOUNTS_FILE, text)
|
|
127
|
+
return text
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def portable_path(workspace: Path, target: Path) -> str:
|
|
131
|
+
"""Prefer ``./relative`` paths for mounts living inside the workspace."""
|
|
132
|
+
target = Path(os.path.abspath(target))
|
|
133
|
+
try:
|
|
134
|
+
rel = target.relative_to(Path(os.path.abspath(workspace)))
|
|
135
|
+
except ValueError:
|
|
136
|
+
return str(target)
|
|
137
|
+
return "./" + rel.as_posix() if rel.as_posix() != "." else "."
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
# slice.toml
|
|
142
|
+
# ---------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass
|
|
146
|
+
class SliceManifest:
|
|
147
|
+
name: str = ""
|
|
148
|
+
version: str = ""
|
|
149
|
+
description: str = ""
|
|
150
|
+
alias: str = ""
|
|
151
|
+
dependencies: list[str] = field(default_factory=list)
|
|
152
|
+
extra: dict[str, Any] = field(default_factory=dict)
|
|
153
|
+
errors: list[str] = field(default_factory=list)
|
|
154
|
+
present: bool = False
|
|
155
|
+
|
|
156
|
+
def to_toml(self) -> dict[str, Any]:
|
|
157
|
+
out: dict[str, Any] = {}
|
|
158
|
+
for key in ("name", "version", "description", "alias"):
|
|
159
|
+
value = getattr(self, key)
|
|
160
|
+
if value:
|
|
161
|
+
out[key] = value
|
|
162
|
+
out["dependencies"] = list(self.dependencies)
|
|
163
|
+
out.update(self.extra)
|
|
164
|
+
return out
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def load_slice_manifest(root: Path) -> SliceManifest:
|
|
168
|
+
path = root / SLICE_FILE
|
|
169
|
+
if not path.is_file():
|
|
170
|
+
return SliceManifest()
|
|
171
|
+
try:
|
|
172
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
173
|
+
except (tomllib.TOMLDecodeError, UnicodeDecodeError) as exc:
|
|
174
|
+
return SliceManifest(errors=[f"invalid slice.toml: {exc}"], present=True)
|
|
175
|
+
sm = SliceManifest(present=True)
|
|
176
|
+
for key in ("name", "version", "description", "alias"):
|
|
177
|
+
value = data.pop(key, "")
|
|
178
|
+
if isinstance(value, str):
|
|
179
|
+
setattr(sm, key, value)
|
|
180
|
+
else:
|
|
181
|
+
sm.errors.append(f"slice.toml: {key!r} must be a string")
|
|
182
|
+
deps = data.pop("dependencies", [])
|
|
183
|
+
if isinstance(deps, list) and all(isinstance(d, str) for d in deps):
|
|
184
|
+
sm.dependencies = list(deps)
|
|
185
|
+
else:
|
|
186
|
+
sm.errors.append("slice.toml: 'dependencies' must be a list of strings")
|
|
187
|
+
sm.extra = data
|
|
188
|
+
return sm
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def dump_slice_manifest(sm: SliceManifest) -> str:
|
|
192
|
+
return tomlfmt.dumps_document(top=sm.to_toml())
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Crash-safe filesystem helpers (atomic writes, durable moves, directory walks)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import errno
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import tempfile
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Iterator
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def fsync_dir(path: Path) -> None:
|
|
14
|
+
"""fsync a directory so renames/unlinks inside it are durable (best effort on non-POSIX)."""
|
|
15
|
+
try:
|
|
16
|
+
fd = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
17
|
+
except OSError:
|
|
18
|
+
return
|
|
19
|
+
try:
|
|
20
|
+
os.fsync(fd)
|
|
21
|
+
except OSError:
|
|
22
|
+
pass
|
|
23
|
+
finally:
|
|
24
|
+
os.close(fd)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def fsync_file(path: Path) -> None:
|
|
28
|
+
"""Flush a file this process just wrote or copied, so its contents are durable before it is published.
|
|
29
|
+
|
|
30
|
+
The handle is opened for writing because Windows can only flush a writable handle: ``os.fsync`` on a
|
|
31
|
+
read-only one fails there with ``EBADF``, although POSIX accepts it.
|
|
32
|
+
"""
|
|
33
|
+
fd = os.open(path, os.O_RDWR | getattr(os, "O_BINARY", 0))
|
|
34
|
+
try:
|
|
35
|
+
os.fsync(fd)
|
|
36
|
+
finally:
|
|
37
|
+
os.close(fd)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def atomic_write_bytes(path: Path, data: bytes) -> None:
|
|
41
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
42
|
+
fd, tmp = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
43
|
+
try:
|
|
44
|
+
with os.fdopen(fd, "wb") as fh:
|
|
45
|
+
fh.write(data)
|
|
46
|
+
fh.flush()
|
|
47
|
+
os.fsync(fh.fileno())
|
|
48
|
+
os.replace(tmp, path)
|
|
49
|
+
except BaseException:
|
|
50
|
+
try:
|
|
51
|
+
os.unlink(tmp)
|
|
52
|
+
except FileNotFoundError:
|
|
53
|
+
pass
|
|
54
|
+
raise
|
|
55
|
+
fsync_dir(path.parent)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def atomic_write_text(path: Path, text: str) -> None:
|
|
59
|
+
atomic_write_bytes(path, text.encode("utf-8"))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def durable_move(src: Path, dst: Path) -> None:
|
|
63
|
+
"""Move ``src`` over ``dst`` atomically, falling back to copy+replace across devices."""
|
|
64
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
65
|
+
try:
|
|
66
|
+
os.replace(src, dst)
|
|
67
|
+
except OSError as exc:
|
|
68
|
+
if exc.errno != errno.EXDEV:
|
|
69
|
+
raise
|
|
70
|
+
atomic_write_bytes(dst, src.read_bytes())
|
|
71
|
+
os.unlink(src)
|
|
72
|
+
fsync_dir(src.parent)
|
|
73
|
+
fsync_dir(dst.parent)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def unlink_if_exists(path: Path) -> bool:
|
|
77
|
+
try:
|
|
78
|
+
os.unlink(path)
|
|
79
|
+
except FileNotFoundError:
|
|
80
|
+
return False
|
|
81
|
+
fsync_dir(path.parent)
|
|
82
|
+
return True
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def prune_empty_dirs(root: Path, start: Path) -> None:
|
|
86
|
+
"""Remove empty directories from ``start`` upward, stopping at (and keeping) ``root``."""
|
|
87
|
+
root = root.resolve()
|
|
88
|
+
current = start.resolve()
|
|
89
|
+
while current != root and root in current.parents:
|
|
90
|
+
try:
|
|
91
|
+
current.rmdir()
|
|
92
|
+
except OSError:
|
|
93
|
+
return
|
|
94
|
+
current = current.parent
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def iter_markdown(root: Path, exclude: tuple[Path, ...] = ()) -> Iterator[tuple[str, os.stat_result]]:
|
|
98
|
+
"""Yield ``(relative_posix_path_without_md, stat)`` for card files under ``root``.
|
|
99
|
+
|
|
100
|
+
Hidden files/directories (``.git``, ``.cache``, ``.manifest.json``...) are skipped,
|
|
101
|
+
as are any directories listed in ``exclude`` (e.g. a nested ``staging/``).
|
|
102
|
+
"""
|
|
103
|
+
if not root.is_dir():
|
|
104
|
+
return
|
|
105
|
+
excluded = {os.path.abspath(p) for p in exclude}
|
|
106
|
+
visited = {os.path.realpath(root)} # guards against directory symlink loops
|
|
107
|
+
stack = [root]
|
|
108
|
+
while stack:
|
|
109
|
+
current = stack.pop()
|
|
110
|
+
try:
|
|
111
|
+
entries = list(os.scandir(current))
|
|
112
|
+
except OSError:
|
|
113
|
+
continue
|
|
114
|
+
for entry in entries:
|
|
115
|
+
if entry.name.startswith("."):
|
|
116
|
+
continue
|
|
117
|
+
try:
|
|
118
|
+
if entry.is_dir(follow_symlinks=True):
|
|
119
|
+
real = os.path.realpath(entry.path)
|
|
120
|
+
if os.path.abspath(entry.path) not in excluded and real not in visited:
|
|
121
|
+
visited.add(real)
|
|
122
|
+
stack.append(Path(entry.path))
|
|
123
|
+
continue
|
|
124
|
+
if not entry.name.endswith(".md") or not entry.is_file(follow_symlinks=True):
|
|
125
|
+
continue
|
|
126
|
+
st = entry.stat(follow_symlinks=True)
|
|
127
|
+
except OSError:
|
|
128
|
+
continue
|
|
129
|
+
rel = Path(entry.path).relative_to(root).as_posix()[:-3]
|
|
130
|
+
yield rel, st
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def clear_directory(path: Path) -> None:
|
|
134
|
+
if not path.exists():
|
|
135
|
+
return
|
|
136
|
+
for entry in path.iterdir():
|
|
137
|
+
if entry.is_dir() and not entry.is_symlink():
|
|
138
|
+
shutil.rmtree(entry)
|
|
139
|
+
else:
|
|
140
|
+
entry.unlink()
|
|
141
|
+
fsync_dir(path)
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Engine-owned JSONL history and recoverable file mutations (no history database).
|
|
2
|
+
|
|
3
|
+
All methods run under the workspace lock. A pending journal contains the exact intended bytes;
|
|
4
|
+
public history contains metadata only. Event IDs make replay after an interrupted append idempotent.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import uuid
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from ..model.card import sha256_bytes
|
|
17
|
+
from ..errors import HistoryError
|
|
18
|
+
from .fsutil import atomic_write_text, fsync_dir, prune_empty_dirs, unlink_if_exists
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def event(kind: str, **fields: Any) -> dict[str, Any]:
|
|
22
|
+
return {"schema_version": 1, "event_id": uuid.uuid4().hex,
|
|
23
|
+
"timestamp": datetime.now(timezone.utc).isoformat(), "event": kind, **fields}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def change(kind: str, **fields: Any) -> dict[str, Any]:
|
|
27
|
+
return event(kind, change_id=uuid.uuid4().hex, **fields)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def transitions(kind: str, ids: list[str], **fields: Any) -> list[dict[str, Any]]:
|
|
31
|
+
transition_id = uuid.uuid4().hex
|
|
32
|
+
return [event(kind, change_id=id, transition_id=transition_id, **fields) for id in ids]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def file_sha(path: Path) -> str | None:
|
|
36
|
+
try:
|
|
37
|
+
return sha256_bytes(path.read_bytes())
|
|
38
|
+
except FileNotFoundError:
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def file_action(path: Path, text: str | None, prune_root: Path | None = None) -> dict[str, Any]:
|
|
43
|
+
return {"path": str(path.absolute()), "before_sha256": file_sha(path), "text": text,
|
|
44
|
+
"after_sha256": sha256_bytes(text.encode("utf-8")) if text is not None else None,
|
|
45
|
+
"prune_root": str(prune_root) if prune_root else None}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ChangeHistory:
|
|
49
|
+
def __init__(self, root: Path) -> None:
|
|
50
|
+
self.root = root
|
|
51
|
+
self.path = root / "changes.jsonl"
|
|
52
|
+
self.journal_path = root / ".pending.json"
|
|
53
|
+
|
|
54
|
+
def has_journal(self) -> bool:
|
|
55
|
+
return self.journal_path.exists()
|
|
56
|
+
|
|
57
|
+
def _read(self) -> tuple[list[dict[str, Any]], int]:
|
|
58
|
+
try:
|
|
59
|
+
data = self.path.read_bytes()
|
|
60
|
+
except FileNotFoundError:
|
|
61
|
+
return [], 0
|
|
62
|
+
events = []
|
|
63
|
+
offset = 0
|
|
64
|
+
for line in data.splitlines(keepends=True):
|
|
65
|
+
if not line.endswith(b"\n"):
|
|
66
|
+
break # an interrupted last append; replay rewrites it from the durable journal
|
|
67
|
+
try:
|
|
68
|
+
item = json.loads(line)
|
|
69
|
+
if not isinstance(item, dict) or not isinstance(item.get("event_id"), str):
|
|
70
|
+
raise ValueError("missing event_id")
|
|
71
|
+
except (ValueError, UnicodeDecodeError) as exc:
|
|
72
|
+
raise HistoryError(f"invalid history record at byte {offset}: {self.path}", path=str(self.path)) from exc
|
|
73
|
+
events.append(item)
|
|
74
|
+
offset += len(line)
|
|
75
|
+
return events, offset
|
|
76
|
+
|
|
77
|
+
def events(self) -> list[dict[str, Any]]:
|
|
78
|
+
return self._read()[0]
|
|
79
|
+
|
|
80
|
+
def append(self, events: list[dict[str, Any]]) -> None:
|
|
81
|
+
if not events:
|
|
82
|
+
return
|
|
83
|
+
existing, complete_bytes = self._read()
|
|
84
|
+
seen = {e["event_id"]: e for e in existing}
|
|
85
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
with self.path.open("a+b") as stream:
|
|
87
|
+
stream.truncate(complete_bytes)
|
|
88
|
+
for item in events:
|
|
89
|
+
old = seen.get(item["event_id"])
|
|
90
|
+
if old is not None:
|
|
91
|
+
if old != item:
|
|
92
|
+
raise HistoryError("history event ID collision", event_id=item["event_id"])
|
|
93
|
+
continue
|
|
94
|
+
stream.write((json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8"))
|
|
95
|
+
seen[item["event_id"]] = item
|
|
96
|
+
stream.flush()
|
|
97
|
+
os.fsync(stream.fileno())
|
|
98
|
+
fsync_dir(self.root)
|
|
99
|
+
|
|
100
|
+
def transact(self, actions: list[dict[str, Any]], events: list[dict[str, Any]],
|
|
101
|
+
*, guards: list[dict[str, Any]] | None = None) -> None:
|
|
102
|
+
if self.has_journal():
|
|
103
|
+
raise HistoryError("an unfinished history transaction must be recovered first", path=str(self.journal_path))
|
|
104
|
+
# Validate existing history before changing any content.
|
|
105
|
+
self.events()
|
|
106
|
+
journal = {"version": 1, "actions": actions, "events": events, "guards": guards or []}
|
|
107
|
+
atomic_write_text(self.journal_path, json.dumps(journal, ensure_ascii=False) + "\n")
|
|
108
|
+
self.recover()
|
|
109
|
+
|
|
110
|
+
def recover(self) -> bool:
|
|
111
|
+
try:
|
|
112
|
+
journal = json.loads(self.journal_path.read_text(encoding="utf-8"))
|
|
113
|
+
if journal.get("version") != 1 or not isinstance(journal.get("actions"), list) or not isinstance(journal.get("events"), list):
|
|
114
|
+
raise ValueError("invalid journal structure")
|
|
115
|
+
except FileNotFoundError:
|
|
116
|
+
return False
|
|
117
|
+
except (ValueError, UnicodeDecodeError, AttributeError) as exc:
|
|
118
|
+
raise HistoryError("invalid history journal; preserved for inspection", path=str(self.journal_path)) from exc
|
|
119
|
+
self.events()
|
|
120
|
+
conflicts = []
|
|
121
|
+
for guard in journal.get("guards", []):
|
|
122
|
+
if file_sha(Path(guard["path"])) != guard["sha256"]:
|
|
123
|
+
conflicts.append({"path": guard["path"], "reason": "baked output changed or missing"})
|
|
124
|
+
for action in journal["actions"]:
|
|
125
|
+
actual = file_sha(Path(action["path"]))
|
|
126
|
+
if actual not in (action["before_sha256"], action["after_sha256"]):
|
|
127
|
+
conflicts.append({"path": action["path"], "current_sha256": actual})
|
|
128
|
+
if conflicts:
|
|
129
|
+
raise HistoryError("history recovery stopped: files changed outside the transaction; "
|
|
130
|
+
"files and journal were preserved", conflicts=conflicts, journal=str(self.journal_path))
|
|
131
|
+
for action in journal["actions"]:
|
|
132
|
+
path = Path(action["path"])
|
|
133
|
+
if file_sha(path) != action["after_sha256"]:
|
|
134
|
+
if action["text"] is None:
|
|
135
|
+
unlink_if_exists(path)
|
|
136
|
+
else:
|
|
137
|
+
atomic_write_text(path, action["text"])
|
|
138
|
+
if action["text"] is None and action.get("prune_root"):
|
|
139
|
+
prune_empty_dirs(Path(action["prune_root"]), path.parent)
|
|
140
|
+
self.append(journal["events"])
|
|
141
|
+
unlink_if_exists(self.journal_path) # last: a crash after append must only deduplicate events
|
|
142
|
+
return True
|
|
143
|
+
|
|
144
|
+
def details(self) -> dict[str, dict[str, Any]]:
|
|
145
|
+
found = {}
|
|
146
|
+
for item in self.events():
|
|
147
|
+
id = item.get("change_id")
|
|
148
|
+
if id and id not in found and item["event"] in ("proposed", "applied"):
|
|
149
|
+
found[id] = {k: item[k] for k in ("change_id", "timestamp", "op", "reason", "before_sha256", "after_sha256")
|
|
150
|
+
if k in item}
|
|
151
|
+
return found
|
|
152
|
+
|
|
153
|
+
def descriptions(self, ids: list[str], *, records=None) -> list[dict[str, Any]]:
|
|
154
|
+
found = self.details() if records is None else records
|
|
155
|
+
return [found[id] for id in ids if id in found]
|
|
156
|
+
|
|
157
|
+
def applied(self) -> dict[str, tuple[str | None, list[str]]]:
|
|
158
|
+
"""Active overlay revision chains; external edits break ancestry instead of inheriting stale reasons."""
|
|
159
|
+
active: dict[str, tuple[str | None, list[str]]] = {}
|
|
160
|
+
groups: dict[str, str | None] = {}
|
|
161
|
+
for item in self.events():
|
|
162
|
+
if item["event"] == "applied":
|
|
163
|
+
address = item["address"]
|
|
164
|
+
tip, ids = active.get(address, (None, []))
|
|
165
|
+
group = item.get("transition_id") or item["event_id"]
|
|
166
|
+
if groups.get(address) != group and tip != item.get("before_sha256"):
|
|
167
|
+
ids = []
|
|
168
|
+
ids = list(ids)
|
|
169
|
+
if item["change_id"] not in ids:
|
|
170
|
+
ids.append(item["change_id"])
|
|
171
|
+
after = item.get("after_sha256")
|
|
172
|
+
active[address] = (after, ids if after is not None else [])
|
|
173
|
+
groups[address] = group
|
|
174
|
+
elif item["event"] in ("baked", "retired"):
|
|
175
|
+
for address in item.get("retired_overlay_cards", []):
|
|
176
|
+
active.pop(address, None)
|
|
177
|
+
groups.pop(address, None)
|
|
178
|
+
return active
|
|
179
|
+
|
|
180
|
+
def changes_for(self, address: str, sha: str | None) -> list[dict[str, Any]]:
|
|
181
|
+
tip, ids = self.applied().get(address, (None, []))
|
|
182
|
+
return self.descriptions(ids) if tip == sha and sha is not None else []
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Reader/writer workspace lock on ``.cache/workspace.lock``.
|
|
2
|
+
|
|
3
|
+
POSIX uses ``fcntl.flock`` (``LOCK_SH`` for readers, ``LOCK_EX`` for writers).
|
|
4
|
+
On Windows an optional ``msvcrt`` adapter takes a byte-range lock; it cannot
|
|
5
|
+
express shared locks, so every acquisition is exclusive there.
|
|
6
|
+
|
|
7
|
+
The lock is re-entrant per ``WorkspaceLock`` instance and serializes threads of
|
|
8
|
+
the same process with an ``RLock``. Requesting ``exclusive`` while holding
|
|
9
|
+
``shared`` converts the lock (flock conversion is not atomic, so callers must
|
|
10
|
+
re-check state after upgrading — the workspace does).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import errno
|
|
16
|
+
import os
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Iterator
|
|
22
|
+
|
|
23
|
+
from ..errors import LockTimeoutError
|
|
24
|
+
|
|
25
|
+
try: # POSIX
|
|
26
|
+
import fcntl
|
|
27
|
+
|
|
28
|
+
_HAVE_FCNTL = True
|
|
29
|
+
except ImportError: # pragma: no cover - Windows
|
|
30
|
+
fcntl = None # type: ignore[assignment]
|
|
31
|
+
_HAVE_FCNTL = False
|
|
32
|
+
import msvcrt # type: ignore[import-not-found]
|
|
33
|
+
|
|
34
|
+
SHARED = "shared"
|
|
35
|
+
EXCLUSIVE = "exclusive"
|
|
36
|
+
_CONTENDED = {errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK, errno.EDEADLK}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class WorkspaceLock:
|
|
40
|
+
def __init__(self, path: Path, timeout: float = 30.0) -> None:
|
|
41
|
+
self.path = path
|
|
42
|
+
self.timeout = timeout
|
|
43
|
+
self._fd: int | None = None
|
|
44
|
+
self._mode: str | None = None
|
|
45
|
+
self._depth = 0
|
|
46
|
+
self._thread_lock = threading.RLock()
|
|
47
|
+
|
|
48
|
+
# -- low level -------------------------------------------------------
|
|
49
|
+
def _open(self) -> int:
|
|
50
|
+
if self._fd is None:
|
|
51
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
self._fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o644)
|
|
53
|
+
return self._fd
|
|
54
|
+
|
|
55
|
+
def _acquire(self, mode: str) -> None:
|
|
56
|
+
fd = self._open()
|
|
57
|
+
deadline = time.monotonic() + self.timeout
|
|
58
|
+
delay = 0.005
|
|
59
|
+
while True:
|
|
60
|
+
try:
|
|
61
|
+
if _HAVE_FCNTL:
|
|
62
|
+
flag = fcntl.LOCK_SH if mode == SHARED else fcntl.LOCK_EX
|
|
63
|
+
fcntl.flock(fd, flag | fcntl.LOCK_NB)
|
|
64
|
+
else: # pragma: no cover - Windows
|
|
65
|
+
if self._mode is None:
|
|
66
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
67
|
+
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
|
|
68
|
+
self._mode = mode
|
|
69
|
+
return
|
|
70
|
+
except OSError as exc:
|
|
71
|
+
if exc.errno not in _CONTENDED:
|
|
72
|
+
raise
|
|
73
|
+
if time.monotonic() >= deadline:
|
|
74
|
+
raise LockTimeoutError(f"timed out waiting for {mode} lock on {self.path}") from None
|
|
75
|
+
time.sleep(delay)
|
|
76
|
+
delay = min(delay * 2, 0.1)
|
|
77
|
+
|
|
78
|
+
def _release(self) -> None:
|
|
79
|
+
if self._fd is None or self._mode is None:
|
|
80
|
+
return
|
|
81
|
+
if _HAVE_FCNTL:
|
|
82
|
+
fcntl.flock(self._fd, fcntl.LOCK_UN)
|
|
83
|
+
else: # pragma: no cover - Windows
|
|
84
|
+
os.lseek(self._fd, 0, os.SEEK_SET)
|
|
85
|
+
msvcrt.locking(self._fd, msvcrt.LK_UNLCK, 1)
|
|
86
|
+
self._mode = None
|
|
87
|
+
|
|
88
|
+
# -- public API ------------------------------------------------------
|
|
89
|
+
@property
|
|
90
|
+
def mode(self) -> str | None:
|
|
91
|
+
return self._mode
|
|
92
|
+
|
|
93
|
+
@contextmanager
|
|
94
|
+
def hold(self, mode: str) -> Iterator[str]:
|
|
95
|
+
"""Hold the lock in ``mode`` (re-entrant; upgrades shared -> exclusive when nested)."""
|
|
96
|
+
with self._thread_lock:
|
|
97
|
+
previous = self._mode
|
|
98
|
+
if previous is None:
|
|
99
|
+
self._acquire(mode)
|
|
100
|
+
elif previous == SHARED and mode == EXCLUSIVE:
|
|
101
|
+
self._acquire(EXCLUSIVE)
|
|
102
|
+
self._depth += 1
|
|
103
|
+
try:
|
|
104
|
+
yield self._mode or mode
|
|
105
|
+
finally:
|
|
106
|
+
self._depth -= 1
|
|
107
|
+
if previous is None:
|
|
108
|
+
self._release()
|
|
109
|
+
elif previous == SHARED and self._mode == EXCLUSIVE:
|
|
110
|
+
self._acquire(SHARED)
|
|
111
|
+
|
|
112
|
+
def shared(self):
|
|
113
|
+
return self.hold(SHARED)
|
|
114
|
+
|
|
115
|
+
def exclusive(self):
|
|
116
|
+
return self.hold(EXCLUSIVE)
|
|
117
|
+
|
|
118
|
+
def close(self) -> None:
|
|
119
|
+
with self._thread_lock:
|
|
120
|
+
if self._depth:
|
|
121
|
+
return
|
|
122
|
+
self._release()
|
|
123
|
+
if self._fd is not None:
|
|
124
|
+
os.close(self._fd)
|
|
125
|
+
self._fd = None
|