sequentialist-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sequentialist/core/__init__.py +84 -0
- sequentialist/core/audit.py +134 -0
- sequentialist/core/canonical.py +50 -0
- sequentialist/core/cli.py +95 -0
- sequentialist/core/engine.py +589 -0
- sequentialist/core/errors.py +40 -0
- sequentialist/core/privacy.py +30 -0
- sequentialist/core/py.typed +0 -0
- sequentialist/core/replay.py +40 -0
- sequentialist/core/report.py +141 -0
- sequentialist/core/schema.py +410 -0
- sequentialist/core/simulate.py +58 -0
- sequentialist/core/sinks.py +98 -0
- sequentialist/core/spec.py +120 -0
- sequentialist/core/storage/__init__.py +24 -0
- sequentialist/core/storage/_lock.py +91 -0
- sequentialist/core/storage/base.py +102 -0
- sequentialist/core/storage/files.py +174 -0
- sequentialist/core/storage/memory.py +64 -0
- sequentialist/core/storage/sqlite.py +150 -0
- sequentialist/core/version.py +4 -0
- sequentialist_core-0.1.0.dist-info/METADATA +129 -0
- sequentialist_core-0.1.0.dist-info/RECORD +28 -0
- sequentialist_core-0.1.0.dist-info/WHEEL +5 -0
- sequentialist_core-0.1.0.dist-info/entry_points.txt +6 -0
- sequentialist_core-0.1.0.dist-info/licenses/COMMERCIAL.md +89 -0
- sequentialist_core-0.1.0.dist-info/licenses/LICENSE +661 -0
- sequentialist_core-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 Novikov Laboratories LLC
|
|
3
|
+
# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
|
|
4
|
+
"""Sinks: where audit entries (decisions, alarms, actions) are announced.
|
|
5
|
+
|
|
6
|
+
The audit log is the record; sinks are notifications. The engine calls
|
|
7
|
+
every sink with every new entry after the checkpoint that made it
|
|
8
|
+
durable. A failing sink is logged and never stops a task. A webhook is
|
|
9
|
+
sent from a background thread, with retries, so a slow receiver does not
|
|
10
|
+
hold up the data.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
import queue
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import urllib.request
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Callable, Dict, Sequence
|
|
22
|
+
|
|
23
|
+
from .audit import AuditEntry
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("sequentialist")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class LogSink:
|
|
29
|
+
"""Logs entries of the given kinds (all but "created" and "segment" by default) as warnings."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, kinds: Sequence[str] = ("decision", "action"), logger_name: str = "sequentialist"):
|
|
32
|
+
self.kinds, self.log = tuple(kinds), logging.getLogger(logger_name)
|
|
33
|
+
|
|
34
|
+
def __call__(self, entry: AuditEntry) -> None:
|
|
35
|
+
if entry.kind in self.kinds:
|
|
36
|
+
self.log.warning("%s %s: %s %s", entry.task_id, entry.kind, entry.decision or "", entry.basis)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class JsonlSink:
|
|
40
|
+
"""Appends every entry as a JSON line to a file."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, path: str | Path):
|
|
43
|
+
self.path = Path(path)
|
|
44
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
self._lock = threading.Lock()
|
|
46
|
+
|
|
47
|
+
def __call__(self, entry: AuditEntry) -> None:
|
|
48
|
+
line = json.dumps(entry.to_json(), sort_keys=True, ensure_ascii=False) + "\n"
|
|
49
|
+
with self._lock, open(self.path, "a", encoding="utf-8") as f:
|
|
50
|
+
f.write(line)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
Post = Callable[[str, bytes, Dict[str, str], float], None]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _post(url: str, body: bytes, headers: dict[str, str], timeout: float) -> None:
|
|
57
|
+
request = urllib.request.Request(url, data=body, headers=headers, method="POST")
|
|
58
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
59
|
+
response.read()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class WebhookSink:
|
|
63
|
+
"""POSTs entries of the given kinds as JSON; retries with exponential backoff, gives up after `attempts`."""
|
|
64
|
+
|
|
65
|
+
def __init__(self, url: str, *, kinds: Sequence[str] = ("decision", "action"), timeout: float = 5.0,
|
|
66
|
+
attempts: int = 5, headers: dict[str, str] | None = None, post: Post | None = None):
|
|
67
|
+
self.url, self.kinds, self.timeout, self.attempts = url, tuple(kinds), float(timeout), int(attempts)
|
|
68
|
+
self.headers = {"Content-Type": "application/json", **(headers or {})}
|
|
69
|
+
self._post = post or _post
|
|
70
|
+
self._queue: queue.Queue = queue.Queue()
|
|
71
|
+
self._thread = threading.Thread(target=self._work, name="sequentialist-webhook", daemon=True)
|
|
72
|
+
self._thread.start()
|
|
73
|
+
|
|
74
|
+
def __call__(self, entry: AuditEntry) -> None:
|
|
75
|
+
if entry.kind in self.kinds:
|
|
76
|
+
self._queue.put(entry)
|
|
77
|
+
|
|
78
|
+
def _work(self) -> None:
|
|
79
|
+
while True:
|
|
80
|
+
entry = self._queue.get()
|
|
81
|
+
if entry is None:
|
|
82
|
+
return
|
|
83
|
+
body = json.dumps(entry.to_json()).encode("utf-8")
|
|
84
|
+
for attempt in range(self.attempts):
|
|
85
|
+
try:
|
|
86
|
+
self._post(self.url, body, self.headers, self.timeout)
|
|
87
|
+
break
|
|
88
|
+
except Exception as error:
|
|
89
|
+
if attempt + 1 == self.attempts:
|
|
90
|
+
logger.error("webhook %s: entry %d of task %s not delivered: %s", self.url, entry.n,
|
|
91
|
+
entry.task_id, error)
|
|
92
|
+
else:
|
|
93
|
+
time.sleep(min(60.0, 0.5 * 2**attempt))
|
|
94
|
+
|
|
95
|
+
def close(self, timeout: float = 10.0) -> None:
|
|
96
|
+
"""Send what is queued, then stop."""
|
|
97
|
+
self._queue.put(None)
|
|
98
|
+
self._thread.join(timeout)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 Novikov Laboratories LLC
|
|
3
|
+
# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
|
|
4
|
+
"""Specs: the declarative description of a task (an experiment, a monitor, a trial).
|
|
5
|
+
|
|
6
|
+
A kind of spec is a subclass of `Spec` with a `kind` (``"ab.experiment/1"``:
|
|
7
|
+
the name and the version of the format) and a `schema`. Reading a spec
|
|
8
|
+
checks it against the schema, then against the subclass's own rules
|
|
9
|
+
(`problems`), and reports every error found at once, each with its JSON
|
|
10
|
+
pointer::
|
|
11
|
+
|
|
12
|
+
spec = load_spec("experiment.yaml", ExperimentSpec)
|
|
13
|
+
spec.data # the normal form: plain JSON, defaults filled in
|
|
14
|
+
spec_id(spec) # SHA-256 of its canonical JSON
|
|
15
|
+
|
|
16
|
+
JSON needs nothing; YAML needs PyYAML (``pip install "sequentialist-core[yaml]"``).
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, ClassVar, Mapping, TypeVar
|
|
23
|
+
|
|
24
|
+
from .canonical import digest
|
|
25
|
+
from .errors import SpecError
|
|
26
|
+
from .schema import INVALID, Errors, Field, Object, String
|
|
27
|
+
|
|
28
|
+
S = TypeVar("S", bound="Spec")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Spec:
|
|
32
|
+
"""Base class of every kind of spec. Subclasses set `kind` and `schema` and may override `problems`."""
|
|
33
|
+
|
|
34
|
+
kind: ClassVar[str] = ""
|
|
35
|
+
schema: ClassVar[Object]
|
|
36
|
+
|
|
37
|
+
def __init__(self, data: dict[str, Any]):
|
|
38
|
+
"""`data` is the normal form (as `parse_spec` returns it, without "kind")."""
|
|
39
|
+
self.data = data
|
|
40
|
+
|
|
41
|
+
def problems(self) -> list[tuple[str, str]]:
|
|
42
|
+
"""Errors the schema cannot express (relations between members), as (JSON pointer, message)."""
|
|
43
|
+
return []
|
|
44
|
+
|
|
45
|
+
def to_json(self) -> dict[str, Any]:
|
|
46
|
+
return {"kind": self.kind, **self.data}
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def id(self) -> str:
|
|
50
|
+
"""The spec id: SHA-256 of the canonical JSON of the normal form."""
|
|
51
|
+
return spec_id(self)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _full_schema(cls: type[Spec]) -> Object:
|
|
55
|
+
fields = {"kind": Field(String(enum=[cls.kind]), doc=f'The format of the spec: "{cls.kind}".')}
|
|
56
|
+
fields.update(cls.schema.fields)
|
|
57
|
+
return Object(fields, doc=cls.schema.doc)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def parse_spec(data: Any, cls: type[S], source: str = "") -> S:
|
|
61
|
+
"""A spec of kind `cls` from parsed JSON or YAML; SpecError lists every problem."""
|
|
62
|
+
errors: Errors = []
|
|
63
|
+
if isinstance(data, dict) and data.get("kind") != cls.kind:
|
|
64
|
+
errors.append(("/kind", f"expected {cls.kind!r}, got {data.get('kind')!r}"))
|
|
65
|
+
raise SpecError(errors, source)
|
|
66
|
+
normal = _full_schema(cls).check(data, "", errors)
|
|
67
|
+
if errors or normal is INVALID:
|
|
68
|
+
raise SpecError(errors, source)
|
|
69
|
+
del normal["kind"]
|
|
70
|
+
spec = cls(normal)
|
|
71
|
+
errors = list(spec.problems())
|
|
72
|
+
if errors:
|
|
73
|
+
raise SpecError(errors, source)
|
|
74
|
+
return spec
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def read_document(source: str | Path) -> Any:
|
|
78
|
+
"""Parse a JSON or YAML file (by its extension) into plain values."""
|
|
79
|
+
path = Path(source)
|
|
80
|
+
text = path.read_text(encoding="utf-8")
|
|
81
|
+
if path.suffix.lower() in (".yaml", ".yml"):
|
|
82
|
+
try:
|
|
83
|
+
import yaml # type: ignore[import-untyped]
|
|
84
|
+
except ImportError:
|
|
85
|
+
raise SpecError([("", "reading YAML needs PyYAML: pip install \"sequentialist-core[yaml]\"")],
|
|
86
|
+
str(path)) from None
|
|
87
|
+
try:
|
|
88
|
+
return yaml.safe_load(text)
|
|
89
|
+
except yaml.YAMLError as error:
|
|
90
|
+
raise SpecError([("", f"not valid YAML: {error}")], str(path)) from None
|
|
91
|
+
try:
|
|
92
|
+
return json.loads(text, parse_constant=_no_constant)
|
|
93
|
+
except ValueError as error:
|
|
94
|
+
raise SpecError([("", f"not valid JSON: {error}")], str(path)) from None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _no_constant(name: str) -> Any:
|
|
98
|
+
raise ValueError(f"{name} is not a JSON number")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def load_spec(source: str | Path | Mapping[str, Any], cls: type[S]) -> S:
|
|
102
|
+
"""A spec from a file (.json, .yaml, .yml) or from an already parsed mapping."""
|
|
103
|
+
if isinstance(source, Mapping):
|
|
104
|
+
return parse_spec(dict(source), cls)
|
|
105
|
+
return parse_spec(read_document(source), cls, str(source))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def spec_id(spec: Spec) -> str:
|
|
109
|
+
return digest(spec.to_json())
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def json_schema(cls: type[Spec]) -> dict[str, Any]:
|
|
113
|
+
"""JSON Schema (draft 2020-12) of the spec format: for editors, other languages and validators."""
|
|
114
|
+
out = _full_schema(cls).json_schema()
|
|
115
|
+
return {
|
|
116
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
117
|
+
"$id": f"https://github.com/andre-a-no/sequentialist/schemas/{cls.kind.replace('/', '-')}.schema.json",
|
|
118
|
+
"title": cls.kind,
|
|
119
|
+
**out,
|
|
120
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 Novikov Laboratories LLC
|
|
3
|
+
# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
|
|
4
|
+
"""Storages of tasks: in memory, in a directory of files, in SQLite. Postgres follows in a later version."""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from .base import Storage, check_task_id
|
|
8
|
+
from .files import FileStorage
|
|
9
|
+
from .memory import MemoryStorage
|
|
10
|
+
from .sqlite import SQLiteStorage
|
|
11
|
+
|
|
12
|
+
__all__ = ["FileStorage", "MemoryStorage", "SQLiteStorage", "Storage", "check_task_id", "open_storage"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def open_storage(url: str, *, sync: str = "fsync") -> Storage:
|
|
16
|
+
"""A storage from a short description: ``memory:``, ``files:<directory>`` or ``sqlite:<file>``."""
|
|
17
|
+
kind, _, where = url.partition(":")
|
|
18
|
+
if kind == "memory" and not where:
|
|
19
|
+
return MemoryStorage()
|
|
20
|
+
if kind == "files" and where:
|
|
21
|
+
return FileStorage(where, sync=sync)
|
|
22
|
+
if kind == "sqlite" and where:
|
|
23
|
+
return SQLiteStorage(where, sync=sync)
|
|
24
|
+
raise ValueError(f"unknown storage {url!r}: use memory:, files:<directory> or sqlite:<file>")
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 Novikov Laboratories LLC
|
|
3
|
+
# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
|
|
4
|
+
"""Exclusive locks on files, released by the operating system when the process ends.
|
|
5
|
+
|
|
6
|
+
A crashed writer leaves no stale lock behind. Two opens of the same lock
|
|
7
|
+
file conflict even within one process, so a task cannot be opened twice by
|
|
8
|
+
mistake either.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import threading
|
|
15
|
+
|
|
16
|
+
from ..errors import TaskLocked
|
|
17
|
+
|
|
18
|
+
if sys.platform == "win32": # pragma: no cover - exercised on Windows only
|
|
19
|
+
import msvcrt
|
|
20
|
+
|
|
21
|
+
def _try_lock(fd: int) -> bool:
|
|
22
|
+
try:
|
|
23
|
+
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
|
|
24
|
+
except OSError:
|
|
25
|
+
return False
|
|
26
|
+
return True
|
|
27
|
+
|
|
28
|
+
def _unlock(fd: int) -> None:
|
|
29
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
30
|
+
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
|
|
31
|
+
else:
|
|
32
|
+
import fcntl
|
|
33
|
+
|
|
34
|
+
def _try_lock(fd: int) -> bool:
|
|
35
|
+
try:
|
|
36
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
37
|
+
except OSError:
|
|
38
|
+
return False
|
|
39
|
+
return True
|
|
40
|
+
|
|
41
|
+
def _unlock(fd: int) -> None:
|
|
42
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Lock:
|
|
46
|
+
"""Held from creation until `release`."""
|
|
47
|
+
|
|
48
|
+
def release(self) -> None: # pragma: no cover - abstract
|
|
49
|
+
raise NotImplementedError
|
|
50
|
+
|
|
51
|
+
def __enter__(self) -> Lock:
|
|
52
|
+
return self
|
|
53
|
+
|
|
54
|
+
def __exit__(self, *exc: object) -> None:
|
|
55
|
+
self.release()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class FileLock(Lock):
|
|
59
|
+
def __init__(self, path: str, what: str):
|
|
60
|
+
self.path = path
|
|
61
|
+
fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o644)
|
|
62
|
+
if not _try_lock(fd):
|
|
63
|
+
os.close(fd)
|
|
64
|
+
raise TaskLocked(f"{what} is open in another writer (lock {path})")
|
|
65
|
+
self._fd: int = fd
|
|
66
|
+
|
|
67
|
+
def release(self) -> None:
|
|
68
|
+
if self._fd >= 0:
|
|
69
|
+
try:
|
|
70
|
+
_unlock(self._fd)
|
|
71
|
+
finally:
|
|
72
|
+
os.close(self._fd)
|
|
73
|
+
self._fd = -1
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class MemoryLock(Lock):
|
|
77
|
+
"""A lock among the users of one object in one process (for in-memory storage)."""
|
|
78
|
+
|
|
79
|
+
def __init__(self, held: set[str], guard: threading.Lock, key: str, what: str):
|
|
80
|
+
with guard:
|
|
81
|
+
if key in held:
|
|
82
|
+
raise TaskLocked(f"{what} is open in another writer")
|
|
83
|
+
held.add(key)
|
|
84
|
+
self._held, self._guard, self._key = held, guard, key
|
|
85
|
+
self._released = False
|
|
86
|
+
|
|
87
|
+
def release(self) -> None:
|
|
88
|
+
with self._guard:
|
|
89
|
+
if not self._released:
|
|
90
|
+
self._held.discard(self._key)
|
|
91
|
+
self._released = True
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 Novikov Laboratories LLC
|
|
3
|
+
# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
|
|
4
|
+
"""What the engine needs from a storage, and the rules every storage follows.
|
|
5
|
+
|
|
6
|
+
Per task a storage keeps:
|
|
7
|
+
|
|
8
|
+
state one JSON document with a version number. `save` replaces it only
|
|
9
|
+
if the stored version is the one the writer read (`expected`),
|
|
10
|
+
so a second writer cannot silently overwrite the first;
|
|
11
|
+
journal JSON entries appended since the last checkpoint: what arrived,
|
|
12
|
+
written before it is processed;
|
|
13
|
+
audit JSON entries appended forever: the decisions and actions;
|
|
14
|
+
documents named JSON documents: archived segments, reports.
|
|
15
|
+
|
|
16
|
+
A task has one writer at a time: `lock` raises TaskLocked while another
|
|
17
|
+
holds it. Values must be plain JSON (`sequentialist.core.canonical.is_plain`).
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import re
|
|
22
|
+
from abc import ABC, abstractmethod
|
|
23
|
+
from typing import Any, Sequence
|
|
24
|
+
|
|
25
|
+
from ..errors import StorageError
|
|
26
|
+
from ._lock import Lock
|
|
27
|
+
|
|
28
|
+
TASK_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}")
|
|
29
|
+
DOCUMENT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}(/[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}){0,7}")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def check_task_id(task_id: str) -> str:
|
|
33
|
+
if not isinstance(task_id, str) or not TASK_ID.fullmatch(task_id):
|
|
34
|
+
raise StorageError(f"task id {task_id!r}: letters, digits, '_', '.' and '-' only, at most 128 characters, "
|
|
35
|
+
f"not starting with '.', '_' or '-' (it names files)")
|
|
36
|
+
return task_id
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def check_document_name(name: str) -> str:
|
|
40
|
+
if not isinstance(name, str) or not DOCUMENT_NAME.fullmatch(name) or ".." in name:
|
|
41
|
+
raise StorageError(f"document name {name!r}: path segments of letters, digits, '_', '.', ':' and '-'")
|
|
42
|
+
return name
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Storage(ABC):
|
|
46
|
+
@abstractmethod
|
|
47
|
+
def lock(self, task_id: str) -> Lock:
|
|
48
|
+
"""Become the only writer of the task until the lock is released; TaskLocked otherwise."""
|
|
49
|
+
|
|
50
|
+
@abstractmethod
|
|
51
|
+
def tasks(self) -> list[str]:
|
|
52
|
+
"""The ids of the stored tasks, sorted."""
|
|
53
|
+
|
|
54
|
+
@abstractmethod
|
|
55
|
+
def load(self, task_id: str) -> tuple[int, dict[str, Any]] | None:
|
|
56
|
+
"""(version, state), or None if the task has never been saved."""
|
|
57
|
+
|
|
58
|
+
@abstractmethod
|
|
59
|
+
def save(self, task_id: str, state: dict[str, Any], *, expected: int, clear_journal: bool = False) -> int:
|
|
60
|
+
"""Replace the state if the stored version is `expected` (0: none yet); return the new version.
|
|
61
|
+
|
|
62
|
+
With `clear_journal` the journal is emptied as well: atomically with
|
|
63
|
+
the state where the storage can (SQLite), right after it otherwise.
|
|
64
|
+
ConflictError if the version differs.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
@abstractmethod
|
|
68
|
+
def journal_append(self, task_id: str, entries: Sequence[dict[str, Any]]) -> None:
|
|
69
|
+
"""Append entries; they are on disk (per the storage's sync mode) when this returns."""
|
|
70
|
+
|
|
71
|
+
@abstractmethod
|
|
72
|
+
def journal_read(self, task_id: str) -> list[dict[str, Any]]:
|
|
73
|
+
"""The entries appended since the journal was last cleared, in order."""
|
|
74
|
+
|
|
75
|
+
@abstractmethod
|
|
76
|
+
def audit_append(self, task_id: str, entry: dict[str, Any]) -> None:
|
|
77
|
+
"""Append one audit entry; it is on disk when this returns."""
|
|
78
|
+
|
|
79
|
+
@abstractmethod
|
|
80
|
+
def audit_read(self, task_id: str) -> list[dict[str, Any]]:
|
|
81
|
+
"""Every audit entry of the task, in order."""
|
|
82
|
+
|
|
83
|
+
@abstractmethod
|
|
84
|
+
def put_document(self, task_id: str, name: str, document: dict[str, Any]) -> None:
|
|
85
|
+
"""Store (or replace) a named document."""
|
|
86
|
+
|
|
87
|
+
@abstractmethod
|
|
88
|
+
def get_document(self, task_id: str, name: str) -> dict[str, Any] | None:
|
|
89
|
+
"""The named document, or None."""
|
|
90
|
+
|
|
91
|
+
@abstractmethod
|
|
92
|
+
def documents(self, task_id: str) -> list[str]:
|
|
93
|
+
"""Names of the task's documents, sorted."""
|
|
94
|
+
|
|
95
|
+
def close(self) -> None: # noqa: B027 - optional
|
|
96
|
+
"""Release resources (connections, files)."""
|
|
97
|
+
|
|
98
|
+
def __enter__(self) -> Storage:
|
|
99
|
+
return self
|
|
100
|
+
|
|
101
|
+
def __exit__(self, *exc: object) -> None:
|
|
102
|
+
self.close()
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 Novikov Laboratories LLC
|
|
3
|
+
# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
|
|
4
|
+
"""Storage in a directory: one subdirectory per task, plain JSON files.
|
|
5
|
+
|
|
6
|
+
<root>/<task>/state.json {"format", "version", "state"}, replaced atomically
|
|
7
|
+
<root>/<task>/journal.jsonl one entry per line
|
|
8
|
+
<root>/<task>/audit.jsonl one entry per line
|
|
9
|
+
<root>/<task>/documents/... one JSON file per document
|
|
10
|
+
<root>/<task>/.lock held by the writer
|
|
11
|
+
|
|
12
|
+
A crash in the middle of an append leaves a torn last line; it is ignored
|
|
13
|
+
when reading and cut off by the next append. `sync="fsync"` (the default)
|
|
14
|
+
forces every append and state to the disk, so they survive a power
|
|
15
|
+
failure; `sync="flush"` survives a crash of the process only.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import tempfile
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Sequence
|
|
24
|
+
|
|
25
|
+
from ..canonical import loads
|
|
26
|
+
from ..errors import ConflictError, StorageError
|
|
27
|
+
from ._lock import FileLock, Lock
|
|
28
|
+
from .base import Storage, check_document_name, check_task_id
|
|
29
|
+
|
|
30
|
+
FORMAT = "sequentialist.files/1"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _fsync_directory(directory: Path) -> None:
|
|
34
|
+
if os.name != "posix":
|
|
35
|
+
return
|
|
36
|
+
fd = os.open(str(directory), os.O_RDONLY)
|
|
37
|
+
try:
|
|
38
|
+
os.fsync(fd)
|
|
39
|
+
finally:
|
|
40
|
+
os.close(fd)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def write_json_atomically(path: Path, value: Any, sync: bool = True) -> None:
|
|
44
|
+
"""Write a file so that a crash leaves either the old content or the new one."""
|
|
45
|
+
fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name, suffix=".tmp")
|
|
46
|
+
try:
|
|
47
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
48
|
+
json.dump(value, f, sort_keys=True, ensure_ascii=False)
|
|
49
|
+
f.flush()
|
|
50
|
+
if sync:
|
|
51
|
+
os.fsync(f.fileno())
|
|
52
|
+
os.replace(tmp, str(path))
|
|
53
|
+
except BaseException:
|
|
54
|
+
if os.path.exists(tmp):
|
|
55
|
+
os.unlink(tmp)
|
|
56
|
+
raise
|
|
57
|
+
if sync:
|
|
58
|
+
_fsync_directory(path.parent)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _read_lines(path: Path) -> list[dict[str, Any]]:
|
|
62
|
+
if not path.exists():
|
|
63
|
+
return []
|
|
64
|
+
out = []
|
|
65
|
+
with open(path, "rb") as f:
|
|
66
|
+
for number, line in enumerate(f, 1):
|
|
67
|
+
if not line.endswith(b"\n"):
|
|
68
|
+
break # a torn last line: the append it belonged to never completed
|
|
69
|
+
try:
|
|
70
|
+
out.append(loads(line.decode("utf-8")))
|
|
71
|
+
except ValueError as error:
|
|
72
|
+
raise StorageError(f"{path}:{number}: not valid JSON ({error})") from None
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class FileStorage(Storage):
|
|
77
|
+
def __init__(self, root: str | Path, *, sync: str = "fsync"):
|
|
78
|
+
if sync not in ("fsync", "flush"):
|
|
79
|
+
raise ValueError("sync must be 'fsync' or 'flush'")
|
|
80
|
+
self.root = Path(root)
|
|
81
|
+
self.sync = sync
|
|
82
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
83
|
+
|
|
84
|
+
def _dir(self, task_id: str, create: bool = False) -> Path:
|
|
85
|
+
directory = self.root / check_task_id(task_id)
|
|
86
|
+
if create:
|
|
87
|
+
(directory / "documents").mkdir(parents=True, exist_ok=True)
|
|
88
|
+
return directory
|
|
89
|
+
|
|
90
|
+
def lock(self, task_id: str) -> Lock:
|
|
91
|
+
return FileLock(str(self._dir(task_id, create=True) / ".lock"), f"task {task_id!r} in {self.root}")
|
|
92
|
+
|
|
93
|
+
def tasks(self) -> list[str]:
|
|
94
|
+
return sorted(p.name for p in self.root.iterdir() if (p / "state.json").exists())
|
|
95
|
+
|
|
96
|
+
def load(self, task_id: str) -> tuple[int, dict[str, Any]] | None:
|
|
97
|
+
path = self._dir(task_id) / "state.json"
|
|
98
|
+
if not path.exists():
|
|
99
|
+
return None
|
|
100
|
+
try:
|
|
101
|
+
stored = loads(path.read_text(encoding="utf-8"))
|
|
102
|
+
except ValueError as error:
|
|
103
|
+
raise StorageError(f"{path}: not valid JSON ({error})") from None
|
|
104
|
+
if not isinstance(stored, dict) or stored.get("format") != FORMAT or type(stored.get("version")) is not int:
|
|
105
|
+
raise StorageError(f"{path}: not a state file of this storage")
|
|
106
|
+
return stored["version"], stored["state"]
|
|
107
|
+
|
|
108
|
+
def save(self, task_id: str, state: dict[str, Any], *, expected: int, clear_journal: bool = False) -> int:
|
|
109
|
+
directory = self._dir(task_id, create=True)
|
|
110
|
+
loaded = self.load(task_id)
|
|
111
|
+
current = 0 if loaded is None else loaded[0]
|
|
112
|
+
if current != expected:
|
|
113
|
+
raise ConflictError(f"task {task_id!r}: stored version {current}, expected {expected}")
|
|
114
|
+
write_json_atomically(directory / "state.json", {"format": FORMAT, "version": current + 1, "state": state},
|
|
115
|
+
self.sync == "fsync")
|
|
116
|
+
if clear_journal:
|
|
117
|
+
journal = directory / "journal.jsonl"
|
|
118
|
+
if journal.exists():
|
|
119
|
+
with open(journal, "r+b") as f:
|
|
120
|
+
f.truncate(0)
|
|
121
|
+
f.flush()
|
|
122
|
+
if self.sync == "fsync":
|
|
123
|
+
os.fsync(f.fileno())
|
|
124
|
+
return current + 1
|
|
125
|
+
|
|
126
|
+
def _append(self, path: Path, entries: Sequence[dict[str, Any]]) -> None:
|
|
127
|
+
data = b"".join((json.dumps(e, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8") for e in entries)
|
|
128
|
+
with open(path, "a+b") as f:
|
|
129
|
+
size = f.seek(0, os.SEEK_END)
|
|
130
|
+
if size:
|
|
131
|
+
f.seek(size - 1)
|
|
132
|
+
if f.read(1) != b"\n": # a torn last line: cut it back to the last complete one
|
|
133
|
+
f.seek(0)
|
|
134
|
+
keep = f.read().rfind(b"\n") + 1
|
|
135
|
+
f.truncate(keep)
|
|
136
|
+
f.write(data)
|
|
137
|
+
f.flush()
|
|
138
|
+
if self.sync == "fsync":
|
|
139
|
+
os.fsync(f.fileno())
|
|
140
|
+
|
|
141
|
+
def journal_append(self, task_id: str, entries: Sequence[dict[str, Any]]) -> None:
|
|
142
|
+
if entries:
|
|
143
|
+
self._append(self._dir(task_id, create=True) / "journal.jsonl", entries)
|
|
144
|
+
|
|
145
|
+
def journal_read(self, task_id: str) -> list[dict[str, Any]]:
|
|
146
|
+
return _read_lines(self._dir(task_id) / "journal.jsonl")
|
|
147
|
+
|
|
148
|
+
def audit_append(self, task_id: str, entry: dict[str, Any]) -> None:
|
|
149
|
+
self._append(self._dir(task_id, create=True) / "audit.jsonl", [entry])
|
|
150
|
+
|
|
151
|
+
def audit_read(self, task_id: str) -> list[dict[str, Any]]:
|
|
152
|
+
return _read_lines(self._dir(task_id) / "audit.jsonl")
|
|
153
|
+
|
|
154
|
+
def _document_path(self, task_id: str, name: str) -> Path:
|
|
155
|
+
return self._dir(task_id) / "documents" / (check_document_name(name) + ".json")
|
|
156
|
+
|
|
157
|
+
def put_document(self, task_id: str, name: str, document: dict[str, Any]) -> None:
|
|
158
|
+
self._dir(task_id, create=True)
|
|
159
|
+
path = self._document_path(task_id, name)
|
|
160
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
161
|
+
write_json_atomically(path, document, self.sync == "fsync")
|
|
162
|
+
|
|
163
|
+
def get_document(self, task_id: str, name: str) -> dict[str, Any] | None:
|
|
164
|
+
path = self._document_path(task_id, name)
|
|
165
|
+
if not path.exists():
|
|
166
|
+
return None
|
|
167
|
+
found: dict[str, Any] = loads(path.read_text(encoding="utf-8"))
|
|
168
|
+
return found
|
|
169
|
+
|
|
170
|
+
def documents(self, task_id: str) -> list[str]:
|
|
171
|
+
base = self._dir(task_id) / "documents"
|
|
172
|
+
if not base.exists():
|
|
173
|
+
return []
|
|
174
|
+
return sorted(p.relative_to(base).as_posix()[:-5] for p in base.rglob("*.json"))
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
# Copyright (C) 2026 Novikov Laboratories LLC
|
|
3
|
+
# Commercial licenses for use outside the AGPL: see COMMERCIAL.md
|
|
4
|
+
"""In-memory storage: for simulations, replays and tests. Nothing survives the process."""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import copy
|
|
8
|
+
import threading
|
|
9
|
+
from typing import Any, Sequence
|
|
10
|
+
|
|
11
|
+
from ..errors import ConflictError
|
|
12
|
+
from ._lock import Lock, MemoryLock
|
|
13
|
+
from .base import Storage, check_document_name, check_task_id
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class MemoryStorage(Storage):
|
|
17
|
+
def __init__(self) -> None:
|
|
18
|
+
self._states: dict[str, tuple[int, dict[str, Any]]] = {}
|
|
19
|
+
self._journals: dict[str, list[dict[str, Any]]] = {}
|
|
20
|
+
self._audits: dict[str, list[dict[str, Any]]] = {}
|
|
21
|
+
self._documents: dict[str, dict[str, dict[str, Any]]] = {}
|
|
22
|
+
self._held: set[str] = set()
|
|
23
|
+
self._guard = threading.Lock()
|
|
24
|
+
|
|
25
|
+
def lock(self, task_id: str) -> Lock:
|
|
26
|
+
return MemoryLock(self._held, self._guard, check_task_id(task_id), f"task {task_id!r}")
|
|
27
|
+
|
|
28
|
+
def tasks(self) -> list[str]:
|
|
29
|
+
return sorted(self._states)
|
|
30
|
+
|
|
31
|
+
def load(self, task_id: str) -> tuple[int, dict[str, Any]] | None:
|
|
32
|
+
stored = self._states.get(check_task_id(task_id))
|
|
33
|
+
return None if stored is None else (stored[0], copy.deepcopy(stored[1]))
|
|
34
|
+
|
|
35
|
+
def save(self, task_id: str, state: dict[str, Any], *, expected: int, clear_journal: bool = False) -> int:
|
|
36
|
+
current = self._states.get(check_task_id(task_id), (0, {}))[0]
|
|
37
|
+
if current != expected:
|
|
38
|
+
raise ConflictError(f"task {task_id!r}: stored version {current}, expected {expected}")
|
|
39
|
+
self._states[task_id] = (current + 1, copy.deepcopy(state))
|
|
40
|
+
if clear_journal:
|
|
41
|
+
self._journals[task_id] = []
|
|
42
|
+
return current + 1
|
|
43
|
+
|
|
44
|
+
def journal_append(self, task_id: str, entries: Sequence[dict[str, Any]]) -> None:
|
|
45
|
+
self._journals.setdefault(check_task_id(task_id), []).extend(copy.deepcopy(list(entries)))
|
|
46
|
+
|
|
47
|
+
def journal_read(self, task_id: str) -> list[dict[str, Any]]:
|
|
48
|
+
return copy.deepcopy(self._journals.get(check_task_id(task_id), []))
|
|
49
|
+
|
|
50
|
+
def audit_append(self, task_id: str, entry: dict[str, Any]) -> None:
|
|
51
|
+
self._audits.setdefault(check_task_id(task_id), []).append(copy.deepcopy(entry))
|
|
52
|
+
|
|
53
|
+
def audit_read(self, task_id: str) -> list[dict[str, Any]]:
|
|
54
|
+
return copy.deepcopy(self._audits.get(check_task_id(task_id), []))
|
|
55
|
+
|
|
56
|
+
def put_document(self, task_id: str, name: str, document: dict[str, Any]) -> None:
|
|
57
|
+
self._documents.setdefault(check_task_id(task_id), {})[check_document_name(name)] = copy.deepcopy(document)
|
|
58
|
+
|
|
59
|
+
def get_document(self, task_id: str, name: str) -> dict[str, Any] | None:
|
|
60
|
+
found = self._documents.get(check_task_id(task_id), {}).get(check_document_name(name))
|
|
61
|
+
return None if found is None else copy.deepcopy(found)
|
|
62
|
+
|
|
63
|
+
def documents(self, task_id: str) -> list[str]:
|
|
64
|
+
return sorted(self._documents.get(check_task_id(task_id), {}))
|