semora-store 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- semora_store-0.1.0/.gitignore +28 -0
- semora_store-0.1.0/PKG-INFO +15 -0
- semora_store-0.1.0/pyproject.toml +26 -0
- semora_store-0.1.0/src/semora_store/__init__.py +44 -0
- semora_store-0.1.0/src/semora_store/context.py +38 -0
- semora_store-0.1.0/src/semora_store/ledger.py +365 -0
- semora_store-0.1.0/src/semora_store/py.typed +0 -0
- semora_store-0.1.0/src/semora_store/transcript.py +163 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
.pytest_cache/
|
|
3
|
+
.mypy_cache/
|
|
4
|
+
.ruff_cache/
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
*.egg-info/
|
|
8
|
+
build/
|
|
9
|
+
dist/
|
|
10
|
+
.coverage
|
|
11
|
+
htmlcov/
|
|
12
|
+
.env
|
|
13
|
+
.env.*
|
|
14
|
+
!.env.example
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# Local tool/editor state — machine-specific, never pushed.
|
|
18
|
+
.claude/
|
|
19
|
+
.codecanvas/
|
|
20
|
+
.vscode/
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Superpowers design/spec scratch — working notes, not project documentation.
|
|
24
|
+
docs/superpowers/
|
|
25
|
+
|
|
26
|
+
# 로컬 자격증명 — 절대 커밋 금지.
|
|
27
|
+
a.txt
|
|
28
|
+
*.token
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: semora-store
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Storage-semantic contracts for Semora execution and append-only transcripts.
|
|
5
|
+
Project-URL: Homepage, https://github.com/donggyun112/semora
|
|
6
|
+
Project-URL: Source, https://github.com/donggyun112/semora
|
|
7
|
+
Project-URL: Changelog, https://github.com/donggyun112/semora/blob/main/CHANGELOG.md
|
|
8
|
+
Author: donggyun112
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Python: >=3.12
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "semora-store"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Storage-semantic contracts for Semora execution and append-only transcripts."
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
authors = [{ name = "donggyun112" }]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"License :: OSI Approved :: MIT License",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Typing :: Typed",
|
|
18
|
+
]
|
|
19
|
+
urls = { Homepage = "https://github.com/donggyun112/semora", Source = "https://github.com/donggyun112/semora", Changelog = "https://github.com/donggyun112/semora/blob/main/CHANGELOG.md" }
|
|
20
|
+
dependencies = []
|
|
21
|
+
# Deliberately empty, and the reason this is its own distribution. Storage adapters implement
|
|
22
|
+
# behavior contracts over opaque values, so they need no message type, control point, or physical
|
|
23
|
+
# schema from `semora`. Anything appearing here means the boundary moved.
|
|
24
|
+
|
|
25
|
+
[tool.hatch.build.targets.wheel]
|
|
26
|
+
packages = ["src/semora_store"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Expose dependency-free step-ledger and transcript contracts."""
|
|
2
|
+
|
|
3
|
+
from .context import ExecutionContext, ScopedStore
|
|
4
|
+
from .ledger import (
|
|
5
|
+
Contended,
|
|
6
|
+
EffectCompletion,
|
|
7
|
+
EffectConflict,
|
|
8
|
+
ExecutionStore,
|
|
9
|
+
ExecutionTransition,
|
|
10
|
+
Fenced,
|
|
11
|
+
Indeterminate,
|
|
12
|
+
InputRecord,
|
|
13
|
+
MemorySteps,
|
|
14
|
+
Step,
|
|
15
|
+
StepLog,
|
|
16
|
+
)
|
|
17
|
+
from .transcript import (
|
|
18
|
+
MODEL_USAGE_FIELDS,
|
|
19
|
+
RUN_FIELDS,
|
|
20
|
+
MemoryTranscript,
|
|
21
|
+
Transcript,
|
|
22
|
+
check_fields,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"MODEL_USAGE_FIELDS",
|
|
27
|
+
"RUN_FIELDS",
|
|
28
|
+
"Contended",
|
|
29
|
+
"EffectCompletion",
|
|
30
|
+
"EffectConflict",
|
|
31
|
+
"ExecutionContext",
|
|
32
|
+
"ExecutionStore",
|
|
33
|
+
"ExecutionTransition",
|
|
34
|
+
"Fenced",
|
|
35
|
+
"Indeterminate",
|
|
36
|
+
"InputRecord",
|
|
37
|
+
"MemorySteps",
|
|
38
|
+
"MemoryTranscript",
|
|
39
|
+
"ScopedStore",
|
|
40
|
+
"Step",
|
|
41
|
+
"StepLog",
|
|
42
|
+
"Transcript",
|
|
43
|
+
"check_fields",
|
|
44
|
+
]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Trusted, storage-neutral scope for one execution."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from types import MappingProxyType
|
|
6
|
+
from typing import Protocol, Self
|
|
7
|
+
|
|
8
|
+
__all__ = ["ExecutionContext", "ScopedStore"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True, slots=True)
|
|
12
|
+
class ExecutionContext:
|
|
13
|
+
"""Carry host-authenticated execution identity without interpreting tenant metadata.
|
|
14
|
+
|
|
15
|
+
``run_id`` is Semora's execution and idempotency coordinate. Every other field is opaque to the
|
|
16
|
+
framework and must originate at the host trust boundary, never in model output or tool input.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
run_id: str
|
|
20
|
+
session_id: str | None = None
|
|
21
|
+
namespace: str | None = None
|
|
22
|
+
actor: str | None = None
|
|
23
|
+
subject: str | None = None
|
|
24
|
+
attributes: Mapping[str, str] = field(default_factory=dict)
|
|
25
|
+
|
|
26
|
+
def __post_init__(self) -> None:
|
|
27
|
+
"""Freeze a copy so caller mutation cannot change authority mid-run."""
|
|
28
|
+
if not self.run_id:
|
|
29
|
+
raise ValueError("execution context requires a non-empty run_id")
|
|
30
|
+
object.__setattr__(self, "attributes", MappingProxyType(dict(self.attributes)))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ScopedStore(Protocol):
|
|
34
|
+
"""Bind a trusted execution scope without prescribing an adapter's physical layout."""
|
|
35
|
+
|
|
36
|
+
def for_execution(self, context: ExecutionContext) -> Self:
|
|
37
|
+
"""Return an adapter view bound to ``context``."""
|
|
38
|
+
...
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
"""Define the durable step ledger and its in-memory implementation.
|
|
2
|
+
|
|
3
|
+
The ledger records opaque step values and distinguishes absent, running, and completed steps.
|
|
4
|
+
It surfaces ambiguous interrupted effects instead of claiming exactly-once execution.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from time import monotonic
|
|
10
|
+
from typing import Any, Literal, NamedTuple, Protocol, Self, runtime_checkable
|
|
11
|
+
|
|
12
|
+
from .context import ExecutionContext, ScopedStore
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"Contended",
|
|
16
|
+
"EffectCompletion",
|
|
17
|
+
"EffectConflict",
|
|
18
|
+
"ExecutionStore",
|
|
19
|
+
"ExecutionTransition",
|
|
20
|
+
"Fenced",
|
|
21
|
+
"Indeterminate",
|
|
22
|
+
"InputRecord",
|
|
23
|
+
"MemorySteps",
|
|
24
|
+
"Step",
|
|
25
|
+
"StepLog",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Fenced(Exception):
|
|
30
|
+
"""Report a write attempted with a stale fencing token."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, run_id: str, presented: int, issued: int) -> None:
|
|
33
|
+
"""Initialize the error with the stale and current tokens."""
|
|
34
|
+
super().__init__(
|
|
35
|
+
f"run {run_id!r} moved on: write presented token {presented}, current is {issued}"
|
|
36
|
+
)
|
|
37
|
+
self.run_id = run_id
|
|
38
|
+
self.presented = presented
|
|
39
|
+
self.issued = issued
|
|
40
|
+
|
|
41
|
+
class Contended(Exception):
|
|
42
|
+
"""Report that another worker holds the run lease."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, run_id: str) -> None:
|
|
45
|
+
"""Initialize the error for the contended run."""
|
|
46
|
+
super().__init__(f"run {run_id!r} is held by another worker")
|
|
47
|
+
self.run_id = run_id
|
|
48
|
+
|
|
49
|
+
class Indeterminate(Exception):
|
|
50
|
+
"""Report a step whose external effect may have occurred."""
|
|
51
|
+
|
|
52
|
+
def __init__(self, run_id: str, step: str) -> None:
|
|
53
|
+
"""Initialize the error for the interrupted step."""
|
|
54
|
+
super().__init__(f"step {step!r} of run {run_id!r} may or may not have happened")
|
|
55
|
+
self.run_id = run_id
|
|
56
|
+
self.step = step
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class EffectConflict(Exception):
|
|
60
|
+
"""Report an effect completion that contradicts its durable state."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, run_id: str, key: str, reason: str) -> None:
|
|
63
|
+
"""Initialize the conflict with its durable coordinates."""
|
|
64
|
+
super().__init__(f"effect {key!r} of run {run_id!r} cannot complete: {reason}")
|
|
65
|
+
self.run_id = run_id
|
|
66
|
+
self.key = key
|
|
67
|
+
self.reason = reason
|
|
68
|
+
|
|
69
|
+
class Step(NamedTuple):
|
|
70
|
+
"""Represent the persisted state and value of one step."""
|
|
71
|
+
|
|
72
|
+
status: Literal["absent", "running", "done"]
|
|
73
|
+
value: Any = None
|
|
74
|
+
|
|
75
|
+
class InputRecord(NamedTuple):
|
|
76
|
+
"""Represent one durable input queue row."""
|
|
77
|
+
|
|
78
|
+
input_id: str
|
|
79
|
+
status: Literal["pending", "claimed", "admitted", "discarded"]
|
|
80
|
+
value: dict[str, Any]
|
|
81
|
+
sequence: int
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class EffectCompletion(NamedTuple):
|
|
85
|
+
"""Complete one immutable effect from the state that authorized the transition."""
|
|
86
|
+
|
|
87
|
+
key: str
|
|
88
|
+
value: Any
|
|
89
|
+
expected: Literal["absent", "running"] = "running"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(frozen=True, slots=True)
|
|
93
|
+
class ExecutionTransition:
|
|
94
|
+
"""Atomically commit effect results, mutable control state, and queued inputs."""
|
|
95
|
+
|
|
96
|
+
effects: tuple[EffectCompletion, ...] = ()
|
|
97
|
+
controls: Mapping[str, Any] = field(default_factory=dict)
|
|
98
|
+
inputs: tuple[tuple[str, dict[str, Any]], ...] = ()
|
|
99
|
+
|
|
100
|
+
@runtime_checkable
|
|
101
|
+
class ExecutionStore(ScopedStore, Protocol):
|
|
102
|
+
"""Persist effect intent, control transitions, run leases, and queued inputs.
|
|
103
|
+
|
|
104
|
+
Implementations commit ``start`` before an effect executes and ``finish_effect`` afterward.
|
|
105
|
+
Completed effect results are immutable. Mutable continuation and protocol state uses
|
|
106
|
+
``write_control`` or ``commit_transition`` instead. Lease-protected writes use the fencing
|
|
107
|
+
token returned by ``acquire``.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
async def read(self, run_id: str, key: str) -> Step:
|
|
111
|
+
"""Return the persisted state of a step."""
|
|
112
|
+
...
|
|
113
|
+
|
|
114
|
+
async def start(self, run_id: str, key: str, token: int = 0) -> bool:
|
|
115
|
+
"""Atomically record new step intent and report whether this caller inserted it."""
|
|
116
|
+
...
|
|
117
|
+
|
|
118
|
+
async def finish_effect(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
|
|
119
|
+
"""Complete a running effect idempotently without replacing a committed result."""
|
|
120
|
+
...
|
|
121
|
+
|
|
122
|
+
async def write_control(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
|
|
123
|
+
"""Upsert mutable framework control state."""
|
|
124
|
+
...
|
|
125
|
+
|
|
126
|
+
async def acquire(self, run_id: str, owner: str, ttl_seconds: float) -> int:
|
|
127
|
+
"""Acquire or renew a run lease and return its fencing token, or zero on contention."""
|
|
128
|
+
...
|
|
129
|
+
|
|
130
|
+
async def release(self, run_id: str, owner: str) -> None:
|
|
131
|
+
"""Release a run lease held by ``owner``."""
|
|
132
|
+
...
|
|
133
|
+
|
|
134
|
+
async def enqueue_input(self, run_id: str, input_id: str, value: dict[str, Any]) -> bool:
|
|
135
|
+
"""Append an input idempotently and report whether it was inserted."""
|
|
136
|
+
...
|
|
137
|
+
|
|
138
|
+
async def list_inputs(self, run_id: str) -> list[InputRecord]:
|
|
139
|
+
"""Return a run's inputs in submission order."""
|
|
140
|
+
...
|
|
141
|
+
|
|
142
|
+
async def claim_input(self, run_id: str, input_id: str, token: int = 0) -> None:
|
|
143
|
+
"""Mark an input as claimed unless it is already terminal."""
|
|
144
|
+
...
|
|
145
|
+
|
|
146
|
+
async def admit_inputs(self, run_id: str, input_ids: list[str], token: int = 0) -> None:
|
|
147
|
+
"""Mark the selected inputs as admitted to model context."""
|
|
148
|
+
...
|
|
149
|
+
|
|
150
|
+
async def discard_inputs(self, run_id: str, input_ids: list[str], token: int = 0) -> None:
|
|
151
|
+
"""Mark screened-out inputs as permanently discarded."""
|
|
152
|
+
...
|
|
153
|
+
|
|
154
|
+
async def commit_transition(
|
|
155
|
+
self,
|
|
156
|
+
run_id: str,
|
|
157
|
+
transition: ExecutionTransition,
|
|
158
|
+
token: int = 0,
|
|
159
|
+
) -> set[str]:
|
|
160
|
+
"""Atomically apply typed effect, control, and input changes."""
|
|
161
|
+
...
|
|
162
|
+
|
|
163
|
+
async def forget(self, run_id: str, key: str, token: int = 0) -> None:
|
|
164
|
+
"""Remove an unfinished step. A `done` step is left alone.
|
|
165
|
+
|
|
166
|
+
Not optional, because clearing is what makes a reported failure retryable: a step that
|
|
167
|
+
raises, a model request that failed before its first chunk, and an aborted stream all end
|
|
168
|
+
by removing their intent. A ledger that silently kept them would turn every one of those
|
|
169
|
+
into a permanent `Indeterminate`.
|
|
170
|
+
|
|
171
|
+
Lease-protected like every other write, and for the sharper reason: erasing intent is the
|
|
172
|
+
one write that can make a *live* worker's step look like it never happened. A replaced
|
|
173
|
+
worker clearing its own abandoned attempt would otherwise delete the successor's running
|
|
174
|
+
intent, and the effect the successor is mid-way through would replay on the attempt after.
|
|
175
|
+
"""
|
|
176
|
+
...
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
StepLog = ExecutionStore
|
|
180
|
+
"""Compatibility name for the execution-store contract."""
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class MemorySteps:
|
|
184
|
+
"""Implement ``ExecutionStore`` with process-local dictionaries."""
|
|
185
|
+
|
|
186
|
+
def __init__(self) -> None:
|
|
187
|
+
"""Initialize empty step, lease, and input stores."""
|
|
188
|
+
self._entries: dict[tuple[str, str], Step] = {}
|
|
189
|
+
self._leases: dict[str, tuple[str, int, float]] = {}
|
|
190
|
+
self._tokens: dict[str, int] = {}
|
|
191
|
+
self._inputs: dict[tuple[str, str], InputRecord] = {}
|
|
192
|
+
self._input_sequence: dict[str, int] = {}
|
|
193
|
+
|
|
194
|
+
def for_execution(self, context: ExecutionContext) -> Self:
|
|
195
|
+
"""Return this scope-neutral in-memory store."""
|
|
196
|
+
del context
|
|
197
|
+
return self
|
|
198
|
+
|
|
199
|
+
async def read(self, run_id: str, key: str) -> Step:
|
|
200
|
+
"""Return the stored step or an absent state."""
|
|
201
|
+
return self._entries.get((run_id, key), Step("absent"))
|
|
202
|
+
|
|
203
|
+
async def start(self, run_id: str, key: str, token: int = 0) -> bool:
|
|
204
|
+
"""Record running intent only when the step is absent."""
|
|
205
|
+
self._fence(run_id, token)
|
|
206
|
+
if (run_id, key) in self._entries:
|
|
207
|
+
return False
|
|
208
|
+
self._entries[run_id, key] = Step("running")
|
|
209
|
+
return True
|
|
210
|
+
|
|
211
|
+
async def finish_effect(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
|
|
212
|
+
"""Complete a running effect while preserving an existing result."""
|
|
213
|
+
self._fence(run_id, token)
|
|
214
|
+
record = self._entries.get((run_id, key), Step("absent"))
|
|
215
|
+
if record.status == "done":
|
|
216
|
+
if record.value == value:
|
|
217
|
+
return
|
|
218
|
+
raise EffectConflict(run_id, key, "a different result is already committed")
|
|
219
|
+
if record.status != "running":
|
|
220
|
+
raise EffectConflict(run_id, key, f"expected 'running', found {record.status!r}")
|
|
221
|
+
self._entries[run_id, key] = Step("done", value)
|
|
222
|
+
|
|
223
|
+
async def write_control(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
|
|
224
|
+
"""Upsert mutable process-local control state."""
|
|
225
|
+
self._fence(run_id, token)
|
|
226
|
+
self._entries[run_id, key] = Step("done", value)
|
|
227
|
+
|
|
228
|
+
async def finish(self, run_id: str, key: str, value: Any, token: int = 0) -> None:
|
|
229
|
+
"""Compatibility alias for ``write_control``."""
|
|
230
|
+
await self.write_control(run_id, key, value, token)
|
|
231
|
+
|
|
232
|
+
async def forget(self, run_id: str, key: str, token: int = 0) -> None:
|
|
233
|
+
"""Remove unfinished step intent while preserving completed results."""
|
|
234
|
+
self._fence(run_id, token)
|
|
235
|
+
if self._entries.get((run_id, key), Step("absent")).status != "done":
|
|
236
|
+
self._entries.pop((run_id, key), None)
|
|
237
|
+
|
|
238
|
+
async def acquire(self, run_id: str, owner: str, ttl_seconds: float = 60.0) -> int:
|
|
239
|
+
"""Acquire or renew a run lease using a monotonic TTL.
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
Current fencing token, or ``0`` when another owner holds the lease.
|
|
243
|
+
"""
|
|
244
|
+
held = self._leases.get(run_id)
|
|
245
|
+
expired = held is not None and held[2] <= monotonic()
|
|
246
|
+
if held is not None and held[0] == owner:
|
|
247
|
+
self._leases[run_id] = (owner, held[1], monotonic() + ttl_seconds)
|
|
248
|
+
return held[1] # the holder renewing keeps its token
|
|
249
|
+
if held is not None and not expired:
|
|
250
|
+
return 0
|
|
251
|
+
# A new lease or a takeover. Either way the token moves, so a previous holder is fenced.
|
|
252
|
+
self._tokens[run_id] = self._tokens.get(run_id, 0) + 1
|
|
253
|
+
self._leases[run_id] = (owner, self._tokens[run_id], monotonic() + ttl_seconds)
|
|
254
|
+
return self._tokens[run_id]
|
|
255
|
+
|
|
256
|
+
async def release(self, run_id: str, owner: str) -> None:
|
|
257
|
+
"""Expire an owned lease without resetting its fencing token."""
|
|
258
|
+
held = self._leases.get(run_id)
|
|
259
|
+
if held is not None and held[0] == owner:
|
|
260
|
+
self._leases[run_id] = ("", held[1], monotonic())
|
|
261
|
+
|
|
262
|
+
async def enqueue_input(self, run_id: str, input_id: str, value: dict[str, Any]) -> bool:
|
|
263
|
+
"""Append an input unless its identifier already exists."""
|
|
264
|
+
key = (run_id, input_id)
|
|
265
|
+
if key in self._inputs:
|
|
266
|
+
return False
|
|
267
|
+
sequence = self._input_sequence.get(run_id, 0)
|
|
268
|
+
self._input_sequence[run_id] = sequence + 1
|
|
269
|
+
self._inputs[key] = InputRecord(input_id, "pending", value, sequence)
|
|
270
|
+
return True
|
|
271
|
+
|
|
272
|
+
async def list_inputs(self, run_id: str) -> list[InputRecord]:
|
|
273
|
+
"""Return a run's inputs in submission order."""
|
|
274
|
+
return sorted(
|
|
275
|
+
(record for (item_run, _), record in self._inputs.items() if item_run == run_id),
|
|
276
|
+
key=lambda record: record.sequence,
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
async def claim_input(self, run_id: str, input_id: str, token: int = 0) -> None:
|
|
280
|
+
"""Mark an input as claimed unless it is already terminal."""
|
|
281
|
+
self._fence(run_id, token)
|
|
282
|
+
key = (run_id, input_id)
|
|
283
|
+
# Absent is a no-op, not a `KeyError`: the durable store expresses this as an `update` that
|
|
284
|
+
# matches no row, and a caller claiming an input that is already gone must not crash on one
|
|
285
|
+
# store and continue on the other.
|
|
286
|
+
record = self._inputs.get(key)
|
|
287
|
+
if record is not None and record.status not in {"admitted", "discarded"}:
|
|
288
|
+
self._inputs[key] = InputRecord(input_id, "claimed", record.value, record.sequence)
|
|
289
|
+
|
|
290
|
+
async def admit_inputs(self, run_id: str, input_ids: list[str], token: int = 0) -> None:
|
|
291
|
+
"""Mark the selected inputs as admitted."""
|
|
292
|
+
self._fence(run_id, token)
|
|
293
|
+
for input_id in input_ids:
|
|
294
|
+
key = (run_id, input_id)
|
|
295
|
+
record = self._inputs.get(key)
|
|
296
|
+
if record is not None and record.status != "discarded":
|
|
297
|
+
self._inputs[key] = InputRecord(
|
|
298
|
+
input_id, "admitted", record.value, record.sequence
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
async def discard_inputs(self, run_id: str, input_ids: list[str], token: int = 0) -> None:
|
|
302
|
+
"""Make screened-out inputs terminal without deleting their idempotency keys."""
|
|
303
|
+
self._fence(run_id, token)
|
|
304
|
+
for input_id in input_ids:
|
|
305
|
+
key = (run_id, input_id)
|
|
306
|
+
record = self._inputs.get(key)
|
|
307
|
+
if record is not None:
|
|
308
|
+
self._inputs[key] = InputRecord(
|
|
309
|
+
input_id, "discarded", record.value, record.sequence
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
async def commit_transition(
|
|
313
|
+
self,
|
|
314
|
+
run_id: str,
|
|
315
|
+
transition: ExecutionTransition,
|
|
316
|
+
token: int = 0,
|
|
317
|
+
) -> set[str]:
|
|
318
|
+
"""Atomically apply a process-local control transition."""
|
|
319
|
+
self._fence(run_id, token)
|
|
320
|
+
completed: list[tuple[str, Any]] = []
|
|
321
|
+
seen: set[str] = set()
|
|
322
|
+
for effect in transition.effects:
|
|
323
|
+
if effect.key in seen:
|
|
324
|
+
raise ValueError(f"effect {effect.key!r} appears twice in one transition")
|
|
325
|
+
seen.add(effect.key)
|
|
326
|
+
record = self._entries.get((run_id, effect.key), Step("absent"))
|
|
327
|
+
if record.status == "done":
|
|
328
|
+
if record.value != effect.value:
|
|
329
|
+
raise EffectConflict(
|
|
330
|
+
run_id, effect.key, "a different result is already committed"
|
|
331
|
+
)
|
|
332
|
+
continue
|
|
333
|
+
if record.status != effect.expected:
|
|
334
|
+
raise EffectConflict(
|
|
335
|
+
run_id,
|
|
336
|
+
effect.key,
|
|
337
|
+
f"expected {effect.expected!r}, found {record.status!r}",
|
|
338
|
+
)
|
|
339
|
+
completed.append((effect.key, effect.value))
|
|
340
|
+
overlap = seen.intersection(transition.controls)
|
|
341
|
+
if overlap:
|
|
342
|
+
raise ValueError(
|
|
343
|
+
f"transition classifies keys as both effect and control: {sorted(overlap)}"
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
inserted: set[str] = set()
|
|
347
|
+
for key, value in completed:
|
|
348
|
+
self._entries[run_id, key] = Step("done", value)
|
|
349
|
+
for key, value in transition.controls.items():
|
|
350
|
+
self._entries[run_id, key] = Step("done", value)
|
|
351
|
+
for input_id, value in transition.inputs:
|
|
352
|
+
input_key = (run_id, input_id)
|
|
353
|
+
if input_key in self._inputs:
|
|
354
|
+
continue
|
|
355
|
+
sequence = self._input_sequence.get(run_id, 0)
|
|
356
|
+
self._input_sequence[run_id] = sequence + 1
|
|
357
|
+
self._inputs[input_key] = InputRecord(input_id, "pending", value, sequence)
|
|
358
|
+
inserted.add(input_id)
|
|
359
|
+
return inserted
|
|
360
|
+
|
|
361
|
+
def _fence(self, run_id: str, token: int) -> None:
|
|
362
|
+
"""Reject stale lease holders while allowing unleased writes with token zero."""
|
|
363
|
+
issued = self._tokens.get(run_id, 0)
|
|
364
|
+
if token and token < issued:
|
|
365
|
+
raise Fenced(run_id, token, issued)
|
|
File without changes
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Claude-style transcript storage without prescribing a physical schema.
|
|
2
|
+
|
|
3
|
+
One adapter owns the ordered conversation, branch markers, run lifecycle, and usage records. Entry
|
|
4
|
+
mappings are opaque to the adapter boundary, so implementations may remap them to any tables,
|
|
5
|
+
documents, or existing API as long as reads reconstruct the same transcript semantics.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from typing import Any, NamedTuple, Protocol, Self, runtime_checkable
|
|
10
|
+
|
|
11
|
+
from .context import ExecutionContext, ScopedStore
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"MODEL_USAGE_FIELDS",
|
|
15
|
+
"RUN_FIELDS",
|
|
16
|
+
"MemoryTranscript",
|
|
17
|
+
"Transcript",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
RUN_FIELDS = frozenset(
|
|
21
|
+
{
|
|
22
|
+
"conversation_id",
|
|
23
|
+
"stop_reason",
|
|
24
|
+
"tool_calls",
|
|
25
|
+
"interrupted_mid_turn",
|
|
26
|
+
"started_at",
|
|
27
|
+
"ended_at",
|
|
28
|
+
}
|
|
29
|
+
)
|
|
30
|
+
"""Fields accepted by run metadata records."""
|
|
31
|
+
|
|
32
|
+
MODEL_USAGE_FIELDS = frozenset(
|
|
33
|
+
{
|
|
34
|
+
"prompt_tokens",
|
|
35
|
+
"completion_tokens",
|
|
36
|
+
"total_tokens",
|
|
37
|
+
"cached_tokens",
|
|
38
|
+
"cache_write_tokens",
|
|
39
|
+
"cost_usd",
|
|
40
|
+
}
|
|
41
|
+
)
|
|
42
|
+
"""Fields accepted by per-model usage records."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class _Row(NamedTuple):
|
|
46
|
+
"""Store one transcript entry and its arrival order."""
|
|
47
|
+
|
|
48
|
+
seq: int
|
|
49
|
+
entry: dict[str, Any]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@runtime_checkable
|
|
53
|
+
class Transcript(ScopedStore, Protocol):
|
|
54
|
+
"""Persist one ordered transcript while leaving its physical representation to the adapter."""
|
|
55
|
+
|
|
56
|
+
async def append(self, entry: dict[str, Any]) -> bool:
|
|
57
|
+
"""Append an entry idempotently.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
entry: Transcript entry containing ``conversation_id`` and ``uuid``.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
``True`` when inserted, or ``False`` when the identity already exists.
|
|
64
|
+
"""
|
|
65
|
+
...
|
|
66
|
+
|
|
67
|
+
async def read(self, conversation_id: str, *, limit: int | None = None) -> list[dict[str, Any]]:
|
|
68
|
+
"""Return entries in arrival order, optionally limited to the newest tail."""
|
|
69
|
+
...
|
|
70
|
+
|
|
71
|
+
async def record_run(self, run_id: str, fields: dict[str, Any]) -> None:
|
|
72
|
+
"""Merge validated lifecycle fields into this transcript's run record.
|
|
73
|
+
|
|
74
|
+
Raises:
|
|
75
|
+
ValueError: If ``fields`` contains a name outside ``RUN_FIELDS``.
|
|
76
|
+
"""
|
|
77
|
+
...
|
|
78
|
+
|
|
79
|
+
async def read_run(self, run_id: str) -> dict[str, Any] | None:
|
|
80
|
+
"""The run record, or `None` if this run was never opened."""
|
|
81
|
+
...
|
|
82
|
+
|
|
83
|
+
async def record_model_usage(self, run_id: str, model: str, counts: dict[str, Any]) -> None:
|
|
84
|
+
"""Merge validated usage fields for one model in this transcript run.
|
|
85
|
+
|
|
86
|
+
Raises:
|
|
87
|
+
ValueError: If ``counts`` contains a name outside ``MODEL_USAGE_FIELDS``.
|
|
88
|
+
"""
|
|
89
|
+
...
|
|
90
|
+
|
|
91
|
+
async def read_model_usage(self, run_id: str) -> dict[str, dict[str, Any]]:
|
|
92
|
+
"""This run's token counts keyed by the model that spent them."""
|
|
93
|
+
...
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def check_fields(table: str, fields: dict[str, Any], allowed: frozenset[str]) -> None:
|
|
97
|
+
"""Raise ``ValueError`` when a record contains unsupported fields."""
|
|
98
|
+
unknown = set(fields) - allowed
|
|
99
|
+
if unknown:
|
|
100
|
+
raise ValueError(f"{table} has no field {sorted(unknown)}")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class MemoryTranscript:
|
|
104
|
+
"""Implement the complete transcript contract with process-local collections."""
|
|
105
|
+
|
|
106
|
+
def __init__(self) -> None:
|
|
107
|
+
"""Initialize empty history collections."""
|
|
108
|
+
self._rows: dict[str, list[_Row]] = {}
|
|
109
|
+
self._seen: set[tuple[str, str]] = set()
|
|
110
|
+
self._runs: dict[str, dict[str, Any]] = {}
|
|
111
|
+
self._model_usage: dict[tuple[str, str], dict[str, Any]] = {}
|
|
112
|
+
self._sequence = 0
|
|
113
|
+
|
|
114
|
+
def for_execution(self, context: ExecutionContext) -> Self:
|
|
115
|
+
"""Ignore routing metadata in this process-local adapter."""
|
|
116
|
+
del context
|
|
117
|
+
return self
|
|
118
|
+
|
|
119
|
+
async def append(self, entry: dict[str, Any]) -> bool:
|
|
120
|
+
"""Append an entry unless its conversation already holds that `uuid`."""
|
|
121
|
+
conversation_id = str(entry.get("conversation_id", ""))
|
|
122
|
+
key = (conversation_id, str(entry.get("uuid", "")))
|
|
123
|
+
if key in self._seen:
|
|
124
|
+
return False
|
|
125
|
+
self._seen.add(key)
|
|
126
|
+
self._sequence += 1
|
|
127
|
+
self._rows.setdefault(conversation_id, []).append(_Row(self._sequence, _frozen(entry)))
|
|
128
|
+
return True
|
|
129
|
+
|
|
130
|
+
async def read(self, conversation_id: str, *, limit: int | None = None) -> list[dict[str, Any]]:
|
|
131
|
+
"""Return entries in arrival order, optionally limited to the newest tail."""
|
|
132
|
+
rows = self._rows.get(conversation_id, [])
|
|
133
|
+
window = rows if limit is None else rows[len(rows) - limit :] if limit > 0 else []
|
|
134
|
+
return [_frozen(row.entry) for row in window]
|
|
135
|
+
|
|
136
|
+
async def record_run(self, run_id: str, fields: dict[str, Any]) -> None:
|
|
137
|
+
"""Merge validated fields into a process-local run record."""
|
|
138
|
+
check_fields("run", fields, RUN_FIELDS)
|
|
139
|
+
self._runs.setdefault(run_id, {}).update(fields)
|
|
140
|
+
|
|
141
|
+
async def read_run(self, run_id: str) -> dict[str, Any] | None:
|
|
142
|
+
"""Return a copied run record."""
|
|
143
|
+
row = self._runs.get(run_id)
|
|
144
|
+
return dict(row) if row is not None else None
|
|
145
|
+
|
|
146
|
+
async def record_model_usage(self, run_id: str, model: str, counts: dict[str, Any]) -> None:
|
|
147
|
+
"""Merge validated token counts for one model of one run."""
|
|
148
|
+
check_fields("model usage", counts, MODEL_USAGE_FIELDS)
|
|
149
|
+
self._model_usage.setdefault((run_id, model), {}).update(counts)
|
|
150
|
+
|
|
151
|
+
async def read_model_usage(self, run_id: str) -> dict[str, dict[str, Any]]:
|
|
152
|
+
"""Return this run's token counts per model."""
|
|
153
|
+
return {
|
|
154
|
+
model: dict(counts)
|
|
155
|
+
for (recorded, model), counts in self._model_usage.items()
|
|
156
|
+
if recorded == run_id
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _frozen(entry: dict[str, Any]) -> dict[str, Any]:
|
|
161
|
+
"""Return a JSON-compatible deep copy of an entry."""
|
|
162
|
+
copied: dict[str, Any] = json.loads(json.dumps(entry))
|
|
163
|
+
return copied
|