nl2data-workflow-postgres 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nl2data_workflow_postgres/__init__.py +25 -0
- nl2data_workflow_postgres/client.py +138 -0
- nl2data_workflow_postgres/config.py +83 -0
- nl2data_workflow_postgres/fake_postgres.py +834 -0
- nl2data_workflow_postgres/py.typed +0 -0
- nl2data_workflow_postgres/schema.py +90 -0
- nl2data_workflow_postgres/store.py +1166 -0
- nl2data_workflow_postgres-0.1.0.dist-info/METADATA +110 -0
- nl2data_workflow_postgres-0.1.0.dist-info/RECORD +11 -0
- nl2data_workflow_postgres-0.1.0.dist-info/WHEEL +5 -0
- nl2data_workflow_postgres-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1166 @@
|
|
|
1
|
+
"""PostgreSQL-backed workflow state, idempotency, and lease store.
|
|
2
|
+
|
|
3
|
+
Implements the replaceable :class:`StateStore` and :class:`IdempotencyStore`
|
|
4
|
+
contracts plus the optional :class:`WorkflowLeaseStore` and
|
|
5
|
+
:class:`FencedStateStore` capabilities, so multiple workers sharing one
|
|
6
|
+
database coordinate through durable snapshots, atomic idempotency keys, and
|
|
7
|
+
lease ownership with monotonic fencing tokens. Safe snapshots reuse the
|
|
8
|
+
core ``serialize_snapshot``/``deserialize_snapshot`` boundary; raw prompts,
|
|
9
|
+
queries, results, credentials, and provider objects are rejected before they
|
|
10
|
+
reach the database.
|
|
11
|
+
|
|
12
|
+
Every mutation is transactional and conditional: compare-and-set updates
|
|
13
|
+
verify workflow identity, expected revision/status, tenant scope, and - when
|
|
14
|
+
ownership is supplied - the current lease owner and fencing token in one
|
|
15
|
+
statement, so a stale worker can never silently overwrite newer state or
|
|
16
|
+
commit after takeover. Backend failures (outages, timeouts, schema
|
|
17
|
+
mismatches) surface as normalized :class:`SharedStoreError` values that never
|
|
18
|
+
leak DSNs, credentials, or raw driver text.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import contextlib
|
|
24
|
+
import re
|
|
25
|
+
from collections.abc import Callable, Iterator
|
|
26
|
+
from datetime import UTC, datetime, timedelta
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from nl2data._redact import REDACTED_VALUE
|
|
30
|
+
from nl2data.errors import NL2DataError
|
|
31
|
+
from nl2data_core.workflow.durable import (
|
|
32
|
+
DurableWorkflowRecord,
|
|
33
|
+
IdempotencyConflictError,
|
|
34
|
+
IdempotencyRecord,
|
|
35
|
+
IdempotencyStatus,
|
|
36
|
+
WorkflowSerializationError,
|
|
37
|
+
deserialize_snapshot,
|
|
38
|
+
serialize_snapshot,
|
|
39
|
+
tenant_scope_namespace,
|
|
40
|
+
)
|
|
41
|
+
from nl2data_core.workflow.lease import (
|
|
42
|
+
WorkflowLease,
|
|
43
|
+
validate_lease_identity,
|
|
44
|
+
)
|
|
45
|
+
from nl2data_core.workflow.models import (
|
|
46
|
+
TERMINAL_STATUSES,
|
|
47
|
+
WorkflowState,
|
|
48
|
+
WorkflowStateError,
|
|
49
|
+
WorkflowStatus,
|
|
50
|
+
WorkflowTransitionError,
|
|
51
|
+
)
|
|
52
|
+
from nl2data_core.workflow.shared_errors import SharedStoreError, SharedStoreErrorCode
|
|
53
|
+
|
|
54
|
+
from .client import (
|
|
55
|
+
build_pool,
|
|
56
|
+
is_connect_error,
|
|
57
|
+
is_duplicate_key_error,
|
|
58
|
+
is_serialization_error,
|
|
59
|
+
is_timeout_error,
|
|
60
|
+
)
|
|
61
|
+
from .config import WorkflowPostgresConfig
|
|
62
|
+
from .schema import MIGRATIONS, SUPPORTED_SCHEMA_VERSION
|
|
63
|
+
|
|
64
|
+
_FINGERPRINT_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
65
|
+
_IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_\-\.]{0,127}$")
|
|
66
|
+
_KEY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
|
|
67
|
+
_TERMINAL_VALUES = tuple(status.value for status in sorted(TERMINAL_STATUSES, key=str))
|
|
68
|
+
|
|
69
|
+
#: Safe migration bootstrap: the schema-metadata table itself is not
|
|
70
|
+
#: versioned and is created before any migration runs.
|
|
71
|
+
_BOOTSTRAP_DDL = """
|
|
72
|
+
CREATE TABLE IF NOT EXISTS {schema}.schema_metadata (
|
|
73
|
+
key TEXT PRIMARY KEY,
|
|
74
|
+
value TEXT NOT NULL
|
|
75
|
+
)
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
#: Every statement the store issues, keyed by a stable name. The ``{schema}``
|
|
79
|
+
#: placeholder is replaced with the quoted deployment namespace at
|
|
80
|
+
#: construction; tests may match against these templates directly.
|
|
81
|
+
SQL_TEMPLATES: dict[str, str] = {
|
|
82
|
+
"read_schema_version": (
|
|
83
|
+
"SELECT value FROM {schema}.schema_metadata WHERE key = 'schema_version'"
|
|
84
|
+
),
|
|
85
|
+
"write_schema_version": (
|
|
86
|
+
"INSERT INTO {schema}.schema_metadata (key, value) VALUES ('schema_version', %s) "
|
|
87
|
+
"ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value"
|
|
88
|
+
),
|
|
89
|
+
"create_workflow": (
|
|
90
|
+
"INSERT INTO {schema}.workflow_states ("
|
|
91
|
+
"workflow_id, request_id, tenant_scope_fingerprint, scope_namespace, "
|
|
92
|
+
"status, revision, schema_version, snapshot, created_at, updated_at, expires_at"
|
|
93
|
+
") VALUES (%s, %s, %s, %s, %s, 1, %s, %s, %s, %s, NULL) "
|
|
94
|
+
"ON CONFLICT (scope_namespace, workflow_id) DO NOTHING"
|
|
95
|
+
),
|
|
96
|
+
"read_snapshot": (
|
|
97
|
+
"SELECT snapshot FROM {schema}.workflow_states "
|
|
98
|
+
"WHERE scope_namespace = %s AND workflow_id = %s"
|
|
99
|
+
),
|
|
100
|
+
"read_checkpoint": (
|
|
101
|
+
"SELECT snapshot FROM {schema}.workflow_states "
|
|
102
|
+
"WHERE scope_namespace = %s AND workflow_id = %s AND request_id = %s"
|
|
103
|
+
),
|
|
104
|
+
"read_revision": (
|
|
105
|
+
"SELECT revision FROM {schema}.workflow_states "
|
|
106
|
+
"WHERE scope_namespace = %s AND workflow_id = %s"
|
|
107
|
+
),
|
|
108
|
+
"read_state_row": (
|
|
109
|
+
"SELECT workflow_id, request_id, tenant_scope_fingerprint, "
|
|
110
|
+
"scope_namespace, status, revision, schema_version, snapshot, "
|
|
111
|
+
"created_at, updated_at, expires_at FROM {schema}.workflow_states "
|
|
112
|
+
"WHERE scope_namespace = %s AND workflow_id = %s FOR UPDATE"
|
|
113
|
+
),
|
|
114
|
+
"update_state": (
|
|
115
|
+
"UPDATE {schema}.workflow_states "
|
|
116
|
+
"SET request_id = %s, tenant_scope_fingerprint = %s, status = %s, "
|
|
117
|
+
"revision = revision + 1, snapshot = %s, updated_at = %s "
|
|
118
|
+
"WHERE scope_namespace = %s AND workflow_id = %s AND status = %s "
|
|
119
|
+
"AND tenant_scope_fingerprint IS NOT DISTINCT FROM %s "
|
|
120
|
+
"AND (%s IS NULL OR revision = %s) "
|
|
121
|
+
"AND status NOT IN (%s, %s, %s)"
|
|
122
|
+
),
|
|
123
|
+
"update_state_fenced": (
|
|
124
|
+
"UPDATE {schema}.workflow_states "
|
|
125
|
+
"SET request_id = %s, tenant_scope_fingerprint = %s, status = %s, "
|
|
126
|
+
"revision = revision + 1, snapshot = %s, updated_at = %s "
|
|
127
|
+
"WHERE scope_namespace = %s AND workflow_id = %s AND status = %s "
|
|
128
|
+
"AND tenant_scope_fingerprint IS NOT DISTINCT FROM %s "
|
|
129
|
+
"AND (%s IS NULL OR revision = %s) "
|
|
130
|
+
"AND status NOT IN (%s, %s, %s) "
|
|
131
|
+
"AND EXISTS (SELECT 1 FROM {schema}.workflow_leases "
|
|
132
|
+
"WHERE scope_namespace = %s AND workflow_id = %s AND owner_id = %s "
|
|
133
|
+
"AND fencing_token = %s AND expires_at > NOW() + make_interval(secs => %s))"
|
|
134
|
+
),
|
|
135
|
+
"list_ids": (
|
|
136
|
+
"SELECT workflow_id FROM {schema}.workflow_states "
|
|
137
|
+
"WHERE scope_namespace = %s ORDER BY workflow_id"
|
|
138
|
+
),
|
|
139
|
+
"delete_states_batch": (
|
|
140
|
+
"DELETE FROM {schema}.workflow_states "
|
|
141
|
+
"WHERE (scope_namespace, workflow_id) IN ("
|
|
142
|
+
"SELECT scope_namespace, workflow_id FROM {schema}.workflow_states "
|
|
143
|
+
"WHERE status IN (%s, %s, %s) AND updated_at <= %s "
|
|
144
|
+
"ORDER BY updated_at, workflow_id LIMIT %s)"
|
|
145
|
+
),
|
|
146
|
+
"delete_idempotency_batch": (
|
|
147
|
+
"DELETE FROM {schema}.idempotency_records "
|
|
148
|
+
"WHERE (scope_namespace, idempotency_key) IN ("
|
|
149
|
+
"SELECT scope_namespace, idempotency_key FROM {schema}.idempotency_records "
|
|
150
|
+
"WHERE expires_at IS NOT NULL AND expires_at <= %s "
|
|
151
|
+
"ORDER BY expires_at, idempotency_key LIMIT %s)"
|
|
152
|
+
),
|
|
153
|
+
"delete_leases_batch": (
|
|
154
|
+
"DELETE FROM {schema}.workflow_leases "
|
|
155
|
+
"WHERE (scope_namespace, workflow_id) IN ("
|
|
156
|
+
"SELECT scope_namespace, workflow_id FROM {schema}.workflow_leases "
|
|
157
|
+
"WHERE expires_at <= %s ORDER BY expires_at, workflow_id LIMIT %s)"
|
|
158
|
+
),
|
|
159
|
+
"insert_idempotency": (
|
|
160
|
+
"INSERT INTO {schema}.idempotency_records ("
|
|
161
|
+
"idempotency_key, request_id, tenant_scope_fingerprint, scope_namespace, "
|
|
162
|
+
"workflow_id, status, terminal_outcome_fingerprint, created_at, updated_at, expires_at"
|
|
163
|
+
") VALUES (%s, %s, %s, %s, %s, %s, NULL, %s, %s, %s) "
|
|
164
|
+
"ON CONFLICT (scope_namespace, idempotency_key) DO NOTHING "
|
|
165
|
+
"RETURNING idempotency_key, request_id, tenant_scope_fingerprint, "
|
|
166
|
+
"scope_namespace, workflow_id, status, terminal_outcome_fingerprint, "
|
|
167
|
+
"created_at, updated_at, expires_at"
|
|
168
|
+
),
|
|
169
|
+
"read_idempotency_row": (
|
|
170
|
+
"SELECT idempotency_key, request_id, tenant_scope_fingerprint, "
|
|
171
|
+
"scope_namespace, workflow_id, status, terminal_outcome_fingerprint, "
|
|
172
|
+
"created_at, updated_at, expires_at FROM {schema}.idempotency_records "
|
|
173
|
+
"WHERE scope_namespace = %s AND idempotency_key = %s FOR UPDATE"
|
|
174
|
+
),
|
|
175
|
+
"read_idempotency_plain": (
|
|
176
|
+
"SELECT idempotency_key, request_id, tenant_scope_fingerprint, "
|
|
177
|
+
"scope_namespace, workflow_id, status, terminal_outcome_fingerprint, "
|
|
178
|
+
"created_at, updated_at, expires_at FROM {schema}.idempotency_records "
|
|
179
|
+
"WHERE scope_namespace = %s AND idempotency_key = %s"
|
|
180
|
+
),
|
|
181
|
+
"delete_idempotency": (
|
|
182
|
+
"DELETE FROM {schema}.idempotency_records "
|
|
183
|
+
"WHERE scope_namespace = %s AND idempotency_key = %s"
|
|
184
|
+
),
|
|
185
|
+
"complete_idempotency": (
|
|
186
|
+
"UPDATE {schema}.idempotency_records "
|
|
187
|
+
"SET workflow_id = %s, status = %s, terminal_outcome_fingerprint = %s, "
|
|
188
|
+
"updated_at = %s "
|
|
189
|
+
"WHERE scope_namespace = %s AND idempotency_key = %s AND status = 'reserved' "
|
|
190
|
+
"RETURNING idempotency_key, request_id, tenant_scope_fingerprint, "
|
|
191
|
+
"scope_namespace, workflow_id, status, terminal_outcome_fingerprint, "
|
|
192
|
+
"created_at, updated_at, expires_at"
|
|
193
|
+
),
|
|
194
|
+
"complete_idempotency_fenced": (
|
|
195
|
+
"UPDATE {schema}.idempotency_records "
|
|
196
|
+
"SET workflow_id = %s, status = %s, terminal_outcome_fingerprint = %s, "
|
|
197
|
+
"updated_at = %s "
|
|
198
|
+
"WHERE scope_namespace = %s AND idempotency_key = %s AND status = 'reserved' "
|
|
199
|
+
"AND EXISTS (SELECT 1 FROM {schema}.workflow_leases "
|
|
200
|
+
"WHERE scope_namespace = %s AND workflow_id = %s AND owner_id = %s "
|
|
201
|
+
"AND fencing_token = %s AND expires_at > NOW() + make_interval(secs => %s)) "
|
|
202
|
+
"RETURNING idempotency_key, request_id, tenant_scope_fingerprint, "
|
|
203
|
+
"scope_namespace, workflow_id, status, terminal_outcome_fingerprint, "
|
|
204
|
+
"created_at, updated_at, expires_at"
|
|
205
|
+
),
|
|
206
|
+
"acquire_lease": (
|
|
207
|
+
"INSERT INTO {schema}.workflow_leases ("
|
|
208
|
+
"scope_namespace, workflow_id, owner_id, fencing_token, expires_at, updated_at"
|
|
209
|
+
") VALUES (%s, %s, %s, 1, %s, NOW()) "
|
|
210
|
+
"ON CONFLICT (scope_namespace, workflow_id) DO UPDATE "
|
|
211
|
+
"SET owner_id = EXCLUDED.owner_id, "
|
|
212
|
+
"fencing_token = {schema}.workflow_leases.fencing_token + 1, "
|
|
213
|
+
"expires_at = EXCLUDED.expires_at, updated_at = NOW() "
|
|
214
|
+
"WHERE {schema}.workflow_leases.expires_at <= NOW() - make_interval(secs => %s) "
|
|
215
|
+
"RETURNING owner_id, fencing_token, expires_at, updated_at"
|
|
216
|
+
),
|
|
217
|
+
"renew_lease": (
|
|
218
|
+
"UPDATE {schema}.workflow_leases "
|
|
219
|
+
"SET expires_at = %s, updated_at = NOW() "
|
|
220
|
+
"WHERE scope_namespace = %s AND workflow_id = %s AND owner_id = %s "
|
|
221
|
+
"AND fencing_token = %s AND expires_at > NOW() + make_interval(secs => %s) "
|
|
222
|
+
"RETURNING owner_id, fencing_token, expires_at, updated_at"
|
|
223
|
+
),
|
|
224
|
+
"release_lease": (
|
|
225
|
+
"DELETE FROM {schema}.workflow_leases "
|
|
226
|
+
"WHERE scope_namespace = %s AND workflow_id = %s AND owner_id = %s "
|
|
227
|
+
"AND fencing_token = %s"
|
|
228
|
+
),
|
|
229
|
+
"inspect_lease": (
|
|
230
|
+
"SELECT owner_id, fencing_token, expires_at, updated_at "
|
|
231
|
+
"FROM {schema}.workflow_leases "
|
|
232
|
+
"WHERE scope_namespace = %s AND workflow_id = %s"
|
|
233
|
+
),
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _utc_now() -> datetime:
|
|
238
|
+
return datetime.now(UTC)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _iso(value: datetime | None) -> str | None:
|
|
242
|
+
return value.isoformat() if value is not None else None
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _parse_dt(value: Any) -> datetime:
|
|
246
|
+
if isinstance(value, datetime):
|
|
247
|
+
return value
|
|
248
|
+
return datetime.fromisoformat(str(value))
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _namespace(fingerprint: str | None) -> str:
|
|
252
|
+
"""The scope namespace for a fingerprint, or the local non-tenant namespace."""
|
|
253
|
+
return tenant_scope_namespace(fingerprint) if fingerprint is not None else ""
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
class PostgreSQLStateStore:
|
|
257
|
+
"""Shared state store persisting safe snapshots to PostgreSQL.
|
|
258
|
+
|
|
259
|
+
Records are keyed by ``(scope_namespace, workflow_id)`` so tenant-scoped
|
|
260
|
+
workflows are isolated and unscoped lookups can never observe them.
|
|
261
|
+
All mutations run in one transaction per operation with conditional
|
|
262
|
+
statements; conflicts surface as :class:`WorkflowStateError` /
|
|
263
|
+
:class:`IdempotencyConflictError`, ownership failures as
|
|
264
|
+
:class:`SharedStoreError` (``LEASE_BUSY``/``FENCING_REJECTED``), and
|
|
265
|
+
backend outages/timeouts as retryable normalized errors.
|
|
266
|
+
"""
|
|
267
|
+
|
|
268
|
+
def __init__(
|
|
269
|
+
self,
|
|
270
|
+
*,
|
|
271
|
+
dsn: str | None = None,
|
|
272
|
+
config: WorkflowPostgresConfig | None = None,
|
|
273
|
+
pool: Any | None = None,
|
|
274
|
+
now: Callable[[], datetime] | None = None,
|
|
275
|
+
) -> None:
|
|
276
|
+
"""Build the store over a DSN (lazy psycopg pool) or an injected pool.
|
|
277
|
+
|
|
278
|
+
Exactly one of ``dsn`` or ``pool`` is required. The injected pool
|
|
279
|
+
seam (fake or host-managed) keeps the store testable without the
|
|
280
|
+
optional driver installed; ``now`` injects the client clock for
|
|
281
|
+
deterministic tests.
|
|
282
|
+
"""
|
|
283
|
+
if (dsn is None) == (pool is None):
|
|
284
|
+
raise ValueError("exactly one of 'dsn' or 'pool' is required")
|
|
285
|
+
self._config = config or WorkflowPostgresConfig(namespace="shared")
|
|
286
|
+
self._schema = self._config.namespace
|
|
287
|
+
self._quoted_schema = f'"{self._schema}"'
|
|
288
|
+
self._sql = {
|
|
289
|
+
name: template.format(schema=self._quoted_schema)
|
|
290
|
+
for name, template in SQL_TEMPLATES.items()
|
|
291
|
+
}
|
|
292
|
+
if pool is not None:
|
|
293
|
+
self._pool = pool
|
|
294
|
+
else:
|
|
295
|
+
assert dsn is not None
|
|
296
|
+
self._pool = build_pool(
|
|
297
|
+
dsn,
|
|
298
|
+
pool_size=self._config.pool_size,
|
|
299
|
+
connect_timeout_seconds=self._config.connect_timeout_seconds,
|
|
300
|
+
command_timeout_seconds=self._config.command_timeout_seconds,
|
|
301
|
+
acquire_timeout_seconds=self._config.pool_acquire_timeout_seconds,
|
|
302
|
+
schema=self._schema,
|
|
303
|
+
)
|
|
304
|
+
self._now_fn = now or _utc_now
|
|
305
|
+
self._closed = False
|
|
306
|
+
self._initialize_schema()
|
|
307
|
+
|
|
308
|
+
# -- schema and connection -------------------------------------------
|
|
309
|
+
|
|
310
|
+
@property
|
|
311
|
+
def schema(self) -> str:
|
|
312
|
+
"""The deployment schema namespace owning every shared table."""
|
|
313
|
+
return self._schema
|
|
314
|
+
|
|
315
|
+
def schema_version(self) -> int:
|
|
316
|
+
"""The persisted schema version read from schema metadata."""
|
|
317
|
+
with self._transaction() as conn:
|
|
318
|
+
cursor = self._execute(conn, "read_schema_version")
|
|
319
|
+
row = cursor.fetchone()
|
|
320
|
+
return int(row["value"]) if row is not None else 0
|
|
321
|
+
|
|
322
|
+
def _initialize_schema(self) -> None:
|
|
323
|
+
with self._transaction() as conn:
|
|
324
|
+
try:
|
|
325
|
+
# The deployment namespace schema is created lazily so a
|
|
326
|
+
# fresh DSN never requires manual DDL before first use.
|
|
327
|
+
self._execute_raw(
|
|
328
|
+
conn, f"CREATE SCHEMA IF NOT EXISTS {self._quoted_schema}"
|
|
329
|
+
)
|
|
330
|
+
self._execute_raw(
|
|
331
|
+
conn, _BOOTSTRAP_DDL.format(schema=self._quoted_schema)
|
|
332
|
+
)
|
|
333
|
+
cursor = self._execute(conn, "read_schema_version")
|
|
334
|
+
row = cursor.fetchone()
|
|
335
|
+
current = int(row["value"]) if row is not None else 0
|
|
336
|
+
target = self._config.schema_version
|
|
337
|
+
if current > target:
|
|
338
|
+
raise SharedStoreError(
|
|
339
|
+
SharedStoreErrorCode.SCHEMA_MISMATCH,
|
|
340
|
+
f"database schema version {current} is newer than the "
|
|
341
|
+
f"configured {target}",
|
|
342
|
+
details={
|
|
343
|
+
"database_schema_version": str(current),
|
|
344
|
+
"configured": str(target),
|
|
345
|
+
},
|
|
346
|
+
)
|
|
347
|
+
if current > SUPPORTED_SCHEMA_VERSION:
|
|
348
|
+
raise SharedStoreError(
|
|
349
|
+
SharedStoreErrorCode.SCHEMA_MISMATCH,
|
|
350
|
+
f"database schema version {current} is newer than supported "
|
|
351
|
+
f"{SUPPORTED_SCHEMA_VERSION}",
|
|
352
|
+
details={
|
|
353
|
+
"database_schema_version": str(current),
|
|
354
|
+
"supported": str(SUPPORTED_SCHEMA_VERSION),
|
|
355
|
+
},
|
|
356
|
+
)
|
|
357
|
+
for version in range(current + 1, target + 1):
|
|
358
|
+
for statement in MIGRATIONS[version]:
|
|
359
|
+
self._execute_raw(conn, statement)
|
|
360
|
+
if current < target:
|
|
361
|
+
self._execute(
|
|
362
|
+
conn, "write_schema_version", (str(target),)
|
|
363
|
+
)
|
|
364
|
+
except SharedStoreError:
|
|
365
|
+
raise
|
|
366
|
+
except NL2DataError:
|
|
367
|
+
raise
|
|
368
|
+
except Exception as error:
|
|
369
|
+
raise self._map_backend_error(
|
|
370
|
+
error, operation="initialize"
|
|
371
|
+
) from error
|
|
372
|
+
|
|
373
|
+
def close(self) -> None:
|
|
374
|
+
"""Close the pool (idempotent); later operations fail closed."""
|
|
375
|
+
if self._closed:
|
|
376
|
+
return
|
|
377
|
+
self._closed = True
|
|
378
|
+
close = getattr(self._pool, "close", None)
|
|
379
|
+
if callable(close):
|
|
380
|
+
close()
|
|
381
|
+
|
|
382
|
+
# -- transaction and error mapping -----------------------------------
|
|
383
|
+
|
|
384
|
+
@contextlib.contextmanager
|
|
385
|
+
def _transaction(self) -> Iterator[Any]:
|
|
386
|
+
if self._closed:
|
|
387
|
+
raise SharedStoreError(
|
|
388
|
+
SharedStoreErrorCode.STORE_UNAVAILABLE,
|
|
389
|
+
"shared state store is closed",
|
|
390
|
+
details={"cause_type": "ClosedStore"},
|
|
391
|
+
)
|
|
392
|
+
with self._pool.connection() as conn:
|
|
393
|
+
try:
|
|
394
|
+
yield conn
|
|
395
|
+
conn.commit()
|
|
396
|
+
except BaseException:
|
|
397
|
+
with contextlib.suppress(Exception):
|
|
398
|
+
conn.rollback()
|
|
399
|
+
raise
|
|
400
|
+
|
|
401
|
+
def _execute(
|
|
402
|
+
self,
|
|
403
|
+
conn: Any,
|
|
404
|
+
name: str,
|
|
405
|
+
params: tuple[Any, ...] = (),
|
|
406
|
+
) -> Any:
|
|
407
|
+
"""Run one named statement with the bounded command timeout."""
|
|
408
|
+
try:
|
|
409
|
+
cursor = conn.cursor()
|
|
410
|
+
self._set_command_timeout(conn, cursor)
|
|
411
|
+
return cursor.execute(self._sql[name], params)
|
|
412
|
+
except Exception as error:
|
|
413
|
+
raise self._map_backend_error(error, operation=name) from error
|
|
414
|
+
|
|
415
|
+
def _execute_raw(self, conn: Any, statement: str) -> Any:
|
|
416
|
+
"""Run a raw migration statement with the bounded command timeout."""
|
|
417
|
+
try:
|
|
418
|
+
cursor = conn.cursor()
|
|
419
|
+
self._set_command_timeout(conn, cursor)
|
|
420
|
+
return cursor.execute(statement, ())
|
|
421
|
+
except Exception as error:
|
|
422
|
+
raise self._map_backend_error(error, operation="migration") from error
|
|
423
|
+
|
|
424
|
+
def _set_command_timeout(self, conn: Any, cursor: Any) -> None:
|
|
425
|
+
"""Apply a command timeout across fake and psycopg cursor APIs."""
|
|
426
|
+
if hasattr(cursor, "timeout"):
|
|
427
|
+
cursor.timeout = self._config.command_timeout_seconds
|
|
428
|
+
return
|
|
429
|
+
timeout_ms = int(self._config.command_timeout_seconds * 1000)
|
|
430
|
+
conn.execute(
|
|
431
|
+
"SELECT set_config('statement_timeout', %s, true)",
|
|
432
|
+
(str(timeout_ms),),
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
def _map_backend_error(
|
|
436
|
+
self, error: Exception, *, operation: str
|
|
437
|
+
) -> Exception:
|
|
438
|
+
"""Normalize a driver failure into a safe structured error."""
|
|
439
|
+
if isinstance(
|
|
440
|
+
error,
|
|
441
|
+
(
|
|
442
|
+
SharedStoreError,
|
|
443
|
+
WorkflowStateError,
|
|
444
|
+
WorkflowTransitionError,
|
|
445
|
+
IdempotencyConflictError,
|
|
446
|
+
WorkflowSerializationError,
|
|
447
|
+
),
|
|
448
|
+
):
|
|
449
|
+
return error
|
|
450
|
+
if is_timeout_error(error):
|
|
451
|
+
return SharedStoreError(
|
|
452
|
+
SharedStoreErrorCode.STORE_TIMEOUT,
|
|
453
|
+
"shared state backend command timed out",
|
|
454
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
455
|
+
cause=error,
|
|
456
|
+
)
|
|
457
|
+
if is_duplicate_key_error(error):
|
|
458
|
+
return SharedStoreError(
|
|
459
|
+
SharedStoreErrorCode.STATE_CONFLICT,
|
|
460
|
+
"shared state backend rejected a duplicate record",
|
|
461
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
462
|
+
cause=error,
|
|
463
|
+
)
|
|
464
|
+
if is_connect_error(error) or is_serialization_error(error):
|
|
465
|
+
return SharedStoreError(
|
|
466
|
+
SharedStoreErrorCode.STORE_UNAVAILABLE,
|
|
467
|
+
"shared state backend is unreachable",
|
|
468
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
469
|
+
cause=error,
|
|
470
|
+
)
|
|
471
|
+
return SharedStoreError(
|
|
472
|
+
SharedStoreErrorCode.STORE_UNAVAILABLE,
|
|
473
|
+
REDACTED_VALUE,
|
|
474
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
475
|
+
cause=error,
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
# -- workflow state operations ---------------------------------------
|
|
479
|
+
|
|
480
|
+
def create(self, state: WorkflowState) -> None:
|
|
481
|
+
namespace = _namespace(state.tenant_scope_fingerprint)
|
|
482
|
+
snapshot = serialize_snapshot(state)
|
|
483
|
+
now = self._now_fn()
|
|
484
|
+
with self._transaction() as conn:
|
|
485
|
+
cursor = self._execute(
|
|
486
|
+
conn,
|
|
487
|
+
"create_workflow",
|
|
488
|
+
(
|
|
489
|
+
state.workflow_id,
|
|
490
|
+
state.request_id,
|
|
491
|
+
state.tenant_scope_fingerprint,
|
|
492
|
+
namespace,
|
|
493
|
+
state.status.value,
|
|
494
|
+
SUPPORTED_SCHEMA_VERSION,
|
|
495
|
+
snapshot,
|
|
496
|
+
_iso(now),
|
|
497
|
+
_iso(now),
|
|
498
|
+
),
|
|
499
|
+
)
|
|
500
|
+
if cursor.rowcount != 1:
|
|
501
|
+
raise WorkflowStateError(
|
|
502
|
+
f"workflow '{state.workflow_id}' already exists",
|
|
503
|
+
details={"workflow_id": state.workflow_id},
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
def get(
|
|
507
|
+
self, workflow_id: str, *, tenant_scope_fingerprint: str | None = None
|
|
508
|
+
) -> WorkflowState | None:
|
|
509
|
+
return self._read_state(
|
|
510
|
+
workflow_id, request_id=None, tenant_scope_fingerprint=tenant_scope_fingerprint
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
def get_revision(
|
|
514
|
+
self, workflow_id: str, *, tenant_scope_fingerprint: str | None = None
|
|
515
|
+
) -> int | None:
|
|
516
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
517
|
+
with self._transaction() as conn:
|
|
518
|
+
cursor = self._execute(
|
|
519
|
+
conn, "read_revision", (namespace, workflow_id)
|
|
520
|
+
)
|
|
521
|
+
row = cursor.fetchone()
|
|
522
|
+
return int(row["revision"]) if row is not None else None
|
|
523
|
+
|
|
524
|
+
def get_checkpoint(
|
|
525
|
+
self,
|
|
526
|
+
workflow_id: str,
|
|
527
|
+
request_id: str,
|
|
528
|
+
*,
|
|
529
|
+
tenant_scope_fingerprint: str | None = None,
|
|
530
|
+
) -> WorkflowState | None:
|
|
531
|
+
return self._read_state(
|
|
532
|
+
workflow_id, request_id=request_id, tenant_scope_fingerprint=tenant_scope_fingerprint
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
def _read_state(
|
|
536
|
+
self,
|
|
537
|
+
workflow_id: str,
|
|
538
|
+
*,
|
|
539
|
+
request_id: str | None,
|
|
540
|
+
tenant_scope_fingerprint: str | None,
|
|
541
|
+
) -> WorkflowState | None:
|
|
542
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
543
|
+
name = "read_checkpoint" if request_id is not None else "read_snapshot"
|
|
544
|
+
params: tuple[Any, ...] = (
|
|
545
|
+
(namespace, workflow_id, request_id)
|
|
546
|
+
if request_id is not None
|
|
547
|
+
else (namespace, workflow_id)
|
|
548
|
+
)
|
|
549
|
+
with self._transaction() as conn:
|
|
550
|
+
cursor = self._execute(conn, name, params)
|
|
551
|
+
row = cursor.fetchone()
|
|
552
|
+
if row is None:
|
|
553
|
+
return None
|
|
554
|
+
try:
|
|
555
|
+
return deserialize_snapshot(row["snapshot"])
|
|
556
|
+
except WorkflowSerializationError as error:
|
|
557
|
+
raise WorkflowSerializationError(
|
|
558
|
+
f"stored snapshot for workflow '{workflow_id}' is malformed",
|
|
559
|
+
code=error.code,
|
|
560
|
+
details={"workflow_id": workflow_id, **error.details},
|
|
561
|
+
) from error
|
|
562
|
+
|
|
563
|
+
def get_record(
|
|
564
|
+
self, workflow_id: str, *, tenant_scope_fingerprint: str | None = None
|
|
565
|
+
) -> DurableWorkflowRecord | None:
|
|
566
|
+
"""Return the full durable record including revision and timestamps."""
|
|
567
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
568
|
+
with self._transaction() as conn:
|
|
569
|
+
cursor = self._execute(conn, "read_state_row", (namespace, workflow_id))
|
|
570
|
+
row = cursor.fetchone()
|
|
571
|
+
if row is None:
|
|
572
|
+
return None
|
|
573
|
+
return DurableWorkflowRecord(
|
|
574
|
+
schema_version=int(row["schema_version"]),
|
|
575
|
+
workflow_id=str(row["workflow_id"]),
|
|
576
|
+
request_id=str(row["request_id"]),
|
|
577
|
+
tenant_scope_fingerprint=row["tenant_scope_fingerprint"],
|
|
578
|
+
scope_namespace=str(row["scope_namespace"]),
|
|
579
|
+
status=WorkflowStatus(str(row["status"])),
|
|
580
|
+
revision=int(row["revision"]),
|
|
581
|
+
snapshot=str(row["snapshot"]),
|
|
582
|
+
created_at=_parse_dt(row["created_at"]),
|
|
583
|
+
updated_at=_parse_dt(row["updated_at"]),
|
|
584
|
+
expires_at=(
|
|
585
|
+
_parse_dt(row["expires_at"]) if row["expires_at"] is not None else None
|
|
586
|
+
),
|
|
587
|
+
)
|
|
588
|
+
|
|
589
|
+
def update(
|
|
590
|
+
self,
|
|
591
|
+
workflow_id: str,
|
|
592
|
+
expected_status: WorkflowStatus,
|
|
593
|
+
state: WorkflowState,
|
|
594
|
+
*,
|
|
595
|
+
expected_version: int | None = None,
|
|
596
|
+
tenant_scope_fingerprint: str | None = None,
|
|
597
|
+
owner_id: str | None = None,
|
|
598
|
+
fencing_token: int | None = None,
|
|
599
|
+
) -> None:
|
|
600
|
+
"""Transactional compare-and-set; ownership is enforced when supplied."""
|
|
601
|
+
if state.workflow_id != workflow_id:
|
|
602
|
+
raise WorkflowStateError(
|
|
603
|
+
"workflow snapshot identity does not match the requested workflow",
|
|
604
|
+
details={"workflow_id": workflow_id},
|
|
605
|
+
)
|
|
606
|
+
if state.tenant_scope_fingerprint != tenant_scope_fingerprint:
|
|
607
|
+
raise WorkflowStateError(
|
|
608
|
+
f"workflow '{workflow_id}' tenant scope mismatch",
|
|
609
|
+
details={"workflow_id": workflow_id},
|
|
610
|
+
)
|
|
611
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
612
|
+
snapshot = serialize_snapshot(state)
|
|
613
|
+
now = self._now_fn()
|
|
614
|
+
fenced = owner_id is not None and fencing_token is not None
|
|
615
|
+
with self._transaction() as conn:
|
|
616
|
+
if fenced:
|
|
617
|
+
cursor = self._execute(
|
|
618
|
+
conn,
|
|
619
|
+
"update_state_fenced",
|
|
620
|
+
(
|
|
621
|
+
state.request_id,
|
|
622
|
+
state.tenant_scope_fingerprint,
|
|
623
|
+
state.status.value,
|
|
624
|
+
snapshot,
|
|
625
|
+
_iso(now),
|
|
626
|
+
namespace,
|
|
627
|
+
workflow_id,
|
|
628
|
+
expected_status.value,
|
|
629
|
+
tenant_scope_fingerprint,
|
|
630
|
+
expected_version,
|
|
631
|
+
expected_version,
|
|
632
|
+
*(status.value for status in TERMINAL_STATUSES),
|
|
633
|
+
namespace,
|
|
634
|
+
workflow_id,
|
|
635
|
+
owner_id,
|
|
636
|
+
fencing_token,
|
|
637
|
+
self._config.clock_tolerance_seconds,
|
|
638
|
+
),
|
|
639
|
+
)
|
|
640
|
+
else:
|
|
641
|
+
cursor = self._execute(
|
|
642
|
+
conn,
|
|
643
|
+
"update_state",
|
|
644
|
+
(
|
|
645
|
+
state.request_id,
|
|
646
|
+
state.tenant_scope_fingerprint,
|
|
647
|
+
state.status.value,
|
|
648
|
+
snapshot,
|
|
649
|
+
_iso(now),
|
|
650
|
+
namespace,
|
|
651
|
+
workflow_id,
|
|
652
|
+
expected_status.value,
|
|
653
|
+
tenant_scope_fingerprint,
|
|
654
|
+
expected_version,
|
|
655
|
+
expected_version,
|
|
656
|
+
*(status.value for status in TERMINAL_STATUSES),
|
|
657
|
+
),
|
|
658
|
+
)
|
|
659
|
+
if cursor.rowcount == 1:
|
|
660
|
+
return
|
|
661
|
+
row = self._execute(
|
|
662
|
+
conn, "read_state_row", (namespace, workflow_id)
|
|
663
|
+
).fetchone()
|
|
664
|
+
if row is None:
|
|
665
|
+
raise WorkflowStateError(
|
|
666
|
+
f"workflow '{workflow_id}' not found",
|
|
667
|
+
details={"workflow_id": workflow_id},
|
|
668
|
+
)
|
|
669
|
+
if row["tenant_scope_fingerprint"] != state.tenant_scope_fingerprint:
|
|
670
|
+
raise WorkflowStateError(
|
|
671
|
+
f"workflow '{workflow_id}' tenant scope mismatch",
|
|
672
|
+
details={"workflow_id": workflow_id},
|
|
673
|
+
)
|
|
674
|
+
current_status = WorkflowStatus(str(row["status"]))
|
|
675
|
+
if current_status in TERMINAL_STATUSES:
|
|
676
|
+
if state.status != current_status:
|
|
677
|
+
raise WorkflowTransitionError(
|
|
678
|
+
f"cannot transition from terminal status '{current_status.value}'",
|
|
679
|
+
details={"from": current_status.value, "to": state.status.value},
|
|
680
|
+
)
|
|
681
|
+
# Idempotent same-status rewrite of a terminal record is a no-op.
|
|
682
|
+
return
|
|
683
|
+
if current_status != expected_status:
|
|
684
|
+
raise WorkflowStateError(
|
|
685
|
+
f"workflow '{workflow_id}' status changed concurrently",
|
|
686
|
+
details={
|
|
687
|
+
"workflow_id": workflow_id,
|
|
688
|
+
"expected": expected_status.value,
|
|
689
|
+
"actual": str(row["status"]),
|
|
690
|
+
},
|
|
691
|
+
)
|
|
692
|
+
if (
|
|
693
|
+
expected_version is not None
|
|
694
|
+
and int(row["revision"]) != expected_version
|
|
695
|
+
):
|
|
696
|
+
raise WorkflowStateError(
|
|
697
|
+
f"workflow '{workflow_id}' version changed concurrently",
|
|
698
|
+
details={
|
|
699
|
+
"workflow_id": workflow_id,
|
|
700
|
+
"expected_version": str(expected_version),
|
|
701
|
+
"actual": str(row["revision"]),
|
|
702
|
+
},
|
|
703
|
+
)
|
|
704
|
+
raise SharedStoreError(
|
|
705
|
+
SharedStoreErrorCode.FENCING_REJECTED,
|
|
706
|
+
f"workflow '{workflow_id}' lease ownership was lost",
|
|
707
|
+
details={"workflow_id": workflow_id},
|
|
708
|
+
)
|
|
709
|
+
|
|
710
|
+
def list_ids(self, *, tenant_scope_fingerprint: str | None = None) -> tuple[str, ...]:
|
|
711
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
712
|
+
with self._transaction() as conn:
|
|
713
|
+
cursor = self._execute(conn, "list_ids", (namespace,))
|
|
714
|
+
return tuple(str(row["workflow_id"]) for row in cursor.fetchall())
|
|
715
|
+
|
|
716
|
+
def cleanup(
|
|
717
|
+
self,
|
|
718
|
+
*,
|
|
719
|
+
terminal_before: datetime,
|
|
720
|
+
expired_before: datetime,
|
|
721
|
+
max_records: int,
|
|
722
|
+
) -> int:
|
|
723
|
+
"""Delete bounded batches of terminal/expired records in one transaction.
|
|
724
|
+
|
|
725
|
+
Only terminal workflow snapshots older than ``terminal_before``,
|
|
726
|
+
idempotency records expired before ``expired_before``, and leases
|
|
727
|
+
expired beyond the clock tolerance are removed; running workflows
|
|
728
|
+
and valid leases are never touched by cleanup.
|
|
729
|
+
"""
|
|
730
|
+
if max_records < 1:
|
|
731
|
+
raise ValueError("max_records must be positive")
|
|
732
|
+
batch = min(max_records, self._config.cleanup_batch_size)
|
|
733
|
+
lease_cutoff = expired_before - timedelta(
|
|
734
|
+
seconds=self._config.clock_tolerance_seconds
|
|
735
|
+
)
|
|
736
|
+
with self._transaction() as conn:
|
|
737
|
+
removed_states = self._execute(
|
|
738
|
+
conn,
|
|
739
|
+
"delete_states_batch",
|
|
740
|
+
(*_TERMINAL_VALUES, _iso(terminal_before), batch),
|
|
741
|
+
).rowcount
|
|
742
|
+
removed_keys = self._execute(
|
|
743
|
+
conn,
|
|
744
|
+
"delete_idempotency_batch",
|
|
745
|
+
(_iso(expired_before), batch),
|
|
746
|
+
).rowcount
|
|
747
|
+
removed_leases = self._execute(
|
|
748
|
+
conn,
|
|
749
|
+
"delete_leases_batch",
|
|
750
|
+
(_iso(lease_cutoff), batch),
|
|
751
|
+
).rowcount
|
|
752
|
+
return (
|
|
753
|
+
int(removed_states) + int(removed_keys) + int(removed_leases)
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
# -- idempotency-key records -----------------------------------------
|
|
757
|
+
|
|
758
|
+
def reserve_idempotency(
|
|
759
|
+
self,
|
|
760
|
+
key: str,
|
|
761
|
+
*,
|
|
762
|
+
request_id: str,
|
|
763
|
+
workflow_id: str,
|
|
764
|
+
tenant_scope_fingerprint: str | None = None,
|
|
765
|
+
expires_at: datetime | None = None,
|
|
766
|
+
) -> IdempotencyRecord:
|
|
767
|
+
"""Atomically bind a key to one request within its scope namespace.
|
|
768
|
+
|
|
769
|
+
Reuse with a different request identity or scope raises
|
|
770
|
+
:class:`IdempotencyConflictError`; an expired record is reusable;
|
|
771
|
+
otherwise the existing record is returned unchanged.
|
|
772
|
+
"""
|
|
773
|
+
self._validate_identity(key, request_id, workflow_id, tenant_scope_fingerprint)
|
|
774
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
775
|
+
now = self._now_fn()
|
|
776
|
+
with self._transaction() as conn:
|
|
777
|
+
cursor = self._execute(
|
|
778
|
+
conn,
|
|
779
|
+
"insert_idempotency",
|
|
780
|
+
(
|
|
781
|
+
key,
|
|
782
|
+
request_id,
|
|
783
|
+
tenant_scope_fingerprint,
|
|
784
|
+
namespace,
|
|
785
|
+
workflow_id,
|
|
786
|
+
IdempotencyStatus.RESERVED.value,
|
|
787
|
+
_iso(now),
|
|
788
|
+
_iso(now),
|
|
789
|
+
_iso(expires_at),
|
|
790
|
+
),
|
|
791
|
+
)
|
|
792
|
+
row = cursor.fetchone()
|
|
793
|
+
if row is not None:
|
|
794
|
+
return self._record_from_row(row)
|
|
795
|
+
row = self._execute(
|
|
796
|
+
conn, "read_idempotency_row", (namespace, key)
|
|
797
|
+
).fetchone()
|
|
798
|
+
if row is None:
|
|
799
|
+
raise WorkflowStateError(
|
|
800
|
+
f"idempotency key '{key}' not found",
|
|
801
|
+
details={"idempotency_key": key},
|
|
802
|
+
)
|
|
803
|
+
stored_expiry = row["expires_at"]
|
|
804
|
+
if stored_expiry is not None and _parse_dt(stored_expiry) <= now:
|
|
805
|
+
self._execute(conn, "delete_idempotency", (namespace, key))
|
|
806
|
+
cursor = self._execute(
|
|
807
|
+
conn,
|
|
808
|
+
"insert_idempotency",
|
|
809
|
+
(
|
|
810
|
+
key,
|
|
811
|
+
request_id,
|
|
812
|
+
tenant_scope_fingerprint,
|
|
813
|
+
namespace,
|
|
814
|
+
workflow_id,
|
|
815
|
+
IdempotencyStatus.RESERVED.value,
|
|
816
|
+
_iso(now),
|
|
817
|
+
_iso(now),
|
|
818
|
+
_iso(expires_at),
|
|
819
|
+
),
|
|
820
|
+
)
|
|
821
|
+
row = cursor.fetchone()
|
|
822
|
+
if row is not None:
|
|
823
|
+
return self._record_from_row(row)
|
|
824
|
+
raise WorkflowStateError(
|
|
825
|
+
f"idempotency key '{key}' not found",
|
|
826
|
+
details={"idempotency_key": key},
|
|
827
|
+
)
|
|
828
|
+
if row["request_id"] != request_id:
|
|
829
|
+
raise IdempotencyConflictError(
|
|
830
|
+
f"idempotency key '{key}' is bound to a different request",
|
|
831
|
+
details={
|
|
832
|
+
"idempotency_key": key,
|
|
833
|
+
"bound_request_id": str(row["request_id"]),
|
|
834
|
+
},
|
|
835
|
+
)
|
|
836
|
+
if row["tenant_scope_fingerprint"] != tenant_scope_fingerprint:
|
|
837
|
+
raise IdempotencyConflictError(
|
|
838
|
+
f"idempotency key '{key}' is bound to a different tenant scope",
|
|
839
|
+
details={"idempotency_key": key},
|
|
840
|
+
)
|
|
841
|
+
if row["workflow_id"] != workflow_id:
|
|
842
|
+
raise IdempotencyConflictError(
|
|
843
|
+
f"idempotency key '{key}' is bound to a different workflow",
|
|
844
|
+
details={"idempotency_key": key},
|
|
845
|
+
)
|
|
846
|
+
return self._record_from_row(row)
|
|
847
|
+
|
|
848
|
+
def complete_idempotency(
|
|
849
|
+
self,
|
|
850
|
+
key: str,
|
|
851
|
+
*,
|
|
852
|
+
workflow_id: str,
|
|
853
|
+
terminal_outcome_fingerprint: str,
|
|
854
|
+
tenant_scope_fingerprint: str | None = None,
|
|
855
|
+
owner_id: str | None = None,
|
|
856
|
+
fencing_token: int | None = None,
|
|
857
|
+
) -> IdempotencyRecord:
|
|
858
|
+
"""Store the terminal outcome reference on a reserved key.
|
|
859
|
+
|
|
860
|
+
Completion is conditional on the reserved status and, when
|
|
861
|
+
ownership is supplied, on the current lease owner/fencing token;
|
|
862
|
+
a completed record is never overwritten.
|
|
863
|
+
"""
|
|
864
|
+
self._validate_identity(key, workflow_id=workflow_id, scope=tenant_scope_fingerprint)
|
|
865
|
+
if _FINGERPRINT_PATTERN.fullmatch(terminal_outcome_fingerprint) is None:
|
|
866
|
+
raise WorkflowStateError(
|
|
867
|
+
"terminal outcome reference must be a sha256 fingerprint",
|
|
868
|
+
details={"idempotency_key": key},
|
|
869
|
+
)
|
|
870
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
871
|
+
now = self._now_fn()
|
|
872
|
+
fenced = owner_id is not None and fencing_token is not None
|
|
873
|
+
with self._transaction() as conn:
|
|
874
|
+
if fenced:
|
|
875
|
+
cursor = self._execute(
|
|
876
|
+
conn,
|
|
877
|
+
"complete_idempotency_fenced",
|
|
878
|
+
(
|
|
879
|
+
workflow_id,
|
|
880
|
+
IdempotencyStatus.COMPLETED.value,
|
|
881
|
+
terminal_outcome_fingerprint,
|
|
882
|
+
_iso(now),
|
|
883
|
+
namespace,
|
|
884
|
+
key,
|
|
885
|
+
namespace,
|
|
886
|
+
workflow_id,
|
|
887
|
+
owner_id,
|
|
888
|
+
fencing_token,
|
|
889
|
+
self._config.clock_tolerance_seconds,
|
|
890
|
+
),
|
|
891
|
+
)
|
|
892
|
+
else:
|
|
893
|
+
cursor = self._execute(
|
|
894
|
+
conn,
|
|
895
|
+
"complete_idempotency",
|
|
896
|
+
(
|
|
897
|
+
workflow_id,
|
|
898
|
+
IdempotencyStatus.COMPLETED.value,
|
|
899
|
+
terminal_outcome_fingerprint,
|
|
900
|
+
_iso(now),
|
|
901
|
+
namespace,
|
|
902
|
+
key,
|
|
903
|
+
),
|
|
904
|
+
)
|
|
905
|
+
row = cursor.fetchone()
|
|
906
|
+
if row is not None:
|
|
907
|
+
return self._record_from_row(row)
|
|
908
|
+
row = self._execute(
|
|
909
|
+
conn, "read_idempotency_row", (namespace, key)
|
|
910
|
+
).fetchone()
|
|
911
|
+
if row is None:
|
|
912
|
+
raise WorkflowStateError(
|
|
913
|
+
f"idempotency key '{key}' not found",
|
|
914
|
+
details={"idempotency_key": key},
|
|
915
|
+
)
|
|
916
|
+
if row["tenant_scope_fingerprint"] != tenant_scope_fingerprint:
|
|
917
|
+
raise IdempotencyConflictError(
|
|
918
|
+
f"idempotency key '{key}' is bound to a different tenant scope",
|
|
919
|
+
details={"idempotency_key": key},
|
|
920
|
+
)
|
|
921
|
+
if row["workflow_id"] != workflow_id:
|
|
922
|
+
raise IdempotencyConflictError(
|
|
923
|
+
f"idempotency key '{key}' is bound to a different workflow",
|
|
924
|
+
details={"idempotency_key": key},
|
|
925
|
+
)
|
|
926
|
+
if row["status"] != IdempotencyStatus.RESERVED.value:
|
|
927
|
+
raise IdempotencyConflictError(
|
|
928
|
+
f"idempotency key '{key}' is already completed",
|
|
929
|
+
details={"idempotency_key": key},
|
|
930
|
+
)
|
|
931
|
+
raise SharedStoreError(
|
|
932
|
+
SharedStoreErrorCode.FENCING_REJECTED,
|
|
933
|
+
f"idempotency key '{key}' lease ownership was lost",
|
|
934
|
+
details={"idempotency_key": key, "workflow_id": workflow_id},
|
|
935
|
+
)
|
|
936
|
+
|
|
937
|
+
def get_idempotency(
|
|
938
|
+
self, key: str, *, tenant_scope_fingerprint: str | None = None
|
|
939
|
+
) -> IdempotencyRecord | None:
|
|
940
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
941
|
+
with self._transaction() as conn:
|
|
942
|
+
cursor = self._execute(conn, "read_idempotency_plain", (namespace, key))
|
|
943
|
+
row = cursor.fetchone()
|
|
944
|
+
return self._record_from_row(row) if row is not None else None
|
|
945
|
+
|
|
946
|
+
# -- lease ownership and fencing -------------------------------------
|
|
947
|
+
|
|
948
|
+
def acquire_lease(
|
|
949
|
+
self,
|
|
950
|
+
workflow_id: str,
|
|
951
|
+
*,
|
|
952
|
+
owner_id: str,
|
|
953
|
+
tenant_scope_fingerprint: str | None = None,
|
|
954
|
+
ttl_seconds: float | None = None,
|
|
955
|
+
) -> WorkflowLease:
|
|
956
|
+
"""Atomically claim the workflow lease.
|
|
957
|
+
|
|
958
|
+
Acquisition succeeds when no lease exists or the stored lease is
|
|
959
|
+
expired beyond the clock tolerance; takeover atomically increments
|
|
960
|
+
the fencing token. A valid lease is reported as ``LEASE_BUSY``
|
|
961
|
+
without changing ownership.
|
|
962
|
+
"""
|
|
963
|
+
validate_lease_identity(
|
|
964
|
+
workflow_id, owner_id=owner_id, scope=tenant_scope_fingerprint
|
|
965
|
+
)
|
|
966
|
+
ttl = (
|
|
967
|
+
self._config.lease_ttl_seconds
|
|
968
|
+
if ttl_seconds is None
|
|
969
|
+
else ttl_seconds
|
|
970
|
+
)
|
|
971
|
+
if ttl < 1.0 or ttl > 86_400.0:
|
|
972
|
+
raise ValueError("lease TTL must be between 1 and 86400 seconds")
|
|
973
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
974
|
+
expires_at = self._now_fn() + timedelta(seconds=ttl)
|
|
975
|
+
with self._transaction() as conn:
|
|
976
|
+
cursor = self._execute(
|
|
977
|
+
conn,
|
|
978
|
+
"acquire_lease",
|
|
979
|
+
(
|
|
980
|
+
namespace,
|
|
981
|
+
workflow_id,
|
|
982
|
+
owner_id,
|
|
983
|
+
_iso(expires_at),
|
|
984
|
+
self._config.clock_tolerance_seconds,
|
|
985
|
+
),
|
|
986
|
+
)
|
|
987
|
+
row = cursor.fetchone()
|
|
988
|
+
if row is not None:
|
|
989
|
+
return WorkflowLease(
|
|
990
|
+
workflow_id=workflow_id,
|
|
991
|
+
tenant_scope_fingerprint=tenant_scope_fingerprint,
|
|
992
|
+
owner_id=str(row["owner_id"]),
|
|
993
|
+
fencing_token=int(row["fencing_token"]),
|
|
994
|
+
expires_at=_parse_dt(row["expires_at"]),
|
|
995
|
+
updated_at=_parse_dt(row["updated_at"]),
|
|
996
|
+
)
|
|
997
|
+
raise SharedStoreError(
|
|
998
|
+
SharedStoreErrorCode.LEASE_BUSY,
|
|
999
|
+
f"workflow '{workflow_id}' lease is busy",
|
|
1000
|
+
details={"workflow_id": workflow_id},
|
|
1001
|
+
)
|
|
1002
|
+
|
|
1003
|
+
def renew_lease(
|
|
1004
|
+
self,
|
|
1005
|
+
workflow_id: str,
|
|
1006
|
+
*,
|
|
1007
|
+
owner_id: str,
|
|
1008
|
+
fencing_token: int,
|
|
1009
|
+
tenant_scope_fingerprint: str | None = None,
|
|
1010
|
+
) -> WorkflowLease:
|
|
1011
|
+
"""Extend the lease only for its current owner and fencing token.
|
|
1012
|
+
|
|
1013
|
+
Renewal is rejected (``FENCING_REJECTED``) when the lease is
|
|
1014
|
+
missing, owned by another worker, carries a superseded token, or
|
|
1015
|
+
has expired - the worker must re-acquire after takeover.
|
|
1016
|
+
"""
|
|
1017
|
+
validate_lease_identity(
|
|
1018
|
+
workflow_id, owner_id=owner_id, scope=tenant_scope_fingerprint
|
|
1019
|
+
)
|
|
1020
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
1021
|
+
expires_at = self._now_fn() + timedelta(
|
|
1022
|
+
seconds=self._config.lease_ttl_seconds
|
|
1023
|
+
)
|
|
1024
|
+
with self._transaction() as conn:
|
|
1025
|
+
cursor = self._execute(
|
|
1026
|
+
conn,
|
|
1027
|
+
"renew_lease",
|
|
1028
|
+
(
|
|
1029
|
+
_iso(expires_at),
|
|
1030
|
+
namespace,
|
|
1031
|
+
workflow_id,
|
|
1032
|
+
owner_id,
|
|
1033
|
+
fencing_token,
|
|
1034
|
+
self._config.clock_tolerance_seconds,
|
|
1035
|
+
),
|
|
1036
|
+
)
|
|
1037
|
+
row = cursor.fetchone()
|
|
1038
|
+
if row is not None:
|
|
1039
|
+
return WorkflowLease(
|
|
1040
|
+
workflow_id=workflow_id,
|
|
1041
|
+
tenant_scope_fingerprint=tenant_scope_fingerprint,
|
|
1042
|
+
owner_id=str(row["owner_id"]),
|
|
1043
|
+
fencing_token=int(row["fencing_token"]),
|
|
1044
|
+
expires_at=_parse_dt(row["expires_at"]),
|
|
1045
|
+
updated_at=_parse_dt(row["updated_at"]),
|
|
1046
|
+
)
|
|
1047
|
+
stored = self._execute(
|
|
1048
|
+
conn, "inspect_lease", (namespace, workflow_id)
|
|
1049
|
+
).fetchone()
|
|
1050
|
+
if stored is None:
|
|
1051
|
+
raise SharedStoreError(
|
|
1052
|
+
SharedStoreErrorCode.FENCING_REJECTED,
|
|
1053
|
+
f"workflow '{workflow_id}' has no lease",
|
|
1054
|
+
details={"workflow_id": workflow_id},
|
|
1055
|
+
)
|
|
1056
|
+
if (
|
|
1057
|
+
str(stored["owner_id"]) != owner_id
|
|
1058
|
+
or int(stored["fencing_token"]) != fencing_token
|
|
1059
|
+
):
|
|
1060
|
+
raise SharedStoreError(
|
|
1061
|
+
SharedStoreErrorCode.FENCING_REJECTED,
|
|
1062
|
+
f"workflow '{workflow_id}' lease ownership was superseded",
|
|
1063
|
+
details={"workflow_id": workflow_id},
|
|
1064
|
+
)
|
|
1065
|
+
raise SharedStoreError(
|
|
1066
|
+
SharedStoreErrorCode.FENCING_REJECTED,
|
|
1067
|
+
f"workflow '{workflow_id}' lease has expired",
|
|
1068
|
+
details={"workflow_id": workflow_id},
|
|
1069
|
+
)
|
|
1070
|
+
|
|
1071
|
+
def release_lease(
|
|
1072
|
+
self,
|
|
1073
|
+
workflow_id: str,
|
|
1074
|
+
*,
|
|
1075
|
+
owner_id: str,
|
|
1076
|
+
fencing_token: int,
|
|
1077
|
+
tenant_scope_fingerprint: str | None = None,
|
|
1078
|
+
) -> bool:
|
|
1079
|
+
"""Release the lease for its current owner/token; ``False`` otherwise."""
|
|
1080
|
+
validate_lease_identity(
|
|
1081
|
+
workflow_id, owner_id=owner_id, scope=tenant_scope_fingerprint
|
|
1082
|
+
)
|
|
1083
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
1084
|
+
with self._transaction() as conn:
|
|
1085
|
+
cursor = self._execute(
|
|
1086
|
+
conn,
|
|
1087
|
+
"release_lease",
|
|
1088
|
+
(namespace, workflow_id, owner_id, fencing_token),
|
|
1089
|
+
)
|
|
1090
|
+
return bool(cursor.rowcount == 1)
|
|
1091
|
+
|
|
1092
|
+
def inspect_lease(
|
|
1093
|
+
self,
|
|
1094
|
+
workflow_id: str,
|
|
1095
|
+
*,
|
|
1096
|
+
tenant_scope_fingerprint: str | None = None,
|
|
1097
|
+
) -> WorkflowLease | None:
|
|
1098
|
+
"""Return the stored lease within the matching scope or ``None``."""
|
|
1099
|
+
namespace = _namespace(tenant_scope_fingerprint)
|
|
1100
|
+
with self._transaction() as conn:
|
|
1101
|
+
cursor = self._execute(conn, "inspect_lease", (namespace, workflow_id))
|
|
1102
|
+
row = cursor.fetchone()
|
|
1103
|
+
if row is None:
|
|
1104
|
+
return None
|
|
1105
|
+
return WorkflowLease(
|
|
1106
|
+
workflow_id=workflow_id,
|
|
1107
|
+
tenant_scope_fingerprint=tenant_scope_fingerprint,
|
|
1108
|
+
owner_id=str(row["owner_id"]),
|
|
1109
|
+
fencing_token=int(row["fencing_token"]),
|
|
1110
|
+
expires_at=_parse_dt(row["expires_at"]),
|
|
1111
|
+
updated_at=_parse_dt(row["updated_at"]),
|
|
1112
|
+
)
|
|
1113
|
+
|
|
1114
|
+
# -- helpers ---------------------------------------------------------
|
|
1115
|
+
|
|
1116
|
+
@staticmethod
|
|
1117
|
+
def _record_from_row(row: Any) -> IdempotencyRecord:
|
|
1118
|
+
return IdempotencyRecord(
|
|
1119
|
+
idempotency_key=str(row["idempotency_key"]),
|
|
1120
|
+
request_id=str(row["request_id"]),
|
|
1121
|
+
tenant_scope_fingerprint=row["tenant_scope_fingerprint"],
|
|
1122
|
+
scope_namespace=str(row["scope_namespace"]),
|
|
1123
|
+
workflow_id=str(row["workflow_id"]),
|
|
1124
|
+
status=IdempotencyStatus(str(row["status"])),
|
|
1125
|
+
terminal_outcome_fingerprint=row["terminal_outcome_fingerprint"],
|
|
1126
|
+
created_at=_parse_dt(row["created_at"]),
|
|
1127
|
+
updated_at=_parse_dt(row["updated_at"]),
|
|
1128
|
+
expires_at=(
|
|
1129
|
+
_parse_dt(row["expires_at"]) if row["expires_at"] is not None else None
|
|
1130
|
+
),
|
|
1131
|
+
)
|
|
1132
|
+
|
|
1133
|
+
@staticmethod
|
|
1134
|
+
def _validate_identity(
|
|
1135
|
+
key: str,
|
|
1136
|
+
request_id: str | None = None,
|
|
1137
|
+
workflow_id: str | None = None,
|
|
1138
|
+
scope: str | None = None,
|
|
1139
|
+
) -> None:
|
|
1140
|
+
if _KEY_PATTERN.fullmatch(key) is None:
|
|
1141
|
+
raise WorkflowStateError(
|
|
1142
|
+
"idempotency key is not identifier-safe",
|
|
1143
|
+
details={"idempotency_key": key[:128]},
|
|
1144
|
+
)
|
|
1145
|
+
if request_id is not None and _IDENTIFIER_PATTERN.fullmatch(request_id) is None:
|
|
1146
|
+
raise WorkflowStateError(
|
|
1147
|
+
"idempotency request identity is not identifier-safe",
|
|
1148
|
+
details={"request_id": request_id[:128]},
|
|
1149
|
+
)
|
|
1150
|
+
if workflow_id is not None and _IDENTIFIER_PATTERN.fullmatch(workflow_id) is None:
|
|
1151
|
+
raise WorkflowStateError(
|
|
1152
|
+
"idempotency workflow identity is not identifier-safe",
|
|
1153
|
+
details={"workflow_id": workflow_id[:128]},
|
|
1154
|
+
)
|
|
1155
|
+
if scope is not None and _FINGERPRINT_PATTERN.fullmatch(scope) is None:
|
|
1156
|
+
raise WorkflowStateError(
|
|
1157
|
+
"idempotency scope must be a sha256 fingerprint",
|
|
1158
|
+
details={"tenant_scope_fingerprint": scope[:128]},
|
|
1159
|
+
)
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
#: Re-exported public surface of this module.
|
|
1163
|
+
__all__ = [
|
|
1164
|
+
"PostgreSQLStateStore",
|
|
1165
|
+
"SQL_TEMPLATES",
|
|
1166
|
+
]
|