phlo-postgres 0.16.0__tar.gz → 0.17.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/PKG-INFO +2 -2
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/pyproject.toml +2 -2
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/exporter_service.yaml +1 -1
- phlo_postgres-0.17.0/src/phlo_postgres/operation_controls.py +350 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/service.yaml +5 -3
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/settings_store.py +13 -1
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres.egg-info/PKG-INFO +2 -2
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres.egg-info/SOURCES.txt +1 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres.egg-info/requires.txt +1 -1
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_integration_postgres.py +8 -5
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/README.md +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/setup.cfg +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/__init__.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/authorization.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/checkpoints.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/cli.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/cli_plugin.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/continuity.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/dataset_state_store.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/governance.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/plugin.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/publish_target.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/resource.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/security_readiness.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/settings.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres/volume_setup.yaml +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres.egg-info/dependency_links.txt +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres.egg-info/entry_points.txt +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres.egg-info/top_level.txt +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_authorization.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_checkpoints.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_continuity.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_dataset_state_store.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_postgres_cli.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_postgres_plugin.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_resource.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/tests/test_settings_store.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phlo-postgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.17.0
|
|
4
4
|
Summary: Postgres service plugin for Phlo
|
|
5
5
|
Author-email: Phlo Team <team@phlo.dev>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.12
|
|
8
8
|
Description-Content-Type: text/plain
|
|
9
|
-
Requires-Dist: phlo<0.
|
|
9
|
+
Requires-Dist: phlo<0.19,>=0.18.0
|
|
10
10
|
Requires-Dist: psycopg2-binary>=2.9.11
|
|
11
11
|
Provides-Extra: dev
|
|
12
12
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
@@ -7,13 +7,13 @@ requires = [
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
dependencies = [
|
|
10
|
-
"phlo>=0.
|
|
10
|
+
"phlo>=0.18.0,<0.19",
|
|
11
11
|
"psycopg2-binary>=2.9.11",
|
|
12
12
|
]
|
|
13
13
|
description = "Postgres service plugin for Phlo"
|
|
14
14
|
name = "phlo-postgres"
|
|
15
15
|
requires-python = ">=3.12"
|
|
16
|
-
version = "0.
|
|
16
|
+
version = "0.17.0"
|
|
17
17
|
|
|
18
18
|
[[project.authors]]
|
|
19
19
|
email = "team@phlo.dev"
|
|
@@ -16,7 +16,7 @@ compose:
|
|
|
16
16
|
phlo.metrics.port: "postgres-exporter:9187"
|
|
17
17
|
phlo.metrics.path: "/metrics"
|
|
18
18
|
environment:
|
|
19
|
-
DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-phlo}:${POSTGRES_PASSWORD
|
|
19
|
+
DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-phlo}:${POSTGRES_PASSWORD:?Run phlo services init to generate credentials}@postgres:5432/${POSTGRES_DB:-phlo}?sslmode=disable
|
|
20
20
|
ports:
|
|
21
21
|
- "${POSTGRES_EXPORTER_PORT:-9187}:9187"
|
|
22
22
|
healthcheck:
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
"""PostgreSQL-owned API operation controls, independent of replica-local paths.
|
|
2
|
+
|
|
3
|
+
Short transactions take a namespace advisory lock before checking absent rows.
|
|
4
|
+
Durable pending/unknown claims, not a connection lease, fence provider calls
|
|
5
|
+
after crashes. No unresolved claim is automatically expired.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Iterator
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import psycopg2
|
|
16
|
+
from psycopg2.extensions import cursor
|
|
17
|
+
|
|
18
|
+
from phlo.operations.journal import OperationJournalEntry, OperationJournalState
|
|
19
|
+
from phlo.plugins.observatory_settings import StorageUnavailableError
|
|
20
|
+
|
|
21
|
+
RATE_BUCKET_CAP = 4096
|
|
22
|
+
|
|
23
|
+
_SCHEMA = """
|
|
24
|
+
CREATE TABLE IF NOT EXISTS phlo_api_operations (
|
|
25
|
+
namespace TEXT NOT NULL, key_hash TEXT NOT NULL, operation TEXT NOT NULL,
|
|
26
|
+
target TEXT NOT NULL, exclusion_target TEXT NOT NULL,
|
|
27
|
+
state TEXT NOT NULL, response_json TEXT NOT NULL DEFAULT '',
|
|
28
|
+
expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + interval '24 hours'),
|
|
29
|
+
PRIMARY KEY (namespace, key_hash, operation, target)
|
|
30
|
+
);
|
|
31
|
+
CREATE UNIQUE INDEX IF NOT EXISTS phlo_api_operations_active_target
|
|
32
|
+
ON phlo_api_operations(namespace, operation, exclusion_target)
|
|
33
|
+
WHERE state IN ('pending', 'unknown');
|
|
34
|
+
CREATE TABLE IF NOT EXISTS phlo_api_operation_resolutions (
|
|
35
|
+
id BIGSERIAL PRIMARY KEY, namespace TEXT NOT NULL, key_hash TEXT NOT NULL,
|
|
36
|
+
operation TEXT NOT NULL, target TEXT NOT NULL, resolution TEXT NOT NULL,
|
|
37
|
+
resolved_by TEXT NOT NULL, evidence_json TEXT NOT NULL,
|
|
38
|
+
resolved_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
39
|
+
);
|
|
40
|
+
CREATE TABLE IF NOT EXISTS phlo_api_rate_buckets (
|
|
41
|
+
namespace TEXT NOT NULL, subject_hash TEXT NOT NULL, operation TEXT NOT NULL,
|
|
42
|
+
events DOUBLE PRECISION[] NOT NULL,
|
|
43
|
+
PRIMARY KEY(namespace, subject_hash, operation)
|
|
44
|
+
);
|
|
45
|
+
CREATE TABLE IF NOT EXISTS phlo_api_operation_journal (
|
|
46
|
+
namespace TEXT NOT NULL, operation_id TEXT NOT NULL,
|
|
47
|
+
action TEXT NOT NULL, target TEXT NOT NULL, state TEXT NOT NULL,
|
|
48
|
+
entry_json TEXT NOT NULL,
|
|
49
|
+
PRIMARY KEY(namespace, operation_id)
|
|
50
|
+
);
|
|
51
|
+
CREATE UNIQUE INDEX IF NOT EXISTS phlo_api_journal_active_target
|
|
52
|
+
ON phlo_api_operation_journal(namespace, action, target)
|
|
53
|
+
WHERE state IN ('claimed', 'submitted', 'unknown');
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class PostgresOperationControls:
|
|
58
|
+
"""Shared controls and the core durable-journal protocol for one project.
|
|
59
|
+
|
|
60
|
+
Connections are opened per transaction and always closed. The namespace
|
|
61
|
+
lock deliberately serialises short control transactions, not provider I/O.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(self, db_url: str, namespace: str) -> None:
|
|
65
|
+
if not namespace.strip():
|
|
66
|
+
raise ValueError("Shared operation controls require a stable namespace")
|
|
67
|
+
self._db_url = db_url
|
|
68
|
+
self._namespace = namespace
|
|
69
|
+
self._schema_ready = False
|
|
70
|
+
|
|
71
|
+
@contextmanager
|
|
72
|
+
def _transaction(self) -> Iterator[cursor]:
|
|
73
|
+
conn = None
|
|
74
|
+
try:
|
|
75
|
+
conn = psycopg2.connect(self._db_url, connect_timeout=5)
|
|
76
|
+
with conn, conn.cursor() as cur:
|
|
77
|
+
cur.execute("SET LOCAL lock_timeout = '5s'")
|
|
78
|
+
cur.execute("SET LOCAL statement_timeout = '10s'")
|
|
79
|
+
# Serialise DDL too: concurrent CREATE IF NOT EXISTS can race
|
|
80
|
+
# on PostgreSQL's catalog unique constraints on first startup.
|
|
81
|
+
if not self._schema_ready:
|
|
82
|
+
cur.execute(
|
|
83
|
+
"SELECT pg_advisory_xact_lock(hashtext('phlo-api-controls-schema'))"
|
|
84
|
+
)
|
|
85
|
+
cur.execute(_SCHEMA)
|
|
86
|
+
cur.execute(
|
|
87
|
+
"SELECT pg_advisory_xact_lock(hashtext(%s))",
|
|
88
|
+
(f"phlo-api-controls:{self._namespace}",),
|
|
89
|
+
)
|
|
90
|
+
yield cur
|
|
91
|
+
self._schema_ready = True
|
|
92
|
+
except psycopg2.Error as exc:
|
|
93
|
+
raise StorageUnavailableError("Shared operation controls are unavailable") from exc
|
|
94
|
+
finally:
|
|
95
|
+
if conn is not None:
|
|
96
|
+
conn.close()
|
|
97
|
+
|
|
98
|
+
def initialize(self) -> None:
|
|
99
|
+
"""Check connectivity and initialise the schema before serving traffic."""
|
|
100
|
+
with self._transaction():
|
|
101
|
+
pass
|
|
102
|
+
|
|
103
|
+
def claim_idempotency(
|
|
104
|
+
self,
|
|
105
|
+
key_hash: str,
|
|
106
|
+
operation: str,
|
|
107
|
+
target: str,
|
|
108
|
+
exclusion_target: str | None = None,
|
|
109
|
+
) -> tuple[bool, str, str]:
|
|
110
|
+
"""Claim identity and target together; return replay or a stable conflict."""
|
|
111
|
+
with self._transaction() as cur:
|
|
112
|
+
cur.execute(
|
|
113
|
+
"DELETE FROM phlo_api_operations WHERE namespace=%s "
|
|
114
|
+
"AND state='completed' AND expires_at < now()",
|
|
115
|
+
(self._namespace,),
|
|
116
|
+
)
|
|
117
|
+
cur.execute(
|
|
118
|
+
"SELECT target, state, response_json FROM phlo_api_operations "
|
|
119
|
+
"WHERE namespace=%s AND key_hash=%s AND operation=%s AND (%s OR target=%s)",
|
|
120
|
+
(self._namespace, key_hash, operation, exclusion_target is not None, target),
|
|
121
|
+
)
|
|
122
|
+
row = cur.fetchone()
|
|
123
|
+
if row is not None:
|
|
124
|
+
if row[0] != target:
|
|
125
|
+
return False, "target_mismatch", ""
|
|
126
|
+
if row[1] != "safe_to_retry":
|
|
127
|
+
return False, row[1], row[2]
|
|
128
|
+
cur.execute(
|
|
129
|
+
"SELECT state FROM phlo_api_operations WHERE namespace=%s "
|
|
130
|
+
"AND operation=%s AND exclusion_target=%s AND state IN ('pending', 'unknown') LIMIT 1",
|
|
131
|
+
(self._namespace, operation, exclusion_target or target),
|
|
132
|
+
)
|
|
133
|
+
active = cur.fetchone()
|
|
134
|
+
if active is not None:
|
|
135
|
+
return False, "operation_" + active[0], ""
|
|
136
|
+
cur.execute(
|
|
137
|
+
"INSERT INTO phlo_api_operations(namespace,key_hash,operation,target,exclusion_target,state) "
|
|
138
|
+
"VALUES (%s,%s,%s,%s,%s,'pending') ON CONFLICT(namespace,key_hash,operation,target) "
|
|
139
|
+
"DO UPDATE SET state='pending', response_json='', expires_at=now()+interval '24 hours'",
|
|
140
|
+
(self._namespace, key_hash, operation, target, exclusion_target or target),
|
|
141
|
+
)
|
|
142
|
+
return True, "pending", ""
|
|
143
|
+
|
|
144
|
+
def finish_idempotency(
|
|
145
|
+
self,
|
|
146
|
+
key_hash: str,
|
|
147
|
+
operation: str,
|
|
148
|
+
target: str,
|
|
149
|
+
state: str,
|
|
150
|
+
response: dict[str, Any] | None = None,
|
|
151
|
+
) -> None:
|
|
152
|
+
"""Record provider completion, or preserve an unknown outcome indefinitely."""
|
|
153
|
+
with self._transaction() as cur:
|
|
154
|
+
cur.execute(
|
|
155
|
+
"UPDATE phlo_api_operations SET state=%s, response_json=%s, "
|
|
156
|
+
"expires_at=now()+interval '24 hours' WHERE namespace=%s AND key_hash=%s "
|
|
157
|
+
"AND operation=%s AND target=%s AND state='pending'",
|
|
158
|
+
(
|
|
159
|
+
state,
|
|
160
|
+
json.dumps(response, sort_keys=True) if response is not None else "",
|
|
161
|
+
self._namespace,
|
|
162
|
+
key_hash,
|
|
163
|
+
operation,
|
|
164
|
+
target,
|
|
165
|
+
),
|
|
166
|
+
)
|
|
167
|
+
if cur.rowcount != 1:
|
|
168
|
+
raise StorageUnavailableError("Operation claim changed before completion")
|
|
169
|
+
|
|
170
|
+
def idempotency_target(self, key_hash: str, operation: str) -> str | None:
|
|
171
|
+
"""Read the canonical key binding across all replicas."""
|
|
172
|
+
with self._transaction() as cur:
|
|
173
|
+
cur.execute(
|
|
174
|
+
"SELECT target FROM phlo_api_operations WHERE namespace=%s AND key_hash=%s AND operation=%s",
|
|
175
|
+
(self._namespace, key_hash, operation),
|
|
176
|
+
)
|
|
177
|
+
row = cur.fetchone()
|
|
178
|
+
return row[0] if row is not None else None
|
|
179
|
+
|
|
180
|
+
def resolve_idempotency(
|
|
181
|
+
self,
|
|
182
|
+
key_hash: str,
|
|
183
|
+
operation: str,
|
|
184
|
+
target: str,
|
|
185
|
+
resolution: str,
|
|
186
|
+
resolved_by: str,
|
|
187
|
+
evidence: dict[str, Any],
|
|
188
|
+
response: dict[str, Any] | None,
|
|
189
|
+
) -> None:
|
|
190
|
+
"""Retain operator evidence; only explicit safe_to_retry permits another call."""
|
|
191
|
+
state = "completed" if resolution == "succeeded" else resolution
|
|
192
|
+
evidence_json = json.dumps(evidence, sort_keys=True)
|
|
193
|
+
with self._transaction() as cur:
|
|
194
|
+
identity = (self._namespace, key_hash, operation, target)
|
|
195
|
+
cur.execute(
|
|
196
|
+
"SELECT state FROM phlo_api_operations WHERE namespace=%s "
|
|
197
|
+
"AND key_hash=%s AND operation=%s AND target=%s",
|
|
198
|
+
identity,
|
|
199
|
+
)
|
|
200
|
+
row = cur.fetchone()
|
|
201
|
+
if row is None or row[0] not in {"pending", "unknown"}:
|
|
202
|
+
cur.execute(
|
|
203
|
+
"SELECT resolution,resolved_by,evidence_json FROM phlo_api_operation_resolutions "
|
|
204
|
+
"WHERE namespace=%s AND key_hash=%s AND operation=%s AND target=%s ORDER BY id DESC LIMIT 1",
|
|
205
|
+
identity,
|
|
206
|
+
)
|
|
207
|
+
prior = cur.fetchone()
|
|
208
|
+
if (
|
|
209
|
+
row is not None
|
|
210
|
+
and row[0] == state
|
|
211
|
+
and prior == (resolution, resolved_by, evidence_json)
|
|
212
|
+
):
|
|
213
|
+
return
|
|
214
|
+
raise ValueError("only pending or unknown idempotency claims can be resolved")
|
|
215
|
+
cur.execute(
|
|
216
|
+
"UPDATE phlo_api_operations SET state=%s,response_json=%s,expires_at=now()+interval '24 hours' "
|
|
217
|
+
"WHERE namespace=%s AND key_hash=%s AND operation=%s AND target=%s",
|
|
218
|
+
(
|
|
219
|
+
state,
|
|
220
|
+
json.dumps(response, sort_keys=True) if response is not None else "",
|
|
221
|
+
*identity,
|
|
222
|
+
),
|
|
223
|
+
)
|
|
224
|
+
cur.execute(
|
|
225
|
+
"INSERT INTO phlo_api_operation_resolutions(namespace,key_hash,operation,target,resolution,resolved_by,evidence_json) "
|
|
226
|
+
"VALUES (%s,%s,%s,%s,%s,%s,%s)",
|
|
227
|
+
(*identity, resolution, resolved_by, evidence_json),
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def rate_limit(self, subject_hash: str, operation: str, limit: int) -> bool:
|
|
231
|
+
"""Bound both bucket cardinality and each sliding window; fail closed at capacity."""
|
|
232
|
+
with self._transaction() as cur:
|
|
233
|
+
cur.execute("SELECT extract(epoch FROM clock_timestamp())::double precision")
|
|
234
|
+
now = cur.fetchone()[0]
|
|
235
|
+
cur.execute(
|
|
236
|
+
"DELETE FROM phlo_api_rate_buckets WHERE namespace=%s AND events[array_length(events,1)] < %s",
|
|
237
|
+
(self._namespace, now - 60),
|
|
238
|
+
)
|
|
239
|
+
identity = (self._namespace, subject_hash, operation)
|
|
240
|
+
cur.execute(
|
|
241
|
+
"SELECT events FROM phlo_api_rate_buckets WHERE namespace=%s AND subject_hash=%s AND operation=%s",
|
|
242
|
+
identity,
|
|
243
|
+
)
|
|
244
|
+
row = cur.fetchone()
|
|
245
|
+
events = [stamp for stamp in row[0] if stamp >= now - 60] if row is not None else []
|
|
246
|
+
if len(events) >= limit:
|
|
247
|
+
return False
|
|
248
|
+
if row is None:
|
|
249
|
+
cur.execute(
|
|
250
|
+
"SELECT count(*) FROM phlo_api_rate_buckets WHERE namespace=%s",
|
|
251
|
+
(self._namespace,),
|
|
252
|
+
)
|
|
253
|
+
if cur.fetchone()[0] >= RATE_BUCKET_CAP:
|
|
254
|
+
return False
|
|
255
|
+
cur.execute(
|
|
256
|
+
"INSERT INTO phlo_api_rate_buckets(namespace,subject_hash,operation,events) VALUES (%s,%s,%s,%s) "
|
|
257
|
+
"ON CONFLICT(namespace,subject_hash,operation) DO UPDATE SET events=EXCLUDED.events",
|
|
258
|
+
(*identity, [*events, now]),
|
|
259
|
+
)
|
|
260
|
+
return True
|
|
261
|
+
|
|
262
|
+
def claim(self, entry: OperationJournalEntry) -> bool:
|
|
263
|
+
"""Claim a journal operation and its active target across replicas."""
|
|
264
|
+
with self._transaction() as cur:
|
|
265
|
+
cur.execute(
|
|
266
|
+
"SELECT 1 FROM phlo_api_operation_journal WHERE namespace=%s AND "
|
|
267
|
+
"(operation_id=%s OR (action=%s AND target=%s AND state IN ('claimed','submitted','unknown'))) LIMIT 1",
|
|
268
|
+
(self._namespace, entry.operation_id, entry.action, entry.target),
|
|
269
|
+
)
|
|
270
|
+
if cur.fetchone() is not None:
|
|
271
|
+
return False
|
|
272
|
+
cur.execute(
|
|
273
|
+
"INSERT INTO phlo_api_operation_journal(namespace,operation_id,action,target,state,entry_json) "
|
|
274
|
+
"VALUES (%s,%s,%s,%s,%s,%s)",
|
|
275
|
+
(
|
|
276
|
+
self._namespace,
|
|
277
|
+
entry.operation_id,
|
|
278
|
+
entry.action,
|
|
279
|
+
entry.target,
|
|
280
|
+
entry.state.value,
|
|
281
|
+
json.dumps(entry.to_dict()),
|
|
282
|
+
),
|
|
283
|
+
)
|
|
284
|
+
return True
|
|
285
|
+
|
|
286
|
+
def transition(
|
|
287
|
+
self,
|
|
288
|
+
operation_id: str,
|
|
289
|
+
state: OperationJournalState,
|
|
290
|
+
result: dict[str, Any] | None = None,
|
|
291
|
+
) -> bool:
|
|
292
|
+
"""Persist a journal transition without releasing unresolved target claims."""
|
|
293
|
+
with self._transaction() as cur:
|
|
294
|
+
cur.execute(
|
|
295
|
+
"SELECT entry_json FROM phlo_api_operation_journal WHERE namespace=%s AND operation_id=%s",
|
|
296
|
+
(self._namespace, operation_id),
|
|
297
|
+
)
|
|
298
|
+
row = cur.fetchone()
|
|
299
|
+
if row is None:
|
|
300
|
+
return False
|
|
301
|
+
record = json.loads(row[0])
|
|
302
|
+
record.update(state=state.value, result=result)
|
|
303
|
+
cur.execute(
|
|
304
|
+
"UPDATE phlo_api_operation_journal SET state=%s,entry_json=%s WHERE namespace=%s AND operation_id=%s",
|
|
305
|
+
(state.value, json.dumps(record), self._namespace, operation_id),
|
|
306
|
+
)
|
|
307
|
+
return True
|
|
308
|
+
|
|
309
|
+
def read(self, operation_id: str) -> OperationJournalEntry | None:
|
|
310
|
+
"""Read journal evidence from any replica."""
|
|
311
|
+
with self._transaction() as cur:
|
|
312
|
+
cur.execute(
|
|
313
|
+
"SELECT entry_json FROM phlo_api_operation_journal WHERE namespace=%s AND operation_id=%s",
|
|
314
|
+
(self._namespace, operation_id),
|
|
315
|
+
)
|
|
316
|
+
row = cur.fetchone()
|
|
317
|
+
if row is None:
|
|
318
|
+
return None
|
|
319
|
+
record = json.loads(row[0])
|
|
320
|
+
record["state"] = OperationJournalState(record["state"])
|
|
321
|
+
return OperationJournalEntry(**record)
|
|
322
|
+
|
|
323
|
+
@contextmanager
|
|
324
|
+
def exclusion(self, operation: str, target: str) -> Iterator[None]:
|
|
325
|
+
"""Non-blocking session advisory lock for a compound provider action.
|
|
326
|
+
|
|
327
|
+
Durable idempotency/journal claims must additionally fence effects whose
|
|
328
|
+
outcome may be unknown after connection loss or process termination.
|
|
329
|
+
"""
|
|
330
|
+
conn = None
|
|
331
|
+
try:
|
|
332
|
+
conn = psycopg2.connect(self._db_url, connect_timeout=5)
|
|
333
|
+
with conn.cursor() as cur:
|
|
334
|
+
cur.execute(
|
|
335
|
+
"SELECT pg_try_advisory_lock(hashtextextended(%s,0))",
|
|
336
|
+
(json.dumps([self._namespace, operation, target]),),
|
|
337
|
+
)
|
|
338
|
+
acquired = cur.fetchone()[0]
|
|
339
|
+
if not acquired:
|
|
340
|
+
raise OperationExclusionConflict("Operation is already in progress")
|
|
341
|
+
yield
|
|
342
|
+
except psycopg2.Error as exc:
|
|
343
|
+
raise StorageUnavailableError("Shared operation exclusion is unavailable") from exc
|
|
344
|
+
finally:
|
|
345
|
+
if conn is not None:
|
|
346
|
+
conn.close()
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class OperationExclusionConflict(RuntimeError):
|
|
350
|
+
"""Another instance holds the compound-action advisory lock."""
|
|
@@ -19,7 +19,7 @@ compose:
|
|
|
19
19
|
phlo.grafana.datasource.database: "${POSTGRES_DB:-phlo}"
|
|
20
20
|
environment:
|
|
21
21
|
POSTGRES_USER: ${POSTGRES_USER:-phlo}
|
|
22
|
-
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD
|
|
22
|
+
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run phlo services init to generate POSTGRES_PASSWORD}
|
|
23
23
|
POSTGRES_DB: ${POSTGRES_DB:-phlo}
|
|
24
24
|
# SSL/TLS
|
|
25
25
|
POSTGRES_SSL_MODE: ${POSTGRES_SSL_MODE:-prefer}
|
|
@@ -28,7 +28,10 @@ compose:
|
|
|
28
28
|
volumes:
|
|
29
29
|
- postgres-data:/var/lib/postgresql
|
|
30
30
|
healthcheck:
|
|
31
|
-
|
|
31
|
+
# Probe over TCP: pg_isready via the unix socket also answers during the
|
|
32
|
+
# image's temporary init server, marking the container healthy before it
|
|
33
|
+
# restarts for real — dependents connecting in the gap lose the connection.
|
|
34
|
+
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U ${POSTGRES_USER:-phlo}"]
|
|
32
35
|
interval: 10s
|
|
33
36
|
timeout: 5s
|
|
34
37
|
retries: 10
|
|
@@ -38,7 +41,6 @@ env_vars:
|
|
|
38
41
|
default: phlo
|
|
39
42
|
description: PostgreSQL username
|
|
40
43
|
POSTGRES_PASSWORD:
|
|
41
|
-
default: phlo
|
|
42
44
|
description: PostgreSQL password
|
|
43
45
|
secret: true
|
|
44
46
|
POSTGRES_DB:
|
|
@@ -27,6 +27,13 @@ from phlo_postgres.settings import get_settings as get_postgres_settings
|
|
|
27
27
|
logger = get_logger(__name__)
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
class _MutationCallbackFailure(Exception):
|
|
31
|
+
"""Carry a user mutation exception through storage error translation."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, error: Exception) -> None:
|
|
34
|
+
self.error = error
|
|
35
|
+
|
|
36
|
+
|
|
30
37
|
def _get_psycopg2():
|
|
31
38
|
try:
|
|
32
39
|
import psycopg2
|
|
@@ -160,7 +167,10 @@ class PostgresSettingsStore:
|
|
|
160
167
|
(scope.value, namespace),
|
|
161
168
|
)
|
|
162
169
|
row = cursor.fetchone()
|
|
163
|
-
|
|
170
|
+
try:
|
|
171
|
+
settings = mutation(row[0] if row else None)
|
|
172
|
+
except Exception as exc:
|
|
173
|
+
raise _MutationCallbackFailure(exc) from exc
|
|
164
174
|
cursor.execute(
|
|
165
175
|
"""
|
|
166
176
|
INSERT INTO phlo_settings (scope, namespace, settings, updated_at)
|
|
@@ -179,6 +189,8 @@ class PostgresSettingsStore:
|
|
|
179
189
|
settings=stored_settings,
|
|
180
190
|
updated_at=updated_at.isoformat() if updated_at else None,
|
|
181
191
|
)
|
|
192
|
+
except _MutationCallbackFailure as exc:
|
|
193
|
+
raise exc.error
|
|
182
194
|
except Exception as exc:
|
|
183
195
|
if isinstance(exc, StorageUnavailableError):
|
|
184
196
|
raise
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phlo-postgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.17.0
|
|
4
4
|
Summary: Postgres service plugin for Phlo
|
|
5
5
|
Author-email: Phlo Team <team@phlo.dev>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.12
|
|
8
8
|
Description-Content-Type: text/plain
|
|
9
|
-
Requires-Dist: phlo<0.
|
|
9
|
+
Requires-Dist: phlo<0.19,>=0.18.0
|
|
10
10
|
Requires-Dist: psycopg2-binary>=2.9.11
|
|
11
11
|
Provides-Extra: dev
|
|
12
12
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
@@ -9,6 +9,7 @@ src/phlo_postgres/continuity.py
|
|
|
9
9
|
src/phlo_postgres/dataset_state_store.py
|
|
10
10
|
src/phlo_postgres/exporter_service.yaml
|
|
11
11
|
src/phlo_postgres/governance.py
|
|
12
|
+
src/phlo_postgres/operation_controls.py
|
|
12
13
|
src/phlo_postgres/plugin.py
|
|
13
14
|
src/phlo_postgres/publish_target.py
|
|
14
15
|
src/phlo_postgres/resource.py
|
|
@@ -10,9 +10,6 @@ from unittest.mock import patch, MagicMock
|
|
|
10
10
|
|
|
11
11
|
import pytest
|
|
12
12
|
|
|
13
|
-
pytestmark = pytest.mark.integration
|
|
14
|
-
|
|
15
|
-
|
|
16
13
|
# =============================================================================
|
|
17
14
|
# Connection Configuration Tests
|
|
18
15
|
# =============================================================================
|
|
@@ -185,12 +182,18 @@ def postgres_connection():
|
|
|
185
182
|
password=settings.postgres_password,
|
|
186
183
|
connect_timeout=5,
|
|
187
184
|
)
|
|
185
|
+
except psycopg2.OperationalError as e:
|
|
186
|
+
if e.pgcode is not None:
|
|
187
|
+
raise
|
|
188
|
+
pytest.skip(f"Postgres not available: {e}")
|
|
189
|
+
|
|
190
|
+
try:
|
|
188
191
|
yield conn
|
|
192
|
+
finally:
|
|
189
193
|
conn.close()
|
|
190
|
-
except Exception as e:
|
|
191
|
-
pytest.skip(f"Postgres not available: {e}")
|
|
192
194
|
|
|
193
195
|
|
|
196
|
+
@pytest.mark.integration
|
|
194
197
|
class TestPostgresIntegrationReal:
|
|
195
198
|
"""Real integration tests against a running Postgres instance."""
|
|
196
199
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{phlo_postgres-0.16.0 → phlo_postgres-0.17.0}/src/phlo_postgres.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|