phlo-postgres 0.16.0__tar.gz → 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/PKG-INFO +2 -2
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/pyproject.toml +2 -2
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/dataset_state_store.py +5 -5
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/exporter_service.yaml +1 -1
- phlo_postgres-0.18.0/src/phlo_postgres/operation_controls.py +407 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/service.yaml +5 -3
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/settings_store.py +13 -1
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/PKG-INFO +2 -2
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/SOURCES.txt +1 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/requires.txt +1 -1
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_authorization.py +3 -42
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_checkpoints.py +25 -1
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_dataset_state_store.py +91 -6
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_integration_postgres.py +78 -82
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/README.md +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/setup.cfg +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/__init__.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/authorization.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/checkpoints.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/cli.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/cli_plugin.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/continuity.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/governance.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/plugin.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/publish_target.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/resource.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/security_readiness.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/settings.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres/volume_setup.yaml +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/dependency_links.txt +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/entry_points.txt +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/top_level.txt +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_continuity.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_postgres_cli.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_postgres_plugin.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_resource.py +0 -0
- {phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/tests/test_settings_store.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phlo-postgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: Postgres service plugin for Phlo
|
|
5
5
|
Author-email: Phlo Team <team@phlo.dev>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.12
|
|
8
8
|
Description-Content-Type: text/plain
|
|
9
|
-
Requires-Dist: phlo<0.
|
|
9
|
+
Requires-Dist: phlo<0.20,>=0.19.0
|
|
10
10
|
Requires-Dist: psycopg2-binary>=2.9.11
|
|
11
11
|
Provides-Extra: dev
|
|
12
12
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
@@ -7,13 +7,13 @@ requires = [
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
dependencies = [
|
|
10
|
-
"phlo>=0.
|
|
10
|
+
"phlo>=0.19.0,<0.20",
|
|
11
11
|
"psycopg2-binary>=2.9.11",
|
|
12
12
|
]
|
|
13
13
|
description = "Postgres service plugin for Phlo"
|
|
14
14
|
name = "phlo-postgres"
|
|
15
15
|
requires-python = ">=3.12"
|
|
16
|
-
version = "0.
|
|
16
|
+
version = "0.18.0"
|
|
17
17
|
|
|
18
18
|
[[project.authors]]
|
|
19
19
|
email = "team@phlo.dev"
|
|
@@ -93,6 +93,7 @@ class SettingsDatasetStateStore:
|
|
|
93
93
|
def compare_and_set(
|
|
94
94
|
self,
|
|
95
95
|
*,
|
|
96
|
+
resource_id: str,
|
|
96
97
|
writes: tuple[StoreWrite, ...],
|
|
97
98
|
action_id: str,
|
|
98
99
|
action: str,
|
|
@@ -100,8 +101,7 @@ class SettingsDatasetStateStore:
|
|
|
100
101
|
) -> StoreWriteResult:
|
|
101
102
|
def apply(state: dict[str, Any] | None) -> dict[str, Any]:
|
|
102
103
|
current = _validated_state(state)
|
|
103
|
-
|
|
104
|
-
committed = current["actions"].get(last.record_id, {}).get(action_id)
|
|
104
|
+
committed = current["actions"].get(resource_id, {}).get(action_id)
|
|
105
105
|
if committed is not None:
|
|
106
106
|
if committed["fingerprint"] == fingerprint:
|
|
107
107
|
self._result = StoreWriteResult(
|
|
@@ -142,9 +142,9 @@ class SettingsDatasetStateStore:
|
|
|
142
142
|
next_state = _copy_state(current)
|
|
143
143
|
for write in writes:
|
|
144
144
|
next_state["records"][write.record_id] = write.next_record.to_read_model()
|
|
145
|
-
next_state["actions"].setdefault(
|
|
145
|
+
next_state["actions"].setdefault(resource_id, {})[action_id] = {
|
|
146
146
|
"action_id": action_id,
|
|
147
|
-
"resource_id":
|
|
147
|
+
"resource_id": resource_id,
|
|
148
148
|
"action": action,
|
|
149
149
|
"fingerprint": fingerprint,
|
|
150
150
|
"outcome_status": StoreWriteStatus.COMMITTED.value,
|
|
@@ -154,7 +154,7 @@ class SettingsDatasetStateStore:
|
|
|
154
154
|
status=StoreWriteStatus.COMMITTED,
|
|
155
155
|
records=tuple(w.next_record for w in writes),
|
|
156
156
|
committed_fingerprint=fingerprint,
|
|
157
|
-
detail=f"{action!r} committed for {
|
|
157
|
+
detail=f"{action!r} committed for {resource_id}.",
|
|
158
158
|
)
|
|
159
159
|
return next_state
|
|
160
160
|
|
|
@@ -16,7 +16,7 @@ compose:
|
|
|
16
16
|
phlo.metrics.port: "postgres-exporter:9187"
|
|
17
17
|
phlo.metrics.path: "/metrics"
|
|
18
18
|
environment:
|
|
19
|
-
DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-phlo}:${POSTGRES_PASSWORD
|
|
19
|
+
DATA_SOURCE_NAME: postgresql://${POSTGRES_USER:-phlo}:${POSTGRES_PASSWORD:?Run phlo services init to generate credentials}@postgres:5432/${POSTGRES_DB:-phlo}?sslmode=disable
|
|
20
20
|
ports:
|
|
21
21
|
- "${POSTGRES_EXPORTER_PORT:-9187}:9187"
|
|
22
22
|
healthcheck:
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
"""PostgreSQL-owned API operation controls, independent of replica-local paths.
|
|
2
|
+
|
|
3
|
+
Short transactions take a namespace advisory lock before checking absent rows.
|
|
4
|
+
Durable pending/unknown claims, not a connection lease, fence provider calls
|
|
5
|
+
after crashes. No unresolved claim is automatically expired.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Iterator
|
|
12
|
+
from contextlib import contextmanager
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import psycopg2
|
|
16
|
+
from psycopg2.extensions import cursor
|
|
17
|
+
|
|
18
|
+
from phlo.operations.journal import (
|
|
19
|
+
OperationJournalEntry,
|
|
20
|
+
OperationJournalState,
|
|
21
|
+
claims_conflict,
|
|
22
|
+
expire_claim,
|
|
23
|
+
maintenance_action_aliases,
|
|
24
|
+
new_claim,
|
|
25
|
+
transition_entry,
|
|
26
|
+
)
|
|
27
|
+
from phlo.plugins.observatory_settings import StorageUnavailableError
|
|
28
|
+
|
|
29
|
+
RATE_BUCKET_CAP = 4096
|
|
30
|
+
|
|
31
|
+
_SCHEMA = """
|
|
32
|
+
CREATE TABLE IF NOT EXISTS phlo_api_operations (
|
|
33
|
+
namespace TEXT NOT NULL, key_hash TEXT NOT NULL, operation TEXT NOT NULL,
|
|
34
|
+
target TEXT NOT NULL, exclusion_target TEXT NOT NULL,
|
|
35
|
+
state TEXT NOT NULL, response_json TEXT NOT NULL DEFAULT '',
|
|
36
|
+
expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + interval '24 hours'),
|
|
37
|
+
PRIMARY KEY (namespace, key_hash, operation, target)
|
|
38
|
+
);
|
|
39
|
+
CREATE UNIQUE INDEX IF NOT EXISTS phlo_api_operations_active_target
|
|
40
|
+
ON phlo_api_operations(namespace, operation, exclusion_target)
|
|
41
|
+
WHERE state IN ('pending', 'unknown');
|
|
42
|
+
CREATE TABLE IF NOT EXISTS phlo_api_operation_resolutions (
|
|
43
|
+
id BIGSERIAL PRIMARY KEY, namespace TEXT NOT NULL, key_hash TEXT NOT NULL,
|
|
44
|
+
operation TEXT NOT NULL, target TEXT NOT NULL, resolution TEXT NOT NULL,
|
|
45
|
+
resolved_by TEXT NOT NULL, evidence_json TEXT NOT NULL,
|
|
46
|
+
resolved_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
47
|
+
);
|
|
48
|
+
CREATE TABLE IF NOT EXISTS phlo_api_rate_buckets (
|
|
49
|
+
namespace TEXT NOT NULL, subject_hash TEXT NOT NULL, operation TEXT NOT NULL,
|
|
50
|
+
events DOUBLE PRECISION[] NOT NULL,
|
|
51
|
+
PRIMARY KEY(namespace, subject_hash, operation)
|
|
52
|
+
);
|
|
53
|
+
CREATE TABLE IF NOT EXISTS phlo_api_operation_journal (
|
|
54
|
+
namespace TEXT NOT NULL, operation_id TEXT NOT NULL,
|
|
55
|
+
action TEXT NOT NULL, target TEXT NOT NULL, state TEXT NOT NULL,
|
|
56
|
+
entry_json TEXT NOT NULL,
|
|
57
|
+
PRIMARY KEY(namespace, operation_id)
|
|
58
|
+
);
|
|
59
|
+
CREATE UNIQUE INDEX IF NOT EXISTS phlo_api_journal_active_target
|
|
60
|
+
ON phlo_api_operation_journal(namespace, action, target)
|
|
61
|
+
WHERE state IN ('claimed', 'submitted', 'unknown');
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class PostgresOperationControls:
|
|
66
|
+
"""Shared controls and the core durable-journal protocol for one project.
|
|
67
|
+
|
|
68
|
+
Connections are opened per transaction and always closed. The namespace
|
|
69
|
+
lock deliberately serialises short control transactions, not provider I/O.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(self, db_url: str, namespace: str) -> None:
|
|
73
|
+
if not namespace.strip():
|
|
74
|
+
raise ValueError("Shared operation controls require a stable namespace")
|
|
75
|
+
self._db_url = db_url
|
|
76
|
+
self._namespace = namespace
|
|
77
|
+
self._schema_ready = False
|
|
78
|
+
|
|
79
|
+
@contextmanager
|
|
80
|
+
def _transaction(self) -> Iterator[cursor]:
|
|
81
|
+
conn = None
|
|
82
|
+
try:
|
|
83
|
+
conn = psycopg2.connect(self._db_url, connect_timeout=5)
|
|
84
|
+
if not self._schema_ready:
|
|
85
|
+
# Commit DDL before waiting for the namespace lock. Otherwise
|
|
86
|
+
# a cold replica's table locks can deadlock with a warm writer
|
|
87
|
+
# that already owns the namespace and needs those tables.
|
|
88
|
+
with conn, conn.cursor() as cur:
|
|
89
|
+
cur.execute("SET LOCAL lock_timeout = '5s'")
|
|
90
|
+
cur.execute("SET LOCAL statement_timeout = '10s'")
|
|
91
|
+
# Concurrent CREATE IF NOT EXISTS can race on catalog keys.
|
|
92
|
+
cur.execute(
|
|
93
|
+
"SELECT pg_advisory_xact_lock(hashtext('phlo-api-controls-schema'))"
|
|
94
|
+
)
|
|
95
|
+
cur.execute(_SCHEMA)
|
|
96
|
+
self._schema_ready = True
|
|
97
|
+
with conn, conn.cursor() as cur:
|
|
98
|
+
cur.execute("SET LOCAL lock_timeout = '5s'")
|
|
99
|
+
cur.execute("SET LOCAL statement_timeout = '10s'")
|
|
100
|
+
cur.execute(
|
|
101
|
+
"SELECT pg_advisory_xact_lock(hashtext(%s))",
|
|
102
|
+
(f"phlo-api-controls:{self._namespace}",),
|
|
103
|
+
)
|
|
104
|
+
yield cur
|
|
105
|
+
except psycopg2.Error as exc:
|
|
106
|
+
raise StorageUnavailableError("Shared operation controls are unavailable") from exc
|
|
107
|
+
finally:
|
|
108
|
+
if conn is not None:
|
|
109
|
+
conn.close()
|
|
110
|
+
|
|
111
|
+
def initialize(self) -> None:
|
|
112
|
+
"""Check connectivity and initialise the schema before serving traffic."""
|
|
113
|
+
with self._transaction():
|
|
114
|
+
pass
|
|
115
|
+
|
|
116
|
+
@contextmanager
|
|
117
|
+
def reclamation_guard(self, identities: list[tuple[str, str]]) -> Iterator[None]:
|
|
118
|
+
"""Hold the claim transaction lock while disposing only quiescent local payloads."""
|
|
119
|
+
with self._transaction() as cur:
|
|
120
|
+
for operation, target in identities:
|
|
121
|
+
cur.execute(
|
|
122
|
+
"SELECT 1 FROM phlo_api_operations WHERE namespace=%s AND operation=%s "
|
|
123
|
+
"AND (target=%s OR exclusion_target=%s) AND state IN ('pending','unknown') LIMIT 1",
|
|
124
|
+
(self._namespace, operation, target, target),
|
|
125
|
+
)
|
|
126
|
+
if cur.fetchone() is not None:
|
|
127
|
+
raise OperationExclusionConflict(
|
|
128
|
+
"An unresolved mutation requires retained inputs"
|
|
129
|
+
)
|
|
130
|
+
yield
|
|
131
|
+
|
|
132
|
+
def claim_idempotency(
|
|
133
|
+
self,
|
|
134
|
+
key_hash: str,
|
|
135
|
+
operation: str,
|
|
136
|
+
target: str,
|
|
137
|
+
exclusion_target: str | None = None,
|
|
138
|
+
) -> tuple[bool, str, str]:
|
|
139
|
+
"""Claim identity and target together; return replay or a stable conflict."""
|
|
140
|
+
with self._transaction() as cur:
|
|
141
|
+
cur.execute(
|
|
142
|
+
"DELETE FROM phlo_api_operations WHERE namespace=%s "
|
|
143
|
+
"AND state='completed' AND expires_at < now()",
|
|
144
|
+
(self._namespace,),
|
|
145
|
+
)
|
|
146
|
+
cur.execute(
|
|
147
|
+
"SELECT target, state, response_json FROM phlo_api_operations "
|
|
148
|
+
"WHERE namespace=%s AND key_hash=%s AND operation=%s AND (%s OR target=%s)",
|
|
149
|
+
(self._namespace, key_hash, operation, exclusion_target is not None, target),
|
|
150
|
+
)
|
|
151
|
+
row = cur.fetchone()
|
|
152
|
+
if row is not None:
|
|
153
|
+
if row[0] != target:
|
|
154
|
+
return False, "target_mismatch", ""
|
|
155
|
+
if row[1] != "safe_to_retry":
|
|
156
|
+
return False, row[1], row[2]
|
|
157
|
+
cur.execute(
|
|
158
|
+
"SELECT state FROM phlo_api_operations WHERE namespace=%s "
|
|
159
|
+
"AND operation=%s AND exclusion_target=%s AND state IN ('pending', 'unknown') LIMIT 1",
|
|
160
|
+
(self._namespace, operation, exclusion_target or target),
|
|
161
|
+
)
|
|
162
|
+
active = cur.fetchone()
|
|
163
|
+
if active is not None:
|
|
164
|
+
return False, "operation_" + active[0], ""
|
|
165
|
+
cur.execute(
|
|
166
|
+
"INSERT INTO phlo_api_operations(namespace,key_hash,operation,target,exclusion_target,state) "
|
|
167
|
+
"VALUES (%s,%s,%s,%s,%s,'pending') ON CONFLICT(namespace,key_hash,operation,target) "
|
|
168
|
+
"DO UPDATE SET state='pending', response_json='', expires_at=now()+interval '24 hours'",
|
|
169
|
+
(self._namespace, key_hash, operation, target, exclusion_target or target),
|
|
170
|
+
)
|
|
171
|
+
return True, "pending", ""
|
|
172
|
+
|
|
173
|
+
def finish_idempotency(
|
|
174
|
+
self,
|
|
175
|
+
key_hash: str,
|
|
176
|
+
operation: str,
|
|
177
|
+
target: str,
|
|
178
|
+
state: str,
|
|
179
|
+
response: dict[str, Any] | None = None,
|
|
180
|
+
) -> None:
|
|
181
|
+
"""Record provider completion, or preserve an unknown outcome indefinitely."""
|
|
182
|
+
with self._transaction() as cur:
|
|
183
|
+
cur.execute(
|
|
184
|
+
"UPDATE phlo_api_operations SET state=%s, response_json=%s, "
|
|
185
|
+
"expires_at=now()+interval '24 hours' WHERE namespace=%s AND key_hash=%s "
|
|
186
|
+
"AND operation=%s AND target=%s AND state='pending'",
|
|
187
|
+
(
|
|
188
|
+
state,
|
|
189
|
+
json.dumps(response, sort_keys=True) if response is not None else "",
|
|
190
|
+
self._namespace,
|
|
191
|
+
key_hash,
|
|
192
|
+
operation,
|
|
193
|
+
target,
|
|
194
|
+
),
|
|
195
|
+
)
|
|
196
|
+
if cur.rowcount != 1:
|
|
197
|
+
raise StorageUnavailableError("Operation claim changed before completion")
|
|
198
|
+
|
|
199
|
+
def idempotency_target(self, key_hash: str, operation: str) -> str | None:
|
|
200
|
+
"""Read the canonical key binding across all replicas."""
|
|
201
|
+
with self._transaction() as cur:
|
|
202
|
+
cur.execute(
|
|
203
|
+
"SELECT target FROM phlo_api_operations WHERE namespace=%s AND key_hash=%s AND operation=%s",
|
|
204
|
+
(self._namespace, key_hash, operation),
|
|
205
|
+
)
|
|
206
|
+
row = cur.fetchone()
|
|
207
|
+
return row[0] if row is not None else None
|
|
208
|
+
|
|
209
|
+
def resolve_idempotency(
|
|
210
|
+
self,
|
|
211
|
+
key_hash: str,
|
|
212
|
+
operation: str,
|
|
213
|
+
target: str,
|
|
214
|
+
resolution: str,
|
|
215
|
+
resolved_by: str,
|
|
216
|
+
evidence: dict[str, Any],
|
|
217
|
+
response: dict[str, Any] | None,
|
|
218
|
+
) -> None:
|
|
219
|
+
"""Retain operator evidence; only explicit safe_to_retry permits another call."""
|
|
220
|
+
state = "completed" if resolution == "succeeded" else resolution
|
|
221
|
+
evidence_json = json.dumps(evidence, sort_keys=True)
|
|
222
|
+
with self._transaction() as cur:
|
|
223
|
+
identity = (self._namespace, key_hash, operation, target)
|
|
224
|
+
cur.execute(
|
|
225
|
+
"SELECT state FROM phlo_api_operations WHERE namespace=%s "
|
|
226
|
+
"AND key_hash=%s AND operation=%s AND target=%s",
|
|
227
|
+
identity,
|
|
228
|
+
)
|
|
229
|
+
row = cur.fetchone()
|
|
230
|
+
if row is None or row[0] not in {"pending", "unknown"}:
|
|
231
|
+
cur.execute(
|
|
232
|
+
"SELECT resolution,resolved_by,evidence_json FROM phlo_api_operation_resolutions "
|
|
233
|
+
"WHERE namespace=%s AND key_hash=%s AND operation=%s AND target=%s ORDER BY id DESC LIMIT 1",
|
|
234
|
+
identity,
|
|
235
|
+
)
|
|
236
|
+
prior = cur.fetchone()
|
|
237
|
+
if (
|
|
238
|
+
row is not None
|
|
239
|
+
and row[0] == state
|
|
240
|
+
and prior == (resolution, resolved_by, evidence_json)
|
|
241
|
+
):
|
|
242
|
+
return
|
|
243
|
+
raise ValueError("only pending or unknown idempotency claims can be resolved")
|
|
244
|
+
cur.execute(
|
|
245
|
+
"UPDATE phlo_api_operations SET state=%s,response_json=%s,expires_at=now()+interval '24 hours' "
|
|
246
|
+
"WHERE namespace=%s AND key_hash=%s AND operation=%s AND target=%s",
|
|
247
|
+
(
|
|
248
|
+
state,
|
|
249
|
+
json.dumps(response, sort_keys=True) if response is not None else "",
|
|
250
|
+
*identity,
|
|
251
|
+
),
|
|
252
|
+
)
|
|
253
|
+
cur.execute(
|
|
254
|
+
"INSERT INTO phlo_api_operation_resolutions(namespace,key_hash,operation,target,resolution,resolved_by,evidence_json) "
|
|
255
|
+
"VALUES (%s,%s,%s,%s,%s,%s,%s)",
|
|
256
|
+
(*identity, resolution, resolved_by, evidence_json),
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
def rate_limit(self, subject_hash: str, operation: str, limit: int) -> bool:
|
|
260
|
+
"""Bound both bucket cardinality and each sliding window; fail closed at capacity."""
|
|
261
|
+
with self._transaction() as cur:
|
|
262
|
+
cur.execute("SELECT extract(epoch FROM clock_timestamp())::double precision")
|
|
263
|
+
now = cur.fetchone()[0]
|
|
264
|
+
cur.execute(
|
|
265
|
+
"DELETE FROM phlo_api_rate_buckets WHERE namespace=%s AND events[array_length(events,1)] < %s",
|
|
266
|
+
(self._namespace, now - 60),
|
|
267
|
+
)
|
|
268
|
+
identity = (self._namespace, subject_hash, operation)
|
|
269
|
+
cur.execute(
|
|
270
|
+
"SELECT events FROM phlo_api_rate_buckets WHERE namespace=%s AND subject_hash=%s AND operation=%s",
|
|
271
|
+
identity,
|
|
272
|
+
)
|
|
273
|
+
row = cur.fetchone()
|
|
274
|
+
events = [stamp for stamp in row[0] if stamp >= now - 60] if row is not None else []
|
|
275
|
+
if len(events) >= limit:
|
|
276
|
+
return False
|
|
277
|
+
if row is None:
|
|
278
|
+
cur.execute(
|
|
279
|
+
"SELECT count(*) FROM phlo_api_rate_buckets WHERE namespace=%s",
|
|
280
|
+
(self._namespace,),
|
|
281
|
+
)
|
|
282
|
+
if cur.fetchone()[0] >= RATE_BUCKET_CAP:
|
|
283
|
+
return False
|
|
284
|
+
cur.execute(
|
|
285
|
+
"INSERT INTO phlo_api_rate_buckets(namespace,subject_hash,operation,events) VALUES (%s,%s,%s,%s) "
|
|
286
|
+
"ON CONFLICT(namespace,subject_hash,operation) DO UPDATE SET events=EXCLUDED.events",
|
|
287
|
+
(*identity, [*events, now]),
|
|
288
|
+
)
|
|
289
|
+
return True
|
|
290
|
+
|
|
291
|
+
def claim(self, entry: OperationJournalEntry) -> bool:
|
|
292
|
+
"""Claim a journal operation and its active target across replicas."""
|
|
293
|
+
entry = new_claim(entry)
|
|
294
|
+
with self._transaction() as cur:
|
|
295
|
+
cur.execute(
|
|
296
|
+
"SELECT entry_json FROM phlo_api_operation_journal WHERE namespace=%s AND "
|
|
297
|
+
"(operation_id=%s OR (action=ANY(%s) AND target=%s AND state IN ('claimed','submitted','unknown')))",
|
|
298
|
+
(
|
|
299
|
+
self._namespace,
|
|
300
|
+
entry.operation_id,
|
|
301
|
+
list(maintenance_action_aliases(entry.action)),
|
|
302
|
+
entry.target,
|
|
303
|
+
),
|
|
304
|
+
)
|
|
305
|
+
for row in cur.fetchall():
|
|
306
|
+
record = json.loads(row[0])
|
|
307
|
+
record["state"] = OperationJournalState(record["state"])
|
|
308
|
+
previous = OperationJournalEntry(**record)
|
|
309
|
+
if previous.operation_id == entry.operation_id:
|
|
310
|
+
return False
|
|
311
|
+
current = expire_claim(previous)
|
|
312
|
+
if current != previous:
|
|
313
|
+
cur.execute(
|
|
314
|
+
"UPDATE phlo_api_operation_journal SET state=%s,entry_json=%s WHERE namespace=%s AND operation_id=%s",
|
|
315
|
+
(
|
|
316
|
+
current.state.value,
|
|
317
|
+
json.dumps(current.to_dict()),
|
|
318
|
+
self._namespace,
|
|
319
|
+
current.operation_id,
|
|
320
|
+
),
|
|
321
|
+
)
|
|
322
|
+
if claims_conflict(current, entry):
|
|
323
|
+
return False
|
|
324
|
+
cur.execute(
|
|
325
|
+
"INSERT INTO phlo_api_operation_journal(namespace,operation_id,action,target,state,entry_json) "
|
|
326
|
+
"VALUES (%s,%s,%s,%s,%s,%s)",
|
|
327
|
+
(
|
|
328
|
+
self._namespace,
|
|
329
|
+
entry.operation_id,
|
|
330
|
+
entry.action,
|
|
331
|
+
entry.target,
|
|
332
|
+
entry.state.value,
|
|
333
|
+
json.dumps(entry.to_dict()),
|
|
334
|
+
),
|
|
335
|
+
)
|
|
336
|
+
return True
|
|
337
|
+
|
|
338
|
+
def compare_and_set(
|
|
339
|
+
self,
|
|
340
|
+
operation_id: str,
|
|
341
|
+
expected: OperationJournalState,
|
|
342
|
+
state: OperationJournalState,
|
|
343
|
+
result: dict[str, Any] | None = None,
|
|
344
|
+
) -> bool:
|
|
345
|
+
"""Persist a journal transition without releasing unresolved target claims."""
|
|
346
|
+
with self._transaction() as cur:
|
|
347
|
+
cur.execute(
|
|
348
|
+
"SELECT entry_json FROM phlo_api_operation_journal WHERE namespace=%s AND operation_id=%s",
|
|
349
|
+
(self._namespace, operation_id),
|
|
350
|
+
)
|
|
351
|
+
row = cur.fetchone()
|
|
352
|
+
if row is None:
|
|
353
|
+
return False
|
|
354
|
+
record = json.loads(row[0])
|
|
355
|
+
record["state"] = OperationJournalState(record["state"])
|
|
356
|
+
entry = OperationJournalEntry(**record)
|
|
357
|
+
if entry.state is not expected:
|
|
358
|
+
return False
|
|
359
|
+
updated = transition_entry(entry, state, result)
|
|
360
|
+
cur.execute(
|
|
361
|
+
"UPDATE phlo_api_operation_journal SET state=%s,entry_json=%s WHERE namespace=%s AND operation_id=%s",
|
|
362
|
+
(state.value, json.dumps(updated.to_dict()), self._namespace, operation_id),
|
|
363
|
+
)
|
|
364
|
+
return True
|
|
365
|
+
|
|
366
|
+
def read(self, operation_id: str) -> OperationJournalEntry | None:
|
|
367
|
+
"""Read journal evidence from any replica."""
|
|
368
|
+
with self._transaction() as cur:
|
|
369
|
+
cur.execute(
|
|
370
|
+
"SELECT entry_json FROM phlo_api_operation_journal WHERE namespace=%s AND operation_id=%s",
|
|
371
|
+
(self._namespace, operation_id),
|
|
372
|
+
)
|
|
373
|
+
row = cur.fetchone()
|
|
374
|
+
if row is None:
|
|
375
|
+
return None
|
|
376
|
+
record = json.loads(row[0])
|
|
377
|
+
record["state"] = OperationJournalState(record["state"])
|
|
378
|
+
return OperationJournalEntry(**record)
|
|
379
|
+
|
|
380
|
+
@contextmanager
|
|
381
|
+
def exclusion(self, operation: str, target: str) -> Iterator[None]:
|
|
382
|
+
"""Non-blocking session advisory lock for a compound provider action.
|
|
383
|
+
|
|
384
|
+
Durable idempotency/journal claims must additionally fence effects whose
|
|
385
|
+
outcome may be unknown after connection loss or process termination.
|
|
386
|
+
"""
|
|
387
|
+
conn = None
|
|
388
|
+
try:
|
|
389
|
+
conn = psycopg2.connect(self._db_url, connect_timeout=5)
|
|
390
|
+
with conn.cursor() as cur:
|
|
391
|
+
cur.execute(
|
|
392
|
+
"SELECT pg_try_advisory_lock(hashtextextended(%s,0))",
|
|
393
|
+
(json.dumps([self._namespace, operation, target]),),
|
|
394
|
+
)
|
|
395
|
+
acquired = cur.fetchone()[0]
|
|
396
|
+
if not acquired:
|
|
397
|
+
raise OperationExclusionConflict("Operation is already in progress")
|
|
398
|
+
yield
|
|
399
|
+
except psycopg2.Error as exc:
|
|
400
|
+
raise StorageUnavailableError("Shared operation exclusion is unavailable") from exc
|
|
401
|
+
finally:
|
|
402
|
+
if conn is not None:
|
|
403
|
+
conn.close()
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
class OperationExclusionConflict(RuntimeError):
|
|
407
|
+
"""Another instance holds the compound-action advisory lock."""
|
|
@@ -19,7 +19,7 @@ compose:
|
|
|
19
19
|
phlo.grafana.datasource.database: "${POSTGRES_DB:-phlo}"
|
|
20
20
|
environment:
|
|
21
21
|
POSTGRES_USER: ${POSTGRES_USER:-phlo}
|
|
22
|
-
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD
|
|
22
|
+
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run phlo services init to generate POSTGRES_PASSWORD}
|
|
23
23
|
POSTGRES_DB: ${POSTGRES_DB:-phlo}
|
|
24
24
|
# SSL/TLS
|
|
25
25
|
POSTGRES_SSL_MODE: ${POSTGRES_SSL_MODE:-prefer}
|
|
@@ -28,7 +28,10 @@ compose:
|
|
|
28
28
|
volumes:
|
|
29
29
|
- postgres-data:/var/lib/postgresql
|
|
30
30
|
healthcheck:
|
|
31
|
-
|
|
31
|
+
# Probe over TCP: pg_isready via the unix socket also answers during the
|
|
32
|
+
# image's temporary init server, marking the container healthy before it
|
|
33
|
+
# restarts for real — dependents connecting in the gap lose the connection.
|
|
34
|
+
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U ${POSTGRES_USER:-phlo}"]
|
|
32
35
|
interval: 10s
|
|
33
36
|
timeout: 5s
|
|
34
37
|
retries: 10
|
|
@@ -38,7 +41,6 @@ env_vars:
|
|
|
38
41
|
default: phlo
|
|
39
42
|
description: PostgreSQL username
|
|
40
43
|
POSTGRES_PASSWORD:
|
|
41
|
-
default: phlo
|
|
42
44
|
description: PostgreSQL password
|
|
43
45
|
secret: true
|
|
44
46
|
POSTGRES_DB:
|
|
@@ -27,6 +27,13 @@ from phlo_postgres.settings import get_settings as get_postgres_settings
|
|
|
27
27
|
logger = get_logger(__name__)
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
class _MutationCallbackFailure(Exception):
|
|
31
|
+
"""Carry a user mutation exception through storage error translation."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, error: Exception) -> None:
|
|
34
|
+
self.error = error
|
|
35
|
+
|
|
36
|
+
|
|
30
37
|
def _get_psycopg2():
|
|
31
38
|
try:
|
|
32
39
|
import psycopg2
|
|
@@ -160,7 +167,10 @@ class PostgresSettingsStore:
|
|
|
160
167
|
(scope.value, namespace),
|
|
161
168
|
)
|
|
162
169
|
row = cursor.fetchone()
|
|
163
|
-
|
|
170
|
+
try:
|
|
171
|
+
settings = mutation(row[0] if row else None)
|
|
172
|
+
except Exception as exc:
|
|
173
|
+
raise _MutationCallbackFailure(exc) from exc
|
|
164
174
|
cursor.execute(
|
|
165
175
|
"""
|
|
166
176
|
INSERT INTO phlo_settings (scope, namespace, settings, updated_at)
|
|
@@ -179,6 +189,8 @@ class PostgresSettingsStore:
|
|
|
179
189
|
settings=stored_settings,
|
|
180
190
|
updated_at=updated_at.isoformat() if updated_at else None,
|
|
181
191
|
)
|
|
192
|
+
except _MutationCallbackFailure as exc:
|
|
193
|
+
raise exc.error
|
|
182
194
|
except Exception as exc:
|
|
183
195
|
if isinstance(exc, StorageUnavailableError):
|
|
184
196
|
raise
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phlo-postgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: Postgres service plugin for Phlo
|
|
5
5
|
Author-email: Phlo Team <team@phlo.dev>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.12
|
|
8
8
|
Description-Content-Type: text/plain
|
|
9
|
-
Requires-Dist: phlo<0.
|
|
9
|
+
Requires-Dist: phlo<0.20,>=0.19.0
|
|
10
10
|
Requires-Dist: psycopg2-binary>=2.9.11
|
|
11
11
|
Provides-Extra: dev
|
|
12
12
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
@@ -9,6 +9,7 @@ src/phlo_postgres/continuity.py
|
|
|
9
9
|
src/phlo_postgres/dataset_state_store.py
|
|
10
10
|
src/phlo_postgres/exporter_service.yaml
|
|
11
11
|
src/phlo_postgres/governance.py
|
|
12
|
+
src/phlo_postgres/operation_controls.py
|
|
12
13
|
src/phlo_postgres/plugin.py
|
|
13
14
|
src/phlo_postgres/publish_target.py
|
|
14
15
|
src/phlo_postgres/resource.py
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"""Tests for the PostgreSQL CLI authorization surface.
|
|
2
2
|
|
|
3
|
-
Package-specific
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
fallbacks delegate to ``phlo_testing.authorization_surface``.
|
|
3
|
+
Package-specific command classification and mapping; the shared adapter
|
|
4
|
+
contract (read allow-through, unknown deny-closed, mutation policy allow/deny)
|
|
5
|
+
delegates to ``phlo_testing.authorization_surface``.
|
|
7
6
|
"""
|
|
8
7
|
|
|
9
8
|
from __future__ import annotations
|
|
@@ -12,56 +11,18 @@ from phlo_testing.authorization_surface import (
|
|
|
12
11
|
assert_mutation_enforcement_allows_and_denies,
|
|
13
12
|
assert_read_commands_allow_without_enforcement,
|
|
14
13
|
assert_unknown_command_denied,
|
|
15
|
-
run_cli_principal_resolver_contract,
|
|
16
14
|
)
|
|
17
15
|
from phlo_postgres.authorization import (
|
|
18
16
|
COMMAND_ACTION_MAP,
|
|
19
17
|
COMMAND_RESOURCE_MAP,
|
|
20
18
|
MUTATION_COMMANDS,
|
|
21
|
-
SURFACE_NAME,
|
|
22
|
-
FRAMEWORK_TYPE,
|
|
23
19
|
PostgresCliSurfaceAdapter,
|
|
24
20
|
)
|
|
25
21
|
|
|
26
22
|
|
|
27
|
-
def test_cli_principal_resolver_environment_fallbacks() -> None:
|
|
28
|
-
"""Shared CliPrincipalResolver environment fallback contract."""
|
|
29
|
-
run_cli_principal_resolver_contract()
|
|
30
|
-
|
|
31
|
-
|
|
32
23
|
class TestPostgresCliSurfaceAdapter:
|
|
33
24
|
"""Tests for PostgreSQL CLI surface adapter."""
|
|
34
25
|
|
|
35
|
-
def test_singleton_instance(self) -> None:
|
|
36
|
-
"""Should return singleton instance."""
|
|
37
|
-
adapter1 = PostgresCliSurfaceAdapter.get_instance()
|
|
38
|
-
adapter2 = PostgresCliSurfaceAdapter.get_instance()
|
|
39
|
-
assert adapter1 is adapter2
|
|
40
|
-
|
|
41
|
-
def test_surface_name(self) -> None:
|
|
42
|
-
"""Should return correct surface name."""
|
|
43
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
44
|
-
assert adapter.surface_name == SURFACE_NAME
|
|
45
|
-
|
|
46
|
-
def test_framework_type(self) -> None:
|
|
47
|
-
"""Should return correct framework type."""
|
|
48
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
49
|
-
assert adapter.framework_type == FRAMEWORK_TYPE
|
|
50
|
-
|
|
51
|
-
def test_list_operations(self) -> None:
|
|
52
|
-
"""Should declare mutation operations."""
|
|
53
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
54
|
-
operations = adapter.list_operations()
|
|
55
|
-
assert len(operations) == len(MUTATION_COMMANDS)
|
|
56
|
-
for op in operations:
|
|
57
|
-
assert op["action"] in COMMAND_ACTION_MAP.values()
|
|
58
|
-
assert op["resource_type"] in COMMAND_RESOURCE_MAP.values()
|
|
59
|
-
|
|
60
|
-
def test_is_active(self) -> None:
|
|
61
|
-
"""Should always return True."""
|
|
62
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
63
|
-
assert adapter.is_active(None) is True
|
|
64
|
-
|
|
65
26
|
def test_read_commands_return_allow(self) -> None:
|
|
66
27
|
"""Read commands should return allow without enforcement."""
|
|
67
28
|
assert_read_commands_allow_without_enforcement(PostgresCliSurfaceAdapter())
|
|
@@ -166,8 +166,24 @@ def test_record_snapshot_and_commit_follow_the_lifecycle() -> None:
|
|
|
166
166
|
staged = store.record_snapshot(checkpoint_id="cp-1", snapshot_id=42, release_id="release-1")
|
|
167
167
|
assert staged.status == "staged"
|
|
168
168
|
assert staged.snapshot_id == 42
|
|
169
|
+
stage_sql = " ".join(resource.cursor_obj.statements[0].split())
|
|
170
|
+
assert "SET status = 'staged', snapshot_id = %s, release_id = %s" in stage_sql
|
|
171
|
+
assert "failure_reason = NULL" in stage_sql
|
|
172
|
+
assert resource.cursor_obj.params[0] == (
|
|
173
|
+
42,
|
|
174
|
+
"release-1",
|
|
175
|
+
"cp-1",
|
|
176
|
+
("claimed", "staged", "failed"),
|
|
177
|
+
)
|
|
178
|
+
|
|
169
179
|
committed = store.commit(checkpoint_id="cp-1")
|
|
170
180
|
assert committed.status == "committed"
|
|
181
|
+
commit_sql = " ".join(resource.cursor_obj.statements[1].split())
|
|
182
|
+
assert "SET status = 'committed'" in commit_sql
|
|
183
|
+
# A commit may only advance a checkpoint that already has an audited snapshot.
|
|
184
|
+
assert "status = 'staged'" in commit_sql
|
|
185
|
+
assert "snapshot_id IS NOT NULL" in commit_sql
|
|
186
|
+
assert resource.cursor_obj.params[1] == ("cp-1",)
|
|
171
187
|
|
|
172
188
|
|
|
173
189
|
def test_commit_without_snapshot_raises_and_refuses_to_advance() -> None:
|
|
@@ -206,6 +222,10 @@ def test_fail_records_reason_but_keeps_ranges() -> None:
|
|
|
206
222
|
assert record.status == "failed"
|
|
207
223
|
assert record.failure_reason == reason
|
|
208
224
|
assert record.ranges, "failed checkpoints retain their claimed ranges"
|
|
225
|
+
update_sql = " ".join(resource.cursor_obj.statements[-1].split())
|
|
226
|
+
assert "SET status = 'failed', failure_reason = %s" in update_sql
|
|
227
|
+
assert "status IN %s" in update_sql
|
|
228
|
+
assert resource.cursor_obj.params[-1] == (reason, "cp-1", ("claimed", "staged", "failed"))
|
|
209
229
|
|
|
210
230
|
|
|
211
231
|
def test_provider_registers_checkpoint_resource() -> None:
|
|
@@ -216,6 +236,10 @@ def test_provider_registers_checkpoint_resource() -> None:
|
|
|
216
236
|
|
|
217
237
|
|
|
218
238
|
def test_real_resource_is_used_when_not_injected() -> None:
|
|
239
|
+
from phlo_postgres.resource import PostgresResource
|
|
240
|
+
|
|
219
241
|
store = PostgresIngestionCheckpointStore()
|
|
220
242
|
assert store._resource is None
|
|
221
|
-
|
|
243
|
+
resource = store._ensure_resource()
|
|
244
|
+
assert isinstance(resource, PostgresResource)
|
|
245
|
+
assert store._ensure_resource() is resource
|
|
@@ -19,7 +19,12 @@ import pytest
|
|
|
19
19
|
|
|
20
20
|
from phlo.dataset import (
|
|
21
21
|
CandidateRecord,
|
|
22
|
+
DatasetPolicy,
|
|
22
23
|
DatasetRecord,
|
|
24
|
+
DatasetService,
|
|
25
|
+
StaticPolicySource,
|
|
26
|
+
TransitionRequest,
|
|
27
|
+
TransitionStatus,
|
|
23
28
|
state_store_namespace,
|
|
24
29
|
)
|
|
25
30
|
from phlo.dataset.models import PublicationState, WorkflowState
|
|
@@ -33,6 +38,7 @@ from phlo_postgres.dataset_state_store import (
|
|
|
33
38
|
from phlo_postgres.plugin import PostgresResourceProvider
|
|
34
39
|
|
|
35
40
|
from _dataset_test_backends import FileLockSettingsStore
|
|
41
|
+
from _dataset_worker import NoEvidence
|
|
36
42
|
|
|
37
43
|
WORKER = Path(__file__).parent / "_dataset_worker.py"
|
|
38
44
|
|
|
@@ -71,22 +77,88 @@ def test_resource_provider_exposes_dataset_state_store() -> None:
|
|
|
71
77
|
# ---------------------------------------------------------------------------
|
|
72
78
|
|
|
73
79
|
|
|
80
|
+
@pytest.mark.parametrize("expected_states", [False, True])
|
|
81
|
+
def test_service_receipts_replay_after_restart(tmp_path: Path, expected_states: bool) -> None:
|
|
82
|
+
from dataclasses import replace
|
|
83
|
+
|
|
84
|
+
path = tmp_path / "state.json"
|
|
85
|
+
store = make_store(path)
|
|
86
|
+
service = DatasetService(
|
|
87
|
+
store=store,
|
|
88
|
+
evidence_source=NoEvidence(),
|
|
89
|
+
policy_source=StaticPolicySource(DatasetPolicy(policy_version="test-policy")),
|
|
90
|
+
)
|
|
91
|
+
requests = [
|
|
92
|
+
TransitionRequest(
|
|
93
|
+
resource_id=resource_id,
|
|
94
|
+
action=action,
|
|
95
|
+
action_id=action,
|
|
96
|
+
expected_state=state if expected_states else None,
|
|
97
|
+
)
|
|
98
|
+
for resource_id, action, state in (
|
|
99
|
+
("candidate:gold.demo", "claim", "open"),
|
|
100
|
+
("candidate:gold.demo", "review", "claimed"),
|
|
101
|
+
("candidate:gold.demo", "promote", "review"),
|
|
102
|
+
("gold.demo", "publish", "draft"),
|
|
103
|
+
("gold.demo", "retire", "published"),
|
|
104
|
+
)
|
|
105
|
+
]
|
|
106
|
+
for request in requests:
|
|
107
|
+
first = service.transition(request)
|
|
108
|
+
assert first.status is TransitionStatus.COMMITTED
|
|
109
|
+
service = DatasetService(
|
|
110
|
+
store=make_store(path),
|
|
111
|
+
evidence_source=NoEvidence(),
|
|
112
|
+
policy_source=StaticPolicySource(DatasetPolicy(policy_version="changed-policy")),
|
|
113
|
+
)
|
|
114
|
+
replay = service.transition(request)
|
|
115
|
+
assert replay.status is TransitionStatus.REPLAYED
|
|
116
|
+
assert replay.after_state == first.after_state
|
|
117
|
+
assert replay.record == first.record
|
|
118
|
+
conflict = service.transition(replace(request, expected_state="different-state"))
|
|
119
|
+
assert conflict.status is TransitionStatus.CONFLICT
|
|
120
|
+
assert "different request" in conflict.message
|
|
121
|
+
assert service.record(first.record.dataset_id) == first.record
|
|
122
|
+
|
|
123
|
+
assert store.committed_action("candidate:gold.demo", "promote") is not None
|
|
124
|
+
assert store.committed_action("gold.demo", "promote") is None
|
|
125
|
+
assert service.record("candidate:gold.demo").current_state == "promoted"
|
|
126
|
+
assert service.record("gold.demo").current_state == "retired"
|
|
127
|
+
assert [event.outcome for event in store.audit_events()] == [
|
|
128
|
+
"committed",
|
|
129
|
+
"replayed",
|
|
130
|
+
"conflict",
|
|
131
|
+
] * 5
|
|
132
|
+
|
|
133
|
+
|
|
74
134
|
def test_compare_and_set_commits_and_replays(tmp_path: Path) -> None:
|
|
75
135
|
store = make_store(tmp_path / "state.json")
|
|
76
136
|
write = StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record())
|
|
77
137
|
first = store.compare_and_set(
|
|
78
|
-
|
|
138
|
+
resource_id="gold.demo",
|
|
139
|
+
writes=(write,),
|
|
140
|
+
action_id="a1",
|
|
141
|
+
action="promote",
|
|
142
|
+
fingerprint="fp-1",
|
|
79
143
|
)
|
|
80
144
|
assert first.status is StoreWriteStatus.COMMITTED
|
|
81
145
|
|
|
82
146
|
replay = store.compare_and_set(
|
|
83
|
-
|
|
147
|
+
resource_id="gold.demo",
|
|
148
|
+
writes=(write,),
|
|
149
|
+
action_id="a1",
|
|
150
|
+
action="promote",
|
|
151
|
+
fingerprint="fp-1",
|
|
84
152
|
)
|
|
85
153
|
assert replay.status is StoreWriteStatus.REPLAYED
|
|
86
154
|
assert replay.committed_fingerprint == "fp-1"
|
|
87
155
|
|
|
88
156
|
conflict = store.compare_and_set(
|
|
89
|
-
|
|
157
|
+
resource_id="gold.demo",
|
|
158
|
+
writes=(write,),
|
|
159
|
+
action_id="a1",
|
|
160
|
+
action="promote",
|
|
161
|
+
fingerprint="fp-2",
|
|
90
162
|
)
|
|
91
163
|
assert conflict.status is StoreWriteStatus.ACTION_CONFLICT
|
|
92
164
|
|
|
@@ -97,7 +169,9 @@ def test_compare_and_set_commits_and_replays(tmp_path: Path) -> None:
|
|
|
97
169
|
def test_compare_and_set_fails_on_precondition_mismatch(tmp_path: Path) -> None:
|
|
98
170
|
store = make_store(tmp_path / "state.json")
|
|
99
171
|
stale = StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record())
|
|
100
|
-
store.compare_and_set(
|
|
172
|
+
store.compare_and_set(
|
|
173
|
+
resource_id="gold.demo", writes=(stale,), action_id="a1", action="promote", fingerprint="fp"
|
|
174
|
+
)
|
|
101
175
|
moved = StoreWrite(
|
|
102
176
|
record_id="gold.demo",
|
|
103
177
|
expected_state="open",
|
|
@@ -106,7 +180,11 @@ def test_compare_and_set_fails_on_precondition_mismatch(tmp_path: Path) -> None:
|
|
|
106
180
|
),
|
|
107
181
|
)
|
|
108
182
|
result = store.compare_and_set(
|
|
109
|
-
|
|
183
|
+
resource_id="gold.demo",
|
|
184
|
+
writes=(moved,),
|
|
185
|
+
action_id="a2",
|
|
186
|
+
action="publish",
|
|
187
|
+
fingerprint="fp2",
|
|
110
188
|
)
|
|
111
189
|
assert result.status is StoreWriteStatus.PRECONDITION_FAILED
|
|
112
190
|
|
|
@@ -117,6 +195,7 @@ def test_promote_writes_both_records_atomically(tmp_path: Path) -> None:
|
|
|
117
195
|
# Seed the candidate in review, the promote pre-state.
|
|
118
196
|
seed = CandidateRecord(dataset_id=f"candidate:{table_id}", table_id=table_id, state="review")
|
|
119
197
|
store.compare_and_set(
|
|
198
|
+
resource_id=f"candidate:{table_id}",
|
|
120
199
|
writes=(
|
|
121
200
|
StoreWrite(record_id=f"candidate:{table_id}", expected_state="open", next_record=seed),
|
|
122
201
|
),
|
|
@@ -145,7 +224,11 @@ def test_promote_writes_both_records_atomically(tmp_path: Path) -> None:
|
|
|
145
224
|
),
|
|
146
225
|
)
|
|
147
226
|
result = store.compare_and_set(
|
|
148
|
-
|
|
227
|
+
resource_id=f"candidate:{table_id}",
|
|
228
|
+
writes=writes,
|
|
229
|
+
action_id="p1",
|
|
230
|
+
action="promote",
|
|
231
|
+
fingerprint="fp",
|
|
149
232
|
)
|
|
150
233
|
assert result.status is StoreWriteStatus.COMMITTED
|
|
151
234
|
assert len(result.records) == 2
|
|
@@ -251,6 +334,7 @@ def test_state_survives_restart(tmp_path: Path) -> None:
|
|
|
251
334
|
path = tmp_path / "state.json"
|
|
252
335
|
store = make_store(path)
|
|
253
336
|
store.compare_and_set(
|
|
337
|
+
resource_id="gold.demo",
|
|
254
338
|
writes=(
|
|
255
339
|
StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record()),
|
|
256
340
|
),
|
|
@@ -300,6 +384,7 @@ def test_two_processes_race_one_publication_transition(tmp_path: Path) -> None:
|
|
|
300
384
|
# Seed the draft dataset the way a first worker's promote would.
|
|
301
385
|
seed = make_store(state_file, project_root)
|
|
302
386
|
seed.compare_and_set(
|
|
387
|
+
resource_id="gold.demo",
|
|
303
388
|
writes=(
|
|
304
389
|
StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record()),
|
|
305
390
|
),
|
|
@@ -10,9 +10,6 @@ from unittest.mock import patch, MagicMock
|
|
|
10
10
|
|
|
11
11
|
import pytest
|
|
12
12
|
|
|
13
|
-
pytestmark = pytest.mark.integration
|
|
14
|
-
|
|
15
|
-
|
|
16
13
|
# =============================================================================
|
|
17
14
|
# Connection Configuration Tests
|
|
18
15
|
# =============================================================================
|
|
@@ -32,30 +29,47 @@ class TestPostgresConfiguration:
|
|
|
32
29
|
assert hasattr(settings, "postgres_db")
|
|
33
30
|
assert hasattr(settings, "postgres_user")
|
|
34
31
|
|
|
35
|
-
def test_postgres_config_has_defaults(self):
|
|
36
|
-
"""Test Postgres config has
|
|
37
|
-
from phlo_postgres.settings import
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
32
|
+
def test_postgres_config_has_defaults(self, monkeypatch, tmp_path):
|
|
33
|
+
"""Test Postgres config has concrete, non-empty defaults."""
|
|
34
|
+
from phlo_postgres.settings import PostgresSettings
|
|
35
|
+
|
|
36
|
+
for name in (
|
|
37
|
+
"POSTGRES_HOST",
|
|
38
|
+
"POSTGRES_PORT",
|
|
39
|
+
"POSTGRES_USER",
|
|
40
|
+
"POSTGRES_DB",
|
|
41
|
+
"POSTGRES_MART_SCHEMA",
|
|
42
|
+
):
|
|
43
|
+
monkeypatch.delenv(name, raising=False)
|
|
44
|
+
with patch("phlo.config.network.socket.gethostbyname", lambda _host: "127.0.0.1"):
|
|
45
|
+
# Isolate from this workspace's .phlo/.env, which may set POSTGRES_PORT.
|
|
46
|
+
settings = PostgresSettings(_project_root=tmp_path)
|
|
47
|
+
|
|
48
|
+
assert settings.postgres_host == "postgres"
|
|
49
|
+
assert settings.postgres_port == 5432
|
|
50
|
+
assert settings.postgres_user == "phlo"
|
|
51
|
+
assert settings.postgres_db == "phlo"
|
|
52
|
+
assert settings.postgres_mart_schema == "marts"
|
|
44
53
|
|
|
45
54
|
def test_connection_string_building(self):
|
|
46
|
-
"""Test
|
|
47
|
-
from phlo_postgres.settings import
|
|
48
|
-
|
|
49
|
-
|
|
55
|
+
"""Test the production connection string encodes credentials correctly."""
|
|
56
|
+
from phlo_postgres.settings import PostgresSettings
|
|
57
|
+
|
|
58
|
+
with patch("phlo.config.network.socket.gethostbyname", lambda _host: "127.0.0.1"):
|
|
59
|
+
settings = PostgresSettings(
|
|
60
|
+
postgres_host="db.internal",
|
|
61
|
+
postgres_port=6543,
|
|
62
|
+
postgres_user="svc",
|
|
63
|
+
postgres_password="p@ss/word",
|
|
64
|
+
postgres_db="lake",
|
|
65
|
+
)
|
|
50
66
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
67
|
+
assert settings.get_postgres_connection_string() == (
|
|
68
|
+
"postgresql://svc:p%40ss%2Fword@db.internal:6543/lake"
|
|
69
|
+
)
|
|
70
|
+
assert settings.get_postgres_connection_string(include_db=False) == (
|
|
71
|
+
"postgresql://svc:p%40ss%2Fword@db.internal:6543"
|
|
55
72
|
)
|
|
56
|
-
|
|
57
|
-
assert "postgresql://" in conn_str
|
|
58
|
-
assert str(settings.postgres_port) in conn_str
|
|
59
73
|
|
|
60
74
|
|
|
61
75
|
# =============================================================================
|
|
@@ -90,24 +104,19 @@ class TestPostgresServicePlugin:
|
|
|
90
104
|
plugin = PostgresServicePlugin()
|
|
91
105
|
service_def = plugin.service_definition
|
|
92
106
|
|
|
93
|
-
assert
|
|
94
|
-
|
|
95
|
-
assert "
|
|
107
|
+
assert service_def["name"] == "postgres"
|
|
108
|
+
assert service_def["category"] == "core"
|
|
109
|
+
assert service_def["compose"]["healthcheck"]["test"][0] == "CMD-SHELL"
|
|
96
110
|
|
|
97
111
|
def test_service_definition_has_image(self):
|
|
98
|
-
"""Test service definition
|
|
112
|
+
"""Test service definition pins the upstream postgres image."""
|
|
99
113
|
from phlo_postgres.plugin import PostgresServicePlugin
|
|
100
114
|
|
|
101
115
|
plugin = PostgresServicePlugin()
|
|
102
116
|
service_def = plugin.service_definition
|
|
103
117
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
if services:
|
|
107
|
-
# Should have postgres service with image
|
|
108
|
-
for svc_name, svc_config in services.items():
|
|
109
|
-
if "postgres" in svc_name.lower():
|
|
110
|
-
assert "image" in svc_config or "build" in svc_config
|
|
118
|
+
assert service_def["image"].startswith("postgres:18.4-alpine3.24@sha256:")
|
|
119
|
+
assert "build" not in service_def
|
|
111
120
|
|
|
112
121
|
|
|
113
122
|
# =============================================================================
|
|
@@ -119,48 +128,36 @@ class TestPostgresConnectionMocked:
|
|
|
119
128
|
"""Test Postgres connections with mocks."""
|
|
120
129
|
|
|
121
130
|
def test_psycopg2_connect_with_config(self):
|
|
122
|
-
"""
|
|
123
|
-
import
|
|
124
|
-
from phlo_postgres.settings import
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
port=settings.postgres_port,
|
|
135
|
-
database=settings.postgres_db,
|
|
136
|
-
user=settings.postgres_user,
|
|
137
|
-
password=settings.postgres_password,
|
|
131
|
+
"""The settings store connects with the DSN built from Postgres settings."""
|
|
132
|
+
from phlo.capabilities.settings import SettingsScope
|
|
133
|
+
from phlo_postgres.settings import PostgresSettings
|
|
134
|
+
from phlo_postgres.settings_store import PostgresSettingsStore
|
|
135
|
+
|
|
136
|
+
with patch("phlo.config.network.socket.gethostbyname", lambda _host: "127.0.0.1"):
|
|
137
|
+
settings = PostgresSettings(
|
|
138
|
+
postgres_host="db.internal",
|
|
139
|
+
postgres_port=6543,
|
|
140
|
+
postgres_user="svc",
|
|
141
|
+
postgres_password="p@ss/word",
|
|
142
|
+
postgres_db="lake",
|
|
138
143
|
)
|
|
139
144
|
|
|
140
|
-
mock_connect.assert_called_once()
|
|
141
|
-
# Verify kwargs
|
|
142
|
-
call_kwargs = mock_connect.call_args.kwargs
|
|
143
|
-
assert call_kwargs["host"] == settings.postgres_host
|
|
144
|
-
assert call_kwargs["database"] == settings.postgres_db
|
|
145
|
-
|
|
146
|
-
def test_connection_with_cursor_context(self):
|
|
147
|
-
"""Test connection with cursor context manager."""
|
|
148
|
-
mock_cursor = MagicMock()
|
|
149
|
-
mock_cursor.fetchall.return_value = [(1,)]
|
|
150
|
-
|
|
151
145
|
mock_conn = MagicMock()
|
|
152
|
-
mock_conn.
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
import psycopg2
|
|
146
|
+
mock_conn.__enter__.return_value = mock_conn
|
|
147
|
+
mock_cursor = MagicMock()
|
|
148
|
+
mock_cursor.fetchone.return_value = None
|
|
149
|
+
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
|
157
150
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
151
|
+
with (
|
|
152
|
+
patch("phlo_postgres.settings_store.get_postgres_settings", return_value=settings),
|
|
153
|
+
patch("phlo_postgres.settings_store.ObservatorySettingsStorageConfig") as config,
|
|
154
|
+
patch("psycopg2.connect", return_value=mock_conn) as mock_connect,
|
|
155
|
+
):
|
|
156
|
+
config.return_value.observatory_settings_db_url = None
|
|
157
|
+
store = PostgresSettingsStore()
|
|
158
|
+
assert store.get(SettingsScope.GLOBAL, "demo") is None
|
|
162
159
|
|
|
163
|
-
|
|
160
|
+
mock_connect.assert_called_once_with("postgresql://svc:p%40ss%2Fword@db.internal:6543/lake")
|
|
164
161
|
|
|
165
162
|
|
|
166
163
|
# =============================================================================
|
|
@@ -185,12 +182,18 @@ def postgres_connection():
|
|
|
185
182
|
password=settings.postgres_password,
|
|
186
183
|
connect_timeout=5,
|
|
187
184
|
)
|
|
185
|
+
except psycopg2.OperationalError as e:
|
|
186
|
+
if e.pgcode is not None:
|
|
187
|
+
raise
|
|
188
|
+
pytest.skip(f"Postgres not available: {e}")
|
|
189
|
+
|
|
190
|
+
try:
|
|
188
191
|
yield conn
|
|
192
|
+
finally:
|
|
189
193
|
conn.close()
|
|
190
|
-
except Exception as e:
|
|
191
|
-
pytest.skip(f"Postgres not available: {e}")
|
|
192
194
|
|
|
193
195
|
|
|
196
|
+
@pytest.mark.integration
|
|
194
197
|
class TestPostgresIntegrationReal:
|
|
195
198
|
"""Real integration tests against a running Postgres instance."""
|
|
196
199
|
|
|
@@ -262,7 +265,7 @@ class TestPostgresIntegrationReal:
|
|
|
262
265
|
# Rollback
|
|
263
266
|
postgres_connection.rollback()
|
|
264
267
|
|
|
265
|
-
#
|
|
268
|
+
# The temporary table must be gone after rollback
|
|
266
269
|
with postgres_connection.cursor() as cur:
|
|
267
270
|
cur.execute("""
|
|
268
271
|
SELECT COUNT(*) FROM information_schema.tables
|
|
@@ -270,8 +273,7 @@ class TestPostgresIntegrationReal:
|
|
|
270
273
|
""")
|
|
271
274
|
result = cur.fetchone()
|
|
272
275
|
|
|
273
|
-
|
|
274
|
-
assert result is not None
|
|
276
|
+
assert result[0] == 0
|
|
275
277
|
|
|
276
278
|
|
|
277
279
|
# =============================================================================
|
|
@@ -287,9 +289,3 @@ class TestPostgresExports:
|
|
|
287
289
|
from phlo_postgres.plugin import PostgresServicePlugin
|
|
288
290
|
|
|
289
291
|
assert PostgresServicePlugin is not None
|
|
290
|
-
|
|
291
|
-
def test_module_importable(self):
|
|
292
|
-
"""Test phlo_postgres module is importable."""
|
|
293
|
-
import phlo_postgres
|
|
294
|
-
|
|
295
|
-
assert phlo_postgres is not None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{phlo_postgres-0.16.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|