phlo-postgres 0.17.0__tar.gz → 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/PKG-INFO +2 -2
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/pyproject.toml +2 -2
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/dataset_state_store.py +5 -5
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/operation_controls.py +73 -16
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/PKG-INFO +2 -2
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/requires.txt +1 -1
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_authorization.py +3 -42
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_checkpoints.py +25 -1
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_dataset_state_store.py +91 -6
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_integration_postgres.py +70 -77
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/README.md +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/setup.cfg +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/__init__.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/authorization.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/checkpoints.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/cli.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/cli_plugin.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/continuity.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/exporter_service.yaml +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/governance.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/plugin.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/publish_target.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/resource.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/security_readiness.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/service.yaml +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/settings.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/settings_store.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres/volume_setup.yaml +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/SOURCES.txt +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/dependency_links.txt +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/entry_points.txt +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/top_level.txt +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_continuity.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_postgres_cli.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_postgres_plugin.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_resource.py +0 -0
- {phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/tests/test_settings_store.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phlo-postgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: Postgres service plugin for Phlo
|
|
5
5
|
Author-email: Phlo Team <team@phlo.dev>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.12
|
|
8
8
|
Description-Content-Type: text/plain
|
|
9
|
-
Requires-Dist: phlo<0.
|
|
9
|
+
Requires-Dist: phlo<0.20,>=0.19.0
|
|
10
10
|
Requires-Dist: psycopg2-binary>=2.9.11
|
|
11
11
|
Provides-Extra: dev
|
|
12
12
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
@@ -7,13 +7,13 @@ requires = [
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
dependencies = [
|
|
10
|
-
"phlo>=0.
|
|
10
|
+
"phlo>=0.19.0,<0.20",
|
|
11
11
|
"psycopg2-binary>=2.9.11",
|
|
12
12
|
]
|
|
13
13
|
description = "Postgres service plugin for Phlo"
|
|
14
14
|
name = "phlo-postgres"
|
|
15
15
|
requires-python = ">=3.12"
|
|
16
|
-
version = "0.
|
|
16
|
+
version = "0.18.0"
|
|
17
17
|
|
|
18
18
|
[[project.authors]]
|
|
19
19
|
email = "team@phlo.dev"
|
|
@@ -93,6 +93,7 @@ class SettingsDatasetStateStore:
|
|
|
93
93
|
def compare_and_set(
|
|
94
94
|
self,
|
|
95
95
|
*,
|
|
96
|
+
resource_id: str,
|
|
96
97
|
writes: tuple[StoreWrite, ...],
|
|
97
98
|
action_id: str,
|
|
98
99
|
action: str,
|
|
@@ -100,8 +101,7 @@ class SettingsDatasetStateStore:
|
|
|
100
101
|
) -> StoreWriteResult:
|
|
101
102
|
def apply(state: dict[str, Any] | None) -> dict[str, Any]:
|
|
102
103
|
current = _validated_state(state)
|
|
103
|
-
|
|
104
|
-
committed = current["actions"].get(last.record_id, {}).get(action_id)
|
|
104
|
+
committed = current["actions"].get(resource_id, {}).get(action_id)
|
|
105
105
|
if committed is not None:
|
|
106
106
|
if committed["fingerprint"] == fingerprint:
|
|
107
107
|
self._result = StoreWriteResult(
|
|
@@ -142,9 +142,9 @@ class SettingsDatasetStateStore:
|
|
|
142
142
|
next_state = _copy_state(current)
|
|
143
143
|
for write in writes:
|
|
144
144
|
next_state["records"][write.record_id] = write.next_record.to_read_model()
|
|
145
|
-
next_state["actions"].setdefault(
|
|
145
|
+
next_state["actions"].setdefault(resource_id, {})[action_id] = {
|
|
146
146
|
"action_id": action_id,
|
|
147
|
-
"resource_id":
|
|
147
|
+
"resource_id": resource_id,
|
|
148
148
|
"action": action,
|
|
149
149
|
"fingerprint": fingerprint,
|
|
150
150
|
"outcome_status": StoreWriteStatus.COMMITTED.value,
|
|
@@ -154,7 +154,7 @@ class SettingsDatasetStateStore:
|
|
|
154
154
|
status=StoreWriteStatus.COMMITTED,
|
|
155
155
|
records=tuple(w.next_record for w in writes),
|
|
156
156
|
committed_fingerprint=fingerprint,
|
|
157
|
-
detail=f"{action!r} committed for {
|
|
157
|
+
detail=f"{action!r} committed for {resource_id}.",
|
|
158
158
|
)
|
|
159
159
|
return next_state
|
|
160
160
|
|
|
@@ -15,7 +15,15 @@ from typing import Any
|
|
|
15
15
|
import psycopg2
|
|
16
16
|
from psycopg2.extensions import cursor
|
|
17
17
|
|
|
18
|
-
from phlo.operations.journal import
|
|
18
|
+
from phlo.operations.journal import (
|
|
19
|
+
OperationJournalEntry,
|
|
20
|
+
OperationJournalState,
|
|
21
|
+
claims_conflict,
|
|
22
|
+
expire_claim,
|
|
23
|
+
maintenance_action_aliases,
|
|
24
|
+
new_claim,
|
|
25
|
+
transition_entry,
|
|
26
|
+
)
|
|
19
27
|
from phlo.plugins.observatory_settings import StorageUnavailableError
|
|
20
28
|
|
|
21
29
|
RATE_BUCKET_CAP = 4096
|
|
@@ -73,22 +81,27 @@ class PostgresOperationControls:
|
|
|
73
81
|
conn = None
|
|
74
82
|
try:
|
|
75
83
|
conn = psycopg2.connect(self._db_url, connect_timeout=5)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
#
|
|
80
|
-
|
|
81
|
-
|
|
84
|
+
if not self._schema_ready:
|
|
85
|
+
# Commit DDL before waiting for the namespace lock. Otherwise
|
|
86
|
+
# a cold replica's table locks can deadlock with a warm writer
|
|
87
|
+
# that already owns the namespace and needs those tables.
|
|
88
|
+
with conn, conn.cursor() as cur:
|
|
89
|
+
cur.execute("SET LOCAL lock_timeout = '5s'")
|
|
90
|
+
cur.execute("SET LOCAL statement_timeout = '10s'")
|
|
91
|
+
# Concurrent CREATE IF NOT EXISTS can race on catalog keys.
|
|
82
92
|
cur.execute(
|
|
83
93
|
"SELECT pg_advisory_xact_lock(hashtext('phlo-api-controls-schema'))"
|
|
84
94
|
)
|
|
85
95
|
cur.execute(_SCHEMA)
|
|
96
|
+
self._schema_ready = True
|
|
97
|
+
with conn, conn.cursor() as cur:
|
|
98
|
+
cur.execute("SET LOCAL lock_timeout = '5s'")
|
|
99
|
+
cur.execute("SET LOCAL statement_timeout = '10s'")
|
|
86
100
|
cur.execute(
|
|
87
101
|
"SELECT pg_advisory_xact_lock(hashtext(%s))",
|
|
88
102
|
(f"phlo-api-controls:{self._namespace}",),
|
|
89
103
|
)
|
|
90
104
|
yield cur
|
|
91
|
-
self._schema_ready = True
|
|
92
105
|
except psycopg2.Error as exc:
|
|
93
106
|
raise StorageUnavailableError("Shared operation controls are unavailable") from exc
|
|
94
107
|
finally:
|
|
@@ -100,6 +113,22 @@ class PostgresOperationControls:
|
|
|
100
113
|
with self._transaction():
|
|
101
114
|
pass
|
|
102
115
|
|
|
116
|
+
@contextmanager
|
|
117
|
+
def reclamation_guard(self, identities: list[tuple[str, str]]) -> Iterator[None]:
|
|
118
|
+
"""Hold the claim transaction lock while disposing only quiescent local payloads."""
|
|
119
|
+
with self._transaction() as cur:
|
|
120
|
+
for operation, target in identities:
|
|
121
|
+
cur.execute(
|
|
122
|
+
"SELECT 1 FROM phlo_api_operations WHERE namespace=%s AND operation=%s "
|
|
123
|
+
"AND (target=%s OR exclusion_target=%s) AND state IN ('pending','unknown') LIMIT 1",
|
|
124
|
+
(self._namespace, operation, target, target),
|
|
125
|
+
)
|
|
126
|
+
if cur.fetchone() is not None:
|
|
127
|
+
raise OperationExclusionConflict(
|
|
128
|
+
"An unresolved mutation requires retained inputs"
|
|
129
|
+
)
|
|
130
|
+
yield
|
|
131
|
+
|
|
103
132
|
def claim_idempotency(
|
|
104
133
|
self,
|
|
105
134
|
key_hash: str,
|
|
@@ -261,14 +290,37 @@ class PostgresOperationControls:
|
|
|
261
290
|
|
|
262
291
|
def claim(self, entry: OperationJournalEntry) -> bool:
|
|
263
292
|
"""Claim a journal operation and its active target across replicas."""
|
|
293
|
+
entry = new_claim(entry)
|
|
264
294
|
with self._transaction() as cur:
|
|
265
295
|
cur.execute(
|
|
266
|
-
"SELECT
|
|
267
|
-
"(operation_id=%s OR (action
|
|
268
|
-
(
|
|
296
|
+
"SELECT entry_json FROM phlo_api_operation_journal WHERE namespace=%s AND "
|
|
297
|
+
"(operation_id=%s OR (action=ANY(%s) AND target=%s AND state IN ('claimed','submitted','unknown')))",
|
|
298
|
+
(
|
|
299
|
+
self._namespace,
|
|
300
|
+
entry.operation_id,
|
|
301
|
+
list(maintenance_action_aliases(entry.action)),
|
|
302
|
+
entry.target,
|
|
303
|
+
),
|
|
269
304
|
)
|
|
270
|
-
|
|
271
|
-
|
|
305
|
+
for row in cur.fetchall():
|
|
306
|
+
record = json.loads(row[0])
|
|
307
|
+
record["state"] = OperationJournalState(record["state"])
|
|
308
|
+
previous = OperationJournalEntry(**record)
|
|
309
|
+
if previous.operation_id == entry.operation_id:
|
|
310
|
+
return False
|
|
311
|
+
current = expire_claim(previous)
|
|
312
|
+
if current != previous:
|
|
313
|
+
cur.execute(
|
|
314
|
+
"UPDATE phlo_api_operation_journal SET state=%s,entry_json=%s WHERE namespace=%s AND operation_id=%s",
|
|
315
|
+
(
|
|
316
|
+
current.state.value,
|
|
317
|
+
json.dumps(current.to_dict()),
|
|
318
|
+
self._namespace,
|
|
319
|
+
current.operation_id,
|
|
320
|
+
),
|
|
321
|
+
)
|
|
322
|
+
if claims_conflict(current, entry):
|
|
323
|
+
return False
|
|
272
324
|
cur.execute(
|
|
273
325
|
"INSERT INTO phlo_api_operation_journal(namespace,operation_id,action,target,state,entry_json) "
|
|
274
326
|
"VALUES (%s,%s,%s,%s,%s,%s)",
|
|
@@ -283,9 +335,10 @@ class PostgresOperationControls:
|
|
|
283
335
|
)
|
|
284
336
|
return True
|
|
285
337
|
|
|
286
|
-
def
|
|
338
|
+
def compare_and_set(
|
|
287
339
|
self,
|
|
288
340
|
operation_id: str,
|
|
341
|
+
expected: OperationJournalState,
|
|
289
342
|
state: OperationJournalState,
|
|
290
343
|
result: dict[str, Any] | None = None,
|
|
291
344
|
) -> bool:
|
|
@@ -299,10 +352,14 @@ class PostgresOperationControls:
|
|
|
299
352
|
if row is None:
|
|
300
353
|
return False
|
|
301
354
|
record = json.loads(row[0])
|
|
302
|
-
record
|
|
355
|
+
record["state"] = OperationJournalState(record["state"])
|
|
356
|
+
entry = OperationJournalEntry(**record)
|
|
357
|
+
if entry.state is not expected:
|
|
358
|
+
return False
|
|
359
|
+
updated = transition_entry(entry, state, result)
|
|
303
360
|
cur.execute(
|
|
304
361
|
"UPDATE phlo_api_operation_journal SET state=%s,entry_json=%s WHERE namespace=%s AND operation_id=%s",
|
|
305
|
-
(state.value, json.dumps(
|
|
362
|
+
(state.value, json.dumps(updated.to_dict()), self._namespace, operation_id),
|
|
306
363
|
)
|
|
307
364
|
return True
|
|
308
365
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: phlo-postgres
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: Postgres service plugin for Phlo
|
|
5
5
|
Author-email: Phlo Team <team@phlo.dev>
|
|
6
6
|
License: MIT
|
|
7
7
|
Requires-Python: >=3.12
|
|
8
8
|
Description-Content-Type: text/plain
|
|
9
|
-
Requires-Dist: phlo<0.
|
|
9
|
+
Requires-Dist: phlo<0.20,>=0.19.0
|
|
10
10
|
Requires-Dist: psycopg2-binary>=2.9.11
|
|
11
11
|
Provides-Extra: dev
|
|
12
12
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"""Tests for the PostgreSQL CLI authorization surface.
|
|
2
2
|
|
|
3
|
-
Package-specific
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
fallbacks delegate to ``phlo_testing.authorization_surface``.
|
|
3
|
+
Package-specific command classification and mapping; the shared adapter
|
|
4
|
+
contract (read allow-through, unknown deny-closed, mutation policy allow/deny)
|
|
5
|
+
delegates to ``phlo_testing.authorization_surface``.
|
|
7
6
|
"""
|
|
8
7
|
|
|
9
8
|
from __future__ import annotations
|
|
@@ -12,56 +11,18 @@ from phlo_testing.authorization_surface import (
|
|
|
12
11
|
assert_mutation_enforcement_allows_and_denies,
|
|
13
12
|
assert_read_commands_allow_without_enforcement,
|
|
14
13
|
assert_unknown_command_denied,
|
|
15
|
-
run_cli_principal_resolver_contract,
|
|
16
14
|
)
|
|
17
15
|
from phlo_postgres.authorization import (
|
|
18
16
|
COMMAND_ACTION_MAP,
|
|
19
17
|
COMMAND_RESOURCE_MAP,
|
|
20
18
|
MUTATION_COMMANDS,
|
|
21
|
-
SURFACE_NAME,
|
|
22
|
-
FRAMEWORK_TYPE,
|
|
23
19
|
PostgresCliSurfaceAdapter,
|
|
24
20
|
)
|
|
25
21
|
|
|
26
22
|
|
|
27
|
-
def test_cli_principal_resolver_environment_fallbacks() -> None:
|
|
28
|
-
"""Shared CliPrincipalResolver environment fallback contract."""
|
|
29
|
-
run_cli_principal_resolver_contract()
|
|
30
|
-
|
|
31
|
-
|
|
32
23
|
class TestPostgresCliSurfaceAdapter:
|
|
33
24
|
"""Tests for PostgreSQL CLI surface adapter."""
|
|
34
25
|
|
|
35
|
-
def test_singleton_instance(self) -> None:
|
|
36
|
-
"""Should return singleton instance."""
|
|
37
|
-
adapter1 = PostgresCliSurfaceAdapter.get_instance()
|
|
38
|
-
adapter2 = PostgresCliSurfaceAdapter.get_instance()
|
|
39
|
-
assert adapter1 is adapter2
|
|
40
|
-
|
|
41
|
-
def test_surface_name(self) -> None:
|
|
42
|
-
"""Should return correct surface name."""
|
|
43
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
44
|
-
assert adapter.surface_name == SURFACE_NAME
|
|
45
|
-
|
|
46
|
-
def test_framework_type(self) -> None:
|
|
47
|
-
"""Should return correct framework type."""
|
|
48
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
49
|
-
assert adapter.framework_type == FRAMEWORK_TYPE
|
|
50
|
-
|
|
51
|
-
def test_list_operations(self) -> None:
|
|
52
|
-
"""Should declare mutation operations."""
|
|
53
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
54
|
-
operations = adapter.list_operations()
|
|
55
|
-
assert len(operations) == len(MUTATION_COMMANDS)
|
|
56
|
-
for op in operations:
|
|
57
|
-
assert op["action"] in COMMAND_ACTION_MAP.values()
|
|
58
|
-
assert op["resource_type"] in COMMAND_RESOURCE_MAP.values()
|
|
59
|
-
|
|
60
|
-
def test_is_active(self) -> None:
|
|
61
|
-
"""Should always return True."""
|
|
62
|
-
adapter = PostgresCliSurfaceAdapter()
|
|
63
|
-
assert adapter.is_active(None) is True
|
|
64
|
-
|
|
65
26
|
def test_read_commands_return_allow(self) -> None:
|
|
66
27
|
"""Read commands should return allow without enforcement."""
|
|
67
28
|
assert_read_commands_allow_without_enforcement(PostgresCliSurfaceAdapter())
|
|
@@ -166,8 +166,24 @@ def test_record_snapshot_and_commit_follow_the_lifecycle() -> None:
|
|
|
166
166
|
staged = store.record_snapshot(checkpoint_id="cp-1", snapshot_id=42, release_id="release-1")
|
|
167
167
|
assert staged.status == "staged"
|
|
168
168
|
assert staged.snapshot_id == 42
|
|
169
|
+
stage_sql = " ".join(resource.cursor_obj.statements[0].split())
|
|
170
|
+
assert "SET status = 'staged', snapshot_id = %s, release_id = %s" in stage_sql
|
|
171
|
+
assert "failure_reason = NULL" in stage_sql
|
|
172
|
+
assert resource.cursor_obj.params[0] == (
|
|
173
|
+
42,
|
|
174
|
+
"release-1",
|
|
175
|
+
"cp-1",
|
|
176
|
+
("claimed", "staged", "failed"),
|
|
177
|
+
)
|
|
178
|
+
|
|
169
179
|
committed = store.commit(checkpoint_id="cp-1")
|
|
170
180
|
assert committed.status == "committed"
|
|
181
|
+
commit_sql = " ".join(resource.cursor_obj.statements[1].split())
|
|
182
|
+
assert "SET status = 'committed'" in commit_sql
|
|
183
|
+
# A commit may only advance a checkpoint that already has an audited snapshot.
|
|
184
|
+
assert "status = 'staged'" in commit_sql
|
|
185
|
+
assert "snapshot_id IS NOT NULL" in commit_sql
|
|
186
|
+
assert resource.cursor_obj.params[1] == ("cp-1",)
|
|
171
187
|
|
|
172
188
|
|
|
173
189
|
def test_commit_without_snapshot_raises_and_refuses_to_advance() -> None:
|
|
@@ -206,6 +222,10 @@ def test_fail_records_reason_but_keeps_ranges() -> None:
|
|
|
206
222
|
assert record.status == "failed"
|
|
207
223
|
assert record.failure_reason == reason
|
|
208
224
|
assert record.ranges, "failed checkpoints retain their claimed ranges"
|
|
225
|
+
update_sql = " ".join(resource.cursor_obj.statements[-1].split())
|
|
226
|
+
assert "SET status = 'failed', failure_reason = %s" in update_sql
|
|
227
|
+
assert "status IN %s" in update_sql
|
|
228
|
+
assert resource.cursor_obj.params[-1] == (reason, "cp-1", ("claimed", "staged", "failed"))
|
|
209
229
|
|
|
210
230
|
|
|
211
231
|
def test_provider_registers_checkpoint_resource() -> None:
|
|
@@ -216,6 +236,10 @@ def test_provider_registers_checkpoint_resource() -> None:
|
|
|
216
236
|
|
|
217
237
|
|
|
218
238
|
def test_real_resource_is_used_when_not_injected() -> None:
|
|
239
|
+
from phlo_postgres.resource import PostgresResource
|
|
240
|
+
|
|
219
241
|
store = PostgresIngestionCheckpointStore()
|
|
220
242
|
assert store._resource is None
|
|
221
|
-
|
|
243
|
+
resource = store._ensure_resource()
|
|
244
|
+
assert isinstance(resource, PostgresResource)
|
|
245
|
+
assert store._ensure_resource() is resource
|
|
@@ -19,7 +19,12 @@ import pytest
|
|
|
19
19
|
|
|
20
20
|
from phlo.dataset import (
|
|
21
21
|
CandidateRecord,
|
|
22
|
+
DatasetPolicy,
|
|
22
23
|
DatasetRecord,
|
|
24
|
+
DatasetService,
|
|
25
|
+
StaticPolicySource,
|
|
26
|
+
TransitionRequest,
|
|
27
|
+
TransitionStatus,
|
|
23
28
|
state_store_namespace,
|
|
24
29
|
)
|
|
25
30
|
from phlo.dataset.models import PublicationState, WorkflowState
|
|
@@ -33,6 +38,7 @@ from phlo_postgres.dataset_state_store import (
|
|
|
33
38
|
from phlo_postgres.plugin import PostgresResourceProvider
|
|
34
39
|
|
|
35
40
|
from _dataset_test_backends import FileLockSettingsStore
|
|
41
|
+
from _dataset_worker import NoEvidence
|
|
36
42
|
|
|
37
43
|
WORKER = Path(__file__).parent / "_dataset_worker.py"
|
|
38
44
|
|
|
@@ -71,22 +77,88 @@ def test_resource_provider_exposes_dataset_state_store() -> None:
|
|
|
71
77
|
# ---------------------------------------------------------------------------
|
|
72
78
|
|
|
73
79
|
|
|
80
|
+
@pytest.mark.parametrize("expected_states", [False, True])
|
|
81
|
+
def test_service_receipts_replay_after_restart(tmp_path: Path, expected_states: bool) -> None:
|
|
82
|
+
from dataclasses import replace
|
|
83
|
+
|
|
84
|
+
path = tmp_path / "state.json"
|
|
85
|
+
store = make_store(path)
|
|
86
|
+
service = DatasetService(
|
|
87
|
+
store=store,
|
|
88
|
+
evidence_source=NoEvidence(),
|
|
89
|
+
policy_source=StaticPolicySource(DatasetPolicy(policy_version="test-policy")),
|
|
90
|
+
)
|
|
91
|
+
requests = [
|
|
92
|
+
TransitionRequest(
|
|
93
|
+
resource_id=resource_id,
|
|
94
|
+
action=action,
|
|
95
|
+
action_id=action,
|
|
96
|
+
expected_state=state if expected_states else None,
|
|
97
|
+
)
|
|
98
|
+
for resource_id, action, state in (
|
|
99
|
+
("candidate:gold.demo", "claim", "open"),
|
|
100
|
+
("candidate:gold.demo", "review", "claimed"),
|
|
101
|
+
("candidate:gold.demo", "promote", "review"),
|
|
102
|
+
("gold.demo", "publish", "draft"),
|
|
103
|
+
("gold.demo", "retire", "published"),
|
|
104
|
+
)
|
|
105
|
+
]
|
|
106
|
+
for request in requests:
|
|
107
|
+
first = service.transition(request)
|
|
108
|
+
assert first.status is TransitionStatus.COMMITTED
|
|
109
|
+
service = DatasetService(
|
|
110
|
+
store=make_store(path),
|
|
111
|
+
evidence_source=NoEvidence(),
|
|
112
|
+
policy_source=StaticPolicySource(DatasetPolicy(policy_version="changed-policy")),
|
|
113
|
+
)
|
|
114
|
+
replay = service.transition(request)
|
|
115
|
+
assert replay.status is TransitionStatus.REPLAYED
|
|
116
|
+
assert replay.after_state == first.after_state
|
|
117
|
+
assert replay.record == first.record
|
|
118
|
+
conflict = service.transition(replace(request, expected_state="different-state"))
|
|
119
|
+
assert conflict.status is TransitionStatus.CONFLICT
|
|
120
|
+
assert "different request" in conflict.message
|
|
121
|
+
assert service.record(first.record.dataset_id) == first.record
|
|
122
|
+
|
|
123
|
+
assert store.committed_action("candidate:gold.demo", "promote") is not None
|
|
124
|
+
assert store.committed_action("gold.demo", "promote") is None
|
|
125
|
+
assert service.record("candidate:gold.demo").current_state == "promoted"
|
|
126
|
+
assert service.record("gold.demo").current_state == "retired"
|
|
127
|
+
assert [event.outcome for event in store.audit_events()] == [
|
|
128
|
+
"committed",
|
|
129
|
+
"replayed",
|
|
130
|
+
"conflict",
|
|
131
|
+
] * 5
|
|
132
|
+
|
|
133
|
+
|
|
74
134
|
def test_compare_and_set_commits_and_replays(tmp_path: Path) -> None:
|
|
75
135
|
store = make_store(tmp_path / "state.json")
|
|
76
136
|
write = StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record())
|
|
77
137
|
first = store.compare_and_set(
|
|
78
|
-
|
|
138
|
+
resource_id="gold.demo",
|
|
139
|
+
writes=(write,),
|
|
140
|
+
action_id="a1",
|
|
141
|
+
action="promote",
|
|
142
|
+
fingerprint="fp-1",
|
|
79
143
|
)
|
|
80
144
|
assert first.status is StoreWriteStatus.COMMITTED
|
|
81
145
|
|
|
82
146
|
replay = store.compare_and_set(
|
|
83
|
-
|
|
147
|
+
resource_id="gold.demo",
|
|
148
|
+
writes=(write,),
|
|
149
|
+
action_id="a1",
|
|
150
|
+
action="promote",
|
|
151
|
+
fingerprint="fp-1",
|
|
84
152
|
)
|
|
85
153
|
assert replay.status is StoreWriteStatus.REPLAYED
|
|
86
154
|
assert replay.committed_fingerprint == "fp-1"
|
|
87
155
|
|
|
88
156
|
conflict = store.compare_and_set(
|
|
89
|
-
|
|
157
|
+
resource_id="gold.demo",
|
|
158
|
+
writes=(write,),
|
|
159
|
+
action_id="a1",
|
|
160
|
+
action="promote",
|
|
161
|
+
fingerprint="fp-2",
|
|
90
162
|
)
|
|
91
163
|
assert conflict.status is StoreWriteStatus.ACTION_CONFLICT
|
|
92
164
|
|
|
@@ -97,7 +169,9 @@ def test_compare_and_set_commits_and_replays(tmp_path: Path) -> None:
|
|
|
97
169
|
def test_compare_and_set_fails_on_precondition_mismatch(tmp_path: Path) -> None:
|
|
98
170
|
store = make_store(tmp_path / "state.json")
|
|
99
171
|
stale = StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record())
|
|
100
|
-
store.compare_and_set(
|
|
172
|
+
store.compare_and_set(
|
|
173
|
+
resource_id="gold.demo", writes=(stale,), action_id="a1", action="promote", fingerprint="fp"
|
|
174
|
+
)
|
|
101
175
|
moved = StoreWrite(
|
|
102
176
|
record_id="gold.demo",
|
|
103
177
|
expected_state="open",
|
|
@@ -106,7 +180,11 @@ def test_compare_and_set_fails_on_precondition_mismatch(tmp_path: Path) -> None:
|
|
|
106
180
|
),
|
|
107
181
|
)
|
|
108
182
|
result = store.compare_and_set(
|
|
109
|
-
|
|
183
|
+
resource_id="gold.demo",
|
|
184
|
+
writes=(moved,),
|
|
185
|
+
action_id="a2",
|
|
186
|
+
action="publish",
|
|
187
|
+
fingerprint="fp2",
|
|
110
188
|
)
|
|
111
189
|
assert result.status is StoreWriteStatus.PRECONDITION_FAILED
|
|
112
190
|
|
|
@@ -117,6 +195,7 @@ def test_promote_writes_both_records_atomically(tmp_path: Path) -> None:
|
|
|
117
195
|
# Seed the candidate in review, the promote pre-state.
|
|
118
196
|
seed = CandidateRecord(dataset_id=f"candidate:{table_id}", table_id=table_id, state="review")
|
|
119
197
|
store.compare_and_set(
|
|
198
|
+
resource_id=f"candidate:{table_id}",
|
|
120
199
|
writes=(
|
|
121
200
|
StoreWrite(record_id=f"candidate:{table_id}", expected_state="open", next_record=seed),
|
|
122
201
|
),
|
|
@@ -145,7 +224,11 @@ def test_promote_writes_both_records_atomically(tmp_path: Path) -> None:
|
|
|
145
224
|
),
|
|
146
225
|
)
|
|
147
226
|
result = store.compare_and_set(
|
|
148
|
-
|
|
227
|
+
resource_id=f"candidate:{table_id}",
|
|
228
|
+
writes=writes,
|
|
229
|
+
action_id="p1",
|
|
230
|
+
action="promote",
|
|
231
|
+
fingerprint="fp",
|
|
149
232
|
)
|
|
150
233
|
assert result.status is StoreWriteStatus.COMMITTED
|
|
151
234
|
assert len(result.records) == 2
|
|
@@ -251,6 +334,7 @@ def test_state_survives_restart(tmp_path: Path) -> None:
|
|
|
251
334
|
path = tmp_path / "state.json"
|
|
252
335
|
store = make_store(path)
|
|
253
336
|
store.compare_and_set(
|
|
337
|
+
resource_id="gold.demo",
|
|
254
338
|
writes=(
|
|
255
339
|
StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record()),
|
|
256
340
|
),
|
|
@@ -300,6 +384,7 @@ def test_two_processes_race_one_publication_transition(tmp_path: Path) -> None:
|
|
|
300
384
|
# Seed the draft dataset the way a first worker's promote would.
|
|
301
385
|
seed = make_store(state_file, project_root)
|
|
302
386
|
seed.compare_and_set(
|
|
387
|
+
resource_id="gold.demo",
|
|
303
388
|
writes=(
|
|
304
389
|
StoreWrite(record_id="gold.demo", expected_state="open", next_record=draft_record()),
|
|
305
390
|
),
|
|
@@ -29,30 +29,47 @@ class TestPostgresConfiguration:
|
|
|
29
29
|
assert hasattr(settings, "postgres_db")
|
|
30
30
|
assert hasattr(settings, "postgres_user")
|
|
31
31
|
|
|
32
|
-
def test_postgres_config_has_defaults(self):
|
|
33
|
-
"""Test Postgres config has
|
|
34
|
-
from phlo_postgres.settings import
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
32
|
+
def test_postgres_config_has_defaults(self, monkeypatch, tmp_path):
|
|
33
|
+
"""Test Postgres config has concrete, non-empty defaults."""
|
|
34
|
+
from phlo_postgres.settings import PostgresSettings
|
|
35
|
+
|
|
36
|
+
for name in (
|
|
37
|
+
"POSTGRES_HOST",
|
|
38
|
+
"POSTGRES_PORT",
|
|
39
|
+
"POSTGRES_USER",
|
|
40
|
+
"POSTGRES_DB",
|
|
41
|
+
"POSTGRES_MART_SCHEMA",
|
|
42
|
+
):
|
|
43
|
+
monkeypatch.delenv(name, raising=False)
|
|
44
|
+
with patch("phlo.config.network.socket.gethostbyname", lambda _host: "127.0.0.1"):
|
|
45
|
+
# Isolate from this workspace's .phlo/.env, which may set POSTGRES_PORT.
|
|
46
|
+
settings = PostgresSettings(_project_root=tmp_path)
|
|
47
|
+
|
|
48
|
+
assert settings.postgres_host == "postgres"
|
|
49
|
+
assert settings.postgres_port == 5432
|
|
50
|
+
assert settings.postgres_user == "phlo"
|
|
51
|
+
assert settings.postgres_db == "phlo"
|
|
52
|
+
assert settings.postgres_mart_schema == "marts"
|
|
41
53
|
|
|
42
54
|
def test_connection_string_building(self):
|
|
43
|
-
"""Test
|
|
44
|
-
from phlo_postgres.settings import
|
|
45
|
-
|
|
46
|
-
|
|
55
|
+
"""Test the production connection string encodes credentials correctly."""
|
|
56
|
+
from phlo_postgres.settings import PostgresSettings
|
|
57
|
+
|
|
58
|
+
with patch("phlo.config.network.socket.gethostbyname", lambda _host: "127.0.0.1"):
|
|
59
|
+
settings = PostgresSettings(
|
|
60
|
+
postgres_host="db.internal",
|
|
61
|
+
postgres_port=6543,
|
|
62
|
+
postgres_user="svc",
|
|
63
|
+
postgres_password="p@ss/word",
|
|
64
|
+
postgres_db="lake",
|
|
65
|
+
)
|
|
47
66
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
67
|
+
assert settings.get_postgres_connection_string() == (
|
|
68
|
+
"postgresql://svc:p%40ss%2Fword@db.internal:6543/lake"
|
|
69
|
+
)
|
|
70
|
+
assert settings.get_postgres_connection_string(include_db=False) == (
|
|
71
|
+
"postgresql://svc:p%40ss%2Fword@db.internal:6543"
|
|
52
72
|
)
|
|
53
|
-
|
|
54
|
-
assert "postgresql://" in conn_str
|
|
55
|
-
assert str(settings.postgres_port) in conn_str
|
|
56
73
|
|
|
57
74
|
|
|
58
75
|
# =============================================================================
|
|
@@ -87,24 +104,19 @@ class TestPostgresServicePlugin:
|
|
|
87
104
|
plugin = PostgresServicePlugin()
|
|
88
105
|
service_def = plugin.service_definition
|
|
89
106
|
|
|
90
|
-
assert
|
|
91
|
-
|
|
92
|
-
assert "
|
|
107
|
+
assert service_def["name"] == "postgres"
|
|
108
|
+
assert service_def["category"] == "core"
|
|
109
|
+
assert service_def["compose"]["healthcheck"]["test"][0] == "CMD-SHELL"
|
|
93
110
|
|
|
94
111
|
def test_service_definition_has_image(self):
|
|
95
|
-
"""Test service definition
|
|
112
|
+
"""Test service definition pins the upstream postgres image."""
|
|
96
113
|
from phlo_postgres.plugin import PostgresServicePlugin
|
|
97
114
|
|
|
98
115
|
plugin = PostgresServicePlugin()
|
|
99
116
|
service_def = plugin.service_definition
|
|
100
117
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if services:
|
|
104
|
-
# Should have postgres service with image
|
|
105
|
-
for svc_name, svc_config in services.items():
|
|
106
|
-
if "postgres" in svc_name.lower():
|
|
107
|
-
assert "image" in svc_config or "build" in svc_config
|
|
118
|
+
assert service_def["image"].startswith("postgres:18.4-alpine3.24@sha256:")
|
|
119
|
+
assert "build" not in service_def
|
|
108
120
|
|
|
109
121
|
|
|
110
122
|
# =============================================================================
|
|
@@ -116,48 +128,36 @@ class TestPostgresConnectionMocked:
|
|
|
116
128
|
"""Test Postgres connections with mocks."""
|
|
117
129
|
|
|
118
130
|
def test_psycopg2_connect_with_config(self):
|
|
119
|
-
"""
|
|
120
|
-
import
|
|
121
|
-
from phlo_postgres.settings import
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
port=settings.postgres_port,
|
|
132
|
-
database=settings.postgres_db,
|
|
133
|
-
user=settings.postgres_user,
|
|
134
|
-
password=settings.postgres_password,
|
|
131
|
+
"""The settings store connects with the DSN built from Postgres settings."""
|
|
132
|
+
from phlo.capabilities.settings import SettingsScope
|
|
133
|
+
from phlo_postgres.settings import PostgresSettings
|
|
134
|
+
from phlo_postgres.settings_store import PostgresSettingsStore
|
|
135
|
+
|
|
136
|
+
with patch("phlo.config.network.socket.gethostbyname", lambda _host: "127.0.0.1"):
|
|
137
|
+
settings = PostgresSettings(
|
|
138
|
+
postgres_host="db.internal",
|
|
139
|
+
postgres_port=6543,
|
|
140
|
+
postgres_user="svc",
|
|
141
|
+
postgres_password="p@ss/word",
|
|
142
|
+
postgres_db="lake",
|
|
135
143
|
)
|
|
136
144
|
|
|
137
|
-
mock_connect.assert_called_once()
|
|
138
|
-
# Verify kwargs
|
|
139
|
-
call_kwargs = mock_connect.call_args.kwargs
|
|
140
|
-
assert call_kwargs["host"] == settings.postgres_host
|
|
141
|
-
assert call_kwargs["database"] == settings.postgres_db
|
|
142
|
-
|
|
143
|
-
def test_connection_with_cursor_context(self):
|
|
144
|
-
"""Test connection with cursor context manager."""
|
|
145
|
-
mock_cursor = MagicMock()
|
|
146
|
-
mock_cursor.fetchall.return_value = [(1,)]
|
|
147
|
-
|
|
148
145
|
mock_conn = MagicMock()
|
|
149
|
-
mock_conn.
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
import psycopg2
|
|
146
|
+
mock_conn.__enter__.return_value = mock_conn
|
|
147
|
+
mock_cursor = MagicMock()
|
|
148
|
+
mock_cursor.fetchone.return_value = None
|
|
149
|
+
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
|
|
154
150
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
151
|
+
with (
|
|
152
|
+
patch("phlo_postgres.settings_store.get_postgres_settings", return_value=settings),
|
|
153
|
+
patch("phlo_postgres.settings_store.ObservatorySettingsStorageConfig") as config,
|
|
154
|
+
patch("psycopg2.connect", return_value=mock_conn) as mock_connect,
|
|
155
|
+
):
|
|
156
|
+
config.return_value.observatory_settings_db_url = None
|
|
157
|
+
store = PostgresSettingsStore()
|
|
158
|
+
assert store.get(SettingsScope.GLOBAL, "demo") is None
|
|
159
159
|
|
|
160
|
-
|
|
160
|
+
mock_connect.assert_called_once_with("postgresql://svc:p%40ss%2Fword@db.internal:6543/lake")
|
|
161
161
|
|
|
162
162
|
|
|
163
163
|
# =============================================================================
|
|
@@ -265,7 +265,7 @@ class TestPostgresIntegrationReal:
|
|
|
265
265
|
# Rollback
|
|
266
266
|
postgres_connection.rollback()
|
|
267
267
|
|
|
268
|
-
#
|
|
268
|
+
# The temporary table must be gone after rollback
|
|
269
269
|
with postgres_connection.cursor() as cur:
|
|
270
270
|
cur.execute("""
|
|
271
271
|
SELECT COUNT(*) FROM information_schema.tables
|
|
@@ -273,8 +273,7 @@ class TestPostgresIntegrationReal:
|
|
|
273
273
|
""")
|
|
274
274
|
result = cur.fetchone()
|
|
275
275
|
|
|
276
|
-
|
|
277
|
-
assert result is not None
|
|
276
|
+
assert result[0] == 0
|
|
278
277
|
|
|
279
278
|
|
|
280
279
|
# =============================================================================
|
|
@@ -290,9 +289,3 @@ class TestPostgresExports:
|
|
|
290
289
|
from phlo_postgres.plugin import PostgresServicePlugin
|
|
291
290
|
|
|
292
291
|
assert PostgresServicePlugin is not None
|
|
293
|
-
|
|
294
|
-
def test_module_importable(self):
|
|
295
|
-
"""Test phlo_postgres module is importable."""
|
|
296
|
-
import phlo_postgres
|
|
297
|
-
|
|
298
|
-
assert phlo_postgres is not None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{phlo_postgres-0.17.0 → phlo_postgres-0.18.0}/src/phlo_postgres.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|