nl2data-semantic-catalog-postgres 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nl2data_semantic_catalog_postgres/__init__.py +18 -0
- nl2data_semantic_catalog_postgres/client.py +139 -0
- nl2data_semantic_catalog_postgres/config.py +136 -0
- nl2data_semantic_catalog_postgres/envelope.py +327 -0
- nl2data_semantic_catalog_postgres/errors.py +188 -0
- nl2data_semantic_catalog_postgres/fake_postgres/__init__.py +53 -0
- nl2data_semantic_catalog_postgres/fake_postgres/driver.py +209 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_audit.py +243 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_drafts.py +84 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_maintenance.py +263 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_publications.py +249 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_schema.py +23 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_snapshots.py +197 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_versions.py +330 -0
- nl2data_semantic_catalog_postgres/fake_postgres/keys.py +99 -0
- nl2data_semantic_catalog_postgres/fake_postgres/pool.py +152 -0
- nl2data_semantic_catalog_postgres/fake_postgres/registry.py +144 -0
- nl2data_semantic_catalog_postgres/maintenance.py +235 -0
- nl2data_semantic_catalog_postgres/py.typed +0 -0
- nl2data_semantic_catalog_postgres/repositories/__init__.py +26 -0
- nl2data_semantic_catalog_postgres/repositories/activation.py +849 -0
- nl2data_semantic_catalog_postgres/repositories/audit_evidence.py +221 -0
- nl2data_semantic_catalog_postgres/repositories/drafts.py +156 -0
- nl2data_semantic_catalog_postgres/repositories/evidence.py +415 -0
- nl2data_semantic_catalog_postgres/repositories/publications.py +486 -0
- nl2data_semantic_catalog_postgres/repositories/snapshots.py +372 -0
- nl2data_semantic_catalog_postgres/schema.py +337 -0
- nl2data_semantic_catalog_postgres/sql.py +520 -0
- nl2data_semantic_catalog_postgres/store.py +716 -0
- nl2data_semantic_catalog_postgres/unit_of_work.py +589 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/METADATA +76 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/RECORD +34 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/WHEEL +5 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
"""Shared unit-of-work, envelope, and error infrastructure for the catalog.
|
|
2
|
+
|
|
3
|
+
One :class:`CatalogUnitOfWork` owns the pool, the resolved SQL statements,
|
|
4
|
+
the command timeout, envelope encoding/decoding bounds, and backend error
|
|
5
|
+
normalization. Repositories receive the unit of work and operate on a
|
|
6
|
+
connection handed to them by a transaction owner; only the transaction
|
|
7
|
+
owner (the store facade for cross-repository operations) commits or
|
|
8
|
+
rolls back.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import contextlib
|
|
14
|
+
from collections.abc import Callable, Iterator
|
|
15
|
+
from datetime import UTC, datetime
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from nl2data_core.assembly.audit_evidence import (
|
|
19
|
+
AssemblyAuditEvidenceEntry,
|
|
20
|
+
PublicationAuditEvidence,
|
|
21
|
+
)
|
|
22
|
+
from nl2data_core.assembly.manifest import AcceptedAssertionManifest
|
|
23
|
+
from nl2data_core.assembly.models import AssemblyDraft, DraftRevisionConflict
|
|
24
|
+
from nl2data_core.bundles.models import BUNDLE_SCHEMA_VERSION, SemanticModelBundle
|
|
25
|
+
from nl2data_core.bundles.publication import PublishAuditRecord
|
|
26
|
+
from nl2data_core.bundles.validation import validate_bundle
|
|
27
|
+
from nl2data_core.canonical import strict_canonical_json, strict_sha256_fingerprint
|
|
28
|
+
from nl2data_core.control_plane.publication.contracts import FrozenReleaseBinding
|
|
29
|
+
from nl2data_core.metadata.models import MetadataSnapshot
|
|
30
|
+
from nl2data_core.metadata.proposals import SemanticProposalSet
|
|
31
|
+
from nl2data_core.verification.models import VerificationSuiteEvidence
|
|
32
|
+
from nl2data_core.workflow.durable import tenant_scope_namespace
|
|
33
|
+
from pydantic import ValidationError
|
|
34
|
+
|
|
35
|
+
from .client import (
|
|
36
|
+
is_connect_error,
|
|
37
|
+
is_duplicate_key_error,
|
|
38
|
+
is_serialization_error,
|
|
39
|
+
is_timeout_error,
|
|
40
|
+
)
|
|
41
|
+
from .config import SemanticCatalogConfig
|
|
42
|
+
from .envelope import (
|
|
43
|
+
ENVELOPE_SCHEMA_VERSION,
|
|
44
|
+
ArtifactKind,
|
|
45
|
+
CatalogEnvelope,
|
|
46
|
+
EnvelopeRejectedError,
|
|
47
|
+
decode_envelope,
|
|
48
|
+
encode_envelope,
|
|
49
|
+
)
|
|
50
|
+
from .errors import SemanticCatalogError, SemanticCatalogErrorCode
|
|
51
|
+
from .sql import SQL_TEMPLATES
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"ENVELOPE_SCHEMA_VERSION",
|
|
55
|
+
"ArtifactKind",
|
|
56
|
+
"CatalogEnvelope",
|
|
57
|
+
"CatalogUnitOfWork",
|
|
58
|
+
"SemanticCatalogError",
|
|
59
|
+
"SemanticCatalogErrorCode",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _utc_now() -> datetime:
|
|
64
|
+
return datetime.now(UTC)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _parse_dt(value: Any) -> datetime:
|
|
68
|
+
if isinstance(value, datetime):
|
|
69
|
+
return value
|
|
70
|
+
return datetime.fromisoformat(str(value))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _namespace(fingerprint: str | None) -> str:
|
|
74
|
+
"""The scope namespace for a fingerprint, or the local non-tenant namespace."""
|
|
75
|
+
return tenant_scope_namespace(fingerprint) if fingerprint is not None else ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class CatalogUnitOfWork:
|
|
79
|
+
"""Transaction, execution, and envelope infrastructure for repositories.
|
|
80
|
+
|
|
81
|
+
The unit of work never persists anything on its own: repositories issue
|
|
82
|
+
statements through :meth:`execute` against a caller-supplied connection,
|
|
83
|
+
and the transaction owner wraps those calls in :meth:`transaction`.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def __init__(
|
|
87
|
+
self,
|
|
88
|
+
*,
|
|
89
|
+
config: SemanticCatalogConfig,
|
|
90
|
+
pool: Any,
|
|
91
|
+
now: Callable[[], datetime] | None = None,
|
|
92
|
+
) -> None:
|
|
93
|
+
self._config = config
|
|
94
|
+
self._schema = config.namespace
|
|
95
|
+
self._quoted_schema = f'"{self._schema}"'
|
|
96
|
+
self._sql = {
|
|
97
|
+
name: template.format(schema=self._quoted_schema)
|
|
98
|
+
for name, template in SQL_TEMPLATES.items()
|
|
99
|
+
}
|
|
100
|
+
self._pool = pool
|
|
101
|
+
self._now_fn = now or _utc_now
|
|
102
|
+
self._closed = False
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def config(self) -> SemanticCatalogConfig:
|
|
106
|
+
"""The catalog configuration owning bounds and retention settings."""
|
|
107
|
+
return self._config
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def schema(self) -> str:
|
|
111
|
+
"""The deployment schema namespace owning every catalog table."""
|
|
112
|
+
return self._schema
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def closed(self) -> bool:
|
|
116
|
+
"""Whether the unit of work (and its pool) is closed."""
|
|
117
|
+
return self._closed
|
|
118
|
+
|
|
119
|
+
def now(self) -> datetime:
|
|
120
|
+
"""The injected client clock reading."""
|
|
121
|
+
return self._now_fn()
|
|
122
|
+
|
|
123
|
+
def close(self) -> None:
|
|
124
|
+
"""Close the pool (idempotent); later operations fail closed."""
|
|
125
|
+
if self._closed:
|
|
126
|
+
return
|
|
127
|
+
self._closed = True
|
|
128
|
+
close = getattr(self._pool, "close", None)
|
|
129
|
+
if callable(close):
|
|
130
|
+
close()
|
|
131
|
+
|
|
132
|
+
def statement(self, name: str) -> str:
|
|
133
|
+
"""The resolved SQL statement for one stable template name."""
|
|
134
|
+
return self._sql[name]
|
|
135
|
+
|
|
136
|
+
# -- transaction and error mapping ------------------------------------
|
|
137
|
+
|
|
138
|
+
@contextlib.contextmanager
|
|
139
|
+
def transaction(self) -> Iterator[Any]:
|
|
140
|
+
"""One transaction-backed connection; commit on success only."""
|
|
141
|
+
if self._closed:
|
|
142
|
+
raise SemanticCatalogError(
|
|
143
|
+
SemanticCatalogErrorCode.CATALOG_UNAVAILABLE,
|
|
144
|
+
"semantic catalog is closed",
|
|
145
|
+
details={"cause_type": "ClosedStore"},
|
|
146
|
+
)
|
|
147
|
+
try:
|
|
148
|
+
with self._pool.connection() as conn:
|
|
149
|
+
try:
|
|
150
|
+
yield conn
|
|
151
|
+
conn.commit()
|
|
152
|
+
except BaseException:
|
|
153
|
+
with contextlib.suppress(Exception):
|
|
154
|
+
conn.rollback()
|
|
155
|
+
raise
|
|
156
|
+
except (DraftRevisionConflict, SemanticCatalogError):
|
|
157
|
+
raise
|
|
158
|
+
except Exception as error:
|
|
159
|
+
# Connection acquisition (pool timeouts, unreachable backends)
|
|
160
|
+
# is normalized like any other backend failure.
|
|
161
|
+
raise self.map_backend_error(error, operation="connect") from error
|
|
162
|
+
|
|
163
|
+
def execute(
|
|
164
|
+
self,
|
|
165
|
+
conn: Any,
|
|
166
|
+
name: str,
|
|
167
|
+
params: tuple[Any, ...] = (),
|
|
168
|
+
) -> Any:
|
|
169
|
+
"""Run one named statement with the bounded command timeout."""
|
|
170
|
+
try:
|
|
171
|
+
cursor = conn.cursor()
|
|
172
|
+
self.set_command_timeout(conn, cursor)
|
|
173
|
+
return cursor.execute(self._sql[name], params)
|
|
174
|
+
except Exception as error:
|
|
175
|
+
raise self.map_backend_error(error, operation=name) from error
|
|
176
|
+
|
|
177
|
+
def execute_raw(self, conn: Any, statement: str) -> Any:
|
|
178
|
+
"""Run a raw migration statement with the bounded command timeout."""
|
|
179
|
+
try:
|
|
180
|
+
cursor = conn.cursor()
|
|
181
|
+
self.set_command_timeout(conn, cursor)
|
|
182
|
+
return cursor.execute(statement, ())
|
|
183
|
+
except Exception as error:
|
|
184
|
+
raise self.map_backend_error(error, operation="migration") from error
|
|
185
|
+
|
|
186
|
+
def set_command_timeout(self, conn: Any, cursor: Any) -> None:
|
|
187
|
+
"""Apply a command timeout across fake and psycopg cursor APIs."""
|
|
188
|
+
if hasattr(cursor, "timeout"):
|
|
189
|
+
cursor.timeout = self._config.command_timeout_seconds
|
|
190
|
+
return
|
|
191
|
+
timeout_ms = int(self._config.command_timeout_seconds * 1000)
|
|
192
|
+
conn.execute(
|
|
193
|
+
"SELECT set_config('statement_timeout', %s, true)",
|
|
194
|
+
(str(timeout_ms),),
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
def map_backend_error(
|
|
198
|
+
self, error: Exception, *, operation: str
|
|
199
|
+
) -> Exception:
|
|
200
|
+
"""Normalize a driver failure into a safe structured error."""
|
|
201
|
+
if isinstance(error, SemanticCatalogError):
|
|
202
|
+
return error
|
|
203
|
+
if isinstance(error, EnvelopeRejectedError):
|
|
204
|
+
return SemanticCatalogError(
|
|
205
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
206
|
+
"catalog artifact was rejected by safe envelope validation",
|
|
207
|
+
details={
|
|
208
|
+
"operation": operation,
|
|
209
|
+
"reason": error.code,
|
|
210
|
+
"cause_type": type(error).__name__,
|
|
211
|
+
},
|
|
212
|
+
cause=error,
|
|
213
|
+
)
|
|
214
|
+
if is_timeout_error(error):
|
|
215
|
+
return SemanticCatalogError(
|
|
216
|
+
SemanticCatalogErrorCode.CATALOG_TIMEOUT,
|
|
217
|
+
"catalog backend command timed out",
|
|
218
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
219
|
+
cause=error,
|
|
220
|
+
)
|
|
221
|
+
if is_duplicate_key_error(error) or is_serialization_error(error):
|
|
222
|
+
return SemanticCatalogError(
|
|
223
|
+
SemanticCatalogErrorCode.CONFLICT,
|
|
224
|
+
"catalog backend rejected a conflicting record",
|
|
225
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
226
|
+
cause=error,
|
|
227
|
+
)
|
|
228
|
+
if is_connect_error(error):
|
|
229
|
+
return SemanticCatalogError(
|
|
230
|
+
SemanticCatalogErrorCode.CATALOG_UNAVAILABLE,
|
|
231
|
+
"catalog backend is unreachable",
|
|
232
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
233
|
+
cause=error,
|
|
234
|
+
)
|
|
235
|
+
return SemanticCatalogError(
|
|
236
|
+
SemanticCatalogErrorCode.CATALOG_UNAVAILABLE,
|
|
237
|
+
"catalog backend operation failed",
|
|
238
|
+
details={"operation": operation, "cause_type": type(error).__name__},
|
|
239
|
+
cause=error,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
# -- envelope encoding/decoding ---------------------------------------
|
|
243
|
+
|
|
244
|
+
def encode(
|
|
245
|
+
self, kind: ArtifactKind, payload: dict[str, Any], fingerprint: str
|
|
246
|
+
) -> str:
|
|
247
|
+
"""Encode one canonical payload under configured byte bounds."""
|
|
248
|
+
try:
|
|
249
|
+
return encode_envelope(
|
|
250
|
+
kind,
|
|
251
|
+
payload,
|
|
252
|
+
fingerprint,
|
|
253
|
+
max_envelope_bytes=self._config.max_envelope_bytes,
|
|
254
|
+
max_payload_bytes=self._config.max_payload_bytes,
|
|
255
|
+
)
|
|
256
|
+
except EnvelopeRejectedError as error:
|
|
257
|
+
code = {
|
|
258
|
+
"fingerprint_mismatch": SemanticCatalogErrorCode.FINGERPRINT_MISMATCH,
|
|
259
|
+
"oversized": SemanticCatalogErrorCode.BOUNDS_EXCEEDED,
|
|
260
|
+
}.get(error.code, SemanticCatalogErrorCode.ENVELOPE_REJECTED)
|
|
261
|
+
raise SemanticCatalogError(
|
|
262
|
+
code,
|
|
263
|
+
"catalog artifact was rejected before persistence",
|
|
264
|
+
details={"reason": error.code},
|
|
265
|
+
cause=error,
|
|
266
|
+
) from error
|
|
267
|
+
|
|
268
|
+
def decode(
|
|
269
|
+
self,
|
|
270
|
+
text: str,
|
|
271
|
+
kind: ArtifactKind,
|
|
272
|
+
*,
|
|
273
|
+
row_schema_version: Any = None,
|
|
274
|
+
) -> CatalogEnvelope:
|
|
275
|
+
"""Decode and revalidate one persisted envelope, failing closed."""
|
|
276
|
+
try:
|
|
277
|
+
envelope = decode_envelope(
|
|
278
|
+
text,
|
|
279
|
+
expected_kind=kind,
|
|
280
|
+
supported_schema_version=ENVELOPE_SCHEMA_VERSION,
|
|
281
|
+
max_envelope_bytes=self._config.max_envelope_bytes,
|
|
282
|
+
max_payload_bytes=self._config.max_payload_bytes,
|
|
283
|
+
)
|
|
284
|
+
except EnvelopeRejectedError as error:
|
|
285
|
+
code = {
|
|
286
|
+
"newer_schema": SemanticCatalogErrorCode.SCHEMA_MISMATCH,
|
|
287
|
+
"fingerprint_mismatch": SemanticCatalogErrorCode.FINGERPRINT_MISMATCH,
|
|
288
|
+
"oversized": SemanticCatalogErrorCode.BOUNDS_EXCEEDED,
|
|
289
|
+
}.get(error.code, SemanticCatalogErrorCode.ENVELOPE_REJECTED)
|
|
290
|
+
raise SemanticCatalogError(
|
|
291
|
+
code,
|
|
292
|
+
"persisted catalog artifact failed revalidation",
|
|
293
|
+
details={"reason": error.code},
|
|
294
|
+
cause=error,
|
|
295
|
+
) from error
|
|
296
|
+
if row_schema_version is not None:
|
|
297
|
+
try:
|
|
298
|
+
row_version = int(row_schema_version)
|
|
299
|
+
except (TypeError, ValueError) as error:
|
|
300
|
+
raise SemanticCatalogError(
|
|
301
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
302
|
+
"persisted catalog artifact has an invalid schema version",
|
|
303
|
+
details={"cause_type": type(error).__name__},
|
|
304
|
+
cause=error,
|
|
305
|
+
) from error
|
|
306
|
+
if row_version > ENVELOPE_SCHEMA_VERSION:
|
|
307
|
+
raise SemanticCatalogError(
|
|
308
|
+
SemanticCatalogErrorCode.SCHEMA_MISMATCH,
|
|
309
|
+
"persisted catalog artifact schema version is newer than "
|
|
310
|
+
"supported",
|
|
311
|
+
details={"row_schema_version": str(row_version)},
|
|
312
|
+
)
|
|
313
|
+
return envelope
|
|
314
|
+
|
|
315
|
+
# -- model reconstruction ----------------------------------------------
|
|
316
|
+
|
|
317
|
+
def snapshot_from_envelope(
|
|
318
|
+
self,
|
|
319
|
+
envelope: CatalogEnvelope,
|
|
320
|
+
*,
|
|
321
|
+
discovered_at: Any = None,
|
|
322
|
+
) -> MetadataSnapshot:
|
|
323
|
+
"""Reconstruct a snapshot, restoring its persisted discovered time.
|
|
324
|
+
|
|
325
|
+
The canonical envelope payload excludes the environmental
|
|
326
|
+
``discovered_at`` timestamp (fingerprint stability), so the row's
|
|
327
|
+
column is applied after reconstruction - otherwise an activation
|
|
328
|
+
policy's freshness bounds would measure from reconstruction time.
|
|
329
|
+
"""
|
|
330
|
+
try:
|
|
331
|
+
snapshot = MetadataSnapshot(**envelope.payload)
|
|
332
|
+
except ValidationError as error:
|
|
333
|
+
raise SemanticCatalogError(
|
|
334
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
335
|
+
"persisted snapshot failed model reconstruction",
|
|
336
|
+
details={"cause_type": "ValidationError"},
|
|
337
|
+
cause=error,
|
|
338
|
+
) from error
|
|
339
|
+
if snapshot.fingerprint != envelope.fingerprint:
|
|
340
|
+
raise SemanticCatalogError(
|
|
341
|
+
SemanticCatalogErrorCode.FINGERPRINT_MISMATCH,
|
|
342
|
+
"persisted snapshot fingerprint does not match its envelope",
|
|
343
|
+
details={"cause_type": "SnapshotFingerprintMismatch"},
|
|
344
|
+
)
|
|
345
|
+
if discovered_at is not None:
|
|
346
|
+
snapshot = snapshot.model_copy(
|
|
347
|
+
update={
|
|
348
|
+
"freshness": snapshot.freshness.model_copy(
|
|
349
|
+
update={"discovered_at": _parse_dt(discovered_at)}
|
|
350
|
+
)
|
|
351
|
+
}
|
|
352
|
+
)
|
|
353
|
+
return snapshot
|
|
354
|
+
|
|
355
|
+
def proposal_set_from_envelope(
|
|
356
|
+
self, envelope: CatalogEnvelope
|
|
357
|
+
) -> SemanticProposalSet:
|
|
358
|
+
try:
|
|
359
|
+
return SemanticProposalSet(**envelope.payload)
|
|
360
|
+
except ValidationError as error:
|
|
361
|
+
raise SemanticCatalogError(
|
|
362
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
363
|
+
"persisted proposal set failed model reconstruction",
|
|
364
|
+
details={"cause_type": "ValidationError"},
|
|
365
|
+
cause=error,
|
|
366
|
+
) from error
|
|
367
|
+
|
|
368
|
+
def draft_from_envelope(self, envelope: CatalogEnvelope) -> AssemblyDraft:
|
|
369
|
+
try:
|
|
370
|
+
return AssemblyDraft.model_validate(envelope.payload)
|
|
371
|
+
except ValidationError as error:
|
|
372
|
+
raise SemanticCatalogError(
|
|
373
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
374
|
+
"persisted assembly draft failed model reconstruction",
|
|
375
|
+
details={"cause_type": "ValidationError"},
|
|
376
|
+
cause=error,
|
|
377
|
+
) from error
|
|
378
|
+
|
|
379
|
+
def manifest_from_envelope(
|
|
380
|
+
self, envelope: CatalogEnvelope
|
|
381
|
+
) -> AcceptedAssertionManifest:
|
|
382
|
+
try:
|
|
383
|
+
return AcceptedAssertionManifest.model_validate(envelope.payload)
|
|
384
|
+
except ValidationError as error:
|
|
385
|
+
raise SemanticCatalogError(
|
|
386
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
387
|
+
"persisted accepted assertion manifest failed reconstruction",
|
|
388
|
+
details={"cause_type": "ValidationError"},
|
|
389
|
+
cause=error,
|
|
390
|
+
) from error
|
|
391
|
+
|
|
392
|
+
def audit_from_envelope(self, envelope: CatalogEnvelope) -> PublishAuditRecord:
|
|
393
|
+
try:
|
|
394
|
+
return PublishAuditRecord.model_validate(envelope.payload)
|
|
395
|
+
except ValidationError as error:
|
|
396
|
+
raise SemanticCatalogError(
|
|
397
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
398
|
+
"persisted publish audit failed reconstruction",
|
|
399
|
+
details={"cause_type": "ValidationError"},
|
|
400
|
+
cause=error,
|
|
401
|
+
) from error
|
|
402
|
+
|
|
403
|
+
def evidence_from_envelope(
|
|
404
|
+
self, envelope: CatalogEnvelope
|
|
405
|
+
) -> VerificationSuiteEvidence:
|
|
406
|
+
payload = envelope.payload.get("evidence", envelope.payload)
|
|
407
|
+
try:
|
|
408
|
+
return VerificationSuiteEvidence.model_validate(payload)
|
|
409
|
+
except ValidationError as error:
|
|
410
|
+
raise SemanticCatalogError(
|
|
411
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
412
|
+
"persisted verification evidence failed reconstruction",
|
|
413
|
+
details={"cause_type": "ValidationError"},
|
|
414
|
+
cause=error,
|
|
415
|
+
) from error
|
|
416
|
+
|
|
417
|
+
def release_binding_from_envelope(
|
|
418
|
+
self, envelope: CatalogEnvelope
|
|
419
|
+
) -> FrozenReleaseBinding | None:
|
|
420
|
+
if "frozen_release_binding" not in envelope.payload:
|
|
421
|
+
return None
|
|
422
|
+
try:
|
|
423
|
+
binding = FrozenReleaseBinding.model_validate(
|
|
424
|
+
envelope.payload["frozen_release_binding"]
|
|
425
|
+
)
|
|
426
|
+
except ValidationError as error:
|
|
427
|
+
raise SemanticCatalogError(
|
|
428
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
429
|
+
"persisted frozen release binding failed reconstruction",
|
|
430
|
+
details={"cause_type": "ValidationError"},
|
|
431
|
+
cause=error,
|
|
432
|
+
) from error
|
|
433
|
+
if binding.fingerprint != envelope.payload.get(
|
|
434
|
+
"frozen_release_binding_fingerprint"
|
|
435
|
+
):
|
|
436
|
+
raise SemanticCatalogError(
|
|
437
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
438
|
+
"persisted frozen release binding fingerprint does not match envelope",
|
|
439
|
+
details={"cause_type": "FrozenReleaseBindingFingerprintMismatch"},
|
|
440
|
+
)
|
|
441
|
+
return binding
|
|
442
|
+
|
|
443
|
+
def verification_evidence_payload(
|
|
444
|
+
self,
|
|
445
|
+
evidence: VerificationSuiteEvidence,
|
|
446
|
+
binding: FrozenReleaseBinding,
|
|
447
|
+
) -> dict[str, Any]:
|
|
448
|
+
return {
|
|
449
|
+
"evidence": evidence.model_dump(mode="json"),
|
|
450
|
+
"frozen_release_binding": binding.canonical_payload(),
|
|
451
|
+
"frozen_release_binding_fingerprint": binding.fingerprint,
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
def publication_audit_evidence_payload(
|
|
455
|
+
self, binding: PublicationAuditEvidence
|
|
456
|
+
) -> dict[str, Any]:
|
|
457
|
+
"""Canonical envelope payload for one publication audit-evidence row."""
|
|
458
|
+
return {
|
|
459
|
+
"publication_audit_evidence": binding.canonical_payload(),
|
|
460
|
+
"publication_audit_evidence_fingerprint": binding.fingerprint,
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
def publication_audit_evidence_from_envelope(
|
|
464
|
+
self, envelope: CatalogEnvelope
|
|
465
|
+
) -> PublicationAuditEvidence:
|
|
466
|
+
try:
|
|
467
|
+
binding = PublicationAuditEvidence.model_validate(
|
|
468
|
+
envelope.payload["publication_audit_evidence"]
|
|
469
|
+
)
|
|
470
|
+
except (KeyError, ValidationError) as error:
|
|
471
|
+
raise SemanticCatalogError(
|
|
472
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
473
|
+
"persisted publication audit evidence failed reconstruction",
|
|
474
|
+
details={"cause_type": "ValidationError"},
|
|
475
|
+
cause=error,
|
|
476
|
+
) from error
|
|
477
|
+
if binding.fingerprint != envelope.payload.get(
|
|
478
|
+
"publication_audit_evidence_fingerprint"
|
|
479
|
+
):
|
|
480
|
+
raise SemanticCatalogError(
|
|
481
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
482
|
+
"persisted publication audit evidence fingerprint does not "
|
|
483
|
+
"match envelope",
|
|
484
|
+
details={"cause_type": "PublicationAuditEvidenceMismatch"},
|
|
485
|
+
)
|
|
486
|
+
return binding
|
|
487
|
+
|
|
488
|
+
def audit_entry_from_envelope(
|
|
489
|
+
self,
|
|
490
|
+
envelope: CatalogEnvelope,
|
|
491
|
+
*,
|
|
492
|
+
occurred_at: Any = None,
|
|
493
|
+
entry_fingerprint: Any = None,
|
|
494
|
+
) -> AssemblyAuditEvidenceEntry:
|
|
495
|
+
"""Reconstruct one audit-evidence entry, restoring occurred_at.
|
|
496
|
+
|
|
497
|
+
The canonical envelope payload excludes the presentation
|
|
498
|
+
``occurred_at`` timestamp (fingerprint stability), so the row's
|
|
499
|
+
column is applied after reconstruction; the entry fingerprint is
|
|
500
|
+
recomputed by the model and must agree with the envelope witness.
|
|
501
|
+
When the row's independent ``entry_fingerprint`` column witness is
|
|
502
|
+
supplied, it must agree too: a swapped envelope would otherwise
|
|
503
|
+
verify against its own fingerprint while silently replacing the
|
|
504
|
+
recorded entry.
|
|
505
|
+
"""
|
|
506
|
+
try:
|
|
507
|
+
entry = AssemblyAuditEvidenceEntry(**envelope.payload)
|
|
508
|
+
except ValidationError as error:
|
|
509
|
+
raise SemanticCatalogError(
|
|
510
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
511
|
+
"persisted audit-evidence entry failed reconstruction",
|
|
512
|
+
details={"cause_type": "ValidationError"},
|
|
513
|
+
cause=error,
|
|
514
|
+
) from error
|
|
515
|
+
if entry.fingerprint != envelope.fingerprint:
|
|
516
|
+
raise SemanticCatalogError(
|
|
517
|
+
SemanticCatalogErrorCode.FINGERPRINT_MISMATCH,
|
|
518
|
+
"persisted audit-evidence entry fingerprint does not match "
|
|
519
|
+
"its envelope",
|
|
520
|
+
details={"cause_type": "AuditEvidenceFingerprintMismatch"},
|
|
521
|
+
)
|
|
522
|
+
if (
|
|
523
|
+
entry_fingerprint is not None
|
|
524
|
+
and entry.fingerprint != entry_fingerprint
|
|
525
|
+
):
|
|
526
|
+
raise SemanticCatalogError(
|
|
527
|
+
SemanticCatalogErrorCode.FINGERPRINT_MISMATCH,
|
|
528
|
+
"persisted audit-evidence entry fingerprint does not match "
|
|
529
|
+
"its row witness",
|
|
530
|
+
details={"cause_type": "AuditEvidenceRowWitnessMismatch"},
|
|
531
|
+
)
|
|
532
|
+
if occurred_at is not None:
|
|
533
|
+
entry = entry.model_copy(update={"occurred_at": _parse_dt(occurred_at)})
|
|
534
|
+
return entry
|
|
535
|
+
|
|
536
|
+
def bundle_from_envelope(self, envelope: CatalogEnvelope) -> SemanticModelBundle:
|
|
537
|
+
try:
|
|
538
|
+
bundle = SemanticModelBundle(**envelope.payload)
|
|
539
|
+
except ValidationError as error:
|
|
540
|
+
raise SemanticCatalogError(
|
|
541
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
542
|
+
"persisted bundle failed model reconstruction",
|
|
543
|
+
details={"cause_type": "ValidationError"},
|
|
544
|
+
cause=error,
|
|
545
|
+
) from error
|
|
546
|
+
validation = validate_bundle(
|
|
547
|
+
bundle, supported_schema_versions=(BUNDLE_SCHEMA_VERSION,)
|
|
548
|
+
)
|
|
549
|
+
if not validation.valid:
|
|
550
|
+
raise SemanticCatalogError(
|
|
551
|
+
SemanticCatalogErrorCode.ENVELOPE_REJECTED,
|
|
552
|
+
"persisted bundle failed structural validation",
|
|
553
|
+
details={"issue_codes": ",".join(validation.issue_codes())},
|
|
554
|
+
)
|
|
555
|
+
return bundle
|
|
556
|
+
|
|
557
|
+
# -- lifecycle events ---------------------------------------------------
|
|
558
|
+
|
|
559
|
+
def insert_event(
|
|
560
|
+
self,
|
|
561
|
+
conn: Any,
|
|
562
|
+
kind: str,
|
|
563
|
+
member_id: str | None,
|
|
564
|
+
*,
|
|
565
|
+
namespace: str,
|
|
566
|
+
occurred_at: datetime,
|
|
567
|
+
) -> None:
|
|
568
|
+
"""Append one bounded lifecycle event (idempotent by identity)."""
|
|
569
|
+
payload = strict_canonical_json(
|
|
570
|
+
{
|
|
571
|
+
"kind": kind,
|
|
572
|
+
"member_id": member_id,
|
|
573
|
+
"scope_namespace": namespace,
|
|
574
|
+
"occurred_at": occurred_at.isoformat(),
|
|
575
|
+
}
|
|
576
|
+
)
|
|
577
|
+
self.execute(
|
|
578
|
+
conn,
|
|
579
|
+
"insert_event",
|
|
580
|
+
(
|
|
581
|
+
namespace,
|
|
582
|
+
strict_sha256_fingerprint(payload),
|
|
583
|
+
kind,
|
|
584
|
+
member_id,
|
|
585
|
+
ENVELOPE_SCHEMA_VERSION,
|
|
586
|
+
payload,
|
|
587
|
+
occurred_at,
|
|
588
|
+
),
|
|
589
|
+
)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nl2data-semantic-catalog-postgres
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Durable PostgreSQL semantic catalog for the nl2data-core metadata-to-Bundle lifecycle.
|
|
5
|
+
Author: NL2Data Contributors
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Keywords: nl2data,semantic catalog,metadata,postgresql,bundles
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
Requires-Dist: nl2data-core>=0.1.0
|
|
18
|
+
Requires-Dist: psycopg[binary,pool]<4,>=3.1
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
21
|
+
Requires-Dist: mypy>=1.10; extra == "dev"
|
|
22
|
+
Requires-Dist: ruff>=0.5; extra == "dev"
|
|
23
|
+
|
|
24
|
+
# nl2data-semantic-catalog-postgres
|
|
25
|
+
|
|
26
|
+
Optional durable PostgreSQL semantic catalog for the
|
|
27
|
+
[`nl2data-core`](https://github.com/nl2data/nl2data-core) metadata-to-Bundle
|
|
28
|
+
lifecycle.
|
|
29
|
+
|
|
30
|
+
The catalog persists safe, versioned representations of metadata snapshots,
|
|
31
|
+
reviewed proposal sets, immutable Semantic Model Bundle publications, active
|
|
32
|
+
pointers, and bounded lifecycle evidence in PostgreSQL, and coordinates
|
|
33
|
+
publish/activate/rollback atomically across processes and workers.
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install nl2data-semantic-catalog-postgres
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The package is optional. Installing or importing `nl2data` / `nl2data-core`
|
|
42
|
+
never requires PostgreSQL; the `psycopg` driver is loaded lazily only when a
|
|
43
|
+
catalog is constructed from a DSN.
|
|
44
|
+
|
|
45
|
+
## Usage
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from nl2data_core.metadata.catalog import SemanticSnapshotCatalog
|
|
49
|
+
from nl2data_semantic_catalog_postgres import PostgreSQLSemanticCatalog
|
|
50
|
+
from nl2data_semantic_catalog_postgres.config import SemanticCatalogConfig
|
|
51
|
+
|
|
52
|
+
catalog: SemanticSnapshotCatalog = PostgreSQLSemanticCatalog(
|
|
53
|
+
dsn=os.environ["NL2DATA_POSTGRES_DSN"], # host-managed secret injection
|
|
54
|
+
config=SemanticCatalogConfig(namespace="my_catalog"),
|
|
55
|
+
)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
- Register snapshots, activate them per source/tenant scope, and reload the
|
|
59
|
+
active content after restart with revalidation.
|
|
60
|
+
- Save reviewed proposal sets bound to their source snapshot fingerprint.
|
|
61
|
+
- Publish, look up, activate, and roll back immutable Semantic Model Bundles
|
|
62
|
+
with core validation, dependency, freshness, drift, and scope checks.
|
|
63
|
+
|
|
64
|
+
## Safety
|
|
65
|
+
|
|
66
|
+
- Only bounded canonical envelopes are persisted: never credentials, DSNs,
|
|
67
|
+
raw prompts, raw queries/results, native objects, or unrestricted source
|
|
68
|
+
values.
|
|
69
|
+
- Every read revalidates schema version, artifact kind, fingerprint, and
|
|
70
|
+
tenant/source scope; newer envelope or migration versions fail closed.
|
|
71
|
+
- Errors are normalized and never leak DSNs or backend exception text.
|
|
72
|
+
- Catalog tables are separate from workflow state tables; the two backends
|
|
73
|
+
never share records.
|
|
74
|
+
|
|
75
|
+
See `docs/operations/services.md` in `nl2data-core` for service profiles and
|
|
76
|
+
migration guidance.
|