nl2data-semantic-catalog-postgres 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nl2data_semantic_catalog_postgres/__init__.py +18 -0
- nl2data_semantic_catalog_postgres/client.py +139 -0
- nl2data_semantic_catalog_postgres/config.py +136 -0
- nl2data_semantic_catalog_postgres/envelope.py +327 -0
- nl2data_semantic_catalog_postgres/errors.py +188 -0
- nl2data_semantic_catalog_postgres/fake_postgres/__init__.py +53 -0
- nl2data_semantic_catalog_postgres/fake_postgres/driver.py +209 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_audit.py +243 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_drafts.py +84 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_maintenance.py +263 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_publications.py +249 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_schema.py +23 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_snapshots.py +197 -0
- nl2data_semantic_catalog_postgres/fake_postgres/handlers_versions.py +330 -0
- nl2data_semantic_catalog_postgres/fake_postgres/keys.py +99 -0
- nl2data_semantic_catalog_postgres/fake_postgres/pool.py +152 -0
- nl2data_semantic_catalog_postgres/fake_postgres/registry.py +144 -0
- nl2data_semantic_catalog_postgres/maintenance.py +235 -0
- nl2data_semantic_catalog_postgres/py.typed +0 -0
- nl2data_semantic_catalog_postgres/repositories/__init__.py +26 -0
- nl2data_semantic_catalog_postgres/repositories/activation.py +849 -0
- nl2data_semantic_catalog_postgres/repositories/audit_evidence.py +221 -0
- nl2data_semantic_catalog_postgres/repositories/drafts.py +156 -0
- nl2data_semantic_catalog_postgres/repositories/evidence.py +415 -0
- nl2data_semantic_catalog_postgres/repositories/publications.py +486 -0
- nl2data_semantic_catalog_postgres/repositories/snapshots.py +372 -0
- nl2data_semantic_catalog_postgres/schema.py +337 -0
- nl2data_semantic_catalog_postgres/sql.py +520 -0
- nl2data_semantic_catalog_postgres/store.py +716 -0
- nl2data_semantic_catalog_postgres/unit_of_work.py +589 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/METADATA +76 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/RECORD +34 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/WHEEL +5 -0
- nl2data_semantic_catalog_postgres-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Optional durable PostgreSQL semantic catalog for nl2data-core.
|
|
2
|
+
|
|
3
|
+
This package implements the core :class:`SemanticSnapshotCatalog` boundary
|
|
4
|
+
with durable PostgreSQL storage for metadata snapshots, reviewed proposal
|
|
5
|
+
sets, immutable Semantic Model Bundle publications, active pointers, and
|
|
6
|
+
bounded lifecycle evidence. It is optional: importing ``nl2data`` or
|
|
7
|
+
``nl2data_core`` never requires PostgreSQL or psycopg, and the psycopg
|
|
8
|
+
driver is loaded lazily only when a catalog is constructed from a DSN.
|
|
9
|
+
|
|
10
|
+
The catalog persists only bounded canonical envelopes - never credentials,
|
|
11
|
+
DSNs, raw prompts, raw queries/results, native driver objects, or
|
|
12
|
+
unrestricted source values - and revalidates fingerprints, tenant/source
|
|
13
|
+
scope, schema versions, and compatibility on every read and activation.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
__all__: list[str] = []
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Lazy optional psycopg driver boundary for the semantic catalog.
|
|
2
|
+
|
|
3
|
+
The ``psycopg``/``psycopg_pool`` packages are loaded only inside this module
|
|
4
|
+
through :func:`importlib.import_module`, so importing ``nl2data``,
|
|
5
|
+
``nl2data_core``, or the catalog module never imports a database driver.
|
|
6
|
+
The catalog accepts an injected pool (fake or host-managed) or a DSN; the
|
|
7
|
+
DSN path is constructed here with bounded connect/command timeouts and a
|
|
8
|
+
bounded pool, and DSNs are never included in errors. Driver errors are
|
|
9
|
+
classified by class name first (so injected fake clients work without the
|
|
10
|
+
driver installed) and by the real driver only when present.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from importlib import import_module
|
|
16
|
+
from importlib.util import find_spec
|
|
17
|
+
from typing import Any, cast
|
|
18
|
+
|
|
19
|
+
from .errors import SemanticCatalogError, SemanticCatalogErrorCode
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def driver_available() -> bool:
|
|
23
|
+
"""Whether the optional ``psycopg`` driver (and pool) is installed."""
|
|
24
|
+
return find_spec("psycopg") is not None and find_spec("psycopg_pool") is not None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def build_pool(
|
|
28
|
+
dsn: str,
|
|
29
|
+
*,
|
|
30
|
+
pool_size: int,
|
|
31
|
+
connect_timeout_seconds: float,
|
|
32
|
+
command_timeout_seconds: float,
|
|
33
|
+
acquire_timeout_seconds: float,
|
|
34
|
+
schema: str,
|
|
35
|
+
) -> Any:
|
|
36
|
+
"""Lazily import the driver and build a bounded PostgreSQL connection pool.
|
|
37
|
+
|
|
38
|
+
Every checkout is pinned to the configured schema namespace through the
|
|
39
|
+
connection ``options`` so all unqualified table names resolve inside the
|
|
40
|
+
deployment's own schema. Raises a normalized ``CATALOG_UNAVAILABLE``
|
|
41
|
+
error when the driver is missing or the pool cannot be constructed; the
|
|
42
|
+
DSN and any driver exception text are never included in the error.
|
|
43
|
+
"""
|
|
44
|
+
if not driver_available():
|
|
45
|
+
raise SemanticCatalogError(
|
|
46
|
+
SemanticCatalogErrorCode.CATALOG_UNAVAILABLE,
|
|
47
|
+
"the psycopg driver is not installed; install the "
|
|
48
|
+
"'nl2data-semantic-catalog-postgres' package",
|
|
49
|
+
details={"cause_type": "ImportError"},
|
|
50
|
+
)
|
|
51
|
+
try:
|
|
52
|
+
pool_module = cast(Any, import_module("psycopg_pool"))
|
|
53
|
+
driver = cast(Any, import_module("psycopg"))
|
|
54
|
+
pool = pool_module.ConnectionPool(
|
|
55
|
+
dsn,
|
|
56
|
+
min_size=1,
|
|
57
|
+
max_size=pool_size,
|
|
58
|
+
open=True,
|
|
59
|
+
timeout=acquire_timeout_seconds,
|
|
60
|
+
kwargs={
|
|
61
|
+
"connect_timeout": connect_timeout_seconds,
|
|
62
|
+
"options": f"-c search_path={schema}",
|
|
63
|
+
"row_factory": driver.rows.dict_row,
|
|
64
|
+
},
|
|
65
|
+
)
|
|
66
|
+
except SemanticCatalogError:
|
|
67
|
+
raise
|
|
68
|
+
except Exception as error:
|
|
69
|
+
raise SemanticCatalogError(
|
|
70
|
+
SemanticCatalogErrorCode.CATALOG_UNAVAILABLE,
|
|
71
|
+
"the PostgreSQL connection pool could not be constructed",
|
|
72
|
+
details={"cause_type": type(error).__name__},
|
|
73
|
+
) from error
|
|
74
|
+
return pool
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def is_connect_error(error: BaseException) -> bool:
|
|
78
|
+
"""Whether the exception signals a lost or unavailable connection."""
|
|
79
|
+
if error.__class__.__name__ in {
|
|
80
|
+
"OperationalError",
|
|
81
|
+
"InterfaceError",
|
|
82
|
+
"ConnectionError",
|
|
83
|
+
"PoolTimeout",
|
|
84
|
+
"PoolClosed",
|
|
85
|
+
}:
|
|
86
|
+
return True
|
|
87
|
+
try:
|
|
88
|
+
pool_module = cast(Any, import_module("psycopg_pool"))
|
|
89
|
+
if isinstance(error, pool_module.PoolTimeout):
|
|
90
|
+
return True
|
|
91
|
+
except ImportError:
|
|
92
|
+
pass
|
|
93
|
+
try:
|
|
94
|
+
exceptions = cast(Any, import_module("psycopg"))
|
|
95
|
+
except ImportError:
|
|
96
|
+
return False
|
|
97
|
+
return isinstance(
|
|
98
|
+
error, (exceptions.OperationalError, exceptions.InterfaceError)
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def is_timeout_error(error: BaseException) -> bool:
|
|
103
|
+
"""Whether the exception signals a query canceled by its timeout."""
|
|
104
|
+
if error.__class__.__name__ in {
|
|
105
|
+
"TimeoutError",
|
|
106
|
+
"QueryCanceledError",
|
|
107
|
+
"QueryCanceled",
|
|
108
|
+
"Timeout",
|
|
109
|
+
}:
|
|
110
|
+
return True
|
|
111
|
+
try:
|
|
112
|
+
exceptions = cast(Any, import_module("psycopg.errors"))
|
|
113
|
+
except ImportError:
|
|
114
|
+
return False
|
|
115
|
+
return isinstance(error, exceptions.QueryCanceled)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def is_duplicate_key_error(error: BaseException) -> bool:
|
|
119
|
+
"""Whether the exception signals a unique-key violation."""
|
|
120
|
+
if error.__class__.__name__ in {"UniqueViolation", "IntegrityError"}:
|
|
121
|
+
return True
|
|
122
|
+
try:
|
|
123
|
+
exceptions = cast(Any, import_module("psycopg.errors"))
|
|
124
|
+
except ImportError:
|
|
125
|
+
return False
|
|
126
|
+
return isinstance(error, exceptions.UniqueViolation)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def is_serialization_error(error: BaseException) -> bool:
|
|
130
|
+
"""Whether the exception signals a retryable transaction conflict."""
|
|
131
|
+
if error.__class__.__name__ in {"SerializationFailure", "DeadlockDetected"}:
|
|
132
|
+
return True
|
|
133
|
+
try:
|
|
134
|
+
exceptions = cast(Any, import_module("psycopg.errors"))
|
|
135
|
+
except ImportError:
|
|
136
|
+
return False
|
|
137
|
+
return isinstance(
|
|
138
|
+
error, (exceptions.SerializationFailure, exceptions.DeadlockDetected)
|
|
139
|
+
)
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Validated bounds for the optional PostgreSQL semantic catalog.
|
|
2
|
+
|
|
3
|
+
The configuration carries only behavior bounds and safe secret *references* -
|
|
4
|
+
never connection strings, DSNs, or credentials. The host injects the actual
|
|
5
|
+
DSN into the catalog constructor from its own secret management; a dumped or
|
|
6
|
+
logged configuration therefore cannot leak PostgreSQL endpoints or secrets.
|
|
7
|
+
Every bound is validated at construction so an unsafe namespace, pool,
|
|
8
|
+
timeout, retention, envelope limit, or schema version fails before any client
|
|
9
|
+
is built, and an unsupported schema version fails closed.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
15
|
+
|
|
16
|
+
from .schema import SUPPORTED_SCHEMA_VERSION
|
|
17
|
+
|
|
18
|
+
#: Schema namespace: bounded identifier so every derived table name stays safe.
|
|
19
|
+
_SCHEMA_PATTERN = r"^[A-Za-z][A-Za-z0-9_]{0,63}$"
|
|
20
|
+
|
|
21
|
+
#: Secret reference: a bounded host-side name (environment variable, vault
|
|
22
|
+
#: key, ...) - never the secret value itself.
|
|
23
|
+
_SECRET_REF_PATTERN = r"^[A-Za-z0-9_][A-Za-z0-9_\-\.]{0,127}$"
|
|
24
|
+
|
|
25
|
+
#: Hard limits for the validated bounds below.
|
|
26
|
+
_MAX_POOL_SIZE = 64
|
|
27
|
+
_MAX_CONNECT_TIMEOUT_SECONDS = 30.0
|
|
28
|
+
_MAX_COMMAND_TIMEOUT_SECONDS = 120.0
|
|
29
|
+
_MAX_ACQUIRE_TIMEOUT_SECONDS = 60.0
|
|
30
|
+
_MAX_SNAPSHOT_RETENTION_SECONDS = 31_536_000.0 # one year
|
|
31
|
+
_MAX_EVENT_RETENTION_SECONDS = 31_536_000.0 # one year
|
|
32
|
+
_MAX_CLEANUP_BATCH = 10_000
|
|
33
|
+
_MAX_MAX_ENVELOPE_BYTES = 16 * 1024 * 1024
|
|
34
|
+
_MIN_MAX_ENVELOPE_BYTES = 4 * 1024
|
|
35
|
+
_MAX_MAX_PAYLOAD_BYTES = 8 * 1024 * 1024
|
|
36
|
+
_MIN_MAX_PAYLOAD_BYTES = 1 * 1024
|
|
37
|
+
_MAX_BUNDLE_HISTORY = 10_000
|
|
38
|
+
_MAX_ACTIVE_POINTERS_PER_SCOPE = 1_024
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SemanticCatalogConfig(BaseModel):
|
|
42
|
+
"""Immutable bounded configuration for one durable catalog.
|
|
43
|
+
|
|
44
|
+
``namespace`` is required and must be unique per application and
|
|
45
|
+
environment: it names the PostgreSQL schema that owns every catalog table,
|
|
46
|
+
so two deployments sharing one database service never observe each
|
|
47
|
+
other's records. ``dsn_secret_ref`` names the host-managed secret that
|
|
48
|
+
holds the DSN at construction time; the config itself never carries the
|
|
49
|
+
DSN. All other fields bound the catalog's work so a pathological pool,
|
|
50
|
+
timeout, retention policy, envelope limit, or history depth can never
|
|
51
|
+
produce unbounded behavior.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
55
|
+
|
|
56
|
+
#: PostgreSQL schema owning all catalog records (never a raw tenant id).
|
|
57
|
+
namespace: str = Field(pattern=_SCHEMA_PATTERN)
|
|
58
|
+
#: Host-side name of the secret holding the DSN (never the DSN itself).
|
|
59
|
+
dsn_secret_ref: str | None = Field(default=None, pattern=_SECRET_REF_PATTERN)
|
|
60
|
+
#: Bounded connection pool size of the lazy psycopg pool.
|
|
61
|
+
pool_size: int = Field(default=5, ge=1, le=_MAX_POOL_SIZE)
|
|
62
|
+
#: Bounded connect timeout for lazy pool connections (seconds).
|
|
63
|
+
connect_timeout_seconds: float = Field(
|
|
64
|
+
default=5.0, ge=0.1, le=_MAX_CONNECT_TIMEOUT_SECONDS
|
|
65
|
+
)
|
|
66
|
+
#: Bounded per-command timeout for every statement (seconds).
|
|
67
|
+
command_timeout_seconds: float = Field(
|
|
68
|
+
default=10.0, ge=0.1, le=_MAX_COMMAND_TIMEOUT_SECONDS
|
|
69
|
+
)
|
|
70
|
+
#: Bounded pool checkout timeout (seconds).
|
|
71
|
+
pool_acquire_timeout_seconds: float = Field(
|
|
72
|
+
default=5.0, ge=0.1, le=_MAX_ACQUIRE_TIMEOUT_SECONDS
|
|
73
|
+
)
|
|
74
|
+
#: Maximum schema version this catalog operates on; newer schemas reject.
|
|
75
|
+
schema_version: int = Field(
|
|
76
|
+
default=SUPPORTED_SCHEMA_VERSION, ge=1, le=SUPPORTED_SCHEMA_VERSION
|
|
77
|
+
)
|
|
78
|
+
#: Default retention applied when a snapshot registration omits one.
|
|
79
|
+
snapshot_retention_seconds: float = Field(
|
|
80
|
+
default=604_800.0, ge=60.0, le=_MAX_SNAPSHOT_RETENTION_SECONDS
|
|
81
|
+
)
|
|
82
|
+
#: Lifecycle events older than this are removed by bounded cleanup.
|
|
83
|
+
event_retention_seconds: float = Field(
|
|
84
|
+
default=604_800.0, ge=60.0, le=_MAX_EVENT_RETENTION_SECONDS
|
|
85
|
+
)
|
|
86
|
+
#: Audit-evidence entries older than this age out unless their bundle
|
|
87
|
+
#: fingerprint belongs to a non-retired published version (or another
|
|
88
|
+
#: protected entry references them as a predecessor).
|
|
89
|
+
audit_retention_seconds: float = Field(
|
|
90
|
+
default=604_800.0, ge=60.0, le=_MAX_EVENT_RETENTION_SECONDS
|
|
91
|
+
)
|
|
92
|
+
#: Maximum records removed by one bounded cleanup pass.
|
|
93
|
+
cleanup_batch_size: int = Field(default=500, ge=1, le=_MAX_CLEANUP_BATCH)
|
|
94
|
+
#: Hard upper bound for one persisted envelope (bytes).
|
|
95
|
+
max_envelope_bytes: int = Field(
|
|
96
|
+
default=1_048_576, ge=_MIN_MAX_ENVELOPE_BYTES, le=_MAX_MAX_ENVELOPE_BYTES
|
|
97
|
+
)
|
|
98
|
+
#: Hard upper bound for the canonical payload inside one envelope (bytes).
|
|
99
|
+
max_payload_bytes: int = Field(
|
|
100
|
+
default=524_288, ge=_MIN_MAX_PAYLOAD_BYTES, le=_MAX_MAX_PAYLOAD_BYTES
|
|
101
|
+
)
|
|
102
|
+
#: Maximum immutable Bundle versions retained per Bundle id.
|
|
103
|
+
max_bundle_history: int = Field(default=100, ge=1, le=_MAX_BUNDLE_HISTORY)
|
|
104
|
+
#: Maximum active Bundle pointers per tenant scope.
|
|
105
|
+
max_active_pointers_per_scope: int = Field(
|
|
106
|
+
default=256, ge=1, le=_MAX_ACTIVE_POINTERS_PER_SCOPE
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
@model_validator(mode="after")
|
|
110
|
+
def _consistent_envelope_bounds(self) -> SemanticCatalogConfig:
|
|
111
|
+
if self.max_payload_bytes > self.max_envelope_bytes:
|
|
112
|
+
raise ValueError(
|
|
113
|
+
"max_payload_bytes must not exceed max_envelope_bytes"
|
|
114
|
+
)
|
|
115
|
+
return self
|
|
116
|
+
|
|
117
|
+
def safe_payload(self) -> dict[str, object]:
|
|
118
|
+
"""Diagnostics dump: bounds and references only, never secrets."""
|
|
119
|
+
return {
|
|
120
|
+
"namespace": self.namespace,
|
|
121
|
+
"dsn_secret_ref": self.dsn_secret_ref,
|
|
122
|
+
"pool_size": self.pool_size,
|
|
123
|
+
"connect_timeout_seconds": self.connect_timeout_seconds,
|
|
124
|
+
"command_timeout_seconds": self.command_timeout_seconds,
|
|
125
|
+
"pool_acquire_timeout_seconds": self.pool_acquire_timeout_seconds,
|
|
126
|
+
"schema_version": self.schema_version,
|
|
127
|
+
"snapshot_retention_seconds": self.snapshot_retention_seconds,
|
|
128
|
+
"event_retention_seconds": self.event_retention_seconds,
|
|
129
|
+
"audit_retention_seconds": self.audit_retention_seconds,
|
|
130
|
+
"cleanup_batch_size": self.cleanup_batch_size,
|
|
131
|
+
"max_envelope_bytes": self.max_envelope_bytes,
|
|
132
|
+
"max_payload_bytes": self.max_payload_bytes,
|
|
133
|
+
"max_bundle_history": self.max_bundle_history,
|
|
134
|
+
"max_active_pointers_per_scope": self.max_active_pointers_per_scope,
|
|
135
|
+
}
|
|
136
|
+
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
"""Bounded canonical JSON envelopes for persisted catalog artifacts.
|
|
2
|
+
|
|
3
|
+
Every artifact persisted by the catalog is wrapped in a versioned envelope::
|
|
4
|
+
|
|
5
|
+
{
|
|
6
|
+
"schema_version": 1,
|
|
7
|
+
"kind": "snapshot" | "proposal_set" | "bundle" |
|
|
8
|
+
"assembly_draft" | "accepted_assertion_manifest" |
|
|
9
|
+
"publish_audit",
|
|
10
|
+
"fingerprint": "sha256:<64 hex>",
|
|
11
|
+
"canonicalization_profile": "jcs-v1" | "legacy-deterministic-json-v1",
|
|
12
|
+
"payload": { ... canonical safe payload ... }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
The canonicalization profile records which canonical JSON algorithm produced
|
|
16
|
+
the stored fingerprint, so reload can recompute and validate the identity
|
|
17
|
+
under the same profile. Records written before profile metadata existed
|
|
18
|
+
carry no profile member and are classified explicitly as the legacy
|
|
19
|
+
profile; an unsupported declared profile fails closed.
|
|
20
|
+
|
|
21
|
+
Encoding validates the payload *before* persistence: the payload must be a
|
|
22
|
+
JSON-native mapping, byte-bounded, and its canonical fingerprint (under the
|
|
23
|
+
declared profile) must equal the declared fingerprint, so an unsafe or
|
|
24
|
+
inconsistent artifact is rejected before any row is written. Decoding
|
|
25
|
+
revalidates everything after a read: schema version (a newer envelope fails
|
|
26
|
+
closed), kind, canonicalization profile, fingerprint, and byte bounds, so a
|
|
27
|
+
tampered, truncated, or forward-incompatible row is never reinterpreted.
|
|
28
|
+
All failures raise :class:`EnvelopeRejectedError` with a bounded safe
|
|
29
|
+
reason code and message - never backend text or payload content.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import json
|
|
35
|
+
import re
|
|
36
|
+
from collections.abc import Mapping
|
|
37
|
+
from enum import StrEnum
|
|
38
|
+
from typing import Any
|
|
39
|
+
|
|
40
|
+
from nl2data_core.canonical import (
|
|
41
|
+
CANONICALIZATION_PROFILE_JCS,
|
|
42
|
+
CanonicalizationError,
|
|
43
|
+
canonical_json,
|
|
44
|
+
profile_fingerprint,
|
|
45
|
+
resolve_canonicalization_profile,
|
|
46
|
+
strict_canonical_json,
|
|
47
|
+
)
|
|
48
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
49
|
+
|
|
50
|
+
#: The only envelope structure version this runtime understands.
|
|
51
|
+
ENVELOPE_SCHEMA_VERSION = 1
|
|
52
|
+
|
|
53
|
+
_FINGERPRINT_PATTERN = r"^sha256:[0-9a-f]{64}$"
|
|
54
|
+
|
|
55
|
+
#: JSON-native scalar types accepted inside a safe payload.
|
|
56
|
+
_JSON_SCALARS = (str, int, float, bool, type(None))
|
|
57
|
+
|
|
58
|
+
#: Bounded reason codes carried by :class:`EnvelopeRejectedError`.
|
|
59
|
+
_MALFORMED = "malformed"
|
|
60
|
+
_UNSAFE_PAYLOAD = "unsafe_payload"
|
|
61
|
+
_UNKNOWN_KIND = "unknown_kind"
|
|
62
|
+
_KIND_MISMATCH = "kind_mismatch"
|
|
63
|
+
_NEWER_SCHEMA = "newer_schema"
|
|
64
|
+
_FINGERPRINT_MISMATCH = "fingerprint_mismatch"
|
|
65
|
+
_OVERSIZED = "oversized"
|
|
66
|
+
_INCOMPATIBLE_PROFILE = "incompatible_profile"
|
|
67
|
+
|
|
68
|
+
#: Profile recorded on every newly encoded envelope. Existing rows without
|
|
69
|
+
#: profile metadata are classified as the legacy profile on reload.
|
|
70
|
+
DEFAULT_CANONICALIZATION_PROFILE = CANONICALIZATION_PROFILE_JCS
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class ArtifactKind(StrEnum):
|
|
74
|
+
"""Kinds of artifacts the catalog persists as envelopes."""
|
|
75
|
+
|
|
76
|
+
SNAPSHOT = "snapshot"
|
|
77
|
+
PROPOSAL_SET = "proposal_set"
|
|
78
|
+
BUNDLE = "bundle"
|
|
79
|
+
ASSEMBLY_DRAFT = "assembly_draft"
|
|
80
|
+
ACCEPTED_ASSERTION_MANIFEST = "accepted_assertion_manifest"
|
|
81
|
+
PUBLISH_AUDIT = "publish_audit"
|
|
82
|
+
VERIFICATION_SUITE_EVIDENCE = "verification_suite_evidence"
|
|
83
|
+
PUBLICATION_AUDIT_EVIDENCE = "publication_audit_evidence"
|
|
84
|
+
ASSEMBLY_AUDIT_EVIDENCE = "assembly_audit_evidence"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class EnvelopeRejectedError(Exception):
|
|
88
|
+
"""A persisted artifact failed safe envelope validation.
|
|
89
|
+
|
|
90
|
+
``code`` is one of the bounded reason codes above; ``message`` is a
|
|
91
|
+
short safe description that never includes payload content or backend
|
|
92
|
+
text. The catalog boundary normalizes this into its public error
|
|
93
|
+
vocabulary without leaking details.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
def __init__(self, code: str, message: str) -> None:
|
|
97
|
+
super().__init__(message)
|
|
98
|
+
self.code = code
|
|
99
|
+
self.message = message
|
|
100
|
+
|
|
101
|
+
def safe_payload(self) -> dict[str, str]:
|
|
102
|
+
return {"code": self.code, "message": self.message}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _assert_json_safe(value: Any, path: str) -> None:
|
|
106
|
+
"""Reject any payload value that is not JSON-native and bounded."""
|
|
107
|
+
if isinstance(value, Mapping):
|
|
108
|
+
if len(value) > 65_536:
|
|
109
|
+
raise EnvelopeRejectedError(
|
|
110
|
+
_UNSAFE_PAYLOAD, "envelope payload mappings are unbounded"
|
|
111
|
+
)
|
|
112
|
+
for key, item in value.items():
|
|
113
|
+
if not isinstance(key, str) or not key or len(key) > 256:
|
|
114
|
+
raise EnvelopeRejectedError(
|
|
115
|
+
_UNSAFE_PAYLOAD, "envelope payload keys must be bounded strings"
|
|
116
|
+
)
|
|
117
|
+
_assert_json_safe(item, f"{path}.{key}")
|
|
118
|
+
return
|
|
119
|
+
if isinstance(value, (list, tuple)):
|
|
120
|
+
if len(value) > 65_536:
|
|
121
|
+
raise EnvelopeRejectedError(
|
|
122
|
+
_UNSAFE_PAYLOAD, "envelope payload collections are unbounded"
|
|
123
|
+
)
|
|
124
|
+
for index, item in enumerate(value):
|
|
125
|
+
_assert_json_safe(item, f"{path}[{index}]")
|
|
126
|
+
return
|
|
127
|
+
if isinstance(value, _JSON_SCALARS):
|
|
128
|
+
if isinstance(value, str) and len(value) > 1_048_576:
|
|
129
|
+
raise EnvelopeRejectedError(
|
|
130
|
+
_UNSAFE_PAYLOAD, "envelope payload strings are unbounded"
|
|
131
|
+
)
|
|
132
|
+
if isinstance(value, float) and (value != value or value in (float("inf"), float("-inf"))): # noqa: E501
|
|
133
|
+
raise EnvelopeRejectedError(
|
|
134
|
+
_UNSAFE_PAYLOAD, "envelope payload floats must be finite"
|
|
135
|
+
)
|
|
136
|
+
return
|
|
137
|
+
raise EnvelopeRejectedError(
|
|
138
|
+
_UNSAFE_PAYLOAD, "envelope payloads must be JSON-native values"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _utf8_bytes(text: str) -> int:
|
|
143
|
+
return len(text.encode("utf-8"))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _profile_canonical_json(profile: str, payload: Mapping[str, Any]) -> str:
|
|
147
|
+
"""Canonical JSON text of a payload under an explicit profile."""
|
|
148
|
+
if profile == CANONICALIZATION_PROFILE_JCS:
|
|
149
|
+
return strict_canonical_json(payload)
|
|
150
|
+
return canonical_json(payload)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _profile_fingerprint_rejected(
|
|
154
|
+
profile: str, payload: Mapping[str, Any]
|
|
155
|
+
) -> str:
|
|
156
|
+
"""Profile fingerprint with encoder rejections mapped to safe errors.
|
|
157
|
+
|
|
158
|
+
The strict encoder raises :class:`CanonicalizationError` for values
|
|
159
|
+
``_assert_json_safe`` cannot see (tuples, deeply prepared enums), and
|
|
160
|
+
the legacy encoder raises ``ValueError`` for NFC-colliding keys; both
|
|
161
|
+
must surface as :class:`EnvelopeRejectedError`, never as raw encoder
|
|
162
|
+
exceptions, so the catalog boundary stays fail-closed and normalized.
|
|
163
|
+
"""
|
|
164
|
+
try:
|
|
165
|
+
return profile_fingerprint(profile, payload)
|
|
166
|
+
except (CanonicalizationError, ValueError) as error:
|
|
167
|
+
raise EnvelopeRejectedError(
|
|
168
|
+
_UNSAFE_PAYLOAD,
|
|
169
|
+
"envelope payload cannot be canonicalized under its profile",
|
|
170
|
+
) from error
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _profile_text_rejected(profile: str, payload: Mapping[str, Any]) -> str:
|
|
174
|
+
"""Profile canonical text with encoder rejections mapped to safe errors."""
|
|
175
|
+
try:
|
|
176
|
+
return _profile_canonical_json(profile, payload)
|
|
177
|
+
except CanonicalizationError as error:
|
|
178
|
+
raise EnvelopeRejectedError(
|
|
179
|
+
_UNSAFE_PAYLOAD,
|
|
180
|
+
"envelope payload cannot be canonicalized under its profile",
|
|
181
|
+
) from error
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _classify_profile(recorded: Any) -> str:
|
|
185
|
+
"""Classify a recorded canonicalization profile; unknown fails closed."""
|
|
186
|
+
if recorded is not None and not isinstance(recorded, str):
|
|
187
|
+
raise EnvelopeRejectedError(
|
|
188
|
+
_MALFORMED, "envelope canonicalization profile is invalid"
|
|
189
|
+
)
|
|
190
|
+
try:
|
|
191
|
+
return resolve_canonicalization_profile(recorded)
|
|
192
|
+
except CanonicalizationError:
|
|
193
|
+
raise EnvelopeRejectedError(
|
|
194
|
+
_INCOMPATIBLE_PROFILE,
|
|
195
|
+
"envelope declares an unsupported canonicalization profile",
|
|
196
|
+
) from None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def encode_envelope(
|
|
200
|
+
kind: ArtifactKind,
|
|
201
|
+
payload: Mapping[str, Any],
|
|
202
|
+
fingerprint: str,
|
|
203
|
+
*,
|
|
204
|
+
canonicalization_profile: str = DEFAULT_CANONICALIZATION_PROFILE,
|
|
205
|
+
max_envelope_bytes: int,
|
|
206
|
+
max_payload_bytes: int,
|
|
207
|
+
) -> str:
|
|
208
|
+
"""Encode one artifact as a validated, bounded canonical envelope.
|
|
209
|
+
|
|
210
|
+
Raises :class:`EnvelopeRejectedError` when the kind is unknown, the
|
|
211
|
+
profile is unsupported, the payload is not a bounded JSON-native
|
|
212
|
+
mapping, its canonical fingerprint (under the declared profile) does
|
|
213
|
+
not match ``fingerprint``, or either byte bound is exceeded.
|
|
214
|
+
"""
|
|
215
|
+
profile = _classify_profile(canonicalization_profile)
|
|
216
|
+
if not isinstance(payload, Mapping):
|
|
217
|
+
raise EnvelopeRejectedError(
|
|
218
|
+
_UNSAFE_PAYLOAD, "envelope payloads must be mappings"
|
|
219
|
+
)
|
|
220
|
+
if re.fullmatch(_FINGERPRINT_PATTERN, fingerprint) is None:
|
|
221
|
+
raise EnvelopeRejectedError(
|
|
222
|
+
_FINGERPRINT_MISMATCH, "envelope fingerprint is malformed"
|
|
223
|
+
)
|
|
224
|
+
_assert_json_safe(payload, "payload")
|
|
225
|
+
if _profile_fingerprint_rejected(profile, payload) != fingerprint:
|
|
226
|
+
raise EnvelopeRejectedError(
|
|
227
|
+
_FINGERPRINT_MISMATCH,
|
|
228
|
+
"envelope fingerprint does not match the canonical payload",
|
|
229
|
+
)
|
|
230
|
+
payload_text = _profile_text_rejected(profile, payload)
|
|
231
|
+
if _utf8_bytes(payload_text) > max_payload_bytes:
|
|
232
|
+
raise EnvelopeRejectedError(_OVERSIZED, "envelope payload exceeds its bound")
|
|
233
|
+
envelope_text = strict_canonical_json(
|
|
234
|
+
{
|
|
235
|
+
"schema_version": ENVELOPE_SCHEMA_VERSION,
|
|
236
|
+
"kind": kind.value,
|
|
237
|
+
"fingerprint": fingerprint,
|
|
238
|
+
"canonicalization_profile": profile,
|
|
239
|
+
"payload": json.loads(payload_text),
|
|
240
|
+
}
|
|
241
|
+
)
|
|
242
|
+
if _utf8_bytes(envelope_text) > max_envelope_bytes:
|
|
243
|
+
raise EnvelopeRejectedError(_OVERSIZED, "envelope exceeds its bound")
|
|
244
|
+
return envelope_text
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class CatalogEnvelope(BaseModel):
|
|
248
|
+
"""Validated envelope read back from the catalog."""
|
|
249
|
+
|
|
250
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
251
|
+
|
|
252
|
+
schema_version: int = Field(ge=1)
|
|
253
|
+
kind: ArtifactKind
|
|
254
|
+
fingerprint: str = Field(pattern=_FINGERPRINT_PATTERN)
|
|
255
|
+
canonicalization_profile: str
|
|
256
|
+
payload: dict[str, Any]
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def decode_envelope(
|
|
260
|
+
text: str,
|
|
261
|
+
*,
|
|
262
|
+
expected_kind: ArtifactKind,
|
|
263
|
+
supported_schema_version: int,
|
|
264
|
+
max_envelope_bytes: int,
|
|
265
|
+
max_payload_bytes: int,
|
|
266
|
+
) -> CatalogEnvelope:
|
|
267
|
+
"""Decode and fully revalidate one persisted envelope.
|
|
268
|
+
|
|
269
|
+
Raises :class:`EnvelopeRejectedError` on any malformed, oversized,
|
|
270
|
+
unknown-kind, newer-schema, kind-mismatched, incompatible-profile, or
|
|
271
|
+
fingerprint-mismatched envelope. A newer schema version, an unknown
|
|
272
|
+
canonicalization profile, or a fingerprint mismatch fails closed: the
|
|
273
|
+
artifact is never returned to callers.
|
|
274
|
+
"""
|
|
275
|
+
if not isinstance(text, str) or not text:
|
|
276
|
+
raise EnvelopeRejectedError(_MALFORMED, "envelope is empty or malformed")
|
|
277
|
+
if _utf8_bytes(text) > max_envelope_bytes:
|
|
278
|
+
raise EnvelopeRejectedError(_OVERSIZED, "envelope exceeds its bound")
|
|
279
|
+
try:
|
|
280
|
+
raw = json.loads(text)
|
|
281
|
+
except ValueError as exc:
|
|
282
|
+
raise EnvelopeRejectedError(_MALFORMED, "envelope is not valid JSON") from exc
|
|
283
|
+
if not isinstance(raw, Mapping):
|
|
284
|
+
raise EnvelopeRejectedError(_MALFORMED, "envelope is not valid JSON")
|
|
285
|
+
required_keys = {"schema_version", "kind", "fingerprint", "payload"}
|
|
286
|
+
optional_keys = {"canonicalization_profile"}
|
|
287
|
+
if set(raw) - optional_keys != required_keys:
|
|
288
|
+
raise EnvelopeRejectedError(_MALFORMED, "envelope structure is invalid")
|
|
289
|
+
schema_version = raw["schema_version"]
|
|
290
|
+
if (
|
|
291
|
+
not isinstance(schema_version, int)
|
|
292
|
+
or isinstance(schema_version, bool)
|
|
293
|
+
or schema_version < 1
|
|
294
|
+
):
|
|
295
|
+
raise EnvelopeRejectedError(_MALFORMED, "envelope schema version is invalid")
|
|
296
|
+
if schema_version > supported_schema_version:
|
|
297
|
+
raise EnvelopeRejectedError(
|
|
298
|
+
_NEWER_SCHEMA, "envelope schema version is newer than supported"
|
|
299
|
+
)
|
|
300
|
+
kind_value = raw["kind"]
|
|
301
|
+
if not isinstance(kind_value, str) or kind_value not in ArtifactKind._value2member_map_:
|
|
302
|
+
raise EnvelopeRejectedError(_UNKNOWN_KIND, "envelope kind is unknown")
|
|
303
|
+
kind = ArtifactKind(kind_value)
|
|
304
|
+
if kind is not expected_kind:
|
|
305
|
+
raise EnvelopeRejectedError(_KIND_MISMATCH, "envelope kind does not match")
|
|
306
|
+
fingerprint = raw["fingerprint"]
|
|
307
|
+
if not isinstance(fingerprint, str) or re.fullmatch(_FINGERPRINT_PATTERN, fingerprint) is None:
|
|
308
|
+
raise EnvelopeRejectedError(_FINGERPRINT_MISMATCH, "envelope fingerprint is malformed")
|
|
309
|
+
payload = raw["payload"]
|
|
310
|
+
if not isinstance(payload, Mapping):
|
|
311
|
+
raise EnvelopeRejectedError(_MALFORMED, "envelope payload is invalid")
|
|
312
|
+
profile = _classify_profile(raw.get("canonicalization_profile"))
|
|
313
|
+
_assert_json_safe(payload, "payload")
|
|
314
|
+
if _utf8_bytes(_profile_text_rejected(profile, payload)) > max_payload_bytes:
|
|
315
|
+
raise EnvelopeRejectedError(_OVERSIZED, "envelope payload exceeds its bound")
|
|
316
|
+
if _profile_fingerprint_rejected(profile, payload) != fingerprint:
|
|
317
|
+
raise EnvelopeRejectedError(
|
|
318
|
+
_FINGERPRINT_MISMATCH,
|
|
319
|
+
"envelope fingerprint does not match the canonical payload",
|
|
320
|
+
)
|
|
321
|
+
return CatalogEnvelope(
|
|
322
|
+
schema_version=schema_version,
|
|
323
|
+
kind=kind,
|
|
324
|
+
fingerprint=fingerprint,
|
|
325
|
+
canonicalization_profile=profile,
|
|
326
|
+
payload=dict(payload),
|
|
327
|
+
)
|