continuo-engine-contract 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_engine_contract-0.7.0/.gitignore +11 -0
- continuo_engine_contract-0.7.0/PKG-INFO +62 -0
- continuo_engine_contract-0.7.0/README.md +51 -0
- continuo_engine_contract-0.7.0/continuo_engine_contract/__init__.py +1 -0
- continuo_engine_contract-0.7.0/continuo_engine_contract/config.py +50 -0
- continuo_engine_contract-0.7.0/continuo_engine_contract/port.py +189 -0
- continuo_engine_contract-0.7.0/continuo_engine_contract/result.py +34 -0
- continuo_engine_contract-0.7.0/continuo_engine_contract/sql.py +86 -0
- continuo_engine_contract-0.7.0/continuo_engine_contract/types.py +23 -0
- continuo_engine_contract-0.7.0/pyproject.toml +19 -0
- continuo_engine_contract-0.7.0/tests/__init__.py +0 -0
- continuo_engine_contract-0.7.0/tests/test_config.py +55 -0
- continuo_engine_contract-0.7.0/tests/test_contract_types.py +25 -0
- continuo_engine_contract-0.7.0/tests/test_port.py +349 -0
- continuo_engine_contract-0.7.0/tests/test_result.py +40 -0
- continuo_engine_contract-0.7.0/tests/test_sql.py +103 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: continuo-engine-contract
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: The contract a Continuo warehouse engine must satisfy: the WarehouseAdapter port and the result-block wire format.
|
|
5
|
+
Author: Simone Carolini
|
|
6
|
+
Classifier: Development Status :: 4 - Beta
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
8
|
+
Requires-Python: >=3.14
|
|
9
|
+
Requires-Dist: sqlglot==30.15.0
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# continuo-engine-contract
|
|
13
|
+
|
|
14
|
+
The contract shared by the Continuo python runtime — the node harness and the
|
|
15
|
+
validation runner alike — and every engine adapter library. Five modules:
|
|
16
|
+
|
|
17
|
+
| Module | What it holds |
|
|
18
|
+
|---|---|
|
|
19
|
+
| `continuo_engine_contract.port` | The `WarehouseAdapter` port — one port covering both the data plane (`fetch`/`ensure_table`/`load`) and validation (engine-specific empty-table DDL) — plus entry-point discovery |
|
|
20
|
+
| `continuo_engine_contract.types` | The SQL column-type grammar that DDL-emitting methods validate against |
|
|
21
|
+
| `continuo_engine_contract.sql` | `ensure_single_read`, the parse-backed gate `check_binds` runs before it lets a declared read near an engine |
|
|
22
|
+
| `continuo_engine_contract.config` | `ensure_known_keys`, the fail-closed gate `build_empty_from_columns` runs before it lets a node's physical-layout `config` block reach DDL |
|
|
23
|
+
| `continuo_engine_contract.result` | The sentinel-framed result-block wire format the runner emits and the k8s-controller parses |
|
|
24
|
+
|
|
25
|
+
Each `continuo-python-runtime-<engine>` library depends on this package and
|
|
26
|
+
implements the port.
|
|
27
|
+
|
|
28
|
+
`ensure_single_read` is why this package has a runtime dependency —
|
|
29
|
+
[sqlglot](https://pypi.org/project/sqlglot/), pinned exactly — where it
|
|
30
|
+
previously had none. Deciding "is this one read query?" means parsing it, and
|
|
31
|
+
that rule is engine-independent: every adapter needs the same one, so it lives
|
|
32
|
+
here rather than being reimplemented per engine. Adapters call it with their
|
|
33
|
+
own dialect (`"postgres"`, `"trino"`, …) and it raises `ValueError` on
|
|
34
|
+
anything that is not exactly one query — no sqlglot exception escapes, so
|
|
35
|
+
callers never import sqlglot to handle a rejected read.
|
|
36
|
+
|
|
37
|
+
## A contract, not a service
|
|
38
|
+
|
|
39
|
+
This package runs nothing of its own — it isn't a service and it isn't the Job
|
|
40
|
+
either. It's the interface both callers agree on: `continuo-python-runtime`
|
|
41
|
+
(the node harness and the validation runner that call it) and every
|
|
42
|
+
`continuo-python-runtime-<engine>` adapter package (which implements it). The
|
|
43
|
+
whole contract is the twelve `WarehouseAdapter` methods (`required_env`,
|
|
44
|
+
`from_env`, `ensure_schema`, `drop_schema`, `build_empty_from_sql`,
|
|
45
|
+
`clone_empty_from_prod`, `build_empty_from_columns`, `check_binds`, `fetch`,
|
|
46
|
+
`ensure_table`, `load`, `close`) plus the sentinel result-block format, both
|
|
47
|
+
reached through the single `continuo_engine.adapters` entry-point group that
|
|
48
|
+
`discover_adapter` resolves against — and the three shared guards
|
|
49
|
+
implementations are required to run rather than reinvent:
|
|
50
|
+
`types.validate_column_type` before interpolating a type into DDL,
|
|
51
|
+
`sql.ensure_single_read` before letting a declared read reach an engine, and
|
|
52
|
+
`config.ensure_known_keys` before building any DDL from a node's
|
|
53
|
+
physical-layout `config` block, to reject keys this engine doesn't recognize.
|
|
54
|
+
|
|
55
|
+
Because the port *is* the contract, this package is very unlikely to need
|
|
56
|
+
changes — new warehouse support is a new adapter package implementing the
|
|
57
|
+
existing methods, not a change here. Touching this file means bumping a
|
|
58
|
+
version that the co-located runtime and adapters in this workspace consume
|
|
59
|
+
immediately, and that external adapter authors pin from PyPI, so treat it as
|
|
60
|
+
a deliberate, coordinated change. The `result` module is stricter still: its
|
|
61
|
+
wire format is frozen against Continuo's Go parser (`pkg/validationresult`)
|
|
62
|
+
and must not change from this side at all.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# continuo-engine-contract
|
|
2
|
+
|
|
3
|
+
The contract shared by the Continuo python runtime — the node harness and the
|
|
4
|
+
validation runner alike — and every engine adapter library. Five modules:
|
|
5
|
+
|
|
6
|
+
| Module | What it holds |
|
|
7
|
+
|---|---|
|
|
8
|
+
| `continuo_engine_contract.port` | The `WarehouseAdapter` port — one port covering both the data plane (`fetch`/`ensure_table`/`load`) and validation (engine-specific empty-table DDL) — plus entry-point discovery |
|
|
9
|
+
| `continuo_engine_contract.types` | The SQL column-type grammar that DDL-emitting methods validate against |
|
|
10
|
+
| `continuo_engine_contract.sql` | `ensure_single_read`, the parse-backed gate `check_binds` runs before it lets a declared read near an engine |
|
|
11
|
+
| `continuo_engine_contract.config` | `ensure_known_keys`, the fail-closed gate `build_empty_from_columns` runs before it lets a node's physical-layout `config` block reach DDL |
|
|
12
|
+
| `continuo_engine_contract.result` | The sentinel-framed result-block wire format the runner emits and the k8s-controller parses |
|
|
13
|
+
|
|
14
|
+
Each `continuo-python-runtime-<engine>` library depends on this package and
|
|
15
|
+
implements the port.
|
|
16
|
+
|
|
17
|
+
`ensure_single_read` is why this package has a runtime dependency —
|
|
18
|
+
[sqlglot](https://pypi.org/project/sqlglot/), pinned exactly — where it
|
|
19
|
+
previously had none. Deciding "is this one read query?" means parsing it, and
|
|
20
|
+
that rule is engine-independent: every adapter needs the same one, so it lives
|
|
21
|
+
here rather than being reimplemented per engine. Adapters call it with their
|
|
22
|
+
own dialect (`"postgres"`, `"trino"`, …) and it raises `ValueError` on
|
|
23
|
+
anything that is not exactly one query — no sqlglot exception escapes, so
|
|
24
|
+
callers never import sqlglot to handle a rejected read.
|
|
25
|
+
|
|
26
|
+
## A contract, not a service
|
|
27
|
+
|
|
28
|
+
This package runs nothing of its own — it isn't a service and it isn't the Job
|
|
29
|
+
either. It's the interface both callers agree on: `continuo-python-runtime`
|
|
30
|
+
(the node harness and the validation runner that call it) and every
|
|
31
|
+
`continuo-python-runtime-<engine>` adapter package (which implements it). The
|
|
32
|
+
whole contract is the twelve `WarehouseAdapter` methods (`required_env`,
|
|
33
|
+
`from_env`, `ensure_schema`, `drop_schema`, `build_empty_from_sql`,
|
|
34
|
+
`clone_empty_from_prod`, `build_empty_from_columns`, `check_binds`, `fetch`,
|
|
35
|
+
`ensure_table`, `load`, `close`) plus the sentinel result-block format, both
|
|
36
|
+
reached through the single `continuo_engine.adapters` entry-point group that
|
|
37
|
+
`discover_adapter` resolves against — and the three shared guards
|
|
38
|
+
implementations are required to run rather than reinvent:
|
|
39
|
+
`types.validate_column_type` before interpolating a type into DDL,
|
|
40
|
+
`sql.ensure_single_read` before letting a declared read reach an engine, and
|
|
41
|
+
`config.ensure_known_keys` before building any DDL from a node's
|
|
42
|
+
physical-layout `config` block, to reject keys this engine doesn't recognize.
|
|
43
|
+
|
|
44
|
+
Because the port *is* the contract, this package is very unlikely to need
|
|
45
|
+
changes — new warehouse support is a new adapter package implementing the
|
|
46
|
+
existing methods, not a change here. Touching this file means bumping a
|
|
47
|
+
version that the co-located runtime and adapters in this workspace consume
|
|
48
|
+
immediately, and that external adapter authors pin from PyPI, so treat it as
|
|
49
|
+
a deliberate, coordinated change. The `result` module is stricter still: its
|
|
50
|
+
wire format is frozen against Continuo's Go parser (`pkg/validationresult`)
|
|
51
|
+
and must not change from this side at all.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Contract for the Continuo warehouse engines: the WarehouseAdapter port and the result-block wire format."""
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""The contract's physical-layout ``config`` guard.
|
|
2
|
+
|
|
3
|
+
A node's ``config`` block is written in the ACTIVE ENGINE's own vocabulary
|
|
4
|
+
(postgres: ``indexes``; trino: ``partitioning``/``sorted_by``/…) rather than
|
|
5
|
+
under an engine namespace: the contract is engine-bound anyway, so there is no
|
|
6
|
+
"someone else's namespace" to excuse a key the engine does not know. An
|
|
7
|
+
unrecognized key is therefore an authoring error and must reject the node — a
|
|
8
|
+
config the author believes is applied but is silently dropped would ship an
|
|
9
|
+
unindexed, unpartitioned table to production behind a green release gate.
|
|
10
|
+
|
|
11
|
+
Adapters call :func:`ensure_known_keys` before building any DDL.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Iterable, Mapping
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def ensure_known_keys(
|
|
19
|
+
config: object, known: Iterable[str], engine: str, where: str = "config"
|
|
20
|
+
) -> None:
|
|
21
|
+
"""Raise ValueError unless *config* is a mapping whose keys all lie in *known*.
|
|
22
|
+
|
|
23
|
+
Parameters
|
|
24
|
+
----------
|
|
25
|
+
config
|
|
26
|
+
The physical-layout block as it arrived from the validation spec.
|
|
27
|
+
Anything that is not a mapping is rejected rather than coerced.
|
|
28
|
+
known
|
|
29
|
+
The keys this engine's adapter recognizes. May be empty.
|
|
30
|
+
engine
|
|
31
|
+
Engine name, quoted into the message so an author reading a failed
|
|
32
|
+
release gate knows which vocabulary was expected.
|
|
33
|
+
where
|
|
34
|
+
Label for the block under inspection, so a nested caller (an index
|
|
35
|
+
entry, say) can point the message at the right part of the config.
|
|
36
|
+
|
|
37
|
+
Raises
|
|
38
|
+
------
|
|
39
|
+
ValueError
|
|
40
|
+
If *config* is not a mapping, or carries any key outside *known*.
|
|
41
|
+
"""
|
|
42
|
+
if not isinstance(config, Mapping):
|
|
43
|
+
raise ValueError(f"{where} must be a mapping, got {type(config).__name__}")
|
|
44
|
+
known_set = set(known)
|
|
45
|
+
unknown = sorted(repr(key) for key in config if key not in known_set)
|
|
46
|
+
if unknown:
|
|
47
|
+
raise ValueError(
|
|
48
|
+
f"unrecognized {where} key(s) {', '.join(unknown)} for engine {engine!r}; "
|
|
49
|
+
f"known keys: {', '.join(sorted(known_set)) or '(none)'}"
|
|
50
|
+
)
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""The warehouse-engine adapter port and entry-point discovery.
|
|
2
|
+
|
|
3
|
+
One port covers both roles an engine plays in Continuo:
|
|
4
|
+
|
|
5
|
+
* **validation DDL** — a release gate builds each node as an EMPTY table in the
|
|
6
|
+
candidate schema and bind-checks its compiled SQL (``ensure_schema``,
|
|
7
|
+
``drop_schema``, ``build_empty_from_sql``, ``build_empty_from_columns``,
|
|
8
|
+
``clone_empty_from_prod``, ``check_binds``);
|
|
9
|
+
* **python-node data-plane I/O** — the runtime harness reads declared inputs and
|
|
10
|
+
writes a node's result (``fetch``, ``ensure_table``, ``load``).
|
|
11
|
+
|
|
12
|
+
Both roles are served by the same connection to the same warehouse, so they are
|
|
13
|
+
one implementation per engine rather than two. Engine packages register a single
|
|
14
|
+
``continuo_engine.adapters`` entry point; each image installs exactly one, so
|
|
15
|
+
discovery — not configuration — selects the adapter.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from abc import ABC, abstractmethod
|
|
20
|
+
from importlib.metadata import entry_points
|
|
21
|
+
from typing import TYPE_CHECKING
|
|
22
|
+
|
|
23
|
+
ENTRY_POINT_GROUP = "continuo_engine.adapters"
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
26
|
+
import pyarrow # type: ignore[import-untyped]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AdapterDiscoveryError(Exception):
|
|
30
|
+
"""Installed adapter plugins do not resolve to exactly one usable engine."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class WarehouseAdapter(ABC):
|
|
34
|
+
"""Port for everything Continuo asks of one warehouse engine.
|
|
35
|
+
|
|
36
|
+
stdout is a parsed channel: the caller prints one sentinel-framed result
|
|
37
|
+
block (see ``result.py``) as its last stdout line. Adapters must log
|
|
38
|
+
diagnostics via the stdlib ``logging`` module (captured from the pod log)
|
|
39
|
+
rather than printing them, must never write to stdout, and must never emit
|
|
40
|
+
the ``===CONTINUO_VALIDATION_RESULT_BEGIN/END===`` marker strings
|
|
41
|
+
themselves.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
@abstractmethod
|
|
46
|
+
def required_env(cls) -> list[str]:
|
|
47
|
+
"""Names of env vars that must be non-empty before connecting."""
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
@abstractmethod
|
|
51
|
+
def from_env(cls) -> "WarehouseAdapter":
|
|
52
|
+
"""Construct a connected adapter from environment variables."""
|
|
53
|
+
|
|
54
|
+
@abstractmethod
|
|
55
|
+
def ensure_schema(self, schema: str) -> None:
|
|
56
|
+
"""Idempotently create *schema*; safe under concurrent callers."""
|
|
57
|
+
|
|
58
|
+
@abstractmethod
|
|
59
|
+
def drop_schema(self, schema: str) -> None:
|
|
60
|
+
"""Idempotently drop *schema* and everything in it; no-op if absent.
|
|
61
|
+
|
|
62
|
+
Symmetric teardown for :meth:`ensure_schema`. The executor schedules this
|
|
63
|
+
as a one-shot engine-image op once a validation reaches its terminal result,
|
|
64
|
+
so the control plane never connects to the warehouse itself. *schema* is
|
|
65
|
+
opaque and mapped to the engine's own dialect.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
@abstractmethod
|
|
69
|
+
def build_empty_from_sql(self, schema: str, table: str, compiled_sql: str) -> None:
|
|
70
|
+
"""Create ``schema.table`` empty, shaped by the compiled SELECT."""
|
|
71
|
+
|
|
72
|
+
@abstractmethod
|
|
73
|
+
def clone_empty_from_prod(self, candidate_schema: str, prod_schema: str, table: str) -> None:
|
|
74
|
+
"""Create ``candidate_schema.table`` empty, shaped like ``prod_schema.table``."""
|
|
75
|
+
|
|
76
|
+
@abstractmethod
|
|
77
|
+
def build_empty_from_columns(
|
|
78
|
+
self, schema: str, table: str, columns: list[dict], config: dict
|
|
79
|
+
) -> None:
|
|
80
|
+
"""Create ``schema.table`` empty from declared typed columns.
|
|
81
|
+
|
|
82
|
+
Rebuild semantics (drop-then-create), matching the other build ops.
|
|
83
|
+
Each column dict carries ``name`` (an identifier — implementations
|
|
84
|
+
MUST quote it with their engine's identifier escaping, never
|
|
85
|
+
interpolate it raw), ``type`` (a string from the contract's type
|
|
86
|
+
grammar — see :mod:`continuo_engine_contract.types`;
|
|
87
|
+
implementations MUST validate before interpolating into DDL), and
|
|
88
|
+
optional ``nullable`` (default True). Bare typed CREATE TABLE is
|
|
89
|
+
engine-portable by design — no dialect families.
|
|
90
|
+
|
|
91
|
+
*config* is the node's physical-layout block — indexes, sort keys,
|
|
92
|
+
partitioning — written in THIS engine's own vocabulary and applied to
|
|
93
|
+
the created table (postgres: ``indexes``; trino: ``partitioning``,
|
|
94
|
+
``sorted_by``, ``format``, ``format_version``). It is deliberately not
|
|
95
|
+
engine-namespaced, so implementations MUST fail closed on any key they
|
|
96
|
+
do not recognize: call
|
|
97
|
+
:func:`continuo_engine_contract.config.ensure_known_keys` with
|
|
98
|
+
this engine's known-key set, and validate every value, BEFORE emitting
|
|
99
|
+
any DDL — a malformed config must never leave a half-built table
|
|
100
|
+
behind. Identifiers and property values taken from *config* MUST be
|
|
101
|
+
escaped or allowlisted before interpolation, exactly as the column
|
|
102
|
+
path already does. An empty *config* MUST emit the same bare
|
|
103
|
+
``CREATE TABLE`` as contract 0.4.0 — no trailing clause, no property
|
|
104
|
+
list, no index statements.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
@abstractmethod
|
|
108
|
+
def check_binds(self, sql: str) -> None:
|
|
109
|
+
"""Verify *sql* binds against the current schema state without scanning data.
|
|
110
|
+
|
|
111
|
+
EXPLAIN/prepare-class mechanism per engine. Raises on missing
|
|
112
|
+
tables/columns or type errors; returns None when the query binds.
|
|
113
|
+
|
|
114
|
+
*sql* is a single read query, and implementations MUST establish that
|
|
115
|
+
by calling :func:`continuo_engine_contract.sql.ensure_single_read`
|
|
116
|
+
with their engine's dialect **before** building anything to send.
|
|
117
|
+
Wrapping the read in a subquery is not a substitute: a read that closes
|
|
118
|
+
the wrap's own parenthesis and reopens it after an injected statement
|
|
119
|
+
leaves the wrap balanced, and a driver that batches ``;``-separated
|
|
120
|
+
statements then executes the injected statement for real.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
@abstractmethod
|
|
124
|
+
def fetch(self, sql: str) -> "pyarrow.Table":
|
|
125
|
+
"""Execute one declared read and return the result as Arrow."""
|
|
126
|
+
|
|
127
|
+
@abstractmethod
|
|
128
|
+
def ensure_table(
|
|
129
|
+
self, schema: str, table: str, columns: list[dict], *, config: dict
|
|
130
|
+
) -> None:
|
|
131
|
+
"""CREATE TABLE IF NOT EXISTS with the typed DDL compiled from columns.
|
|
132
|
+
|
|
133
|
+
Each column dict carries ``name`` (an identifier — implementations
|
|
134
|
+
MUST quote it with their engine's identifier escaping, never
|
|
135
|
+
interpolate it raw), ``type`` (a string from the contract's type
|
|
136
|
+
grammar — see :mod:`continuo_engine_contract.types`;
|
|
137
|
+
implementations MUST validate before interpolating into DDL), and
|
|
138
|
+
``nullable`` (bool).
|
|
139
|
+
|
|
140
|
+
*config* is the node's physical-layout block in this engine's own
|
|
141
|
+
vocabulary (e.g. ``indexes``), applied when the table is created;
|
|
142
|
+
callers pass ``{}`` for none. It is keyword-only: the harness passes
|
|
143
|
+
it as ``config=`` unconditionally, and a positional-friendly
|
|
144
|
+
signature would let a caller regress to positional passing unnoticed.
|
|
145
|
+
Implementations MUST fail closed on any config key they do not
|
|
146
|
+
recognize, and MUST validate config values before interpolating
|
|
147
|
+
anything into DDL — the same discipline
|
|
148
|
+
:meth:`build_empty_from_columns` follows.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
@abstractmethod
|
|
152
|
+
def load(self, schema: str, table: str, data: "pyarrow.Table") -> None:
|
|
153
|
+
"""Atomically replace the table's contents with *data*."""
|
|
154
|
+
|
|
155
|
+
@abstractmethod
|
|
156
|
+
def close(self) -> None:
|
|
157
|
+
"""Release the underlying connection."""
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def discover_adapter() -> tuple[str, type[WarehouseAdapter]]:
|
|
161
|
+
"""Return ``(engine_name, adapter_class)`` from the single installed plugin.
|
|
162
|
+
|
|
163
|
+
Raises
|
|
164
|
+
------
|
|
165
|
+
AdapterDiscoveryError
|
|
166
|
+
If zero or multiple adapters are installed, if the entry point fails to
|
|
167
|
+
import, or if it does not resolve to a WarehouseAdapter subclass.
|
|
168
|
+
"""
|
|
169
|
+
eps = list(entry_points(group=ENTRY_POINT_GROUP))
|
|
170
|
+
if not eps:
|
|
171
|
+
raise AdapterDiscoveryError(
|
|
172
|
+
f"no engine adapter installed (entry-point group {ENTRY_POINT_GROUP!r} is "
|
|
173
|
+
f"empty); install exactly one engine package"
|
|
174
|
+
)
|
|
175
|
+
if len(eps) > 1:
|
|
176
|
+
names = ", ".join(sorted(ep.name for ep in eps))
|
|
177
|
+
raise AdapterDiscoveryError(
|
|
178
|
+
f"multiple engine adapters installed ({names}); an image must install exactly one"
|
|
179
|
+
)
|
|
180
|
+
ep = eps[0]
|
|
181
|
+
try:
|
|
182
|
+
cls = ep.load()
|
|
183
|
+
except Exception as exc:
|
|
184
|
+
raise AdapterDiscoveryError(f"failed to load adapter entry point {ep.name!r}: {exc}") from exc
|
|
185
|
+
if not (isinstance(cls, type) and issubclass(cls, WarehouseAdapter)):
|
|
186
|
+
raise AdapterDiscoveryError(
|
|
187
|
+
f"entry point {ep.name!r} does not resolve to a WarehouseAdapter subclass"
|
|
188
|
+
)
|
|
189
|
+
return ep.name, cls
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Cross-language contract for one validation node's structured result.
|
|
2
|
+
|
|
3
|
+
validation_runner.py prints exactly one sentinel-framed block on stdout as its LAST
|
|
4
|
+
output. The k8s-controller extracts the block from the pod log, uploads the JSON to
|
|
5
|
+
S3, and surfaces it as run_results_uri. The remediation classifier reads status +
|
|
6
|
+
message deterministically, falling back to the text log when the block is absent.
|
|
7
|
+
|
|
8
|
+
status uses dbt's RunStatus vocabulary (success | error | fail | skipped).
|
|
9
|
+
"""
|
|
10
|
+
import json
|
|
11
|
+
|
|
12
|
+
SENTINEL_BEGIN = "===CONTINUO_VALIDATION_RESULT_BEGIN==="
|
|
13
|
+
SENTINEL_END = "===CONTINUO_VALIDATION_RESULT_END==="
|
|
14
|
+
|
|
15
|
+
SCHEMA_VERSION = 1
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def result_block(status: str, message: str = "", failures: int = 0, unique_id: str = "") -> str:
|
|
19
|
+
"""Return the sentinel-framed, single-line-JSON block for one node.
|
|
20
|
+
|
|
21
|
+
The JSON is compact (no embedded newlines) so the consumer can split the
|
|
22
|
+
block out of a multi-line pod log by marker lines alone.
|
|
23
|
+
"""
|
|
24
|
+
body = json.dumps(
|
|
25
|
+
{
|
|
26
|
+
"schema_version": SCHEMA_VERSION,
|
|
27
|
+
"status": status,
|
|
28
|
+
"message": message or "",
|
|
29
|
+
"failures": int(failures or 0),
|
|
30
|
+
"unique_id": unique_id or "",
|
|
31
|
+
},
|
|
32
|
+
separators=(",", ":"),
|
|
33
|
+
)
|
|
34
|
+
return f"{SENTINEL_BEGIN}\n{body}\n{SENTINEL_END}"
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""The contract's single-read gate for :meth:`WarehouseAdapter.check_binds`.
|
|
2
|
+
|
|
3
|
+
``check_binds`` accepts a single read query, raises on any bind failure, and
|
|
4
|
+
scans zero data. Enforcing "a single read" by wrapping the text in a subquery
|
|
5
|
+
and letting the engine's parser object is not enough: a read that closes the
|
|
6
|
+
wrap's own parenthesis and reopens it after an injected statement leaves the
|
|
7
|
+
wrap balanced, and an engine whose driver batches ``;``-separated statements
|
|
8
|
+
(psycopg2's simple-query protocol does) then executes the injected statement
|
|
9
|
+
for real. So the shape is decided here, by parsing, before any text reaches an
|
|
10
|
+
engine.
|
|
11
|
+
|
|
12
|
+
This lives in the contract rather than in each adapter because the rule is
|
|
13
|
+
engine-independent and every adapter — including third-party ones — needs the
|
|
14
|
+
same one.
|
|
15
|
+
"""
|
|
16
|
+
import logging
|
|
17
|
+
|
|
18
|
+
import sqlglot
|
|
19
|
+
|
|
20
|
+
from sqlglot import exp
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
# The expression types a legitimate read parses to. Validated against the full
|
|
25
|
+
# set of read shapes a spec may declare (plain/star selects, WHERE/ORDER
|
|
26
|
+
# BY/LIMIT, CTEs, UNION ALL, GROUP BY, joins, FROM-less selects, VALUES,
|
|
27
|
+
# duplicate output names, trailing semicolons, trailing line comments). Narrow
|
|
28
|
+
# it further and legitimate reads start getting rejected.
|
|
29
|
+
_READ_TYPES = (exp.Select, exp.Union, exp.Values, exp.Subquery)
|
|
30
|
+
|
|
31
|
+
# Rejection messages quote the offending read, and ride back to the controller
|
|
32
|
+
# inside the runner's result block — so bound every variable-length part of
|
|
33
|
+
# them, both the read itself and any parser detail appended to it.
|
|
34
|
+
_MAX_EXCERPT = 200
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _excerpt(text: str) -> str:
|
|
38
|
+
"""Render *text* for an error message: whitespace collapsed, length bounded."""
|
|
39
|
+
collapsed = " ".join(text.split())
|
|
40
|
+
if len(collapsed) > _MAX_EXCERPT:
|
|
41
|
+
return f"{collapsed[:_MAX_EXCERPT]}... (truncated)"
|
|
42
|
+
return collapsed
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def ensure_single_read(sql: str, dialect: str | None = None) -> None:
|
|
46
|
+
"""Raise ValueError unless *sql* is exactly one read query.
|
|
47
|
+
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
sql
|
|
51
|
+
The declared read, as written in the spec.
|
|
52
|
+
dialect
|
|
53
|
+
A sqlglot dialect name (e.g. ``"postgres"``, ``"trino"``). Adapters
|
|
54
|
+
pass their engine's dialect; ``None`` parses with sqlglot's default.
|
|
55
|
+
|
|
56
|
+
Raises
|
|
57
|
+
------
|
|
58
|
+
ValueError
|
|
59
|
+
If *sql* does not parse, parses to anything other than exactly one
|
|
60
|
+
statement, or parses to a statement that is not a query. Every
|
|
61
|
+
rejection is a ValueError — a sqlglot ``ParseError`` never escapes, so
|
|
62
|
+
callers need not import sqlglot to handle a rejected read.
|
|
63
|
+
"""
|
|
64
|
+
try:
|
|
65
|
+
parsed = sqlglot.parse(sql, dialect=dialect)
|
|
66
|
+
except sqlglot.ParseError as exc:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"check_binds accepts a single read query, and this read does not parse "
|
|
69
|
+
f"as {dialect or 'SQL'}: {_excerpt(sql)} [{_excerpt(str(exc))}]"
|
|
70
|
+
) from exc
|
|
71
|
+
|
|
72
|
+
statements = [statement for statement in parsed if statement is not None]
|
|
73
|
+
if len(statements) != 1:
|
|
74
|
+
raise ValueError(
|
|
75
|
+
f"check_binds accepts exactly one read query, but this read parses as "
|
|
76
|
+
f"{len(statements)} statements: {_excerpt(sql)}"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
statement = statements[0]
|
|
80
|
+
if not isinstance(statement, _READ_TYPES):
|
|
81
|
+
raise ValueError(
|
|
82
|
+
f"check_binds accepts a single read query, but this read is a "
|
|
83
|
+
f"{statement.key.upper()} statement: {_excerpt(sql)}"
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
logger.debug("read accepted as a single query by the %s gate", dialect or "default")
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""The contract's SQL column-type grammar.
|
|
2
|
+
|
|
3
|
+
The matched text is interpolated directly into engine DDL by the adapters, so
|
|
4
|
+
this is an injection guard, not just validation — anything outside this shape
|
|
5
|
+
is rejected outright. Kept byte-identical to the runtime adapters' grammar in
|
|
6
|
+
continuo-python-runtime so both planes accept the same type set.
|
|
7
|
+
"""
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
TYPE_RE = re.compile(
|
|
11
|
+
r"^("
|
|
12
|
+
r"BIGINT|INT|INTEGER|DOUBLE PRECISION|TEXT|TIMESTAMP|DATE|BOOLEAN|"
|
|
13
|
+
r"(NUMERIC|DECIMAL)\(\d+,\s*\d+\)|"
|
|
14
|
+
r"(VARCHAR|CHAR)\(\d+\)"
|
|
15
|
+
r")\Z",
|
|
16
|
+
re.IGNORECASE | re.ASCII,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def validate_column_type(type_str: str) -> None:
|
|
21
|
+
"""Raise ValueError unless *type_str* matches the contract's SQL type grammar."""
|
|
22
|
+
if not TYPE_RE.match(type_str):
|
|
23
|
+
raise ValueError(f"unsupported or invalid SQL type: {type_str!r}")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
version = "0.7.0"
|
|
3
|
+
name = "continuo-engine-contract"
|
|
4
|
+
description = "The contract a Continuo warehouse engine must satisfy: the WarehouseAdapter port and the result-block wire format."
|
|
5
|
+
authors = [{ name = "Simone Carolini" }]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
requires-python = ">= 3.14"
|
|
8
|
+
dependencies = ["sqlglot==30.15.0"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 4 - Beta",
|
|
11
|
+
"Programming Language :: Python :: 3.14",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[build-system]
|
|
15
|
+
requires = ["hatchling"]
|
|
16
|
+
build-backend = "hatchling.build"
|
|
17
|
+
|
|
18
|
+
[tool.hatch.build.targets.wheel]
|
|
19
|
+
packages = ["continuo_engine_contract"]
|
|
File without changes
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Unit tests for the physical-layout config guard."""
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from continuo_engine_contract.config import ensure_known_keys
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_empty_config_is_accepted_even_with_no_known_keys():
|
|
8
|
+
"""An absent layout block is the common case and must never raise."""
|
|
9
|
+
ensure_known_keys({}, (), "postgres")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_known_keys_are_accepted():
|
|
13
|
+
"""Every key inside the engine's vocabulary passes."""
|
|
14
|
+
ensure_known_keys({"indexes": [], "extra": 1}, ("indexes", "extra"), "postgres")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_unknown_key_is_rejected_naming_key_engine_and_vocabulary():
|
|
18
|
+
"""The message must tell the author what they wrote and what the engine knows."""
|
|
19
|
+
with pytest.raises(ValueError) as exc:
|
|
20
|
+
ensure_known_keys({"sortkey": ["id"]}, ("indexes",), "postgres")
|
|
21
|
+
message = str(exc.value)
|
|
22
|
+
assert "'sortkey'" in message
|
|
23
|
+
assert "'postgres'" in message
|
|
24
|
+
assert "indexes" in message
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_multiple_unknown_keys_are_reported_together_and_sorted():
|
|
28
|
+
"""All authoring errors surface in one gate failure, deterministically ordered."""
|
|
29
|
+
with pytest.raises(ValueError) as exc:
|
|
30
|
+
ensure_known_keys({"zeta": 1, "alpha": 2}, ("indexes",), "postgres")
|
|
31
|
+
message = str(exc.value)
|
|
32
|
+
assert message.index("'alpha'") < message.index("'zeta'")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_empty_known_set_renders_as_none():
|
|
36
|
+
"""An engine with no vocabulary yet still produces a readable message."""
|
|
37
|
+
with pytest.raises(ValueError, match=r"\(none\)"):
|
|
38
|
+
ensure_known_keys({"indexes": []}, (), "trino")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@pytest.mark.parametrize(
|
|
42
|
+
"bad", [[], "indexes", 3, None, True], ids=["list", "str", "int", "none", "bool"]
|
|
43
|
+
)
|
|
44
|
+
def test_non_mapping_config_is_rejected(bad):
|
|
45
|
+
"""Config must be a mapping; a list or scalar is an authoring error, not a default."""
|
|
46
|
+
with pytest.raises(ValueError, match="must be a mapping"):
|
|
47
|
+
ensure_known_keys(bad, ("indexes",), "postgres")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_where_label_appears_in_both_messages():
|
|
51
|
+
"""Nested callers relabel the subject so the error points at the right block."""
|
|
52
|
+
with pytest.raises(ValueError, match="index entry must be a mapping"):
|
|
53
|
+
ensure_known_keys([], ("columns",), "postgres", where="index entry")
|
|
54
|
+
with pytest.raises(ValueError, match="unrecognized index entry key"):
|
|
55
|
+
ensure_known_keys({"nope": 1}, ("columns",), "postgres", where="index entry")
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Unit tests for the SQL column-type grammar contract."""
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from continuo_engine_contract.types import validate_column_type
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.mark.parametrize("ok", [
|
|
8
|
+
"BIGINT", "INT", "INTEGER", "DOUBLE PRECISION", "TEXT",
|
|
9
|
+
"TIMESTAMP", "DATE", "BOOLEAN",
|
|
10
|
+
"NUMERIC(10,2)", "DECIMAL(38, 0)", "VARCHAR(255)", "CHAR(1)",
|
|
11
|
+
"bigint", "varchar(9)", # case-insensitive
|
|
12
|
+
])
|
|
13
|
+
def test_supported_types_pass(ok: str) -> None:
|
|
14
|
+
"""Supported types must not raise."""
|
|
15
|
+
validate_column_type(ok) # must not raise
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.mark.parametrize("bad", [
|
|
19
|
+
"", "FLOAT", "VARCHAR", "NUMERIC", "NUMERIC(10)", "VARCHAR(255); DROP TABLE x",
|
|
20
|
+
"TEXT )", "INTEGER--", "SERIAL", "JSONB", "TIMESTAMP WITH TIME ZONE",
|
|
21
|
+
])
|
|
22
|
+
def test_unsupported_or_malformed_types_raise(bad: str) -> None:
|
|
23
|
+
"""Unsupported or malformed types must raise ValueError."""
|
|
24
|
+
with pytest.raises(ValueError):
|
|
25
|
+
validate_column_type(bad)
|
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
"""Discovery tests for the WarehouseAdapter port.
|
|
2
|
+
|
|
3
|
+
These install REAL adapter packages (with genuine ``continuo_engine.adapters``
|
|
4
|
+
entry points) into throwaway virtualenvs and run ``discover_adapter`` against the
|
|
5
|
+
real ``importlib.metadata`` machinery — no monkeypatching of ``entry_points``, so the
|
|
6
|
+
tests exercise actual entry-point loading rather than an assumed return shape.
|
|
7
|
+
"""
|
|
8
|
+
import subprocess
|
|
9
|
+
import textwrap
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
from continuo_engine_contract.port import ENTRY_POINT_GROUP, WarehouseAdapter
|
|
16
|
+
|
|
17
|
+
CONTRACT_ROOT = Path(__file__).resolve().parent.parent # contract/
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_warehouse_adapter_cannot_be_instantiated():
|
|
21
|
+
"""The port is abstract — a concrete adapter must implement every method."""
|
|
22
|
+
with pytest.raises(TypeError):
|
|
23
|
+
WarehouseAdapter() # type: ignore[abstract]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_warehouse_adapter_requires_drop_schema():
|
|
27
|
+
"""drop_schema is part of the contract: omitting it blocks instantiation.
|
|
28
|
+
|
|
29
|
+
A concrete adapter that implements every method except the teardown one must
|
|
30
|
+
still be abstract, so the executor can rely on drop_schema existing on every
|
|
31
|
+
installed engine.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
class MissingDrop(WarehouseAdapter):
|
|
35
|
+
@classmethod
|
|
36
|
+
def required_env(cls):
|
|
37
|
+
return []
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_env(cls):
|
|
41
|
+
return cls()
|
|
42
|
+
|
|
43
|
+
def ensure_schema(self, schema):
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
def build_empty_from_sql(self, schema, table, compiled_sql):
|
|
47
|
+
...
|
|
48
|
+
|
|
49
|
+
def clone_empty_from_prod(self, candidate_schema, prod_schema, table):
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
def build_empty_from_columns(self, schema, table, columns, config):
|
|
53
|
+
...
|
|
54
|
+
|
|
55
|
+
def check_binds(self, sql):
|
|
56
|
+
...
|
|
57
|
+
|
|
58
|
+
def fetch(self, sql):
|
|
59
|
+
...
|
|
60
|
+
|
|
61
|
+
def ensure_table(self, schema, table, columns, *, config):
|
|
62
|
+
...
|
|
63
|
+
|
|
64
|
+
def load(self, schema, table, data):
|
|
65
|
+
...
|
|
66
|
+
|
|
67
|
+
def close(self):
|
|
68
|
+
...
|
|
69
|
+
|
|
70
|
+
with pytest.raises(TypeError, match="drop_schema"):
|
|
71
|
+
MissingDrop() # type: ignore[abstract]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_warehouse_adapter_requires_columns_and_binds_methods():
|
|
75
|
+
"""A subclass missing the 0.4.0 methods cannot be instantiated.
|
|
76
|
+
|
|
77
|
+
Adapters implementing only the 0.3.0 surface (required_env, from_env,
|
|
78
|
+
ensure_schema, drop_schema, build_empty_from_sql, clone_empty_from_prod,
|
|
79
|
+
close) must still be abstract, so callers can rely on build_empty_from_columns
|
|
80
|
+
and check_binds existing on every installed engine.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
class Partial(WarehouseAdapter):
|
|
84
|
+
@classmethod
|
|
85
|
+
def required_env(cls):
|
|
86
|
+
return []
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def from_env(cls):
|
|
90
|
+
return cls()
|
|
91
|
+
|
|
92
|
+
def ensure_schema(self, schema):
|
|
93
|
+
...
|
|
94
|
+
|
|
95
|
+
def drop_schema(self, schema):
|
|
96
|
+
...
|
|
97
|
+
|
|
98
|
+
def build_empty_from_sql(self, schema, table, compiled_sql):
|
|
99
|
+
...
|
|
100
|
+
|
|
101
|
+
def clone_empty_from_prod(self, candidate_schema, prod_schema, table):
|
|
102
|
+
...
|
|
103
|
+
|
|
104
|
+
def close(self):
|
|
105
|
+
...
|
|
106
|
+
|
|
107
|
+
with pytest.raises(TypeError, match="build_empty_from_columns|check_binds"):
|
|
108
|
+
Partial() # type: ignore[abstract]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_partial_engine_cannot_be_instantiated():
|
|
112
|
+
"""A class implementing only the validation half must fail at construction.
|
|
113
|
+
|
|
114
|
+
One port means one class per engine covering both roles: an adapter that
|
|
115
|
+
still only speaks validation DDL is abstract, so a half-migrated engine
|
|
116
|
+
package cannot be constructed and silently fail later, at the first
|
|
117
|
+
python-node read.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
class HalfEngine(WarehouseAdapter):
|
|
121
|
+
@classmethod
|
|
122
|
+
def required_env(cls):
|
|
123
|
+
return []
|
|
124
|
+
|
|
125
|
+
@classmethod
|
|
126
|
+
def from_env(cls):
|
|
127
|
+
return cls()
|
|
128
|
+
|
|
129
|
+
def ensure_schema(self, schema):
|
|
130
|
+
...
|
|
131
|
+
|
|
132
|
+
def drop_schema(self, schema):
|
|
133
|
+
...
|
|
134
|
+
|
|
135
|
+
def build_empty_from_sql(self, schema, table, compiled_sql):
|
|
136
|
+
...
|
|
137
|
+
|
|
138
|
+
def build_empty_from_columns(self, schema, table, columns, config):
|
|
139
|
+
...
|
|
140
|
+
|
|
141
|
+
def clone_empty_from_prod(self, schema, prod_schema, table):
|
|
142
|
+
...
|
|
143
|
+
|
|
144
|
+
def check_binds(self, sql):
|
|
145
|
+
...
|
|
146
|
+
|
|
147
|
+
def close(self):
|
|
148
|
+
...
|
|
149
|
+
|
|
150
|
+
# fetch / ensure_table / load deliberately missing
|
|
151
|
+
|
|
152
|
+
with pytest.raises(TypeError):
|
|
153
|
+
HalfEngine() # type: ignore[abstract]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_single_entry_point_group_and_single_discovery():
|
|
157
|
+
"""One port, one entry-point group, one discovery function — no second half left."""
|
|
158
|
+
from continuo_engine_contract import port
|
|
159
|
+
|
|
160
|
+
assert port.ENTRY_POINT_GROUP == "continuo_engine.adapters"
|
|
161
|
+
public = {name for name in dir(port) if not name.startswith("_")}
|
|
162
|
+
# Asserted as exhaustive sets rather than as absent legacy spellings: the
|
|
163
|
+
# merge left exactly one entry-point group, one discovery function, and
|
|
164
|
+
# one port class, so a second of any of them fails here whatever it is
|
|
165
|
+
# called. (tests/test_no_legacy_names.py owns the old spellings.)
|
|
166
|
+
assert {n for n in public if n.endswith("ENTRY_POINT_GROUP")} == {"ENTRY_POINT_GROUP"}
|
|
167
|
+
assert {n for n in public if n.startswith("discover")} == {"discover_adapter"}
|
|
168
|
+
assert {n for n in public if n.endswith("Adapter")} == {"WarehouseAdapter"}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_ensure_table_signature_declares_config_keyword_only():
|
|
172
|
+
"""The port itself declares the physical-layout config parameter.
|
|
173
|
+
|
|
174
|
+
Adapters receive the node's config through the abstract signature — they
|
|
175
|
+
must never have to extend beyond the ABC to accept it. It is required and
|
|
176
|
+
keyword-only: the harness passes ``config=`` unconditionally, and a
|
|
177
|
+
positional-friendly signature would let callers regress to positional
|
|
178
|
+
passing unnoticed.
|
|
179
|
+
"""
|
|
180
|
+
import inspect
|
|
181
|
+
|
|
182
|
+
sig = inspect.signature(WarehouseAdapter.ensure_table)
|
|
183
|
+
params = list(sig.parameters)
|
|
184
|
+
assert params == ["self", "schema", "table", "columns", "config"]
|
|
185
|
+
config = sig.parameters["config"]
|
|
186
|
+
assert config.kind is inspect.Parameter.KEYWORD_ONLY
|
|
187
|
+
assert config.default is inspect.Parameter.empty
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# --- Real installed-adapter discovery via throwaway venvs --------------------
|
|
191
|
+
|
|
192
|
+
_FIXTURE_MODULE = '''\
|
|
193
|
+
from continuo_engine_contract.port import WarehouseAdapter
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class OneAdapter(WarehouseAdapter):
|
|
197
|
+
@classmethod
|
|
198
|
+
def required_env(cls):
|
|
199
|
+
return []
|
|
200
|
+
|
|
201
|
+
@classmethod
|
|
202
|
+
def from_env(cls):
|
|
203
|
+
return cls()
|
|
204
|
+
|
|
205
|
+
def ensure_schema(self, schema):
|
|
206
|
+
...
|
|
207
|
+
|
|
208
|
+
def drop_schema(self, schema):
|
|
209
|
+
...
|
|
210
|
+
|
|
211
|
+
def build_empty_from_sql(self, schema, table, compiled_sql):
|
|
212
|
+
...
|
|
213
|
+
|
|
214
|
+
def clone_empty_from_prod(self, candidate_schema, prod_schema, table):
|
|
215
|
+
...
|
|
216
|
+
|
|
217
|
+
def build_empty_from_columns(self, schema, table, columns, config):
|
|
218
|
+
...
|
|
219
|
+
|
|
220
|
+
def check_binds(self, sql):
|
|
221
|
+
...
|
|
222
|
+
|
|
223
|
+
def fetch(self, sql):
|
|
224
|
+
...
|
|
225
|
+
|
|
226
|
+
def ensure_table(self, schema, table, columns, *, config):
|
|
227
|
+
...
|
|
228
|
+
|
|
229
|
+
def load(self, schema, table, data):
|
|
230
|
+
...
|
|
231
|
+
|
|
232
|
+
def close(self):
|
|
233
|
+
...
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class TwoAdapter(OneAdapter):
|
|
237
|
+
...
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
not_an_adapter = object()
|
|
241
|
+
'''
|
|
242
|
+
|
|
243
|
+
_DISCOVER_SCRIPT = textwrap.dedent(
|
|
244
|
+
"""
|
|
245
|
+
from continuo_engine_contract.port import discover_adapter, AdapterDiscoveryError
|
|
246
|
+
try:
|
|
247
|
+
name, cls = discover_adapter()
|
|
248
|
+
print("OK", name, cls.__name__)
|
|
249
|
+
except AdapterDiscoveryError as exc:
|
|
250
|
+
print("ERR", exc)
|
|
251
|
+
"""
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _run_discovery(
|
|
256
|
+
tmp_path: Path,
|
|
257
|
+
entry_points: dict,
|
|
258
|
+
module_body: str = _FIXTURE_MODULE,
|
|
259
|
+
group: str = ENTRY_POINT_GROUP,
|
|
260
|
+
script: str = _DISCOVER_SCRIPT,
|
|
261
|
+
) -> str:
|
|
262
|
+
"""Run discovery in a fresh venv holding the contract + a fixture adapter.
|
|
263
|
+
|
|
264
|
+
Installs a generated fixture package declaring *entry_points* under *group*
|
|
265
|
+
with *module_body* as its module source, then runs *script* (which calls
|
|
266
|
+
``discover_adapter``) and returns its stdout.
|
|
267
|
+
"""
|
|
268
|
+
fixture = tmp_path / "fixture"
|
|
269
|
+
(fixture / "cvfixtureadapters").mkdir(parents=True)
|
|
270
|
+
(fixture / "cvfixtureadapters" / "__init__.py").write_text(module_body)
|
|
271
|
+
eps = "\n".join(f'{name} = "cvfixtureadapters:{target}"' for name, target in entry_points.items())
|
|
272
|
+
(fixture / "pyproject.toml").write_text(
|
|
273
|
+
textwrap.dedent(
|
|
274
|
+
f"""
|
|
275
|
+
[project]
|
|
276
|
+
name = "cvfixtureadapters"
|
|
277
|
+
version = "0.0.0"
|
|
278
|
+
requires-python = ">= 3.14"
|
|
279
|
+
|
|
280
|
+
[project.entry-points."{group}"]
|
|
281
|
+
{eps}
|
|
282
|
+
|
|
283
|
+
[build-system]
|
|
284
|
+
requires = ["hatchling"]
|
|
285
|
+
build-backend = "hatchling.build"
|
|
286
|
+
|
|
287
|
+
[tool.hatch.build.targets.wheel]
|
|
288
|
+
packages = ["cvfixtureadapters"]
|
|
289
|
+
"""
|
|
290
|
+
)
|
|
291
|
+
)
|
|
292
|
+
venv = tmp_path / "venv"
|
|
293
|
+
subprocess.run(["uv", "venv", str(venv), "-p", "3.14"], check=True, capture_output=True)
|
|
294
|
+
subprocess.run(
|
|
295
|
+
["uv", "pip", "install", "--python", str(venv), str(CONTRACT_ROOT), str(fixture)],
|
|
296
|
+
check=True,
|
|
297
|
+
capture_output=True,
|
|
298
|
+
)
|
|
299
|
+
py = venv / "bin" / "python"
|
|
300
|
+
proc = subprocess.run([str(py), "-c", script], capture_output=True, text=True)
|
|
301
|
+
return proc.stdout
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def test_discover_adapter_raises_when_none_installed(tmp_path):
|
|
305
|
+
"""With no adapter package installed, discovery raises cleanly.
|
|
306
|
+
|
|
307
|
+
This runs in a throwaway venv rather than the ambient one: the ambient
|
|
308
|
+
environment in this workspace also has other, unrelated adapter packages
|
|
309
|
+
installed alongside the contract, so it can never exercise the genuine
|
|
310
|
+
zero-adapter path.
|
|
311
|
+
"""
|
|
312
|
+
out = _run_discovery(tmp_path, {})
|
|
313
|
+
assert out.startswith("ERR")
|
|
314
|
+
assert "no engine adapter installed" in out
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def test_discover_single_installed_adapter_is_found(tmp_path):
|
|
318
|
+
"""One installed adapter package → discovery returns its name and class."""
|
|
319
|
+
out = _run_discovery(tmp_path, {"one": "OneAdapter"})
|
|
320
|
+
assert out.strip() == "OK one OneAdapter"
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def test_discover_multiple_installed_adapters_raises_naming_them(tmp_path):
|
|
324
|
+
"""Two installed adapters → discovery refuses and names both."""
|
|
325
|
+
out = _run_discovery(tmp_path, {"one": "OneAdapter", "two": "TwoAdapter"})
|
|
326
|
+
assert out.startswith("ERR")
|
|
327
|
+
assert "one, two" in out
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def test_discover_non_adapter_entry_point_raises(tmp_path):
|
|
331
|
+
"""An entry point that is not a WarehouseAdapter subclass is rejected."""
|
|
332
|
+
out = _run_discovery(tmp_path, {"broken": "not_an_adapter"})
|
|
333
|
+
assert out.startswith("ERR")
|
|
334
|
+
assert "WarehouseAdapter subclass" in out
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def test_discover_wraps_adapter_import_failure(tmp_path):
|
|
338
|
+
"""A failing adapter import surfaces as AdapterDiscoveryError, not a raw ImportError.
|
|
339
|
+
|
|
340
|
+
That lets the runner still emit a structured result block for a malformed engine
|
|
341
|
+
image rather than crashing with a traceback.
|
|
342
|
+
"""
|
|
343
|
+
out = _run_discovery(
|
|
344
|
+
tmp_path,
|
|
345
|
+
{"one": "OneAdapter"},
|
|
346
|
+
module_body="import definitely_missing_pkg_zzz # noqa: F401\n",
|
|
347
|
+
)
|
|
348
|
+
assert out.startswith("ERR")
|
|
349
|
+
assert "failed to load adapter entry point" in out
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Unit tests for the validation-result contract helper."""
|
|
2
|
+
import json
|
|
3
|
+
|
|
4
|
+
from continuo_engine_contract import result
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def test_sentinel_markers_are_the_wire_contract():
|
|
8
|
+
"""Pin the sentinel strings to their literal wire values.
|
|
9
|
+
|
|
10
|
+
The Go side (pkg/validationresult) pins the same literals independently, so an
|
|
11
|
+
edit to either marker on either side fails its own suite — the cross-language
|
|
12
|
+
guard cannot be silently defeated by editing only the Python constant.
|
|
13
|
+
"""
|
|
14
|
+
assert result.SENTINEL_BEGIN == "===CONTINUO_VALIDATION_RESULT_BEGIN==="
|
|
15
|
+
assert result.SENTINEL_END == "===CONTINUO_VALIDATION_RESULT_END==="
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _extract(block: str) -> dict:
|
|
19
|
+
lines = block.splitlines()
|
|
20
|
+
assert lines[0] == result.SENTINEL_BEGIN
|
|
21
|
+
assert lines[-1] == result.SENTINEL_END
|
|
22
|
+
return json.loads(lines[1])
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_result_block_is_sentinel_framed_single_line_json():
|
|
26
|
+
"""A populated block round-trips through the sentinel framing intact."""
|
|
27
|
+
block = result.result_block(status="error", message="boom", unique_id="model.svc.x")
|
|
28
|
+
assert _extract(block) == {
|
|
29
|
+
"schema_version": 1, "status": "error", "message": "boom",
|
|
30
|
+
"failures": 0, "unique_id": "model.svc.x",
|
|
31
|
+
}
|
|
32
|
+
assert len(block.splitlines()) == 3
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_result_block_success_defaults():
|
|
36
|
+
"""Omitted message/failures default to empty string and zero."""
|
|
37
|
+
doc = _extract(result.result_block(status="success"))
|
|
38
|
+
assert doc["status"] == "success"
|
|
39
|
+
assert doc["message"] == ""
|
|
40
|
+
assert doc["failures"] == 0
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Unit tests for the single-read gate every adapter runs before EXPLAIN."""
|
|
2
|
+
import pytest
|
|
3
|
+
|
|
4
|
+
from continuo_engine_contract.sql import ensure_single_read
|
|
5
|
+
|
|
6
|
+
DIALECTS = ["postgres", "trino", None]
|
|
7
|
+
|
|
8
|
+
# The read shapes a legitimate spec is allowed to declare. Every one of these
|
|
9
|
+
# must survive the gate on every dialect — this list is the regression surface.
|
|
10
|
+
LEGIT_READS = [
|
|
11
|
+
"select a, b from s.up",
|
|
12
|
+
"select * from s.up",
|
|
13
|
+
"select a from s.up where b = 'x;y' order by a limit 10",
|
|
14
|
+
"with c as (select a from s.up) select a from c",
|
|
15
|
+
"select a from s.up union all select a from s.up2",
|
|
16
|
+
"select a, count(*) from s.up group by a",
|
|
17
|
+
"select l.a from s.up l join s.up2 r on l.a = r.a",
|
|
18
|
+
"select 1",
|
|
19
|
+
"VALUES (1), (2)",
|
|
20
|
+
"select a as x, b as x from s.up",
|
|
21
|
+
"select a from s.up;",
|
|
22
|
+
"select a from s.up -- trailing comment",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
# The paren-balanced escapes: the wrap alone lets these through, because the
|
|
26
|
+
# read closes the wrap's own paren and reopens it after the injected statement.
|
|
27
|
+
PAREN_ESCAPE_DELETE = "select 1) AS x; DELETE FROM s.target; SELECT * FROM (SELECT 1"
|
|
28
|
+
PAREN_ESCAPE_DROP = "select 1) AS x; DROP TABLE s.target; SELECT * FROM (SELECT 1"
|
|
29
|
+
|
|
30
|
+
STACKED = "select a from s.up; delete from s.up"
|
|
31
|
+
|
|
32
|
+
BARE_DML = [
|
|
33
|
+
"delete from s.up",
|
|
34
|
+
"insert into s.up values (1)",
|
|
35
|
+
"update s.up set a = 1",
|
|
36
|
+
"drop table s.up",
|
|
37
|
+
"create table s.x (a int)",
|
|
38
|
+
"truncate table s.up",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.mark.parametrize("dialect", DIALECTS)
|
|
43
|
+
@pytest.mark.parametrize("sql", LEGIT_READS)
|
|
44
|
+
def test_legitimate_reads_pass(sql: str, dialect: str | None) -> None:
|
|
45
|
+
"""Every legitimate read shape must survive the gate unchanged."""
|
|
46
|
+
ensure_single_read(sql, dialect=dialect) # must not raise
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.mark.parametrize("dialect", DIALECTS)
|
|
50
|
+
@pytest.mark.parametrize("sql", [PAREN_ESCAPE_DELETE, PAREN_ESCAPE_DROP])
|
|
51
|
+
def test_paren_balanced_escape_is_rejected(sql: str, dialect: str | None) -> None:
|
|
52
|
+
"""A read that balances the subquery wrap's parens must not reach the engine."""
|
|
53
|
+
with pytest.raises(ValueError):
|
|
54
|
+
ensure_single_read(sql, dialect=dialect)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@pytest.mark.parametrize("dialect", DIALECTS)
|
|
58
|
+
def test_stacked_statement_is_rejected(dialect: str | None) -> None:
|
|
59
|
+
"""A naively stacked second statement is rejected as more than one statement."""
|
|
60
|
+
with pytest.raises(ValueError, match="exactly one"):
|
|
61
|
+
ensure_single_read(STACKED, dialect=dialect)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@pytest.mark.parametrize("dialect", DIALECTS)
|
|
65
|
+
@pytest.mark.parametrize("sql", BARE_DML)
|
|
66
|
+
def test_bare_non_query_statements_are_rejected(sql: str, dialect: str | None) -> None:
|
|
67
|
+
"""A single statement that is not a read is rejected as not a query."""
|
|
68
|
+
with pytest.raises(ValueError, match="read query"):
|
|
69
|
+
ensure_single_read(sql, dialect=dialect)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@pytest.mark.parametrize("sql", [PAREN_ESCAPE_DELETE, STACKED, "delete from s.up"])
|
|
73
|
+
def test_rejection_message_names_the_offending_read(sql: str) -> None:
|
|
74
|
+
"""Every rejection must be actionable: it quotes the read that was rejected."""
|
|
75
|
+
with pytest.raises(ValueError) as excinfo:
|
|
76
|
+
ensure_single_read(sql, dialect="postgres")
|
|
77
|
+
message = str(excinfo.value)
|
|
78
|
+
assert "read" in message
|
|
79
|
+
assert sql[:30] in message
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_unparseable_read_is_rejected_as_valueerror_not_a_sqlglot_error() -> None:
|
|
83
|
+
"""Callers catch ValueError; a sqlglot ParseError must never escape the gate."""
|
|
84
|
+
with pytest.raises(ValueError):
|
|
85
|
+
ensure_single_read("select from from where", dialect="postgres")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_empty_read_is_rejected() -> None:
|
|
89
|
+
"""An empty read parses to zero statements, which is not exactly one."""
|
|
90
|
+
with pytest.raises(ValueError, match="exactly one"):
|
|
91
|
+
ensure_single_read(" ", dialect="postgres")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@pytest.mark.parametrize("sql", [
|
|
95
|
+
"delete from s.up where a in (" + ", ".join(str(i) for i in range(500)) + ")",
|
|
96
|
+
"select from from where " + ", ".join(str(i) for i in range(500)), # ParseError path
|
|
97
|
+
"select a from s.up; " * 200, # many-statements path
|
|
98
|
+
])
|
|
99
|
+
def test_long_read_is_truncated_in_the_message(sql: str) -> None:
|
|
100
|
+
"""Every rejection message is bounded: it rides the result block to the controller."""
|
|
101
|
+
with pytest.raises(ValueError) as excinfo:
|
|
102
|
+
ensure_single_read(sql, dialect="postgres")
|
|
103
|
+
assert len(str(excinfo.value)) < 600
|