continuo-engine-contract 0.7.2__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/.gitignore +2 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/PKG-INFO +15 -4
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/README.md +10 -2
- continuo_engine_contract-0.8.0/continuo_engine_contract/keys.py +87 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/port.py +98 -2
- continuo_engine_contract-0.8.0/continuo_engine_contract/testing/__init__.py +4 -0
- continuo_engine_contract-0.8.0/continuo_engine_contract/testing/conformance.py +306 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/pyproject.toml +8 -2
- continuo_engine_contract-0.8.0/tests/test_conformance_suite.py +168 -0
- continuo_engine_contract-0.8.0/tests/test_contract_keys.py +95 -0
- continuo_engine_contract-0.8.0/tests/test_contract_packaging.py +26 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/tests/test_port.py +122 -2
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/LICENSE +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/NOTICE +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/__init__.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/config.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/result.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/sql.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/types.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/tests/__init__.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/tests/test_config.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/tests/test_contract_types.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/tests/test_result.py +0 -0
- {continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/tests/test_sql.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: continuo-engine-contract
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: The contract a Continuo warehouse engine must satisfy: the WarehouseAdapter port and the result-block wire format.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -9,21 +9,26 @@ License-File: NOTICE
|
|
|
9
9
|
Classifier: Development Status :: 4 - Beta
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.14
|
|
11
11
|
Requires-Python: >=3.14
|
|
12
|
-
Requires-Dist: sqlglot==30.
|
|
12
|
+
Requires-Dist: sqlglot==30.17.0
|
|
13
|
+
Provides-Extra: testing
|
|
14
|
+
Requires-Dist: pyarrow==25.0.1; extra == 'testing'
|
|
15
|
+
Requires-Dist: pytest==9.1.1; extra == 'testing'
|
|
13
16
|
Description-Content-Type: text/markdown
|
|
14
17
|
|
|
15
18
|
# continuo-engine-contract
|
|
16
19
|
|
|
17
20
|
The contract shared by the Continuo python runtime — the node harness and the
|
|
18
|
-
validation runner alike — and every engine adapter library.
|
|
21
|
+
validation runner alike — and every engine adapter library. Seven modules:
|
|
19
22
|
|
|
20
23
|
| Module | What it holds |
|
|
21
24
|
|---|---|
|
|
22
|
-
| `continuo_engine_contract.port` | The `WarehouseAdapter` port — one port covering both the data plane (`fetch`/`ensure_table`/`load`) and validation (engine-specific empty-table DDL) — plus entry-point discovery |
|
|
25
|
+
| `continuo_engine_contract.port` | The `WarehouseAdapter` port — one port covering both the data plane (`fetch`/`ensure_table`/`load`) and validation (engine-specific empty-table DDL) — the opt-in write capabilities (`TableIntrospection`, `AppendCapable`, `UpsertCapable`, with `LiveColumn`), plus entry-point discovery |
|
|
26
|
+
| `continuo_engine_contract.keys` | `ensure_unique_key` and `DuplicateKeyError`: the primary-key check `UpsertCapable.upsert` runs before any engine sees the rows |
|
|
23
27
|
| `continuo_engine_contract.types` | The SQL column-type grammar that DDL-emitting methods validate against |
|
|
24
28
|
| `continuo_engine_contract.sql` | `ensure_single_read`, the parse-backed gate `check_binds` runs before it lets a declared read near an engine |
|
|
25
29
|
| `continuo_engine_contract.config` | `ensure_known_keys`, the fail-closed gate `build_empty_from_columns` runs before it lets a node's physical-layout `config` block reach DDL |
|
|
26
30
|
| `continuo_engine_contract.result` | The sentinel-framed result-block wire format the runner emits and the k8s-controller parses |
|
|
31
|
+
| `continuo_engine_contract.testing.conformance` | The pytest conformance suite for the write capabilities — `IntrospectionConformance`, `AppendConformance`, `UpsertConformance` — that an adapter's integration tests subclass, providing `adapter` and `schema` fixtures. Needs the `testing` extra |
|
|
27
32
|
|
|
28
33
|
Each `continuo-python-runtime-<engine>` library depends on this package and
|
|
29
34
|
implements the port.
|
|
@@ -54,6 +59,12 @@ implementations are required to run rather than reinvent:
|
|
|
54
59
|
`sql.ensure_single_read` before letting a declared read reach an engine, and
|
|
55
60
|
`config.ensure_known_keys` before building any DDL from a node's
|
|
56
61
|
physical-layout `config` block, to reject keys this engine doesn't recognize.
|
|
62
|
+
Three capability interfaces are opt-in on top of the port. An adapter that
|
|
63
|
+
also inherits `AppendCapable` (`table_columns`, `append`) or `UpsertCapable`
|
|
64
|
+
(`table_columns`, `_upsert`; the port's own `upsert` checks key uniqueness
|
|
65
|
+
first) can run `append` / `upsert` nodes; an adapter that inherits neither
|
|
66
|
+
still runs every `replace` node. `ensure_table` takes an optional
|
|
67
|
+
keyword-only `primary_key`, passed only for upsert nodes.
|
|
57
68
|
|
|
58
69
|
Because the port *is* the contract, this package is very unlikely to need
|
|
59
70
|
changes — new warehouse support is a new adapter package implementing the
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
# continuo-engine-contract
|
|
2
2
|
|
|
3
3
|
The contract shared by the Continuo python runtime — the node harness and the
|
|
4
|
-
validation runner alike — and every engine adapter library.
|
|
4
|
+
validation runner alike — and every engine adapter library. Seven modules:
|
|
5
5
|
|
|
6
6
|
| Module | What it holds |
|
|
7
7
|
|---|---|
|
|
8
|
-
| `continuo_engine_contract.port` | The `WarehouseAdapter` port — one port covering both the data plane (`fetch`/`ensure_table`/`load`) and validation (engine-specific empty-table DDL) — plus entry-point discovery |
|
|
8
|
+
| `continuo_engine_contract.port` | The `WarehouseAdapter` port — one port covering both the data plane (`fetch`/`ensure_table`/`load`) and validation (engine-specific empty-table DDL) — the opt-in write capabilities (`TableIntrospection`, `AppendCapable`, `UpsertCapable`, with `LiveColumn`), plus entry-point discovery |
|
|
9
|
+
| `continuo_engine_contract.keys` | `ensure_unique_key` and `DuplicateKeyError`: the primary-key check `UpsertCapable.upsert` runs before any engine sees the rows |
|
|
9
10
|
| `continuo_engine_contract.types` | The SQL column-type grammar that DDL-emitting methods validate against |
|
|
10
11
|
| `continuo_engine_contract.sql` | `ensure_single_read`, the parse-backed gate `check_binds` runs before it lets a declared read near an engine |
|
|
11
12
|
| `continuo_engine_contract.config` | `ensure_known_keys`, the fail-closed gate `build_empty_from_columns` runs before it lets a node's physical-layout `config` block reach DDL |
|
|
12
13
|
| `continuo_engine_contract.result` | The sentinel-framed result-block wire format the runner emits and the k8s-controller parses |
|
|
14
|
+
| `continuo_engine_contract.testing.conformance` | The pytest conformance suite for the write capabilities — `IntrospectionConformance`, `AppendConformance`, `UpsertConformance` — that an adapter's integration tests subclass, providing `adapter` and `schema` fixtures. Needs the `testing` extra |
|
|
13
15
|
|
|
14
16
|
Each `continuo-python-runtime-<engine>` library depends on this package and
|
|
15
17
|
implements the port.
|
|
@@ -40,6 +42,12 @@ implementations are required to run rather than reinvent:
|
|
|
40
42
|
`sql.ensure_single_read` before letting a declared read reach an engine, and
|
|
41
43
|
`config.ensure_known_keys` before building any DDL from a node's
|
|
42
44
|
physical-layout `config` block, to reject keys this engine doesn't recognize.
|
|
45
|
+
Three capability interfaces are opt-in on top of the port. An adapter that
|
|
46
|
+
also inherits `AppendCapable` (`table_columns`, `append`) or `UpsertCapable`
|
|
47
|
+
(`table_columns`, `_upsert`; the port's own `upsert` checks key uniqueness
|
|
48
|
+
first) can run `append` / `upsert` nodes; an adapter that inherits neither
|
|
49
|
+
still runs every `replace` node. `ensure_table` takes an optional
|
|
50
|
+
keyword-only `primary_key`, passed only for upsert nodes.
|
|
43
51
|
|
|
44
52
|
Because the port *is* the contract, this package is very unlikely to need
|
|
45
53
|
changes — new warehouse support is a new adapter package implementing the
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""The primary-key invariant every upsert relies on, checked once for every engine.
|
|
2
|
+
|
|
3
|
+
An upsert whose own rows repeat a key cannot be written consistently: DuckLake's
|
|
4
|
+
MERGE inserts both rows of a repeated new key and keeps the last row of a
|
|
5
|
+
repeated existing one, Trino fails only when two rows match one existing row,
|
|
6
|
+
and postgres's ON CONFLICT fails within one statement but keeps the last row
|
|
7
|
+
across statements. ``UpsertCapable.upsert`` therefore calls
|
|
8
|
+
:func:`ensure_unique_key` before any engine sees the rows.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
16
|
+
import pyarrow # type: ignore[import-untyped]
|
|
17
|
+
|
|
18
|
+
# How many repeated keys the error message names before summarizing the rest.
|
|
19
|
+
_SHOWN_KEYS = 5
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DuplicateKeyError(ValueError):
|
|
23
|
+
"""The rows to upsert repeat a primary key; nothing was written."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def ensure_unique_key(data: "pyarrow.Table", primary_key: Sequence[str]) -> None:
|
|
27
|
+
"""Raise :class:`DuplicateKeyError` if two rows of *data* share a *primary_key* value.
|
|
28
|
+
|
|
29
|
+
Groups the rows by the key columns. NULL counts as a value here; the
|
|
30
|
+
contract requires every key column to be ``nullable: false`` and the
|
|
31
|
+
harness's ``conform()`` rejects NULLs in such columns before this runs.
|
|
32
|
+
The message names the first repeated keys, in order of first appearance,
|
|
33
|
+
with their row counts. Only methods of *data* are called, so this module
|
|
34
|
+
needs no pyarrow import.
|
|
35
|
+
|
|
36
|
+
Raises:
|
|
37
|
+
ValueError: If *primary_key* is empty, names a column *data* lacks, or
|
|
38
|
+
includes a floating-point column (a warehouse treats ``0.0`` and
|
|
39
|
+
``-0.0`` as one key, which this check cannot see).
|
|
40
|
+
DuplicateKeyError: If a key value appears in more than one row.
|
|
41
|
+
"""
|
|
42
|
+
key = list(primary_key)
|
|
43
|
+
if not key:
|
|
44
|
+
raise ValueError("primary_key must name at least one column")
|
|
45
|
+
missing = [name for name in key if name not in data.column_names]
|
|
46
|
+
if missing:
|
|
47
|
+
raise ValueError(
|
|
48
|
+
f"primary key column(s) {missing} are not in the data's columns {data.column_names}"
|
|
49
|
+
)
|
|
50
|
+
floating = [
|
|
51
|
+
name
|
|
52
|
+
for name in key
|
|
53
|
+
if str(data.schema.field(name).type) in {"halffloat", "float", "double"}
|
|
54
|
+
]
|
|
55
|
+
if floating:
|
|
56
|
+
raise ValueError(
|
|
57
|
+
f"primary key column(s) {floating} are floating-point; floating-point "
|
|
58
|
+
"columns cannot be a primary key because the warehouse treats values like "
|
|
59
|
+
"0.0 and -0.0 as equal while this check would see them as distinct"
|
|
60
|
+
)
|
|
61
|
+
if data.num_rows < 2:
|
|
62
|
+
return
|
|
63
|
+
# Positional names: a key column called "count_all" must not collide with
|
|
64
|
+
# the aggregate's own output column.
|
|
65
|
+
positional = [f"k{index}" for index in range(len(key))]
|
|
66
|
+
counts = (
|
|
67
|
+
data.select(key)
|
|
68
|
+
.rename_columns(positional)
|
|
69
|
+
.group_by(positional, use_threads=False)
|
|
70
|
+
.aggregate([([], "count_all")])
|
|
71
|
+
)
|
|
72
|
+
if counts.num_rows == data.num_rows:
|
|
73
|
+
return
|
|
74
|
+
repeated = [row for row in counts.to_pylist() if row["count_all"] > 1]
|
|
75
|
+
shown = "; ".join(
|
|
76
|
+
f"({', '.join(str(row[name]) for name in positional)}) appears {row['count_all']} times"
|
|
77
|
+
for row in repeated[:_SHOWN_KEYS]
|
|
78
|
+
)
|
|
79
|
+
more = (
|
|
80
|
+
f"; and {len(repeated) - _SHOWN_KEYS} more repeated key(s)"
|
|
81
|
+
if len(repeated) > _SHOWN_KEYS
|
|
82
|
+
else ""
|
|
83
|
+
)
|
|
84
|
+
raise DuplicateKeyError(
|
|
85
|
+
f"primary key ({', '.join(key)}) is not unique in the rows to write: "
|
|
86
|
+
f"{shown}{more}; nothing was written"
|
|
87
|
+
)
|
{continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/port.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""The warehouse-engine adapter port
|
|
1
|
+
"""The warehouse-engine adapter port, its opt-in write capabilities, and discovery.
|
|
2
2
|
|
|
3
3
|
One port covers both roles an engine plays in Continuo:
|
|
4
4
|
|
|
@@ -9,6 +9,11 @@ One port covers both roles an engine plays in Continuo:
|
|
|
9
9
|
* **python-node data-plane I/O** — the runtime harness reads declared inputs and
|
|
10
10
|
writes a node's result (``fetch``, ``ensure_table``, ``load``).
|
|
11
11
|
|
|
12
|
+
Writing modes beyond replace are opt-in capabilities an adapter declares by
|
|
13
|
+
inheritance (``TableIntrospection``, ``AppendCapable``, ``UpsertCapable``): an
|
|
14
|
+
adapter that declares none still runs every ``replace`` node, and one that
|
|
15
|
+
declares a capability must implement all of it to be instantiated.
|
|
16
|
+
|
|
12
17
|
Both roles are served by the same connection to the same warehouse, so they are
|
|
13
18
|
one implementation per engine rather than two. Engine packages register a single
|
|
14
19
|
``continuo_engine.adapters`` entry point; each image installs exactly one, so
|
|
@@ -17,9 +22,13 @@ discovery — not configuration — selects the adapter.
|
|
|
17
22
|
from __future__ import annotations
|
|
18
23
|
|
|
19
24
|
from abc import ABC, abstractmethod
|
|
25
|
+
from collections.abc import Sequence
|
|
26
|
+
from dataclasses import dataclass
|
|
20
27
|
from importlib.metadata import entry_points
|
|
21
28
|
from typing import TYPE_CHECKING
|
|
22
29
|
|
|
30
|
+
from continuo_engine_contract.keys import ensure_unique_key
|
|
31
|
+
|
|
23
32
|
ENTRY_POINT_GROUP = "continuo_engine.adapters"
|
|
24
33
|
|
|
25
34
|
if TYPE_CHECKING: # pragma: no cover
|
|
@@ -126,7 +135,13 @@ class WarehouseAdapter(ABC):
|
|
|
126
135
|
|
|
127
136
|
@abstractmethod
|
|
128
137
|
def ensure_table(
|
|
129
|
-
self,
|
|
138
|
+
self,
|
|
139
|
+
schema: str,
|
|
140
|
+
table: str,
|
|
141
|
+
columns: list[dict],
|
|
142
|
+
*,
|
|
143
|
+
config: dict,
|
|
144
|
+
primary_key: Sequence[str] = (),
|
|
130
145
|
) -> None:
|
|
131
146
|
"""CREATE TABLE IF NOT EXISTS with the typed DDL compiled from columns.
|
|
132
147
|
|
|
@@ -146,6 +161,13 @@ class WarehouseAdapter(ABC):
|
|
|
146
161
|
recognize, and MUST validate config values before interpolating
|
|
147
162
|
anything into DDL — the same discipline
|
|
148
163
|
:meth:`build_empty_from_columns` follows.
|
|
164
|
+
|
|
165
|
+
*primary_key* names an upsert node's key columns, spelled exactly as in
|
|
166
|
+
*columns*. The harness passes it only for ``upsert`` nodes, so an
|
|
167
|
+
adapter that is not :class:`UpsertCapable` never receives it. An engine
|
|
168
|
+
that can enforce a key declares it on the table it creates (postgres:
|
|
169
|
+
``PRIMARY KEY``); an engine without constraints ignores it. Like
|
|
170
|
+
*config*, it applies only when this call creates the table.
|
|
149
171
|
"""
|
|
150
172
|
|
|
151
173
|
@abstractmethod
|
|
@@ -157,6 +179,80 @@ class WarehouseAdapter(ABC):
|
|
|
157
179
|
"""Release the underlying connection."""
|
|
158
180
|
|
|
159
181
|
|
|
182
|
+
@dataclass(frozen=True)
|
|
183
|
+
class LiveColumn:
|
|
184
|
+
"""One column of an existing table, as the engine reports it."""
|
|
185
|
+
|
|
186
|
+
name: str
|
|
187
|
+
nullable: bool
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class TableIntrospection(ABC):
|
|
191
|
+
"""Reads the shape of an existing table.
|
|
192
|
+
|
|
193
|
+
The base of every write capability: before an append or upsert the
|
|
194
|
+
harness compares the live table with the node's declared columns.
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
@abstractmethod
|
|
198
|
+
def table_columns(self, schema: str, table: str) -> list[LiveColumn]:
|
|
199
|
+
"""Columns of an existing table. Raises if the table does not exist.
|
|
200
|
+
|
|
201
|
+
Raises ``LookupError`` naming ``schema.table`` when it does not exist.
|
|
202
|
+
Column order is not part of the contract; callers compare by name.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
class AppendCapable(TableIntrospection):
|
|
207
|
+
"""An engine that can add rows to a table and keep the rows already there.
|
|
208
|
+
|
|
209
|
+
Declare it by inheritance, e.g. ``class MyAdapter(WarehouseAdapter,
|
|
210
|
+
AppendCapable)``; the harness runs a ``mode_type: append`` node only on an
|
|
211
|
+
adapter that is an instance of this class.
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
@abstractmethod
|
|
215
|
+
def append(self, schema: str, table: str, data: "pyarrow.Table") -> None:
|
|
216
|
+
"""Atomically insert every row of data. All rows or none.
|
|
217
|
+
|
|
218
|
+
The insert names *data*'s columns, so their order need not match the
|
|
219
|
+
table's. A 0-row *data* writes nothing.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class UpsertCapable(TableIntrospection):
|
|
224
|
+
"""An engine that can insert new keys and overwrite existing ones in one write.
|
|
225
|
+
|
|
226
|
+
Declare it by inheritance. :meth:`upsert` is a template method: the key
|
|
227
|
+
invariant is checked here, once for every engine, and the rows then go to
|
|
228
|
+
the engine's :meth:`_upsert`.
|
|
229
|
+
"""
|
|
230
|
+
|
|
231
|
+
def upsert(
|
|
232
|
+
self, schema: str, table: str, data: "pyarrow.Table", primary_key: Sequence[str]
|
|
233
|
+
) -> None:
|
|
234
|
+
"""Atomically insert rows with a new key and overwrite rows whose key exists.
|
|
235
|
+
|
|
236
|
+
Rows whose key is not in data are untouched. *primary_key* names
|
|
237
|
+
columns of *data* exactly as spelled there.
|
|
238
|
+
|
|
239
|
+
Raises:
|
|
240
|
+
DuplicateKeyError: If *data* repeats a key; nothing is written.
|
|
241
|
+
"""
|
|
242
|
+
ensure_unique_key(data, primary_key)
|
|
243
|
+
self._upsert(schema, table, data, primary_key)
|
|
244
|
+
|
|
245
|
+
@abstractmethod
|
|
246
|
+
def _upsert(
|
|
247
|
+
self, schema: str, table: str, data: "pyarrow.Table", primary_key: Sequence[str]
|
|
248
|
+
) -> None:
|
|
249
|
+
"""Engine-specific write. Keys in data are already unique.
|
|
250
|
+
|
|
251
|
+
All rows or none; a 0-row *data* writes nothing. When every column is
|
|
252
|
+
in the key there is nothing to overwrite: insert only the new keys.
|
|
253
|
+
"""
|
|
254
|
+
|
|
255
|
+
|
|
160
256
|
def discover_adapter() -> tuple[str, type[WarehouseAdapter]]:
|
|
161
257
|
"""Return ``(engine_name, adapter_class)`` from the single installed plugin.
|
|
162
258
|
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""Conformance tests for the engine contract's write capabilities.
|
|
2
|
+
|
|
3
|
+
One test class per capability. An adapter's integration suite subclasses the
|
|
4
|
+
classes for exactly the capabilities it declares and runs them against its
|
|
5
|
+
real engine; a bring-your-own adapter can do the same::
|
|
6
|
+
|
|
7
|
+
from continuo_engine_contract.testing.conformance import AppendConformance
|
|
8
|
+
|
|
9
|
+
class TestMyEngineAppend(AppendConformance):
|
|
10
|
+
pass
|
|
11
|
+
|
|
12
|
+
The subclass's module (or a conftest.py above it) provides two fixtures:
|
|
13
|
+
|
|
14
|
+
``adapter``
|
|
15
|
+
A connected adapter instance that declares the capability under test.
|
|
16
|
+
``schema``
|
|
17
|
+
A schema name the tests may create tables in (``ensure_table`` creates
|
|
18
|
+
it). The fixture drops it after the test.
|
|
19
|
+
|
|
20
|
+
Rows are read back through ``adapter.fetch`` with ANSI double-quoted
|
|
21
|
+
identifiers; an engine that quotes differently overrides
|
|
22
|
+
:meth:`ConformanceBase.read_rows`. Comparisons raise ``AssertionError`` with
|
|
23
|
+
both sides in the message, so no pytest assertion rewriting is needed.
|
|
24
|
+
|
|
25
|
+
Needs the ``testing`` extra: ``pip install "continuo-engine-contract[testing]"``.
|
|
26
|
+
"""
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import datetime
|
|
30
|
+
import uuid
|
|
31
|
+
from collections.abc import Iterable, Sequence
|
|
32
|
+
from decimal import Decimal
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
import pyarrow as pa # type: ignore[import-untyped]
|
|
36
|
+
import pytest
|
|
37
|
+
|
|
38
|
+
from continuo_engine_contract.keys import DuplicateKeyError
|
|
39
|
+
from continuo_engine_contract.port import LiveColumn
|
|
40
|
+
|
|
41
|
+
# More rows than one insert statement carries in any shipped adapter (postgres
|
|
42
|
+
# pages 1000 rows, trino batches 500), so an atomicity case spans statements.
|
|
43
|
+
BULK_ROWS = 1200
|
|
44
|
+
|
|
45
|
+
COLUMNS = [
|
|
46
|
+
{"name": "id", "type": "BIGINT", "nullable": False},
|
|
47
|
+
{"name": "name", "type": "VARCHAR(20)", "nullable": False},
|
|
48
|
+
{"name": "score", "type": "BIGINT", "nullable": True},
|
|
49
|
+
]
|
|
50
|
+
NAMES = ("id", "name", "score")
|
|
51
|
+
ARROW = pa.schema([("id", pa.int64()), ("name", pa.string()), ("score", pa.int64())])
|
|
52
|
+
|
|
53
|
+
# Every column in the key: an upsert has nothing to overwrite.
|
|
54
|
+
KEY_ONLY_COLUMNS = [
|
|
55
|
+
{"name": "a", "type": "BIGINT", "nullable": False},
|
|
56
|
+
{"name": "b", "type": "VARCHAR(20)", "nullable": False},
|
|
57
|
+
]
|
|
58
|
+
KEY_ONLY_ARROW = pa.schema([("a", pa.int64()), ("b", pa.string())])
|
|
59
|
+
|
|
60
|
+
# Names that need quoting: a reserved word, mixed case, a percent sign.
|
|
61
|
+
AWKWARD_COLUMNS = [
|
|
62
|
+
{"name": "order", "type": "BIGINT", "nullable": False},
|
|
63
|
+
{"name": "Rate", "type": "VARCHAR(20)", "nullable": False},
|
|
64
|
+
{"name": "pct%", "type": "BIGINT", "nullable": True},
|
|
65
|
+
]
|
|
66
|
+
AWKWARD_NAMES = ("order", "Rate", "pct%")
|
|
67
|
+
AWKWARD_ARROW = pa.schema([("order", pa.int64()), ("Rate", pa.string()), ("pct%", pa.int64())])
|
|
68
|
+
|
|
69
|
+
# The shape of the first upsert node (daily FX rates).
|
|
70
|
+
FX_COLUMNS = [
|
|
71
|
+
{"name": "rate_date", "type": "DATE", "nullable": False},
|
|
72
|
+
{"name": "base", "type": "VARCHAR(3)", "nullable": False},
|
|
73
|
+
{"name": "rate", "type": "DECIMAL(12,6)", "nullable": True},
|
|
74
|
+
]
|
|
75
|
+
FX_NAMES = ("rate_date", "base", "rate")
|
|
76
|
+
FX_ARROW = pa.schema(
|
|
77
|
+
[("rate_date", pa.date32()), ("base", pa.string()), ("rate", pa.decimal128(12, 6))]
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def quote(name: str) -> str:
|
|
82
|
+
"""*name* as an ANSI double-quoted identifier."""
|
|
83
|
+
return '"' + name.replace('"', '""') + '"'
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def table_of(schema: Any, rows: Sequence[Sequence[Any]]) -> Any:
|
|
87
|
+
"""An Arrow table of *rows* (tuples in *schema*'s column order)."""
|
|
88
|
+
return pa.Table.from_pylist([dict(zip(schema.names, row)) for row in rows], schema=schema)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def bulk_rows(start: int) -> list[tuple[Any, ...]]:
|
|
92
|
+
"""BULK_ROWS valid ``ARROW`` rows with ids from *start*, then one whose NOT NULL name is NULL."""
|
|
93
|
+
rows: list[tuple[Any, ...]] = [(i, f"n{i}", i) for i in range(start, start + BULK_ROWS)]
|
|
94
|
+
rows.append((start + BULK_ROWS, None, 0))
|
|
95
|
+
return rows
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def canonical(rows: Iterable[Sequence[Any]]) -> list[tuple[Any, ...]]:
|
|
99
|
+
"""*rows* as tuples in a deterministic order, for comparing a table's contents."""
|
|
100
|
+
return sorted((tuple(row) for row in rows), key=repr)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class ConformanceBase:
|
|
104
|
+
"""Helpers shared by the capability classes (fixtures: see the module docstring)."""
|
|
105
|
+
|
|
106
|
+
def new_table(
|
|
107
|
+
self,
|
|
108
|
+
adapter: Any,
|
|
109
|
+
schema: str,
|
|
110
|
+
columns: list[dict[str, Any]],
|
|
111
|
+
primary_key: Sequence[str] = (),
|
|
112
|
+
) -> str:
|
|
113
|
+
"""Create a fresh table through ``ensure_table`` and return its name."""
|
|
114
|
+
table = f"conf_{uuid.uuid4().hex[:12]}"
|
|
115
|
+
if primary_key:
|
|
116
|
+
adapter.ensure_table(schema, table, columns, config={}, primary_key=primary_key)
|
|
117
|
+
else:
|
|
118
|
+
adapter.ensure_table(schema, table, columns, config={})
|
|
119
|
+
return table
|
|
120
|
+
|
|
121
|
+
def read_rows(
|
|
122
|
+
self, adapter: Any, schema: str, table: str, names: Sequence[str]
|
|
123
|
+
) -> list[tuple[Any, ...]]:
|
|
124
|
+
"""Every row of ``schema.table`` as a tuple in *names* order, canonically sorted."""
|
|
125
|
+
select = ", ".join(f"{quote(name)} AS c{index}" for index, name in enumerate(names))
|
|
126
|
+
data = adapter.fetch(f"SELECT {select} FROM {quote(schema)}.{quote(table)}")
|
|
127
|
+
return canonical(
|
|
128
|
+
tuple(row[f"c{index}"] for index in range(len(names))) for row in data.to_pylist()
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
def expect_rows(
|
|
132
|
+
self,
|
|
133
|
+
adapter: Any,
|
|
134
|
+
schema: str,
|
|
135
|
+
table: str,
|
|
136
|
+
names: Sequence[str],
|
|
137
|
+
expected: Iterable[Sequence[Any]],
|
|
138
|
+
) -> None:
|
|
139
|
+
"""Fail unless ``schema.table`` holds exactly *expected*, in any order."""
|
|
140
|
+
actual = self.read_rows(adapter, schema, table, names)
|
|
141
|
+
wanted = canonical(expected)
|
|
142
|
+
if actual != wanted:
|
|
143
|
+
raise AssertionError(f"{schema}.{table} holds {actual}, expected {wanted}")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class IntrospectionConformance(ConformanceBase):
|
|
147
|
+
"""For an adapter that is a ``TableIntrospection`` (every write capability is)."""
|
|
148
|
+
|
|
149
|
+
def test_table_columns_reports_names_and_nullability(self, adapter, schema):
|
|
150
|
+
table = self.new_table(adapter, schema, COLUMNS)
|
|
151
|
+
reported = sorted(adapter.table_columns(schema, table), key=lambda column: column.name)
|
|
152
|
+
expected = [LiveColumn("id", False), LiveColumn("name", False), LiveColumn("score", True)]
|
|
153
|
+
if reported != expected:
|
|
154
|
+
raise AssertionError(f"table_columns reported {reported}, expected {expected}")
|
|
155
|
+
|
|
156
|
+
def test_table_columns_of_a_missing_table_raises_lookup_error(self, adapter, schema):
|
|
157
|
+
self.new_table(adapter, schema, COLUMNS) # the schema exists; only the table is missing
|
|
158
|
+
with pytest.raises(LookupError):
|
|
159
|
+
adapter.table_columns(schema, f"conf_missing_{uuid.uuid4().hex[:8]}")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class AppendConformance(ConformanceBase):
|
|
163
|
+
"""For an adapter that is an ``AppendCapable``."""
|
|
164
|
+
|
|
165
|
+
def test_append_into_an_empty_table(self, adapter, schema):
|
|
166
|
+
table = self.new_table(adapter, schema, COLUMNS)
|
|
167
|
+
adapter.append(schema, table, table_of(ARROW, [(1, "a", 10), (2, "b", None)]))
|
|
168
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10), (2, "b", None)])
|
|
169
|
+
|
|
170
|
+
def test_two_appends_accumulate(self, adapter, schema):
|
|
171
|
+
table = self.new_table(adapter, schema, COLUMNS)
|
|
172
|
+
adapter.append(schema, table, table_of(ARROW, [(1, "a", 10)]))
|
|
173
|
+
adapter.append(schema, table, table_of(ARROW, [(2, "b", 20), (3, "c", 30)]))
|
|
174
|
+
self.expect_rows(
|
|
175
|
+
adapter, schema, table, NAMES, [(1, "a", 10), (2, "b", 20), (3, "c", 30)]
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
def test_append_of_zero_rows_writes_nothing(self, adapter, schema):
|
|
179
|
+
table = self.new_table(adapter, schema, COLUMNS)
|
|
180
|
+
adapter.append(schema, table, table_of(ARROW, [(1, "a", 10)]))
|
|
181
|
+
adapter.append(schema, table, table_of(ARROW, []))
|
|
182
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10)])
|
|
183
|
+
|
|
184
|
+
def test_append_is_all_rows_or_none(self, adapter, schema):
|
|
185
|
+
"""More rows than one insert statement carries; the last violates NOT NULL."""
|
|
186
|
+
table = self.new_table(adapter, schema, COLUMNS)
|
|
187
|
+
adapter.append(schema, table, table_of(ARROW, [(1, "a", 10)]))
|
|
188
|
+
with pytest.raises(Exception):
|
|
189
|
+
adapter.append(schema, table, table_of(ARROW, bulk_rows(100)))
|
|
190
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10)])
|
|
191
|
+
|
|
192
|
+
def test_append_names_its_columns(self, adapter, schema):
|
|
193
|
+
"""The data's column order need not match the table's."""
|
|
194
|
+
table = self.new_table(adapter, schema, COLUMNS)
|
|
195
|
+
reordered = pa.table({
|
|
196
|
+
"score": pa.array([10], pa.int64()),
|
|
197
|
+
"name": pa.array(["a"], pa.string()),
|
|
198
|
+
"id": pa.array([1], pa.int64()),
|
|
199
|
+
})
|
|
200
|
+
adapter.append(schema, table, reordered)
|
|
201
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10)])
|
|
202
|
+
|
|
203
|
+
def test_append_quotes_awkward_identifiers(self, adapter, schema):
|
|
204
|
+
table = self.new_table(adapter, schema, AWKWARD_COLUMNS)
|
|
205
|
+
adapter.append(schema, table, table_of(AWKWARD_ARROW, [(1, "x", 5), (2, "y", None)]))
|
|
206
|
+
self.expect_rows(adapter, schema, table, AWKWARD_NAMES, [(1, "x", 5), (2, "y", None)])
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class UpsertConformance(ConformanceBase):
|
|
210
|
+
"""For an adapter that is an ``UpsertCapable``. Tables are made by ``ensure_table`` with the key."""
|
|
211
|
+
|
|
212
|
+
def test_upsert_inserts_new_keys_into_a_table_created_in_the_same_run(self, adapter, schema):
|
|
213
|
+
table = self.new_table(adapter, schema, COLUMNS, primary_key=["id"])
|
|
214
|
+
adapter.upsert(schema, table, table_of(ARROW, [(1, "a", 10), (2, "b", None)]), ["id"])
|
|
215
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10), (2, "b", None)])
|
|
216
|
+
|
|
217
|
+
def test_upsert_overwrites_existing_keys_and_leaves_other_rows_untouched(self, adapter, schema):
|
|
218
|
+
table = self.new_table(adapter, schema, COLUMNS, primary_key=["id"])
|
|
219
|
+
adapter.upsert(
|
|
220
|
+
schema, table, table_of(ARROW, [(1, "a", 10), (2, "b", 20), (3, "c", 30)]), ["id"]
|
|
221
|
+
)
|
|
222
|
+
adapter.upsert(schema, table, table_of(ARROW, [(2, "B", None), (4, "d", 40)]), ["id"])
|
|
223
|
+
self.expect_rows(
|
|
224
|
+
adapter, schema, table, NAMES,
|
|
225
|
+
[(1, "a", 10), (2, "B", None), (3, "c", 30), (4, "d", 40)],
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
def test_upsert_rerun_is_idempotent(self, adapter, schema):
|
|
229
|
+
table = self.new_table(adapter, schema, COLUMNS, primary_key=["id"])
|
|
230
|
+
rows = [(1, "a", 10), (2, "b", 20)]
|
|
231
|
+
adapter.upsert(schema, table, table_of(ARROW, rows), ["id"])
|
|
232
|
+
adapter.upsert(schema, table, table_of(ARROW, rows), ["id"])
|
|
233
|
+
self.expect_rows(adapter, schema, table, NAMES, rows)
|
|
234
|
+
|
|
235
|
+
def test_upsert_with_a_repeated_key_raises_and_writes_nothing(self, adapter, schema):
|
|
236
|
+
table = self.new_table(adapter, schema, COLUMNS, primary_key=["id"])
|
|
237
|
+
adapter.upsert(schema, table, table_of(ARROW, [(1, "a", 10)]), ["id"])
|
|
238
|
+
with pytest.raises(DuplicateKeyError):
|
|
239
|
+
adapter.upsert(
|
|
240
|
+
schema, table, table_of(ARROW, [(1, "x", 1), (5, "y", 1), (5, "z", 2)]), ["id"]
|
|
241
|
+
)
|
|
242
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10)])
|
|
243
|
+
|
|
244
|
+
def test_upsert_when_every_column_is_in_the_key(self, adapter, schema):
|
|
245
|
+
table = self.new_table(adapter, schema, KEY_ONLY_COLUMNS, primary_key=["a", "b"])
|
|
246
|
+
adapter.upsert(schema, table, table_of(KEY_ONLY_ARROW, [(1, "x"), (1, "y")]), ["a", "b"])
|
|
247
|
+
adapter.upsert(schema, table, table_of(KEY_ONLY_ARROW, [(1, "x"), (2, "x")]), ["a", "b"])
|
|
248
|
+
self.expect_rows(adapter, schema, table, ("a", "b"), [(1, "x"), (1, "y"), (2, "x")])
|
|
249
|
+
|
|
250
|
+
def test_upsert_is_all_rows_or_none(self, adapter, schema):
|
|
251
|
+
"""Updates of existing keys plus more rows than one statement; the last violates NOT NULL."""
|
|
252
|
+
table = self.new_table(adapter, schema, COLUMNS, primary_key=["id"])
|
|
253
|
+
adapter.upsert(schema, table, table_of(ARROW, [(1, "a", 10), (2, "b", 20)]), ["id"])
|
|
254
|
+
rows = [(1, "changed", 1), (2, "changed", 2)] + bulk_rows(100)
|
|
255
|
+
with pytest.raises(Exception):
|
|
256
|
+
adapter.upsert(schema, table, table_of(ARROW, rows), ["id"])
|
|
257
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10), (2, "b", 20)])
|
|
258
|
+
|
|
259
|
+
def test_upsert_of_zero_rows_writes_nothing(self, adapter, schema):
|
|
260
|
+
table = self.new_table(adapter, schema, COLUMNS, primary_key=["id"])
|
|
261
|
+
adapter.upsert(schema, table, table_of(ARROW, [(1, "a", 10)]), ["id"])
|
|
262
|
+
adapter.upsert(schema, table, table_of(ARROW, []), ["id"])
|
|
263
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "a", 10)])
|
|
264
|
+
|
|
265
|
+
def test_upsert_names_its_columns(self, adapter, schema):
|
|
266
|
+
table = self.new_table(adapter, schema, COLUMNS, primary_key=["id"])
|
|
267
|
+
adapter.upsert(schema, table, table_of(ARROW, [(1, "a", 10)]), ["id"])
|
|
268
|
+
reordered = pa.table({
|
|
269
|
+
"score": pa.array([11, 20], pa.int64()),
|
|
270
|
+
"name": pa.array(["A", "b"], pa.string()),
|
|
271
|
+
"id": pa.array([1, 2], pa.int64()),
|
|
272
|
+
})
|
|
273
|
+
adapter.upsert(schema, table, reordered, ["id"])
|
|
274
|
+
self.expect_rows(adapter, schema, table, NAMES, [(1, "A", 11), (2, "b", 20)])
|
|
275
|
+
|
|
276
|
+
def test_upsert_quotes_awkward_identifiers(self, adapter, schema):
|
|
277
|
+
key = ["order", "Rate"]
|
|
278
|
+
table = self.new_table(adapter, schema, AWKWARD_COLUMNS, primary_key=key)
|
|
279
|
+
adapter.upsert(schema, table, table_of(AWKWARD_ARROW, [(1, "x", 5), (1, "y", 6)]), key)
|
|
280
|
+
adapter.upsert(schema, table, table_of(AWKWARD_ARROW, [(1, "x", 50), (2, "x", None)]), key)
|
|
281
|
+
self.expect_rows(
|
|
282
|
+
adapter, schema, table, AWKWARD_NAMES, [(1, "x", 50), (1, "y", 6), (2, "x", None)]
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def test_upsert_with_a_date_and_text_key_and_a_decimal_value(self, adapter, schema):
|
|
286
|
+
key = ["rate_date", "base"]
|
|
287
|
+
day1, day2 = datetime.date(2026, 10, 8), datetime.date(2026, 10, 9)
|
|
288
|
+
table = self.new_table(adapter, schema, FX_COLUMNS, primary_key=key)
|
|
289
|
+
adapter.upsert(
|
|
290
|
+
schema, table,
|
|
291
|
+
table_of(FX_ARROW, [(day1, "EUR", Decimal("1.081200")), (day1, "GBP", Decimal("0.861000"))]),
|
|
292
|
+
key,
|
|
293
|
+
)
|
|
294
|
+
adapter.upsert(
|
|
295
|
+
schema, table,
|
|
296
|
+
table_of(FX_ARROW, [(day1, "EUR", Decimal("1.090000")), (day2, "EUR", None)]),
|
|
297
|
+
key,
|
|
298
|
+
)
|
|
299
|
+
self.expect_rows(
|
|
300
|
+
adapter, schema, table, FX_NAMES,
|
|
301
|
+
[
|
|
302
|
+
(day1, "EUR", Decimal("1.090000")),
|
|
303
|
+
(day1, "GBP", Decimal("0.861000")),
|
|
304
|
+
(day2, "EUR", None),
|
|
305
|
+
],
|
|
306
|
+
)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[project]
|
|
2
|
-
version = "0.
|
|
2
|
+
version = "0.8.0"
|
|
3
3
|
name = "continuo-engine-contract"
|
|
4
4
|
description = "The contract a Continuo warehouse engine must satisfy: the WarehouseAdapter port and the result-block wire format."
|
|
5
5
|
authors = [{ name = "Simone Carolini" }]
|
|
@@ -7,12 +7,18 @@ readme = "README.md"
|
|
|
7
7
|
requires-python = ">= 3.14"
|
|
8
8
|
license = "Apache-2.0"
|
|
9
9
|
license-files = ["LICENSE", "NOTICE"]
|
|
10
|
-
dependencies = ["sqlglot==30.
|
|
10
|
+
dependencies = ["sqlglot==30.17.0"]
|
|
11
11
|
classifiers = [
|
|
12
12
|
"Development Status :: 4 - Beta",
|
|
13
13
|
"Programming Language :: Python :: 3.14",
|
|
14
14
|
]
|
|
15
15
|
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
# The conformance suite (continuo_engine_contract.testing.conformance) for adapter test
|
|
18
|
+
# suites, this repo's and bring-your-own: pytest, plus pyarrow to build the rows.
|
|
19
|
+
# Pinned exactly, like every in-repo dependency.
|
|
20
|
+
testing = ["pytest==9.1.1", "pyarrow==25.0.1"]
|
|
21
|
+
|
|
16
22
|
[build-system]
|
|
17
23
|
requires = ["hatchling"]
|
|
18
24
|
build-backend = "hatchling.build"
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""The conformance suite, run against an in-memory reference adapter.
|
|
2
|
+
|
|
3
|
+
Proves the suite's own cases agree with each other before any engine runs them,
|
|
4
|
+
shows a bring-your-own adapter how to use it, and checks with two broken
|
|
5
|
+
adapters that the suite really catches a non-atomic append and an upsert that
|
|
6
|
+
loses rows.
|
|
7
|
+
"""
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from continuo_engine_contract.port import (
|
|
11
|
+
AppendCapable,
|
|
12
|
+
LiveColumn,
|
|
13
|
+
UpsertCapable,
|
|
14
|
+
WarehouseAdapter,
|
|
15
|
+
)
|
|
16
|
+
from continuo_engine_contract.testing.conformance import (
|
|
17
|
+
AppendConformance,
|
|
18
|
+
IntrospectionConformance,
|
|
19
|
+
UpsertConformance,
|
|
20
|
+
canonical,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _MemoryWarehouse(WarehouseAdapter, AppendCapable, UpsertCapable):
|
|
25
|
+
"""A reference adapter over Python lists: NOT NULL enforced, every write all-or-nothing."""
|
|
26
|
+
|
|
27
|
+
def __init__(self):
|
|
28
|
+
self.tables: dict[tuple[str, str], dict] = {}
|
|
29
|
+
|
|
30
|
+
@classmethod
|
|
31
|
+
def required_env(cls):
|
|
32
|
+
return []
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def from_env(cls):
|
|
36
|
+
return cls()
|
|
37
|
+
|
|
38
|
+
def ensure_schema(self, schema):
|
|
39
|
+
...
|
|
40
|
+
|
|
41
|
+
def drop_schema(self, schema):
|
|
42
|
+
for key in [key for key in self.tables if key[0] == schema]:
|
|
43
|
+
del self.tables[key]
|
|
44
|
+
|
|
45
|
+
def build_empty_from_sql(self, schema, table, compiled_sql):
|
|
46
|
+
...
|
|
47
|
+
|
|
48
|
+
def clone_empty_from_prod(self, candidate_schema, prod_schema, table):
|
|
49
|
+
...
|
|
50
|
+
|
|
51
|
+
def build_empty_from_columns(self, schema, table, columns, config):
|
|
52
|
+
...
|
|
53
|
+
|
|
54
|
+
def check_binds(self, sql):
|
|
55
|
+
...
|
|
56
|
+
|
|
57
|
+
def fetch(self, sql):
|
|
58
|
+
raise AssertionError("the memory adapter is read through read_rows, never fetch")
|
|
59
|
+
|
|
60
|
+
def ensure_table(self, schema, table, columns, *, config, primary_key=()):
|
|
61
|
+
self.tables.setdefault(
|
|
62
|
+
(schema, table), {"columns": [dict(c) for c in columns], "rows": []}
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def load(self, schema, table, data):
|
|
66
|
+
self.tables[(schema, table)]["rows"] = self._rows(schema, table, data.to_pylist())
|
|
67
|
+
|
|
68
|
+
def close(self):
|
|
69
|
+
...
|
|
70
|
+
|
|
71
|
+
def table_columns(self, schema, table):
|
|
72
|
+
if (schema, table) not in self.tables:
|
|
73
|
+
raise LookupError(f"table {schema}.{table} does not exist")
|
|
74
|
+
return [
|
|
75
|
+
LiveColumn(c["name"], c.get("nullable", True))
|
|
76
|
+
for c in self.tables[(schema, table)]["columns"]
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
def _rows(self, schema, table, rows):
|
|
80
|
+
"""*rows* projected onto the table's columns; ValueError on a NULL in a NOT NULL column."""
|
|
81
|
+
columns = self.tables[(schema, table)]["columns"]
|
|
82
|
+
for row in rows:
|
|
83
|
+
for column in columns:
|
|
84
|
+
if not column.get("nullable", True) and row.get(column["name"]) is None:
|
|
85
|
+
raise ValueError(f"NOT NULL violated: {column['name']}")
|
|
86
|
+
return [{c["name"]: row.get(c["name"]) for c in columns} for row in rows]
|
|
87
|
+
|
|
88
|
+
def append(self, schema, table, data):
|
|
89
|
+
rows = self._rows(schema, table, data.to_pylist()) # validate everything first
|
|
90
|
+
self.tables[(schema, table)]["rows"].extend(rows)
|
|
91
|
+
|
|
92
|
+
def _upsert(self, schema, table, data, primary_key):
|
|
93
|
+
rows = self._rows(schema, table, data.to_pylist())
|
|
94
|
+
stored = self.tables[(schema, table)]["rows"]
|
|
95
|
+
index = {tuple(row[k] for k in primary_key): i for i, row in enumerate(stored)}
|
|
96
|
+
for row in rows:
|
|
97
|
+
key = tuple(row[k] for k in primary_key)
|
|
98
|
+
if key in index:
|
|
99
|
+
stored[index[key]] = row
|
|
100
|
+
else:
|
|
101
|
+
index[key] = len(stored)
|
|
102
|
+
stored.append(row)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class _LeakyAppendWarehouse(_MemoryWarehouse):
|
|
106
|
+
"""Appends row by row, so a failing row leaves the earlier ones behind."""
|
|
107
|
+
|
|
108
|
+
def append(self, schema, table, data):
|
|
109
|
+
for row in data.to_pylist():
|
|
110
|
+
self.tables[(schema, table)]["rows"].extend(self._rows(schema, table, [row]))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class _ReplacingUpsertWarehouse(_MemoryWarehouse):
|
|
114
|
+
"""Upserts by replacing the whole table, losing rows whose key is not in the data."""
|
|
115
|
+
|
|
116
|
+
def _upsert(self, schema, table, data, primary_key):
|
|
117
|
+
self.tables[(schema, table)]["rows"] = self._rows(schema, table, data.to_pylist())
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _ReadsMemory:
|
|
121
|
+
"""Reads the memory adapter's rows directly instead of through fetch."""
|
|
122
|
+
|
|
123
|
+
def read_rows(self, adapter, schema, table, names):
|
|
124
|
+
rows = adapter.tables[(schema, table)]["rows"]
|
|
125
|
+
return canonical(tuple(row[name] for name in names) for row in rows)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@pytest.fixture
|
|
129
|
+
def adapter():
|
|
130
|
+
return _MemoryWarehouse()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@pytest.fixture
|
|
134
|
+
def schema(adapter):
|
|
135
|
+
yield "conf"
|
|
136
|
+
adapter.drop_schema("conf")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class TestMemoryIntrospection(_ReadsMemory, IntrospectionConformance):
|
|
140
|
+
pass
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class TestMemoryAppend(_ReadsMemory, AppendConformance):
|
|
144
|
+
pass
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class TestMemoryUpsert(_ReadsMemory, UpsertConformance):
|
|
148
|
+
pass
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class _MemoryAppendCase(_ReadsMemory, AppendConformance):
|
|
152
|
+
pass
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class _MemoryUpsertCase(_ReadsMemory, UpsertConformance):
|
|
156
|
+
pass
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def test_the_suite_catches_a_non_atomic_append():
|
|
160
|
+
with pytest.raises(AssertionError, match="holds"):
|
|
161
|
+
_MemoryAppendCase().test_append_is_all_rows_or_none(_LeakyAppendWarehouse(), "conf")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def test_the_suite_catches_an_upsert_that_loses_other_rows():
|
|
165
|
+
with pytest.raises(AssertionError, match="holds"):
|
|
166
|
+
_MemoryUpsertCase().test_upsert_overwrites_existing_keys_and_leaves_other_rows_untouched(
|
|
167
|
+
_ReplacingUpsertWarehouse(), "conf"
|
|
168
|
+
)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""ensure_unique_key: the primary-key invariant every upsert checks first."""
|
|
2
|
+
import datetime
|
|
3
|
+
from decimal import Decimal
|
|
4
|
+
|
|
5
|
+
import pyarrow as pa
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from continuo_engine_contract.keys import DuplicateKeyError, ensure_unique_key
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_duplicate_key_error_is_a_value_error():
|
|
12
|
+
assert issubclass(DuplicateKeyError, ValueError)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_a_unique_single_column_key_passes():
|
|
16
|
+
ensure_unique_key(pa.table({"id": [1, 2, 3], "v": ["a", "a", "a"]}), ["id"])
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_a_unique_composite_key_passes():
|
|
20
|
+
ensure_unique_key(pa.table({"d": [1, 1, 2], "b": ["EUR", "USD", "EUR"]}), ("d", "b"))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_empty_and_single_row_tables_pass():
|
|
24
|
+
ensure_unique_key(pa.table({"id": pa.array([], pa.int64())}), ["id"])
|
|
25
|
+
ensure_unique_key(pa.table({"id": [1]}), ["id"])
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_a_repeated_key_raises_naming_the_key_and_its_count():
|
|
29
|
+
data = pa.table({
|
|
30
|
+
"rate_date": [datetime.date(2026, 10, 8)] * 2 + [datetime.date(2026, 10, 9)],
|
|
31
|
+
"base": ["EUR", "EUR", "EUR"],
|
|
32
|
+
"rate": pa.array([Decimal("1.1")] * 3, pa.decimal128(12, 6)),
|
|
33
|
+
})
|
|
34
|
+
with pytest.raises(
|
|
35
|
+
DuplicateKeyError,
|
|
36
|
+
match=(
|
|
37
|
+
r"^primary key \(rate_date, base\) is not unique in the rows to write: "
|
|
38
|
+
r"\(2026-10-08, EUR\) appears 2 times; nothing was written$"
|
|
39
|
+
),
|
|
40
|
+
):
|
|
41
|
+
ensure_unique_key(data, ["rate_date", "base"])
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_repeated_keys_are_listed_in_order_of_first_appearance_and_capped_at_five():
|
|
45
|
+
ids = [7, 7, 3, 3, 3] + [n for n in range(10, 16) for _ in range(2)]
|
|
46
|
+
with pytest.raises(DuplicateKeyError) as caught:
|
|
47
|
+
ensure_unique_key(pa.table({"id": ids}), ["id"])
|
|
48
|
+
message = str(caught.value)
|
|
49
|
+
assert message.index("(7) appears 2 times") < message.index("(3) appears 3 times")
|
|
50
|
+
assert "(12) appears 2 times" in message
|
|
51
|
+
assert "(13) appears" not in message
|
|
52
|
+
assert "; and 3 more repeated key(s); nothing was written" in message
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_a_key_column_named_like_the_aggregate_still_works():
|
|
56
|
+
with pytest.raises(DuplicateKeyError, match=r"\(1\) appears 2 times"):
|
|
57
|
+
ensure_unique_key(pa.table({"count_all": [1, 1]}), ["count_all"])
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_an_empty_key_is_rejected():
|
|
61
|
+
with pytest.raises(ValueError, match="at least one column"):
|
|
62
|
+
ensure_unique_key(pa.table({"id": [1]}), [])
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_a_key_column_missing_from_the_data_is_rejected():
|
|
66
|
+
with pytest.raises(ValueError, match=r"primary key column\(s\) \['quote'\]") as caught:
|
|
67
|
+
ensure_unique_key(pa.table({"base": ["EUR"]}), ["base", "quote"])
|
|
68
|
+
assert type(caught.value) is ValueError
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_a_floating_point_key_is_rejected():
|
|
72
|
+
# Arrow groups 0.0 and -0.0 as distinct; a warehouse treats them as one key.
|
|
73
|
+
data = pa.table({"id": pa.array([0.0, -0.0], pa.float64())})
|
|
74
|
+
with pytest.raises(ValueError, match="floating-point") as caught:
|
|
75
|
+
ensure_unique_key(data, ["id"])
|
|
76
|
+
assert type(caught.value) is ValueError
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_a_float32_key_is_rejected():
|
|
80
|
+
data = pa.table({"id": pa.array([1.0, 2.0], pa.float32())})
|
|
81
|
+
with pytest.raises(ValueError, match="floating-point") as caught:
|
|
82
|
+
ensure_unique_key(data, ["id"])
|
|
83
|
+
assert type(caught.value) is ValueError
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_a_composite_key_with_a_float_column_is_rejected():
|
|
87
|
+
data = pa.table({"base": ["EUR", "USD"], "rate": pa.array([1.5, 2.5], pa.float64())})
|
|
88
|
+
with pytest.raises(ValueError, match=r"column\(s\) \['rate'\] are floating-point") as caught:
|
|
89
|
+
ensure_unique_key(data, ["base", "rate"])
|
|
90
|
+
assert type(caught.value) is ValueError
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_a_decimal_key_is_allowed():
|
|
94
|
+
data = pa.table({"k": pa.array([Decimal("1.0"), Decimal("2.0")], pa.decimal128(12, 6))})
|
|
95
|
+
ensure_unique_key(data, ["k"])
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""The published wheel ships the conformance suite and declares the testing extra."""
|
|
2
|
+
import re
|
|
3
|
+
import subprocess
|
|
4
|
+
import zipfile
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
CONTRACT_ROOT = Path(__file__).resolve().parent.parent # contract/
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_wheel_ships_the_conformance_suite_and_the_testing_extra(tmp_path):
|
|
11
|
+
subprocess.run(
|
|
12
|
+
["uv", "build", "--wheel", "--out-dir", str(tmp_path), str(CONTRACT_ROOT)],
|
|
13
|
+
check=True,
|
|
14
|
+
capture_output=True,
|
|
15
|
+
)
|
|
16
|
+
(wheel,) = tmp_path.glob("continuo_engine_contract-*.whl")
|
|
17
|
+
with zipfile.ZipFile(wheel) as archive:
|
|
18
|
+
names = set(archive.namelist())
|
|
19
|
+
(metadata_name,) = [n for n in names if n.endswith(".dist-info/METADATA")]
|
|
20
|
+
metadata = archive.read(metadata_name).decode()
|
|
21
|
+
assert "continuo_engine_contract/testing/__init__.py" in names
|
|
22
|
+
assert "continuo_engine_contract/testing/conformance.py" in names
|
|
23
|
+
assert "continuo_engine_contract/keys.py" in names
|
|
24
|
+
assert "Provides-Extra: testing" in metadata
|
|
25
|
+
assert re.search(r"Requires-Dist: pytest==9\.1\.1; extra == ['\"]testing['\"]", metadata)
|
|
26
|
+
assert re.search(r"Requires-Dist: pyarrow==25\.0\.1; extra == ['\"]testing['\"]", metadata)
|
|
@@ -5,14 +5,24 @@ entry points) into throwaway virtualenvs and run ``discover_adapter`` against th
|
|
|
5
5
|
real ``importlib.metadata`` machinery — no monkeypatching of ``entry_points``, so the
|
|
6
6
|
tests exercise actual entry-point loading rather than an assumed return shape.
|
|
7
7
|
"""
|
|
8
|
+
import dataclasses
|
|
8
9
|
import subprocess
|
|
9
10
|
import textwrap
|
|
10
11
|
|
|
11
12
|
from pathlib import Path
|
|
12
13
|
|
|
14
|
+
import pyarrow as pa
|
|
13
15
|
import pytest
|
|
14
16
|
|
|
15
|
-
from continuo_engine_contract.
|
|
17
|
+
from continuo_engine_contract.keys import DuplicateKeyError
|
|
18
|
+
from continuo_engine_contract.port import (
|
|
19
|
+
ENTRY_POINT_GROUP,
|
|
20
|
+
AppendCapable,
|
|
21
|
+
LiveColumn,
|
|
22
|
+
TableIntrospection,
|
|
23
|
+
UpsertCapable,
|
|
24
|
+
WarehouseAdapter,
|
|
25
|
+
)
|
|
16
26
|
|
|
17
27
|
CONTRACT_ROOT = Path(__file__).resolve().parent.parent # contract/
|
|
18
28
|
|
|
@@ -181,12 +191,122 @@ def test_ensure_table_signature_declares_config_keyword_only():
|
|
|
181
191
|
|
|
182
192
|
sig = inspect.signature(WarehouseAdapter.ensure_table)
|
|
183
193
|
params = list(sig.parameters)
|
|
184
|
-
assert params == ["self", "schema", "table", "columns", "config"]
|
|
194
|
+
assert params == ["self", "schema", "table", "columns", "config", "primary_key"]
|
|
185
195
|
config = sig.parameters["config"]
|
|
186
196
|
assert config.kind is inspect.Parameter.KEYWORD_ONLY
|
|
187
197
|
assert config.default is inspect.Parameter.empty
|
|
188
198
|
|
|
189
199
|
|
|
200
|
+
def test_ensure_table_accepts_an_optional_keyword_only_primary_key():
|
|
201
|
+
"""The harness passes primary_key only for upsert nodes, so it must default to ()."""
|
|
202
|
+
import inspect
|
|
203
|
+
|
|
204
|
+
primary_key = inspect.signature(WarehouseAdapter.ensure_table).parameters["primary_key"]
|
|
205
|
+
assert primary_key.kind is inspect.Parameter.KEYWORD_ONLY
|
|
206
|
+
assert primary_key.default == ()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def test_capabilities_declare_exactly_their_own_abstract_methods():
|
|
210
|
+
"""No capability ships a method that exists but raises 'unsupported'."""
|
|
211
|
+
assert TableIntrospection.__abstractmethods__ == frozenset({"table_columns"})
|
|
212
|
+
assert AppendCapable.__abstractmethods__ == frozenset({"table_columns", "append"})
|
|
213
|
+
assert UpsertCapable.__abstractmethods__ == frozenset({"table_columns", "_upsert"})
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def test_a_partially_implemented_append_capability_cannot_be_instantiated():
|
|
217
|
+
class NoIntrospection(AppendCapable):
|
|
218
|
+
def append(self, schema, table, data):
|
|
219
|
+
...
|
|
220
|
+
|
|
221
|
+
with pytest.raises(TypeError, match="table_columns"):
|
|
222
|
+
NoIntrospection() # type: ignore[abstract]
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def test_a_partially_implemented_upsert_capability_cannot_be_instantiated():
|
|
226
|
+
class NoEngineWrite(UpsertCapable):
|
|
227
|
+
def table_columns(self, schema, table):
|
|
228
|
+
return []
|
|
229
|
+
|
|
230
|
+
with pytest.raises(TypeError, match="_upsert"):
|
|
231
|
+
NoEngineWrite() # type: ignore[abstract]
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class _RecordingUpsert(UpsertCapable):
|
|
235
|
+
def __init__(self):
|
|
236
|
+
self.writes = []
|
|
237
|
+
|
|
238
|
+
def table_columns(self, schema, table):
|
|
239
|
+
return []
|
|
240
|
+
|
|
241
|
+
def _upsert(self, schema, table, data, primary_key):
|
|
242
|
+
self.writes.append((schema, table, data, tuple(primary_key)))
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def test_upsert_hands_unique_rows_to_the_engine_write():
|
|
246
|
+
adapter = _RecordingUpsert()
|
|
247
|
+
data = pa.table({"id": [1, 2], "v": ["a", "b"]})
|
|
248
|
+
adapter.upsert("s", "t", data, ["id"])
|
|
249
|
+
assert adapter.writes == [("s", "t", data, ("id",))]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def test_upsert_checks_the_key_before_the_engine_sees_any_row():
|
|
253
|
+
adapter = _RecordingUpsert()
|
|
254
|
+
with pytest.raises(DuplicateKeyError):
|
|
255
|
+
adapter.upsert("s", "t", pa.table({"id": [1, 1]}), ["id"])
|
|
256
|
+
assert adapter.writes == []
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class _PortOnly(WarehouseAdapter):
|
|
260
|
+
@classmethod
|
|
261
|
+
def required_env(cls):
|
|
262
|
+
return []
|
|
263
|
+
|
|
264
|
+
@classmethod
|
|
265
|
+
def from_env(cls):
|
|
266
|
+
return cls()
|
|
267
|
+
|
|
268
|
+
def ensure_schema(self, schema):
|
|
269
|
+
...
|
|
270
|
+
|
|
271
|
+
def drop_schema(self, schema):
|
|
272
|
+
...
|
|
273
|
+
|
|
274
|
+
def build_empty_from_sql(self, schema, table, compiled_sql):
|
|
275
|
+
...
|
|
276
|
+
|
|
277
|
+
def clone_empty_from_prod(self, candidate_schema, prod_schema, table):
|
|
278
|
+
...
|
|
279
|
+
|
|
280
|
+
def build_empty_from_columns(self, schema, table, columns, config):
|
|
281
|
+
...
|
|
282
|
+
|
|
283
|
+
def check_binds(self, sql):
|
|
284
|
+
...
|
|
285
|
+
|
|
286
|
+
def fetch(self, sql):
|
|
287
|
+
...
|
|
288
|
+
|
|
289
|
+
def ensure_table(self, schema, table, columns, *, config, primary_key=()):
|
|
290
|
+
...
|
|
291
|
+
|
|
292
|
+
def load(self, schema, table, data):
|
|
293
|
+
...
|
|
294
|
+
|
|
295
|
+
def close(self):
|
|
296
|
+
...
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def test_capabilities_are_opt_in():
|
|
300
|
+
adapter = _PortOnly()
|
|
301
|
+
assert not isinstance(adapter, (TableIntrospection, AppendCapable, UpsertCapable))
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def test_live_column_is_a_frozen_value():
|
|
305
|
+
assert LiveColumn("id", False) == LiveColumn(name="id", nullable=False)
|
|
306
|
+
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
307
|
+
LiveColumn("id", False).name = "x" # type: ignore[misc]
|
|
308
|
+
|
|
309
|
+
|
|
190
310
|
# --- Real installed-adapter discovery via throwaway venvs --------------------
|
|
191
311
|
|
|
192
312
|
_FIXTURE_MODULE = '''\
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/config.py
RENAMED
|
File without changes
|
{continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/result.py
RENAMED
|
File without changes
|
{continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/sql.py
RENAMED
|
File without changes
|
{continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/continuo_engine_contract/types.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{continuo_engine_contract-0.7.2 → continuo_engine_contract-0.8.0}/tests/test_contract_types.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|