continuo-python-runtime 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

continuo_python_runtime/contract/kinds.py

@@ -1,8 +1,9 @@
 """Per-kind contract rules: which fields a node of each kind must carry.
 
 Each kind is one ``KindRules`` entry composed of a script policy, a reads rule,
-and whether it may name a Secret. Adding a kind means adding one entry to
-``RULES``; nothing that parses a contract branches on the kind's name.
+whether it may name a Secret, and which write modes it may declare. Adding a
+kind means adding one entry to ``RULES``; nothing that parses a contract
+branches on the kind's name.
 """
 
 from __future__ import annotations
@@ -14,7 +15,7 @@ from typing import Any
 from continuo_engine_contract.sql import ensure_single_read  # type: ignore[import-untyped]
 from sqlglot.errors import TokenError
 
-from continuo_python_runtime.contract.model import KINDS
+from continuo_python_runtime.contract.model import DEFAULT_MODE_TYPE, KINDS, MODE_TYPES
 from continuo_python_runtime.csv_source import parse_csv_uri
 from continuo_python_runtime.errors import ContractError
 
@@ -108,12 +109,18 @@ class KindRules:
     script_required: bool
     reads: ReadsRule
     secret_allowed: bool = False
+    # The write modes a node of this kind may declare. A new kind gets replace
+    # only unless its entry opts into more.
+    mode_types: frozenset[str] = frozenset({DEFAULT_MODE_TYPE})
 
 
 RULES: dict[str, KindRules] = {
-    "python-node": KindRules(script_required=True, reads=SqlReads()),
+    "python-node": KindRules(script_required=True, reads=SqlReads(), mode_types=MODE_TYPES),
+    # A csv drop is a snapshot of its source: replace only (spec D2).
     "python-csv": KindRules(script_required=False, reads=CsvRead()),
-    "python-api": KindRules(script_required=True, reads=NoReads(), secret_allowed=True),
+    "python-api": KindRules(
+        script_required=True, reads=NoReads(), secret_allowed=True, mode_types=MODE_TYPES
+    ),
 }
 
 

continuo_python_runtime/contract/loader.py

@@ -17,7 +17,9 @@ from continuo_python_runtime.contract.kinds import rules_for
 from continuo_python_runtime.contract.model import (
     CRITICALITIES,
     DEFAULT_KIND,
+    DEFAULT_MODE_TYPE,
     EXTRA_COLUMNS_POLICIES,
+    MODE_TYPES,
     Column,
     Node,
 )
@@ -40,6 +42,8 @@ _ALLOWED_KEYS = {
     "config",
     "content_hash",
     "secret_ref",
+    "mode_type",
+    "primary_key",
 }
 
 _REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule")
@@ -140,6 +144,67 @@ def _validate_config(raw: Any, label: str) -> dict[str, Any]:
     return raw
 
 
+def _validate_primary_key(
+    raw: dict[str, Any], mode_type: str, columns: list[Column], label: str
+) -> tuple[str, ...]:
+    """Validate ``primary_key`` against ``mode_type`` and the declared columns.
+
+    Required for ``upsert`` and rejected otherwise: a key on ``append`` or
+    ``replace`` would look enforced when it is not. Names keep their order,
+    which is what the wire entry carries. Each must be spelled exactly as an
+    output column (postgres quotes identifiers, so ``Rate_Date`` is not
+    ``rate_date``) declared ``nullable: false``, because a NULL key never
+    matches an existing row. Repeats are refused case-insensitively too, so a
+    key cannot name two columns that differ only in case. A floating-point
+    column is refused as well: the warehouse treats ``0.0`` and ``-0.0`` as
+    one key, so ``ensure_unique_key`` would reject the rows at write time --
+    failing here instead keeps an unwritable upsert out of the contract.
+    """
+    if mode_type != "upsert":
+        if "primary_key" in raw:
+            raise ContractError(
+                f"{label}: 'primary_key' is only allowed when mode_type is upsert "
+                f"(got {mode_type})"
+            )
+        return ()
+    if "primary_key" not in raw:
+        raise ContractError(f"{label}: 'primary_key' is required when mode_type is upsert")
+    raw_key = raw["primary_key"]
+    if not isinstance(raw_key, list) or not raw_key:
+        raise ContractError(
+            f"{label}: 'primary_key' must be a non-empty list of output column names"
+        )
+    by_name = {column.name: column for column in columns}
+    seen: set[str] = set()
+    for name in raw_key:
+        if not isinstance(name, str) or not name.strip():
+            raise ContractError(
+                f"{label}: 'primary_key' entry {name!r} must be a non-empty string"
+            )
+        folded = name.lower()  # same folding as continuo's parser
+        if folded in seen:
+            raise ContractError(
+                f"{label}: 'primary_key' names {name!r} more than once "
+                "(names are compared case-insensitively)"
+            )
+        seen.add(folded)
+        column = by_name.get(name)
+        if column is None:
+            raise ContractError(
+                f"{label}: 'primary_key' column {name!r} is not in output_columns"
+            )
+        if column.nullable:
+            raise ContractError(
+                f"{label}: 'primary_key' column {name!r} must be declared nullable: false"
+            )
+        if parse_sql_type(column.type).base == "DOUBLE_PRECISION":
+            raise ContractError(
+                f"{label}: 'primary_key' column {name!r} is floating-point "
+                f"({column.type}); a floating-point column cannot be a primary key"
+            )
+    return tuple(raw_key)
+
+
 def parse_node(
     raw: dict[str, Any],
     source: str,
@@ -158,7 +223,8 @@ def parse_node(
     ``check_reads=False`` skips *only* that :func:`ensure_single_read` call —
     the read-shape gate — leaving every other rule here (required fields,
     criticality, the ``reads`` map's own shape, output-column types and
-    uniqueness, ``config``, ``content_hash``) in force. It exists for
+    uniqueness, ``mode_type`` and ``primary_key``, ``config``,
+    ``content_hash``) in force. It exists for
     :func:`~continuo_python_runtime.harness.run_node`; see
     :func:`load_contract_dir` for why the runtime opts out.
     """
@@ -229,6 +295,17 @@ def parse_node(
             raise ContractError(f"{label}: 'secret_ref' is not allowed for kind {kind}")
         secret_ref = validate_secret_ref(raw["secret_ref"], label)
 
+    mode_type = raw.get("mode_type", DEFAULT_MODE_TYPE)
+    if not isinstance(mode_type, str) or mode_type not in MODE_TYPES:
+        raise ContractError(
+            f"{label}: 'mode_type' must be one of {sorted(MODE_TYPES)}, got {mode_type!r}"
+        )
+    if mode_type not in rules.mode_types:
+        raise ContractError(
+            f"{label}: 'mode_type' {mode_type} is not allowed for kind {kind} "
+            f"(allowed: {sorted(rules.mode_types)})"
+        )
+
     raw_columns = raw.get("output_columns")
     if not isinstance(raw_columns, list) or not raw_columns:
         raise ContractError(f"{label}: 'output_columns' must be a non-empty list")
@@ -269,6 +346,8 @@ def parse_node(
         seen_names.add(name)
         columns.append(Column(name=name, type=col_type, nullable=nullable))
 
+    primary_key = _validate_primary_key(raw, mode_type, columns, label)
+
     description = raw.get("description", "")
     if not isinstance(description, str):
         raise ContractError(f"{label}: 'description' must be a string")
@@ -294,6 +373,8 @@ def parse_node(
         content_hash=content_hash,
         kind=kind,
         secret_ref=secret_ref,
+        mode_type=mode_type,
+        primary_key=primary_key,
     )
 
 

continuo_python_runtime/contract/merge.py

@@ -7,7 +7,7 @@ import yaml
 
 from continuo_python_runtime.closure import resolve_closure
 from continuo_python_runtime.contract.loader import load_contract_dir
-from continuo_python_runtime.contract.model import CONTRACT_VERSION, Node
+from continuo_python_runtime.contract.model import CONTRACT_VERSION, DEFAULT_MODE_TYPE, Node
 from continuo_python_runtime.contract.paths import resolve_script_path
 from continuo_python_runtime.errors import ContractError
 from continuo_python_runtime.hashing import hash_parts
@@ -20,7 +20,10 @@ def node_entry(node: Node) -> dict:
     Returns the node as a dict with all fields, output_columns as list of
     dicts. The entry carries no hash fields: build_wire_contract adds all
     four (source_hash, shared_code_hash, config_hash, content_hash). The
-    nullable field is always present in output_columns.
+    nullable field is always present in output_columns. ``secret_ref``,
+    ``mode_type`` and ``primary_key`` appear only when they differ from their
+    default, so a node that does not use them has the entry (and hashes) it
+    had before they existed.
     """
     entry = {
         "schema": node.schema,
@@ -47,6 +50,12 @@ def node_entry(node: Node) -> dict:
     # only when set keeps every other node's config_hash unchanged.
     if node.secret_ref:
         entry["secret_ref"] = node.secret_ref
+    # Same reasoning: every replace node's entry, config_hash and content_hash
+    # stay byte-identical, so contract_version stays 1.
+    if node.mode_type != DEFAULT_MODE_TYPE:
+        entry["mode_type"] = node.mode_type
+    if node.mode_type == "upsert":
+        entry["primary_key"] = list(node.primary_key)
     return entry
 
 

continuo_python_runtime/contract/model.py

@@ -8,6 +8,11 @@ CRITICALITIES = frozenset({"REGULATORY", "CORE", "SECONDARY"})
 EXTRA_COLUMNS_POLICIES = frozenset({"raise", "warn"})
 KINDS = frozenset({"python-node", "python-csv", "python-api"})
 DEFAULT_KIND = "python-node"
+# How the harness writes a node's table on each run: replace empties it first,
+# append keeps the rows of earlier runs, upsert overwrites rows whose
+# primary_key exists and keeps the rest.
+MODE_TYPES = frozenset({"append", "upsert", "replace"})
+DEFAULT_MODE_TYPE = "replace"
 CONTRACT_VERSION = 1
 
 
@@ -38,6 +43,8 @@ class Node:
     content_hash: str | None = None
     kind: str = DEFAULT_KIND
     secret_ref: str = ""
+    mode_type: str = DEFAULT_MODE_TYPE
+    primary_key: tuple[str, ...] = ()
 
     @property
     def relation(self) -> str:

continuo_python_runtime/harness.py

@@ -3,7 +3,8 @@
 ``run_node`` is the sole write sink for a python node. It resolves the
 node from the contract, loads and executes its script inside a
 :class:`~continuo_python_runtime.context.RunContext`, conforms the result to
-the declared schema, and writes it through the runtime adapter. Exactly one
+the declared schema, and writes it through the runtime adapter with the
+node's ``mode_type``: ``load`` (replace), ``append`` or ``upsert``. Exactly one
 sentinel-framed result block is printed to stdout per run; every other
 diagnostic goes to stderr via ``logging``.
 """
@@ -17,11 +18,15 @@ import os
 import sys
 import uuid
 from collections.abc import Callable, Mapping
+from dataclasses import dataclass
 from pathlib import Path
 from types import ModuleType
 from typing import Any
 
+from continuo_engine_contract.keys import DuplicateKeyError  # type: ignore[import-untyped]
 from continuo_engine_contract.port import (  # type: ignore[import-untyped]
+    AppendCapable,
+    UpsertCapable,
     discover_adapter,
 )
 from continuo_engine_contract.result import result_block  # type: ignore[import-untyped]
@@ -33,7 +38,14 @@ from continuo_python_runtime.contract.model import Node
 from continuo_python_runtime.contract.paths import resolve_script_path
 from continuo_python_runtime.csv_loader import produce_csv
 from continuo_python_runtime.csv_source import CsvSourceReader
-from continuo_python_runtime.errors import ContractError, HarnessError, LoadError, ScriptError
+from continuo_python_runtime.errors import (
+    ConformError,
+    ContractError,
+    HarnessError,
+    LoadError,
+    ScriptError,
+)
+from continuo_python_runtime.shape import shape_diff
 
 logger = logging.getLogger("continuo_python_runtime.harness")
 
@@ -232,6 +244,75 @@ _PRODUCERS: dict[str, Callable[[Node, Any, Path, CsvSourceReader | None], Any]]
 }
 
 
+def _write_replace(adapter: Any, schema: str, table: str, data: Any, key: tuple[str, ...]) -> None:
+    adapter.load(schema, table, data)
+
+
+def _write_append(adapter: Any, schema: str, table: str, data: Any, key: tuple[str, ...]) -> None:
+    adapter.append(schema, table, data)
+
+
+def _write_upsert(adapter: Any, schema: str, table: str, data: Any, key: tuple[str, ...]) -> None:
+    adapter.upsert(schema, table, data, key)
+
+
+@dataclass(frozen=True)
+class _Writer:
+    """How one ``mode_type`` is written.
+
+    ``capability`` is the engine-contract interface the adapter must declare
+    (``None``: every ``WarehouseAdapter`` can, through ``load``). Every
+    capability extends ``TableIntrospection``, so a capability-gated write is
+    also a shape-checked one.
+    """
+
+    capability: type | None
+    write: Callable[[Any, str, str, Any, tuple[str, ...]], None]
+
+
+# One entry per mode_type, beside _PRODUCERS: a new mode is one capability
+# interface in the engine contract plus one entry here.
+_WRITERS: dict[str, _Writer] = {
+    "replace": _Writer(capability=None, write=_write_replace),
+    "append": _Writer(capability=AppendCapable, write=_write_append),
+    "upsert": _Writer(capability=UpsertCapable, write=_write_upsert),
+}
+
+
+def _writer_for(adapter: Any, node: Node) -> _Writer:
+    """The node's writer, if the adapter declares the capability its mode needs.
+
+    Called right after the config tripwire, before the script runs or the
+    warehouse is touched: an engine that cannot append fails in the first
+    second, not after the whole run.
+
+    Raises:
+        LoadError: ``<Adapter> does not support <mode>``.
+    """
+    writer = _WRITERS[node.mode_type]
+    if writer.capability is not None and not isinstance(adapter, writer.capability):
+        raise LoadError(f"{type(adapter).__name__} does not support {node.mode_type}")
+    return writer
+
+
+def _check_shape(adapter: Any, node: Node, schema: str, table: str) -> None:
+    """Fail unless the live target table still has the node's declared shape.
+
+    Runs before an append or upsert only: those keep earlier rows, so a table
+    whose columns or nullability drifted (changed outside the contract, or
+    created before the release gate recorded shapes) must not be written. It
+    inspects the table actually written -- ``TARGET_SCHEMA.TABLE_NAME`` -- not
+    ``node.relation``.
+
+    Raises:
+        LoadError: Naming every missing column, unexpected column and
+            nullability mismatch.
+    """
+    diff = shape_diff(node.output_columns, adapter.table_columns(schema, table))
+    if diff:
+        raise LoadError(diff.message(f"{schema}.{table}"))
+
+
 def run_node(
     env: Mapping[str, str], adapter: Any = None, reader: CsvSourceReader | None = None
 ) -> int:
@@ -276,6 +357,7 @@ def run_node(
                 raise LoadError(f"adapter construction failed: {exc}") from exc
 
         _validate_config_early(active_adapter, node)
+        writer = _writer_for(active_adapter, node)
 
         table = _PRODUCERS[node.kind](node, active_adapter, app_root, reader)
 
@@ -285,11 +367,25 @@ def run_node(
             {"name": c.name, "type": c.type, "nullable": c.nullable}
             for c in node.output_columns
         ]
+        # Spelled exactly as in output_columns (the loader enforces it). Empty
+        # for every mode but upsert, so primary_key reaches ensure_table only
+        # for upsert nodes.
+        key = node.primary_key
         try:
-            active_adapter.ensure_table(target_schema, table_name, columns, config=node.config)
-            active_adapter.load(target_schema, table_name, conformed)
+            if key:
+                active_adapter.ensure_table(
+                    target_schema, table_name, columns, config=node.config, primary_key=key
+                )
+            else:
+                active_adapter.ensure_table(target_schema, table_name, columns, config=node.config)
+            if writer.capability is not None:
+                _check_shape(active_adapter, node, target_schema, table_name)
+            writer.write(active_adapter, target_schema, table_name, conformed, key)
         except HarnessError:
             raise
+        except DuplicateKeyError as exc:
+            # The script's rows repeat a key: its output is at fault, not the warehouse.
+            raise ConformError(str(exc)) from exc
         except Exception as exc:
             raise LoadError(f"failed to write {target_schema}.{table_name}: {exc}") from exc
 

continuo_python_runtime/shape.py

@@ -0,0 +1,95 @@
+"""Compare a live table's columns with a node's declared output columns.
+
+The harness runs this before every append or upsert: those modes keep earlier
+rows, so writing into a table whose shape drifted from the contract would mix
+two shapes in one table. Names are compared case-insensitively (Trino folds
+names, DuckDB matches them case-insensitively) and nullability exactly. Order
+is ignored, because every insert names its columns; types are ignored,
+because engines report them in their own spelling (Trino and DuckDB drop a
+VARCHAR's length) and there is no engine-to-contract type mapping. Type
+changes are the release gate's job.
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import NamedTuple
+
+from continuo_engine_contract.port import LiveColumn  # type: ignore[import-untyped]
+
+from continuo_python_runtime.contract.model import Column
+
+
+class NullabilityDiff(NamedTuple):
+    """A column whose nullability differs: as declared vs as the live table has it."""
+
+    name: str
+    declared: bool
+    live: bool
+
+
+def _yaml_bool(value: bool) -> str:
+    return "true" if value else "false"
+
+
+@dataclass(frozen=True)
+class ShapeDiff:
+    """What separates a live table from the declared columns; falsy when nothing does."""
+
+    missing: tuple[str, ...] = ()
+    unexpected: tuple[str, ...] = ()
+    nullability: tuple[NullabilityDiff, ...] = ()
+
+    def __bool__(self) -> bool:
+        return bool(self.missing or self.unexpected or self.nullability)
+
+    def message(self, target: str) -> str:
+        """One line naming every difference, for a LoadError about *target* (``schema.table``)."""
+        parts: list[str] = []
+        if self.missing:
+            parts.append(f"missing column(s) {list(self.missing)}")
+        if self.unexpected:
+            parts.append(f"unexpected column(s) {list(self.unexpected)}")
+        parts.extend(
+            f"column {diff.name} is nullable: {_yaml_bool(diff.live)} in the table "
+            f"but nullable: {_yaml_bool(diff.declared)} in the contract"
+            for diff in self.nullability
+        )
+        return (
+            f"table {target} does not match the contract's output_columns: {'; '.join(parts)}. "
+            "An append or upsert table keeps its rows, so its shape cannot change: "
+            "write to a new table (a _v2 name) under a new contract"
+        )
+
+
+def shape_diff(
+    contract_columns: Sequence[Column], live_columns: Sequence[LiveColumn]
+) -> ShapeDiff:
+    """Compare the declared columns with a live table's.
+
+    Each declared column is paired with a live column of the exact same name
+    if there is one, else with one whose name matches case-insensitively, so
+    two live columns that differ only in case pair with the right declared
+    column.
+    """
+    remaining = list(live_columns)
+    missing: list[str] = []
+    nullability: list[NullabilityDiff] = []
+    for declared in contract_columns:
+        live = next((column for column in remaining if column.name == declared.name), None)
+        if live is None:
+            folded = declared.name.casefold()
+            live = next(
+                (column for column in remaining if column.name.casefold() == folded), None
+            )
+        if live is None:
+            missing.append(declared.name)
+            continue
+        remaining.remove(live)
+        if live.nullable != declared.nullable:
+            nullability.append(NullabilityDiff(declared.name, declared.nullable, live.nullable))
+    return ShapeDiff(
+        missing=tuple(missing),
+        unexpected=tuple(column.name for column in remaining),
+        nullability=tuple(nullability),
+    )

{continuo_python_runtime-0.7.0.dist-info → continuo_python_runtime-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: continuo-python-runtime
-Version: 0.7.0
+Version: 0.9.0
 Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
 Author: Simone Carolini
 Maintainer: Simone Carolini
@@ -12,7 +12,7 @@ Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: >=3.14
 Requires-Dist: boto3==1.43.85
-Requires-Dist: continuo-engine-contract==0.7.3
+Requires-Dist: continuo-engine-contract==0.8.0
 Requires-Dist: pyarrow==25.0.1
 Requires-Dist: pyyaml==6.0.3
 Requires-Dist: sqlglot==30.17.0
@@ -41,15 +41,16 @@ Five artifacts come out of this repository:
 - **The `continuo-engine-contract` PyPI package** — the `WarehouseAdapter`
   port, the contract schema, the shared SQL/type/config guards, and the
   sentinel result-block format. Adapter authors outside this repo pin it.
-- **The two engine-adapter PyPI packages** (`continuo-postgres-adapter`,
-  `continuo-trino-adapter`) — one `WarehouseAdapter` implementation per
+- **The three engine-adapter PyPI packages** (`continuo-postgres-adapter`,
+  `continuo-trino-adapter`, `continuo-duckdb-adapter`) — one `WarehouseAdapter` implementation per
   warehouse engine, each published independently under the same tag. A
   domain repo normally never installs these directly (the engine image
   already has the matching one baked in); they exist as standalone PyPI
   packages for the "build your own container" shape (see below) and for
   third-party adapter authors to reference.
 - **Per-engine base images**, one per warehouse engine
-  (`continuo-python-runtime-postgres`, `continuo-python-runtime-trino`), that
+  (`continuo-python-runtime-postgres`, `continuo-python-runtime-trino`,
+  `continuo-python-runtime-duckdb`), that
   domain repos build `FROM`. Each image bakes in the runtime and a single
   `WarehouseAdapter` for that engine, and serves both roles that adapter has:
   the node harness (`ENTRYPOINT ["continuo-runtime"]`, `CMD ["run"]`) and the
@@ -57,20 +58,23 @@ Five artifacts come out of this repository:
 - **`template/`** — a copy-ready domain repo: `Dockerfile`, `contracts/`,
   `scripts/`, and the `release.yml` CI/CD workflow.
 
-One `vX.Y.Z` git tag releases all of it: `publish-pypi.yml` builds all four
-PyPI distributions into a single `dist/` and publishes them together, and
-`images.yml` builds and pushes both engine images — each installing its
-matching pinned adapter version from that same release — multi-arch under
-the same tag.
+One `vX.Y.Z` git tag releases all of it: `publish-pypi.yml` builds all five
+PyPI distributions into a single `dist/`, verifies them (and full engine
+images built from them) on amd64 and arm64, and only then publishes them
+together. It then builds all three engine images natively per architecture —
+each installing its matching pinned adapter version from that same release —
+pulls and smoke-tests the exact pushed digests, and finally gives the verified
+amd64 + arm64 digests one multi-arch tag. See `CONTRIBUTING.md` (Releasing).
 
 ### What this repo owns
 
-This repository owns the entire python-node surface: the engine contract, both
+This repository owns the entire python-node surface: the engine contract, the three
 engine adapters, the validation runner, and the node harness. The former
 `continuo-validation` repository was merged in — there is no longer a separate
 validation-side port, adapter class, entry-point group, or image. One
 `WarehouseAdapter` per engine serves both the data plane (`fetch` /
-`ensure_table` / `load`) and validation (`ensure_schema` / `drop_schema` /
+`ensure_table` / `load`, plus the opt-in `table_columns` / `append` / `upsert`
+capabilities) and validation (`ensure_schema` / `drop_schema` /
 `build_empty_from_sql` / `build_empty_from_columns` / `clone_empty_from_prod` /
 `check_binds`), and one image per engine runs both roles.
 
@@ -80,16 +84,18 @@ validation-side port, adapter class, entry-point group, or image. One
 | `continuo-engine-contract` | `continuo_engine_contract` | this repo, `contract/` | The `WarehouseAdapter` port, contract schema, the SQL/type/config guards adapters must run, and the result-block format. Published to PyPI. |
 | `continuo-postgres-adapter` | `continuo_postgres_adapter` | this repo, `adapters/postgres/` | `PostgresAdapter` — one class, both roles. Published to PyPI. |
 | `continuo-trino-adapter` | `continuo_trino_adapter` | this repo, `adapters/trino/` | `TrinoAdapter` — one class, both roles, for Trino/Iceberg. Published to PyPI. |
+| `continuo-duckdb-adapter` | `continuo_duckdb_adapter` | this repo, `adapters/duckdb/` | `DuckDBAdapter` — one class, both roles, for DuckDB on a DuckLake (Postgres catalog, Parquet on S3). Published to PyPI. |
 
-All four are uv workspace members (`[tool.uv.workspace]` in the root
+All five are uv workspace members (`[tool.uv.workspace]` in the root
 `pyproject.toml`), so `uv sync --all-packages --all-groups` at the repo root
 installs everything for local development.
 
-**All four packages in the table above are published to PyPI**, under the
-same `vX.Y.Z` tag. The two engine images then **install the matching pinned
+**All five packages in the table above are published to PyPI**, under the
+same `vX.Y.Z` tag. The three engine images then **install the matching pinned
 adapter version from PyPI** — `Dockerfile.postgres` installs
 `continuo-postgres-adapter==X.Y.Z`, `Dockerfile.trino` installs
-`continuo-trino-adapter==X.Y.Z` — rather than building it from this repo's
+`continuo-trino-adapter==X.Y.Z` and `Dockerfile.duckdb` installs
+`continuo-duckdb-adapter==X.Y.Z` — rather than building it from this repo's
 source tree, so each image still ships exactly one adapter and the runtime
 still discovers it through the `continuo_engine.adapters` entry-point group
 at run time. The image **name** (`continuo-python-runtime-<engine>`) and the
@@ -97,7 +103,7 @@ adapter's pip **distribution** name (`continuo-<engine>-adapter`) are two
 different artifacts of the same adapter — same engine, same version, same
 runtime behavior, different packaging; see "Build your own container" below
 for a build shape that installs the pip package directly instead of `FROM`
-the image. All four packages are still built, type-checked, and tested by CI
+the image. All five packages are still built, type-checked, and tested by CI
 on every change.
 
 ### The result block is a frozen wire contract
@@ -116,9 +122,13 @@ the Go parser has not been taught is a production outage, not a refactor.
    service name (one service name per domain repo).
 3. Configure repository variables in GitHub (Settings → Secrets and
    variables → Actions): `REGISTRY` (your Docker registry), `BUCKET` (your
-   S3 bucket for contract artifacts), `RELEASE_ENDPOINT` (the release
-   webhook endpoint). `RELEASE_ENDPOINT` is the **base URL** of the Continuo
-   API (no `/releases` suffix) — the workflow appends `/releases` itself.
+   S3 bucket for contract artifacts), `RELEASE_ENDPOINT` (the base
+   URL of your continuo install, `scheme://host[:port]` with no path, the origin of continuo's
+   `auth.publicUrl`). The
+   workflow calls `<RELEASE_ENDPOINT>/api/v1/releases` with its GitHub Actions
+   OIDC token, so the repository must be bound to your service in continuo's
+   `ciAuth.bindings` (see [Releasing from CI](https://github.com/carolsimone/continuo/blob/main/deploy/README.md#releasing-from-ci-github-actions)),
+   and an operator bootstraps the service's first release.
 4. Configure repository secrets: `AWS_ACCESS_KEY_ID` and
    `AWS_SECRET_ACCESS_KEY` for the S3 upload. The template workflow pushes
    the built image to GHCR using the workflow's own `GITHUB_TOKEN` (granted
@@ -149,7 +159,7 @@ the pre-flight check once before your next release; it reports every affected
 read at once:
 
 ```bash
-continuo-runtime validate contracts/ --dialect postgres   # or trino
+continuo-runtime validate contracts/ --dialect postgres   # or trino, duckdb
 ```
 
 The runtime image does not re-run this gate, so a read that passes here is
@@ -186,6 +196,62 @@ produced, never how it is checked or written.
   script reads with `os.environ[...]`. The contract never holds a secret value
   or env var name; the Secret itself is created in the cluster by the operator.
 
+## Write modes
+
+`mode_type:` chooses how the harness writes a node's table on each run:
+
+| `mode_type` | Each run | Rows from earlier runs |
+| --- | --- | --- |
+| `replace` (default) | empties the table, inserts this run's rows | gone |
+| `append` | inserts this run's rows | kept |
+| `upsert` | inserts rows with a new key, overwrites rows whose key exists | kept unless overwritten; never deleted |
+
+```yaml
+- schema: fx
+  table: fx_rates
+  kind: python-api
+  mode_type: upsert
+  primary_key: [rate_date, base, quote]
+  output_columns:
+    - {name: rate_date, type: DATE,       nullable: false}
+    - {name: base,      type: VARCHAR(3), nullable: false}
+    - {name: quote,     type: VARCHAR(3), nullable: false}
+    - {name: rate,      type: DECIMAL(12,6)}
+```
+
+- `python-csv` nodes only `replace`.
+- `primary_key` is required for `upsert` and rejected otherwise: a non-empty
+  list of output columns declared `nullable: false` (a NULL key never
+  matches), each spelled exactly as in `output_columns`, names unique
+  case-insensitively. A Postgres `replace` table that
+  wants uniqueness uses `config.indexes` with `unique: true`.
+- Every write is atomic: all of the run's rows or none. An `append` or
+  `upsert` run with zero rows writes nothing (a `replace` run with zero rows
+  still empties the table).
+- `append` is a plain insert: a run that committed but was reported failed
+  and is retried stores its rows twice. `upsert` is idempotent.
+- An `upsert` run whose own rows repeat a key fails with `ConformError` and
+  writes nothing.
+- Before an `append` or `upsert` the harness compares the live table with
+  `output_columns` — names case-insensitively, nullability exactly; order and
+  types are not compared — and fails with `LoadError` naming every missing
+  column, unexpected column and nullability mismatch. Changing an `append` or
+  `upsert` table (columns, types, nullability, key or mode) needs a new table
+  name (`_v2`) under a new contract; continuo rejects the release otherwise.
+  Adding a nullable column is no exception: the existing rows cannot be
+  backfilled.
+- The installed engine must support the mode; all three shipped adapters do.
+  A bring-your-own adapter declares `AppendCapable` / `UpsertCapable` from
+  `continuo-engine-contract` (and can run its conformance suite); without them
+  an `append`/`upsert` node fails with `LoadError: <Adapter> does not support
+  <mode>` before its script runs.
+- Engines: on Postgres the key is the table's `PRIMARY KEY`, created with the
+  table; DuckLake and Trino/Iceberg have no constraints, so the key is
+  guaranteed by the write path only. On Trino an `upsert` table needs Iceberg
+  `format_version` 2 (the default).
+- `full_refresh` stays refused for every python node: a rebuild would destroy
+  an accumulating table.
+
 ## The script API
 
 A node script is a Python file with exactly one required entry point:
@@ -272,8 +338,8 @@ text.
 | `ContractError` | Contract missing or invalid, node not found for `NODE_ID`, or the declared script is missing/unreachable. | The contract yaml or the `script:` path. |
 | `ReadError` | `ctx.read()` was called with an undeclared name, or a declared read failed at the warehouse. | The `reads:` map, or the upstream query/warehouse access. |
 | `ScriptError` | `run()` raised, has no callable `run`, or returned a value that isn't Arrow-convertible. | The node script. |
-| `ConformError` | Structural mismatch (extra/missing/duplicate columns), a strict-cast failure, a not-null violation, or a VARCHAR/CHAR overflow. | The script's output shape, or the `output_columns` declaration. |
-| `LoadError` | Adapter construction failed, or the DDL/INSERT failed at the warehouse during the write. | Warehouse connectivity/permissions, or the target table. |
+| `ConformError` | Structural mismatch (extra/missing/duplicate columns), a strict-cast failure, a not-null violation, a VARCHAR/CHAR overflow, or an `upsert` run whose rows repeat a primary key. | The script's output shape, or the `output_columns` declaration. |
+| `LoadError` | Adapter construction failed, the installed engine does not support the node's `mode_type`, the live table's shape differs from `output_columns` (append/upsert), or the DDL/INSERT failed at the warehouse during the write. | Warehouse connectivity/permissions, the engine image, or the target table. |
 
 ## Engine selection
 
@@ -281,20 +347,23 @@ A domain repo picks its warehouse engine by which base image it builds
 `FROM`:
 
 ```dockerfile
-FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.7.0
+FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.9.0
+# or
+FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.9.0
 # or
-FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.7.0
+FROM ghcr.io/carolsimone/continuo-python-runtime-duckdb:v0.9.0
 ```
 
 The engine is part of the image **name**; the tag is the bare version, so
 Continuo's Helm chart can pin an image as `<name>:vX.Y.Z@sha256:<digest>`.
 
 Each image bakes in exactly one `WarehouseAdapter` for that engine — the
-pinned PyPI version of the `continuo-postgres-adapter` or
-`continuo-trino-adapter` package built from this repo's `adapters/postgres/`
-or `adapters/trino/` source (see the table above) — registered under the
+pinned PyPI version of the `continuo-postgres-adapter`,
+`continuo-trino-adapter` or `continuo-duckdb-adapter` package built from this
+repo's `adapters/postgres/`, `adapters/trino/` or `adapters/duckdb/` source
+(see the table above) — registered under the
 `continuo_engine.adapters` entry-point group (entry names `postgres` /
-`trino`). The runtime discovers it via `discover_adapter()` at run time, so a
+`trino` / `duckdb`). The runtime discovers it via `discover_adapter()` at run time, so a
 single image serves every node in the service and the release-time
 validation Job for it. The executor injects the warehouse connection as
 environment variables (engine-native, e.g.

{continuo_python_runtime-0.7.0.dist-info → continuo_python_runtime-0.9.0.dist-info}/RECORD

@@ -6,15 +6,16 @@ continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7D
 continuo_python_runtime/csv_loader.py,sha256=a5Hwu-YIzIiko8X1_vqShsnzjs2-OsKEUSCHLXg_dio,3346
 continuo_python_runtime/csv_source.py,sha256=7g8Tu6ieiJLvWH4Ep4nwYlOdGkjeePr12K5UHuO8TNc,4136
 continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
-continuo_python_runtime/harness.py,sha256=vsoahI38OCaCEpg9Il7xcHYcsXp6A5cxY_fjCC8xGJk,13046
+continuo_python_runtime/harness.py,sha256=vm2wL7cOHT2uvrIgDZkNXAV7aT-6x7dQoCKirr41oJ0,16798
 continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
 continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
+continuo_python_runtime/shape.py,sha256=C-5ClSxPYgoO9pUiGqw7pCnmlBbsEl-sIm7Twbf-H4o,3693
 continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
 continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
-continuo_python_runtime/contract/kinds.py,sha256=Ht-hCT7R5A1tEnPlbGL-b9FyJQBvpDWrXK9PBt7yA28,4938
-continuo_python_runtime/contract/loader.py,sha256=9HsV7dJhCwudjOPXBapYFKiZtslyXIkiZ_WfeuLh5-E,14858
-continuo_python_runtime/contract/merge.py,sha256=Zw--_-v3ERE02Nu7qqLtbHa3Hr__0a1SF-2PRqUfXAY,7594
-continuo_python_runtime/contract/model.py,sha256=WB_Qw4koP8BEwuiAZwIEsKFQRa_6QHHBK4xi4eGvPQo,1082
+continuo_python_runtime/contract/kinds.py,sha256=nirtluv9GU0sZzGF8_-qRIdbeCRtBul_ppNnr6CIEC0,5320
+continuo_python_runtime/contract/loader.py,sha256=uDlOYN6V7KhYhHk2aK34MHxSo3Z8riZw3FZ31Xf1Aqk,18386
+continuo_python_runtime/contract/merge.py,sha256=vN1hA0WSHbUnDyyobW1Z2YYZvU4a0eqwDLS8d_xNGQo,8124
+continuo_python_runtime/contract/model.py,sha256=Dq95CuAcSopumNqP9SlnrxXKhDVeYA55JwBKorhYde4,1435
 continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
 continuo_python_runtime/contract/secret_ref.py,sha256=96jh2G81clpabiEWGIsxbd9r5GKEGLb4Sa1-WJ9G59Q,1131
 continuo_python_runtime/csv_readers/__init__.py,sha256=9TKvjC37C77uRaJQZuO0--zVj2OvuNwpJE6nWx7j2S8,806
@@ -23,9 +24,9 @@ continuo_python_runtime/csv_readers/s3.py,sha256=zmvHs3TxlN2alPKrdaHzPyREmkve2nB
 continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
 continuo_python_runtime/validation/runner.py,sha256=7pbksf7MXfTNcdklCOkOXg9qfRiJtyI_dbgbL3a9LWk,14072
 continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
-continuo_python_runtime-0.7.0.dist-info/METADATA,sha256=Uo2XMCZawYbKgpER3qDn-eF_0cKJUZTultaxxcBAhuQ,19281
-continuo_python_runtime-0.7.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-continuo_python_runtime-0.7.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
-continuo_python_runtime-0.7.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
-continuo_python_runtime-0.7.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
-continuo_python_runtime-0.7.0.dist-info/RECORD,,
+continuo_python_runtime-0.9.0.dist-info/METADATA,sha256=wUHVQEfqOyqKM7xdRqUXXsJE4AI1aDZQ3x9q3_9zon4,23389
+continuo_python_runtime-0.9.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+continuo_python_runtime-0.9.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
+continuo_python_runtime-0.9.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
+continuo_python_runtime-0.9.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
+continuo_python_runtime-0.9.0.dist-info/RECORD,,