continuo-python-runtime 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

continuo_python_runtime/contract/kinds.py

@@ -1,8 +1,8 @@
 """Per-kind contract rules: which fields a node of each kind must carry.
 
-Each kind is one ``KindRules`` entry composed of a script policy and a reads
-rule. Adding a kind means adding one entry to ``RULES``; nothing that parses a
-contract branches on the kind's name.
+Each kind is one ``KindRules`` entry composed of a script policy, a reads rule,
+and whether it may name a Secret. Adding a kind means adding one entry to
+``RULES``; nothing that parses a contract branches on the kind's name.
 """
 
 from __future__ import annotations
@@ -88,17 +88,32 @@ class CsvRead(ReadsRule):
         return dict(reads)
 
 
+class NoReads(ReadsRule):
+    """No declared reads: the node's script fetches its own data."""
+
+    def validate(
+        self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
+    ) -> dict[str, str]:
+        if reads is None or (isinstance(reads, dict) and not reads):
+            return {}
+        raise ContractError(
+            f"{label}: a python-api node declares no 'reads' (its script fetches its own data)"
+        )
+
+
 @dataclass(frozen=True)
 class KindRules:
     """What a node of one kind must declare."""
 
     script_required: bool
     reads: ReadsRule
+    secret_allowed: bool = False
 
 
 RULES: dict[str, KindRules] = {
     "python-node": KindRules(script_required=True, reads=SqlReads()),
     "python-csv": KindRules(script_required=False, reads=CsvRead()),
+    "python-api": KindRules(script_required=True, reads=NoReads(), secret_allowed=True),
 }
 
 

continuo_python_runtime/contract/loader.py

@@ -21,6 +21,7 @@ from continuo_python_runtime.contract.model import (
     Column,
     Node,
 )
+from continuo_python_runtime.contract.secret_ref import validate_secret_ref
 from continuo_python_runtime.errors import ContractError
 from continuo_python_runtime.types import parse_sql_type
 
@@ -38,6 +39,7 @@ _ALLOWED_KEYS = {
     "output_columns",
     "config",
     "content_hash",
+    "secret_ref",
 }
 
 _REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule")
@@ -221,6 +223,12 @@ def parse_node(
         raw.get("reads"), label, dialect=dialect, check_reads=check_reads
     )
 
+    secret_ref = ""
+    if "secret_ref" in raw:
+        if not rules.secret_allowed:
+            raise ContractError(f"{label}: 'secret_ref' is not allowed for kind {kind}")
+        secret_ref = validate_secret_ref(raw["secret_ref"], label)
+
     raw_columns = raw.get("output_columns")
     if not isinstance(raw_columns, list) or not raw_columns:
         raise ContractError(f"{label}: 'output_columns' must be a non-empty list")
@@ -285,6 +293,7 @@ def parse_node(
         config=config,
         content_hash=content_hash,
         kind=kind,
+        secret_ref=secret_ref,
     )
 
 

continuo_python_runtime/contract/merge.py

@@ -22,7 +22,7 @@ def node_entry(node: Node) -> dict:
     four (source_hash, shared_code_hash, config_hash, content_hash). The
     nullable field is always present in output_columns.
     """
-    return {
+    entry = {
         "schema": node.schema,
         "table": node.table,
         "owner": node.owner,
@@ -43,6 +43,11 @@ def node_entry(node: Node) -> dict:
         "extra_columns": node.extra_columns,
         "config": dict(node.config),
     }
+    # Only a kind that may name a Secret ever carries one; emitting the key
+    # only when set keeps every other node's config_hash unchanged.
+    if node.secret_ref:
+        entry["secret_ref"] = node.secret_ref
+    return entry
 
 
 def _lint_node_closure(
@@ -131,6 +136,7 @@ def _csv_hash_inputs(node: Node, repo_root: Path) -> tuple[bytes, list[bytes]]:
 _HASH_INPUTS: dict[str, Callable[[Node, Path], tuple[bytes, list[bytes]]]] = {
     "python-node": _script_hash_inputs,
     "python-csv": _csv_hash_inputs,
+    "python-api": _script_hash_inputs,
 }
 
 

continuo_python_runtime/contract/model.py

@@ -6,7 +6,7 @@ from typing import Any
 # Module-level constants
 CRITICALITIES = frozenset({"REGULATORY", "CORE", "SECONDARY"})
 EXTRA_COLUMNS_POLICIES = frozenset({"raise", "warn"})
-KINDS = frozenset({"python-node", "python-csv"})
+KINDS = frozenset({"python-node", "python-csv", "python-api"})
 DEFAULT_KIND = "python-node"
 CONTRACT_VERSION = 1
 
@@ -37,6 +37,7 @@ class Node:
     config: dict[str, Any] = field(default_factory=dict)
     content_hash: str | None = None
     kind: str = DEFAULT_KIND
+    secret_ref: str = ""
 
     @property
     def relation(self) -> str:

continuo_python_runtime/contract/secret_ref.py

@@ -0,0 +1,31 @@
+"""The name of the Kubernetes Secret a node's pod receives as env vars.
+
+The ``continuo-api-`` prefix keeps a contract from naming any other Secret in
+the namespace (the warehouse credentials, the platform's own credentials).
+continuo's topology-controller and execution-controller enforce the same
+pattern; this check gives the author the error in CI instead.
+"""
+
+from __future__ import annotations
+
+import re
+
+from continuo_python_runtime.errors import ContractError
+
+SECRET_REF_PATTERN = re.compile(r"^continuo-api-[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
+SECRET_REF_MAX_LEN = 253
+
+
+def validate_secret_ref(value: object, label: str) -> str:
+    """Return ``value`` if it is a valid ``continuo-api-*`` Secret name."""
+    if (
+        not isinstance(value, str)
+        or len(value) > SECRET_REF_MAX_LEN
+        or not SECRET_REF_PATTERN.fullmatch(value)
+    ):
+        raise ContractError(
+            f"{label}: 'secret_ref' must be a Kubernetes Secret name starting with "
+            f"'continuo-api-' (lowercase letters, digits, '-'; at most "
+            f"{SECRET_REF_MAX_LEN} chars), got {value!r}"
+        )
+    return value

continuo_python_runtime/harness.py

@@ -1,6 +1,6 @@
 """Container entrypoint: dispatches a single node's script and writes its output.
 
-``run_node`` is the sole write sink for a python-node node. It resolves the
+``run_node`` is the sole write sink for a python node. It resolves the
 node from the contract, loads and executes its script inside a
 :class:`~continuo_python_runtime.context.RunContext`, conforms the result to
 the declared schema, and writes it through the runtime adapter. Exactly one
@@ -227,6 +227,7 @@ def _produce_from_csv(
 
 _PRODUCERS: dict[str, Callable[[Node, Any, Path, CsvSourceReader | None], Any]] = {
     "python-node": _produce_from_script,
+    "python-api": _produce_from_script,
     "python-csv": _produce_from_csv,
 }
 

{continuo_python_runtime-0.6.0.dist-info → continuo_python_runtime-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: continuo-python-runtime
-Version: 0.6.0
+Version: 0.8.0
 Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
 Author: Simone Carolini
 Maintainer: Simone Carolini
@@ -41,15 +41,16 @@ Five artifacts come out of this repository:
 - **The `continuo-engine-contract` PyPI package** — the `WarehouseAdapter`
   port, the contract schema, the shared SQL/type/config guards, and the
   sentinel result-block format. Adapter authors outside this repo pin it.
-- **The two engine-adapter PyPI packages** (`continuo-postgres-adapter`,
-  `continuo-trino-adapter`) — one `WarehouseAdapter` implementation per
+- **The three engine-adapter PyPI packages** (`continuo-postgres-adapter`,
+  `continuo-trino-adapter`, `continuo-duckdb-adapter`) — one `WarehouseAdapter` implementation per
   warehouse engine, each published independently under the same tag. A
   domain repo normally never installs these directly (the engine image
   already has the matching one baked in); they exist as standalone PyPI
   packages for the "build your own container" shape (see below) and for
   third-party adapter authors to reference.
 - **Per-engine base images**, one per warehouse engine
-  (`continuo-python-runtime-postgres`, `continuo-python-runtime-trino`), that
+  (`continuo-python-runtime-postgres`, `continuo-python-runtime-trino`,
+  `continuo-python-runtime-duckdb`), that
   domain repos build `FROM`. Each image bakes in the runtime and a single
   `WarehouseAdapter` for that engine, and serves both roles that adapter has:
   the node harness (`ENTRYPOINT ["continuo-runtime"]`, `CMD ["run"]`) and the
@@ -57,15 +58,17 @@ Five artifacts come out of this repository:
 - **`template/`** — a copy-ready domain repo: `Dockerfile`, `contracts/`,
   `scripts/`, and the `release.yml` CI/CD workflow.
 
-One `vX.Y.Z` git tag releases all of it: `publish-pypi.yml` builds all four
-PyPI distributions into a single `dist/` and publishes them together, and
-`images.yml` builds and pushes both engine images — each installing its
-matching pinned adapter version from that same release — multi-arch under
-the same tag.
+One `vX.Y.Z` git tag releases all of it: `publish-pypi.yml` builds all five
+PyPI distributions into a single `dist/`, verifies them (and full engine
+images built from them) on amd64 and arm64, and only then publishes them
+together. It then builds all three engine images natively per architecture —
+each installing its matching pinned adapter version from that same release —
+pulls and smoke-tests the exact pushed digests, and finally gives the verified
+amd64 + arm64 digests one multi-arch tag. See `CONTRIBUTING.md` (Releasing).
 
 ### What this repo owns
 
-This repository owns the entire python-node surface: the engine contract, both
+This repository owns the entire python-node surface: the engine contract, the three
 engine adapters, the validation runner, and the node harness. The former
 `continuo-validation` repository was merged in — there is no longer a separate
 validation-side port, adapter class, entry-point group, or image. One
@@ -80,16 +83,18 @@ validation-side port, adapter class, entry-point group, or image. One
 | `continuo-engine-contract` | `continuo_engine_contract` | this repo, `contract/` | The `WarehouseAdapter` port, contract schema, the SQL/type/config guards adapters must run, and the result-block format. Published to PyPI. |
 | `continuo-postgres-adapter` | `continuo_postgres_adapter` | this repo, `adapters/postgres/` | `PostgresAdapter` — one class, both roles. Published to PyPI. |
 | `continuo-trino-adapter` | `continuo_trino_adapter` | this repo, `adapters/trino/` | `TrinoAdapter` — one class, both roles, for Trino/Iceberg. Published to PyPI. |
+| `continuo-duckdb-adapter` | `continuo_duckdb_adapter` | this repo, `adapters/duckdb/` | `DuckDBAdapter` — one class, both roles, for DuckDB on a DuckLake (Postgres catalog, Parquet on S3). Published to PyPI. |
 
-All four are uv workspace members (`[tool.uv.workspace]` in the root
+All five are uv workspace members (`[tool.uv.workspace]` in the root
 `pyproject.toml`), so `uv sync --all-packages --all-groups` at the repo root
 installs everything for local development.
 
-**All four packages in the table above are published to PyPI**, under the
-same `vX.Y.Z` tag. The two engine images then **install the matching pinned
+**All five packages in the table above are published to PyPI**, under the
+same `vX.Y.Z` tag. The three engine images then **install the matching pinned
 adapter version from PyPI** — `Dockerfile.postgres` installs
 `continuo-postgres-adapter==X.Y.Z`, `Dockerfile.trino` installs
-`continuo-trino-adapter==X.Y.Z` — rather than building it from this repo's
+`continuo-trino-adapter==X.Y.Z` and `Dockerfile.duckdb` installs
+`continuo-duckdb-adapter==X.Y.Z` — rather than building it from this repo's
 source tree, so each image still ships exactly one adapter and the runtime
 still discovers it through the `continuo_engine.adapters` entry-point group
 at run time. The image **name** (`continuo-python-runtime-<engine>`) and the
@@ -97,7 +102,7 @@ adapter's pip **distribution** name (`continuo-<engine>-adapter`) are two
 different artifacts of the same adapter — same engine, same version, same
 runtime behavior, different packaging; see "Build your own container" below
 for a build shape that installs the pip package directly instead of `FROM`
-the image. All four packages are still built, type-checked, and tested by CI
+the image. All five packages are still built, type-checked, and tested by CI
 on every change.
 
 ### The result block is a frozen wire contract
@@ -149,7 +154,7 @@ the pre-flight check once before your next release; it reports every affected
 read at once:
 
 ```bash
-continuo-runtime validate contracts/ --dialect postgres   # or trino
+continuo-runtime validate contracts/ --dialect postgres   # or trino, duckdb
 ```
 
 The runtime image does not re-run this gate, so a read that passes here is
@@ -179,6 +184,12 @@ produced, never how it is checked or written.
   header but not declared are governed by the same `extra_columns` policy as
   a script node's output — `raise` (default) fails the run, `warn` drops
   them and logs a warning. See `template/contracts/example_csv.yml`.
+- **`python-api`** — a script node that fetches its own data (typically over
+  HTTP) and declares no `reads:`. `ctx.read` has nothing to return. It may name
+  one Kubernetes Secret with `secret_ref: continuo-api-<name>`; continuo
+  attaches that Secret to the node's pod, so each of its keys is an env var the
+  script reads with `os.environ[...]`. The contract never holds a secret value
+  or env var name; the Secret itself is created in the cluster by the operator.
 
 ## The script API
 
@@ -275,9 +286,9 @@ A domain repo picks its warehouse engine by which base image it builds
 `FROM`:
 
 ```dockerfile
-FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.6.0
+FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.8.0
 # or
-FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.6.0
+FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.8.0
 ```
 
 The engine is part of the image **name**; the tag is the bare version, so
@@ -288,7 +299,7 @@ pinned PyPI version of the `continuo-postgres-adapter` or
 `continuo-trino-adapter` package built from this repo's `adapters/postgres/`
 or `adapters/trino/` source (see the table above) — registered under the
 `continuo_engine.adapters` entry-point group (entry names `postgres` /
-`trino`). The runtime discovers it via `discover_adapter()` at run time, so a
+`trino` / `duckdb`). The runtime discovers it via `discover_adapter()` at run time, so a
 single image serves every node in the service and the release-time
 validation Job for it. The executor injects the warehouse connection as
 environment variables (engine-native, e.g.

{continuo_python_runtime-0.6.0.dist-info → continuo_python_runtime-0.8.0.dist-info}/RECORD

@@ -6,25 +6,26 @@ continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7D
 continuo_python_runtime/csv_loader.py,sha256=a5Hwu-YIzIiko8X1_vqShsnzjs2-OsKEUSCHLXg_dio,3346
 continuo_python_runtime/csv_source.py,sha256=7g8Tu6ieiJLvWH4Ep4nwYlOdGkjeePr12K5UHuO8TNc,4136
 continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
-continuo_python_runtime/harness.py,sha256=1NDlhHVoDF9Q1jg3LQmhA56XgRZCBUHzcknntgOwqV0,13011
+continuo_python_runtime/harness.py,sha256=vsoahI38OCaCEpg9Il7xcHYcsXp6A5cxY_fjCC8xGJk,13046
 continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
 continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
 continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
 continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
-continuo_python_runtime/contract/kinds.py,sha256=nFyCZplwpxYK8AKzKRfJp0-V49jFVZcJAW8Q4-3ezHE,4338
-continuo_python_runtime/contract/loader.py,sha256=1JARlZJSgadN0h_wvvhFV9q5nqwoDbtvwVsAGDSiXg8,14491
-continuo_python_runtime/contract/merge.py,sha256=QjHavaFe_HviR3vPtAmF5-4_3XdmgNbhl-JmrpVM8iI,7323
-continuo_python_runtime/contract/model.py,sha256=-tf5DFqRidMx_Pt8NWyJZ9i8Yystg9-YqBjqHAeX4Xg,1043
+continuo_python_runtime/contract/kinds.py,sha256=Ht-hCT7R5A1tEnPlbGL-b9FyJQBvpDWrXK9PBt7yA28,4938
+continuo_python_runtime/contract/loader.py,sha256=9HsV7dJhCwudjOPXBapYFKiZtslyXIkiZ_WfeuLh5-E,14858
+continuo_python_runtime/contract/merge.py,sha256=Zw--_-v3ERE02Nu7qqLtbHa3Hr__0a1SF-2PRqUfXAY,7594
+continuo_python_runtime/contract/model.py,sha256=WB_Qw4koP8BEwuiAZwIEsKFQRa_6QHHBK4xi4eGvPQo,1082
 continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
+continuo_python_runtime/contract/secret_ref.py,sha256=96jh2G81clpabiEWGIsxbd9r5GKEGLb4Sa1-WJ9G59Q,1131
 continuo_python_runtime/csv_readers/__init__.py,sha256=9TKvjC37C77uRaJQZuO0--zVj2OvuNwpJE6nWx7j2S8,806
 continuo_python_runtime/csv_readers/https.py,sha256=tMJ9k5F0b9SAydpon0ZlOznfjvjhF2BBLltyVmvRneE,4998
 continuo_python_runtime/csv_readers/s3.py,sha256=zmvHs3TxlN2alPKrdaHzPyREmkve2nBiAsHRXXe1mE0,2246
 continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
 continuo_python_runtime/validation/runner.py,sha256=7pbksf7MXfTNcdklCOkOXg9qfRiJtyI_dbgbL3a9LWk,14072
 continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
-continuo_python_runtime-0.6.0.dist-info/METADATA,sha256=kX73HGlQoFnSWLVicVKPNyVo-ou6HDH4360MHrcduaM,18810
-continuo_python_runtime-0.6.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-continuo_python_runtime-0.6.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
-continuo_python_runtime-0.6.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
-continuo_python_runtime-0.6.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
-continuo_python_runtime-0.6.0.dist-info/RECORD,,
+continuo_python_runtime-0.8.0.dist-info/METADATA,sha256=q2tb518F9n_Bo4LIzbhhGSGm5yJPxbrOkWJKJ6y9gxQ,19874
+continuo_python_runtime-0.8.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+continuo_python_runtime-0.8.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
+continuo_python_runtime-0.8.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
+continuo_python_runtime-0.8.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
+continuo_python_runtime-0.8.0.dist-info/RECORD,,