continuo-python-runtime 0.5.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/workflows/images.yml +2 -2
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/CHANGELOG.md +23 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/PKG-INFO +10 -4
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/README.md +9 -3
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/pyproject.toml +1 -1
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/tests/conftest.py +12 -2
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/tests/test_integration_runtime_postgres.py +1 -1
- continuo_python_runtime-0.7.0/continuo_python_runtime/contract/kinds.py +124 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/contract/loader.py +25 -63
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/contract/merge.py +33 -14
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/contract/model.py +4 -2
- continuo_python_runtime-0.7.0/continuo_python_runtime/contract/secret_ref.py +31 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/harness.py +29 -14
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/docs/boundary-contract.md +12 -3
- continuo_python_runtime-0.7.0/image-requirements-postgres.txt +2 -0
- continuo_python_runtime-0.7.0/image-requirements-trino.txt +2 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/pyproject.toml +1 -1
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/Dockerfile +1 -1
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/conftest.py +12 -2
- continuo_python_runtime-0.7.0/tests/contract/test_kinds.py +96 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/contract/test_loader.py +55 -10
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/contract/test_merge.py +50 -1
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/contract/test_model.py +5 -5
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/smoke/trino-stack/docker-compose.yml +14 -3
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_cli.py +1 -1
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_contract_loader.py +8 -3
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_harness.py +51 -2
- continuo_python_runtime-0.7.0/tests/test_kind_registries.py +22 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_no_legacy_names.py +10 -2
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/uv.lock +2 -2
- continuo_python_runtime-0.5.0/image-requirements-postgres.txt +0 -2
- continuo_python_runtime-0.5.0/image-requirements-trino.txt +0 -2
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.dockerignore +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/CODEOWNERS +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/dependabot.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/workflows/ci.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/workflows/dco.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/workflows/publish-pypi.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/workflows/release.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/workflows/security.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.gitignore +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.gitleaks.toml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/CLAUDE.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/CODE_OF_CONDUCT.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/CONTRIBUTING.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/DCO +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/Dockerfile.postgres +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/Dockerfile.trino +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/LICENSE +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/NOTICE +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/SECURITY.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/README.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/continuo_postgres_adapter/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/continuo_postgres_adapter/adapter.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/tests/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/tests/test_adapter_postgres_validation.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/tests/test_adapter_runtime_postgres.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/tests/test_integration_postgres_validation.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/README.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/continuo_trino_adapter/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/continuo_trino_adapter/adapter.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/pyproject.toml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/tests/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/tests/test_adapter_runtime_trino.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/tests/test_adapter_trino_validation.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/tests/test_integration_runtime_trino.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/trino/tests/test_integration_trino_validation.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/cli.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/closure.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/conform.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/context.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/contract/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/contract/paths.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/csv_loader.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/csv_readers/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/csv_readers/https.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/csv_readers/s3.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/csv_source.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/errors.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/hashing.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/lint.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/types.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/validation/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/validation/runner.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/validation/s3.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/LICENSE +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/NOTICE +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/README.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/continuo_engine_contract/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/continuo_engine_contract/config.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/continuo_engine_contract/port.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/continuo_engine_contract/result.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/continuo_engine_contract/sql.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/continuo_engine_contract/types.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/pyproject.toml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/tests/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/tests/test_config.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/tests/test_contract_types.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/tests/test_port.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/tests/test_result.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/contract/tests/test_sql.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/scripts/check_version_bumps.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/scripts/security-scan.sh +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/.github/workflows/release.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/Dockerfile.pip +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/README.md +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/contracts/example.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/contracts/example_csv.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/requirements.lock +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/template/scripts/example.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/contract/__init__.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/smoke/node_smoke/contracts/smoke.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/smoke/node_smoke/scripts/smoke.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/smoke/postgres-stack/docker-compose.yml +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/smoke/trino-stack/catalog/iceberg.properties +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_adapter_naming.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_closure.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_conform.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_context.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_contract_pin_consistency.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_csv_loader.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_csv_readers_https.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_csv_readers_integration.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_csv_source.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_errors.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_hashing.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_image_requirements_sync.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_image_smoke_validation.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_lint.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_package.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_template.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_types.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_validation_runner.py +0 -0
- {continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/tests/test_validation_s3.py +0 -0
{continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/.github/workflows/images.yml
RENAMED
|
@@ -83,7 +83,7 @@ jobs:
|
|
|
83
83
|
run: |
|
|
84
84
|
docker run --rm --network host \
|
|
85
85
|
-v "$PWD/tests/smoke/node_smoke:/app" \
|
|
86
|
-
-e NODE_ID=python-
|
|
86
|
+
-e NODE_ID=python-node.smoke.analytics.smoke \
|
|
87
87
|
-e TABLE_NAME=smoke \
|
|
88
88
|
-e TARGET_SCHEMA=analytics \
|
|
89
89
|
-e POSTGRES_HOST=localhost \
|
|
@@ -134,7 +134,7 @@ jobs:
|
|
|
134
134
|
run: |
|
|
135
135
|
docker run --rm \
|
|
136
136
|
-v "$PWD/tests/smoke/node_smoke:/app" \
|
|
137
|
-
-e NODE_ID=python-
|
|
137
|
+
-e NODE_ID=python-node.smoke.analytics.smoke \
|
|
138
138
|
-e TABLE_NAME=smoke \
|
|
139
139
|
-e TARGET_SCHEMA=analytics \
|
|
140
140
|
-e TRINO_HOST=host.docker.internal \
|
|
@@ -5,6 +5,29 @@ follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [0.7.0] - 2026-09-29
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `python-api` node kind: a script node with no declared reads. It may name a
|
|
13
|
+
Kubernetes Secret with `secret_ref` (must match `continuo-api-*`), which
|
|
14
|
+
continuo attaches to the node's pod as env vars. `secret_ref` is rejected on
|
|
15
|
+
every other kind. Existing contracts and their content hashes are unchanged.
|
|
16
|
+
|
|
17
|
+
## [0.6.0] - 2026-09-28
|
|
18
|
+
|
|
19
|
+
### Breaking
|
|
20
|
+
|
|
21
|
+
- The script node kind is named `python-node` (the default when `kind` is
|
|
22
|
+
omitted). A contract declaring `kind: python-model` is rejected at load.
|
|
23
|
+
Re-running `continuo-runtime merge` changes every python node's
|
|
24
|
+
`config_hash`, so each node is re-validated once on its next release.
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- Contract rules, hash inputs and run producers are looked up per kind from
|
|
29
|
+
registries pinned to `KINDS`.
|
|
30
|
+
|
|
8
31
|
## [0.5.0] - 2026-09-09
|
|
9
32
|
|
|
10
33
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: continuo-python-runtime
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
Maintainer: Simone Carolini
|
|
@@ -163,7 +163,7 @@ Every rule below (`extra_columns`, `output_columns`, "Conform rules") applies
|
|
|
163
163
|
to both kinds identically — `kind` only changes how the pre-conform table is
|
|
164
164
|
produced, never how it is checked or written.
|
|
165
165
|
|
|
166
|
-
- **`python-
|
|
166
|
+
- **`python-node`** (the default; the field may be omitted) — a script node.
|
|
167
167
|
It requires `script:` and a `reads:` map of one or more named SQL queries,
|
|
168
168
|
as described in "The script API" below.
|
|
169
169
|
- **`python-csv`** — a contract-only node: it has no script and its `reads:`
|
|
@@ -179,6 +179,12 @@ produced, never how it is checked or written.
|
|
|
179
179
|
header but not declared are governed by the same `extra_columns` policy as
|
|
180
180
|
a script node's output — `raise` (default) fails the run, `warn` drops
|
|
181
181
|
them and logs a warning. See `template/contracts/example_csv.yml`.
|
|
182
|
+
- **`python-api`** — a script node that fetches its own data (typically over
|
|
183
|
+
HTTP) and declares no `reads:`. `ctx.read` has nothing to return. It may name
|
|
184
|
+
one Kubernetes Secret with `secret_ref: continuo-api-<name>`; continuo
|
|
185
|
+
attaches that Secret to the node's pod, so each of its keys is an env var the
|
|
186
|
+
script reads with `os.environ[...]`. The contract never holds a secret value
|
|
187
|
+
or env var name; the Secret itself is created in the cluster by the operator.
|
|
182
188
|
|
|
183
189
|
## The script API
|
|
184
190
|
|
|
@@ -275,9 +281,9 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
275
281
|
`FROM`:
|
|
276
282
|
|
|
277
283
|
```dockerfile
|
|
278
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.
|
|
284
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.7.0
|
|
279
285
|
# or
|
|
280
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.
|
|
286
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.7.0
|
|
281
287
|
```
|
|
282
288
|
|
|
283
289
|
The engine is part of the image **name**; the tag is the bare version, so
|
|
@@ -143,7 +143,7 @@ Every rule below (`extra_columns`, `output_columns`, "Conform rules") applies
|
|
|
143
143
|
to both kinds identically — `kind` only changes how the pre-conform table is
|
|
144
144
|
produced, never how it is checked or written.
|
|
145
145
|
|
|
146
|
-
- **`python-
|
|
146
|
+
- **`python-node`** (the default; the field may be omitted) — a script node.
|
|
147
147
|
It requires `script:` and a `reads:` map of one or more named SQL queries,
|
|
148
148
|
as described in "The script API" below.
|
|
149
149
|
- **`python-csv`** — a contract-only node: it has no script and its `reads:`
|
|
@@ -159,6 +159,12 @@ produced, never how it is checked or written.
|
|
|
159
159
|
header but not declared are governed by the same `extra_columns` policy as
|
|
160
160
|
a script node's output — `raise` (default) fails the run, `warn` drops
|
|
161
161
|
them and logs a warning. See `template/contracts/example_csv.yml`.
|
|
162
|
+
- **`python-api`** — a script node that fetches its own data (typically over
|
|
163
|
+
HTTP) and declares no `reads:`. `ctx.read` has nothing to return. It may name
|
|
164
|
+
one Kubernetes Secret with `secret_ref: continuo-api-<name>`; continuo
|
|
165
|
+
attaches that Secret to the node's pod, so each of its keys is an env var the
|
|
166
|
+
script reads with `os.environ[...]`. The contract never holds a secret value
|
|
167
|
+
or env var name; the Secret itself is created in the cluster by the operator.
|
|
162
168
|
|
|
163
169
|
## The script API
|
|
164
170
|
|
|
@@ -255,9 +261,9 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
255
261
|
`FROM`:
|
|
256
262
|
|
|
257
263
|
```dockerfile
|
|
258
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.
|
|
264
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.7.0
|
|
259
265
|
# or
|
|
260
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.
|
|
266
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.7.0
|
|
261
267
|
```
|
|
262
268
|
|
|
263
269
|
The engine is part of the image **name**; the tag is the bare version, so
|
{continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/adapters/postgres/tests/conftest.py
RENAMED
|
@@ -43,8 +43,18 @@ def minio_container():
|
|
|
43
43
|
"-p", "0:9000",
|
|
44
44
|
"-e", "MINIO_ROOT_USER=minioadmin",
|
|
45
45
|
"-e", "MINIO_ROOT_PASSWORD=minioadmin",
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
# ghcr mirror of Bitnami's minio image (Docker Hub's minio/minio
|
|
47
|
+
# is no longer anonymously pullable). Its ENTRYPOINT is
|
|
48
|
+
# Bitnami's own setup script, not the `minio` binary, so it
|
|
49
|
+
# must be overridden for these args to reach `minio` directly;
|
|
50
|
+
# its data dir also has to be Bitnami's own
|
|
51
|
+
# /bitnami/minio/data (world-root-group-writable for the
|
|
52
|
+
# image's non-root user) since plain /data doesn't exist in
|
|
53
|
+
# this image and the binary can't create it at the
|
|
54
|
+
# filesystem root.
|
|
55
|
+
"--entrypoint", "minio",
|
|
56
|
+
"ghcr.io/carolsimone/continuo-minio:2025.7.23",
|
|
57
|
+
"server", "/bitnami/minio/data", "--address", ":9000",
|
|
48
58
|
],
|
|
49
59
|
check=True, capture_output=True, text=True, timeout=60,
|
|
50
60
|
)
|
|
@@ -354,7 +354,7 @@ def test_run_node_csv_kind_loads_minio_csv_into_postgres(clean_schema, csv_minio
|
|
|
354
354
|
doubles here -- the S3CsvSourceReader from csv_readers.reader_for talks
|
|
355
355
|
to the real minio container, and PostgresAdapter writes to the real
|
|
356
356
|
postgres stack), then conform()/ensure_table()/load() proceed exactly as
|
|
357
|
-
for a python-
|
|
357
|
+
for a python-node node.
|
|
358
358
|
"""
|
|
359
359
|
monkeypatch.setenv("S3_ENDPOINT_URL", csv_minio)
|
|
360
360
|
repo = _csv_contract_dir(tmp_path, clean_schema)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Per-kind contract rules: which fields a node of each kind must carry.
|
|
2
|
+
|
|
3
|
+
Each kind is one ``KindRules`` entry composed of a script policy, a reads rule,
|
|
4
|
+
and whether it may name a Secret. Adding a kind means adding one entry to
|
|
5
|
+
``RULES``; nothing that parses a contract branches on the kind's name.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from continuo_engine_contract.sql import ensure_single_read # type: ignore[import-untyped]
|
|
15
|
+
from sqlglot.errors import TokenError
|
|
16
|
+
|
|
17
|
+
from continuo_python_runtime.contract.model import KINDS
|
|
18
|
+
from continuo_python_runtime.csv_source import parse_csv_uri
|
|
19
|
+
from continuo_python_runtime.errors import ContractError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ReadsRule(ABC):
|
|
23
|
+
"""Validates a node's ``reads`` mapping and returns it unchanged."""
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def validate(
|
|
27
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
28
|
+
) -> dict[str, str]: ...
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SqlReads(ReadsRule):
|
|
32
|
+
"""A non-empty mapping of read name -> single SELECT."""
|
|
33
|
+
|
|
34
|
+
def validate(
|
|
35
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
36
|
+
) -> dict[str, str]:
|
|
37
|
+
if not isinstance(reads, dict) or not reads:
|
|
38
|
+
raise ContractError(
|
|
39
|
+
f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
|
|
40
|
+
)
|
|
41
|
+
for name, sql in reads.items():
|
|
42
|
+
if not isinstance(name, str) or not name.strip():
|
|
43
|
+
raise ContractError(
|
|
44
|
+
f"{label}: 'reads' name {name!r} must be a non-empty string"
|
|
45
|
+
)
|
|
46
|
+
if not isinstance(sql, str) or not sql.strip():
|
|
47
|
+
raise ContractError(
|
|
48
|
+
f"{label}: 'reads.{name}' must be a non-empty SQL string"
|
|
49
|
+
)
|
|
50
|
+
if not check_reads:
|
|
51
|
+
continue
|
|
52
|
+
try:
|
|
53
|
+
ensure_single_read(sql, dialect)
|
|
54
|
+
except (ValueError, TokenError) as exc:
|
|
55
|
+
# ensure_single_read's own message is phrased for check_binds
|
|
56
|
+
# (its only other caller today), so it's wrapped rather than
|
|
57
|
+
# surfaced bare here. TokenError is also caught: an unterminated
|
|
58
|
+
# string literal or comment fails sqlglot's tokenizer with a
|
|
59
|
+
# TokenError, a SqlglotError sibling of ParseError and not a
|
|
60
|
+
# subclass of ValueError -- despite ensure_single_read's
|
|
61
|
+
# docstring promising every rejection is a ValueError. Only
|
|
62
|
+
# TokenError, not the broader SqlglotError, is caught here: by
|
|
63
|
+
# the time control reaches this point `dialect` has already
|
|
64
|
+
# been validated once in load_contract_dir, so any other
|
|
65
|
+
# SqlglotError a future sqlglot version might raise from this
|
|
66
|
+
# call should surface as itself, not get relabeled as a
|
|
67
|
+
# rejected read.
|
|
68
|
+
raise ContractError(
|
|
69
|
+
f"{label}: 'reads.{name}' must be a single read query ({exc})"
|
|
70
|
+
) from exc
|
|
71
|
+
return dict(reads)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class CsvRead(ReadsRule):
|
|
75
|
+
"""Exactly ``{csv: <uri>}``, the uri in the grammar ``parse_csv_uri`` accepts."""
|
|
76
|
+
|
|
77
|
+
def validate(
|
|
78
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
79
|
+
) -> dict[str, str]:
|
|
80
|
+
if not isinstance(reads, dict) or set(reads) != {"csv"}:
|
|
81
|
+
raise ContractError(
|
|
82
|
+
f"{label}: a python-csv node's 'reads' must be exactly {{csv: <uri>}}"
|
|
83
|
+
)
|
|
84
|
+
try:
|
|
85
|
+
parse_csv_uri(reads["csv"])
|
|
86
|
+
except (ValueError, TypeError) as exc:
|
|
87
|
+
raise ContractError(f"{label}: invalid csv uri: {exc}") from exc
|
|
88
|
+
return dict(reads)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class NoReads(ReadsRule):
|
|
92
|
+
"""No declared reads: the node's script fetches its own data."""
|
|
93
|
+
|
|
94
|
+
def validate(
|
|
95
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
96
|
+
) -> dict[str, str]:
|
|
97
|
+
if reads is None or (isinstance(reads, dict) and not reads):
|
|
98
|
+
return {}
|
|
99
|
+
raise ContractError(
|
|
100
|
+
f"{label}: a python-api node declares no 'reads' (its script fetches its own data)"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class KindRules:
|
|
106
|
+
"""What a node of one kind must declare."""
|
|
107
|
+
|
|
108
|
+
script_required: bool
|
|
109
|
+
reads: ReadsRule
|
|
110
|
+
secret_allowed: bool = False
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
RULES: dict[str, KindRules] = {
|
|
114
|
+
"python-node": KindRules(script_required=True, reads=SqlReads()),
|
|
115
|
+
"python-csv": KindRules(script_required=False, reads=CsvRead()),
|
|
116
|
+
"python-api": KindRules(script_required=True, reads=NoReads(), secret_allowed=True),
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def rules_for(kind: object, label: str) -> KindRules:
|
|
121
|
+
"""Return the rules for ``kind``; raise ``ContractError`` for any other value."""
|
|
122
|
+
if isinstance(kind, str) and kind in RULES:
|
|
123
|
+
return RULES[kind]
|
|
124
|
+
raise ContractError(f"{label}: 'kind' must be one of {sorted(KINDS)}, got {kind!r}")
|
|
@@ -11,18 +11,17 @@ from pathlib import Path
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
|
|
13
13
|
import yaml
|
|
14
|
-
from continuo_engine_contract.sql import ensure_single_read # type: ignore[import-untyped]
|
|
15
14
|
from sqlglot import Dialect
|
|
16
|
-
from sqlglot.errors import TokenError
|
|
17
15
|
|
|
16
|
+
from continuo_python_runtime.contract.kinds import rules_for
|
|
18
17
|
from continuo_python_runtime.contract.model import (
|
|
19
18
|
CRITICALITIES,
|
|
19
|
+
DEFAULT_KIND,
|
|
20
20
|
EXTRA_COLUMNS_POLICIES,
|
|
21
|
-
KINDS,
|
|
22
21
|
Column,
|
|
23
22
|
Node,
|
|
24
23
|
)
|
|
25
|
-
from continuo_python_runtime.
|
|
24
|
+
from continuo_python_runtime.contract.secret_ref import validate_secret_ref
|
|
26
25
|
from continuo_python_runtime.errors import ContractError
|
|
27
26
|
from continuo_python_runtime.types import parse_sql_type
|
|
28
27
|
|
|
@@ -40,6 +39,7 @@ _ALLOWED_KEYS = {
|
|
|
40
39
|
"output_columns",
|
|
41
40
|
"config",
|
|
42
41
|
"content_hash",
|
|
42
|
+
"secret_ref",
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
_REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule")
|
|
@@ -173,11 +173,8 @@ def parse_node(
|
|
|
173
173
|
if unknown:
|
|
174
174
|
raise ContractError(f"{label}: unknown key(s) {sorted(unknown)}")
|
|
175
175
|
|
|
176
|
-
kind = raw.get("kind",
|
|
177
|
-
|
|
178
|
-
raise ContractError(
|
|
179
|
-
f"{label}: 'kind' must be one of {sorted(KINDS)}, got {kind!r}"
|
|
180
|
-
)
|
|
176
|
+
kind = raw.get("kind", DEFAULT_KIND)
|
|
177
|
+
rules = rules_for(kind, label)
|
|
181
178
|
|
|
182
179
|
for field in _REQUIRED_STRING_FIELDS:
|
|
183
180
|
value = raw.get(field)
|
|
@@ -191,20 +188,20 @@ def parse_node(
|
|
|
191
188
|
owner = raw["owner"]
|
|
192
189
|
schedule = raw["schedule"]
|
|
193
190
|
|
|
194
|
-
if
|
|
195
|
-
if "script" in raw:
|
|
196
|
-
raise ContractError(
|
|
197
|
-
f"{label}: 'script' is forbidden for kind python-csv "
|
|
198
|
-
"(csv nodes are contract-only)"
|
|
199
|
-
)
|
|
200
|
-
script = ""
|
|
201
|
-
else:
|
|
191
|
+
if rules.script_required:
|
|
202
192
|
raw_script = raw.get("script")
|
|
203
193
|
if not isinstance(raw_script, str) or not raw_script.strip():
|
|
204
194
|
raise ContractError(
|
|
205
195
|
f"{label}: required field 'script' must be a non-empty string"
|
|
206
196
|
)
|
|
207
197
|
script = raw_script
|
|
198
|
+
else:
|
|
199
|
+
if "script" in raw:
|
|
200
|
+
raise ContractError(
|
|
201
|
+
f"{label}: 'script' is forbidden for kind {kind} "
|
|
202
|
+
"(it is a contract-only kind)"
|
|
203
|
+
)
|
|
204
|
+
script = ""
|
|
208
205
|
|
|
209
206
|
criticality = raw.get("criticality")
|
|
210
207
|
if not isinstance(criticality, str) or criticality not in CRITICALITIES:
|
|
@@ -222,51 +219,15 @@ def parse_node(
|
|
|
222
219
|
f"got {extra_columns!r}"
|
|
223
220
|
)
|
|
224
221
|
|
|
225
|
-
reads =
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
raise ContractError(f"{label}: invalid csv uri: {exc}") from exc
|
|
235
|
-
else:
|
|
236
|
-
if not isinstance(reads, dict) or not reads:
|
|
237
|
-
raise ContractError(
|
|
238
|
-
f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
|
|
239
|
-
)
|
|
240
|
-
for name, sql in reads.items():
|
|
241
|
-
if not isinstance(name, str) or not name.strip():
|
|
242
|
-
raise ContractError(
|
|
243
|
-
f"{label}: 'reads' name {name!r} must be a non-empty string"
|
|
244
|
-
)
|
|
245
|
-
if not isinstance(sql, str) or not sql.strip():
|
|
246
|
-
raise ContractError(
|
|
247
|
-
f"{label}: 'reads.{name}' must be a non-empty SQL string"
|
|
248
|
-
)
|
|
249
|
-
if not check_reads:
|
|
250
|
-
continue
|
|
251
|
-
try:
|
|
252
|
-
ensure_single_read(sql, dialect)
|
|
253
|
-
except (ValueError, TokenError) as exc:
|
|
254
|
-
# ensure_single_read's own message is phrased for check_binds
|
|
255
|
-
# (its only other caller today), so it's wrapped rather than
|
|
256
|
-
# surfaced bare here. TokenError is also caught: an unterminated
|
|
257
|
-
# string literal or comment fails sqlglot's tokenizer with a
|
|
258
|
-
# TokenError, a SqlglotError sibling of ParseError and not a
|
|
259
|
-
# subclass of ValueError -- despite ensure_single_read's
|
|
260
|
-
# docstring promising every rejection is a ValueError. Only
|
|
261
|
-
# TokenError, not the broader SqlglotError, is caught here: by
|
|
262
|
-
# the time control reaches this point `dialect` has already
|
|
263
|
-
# been validated once in load_contract_dir, so any other
|
|
264
|
-
# SqlglotError a future sqlglot version might raise from this
|
|
265
|
-
# call should surface as itself, not get relabeled as a
|
|
266
|
-
# rejected read.
|
|
267
|
-
raise ContractError(
|
|
268
|
-
f"{label}: 'reads.{name}' must be a single read query ({exc})"
|
|
269
|
-
) from exc
|
|
222
|
+
reads = rules.reads.validate(
|
|
223
|
+
raw.get("reads"), label, dialect=dialect, check_reads=check_reads
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
secret_ref = ""
|
|
227
|
+
if "secret_ref" in raw:
|
|
228
|
+
if not rules.secret_allowed:
|
|
229
|
+
raise ContractError(f"{label}: 'secret_ref' is not allowed for kind {kind}")
|
|
230
|
+
secret_ref = validate_secret_ref(raw["secret_ref"], label)
|
|
270
231
|
|
|
271
232
|
raw_columns = raw.get("output_columns")
|
|
272
233
|
if not isinstance(raw_columns, list) or not raw_columns:
|
|
@@ -325,13 +286,14 @@ def parse_node(
|
|
|
325
286
|
schedule=schedule,
|
|
326
287
|
criticality=criticality,
|
|
327
288
|
script=script,
|
|
328
|
-
reads=
|
|
289
|
+
reads=reads,
|
|
329
290
|
output_columns=tuple(columns),
|
|
330
291
|
description=description,
|
|
331
292
|
extra_columns=extra_columns,
|
|
332
293
|
config=config,
|
|
333
294
|
content_hash=content_hash,
|
|
334
295
|
kind=kind,
|
|
296
|
+
secret_ref=secret_ref,
|
|
335
297
|
)
|
|
336
298
|
|
|
337
299
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Contract v1 merger: wire contract builder."""
|
|
2
2
|
|
|
3
|
+
from collections.abc import Callable
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import yaml
|
|
@@ -21,7 +22,7 @@ def node_entry(node: Node) -> dict:
|
|
|
21
22
|
four (source_hash, shared_code_hash, config_hash, content_hash). The
|
|
22
23
|
nullable field is always present in output_columns.
|
|
23
24
|
"""
|
|
24
|
-
|
|
25
|
+
entry = {
|
|
25
26
|
"schema": node.schema,
|
|
26
27
|
"table": node.table,
|
|
27
28
|
"owner": node.owner,
|
|
@@ -42,6 +43,11 @@ def node_entry(node: Node) -> dict:
|
|
|
42
43
|
"extra_columns": node.extra_columns,
|
|
43
44
|
"config": dict(node.config),
|
|
44
45
|
}
|
|
46
|
+
# Only a kind that may name a Secret ever carries one; emitting the key
|
|
47
|
+
# only when set keeps every other node's config_hash unchanged.
|
|
48
|
+
if node.secret_ref:
|
|
49
|
+
entry["secret_ref"] = node.secret_ref
|
|
50
|
+
return entry
|
|
45
51
|
|
|
46
52
|
|
|
47
53
|
def _lint_node_closure(
|
|
@@ -110,6 +116,30 @@ def _lint_node_closure(
|
|
|
110
116
|
)
|
|
111
117
|
|
|
112
118
|
|
|
119
|
+
def _script_hash_inputs(node: Node, repo_root: Path) -> tuple[bytes, list[bytes]]:
|
|
120
|
+
"""A script node's source is its script plus its in-repo import closure,
|
|
121
|
+
every file of which is held to the closure lint."""
|
|
122
|
+
script_path = resolve_script_path(node.script, repo_root, context=node.relation)
|
|
123
|
+
script_bytes = script_path.read_bytes()
|
|
124
|
+
closure = resolve_closure(script_path, repo_root)
|
|
125
|
+
member_bytes = [member.read_bytes() for member in closure]
|
|
126
|
+
_lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
|
|
127
|
+
return script_bytes, member_bytes
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _csv_hash_inputs(node: Node, repo_root: Path) -> tuple[bytes, list[bytes]]:
|
|
131
|
+
"""A csv node has no script and no import closure: its source is the uri.
|
|
132
|
+
New file content at the same uri is new data, not a new node version."""
|
|
133
|
+
return node.reads["csv"].encode(), []
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
_HASH_INPUTS: dict[str, Callable[[Node, Path], tuple[bytes, list[bytes]]]] = {
|
|
137
|
+
"python-node": _script_hash_inputs,
|
|
138
|
+
"python-csv": _csv_hash_inputs,
|
|
139
|
+
"python-api": _script_hash_inputs,
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
113
143
|
def build_wire_contract(
|
|
114
144
|
contract_dir: Path, repo_root: Path, service: str, *, dialect: str | None = None
|
|
115
145
|
) -> dict:
|
|
@@ -129,19 +159,8 @@ def build_wire_contract(
|
|
|
129
159
|
for node in nodes:
|
|
130
160
|
entry = node_entry(node)
|
|
131
161
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
# the uri -- new file content at the same uri is new data, not a
|
|
135
|
-
# new node version.
|
|
136
|
-
uri_bytes = node.reads["csv"].encode()
|
|
137
|
-
entry.update(hash_parts(entry, uri_bytes, []))
|
|
138
|
-
else:
|
|
139
|
-
script_path = resolve_script_path(node.script, repo_root, context=node.relation)
|
|
140
|
-
script_bytes = script_path.read_bytes()
|
|
141
|
-
closure = resolve_closure(script_path, repo_root)
|
|
142
|
-
member_bytes = [member.read_bytes() for member in closure]
|
|
143
|
-
_lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
|
|
144
|
-
entry.update(hash_parts(entry, script_bytes, member_bytes))
|
|
162
|
+
primary_bytes, member_bytes = _HASH_INPUTS[node.kind](node, repo_root)
|
|
163
|
+
entry.update(hash_parts(entry, primary_bytes, member_bytes))
|
|
145
164
|
|
|
146
165
|
wire_nodes.append(entry)
|
|
147
166
|
|
|
@@ -6,7 +6,8 @@ from typing import Any
|
|
|
6
6
|
# Module-level constants
|
|
7
7
|
CRITICALITIES = frozenset({"REGULATORY", "CORE", "SECONDARY"})
|
|
8
8
|
EXTRA_COLUMNS_POLICIES = frozenset({"raise", "warn"})
|
|
9
|
-
KINDS = frozenset({"python-
|
|
9
|
+
KINDS = frozenset({"python-node", "python-csv", "python-api"})
|
|
10
|
+
DEFAULT_KIND = "python-node"
|
|
10
11
|
CONTRACT_VERSION = 1
|
|
11
12
|
|
|
12
13
|
|
|
@@ -35,7 +36,8 @@ class Node:
|
|
|
35
36
|
extra_columns: str = "raise"
|
|
36
37
|
config: dict[str, Any] = field(default_factory=dict)
|
|
37
38
|
content_hash: str | None = None
|
|
38
|
-
kind: str =
|
|
39
|
+
kind: str = DEFAULT_KIND
|
|
40
|
+
secret_ref: str = ""
|
|
39
41
|
|
|
40
42
|
@property
|
|
41
43
|
def relation(self) -> str:
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""The name of the Kubernetes Secret a node's pod receives as env vars.
|
|
2
|
+
|
|
3
|
+
The ``continuo-api-`` prefix keeps a contract from naming any other Secret in
|
|
4
|
+
the namespace (the warehouse credentials, the platform's own credentials).
|
|
5
|
+
continuo's topology-controller and execution-controller enforce the same
|
|
6
|
+
pattern; this check gives the author the error in CI instead.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from continuo_python_runtime.errors import ContractError
|
|
14
|
+
|
|
15
|
+
SECRET_REF_PATTERN = re.compile(r"^continuo-api-[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
|
|
16
|
+
SECRET_REF_MAX_LEN = 253
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def validate_secret_ref(value: object, label: str) -> str:
|
|
20
|
+
"""Return ``value`` if it is a valid ``continuo-api-*`` Secret name."""
|
|
21
|
+
if (
|
|
22
|
+
not isinstance(value, str)
|
|
23
|
+
or len(value) > SECRET_REF_MAX_LEN
|
|
24
|
+
or not SECRET_REF_PATTERN.fullmatch(value)
|
|
25
|
+
):
|
|
26
|
+
raise ContractError(
|
|
27
|
+
f"{label}: 'secret_ref' must be a Kubernetes Secret name starting with "
|
|
28
|
+
f"'continuo-api-' (lowercase letters, digits, '-'; at most "
|
|
29
|
+
f"{SECRET_REF_MAX_LEN} chars), got {value!r}"
|
|
30
|
+
)
|
|
31
|
+
return value
|
{continuo_python_runtime-0.5.0 → continuo_python_runtime-0.7.0}/continuo_python_runtime/harness.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Container entrypoint: dispatches a single node's script and writes its output.
|
|
2
2
|
|
|
3
|
-
``run_node`` is the sole write sink for a python
|
|
3
|
+
``run_node`` is the sole write sink for a python node. It resolves the
|
|
4
4
|
node from the contract, loads and executes its script inside a
|
|
5
5
|
:class:`~continuo_python_runtime.context.RunContext`, conforms the result to
|
|
6
6
|
the declared schema, and writes it through the runtime adapter. Exactly one
|
|
@@ -16,7 +16,7 @@ import logging
|
|
|
16
16
|
import os
|
|
17
17
|
import sys
|
|
18
18
|
import uuid
|
|
19
|
-
from collections.abc import Mapping
|
|
19
|
+
from collections.abc import Callable, Mapping
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
from types import ModuleType
|
|
22
22
|
from typing import Any
|
|
@@ -208,15 +208,37 @@ def _validate_config_early(adapter: Any, node: Node) -> None:
|
|
|
208
208
|
) from exc
|
|
209
209
|
|
|
210
210
|
|
|
211
|
+
def _produce_from_script(
|
|
212
|
+
node: Node, adapter: Any, app_root: Path, reader: CsvSourceReader | None
|
|
213
|
+
) -> Any:
|
|
214
|
+
"""Import the node's script and return its ``run(ctx)`` result as Arrow."""
|
|
215
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
216
|
+
module = load_script(node, app_root)
|
|
217
|
+
ctx = RunContext(node, adapter)
|
|
218
|
+
return to_arrow(_execute_script(module, ctx))
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _produce_from_csv(
|
|
222
|
+
node: Node, adapter: Any, app_root: Path, reader: CsvSourceReader | None
|
|
223
|
+
) -> Any:
|
|
224
|
+
"""Load the node's declared csv source."""
|
|
225
|
+
return produce_csv(node, reader=reader)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
_PRODUCERS: dict[str, Callable[[Node, Any, Path, CsvSourceReader | None], Any]] = {
|
|
229
|
+
"python-node": _produce_from_script,
|
|
230
|
+
"python-api": _produce_from_script,
|
|
231
|
+
"python-csv": _produce_from_csv,
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
|
|
211
235
|
def run_node(
|
|
212
236
|
env: Mapping[str, str], adapter: Any = None, reader: CsvSourceReader | None = None
|
|
213
237
|
) -> int:
|
|
214
238
|
"""Run a single node end-to-end and print exactly one sentinel result block.
|
|
215
239
|
|
|
216
|
-
``reader``
|
|
217
|
-
|
|
218
|
-
letting that function pick a reader via ``reader_for``. Ignored for a
|
|
219
|
-
python-model node.
|
|
240
|
+
``reader`` is threaded into the python-csv producer instead of letting
|
|
241
|
+
it pick a reader via ``reader_for``; the script producer ignores it.
|
|
220
242
|
|
|
221
243
|
Returns 0 on success, 1 on any :class:`HarnessError`.
|
|
222
244
|
"""
|
|
@@ -255,14 +277,7 @@ def run_node(
|
|
|
255
277
|
|
|
256
278
|
_validate_config_early(active_adapter, node)
|
|
257
279
|
|
|
258
|
-
|
|
259
|
-
table = produce_csv(node, reader=reader)
|
|
260
|
-
else:
|
|
261
|
-
with contextlib.redirect_stdout(sys.stderr):
|
|
262
|
-
module = load_script(node, app_root)
|
|
263
|
-
ctx = RunContext(node, active_adapter)
|
|
264
|
-
raw_result = _execute_script(module, ctx)
|
|
265
|
-
table = to_arrow(raw_result)
|
|
280
|
+
table = _PRODUCERS[node.kind](node, active_adapter, app_root, reader)
|
|
266
281
|
|
|
267
282
|
conformed = conform(table, node.output_columns, node.extra_columns)
|
|
268
283
|
|