continuo-python-runtime 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/.github/workflows/ci.yml +1 -1
  2. continuo_python_runtime-0.2.0/.github/workflows/publish-pypi.yml +65 -0
  3. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/Dockerfile.postgres +6 -2
  4. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/Dockerfile.trino +6 -2
  5. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/PKG-INFO +50 -7
  6. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/README.md +47 -5
  7. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/cli.py +27 -9
  8. continuo_python_runtime-0.2.0/continuo_python_runtime/closure.py +268 -0
  9. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/contract/loader.py +155 -129
  10. continuo_python_runtime-0.2.0/continuo_python_runtime/contract/merge.py +157 -0
  11. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/contract/model.py +3 -1
  12. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/harness.py +93 -5
  13. continuo_python_runtime-0.2.0/continuo_python_runtime/hashing.py +89 -0
  14. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/lint.py +40 -26
  15. continuo_python_runtime-0.2.0/continuo_python_runtime/types.py +144 -0
  16. continuo_python_runtime-0.2.0/docs/boundary-contract.md +380 -0
  17. continuo_python_runtime-0.2.0/docs/superpowers/plans/2026-08-07-step-3b-config-three-part-hash.md +852 -0
  18. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/pyproject.toml +13 -2
  19. continuo_python_runtime-0.2.0/python-runtime-postgres/continuo_python_runtime_postgres/adapter.py +442 -0
  20. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-postgres/pyproject.toml +2 -2
  21. continuo_python_runtime-0.2.0/python-runtime-postgres/tests/test_adapter_runtime_postgres.py +543 -0
  22. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-trino/continuo_python_runtime_trino/adapter.py +125 -35
  23. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-trino/pyproject.toml +2 -2
  24. continuo_python_runtime-0.2.0/python-runtime-trino/tests/test_adapter_runtime_trino.py +501 -0
  25. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/template/.github/workflows/release.yml +8 -0
  26. continuo_python_runtime-0.2.0/template/Dockerfile +14 -0
  27. continuo_python_runtime-0.2.0/template/contracts/example.yml +38 -0
  28. continuo_python_runtime-0.2.0/template/scripts/example.py +17 -0
  29. continuo_python_runtime-0.2.0/tests/conftest.py +143 -0
  30. continuo_python_runtime-0.2.0/tests/contract/test_loader.py +625 -0
  31. continuo_python_runtime-0.2.0/tests/contract/test_merge.py +430 -0
  32. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/contract/test_model.py +6 -0
  33. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_cli.py +126 -0
  34. continuo_python_runtime-0.2.0/tests/test_closure.py +448 -0
  35. continuo_python_runtime-0.2.0/tests/test_harness.py +614 -0
  36. continuo_python_runtime-0.2.0/tests/test_hashing.py +188 -0
  37. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_lint.py +80 -0
  38. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_template.py +17 -2
  39. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_types.py +54 -0
  40. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/uv.lock +23 -9
  41. continuo_python_runtime-0.1.0/.github/workflows/publish-pypi.yml +0 -129
  42. continuo_python_runtime-0.1.0/continuo_python_runtime/contract/merge.py +0 -80
  43. continuo_python_runtime-0.1.0/continuo_python_runtime/hashing.py +0 -34
  44. continuo_python_runtime-0.1.0/continuo_python_runtime/types.py +0 -208
  45. continuo_python_runtime-0.1.0/docs/boundary-contract.md +0 -152
  46. continuo_python_runtime-0.1.0/python-runtime-postgres/continuo_python_runtime_postgres/adapter.py +0 -229
  47. continuo_python_runtime-0.1.0/python-runtime-postgres/tests/test_adapter_runtime_postgres.py +0 -115
  48. continuo_python_runtime-0.1.0/python-runtime-trino/tests/test_adapter_runtime_trino.py +0 -210
  49. continuo_python_runtime-0.1.0/template/Dockerfile +0 -8
  50. continuo_python_runtime-0.1.0/template/contracts/example.yml +0 -12
  51. continuo_python_runtime-0.1.0/template/scripts/example.py +0 -3
  52. continuo_python_runtime-0.1.0/tests/conftest.py +0 -81
  53. continuo_python_runtime-0.1.0/tests/contract/test_loader.py +0 -344
  54. continuo_python_runtime-0.1.0/tests/contract/test_merge.py +0 -118
  55. continuo_python_runtime-0.1.0/tests/test_harness.py +0 -261
  56. continuo_python_runtime-0.1.0/tests/test_hashing.py +0 -53
  57. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/.dockerignore +0 -0
  58. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/.github/workflows/images.yml +0 -0
  59. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/.gitignore +0 -0
  60. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/__init__.py +0 -0
  61. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/conform.py +0 -0
  62. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/context.py +0 -0
  63. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/contract/__init__.py +0 -0
  64. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/contract/paths.py +0 -0
  65. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/continuo_python_runtime/errors.py +0 -0
  66. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/docs/superpowers/plans/2026-07-31-python-runtime.md +0 -0
  67. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/docs/superpowers/specs/2026-07-31-python-runtime-design.md +0 -0
  68. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-postgres/README.md +0 -0
  69. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-postgres/continuo_python_runtime_postgres/__init__.py +0 -0
  70. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-postgres/tests/__init__.py +0 -0
  71. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-postgres/tests/test_integration_runtime_postgres.py +0 -0
  72. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-trino/README.md +0 -0
  73. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-trino/continuo_python_runtime_trino/__init__.py +0 -0
  74. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-trino/tests/__init__.py +0 -0
  75. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/python-runtime-trino/tests/test_integration_runtime_trino.py +0 -0
  76. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/template/README.md +0 -0
  77. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/__init__.py +0 -0
  78. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/contract/__init__.py +0 -0
  79. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/smoke/node_smoke/contracts/smoke.yml +0 -0
  80. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/smoke/node_smoke/scripts/smoke.py +0 -0
  81. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/smoke/postgres-stack/docker-compose.yml +0 -0
  82. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/smoke/trino-stack/catalog/iceberg.properties +0 -0
  83. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/smoke/trino-stack/docker-compose.yml +0 -0
  84. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_conform.py +0 -0
  85. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_context.py +0 -0
  86. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_errors.py +0 -0
  87. {continuo_python_runtime-0.1.0 → continuo_python_runtime-0.2.0}/tests/test_package.py +0 -0
@@ -8,7 +8,7 @@ jobs:
8
8
  steps:
9
9
  - uses: actions/checkout@v4
10
10
  - uses: astral-sh/setup-uv@v5
11
- # continuo-validation-contract resolves from PyPI (==0.3.0).
11
+ # continuo-validation-contract resolves from PyPI (==0.4.0).
12
12
  - name: Sync
13
13
  run: uv sync --all-packages --all-groups
14
14
  - name: Lint
@@ -0,0 +1,65 @@
1
+ name: publish-pypi
2
+
3
+ # Publishes the continuo-python-runtime package via PyPI Trusted Publishing
4
+ # (OIDC, no stored token).
5
+ #
6
+ # The two engine adapters (continuo-python-runtime-postgres /
7
+ # continuo-python-runtime-trino) are deliberately NOT published here. Nothing
8
+ # installs them from an index: the runtime images build them from the build
9
+ # context (see Dockerfile.postgres / Dockerfile.trino and images.yml), and the
10
+ # harness package does not depend on them — adapters are found at run time
11
+ # through the `continuo_runtime.adapters` entry-point group, inside the image
12
+ # that installed exactly one. Publishing them would only add releases with no
13
+ # consumer. They remain uv workspace members and are still built, typed, and
14
+ # tested by ci.yml.
15
+ #
16
+ # Tag glob note: "python-runtime-v*" requires a literal "v" immediately after
17
+ # the dash, so it matches only the harness tag (python-runtime-v1.2.3[-testN]).
18
+ # Tag `<pkg>-v<ver>-test<n>` publishes to TestPyPI; `<pkg>-v<ver>` publishes to
19
+ # real PyPI. The GitHub environment name is what the PyPI "pending publisher"
20
+ # is registered against.
21
+ #
22
+ # Ordering: continuo-validation-contract==0.4.0 must exist on the target index
23
+ # before publishing here — consumers of the wheel resolve it from that index
24
+ # ([tool.uv.sources] is dev-only and not embedded in the wheel).
25
+ on:
26
+ push:
27
+ tags:
28
+ - "python-runtime-v*"
29
+
30
+ permissions: {}
31
+
32
+ jobs:
33
+ publish-runtime:
34
+ if: startsWith(github.ref_name, 'python-runtime-v')
35
+ runs-on: ubuntu-latest
36
+ environment: ${{ contains(github.ref_name, '-test') && 'testpypi' || 'pypi' }}
37
+ permissions:
38
+ id-token: write # OIDC token for Trusted Publishing
39
+ contents: read # actions/checkout needs read access to the repo
40
+ steps:
41
+ - uses: actions/checkout@v4
42
+ - uses: astral-sh/setup-uv@v5
43
+ - name: Test before publishing
44
+ run: |
45
+ uv sync --all-groups
46
+ uv run pytest -q
47
+ - name: Build the sdist + wheel
48
+ run: uv build -o dist
49
+ - name: Publish to TestPyPI
50
+ if: contains(github.ref_name, '-test')
51
+ # Pinned to a commit SHA. release/v1 is a moving branch, and this job
52
+ # holds id-token: write for PyPI Trusted Publishing — a compromised
53
+ # revision could mint a token and publish under this project's name.
54
+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
55
+ with:
56
+ repository-url: https://test.pypi.org/legacy/
57
+ packages-dir: dist
58
+ - name: Publish to PyPI
59
+ if: ${{ !contains(github.ref_name, '-test') }}
60
+ # Pinned to a commit SHA. release/v1 is a moving branch, and this job
61
+ # holds id-token: write for PyPI Trusted Publishing — a compromised
62
+ # revision could mint a token and publish under this project's name.
63
+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
64
+ with:
65
+ packages-dir: dist
@@ -5,9 +5,13 @@ COPY pyproject.toml README.md /src/
5
5
  COPY continuo_python_runtime /src/continuo_python_runtime
6
6
  COPY python-runtime-postgres /src/python-runtime-postgres
7
7
  RUN pip install --no-cache-dir \
8
- "continuo-validation-contract==0.3.0" \
8
+ "continuo-validation-contract==0.4.0" \
9
9
  /src/python-runtime-postgres \
10
10
  /src
11
- ENV CONTRACT_DIR=/app/contracts APP_ROOT=/app
11
+ # PYTHONPATH is belt-and-braces: the harness also inserts APP_ROOT and the
12
+ # node script's own directory at the front of sys.path before executing it,
13
+ # so a script's in-repo helper imports resolve. Neither WORKDIR nor the
14
+ # console-script entrypoint puts /app on sys.path by itself.
15
+ ENV CONTRACT_DIR=/app/contracts APP_ROOT=/app PYTHONPATH=/app
12
16
  WORKDIR /app
13
17
  ENTRYPOINT ["continuo-runtime", "run"]
@@ -5,9 +5,13 @@ COPY pyproject.toml README.md /src/
5
5
  COPY continuo_python_runtime /src/continuo_python_runtime
6
6
  COPY python-runtime-trino /src/python-runtime-trino
7
7
  RUN pip install --no-cache-dir \
8
- "continuo-validation-contract==0.3.0" \
8
+ "continuo-validation-contract==0.4.0" \
9
9
  /src/python-runtime-trino \
10
10
  /src
11
- ENV CONTRACT_DIR=/app/contracts APP_ROOT=/app
11
+ # PYTHONPATH is belt-and-braces: the harness also inserts APP_ROOT and the
12
+ # node script's own directory at the front of sys.path before executing it,
13
+ # so a script's in-repo helper imports resolve. Neither WORKDIR nor the
14
+ # console-script entrypoint puts /app on sys.path by itself.
15
+ ENV CONTRACT_DIR=/app/contracts APP_ROOT=/app PYTHONPATH=/app
12
16
  WORKDIR /app
13
17
  ENTRYPOINT ["continuo-runtime", "run"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: continuo-python-runtime
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
5
5
  Author: Simone Carolini
6
6
  Maintainer: Simone Carolini
@@ -8,9 +8,10 @@ Classifier: Development Status :: 4 - Beta
8
8
  Classifier: Intended Audience :: Developers
9
9
  Classifier: Programming Language :: Python :: 3.14
10
10
  Requires-Python: >=3.14
11
- Requires-Dist: continuo-validation-contract==0.3.0
11
+ Requires-Dist: continuo-validation-contract==0.4.0
12
12
  Requires-Dist: pyarrow==25.0.0
13
13
  Requires-Dist: pyyaml==6.0.3
14
+ Requires-Dist: sqlglot==30.15.0
14
15
  Description-Content-Type: text/markdown
15
16
 
16
17
  # Continuo Python Runtime
@@ -51,17 +52,26 @@ data-plane adapters here.
51
52
  | Package (distribution name) | Module | Lives in | Role |
52
53
  | --- | --- | --- | --- |
53
54
  | `continuo-python-runtime` | `continuo_python_runtime` | this repo (root) | Harness: CLI, `conform()`, `RunContext`, error taxonomy. |
54
- | `continuo-python-runtime-postgres` | `continuo_python_runtime_postgres` | this repo, `python-runtime-postgres/` | Data-plane `RuntimeAdapter` for Postgres (`fetch`/`ensure_table`/`load`). |
55
- | `continuo-python-runtime-trino` | `continuo_python_runtime_trino` | this repo, `python-runtime-trino/` | Data-plane `RuntimeAdapter` for Trino/Iceberg. |
55
+ | `continuo-python-runtime-postgres` | `continuo_python_runtime_postgres` | this repo, `python-runtime-postgres/` | Data-plane `RuntimeAdapter` for Postgres (`fetch`/`ensure_table`/`load`). **Not published to PyPI** — built from source into the image. |
56
+ | `continuo-python-runtime-trino` | `continuo_python_runtime_trino` | this repo, `python-runtime-trino/` | Data-plane `RuntimeAdapter` for Trino/Iceberg. **Not published to PyPI** — built from source into the image. |
56
57
  | `continuo-validation-contract` | `continuo_validation_contract` | `continuo-validation-runners` | The published contract (schema, `RuntimeAdapter` port, result-block format) both sides depend on. |
57
58
  | `continuo-validation-postgres` / `continuo-validation-trino` | `continuo_validation_postgres` / `continuo_validation_trino` | `continuo-validation-runners` | Validation-side (lint/merge, no live warehouse I/O) adapters — not to be confused with the data-plane adapters above. |
58
59
 
59
60
  All three packages built in this repo resolve `continuo-validation-contract`
60
- from PyPI (`==0.3.0`); the two adapter packages are uv workspace members
61
+ from PyPI (`==0.4.0`); the two adapter packages are uv workspace members
61
62
  (`[tool.uv.workspace]` in the root `pyproject.toml`), so `uv sync
62
63
  --all-packages --all-groups` at the repo root installs everything for local
63
64
  development.
64
65
 
66
+ **Only `continuo-python-runtime` is published to PyPI.** The two engine
67
+ adapters are built **from source into the runtime images**: `Dockerfile.postgres`
68
+ and `Dockerfile.trino` `pip install` them out of the build context, so each
69
+ image ships exactly one adapter and the harness discovers it through the
70
+ `continuo_runtime.adapters` entry-point group at run time. Nothing installs
71
+ them from an index — the harness package does not depend on them, and domain
72
+ repos get their adapter by building `FROM` a published base image. They are
73
+ still built, type-checked, and tested by CI on every change.
74
+
65
75
  ## Quickstart for domain teams
66
76
 
67
77
  1. Copy `template/` into a new repository.
@@ -85,6 +95,28 @@ development.
85
95
  and merges the contracts, builds and pushes the image, uploads the merged
86
96
  contract to S3, and POSTs the release.
87
97
 
98
+ ### Upgrading an existing domain repo
99
+
100
+ `validate` / `merge` / `hash` now hand every declared read to a real SQL
101
+ parser (sqlglot, via `continuo_validation_contract.sql.ensure_single_read`)
102
+ instead of scanning it for a leading `SELECT`/`WITH`. Two things follow for a
103
+ repo written before this, on its next release: SQL a driver would accept but
104
+ a parser will not — most commonly a driver-specific bind placeholder like
105
+ psycopg2's `%(name)s`, which `ctx.read(name)` could never have used anyway —
106
+ now fails validation, and engine-specific syntax (postgres `~`, `@>`, …)
107
+ needs `--dialect <engine>`, which a repo should be passing regardless since
108
+ Continuo bind-checks every read in the install's own warehouse dialect. Run
109
+ the pre-flight check once before your next release; it reports every affected
110
+ read at once:
111
+
112
+ ```bash
113
+ continuo-runtime validate contracts/ --dialect postgres # or trino
114
+ ```
115
+
116
+ The runtime image does not re-run this gate, so a read that passes here is
117
+ not re-judged under a different grammar in production. See
118
+ `docs/boundary-contract.md` §13.1.
119
+
88
120
  ## The script API
89
121
 
90
122
  A node script is a Python file with exactly one required entry point:
@@ -125,6 +157,17 @@ def run(ctx):
125
157
  are the driver-import and data-access-call rules, together with the fact
126
158
  that `RunContext` only exposes `ctx.read()` — all warehouse access goes
127
159
  through it.
160
+ - Scripts may import shared in-repo helpers. Before executing a script the
161
+ harness puts the repo root (`APP_ROOT`) and the script's own directory on
162
+ `sys.path`, so both `import helpers` (a sibling of the script) and
163
+ `from lib.shared import ...` (anywhere under the repo root) work, including
164
+ from inside `run()`. Every helper a script reaches transitively is folded
165
+ into `shared_code_hash`, so editing one re-fingerprints the node — but the
166
+ hash does not put the file in the image: **`COPY` every directory your
167
+ scripts import from in your `Dockerfile`**, or the release is valid and the
168
+ node dies with `ModuleNotFoundError` on its first run. Because the repo root
169
+ precedes the standard library on `sys.path`, avoid naming a top-level module
170
+ after a stdlib one (`types.py`, `json.py`, `logging.py`, …).
128
171
  - The harness — not the script — performs the write. It calls `conform()`
129
172
  on whatever `run()` returned and issues the only INSERT; the script never
130
173
  writes directly.
@@ -165,9 +208,9 @@ A domain repo picks its warehouse engine by which base image it builds
165
208
  `FROM`:
166
209
 
167
210
  ```dockerfile
168
- FROM ghcr.io/carolsimone/continuo-python-runtime:v0.1.0-postgres
211
+ FROM ghcr.io/carolsimone/continuo-python-runtime:v0.2.0-postgres
169
212
  # or
170
- FROM ghcr.io/carolsimone/continuo-python-runtime:v0.1.0-trino
213
+ FROM ghcr.io/carolsimone/continuo-python-runtime:v0.2.0-trino
171
214
  ```
172
215
 
173
216
  Each image bakes in exactly one `RuntimeAdapter` for that engine — installed
@@ -36,17 +36,26 @@ data-plane adapters here.
36
36
  | Package (distribution name) | Module | Lives in | Role |
37
37
  | --- | --- | --- | --- |
38
38
  | `continuo-python-runtime` | `continuo_python_runtime` | this repo (root) | Harness: CLI, `conform()`, `RunContext`, error taxonomy. |
39
- | `continuo-python-runtime-postgres` | `continuo_python_runtime_postgres` | this repo, `python-runtime-postgres/` | Data-plane `RuntimeAdapter` for Postgres (`fetch`/`ensure_table`/`load`). |
40
- | `continuo-python-runtime-trino` | `continuo_python_runtime_trino` | this repo, `python-runtime-trino/` | Data-plane `RuntimeAdapter` for Trino/Iceberg. |
39
+ | `continuo-python-runtime-postgres` | `continuo_python_runtime_postgres` | this repo, `python-runtime-postgres/` | Data-plane `RuntimeAdapter` for Postgres (`fetch`/`ensure_table`/`load`). **Not published to PyPI** — built from source into the image. |
40
+ | `continuo-python-runtime-trino` | `continuo_python_runtime_trino` | this repo, `python-runtime-trino/` | Data-plane `RuntimeAdapter` for Trino/Iceberg. **Not published to PyPI** — built from source into the image. |
41
41
  | `continuo-validation-contract` | `continuo_validation_contract` | `continuo-validation-runners` | The published contract (schema, `RuntimeAdapter` port, result-block format) both sides depend on. |
42
42
  | `continuo-validation-postgres` / `continuo-validation-trino` | `continuo_validation_postgres` / `continuo_validation_trino` | `continuo-validation-runners` | Validation-side (lint/merge, no live warehouse I/O) adapters — not to be confused with the data-plane adapters above. |
43
43
 
44
44
  All three packages built in this repo resolve `continuo-validation-contract`
45
- from PyPI (`==0.3.0`); the two adapter packages are uv workspace members
45
+ from PyPI (`==0.4.0`); the two adapter packages are uv workspace members
46
46
  (`[tool.uv.workspace]` in the root `pyproject.toml`), so `uv sync
47
47
  --all-packages --all-groups` at the repo root installs everything for local
48
48
  development.
49
49
 
50
+ **Only `continuo-python-runtime` is published to PyPI.** The two engine
51
+ adapters are built **from source into the runtime images**: `Dockerfile.postgres`
52
+ and `Dockerfile.trino` `pip install` them out of the build context, so each
53
+ image ships exactly one adapter and the harness discovers it through the
54
+ `continuo_runtime.adapters` entry-point group at run time. Nothing installs
55
+ them from an index — the harness package does not depend on them, and domain
56
+ repos get their adapter by building `FROM` a published base image. They are
57
+ still built, type-checked, and tested by CI on every change.
58
+
50
59
  ## Quickstart for domain teams
51
60
 
52
61
  1. Copy `template/` into a new repository.
@@ -70,6 +79,28 @@ development.
70
79
  and merges the contracts, builds and pushes the image, uploads the merged
71
80
  contract to S3, and POSTs the release.
72
81
 
82
+ ### Upgrading an existing domain repo
83
+
84
+ `validate` / `merge` / `hash` now hand every declared read to a real SQL
85
+ parser (sqlglot, via `continuo_validation_contract.sql.ensure_single_read`)
86
+ instead of scanning it for a leading `SELECT`/`WITH`. Two things follow for a
87
+ repo written before this, on its next release: SQL a driver would accept but
88
+ a parser will not — most commonly a driver-specific bind placeholder like
89
+ psycopg2's `%(name)s`, which `ctx.read(name)` could never have used anyway —
90
+ now fails validation, and engine-specific syntax (postgres `~`, `@>`, …)
91
+ needs `--dialect <engine>`, which a repo should be passing regardless since
92
+ Continuo bind-checks every read in the install's own warehouse dialect. Run
93
+ the pre-flight check once before your next release; it reports every affected
94
+ read at once:
95
+
96
+ ```bash
97
+ continuo-runtime validate contracts/ --dialect postgres # or trino
98
+ ```
99
+
100
+ The runtime image does not re-run this gate, so a read that passes here is
101
+ not re-judged under a different grammar in production. See
102
+ `docs/boundary-contract.md` §13.1.
103
+
73
104
  ## The script API
74
105
 
75
106
  A node script is a Python file with exactly one required entry point:
@@ -110,6 +141,17 @@ def run(ctx):
110
141
  are the driver-import and data-access-call rules, together with the fact
111
142
  that `RunContext` only exposes `ctx.read()` — all warehouse access goes
112
143
  through it.
144
+ - Scripts may import shared in-repo helpers. Before executing a script the
145
+ harness puts the repo root (`APP_ROOT`) and the script's own directory on
146
+ `sys.path`, so both `import helpers` (a sibling of the script) and
147
+ `from lib.shared import ...` (anywhere under the repo root) work, including
148
+ from inside `run()`. Every helper a script reaches transitively is folded
149
+ into `shared_code_hash`, so editing one re-fingerprints the node — but the
150
+ hash does not put the file in the image: **`COPY` every directory your
151
+ scripts import from in your `Dockerfile`**, or the release is valid and the
152
+ node dies with `ModuleNotFoundError` on its first run. Because the repo root
153
+ precedes the standard library on `sys.path`, avoid naming a top-level module
154
+ after a stdlib one (`types.py`, `json.py`, `logging.py`, …).
113
155
  - The harness — not the script — performs the write. It calls `conform()`
114
156
  on whatever `run()` returned and issues the only INSERT; the script never
115
157
  writes directly.
@@ -150,9 +192,9 @@ A domain repo picks its warehouse engine by which base image it builds
150
192
  `FROM`:
151
193
 
152
194
  ```dockerfile
153
- FROM ghcr.io/carolsimone/continuo-python-runtime:v0.1.0-postgres
195
+ FROM ghcr.io/carolsimone/continuo-python-runtime:v0.2.0-postgres
154
196
  # or
155
- FROM ghcr.io/carolsimone/continuo-python-runtime:v0.1.0-trino
197
+ FROM ghcr.io/carolsimone/continuo-python-runtime:v0.2.0-trino
156
198
  ```
157
199
 
158
200
  Each image bakes in exactly one `RuntimeAdapter` for that engine — installed
@@ -30,11 +30,17 @@ def main(argv: list[str] | None = None) -> int:
30
30
  parser = argparse.ArgumentParser(prog="continuo-runtime")
31
31
  subparsers = parser.add_subparsers(dest="command", required=True)
32
32
 
33
+ dialect_help = (
34
+ "sqlglot dialect the reads are authored in (e.g. postgres, trino); "
35
+ "defaults to sqlglot's dialect-neutral parser."
36
+ )
37
+
33
38
  # validate subcommand
34
39
  validate_parser = subparsers.add_parser(
35
40
  "validate", help="Validate contract directory"
36
41
  )
37
42
  validate_parser.add_argument("contract_dir", help="Path to contract directory")
43
+ validate_parser.add_argument("--dialect", default=None, help=dialect_help)
38
44
 
39
45
  # merge subcommand
40
46
  merge_parser = subparsers.add_parser(
@@ -44,6 +50,7 @@ def main(argv: list[str] | None = None) -> int:
44
50
  merge_parser.add_argument("--service", required=True, help="Service name")
45
51
  merge_parser.add_argument("--repo-root", required=True, help="Repository root path")
46
52
  merge_parser.add_argument("--out", required=True, help="Output file path")
53
+ merge_parser.add_argument("--dialect", default=None, help=dialect_help)
47
54
 
48
55
  # hash subcommand
49
56
  hash_parser = subparsers.add_parser(
@@ -53,6 +60,7 @@ def main(argv: list[str] | None = None) -> int:
53
60
  hash_parser.add_argument(
54
61
  "--repo-root", required=True, help="Repository root path"
55
62
  )
63
+ hash_parser.add_argument("--dialect", default=None, help=dialect_help)
56
64
 
57
65
  # lint subcommand
58
66
  lint_parser = subparsers.add_parser(
@@ -69,11 +77,13 @@ def main(argv: list[str] | None = None) -> int:
69
77
 
70
78
  try:
71
79
  if args.command == "validate":
72
- return cmd_validate(args.contract_dir)
80
+ return cmd_validate(args.contract_dir, args.dialect)
73
81
  elif args.command == "merge":
74
- return cmd_merge(args.contract_dir, args.service, args.repo_root, args.out)
82
+ return cmd_merge(
83
+ args.contract_dir, args.service, args.repo_root, args.out, args.dialect
84
+ )
75
85
  elif args.command == "hash":
76
- return cmd_hash(args.contract_dir, args.repo_root)
86
+ return cmd_hash(args.contract_dir, args.repo_root, args.dialect)
77
87
  elif args.command == "lint":
78
88
  return cmd_lint(args.path)
79
89
  elif args.command == "run":
@@ -88,27 +98,35 @@ def main(argv: list[str] | None = None) -> int:
88
98
  return 0
89
99
 
90
100
 
91
- def cmd_validate(contract_dir: str) -> int:
101
+ def cmd_validate(contract_dir: str, dialect: str | None = None) -> int:
92
102
  """Validate contracts in a directory."""
93
- load_contract_dir(Path(contract_dir))
103
+ load_contract_dir(Path(contract_dir), dialect=dialect)
94
104
  return 0
95
105
 
96
106
 
97
- def cmd_merge(contract_dir: str, service: str, repo_root: str, out: str) -> int:
107
+ def cmd_merge(
108
+ contract_dir: str,
109
+ service: str,
110
+ repo_root: str,
111
+ out: str,
112
+ dialect: str | None = None,
113
+ ) -> int:
98
114
  """Merge contracts into a wire contract file."""
99
- doc = build_wire_contract(Path(contract_dir), Path(repo_root), service)
115
+ doc = build_wire_contract(Path(contract_dir), Path(repo_root), service, dialect=dialect)
100
116
  write_wire_contract(doc, Path(out))
101
117
  return 0
102
118
 
103
119
 
104
- def cmd_hash(contract_dir: str, repo_root: str) -> int:
120
+ def cmd_hash(contract_dir: str, repo_root: str, dialect: str | None = None) -> int:
105
121
  """Print relation and content hash for each node.
106
122
 
107
123
  Reuses build_wire_contract for consistent hashing and error handling.
108
124
  Service value is irrelevant to per-node hashes since entries don't include it.
109
125
  """
110
126
  # Reuse build_wire_contract for consistent hashing and error handling
111
- doc = build_wire_contract(Path(contract_dir), Path(repo_root), service="_hash")
127
+ doc = build_wire_contract(
128
+ Path(contract_dir), Path(repo_root), service="_hash", dialect=dialect
129
+ )
112
130
 
113
131
  # Print relation\thash from wire contract nodes (already sorted by relation)
114
132
  for node_entry in doc["nodes"]:
@@ -0,0 +1,268 @@
1
+ """Static in-repo import-closure resolver.
2
+
3
+ CI's ``content_hash`` is the sole change detector Continuo uses to decide
4
+ whether a node needs revalidation. Until this module existed, that hash
5
+ covered only the node's own script file, so a byte edit to a shared helper
6
+ module the script imports changed nothing — the node kept running against a
7
+ stale fingerprint in production. ``resolve_closure`` closes that gap: it
8
+ returns the transitive set of in-repo Python files a script reaches through
9
+ its ``import`` statements, so a later stage can fold their bytes into the
10
+ hash too.
11
+
12
+ That framing decides every ambiguous call in the algorithm below:
13
+ under-inclusion is a correctness bug (a stale node silently running in
14
+ production), while over-inclusion is merely a spurious revalidation.
15
+
16
+ - Search roots (rule 4) are the fixed, ordered pair ``(repo_root,
17
+ script_dir)`` — ``script_dir`` computed once from the seed script and
18
+ reused, unchanged, for every resolution in the traversal, never the
19
+ current file's own directory. This mirrors
20
+ :func:`continuo_python_runtime.harness.ensure_import_paths` exactly, which
21
+ puts that same pair on ``sys.path`` for the process lifetime: the static
22
+ model and the runtime model are provably the same list. A module importing
23
+ a sibling that is on neither root (e.g. ``lib/shared.py`` doing ``import
24
+ sibling`` for a file at ``lib/sibling.py``, when the script itself lives
25
+ elsewhere) raises ``ImportError`` at run time, so excluding that sibling
26
+ from the closure cannot hide a stale-node bug — that node cannot run at
27
+ all. When the script lives at ``repo_root`` itself, the pair collapses to
28
+ a single root so the same candidate is not probed twice.
29
+ - Within a search root, the package form (``<root>/a/b/c/__init__.py``) is
30
+ tried before the module-file form (``<root>/a/b/c.py``), matching
31
+ Python's own lookup order: when both exist, ``import a.b.c`` binds the
32
+ package, and the module file of the same name is never executed.
33
+ - ``from pkg import name`` (rule 3) is expanded to include both ``pkg`` and
34
+ ``pkg.name`` as candidate dotted names, because ``name`` may be a
35
+ submodule (a file) rather than an attribute of ``pkg`` — the two are
36
+ indistinguishable from the import statement's syntax alone. The same
37
+ applies to relative imports: ``from . import name`` always includes the
38
+ bare enclosing package as a candidate too, not only when the import also
39
+ names a submodule (``from .mod import name``) — real Python executes the
40
+ package's ``__init__.py`` either way, so treating the two forms
41
+ asymmetrically would under-include exactly the file this module exists to
42
+ stop missing. This choice leans deliberately toward over-inclusion:
43
+ ``name`` may turn out to be a plain attribute rather than a submodule, in
44
+ which case the extra candidate simply fails to resolve.
45
+
46
+ Dynamic-import constructs (``importlib``, ``builtins``, ``__builtins__``,
47
+ ``__import__``, ``exec``, ``eval``, ``.import_module``) are rejected outright
48
+ rather than degrading to "resolve what we can": whatever a script imports
49
+ through one of those, this static analysis cannot see, so the hash could
50
+ never be trusted to reflect it.
51
+ """
52
+
53
+ from __future__ import annotations
54
+
55
+ import ast
56
+ from collections import deque
57
+ from pathlib import Path
58
+
59
+ from continuo_python_runtime.errors import ContractError
60
+
61
+ _DYNAMIC_IMPORT_MODULES = frozenset({"importlib", "builtins"})
62
+ _DYNAMIC_IMPORT_NAMES = frozenset({"__import__", "exec", "eval", "__builtins__"})
63
+ _DYNAMIC_IMPORT_ATTRS = frozenset({"import_module", "__import__"})
64
+
65
+
66
+ def dynamic_import_violations(tree: ast.AST) -> list[tuple[int, str]]:
67
+ """Return `(lineno, construct)` for every dynamic-import construct in *tree*.
68
+
69
+ Flags, per AST node type:
70
+
71
+ - `ast.Import`: any alias whose root module (`name.split(".")[0]`) is
72
+ `importlib` or `builtins` — e.g. `import importlib.util`, `import
73
+ builtins`. `construct` is that root module name.
74
+ - `ast.ImportFrom`: if the root module is `importlib` or `builtins`,
75
+ one violation naming that module — and nothing else from the same
76
+ statement (one statement, one violation; its aliases are not also
77
+ inspected). Otherwise, any `alias.name` in `{"__import__", "exec",
78
+ "eval"}` is a violation named for that alias — this catches
79
+ re-exports of those names from a module that is neither `importlib`
80
+ nor `builtins` (e.g. `from somewhere import exec as e`), which no
81
+ other rule here would see.
82
+ - `ast.Name`: `id` in `{"__import__", "exec", "eval", "__builtins__"}` (a
83
+ bare reference or call). `__builtins__` is included because it is
84
+ injected into every module's globals by the harness's own loader
85
+ (`spec_from_file_location` + `exec_module`, see `harness.py`) as the
86
+ builtins *dict* — so `__builtins__['exec'](...)` and
87
+ `__builtins__['__import__'](...)` need no preceding `import builtins`
88
+ at all, and flagging the bare name catches every subscript form in one
89
+ rule.
90
+ - `ast.Attribute`: `attr` in `{"import_module", "__import__"}`.
91
+ Deliberately NOT `exec`/`eval`: those are legitimate method names on
92
+ arbitrary objects (pandas `DataFrame.eval`/`DataFrame.query`, used in
93
+ ordinary node scripts that return a dataframe), and flagging them here
94
+ would reject that legitimate user code. `builtins.exec` /
95
+ `builtins.eval` (attribute access on the `builtins` module after an
96
+ explicit `import builtins`) are still caught, because that `import` is
97
+ itself an `ast.Import` violation above — but `__builtins__['exec']`
98
+ needs no such import, which is exactly what the `ast.Name` rule above
99
+ exists to close.
100
+
101
+ `construct` is the offending name as written: "importlib", "builtins",
102
+ "__import__", "exec", "eval", "__builtins__", or "import_module". Sorted
103
+ by lineno, then construct.
104
+ """
105
+ violations: list[tuple[int, str]] = []
106
+ for node in ast.walk(tree):
107
+ if isinstance(node, ast.Import):
108
+ for alias in node.names:
109
+ root = alias.name.split(".")[0]
110
+ if root in _DYNAMIC_IMPORT_MODULES:
111
+ violations.append((node.lineno, root))
112
+ elif isinstance(node, ast.ImportFrom):
113
+ from_root = node.module.split(".")[0] if node.module is not None else None
114
+ if from_root in _DYNAMIC_IMPORT_MODULES:
115
+ violations.append((node.lineno, from_root))
116
+ continue # one statement, one violation - aliases not also checked
117
+ for alias in node.names:
118
+ if alias.name in _DYNAMIC_IMPORT_NAMES:
119
+ violations.append((node.lineno, alias.name))
120
+ elif isinstance(node, ast.Name):
121
+ if node.id in _DYNAMIC_IMPORT_NAMES:
122
+ violations.append((node.lineno, node.id))
123
+ elif isinstance(node, ast.Attribute):
124
+ if node.attr in _DYNAMIC_IMPORT_ATTRS:
125
+ violations.append((node.lineno, node.attr))
126
+ return sorted(violations)
127
+
128
+
129
+ def _package_components(importing_file: Path, repo_root: Path) -> tuple[str, ...]:
130
+ """Dotted-path components of *importing_file*'s parent directory,
131
+ relative to *repo_root* (empty tuple when the file lives at the root)."""
132
+ rel_parent = importing_file.parent.relative_to(repo_root)
133
+ if rel_parent == Path("."):
134
+ return ()
135
+ return rel_parent.parts
136
+
137
+
138
+ def _names_from_import(node: ast.Import) -> list[str]:
139
+ return [alias.name for alias in node.names]
140
+
141
+
142
+ def _names_from_import_from(
143
+ node: ast.ImportFrom, importing_file: Path, repo_root: Path
144
+ ) -> list[str]:
145
+ if node.level == 0:
146
+ # level == 0 always carries a module per the grammar (`from X import Y`).
147
+ base = node.module
148
+ assert base is not None
149
+ return [base] + [f"{base}.{alias.name}" for alias in node.names]
150
+
151
+ components = _package_components(importing_file, repo_root)
152
+ walk_up = node.level - 1
153
+ if walk_up > len(components):
154
+ return [] # would escape repo_root - cannot name an in-repo file
155
+
156
+ prefix_parts = components[: len(components) - walk_up]
157
+ prefix = ".".join(prefix_parts)
158
+ base = ".".join(part for part in (prefix, node.module) if part)
159
+
160
+ # Real Python executes the package's __init__.py for `from . import name`
161
+ # regardless of whether `name` is a submodule file or just an attribute
162
+ # defined inside __init__.py - the two are indistinguishable from the
163
+ # import statement's syntax alone, so the bare prefix/base is always a
164
+ # candidate (symmetric with the level == 0 branch above), not only when
165
+ # node.module is also present. Missing __init__.py here would be
166
+ # under-inclusion (a stale node in production); including it when it was
167
+ # already reachable is a no-op, and including it spuriously costs at
168
+ # most one revalidation - see the module docstring.
169
+ names = [base] if base else []
170
+ names.extend(f"{base}.{alias.name}" if base else alias.name for alias in node.names)
171
+ return names
172
+
173
+
174
+ def _collect_import_names(tree: ast.AST, importing_file: Path, repo_root: Path) -> list[str]:
175
+ names: list[str] = []
176
+ for node in ast.walk(tree):
177
+ if isinstance(node, ast.Import):
178
+ names.extend(_names_from_import(node))
179
+ elif isinstance(node, ast.ImportFrom):
180
+ names.extend(_names_from_import_from(node, importing_file, repo_root))
181
+ return names
182
+
183
+
184
+ def _resolve_name(
185
+ name: str, search_roots: tuple[Path, ...]
186
+ ) -> tuple[Path, Path, tuple[str, ...]] | None:
187
+ """Resolve a dotted module name to (root, resolved file, name parts).
188
+
189
+ Tries each root in *search_roots*, in order, and for each, the package
190
+ form `<root>/a/b/c/__init__.py` before the module-file form
191
+ `<root>/a/b/c.py` - matching Python's own package-before-module lookup
192
+ order (when both exist, `import a.b.c` binds the package and the module
193
+ file of the same name is never executed). Returns None when the name
194
+ resolves under no root (it's external).
195
+ """
196
+ parts = tuple(name.split("."))
197
+ for root in search_roots:
198
+ package_init = root.joinpath(*parts, "__init__.py")
199
+ if package_init.is_file():
200
+ return root, package_init.resolve(), parts
201
+ module_file = root.joinpath(*parts).with_suffix(".py")
202
+ if module_file.is_file():
203
+ return root, module_file.resolve(), parts
204
+ return None
205
+
206
+
207
+ def resolve_closure(script_path: Path, repo_root: Path) -> list[Path]:
208
+ """Return the transitive in-repo import closure of *script_path*.
209
+
210
+ Absolute, resolved paths, sorted, with *script_path* itself EXCLUDED (it is
211
+ `source_hash`). Files that do not resolve under *repo_root* — stdlib,
212
+ site-packages, anything installed — are not closure members: external deps
213
+ are the image's concern (`image_tag`), not the hash's.
214
+
215
+ Search roots are the fixed, ordered pair `(repo_root, script_dir)` -
216
+ `script_dir` is *script_path*'s own directory, computed once here and
217
+ reused for every resolution in the traversal, matching what
218
+ `harness.ensure_import_paths` puts on `sys.path` for the whole process.
219
+ When the script lives at *repo_root* itself the pair collapses to a
220
+ single root, so it is not probed twice.
221
+ """
222
+ repo_root = repo_root.resolve()
223
+ script_resolved = script_path.resolve()
224
+ script_dir = script_resolved.parent
225
+ search_roots = (repo_root,) if script_dir == repo_root else (repo_root, script_dir)
226
+
227
+ seen: set[Path] = {script_resolved}
228
+ queue: deque[Path] = deque([script_resolved])
229
+
230
+ while queue:
231
+ current = queue.popleft()
232
+ rel = current.relative_to(repo_root)
233
+
234
+ source = current.read_bytes()
235
+ try:
236
+ tree = ast.parse(source)
237
+ except SyntaxError as exc:
238
+ raise ContractError(f"{rel}: syntax error: {exc.msg}") from exc
239
+
240
+ violations = dynamic_import_violations(tree)
241
+ if violations:
242
+ lineno, construct = violations[0]
243
+ raise ContractError(
244
+ f"{rel}:{lineno}: dynamic import construct {construct!r} is not "
245
+ "allowed — the content hash cannot see it"
246
+ )
247
+
248
+ for name in _collect_import_names(tree, current, repo_root):
249
+ match = _resolve_name(name, search_roots)
250
+ if match is None:
251
+ continue
252
+ root, resolved_file, parts = match
253
+
254
+ candidates = [resolved_file]
255
+ for depth in range(1, len(parts)):
256
+ init_candidate = root.joinpath(*parts[:depth], "__init__.py")
257
+ if init_candidate.is_file():
258
+ candidates.append(init_candidate.resolve())
259
+
260
+ for candidate in candidates:
261
+ if not candidate.is_relative_to(repo_root):
262
+ continue # symlink escape
263
+ if candidate in seen:
264
+ continue
265
+ seen.add(candidate)
266
+ queue.append(candidate)
267
+
268
+ return sorted(seen - {script_resolved})