continuo-python-runtime 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/cli.py +38 -9
- continuo_python_runtime/closure.py +268 -0
- continuo_python_runtime/context.py +2 -2
- continuo_python_runtime/contract/loader.py +155 -129
- continuo_python_runtime/contract/merge.py +86 -9
- continuo_python_runtime/contract/model.py +3 -1
- continuo_python_runtime/harness.py +97 -10
- continuo_python_runtime/hashing.py +78 -23
- continuo_python_runtime/lint.py +40 -26
- continuo_python_runtime/types.py +32 -96
- continuo_python_runtime/validation/__init__.py +1 -0
- continuo_python_runtime/validation/runner.py +245 -0
- continuo_python_runtime/validation/s3.py +45 -0
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.3.0.dist-info}/METADATA +108 -43
- continuo_python_runtime-0.3.0.dist-info/RECORD +22 -0
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.3.0.dist-info}/WHEEL +1 -1
- continuo_python_runtime-0.1.0.dist-info/RECORD +0 -18
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.3.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Build a single node as an empty table in the candidate schema (blue/green validation).
|
|
2
|
+
|
|
3
|
+
Dispatches on ``VALIDATION_OP`` env var (default ``build_from_sql``):
|
|
4
|
+
- ``build_from_sql``: fetch the node's compiled SQL from S3 (``CANDIDATE_SQL_URI``) and
|
|
5
|
+
materialize it empty (models/snapshots).
|
|
6
|
+
- ``clone_from_prod``: clone an existing prod table's shape empty from ``PROD_SCHEMA``
|
|
7
|
+
(unchanged upstreams, including seeds).
|
|
8
|
+
- ``build_from_columns``: for python nodes, which have no SELECT to shape their output
|
|
9
|
+
from. Fetch the node's validation spec JSON from S3 (``CANDIDATE_SPEC_URI`` —
|
|
10
|
+
``{"reads": [sql, ...], "output_columns": [{"name","type","nullable"}, ...],
|
|
11
|
+
"config": {...}}``; ``config`` is optional and defaults to ``{}``), bind-check every
|
|
12
|
+
declared read against the candidate schema so an upstream that
|
|
13
|
+
dropped a column the script reads fails the release gate, then materialize the
|
|
14
|
+
output table empty from the declared typed columns and the declared physical layout.
|
|
15
|
+
|
|
16
|
+
The engine adapter is discovered from the single installed
|
|
17
|
+
``continuo_engine.adapters`` entry point — each runner image installs exactly one.
|
|
18
|
+
stdout is reserved exclusively for the runner's one structured ``result_block``,
|
|
19
|
+
printed as its last line; all diagnostics go to stderr via the ``logging`` module.
|
|
20
|
+
A non-zero exit marks the node failed.
|
|
21
|
+
"""
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
|
|
27
|
+
from continuo_engine_contract import result # type: ignore[import-untyped]
|
|
28
|
+
from continuo_engine_contract.port import ( # type: ignore[import-untyped]
|
|
29
|
+
AdapterDiscoveryError,
|
|
30
|
+
discover_adapter,
|
|
31
|
+
)
|
|
32
|
+
from continuo_python_runtime.validation import s3
|
|
33
|
+
|
|
34
|
+
logger = logging.getLogger("validation_runner")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _node_id() -> str:
|
|
38
|
+
"""Best-known node identity for a result block: ``NODE_ID`` env, else ``""``.
|
|
39
|
+
|
|
40
|
+
The single source of NODE_ID resolution shared by every block-emitting path.
|
|
41
|
+
``main`` layers a ``model.{table}`` fallback on top once TABLE_NAME is known;
|
|
42
|
+
the early ``_require`` failures (which may be the missing TABLE_NAME itself)
|
|
43
|
+
cannot, so they fall back to this bare value.
|
|
44
|
+
"""
|
|
45
|
+
return os.environ.get("NODE_ID", "")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _require(name: str) -> str:
|
|
49
|
+
value = os.environ.get(name)
|
|
50
|
+
if not value:
|
|
51
|
+
logger.error("missing required env var %s", name)
|
|
52
|
+
print(
|
|
53
|
+
result.result_block(
|
|
54
|
+
"error", f"missing required env var {name}",
|
|
55
|
+
unique_id=_node_id(),
|
|
56
|
+
),
|
|
57
|
+
flush=True,
|
|
58
|
+
)
|
|
59
|
+
sys.exit(2)
|
|
60
|
+
return value
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def load_candidate_sql() -> str:
|
|
64
|
+
"""Fetch this node's candidate SQL from S3 at ``CANDIDATE_SQL_URI``.
|
|
65
|
+
|
|
66
|
+
Returns the raw UTF-8 body (no stripping; the caller normalizes). Returns ``""``
|
|
67
|
+
when ``CANDIDATE_SQL_URI`` is unset/empty (nothing to validate). Raises on
|
|
68
|
+
invalid-URI or S3-download errors so ``main`` maps them to a structured block.
|
|
69
|
+
"""
|
|
70
|
+
uri = os.environ.get("CANDIDATE_SQL_URI", "")
|
|
71
|
+
if not uri:
|
|
72
|
+
return ""
|
|
73
|
+
bucket, key = s3.parse_s3_uri(uri)
|
|
74
|
+
client = s3.make_s3_client()
|
|
75
|
+
body = client.get_object(Bucket=bucket, Key=key)["Body"].read()
|
|
76
|
+
decoded: str = body.decode("utf-8")
|
|
77
|
+
return decoded
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def load_candidate_spec() -> dict:
|
|
81
|
+
"""Fetch this node's validation spec JSON from S3 at ``CANDIDATE_SPEC_URI``.
|
|
82
|
+
|
|
83
|
+
Returns the parsed dict. Raises on unset/empty URI, invalid URI, S3
|
|
84
|
+
download errors, invalid JSON, or JSON that parses to something other than
|
|
85
|
+
an object (e.g. a list, ``null``, a number, or a string) — ``main`` maps
|
|
86
|
+
every failure to a structured block.
|
|
87
|
+
"""
|
|
88
|
+
uri = os.environ.get("CANDIDATE_SPEC_URI", "")
|
|
89
|
+
if not uri:
|
|
90
|
+
raise ValueError("CANDIDATE_SPEC_URI is unset or empty")
|
|
91
|
+
bucket, key = s3.parse_s3_uri(uri)
|
|
92
|
+
client = s3.make_s3_client()
|
|
93
|
+
body = client.get_object(Bucket=bucket, Key=key)["Body"].read()
|
|
94
|
+
spec = json.loads(body.decode("utf-8"))
|
|
95
|
+
if not isinstance(spec, dict):
|
|
96
|
+
raise ValueError(
|
|
97
|
+
f"candidate spec must be a JSON object, got {type(spec).__name__}"
|
|
98
|
+
)
|
|
99
|
+
return spec
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
_NODE_OPS = ("build_from_sql", "clone_from_prod", "build_from_columns")
|
|
103
|
+
_SCHEMA_OPS = ("ensure_schema", "drop_schema")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def main() -> None:
|
|
107
|
+
"""Run one validation op end to end; exits non-zero on failure.
|
|
108
|
+
|
|
109
|
+
Node ops (``build_from_sql``/``clone_from_prod``/``build_from_columns``)
|
|
110
|
+
materialize one empty node table and require ``TABLE_NAME``. Schema ops
|
|
111
|
+
(``ensure_schema``/``drop_schema``) act on the whole candidate schema —
|
|
112
|
+
the executor schedules them as one-shot engine-image Jobs to own the
|
|
113
|
+
candidate-schema lifecycle without connecting to the warehouse itself —
|
|
114
|
+
and take no table.
|
|
115
|
+
"""
|
|
116
|
+
logging.basicConfig(
|
|
117
|
+
level=logging.INFO, stream=sys.stderr, format="%(levelname)s %(name)s: %(message)s"
|
|
118
|
+
)
|
|
119
|
+
schema = _require("DBT_TARGET_SCHEMA")
|
|
120
|
+
op = os.environ.get("VALIDATION_OP", "build_from_sql")
|
|
121
|
+
|
|
122
|
+
# Gather op-specific inputs and identity BEFORE touching the adapter, surfacing
|
|
123
|
+
# input errors as a structured block (preserves the prior contract + exit codes).
|
|
124
|
+
table = None
|
|
125
|
+
candidate_sql = None
|
|
126
|
+
prod_schema = None
|
|
127
|
+
spec: dict | None = None
|
|
128
|
+
config: dict = {}
|
|
129
|
+
if op in _NODE_OPS:
|
|
130
|
+
table = _require("TABLE_NAME")
|
|
131
|
+
unique_id = _node_id() or f"model.{table}"
|
|
132
|
+
if op == "build_from_sql":
|
|
133
|
+
try:
|
|
134
|
+
raw_sql = load_candidate_sql()
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
uri = os.environ.get("CANDIDATE_SQL_URI", "")
|
|
137
|
+
logger.error("ERROR fetching candidate SQL from %r: %s", uri, exc)
|
|
138
|
+
print(result.result_block("error", str(exc), unique_id=unique_id), flush=True)
|
|
139
|
+
sys.exit(1)
|
|
140
|
+
if not raw_sql:
|
|
141
|
+
logger.error(
|
|
142
|
+
"CANDIDATE_SQL_URI is unset or the object is empty for a "
|
|
143
|
+
"build_from_sql node; cannot validate"
|
|
144
|
+
)
|
|
145
|
+
print(result.result_block("error", "CANDIDATE_SQL_URI is unset or empty",
|
|
146
|
+
unique_id=unique_id), flush=True)
|
|
147
|
+
sys.exit(2)
|
|
148
|
+
candidate_sql = raw_sql
|
|
149
|
+
elif op == "build_from_columns":
|
|
150
|
+
try:
|
|
151
|
+
spec = load_candidate_spec()
|
|
152
|
+
except (ValueError, json.JSONDecodeError) as exc:
|
|
153
|
+
logger.error("invalid candidate spec: %s", exc)
|
|
154
|
+
print(result.result_block("error", str(exc), unique_id=unique_id), flush=True)
|
|
155
|
+
sys.exit(2)
|
|
156
|
+
except Exception as exc:
|
|
157
|
+
uri = os.environ.get("CANDIDATE_SPEC_URI", "")
|
|
158
|
+
logger.error("ERROR fetching candidate spec from %r: %s", uri, exc)
|
|
159
|
+
print(result.result_block("error", str(exc), unique_id=unique_id), flush=True)
|
|
160
|
+
sys.exit(1)
|
|
161
|
+
if not spec.get("output_columns"):
|
|
162
|
+
logger.error("candidate spec has no output_columns; cannot validate")
|
|
163
|
+
print(result.result_block("error", "candidate spec has no output_columns",
|
|
164
|
+
unique_id=unique_id), flush=True)
|
|
165
|
+
sys.exit(2)
|
|
166
|
+
raw_config = spec.get("config")
|
|
167
|
+
if raw_config is not None and not isinstance(raw_config, dict):
|
|
168
|
+
# Engine-blind: the runner only decides that the block is a JSON
|
|
169
|
+
# object. Which keys are legal is the installed adapter's call.
|
|
170
|
+
msg = (
|
|
171
|
+
f"candidate spec 'config' must be a JSON object, "
|
|
172
|
+
f"got {type(raw_config).__name__}"
|
|
173
|
+
)
|
|
174
|
+
logger.error("%s", msg)
|
|
175
|
+
print(result.result_block("error", msg, unique_id=unique_id), flush=True)
|
|
176
|
+
sys.exit(2)
|
|
177
|
+
config = raw_config or {}
|
|
178
|
+
else:
|
|
179
|
+
prod_schema = _require("PROD_SCHEMA")
|
|
180
|
+
elif op in _SCHEMA_OPS:
|
|
181
|
+
unique_id = _node_id() or f"schema.{schema}"
|
|
182
|
+
else:
|
|
183
|
+
unique_id = _node_id() or f"schema.{schema}"
|
|
184
|
+
logger.error("unknown VALIDATION_OP %r", op)
|
|
185
|
+
print(result.result_block("error", f"unknown VALIDATION_OP {op!r}",
|
|
186
|
+
unique_id=unique_id), flush=True)
|
|
187
|
+
sys.exit(2)
|
|
188
|
+
|
|
189
|
+
# Engine selection is discovery, not configuration: the image installs one adapter.
|
|
190
|
+
try:
|
|
191
|
+
engine, adapter_cls = discover_adapter()
|
|
192
|
+
except AdapterDiscoveryError as exc:
|
|
193
|
+
logger.error("%s", exc)
|
|
194
|
+
print(result.result_block("error", str(exc), unique_id=unique_id), flush=True)
|
|
195
|
+
sys.exit(2)
|
|
196
|
+
|
|
197
|
+
missing = [v for v in adapter_cls.required_env() if not os.environ.get(v)]
|
|
198
|
+
if missing:
|
|
199
|
+
msg = f"missing required env for engine {engine!r}: {', '.join(missing)}"
|
|
200
|
+
logger.error("%s", msg)
|
|
201
|
+
print(result.result_block("error", msg, unique_id=unique_id), flush=True)
|
|
202
|
+
sys.exit(2)
|
|
203
|
+
|
|
204
|
+
# close() runs exactly once, in the finally, on every path: the success case,
|
|
205
|
+
# the op error (sys.exit raises SystemExit, which still unwinds finally), and
|
|
206
|
+
# a from_env() failure (adapter stays None). A close() failure only logs — the
|
|
207
|
+
# primary error, if any, is already the SystemExit propagating through.
|
|
208
|
+
adapter = None
|
|
209
|
+
try:
|
|
210
|
+
adapter = adapter_cls.from_env()
|
|
211
|
+
if op == "ensure_schema":
|
|
212
|
+
adapter.ensure_schema(schema)
|
|
213
|
+
elif op == "drop_schema":
|
|
214
|
+
adapter.drop_schema(schema)
|
|
215
|
+
else:
|
|
216
|
+
adapter.ensure_schema(schema)
|
|
217
|
+
assert table is not None, "table must be set for a node op"
|
|
218
|
+
if op == "build_from_sql":
|
|
219
|
+
assert candidate_sql is not None, "candidate_sql must be set for build_from_sql"
|
|
220
|
+
adapter.build_empty_from_sql(schema, table, candidate_sql)
|
|
221
|
+
elif op == "build_from_columns":
|
|
222
|
+
assert spec is not None, "spec must be set for build_from_columns"
|
|
223
|
+
for read_sql in spec.get("reads", []):
|
|
224
|
+
adapter.check_binds(read_sql)
|
|
225
|
+
adapter.build_empty_from_columns(schema, table, spec["output_columns"], config)
|
|
226
|
+
else:
|
|
227
|
+
assert prod_schema is not None, "prod_schema must be set for clone_from_prod"
|
|
228
|
+
adapter.clone_empty_from_prod(schema, prod_schema, table)
|
|
229
|
+
except Exception as exc:
|
|
230
|
+
logger.error("ERROR op=%s on schema %s: %s", op, schema, exc)
|
|
231
|
+
print(result.result_block("error", str(exc), unique_id=unique_id), flush=True)
|
|
232
|
+
sys.exit(1)
|
|
233
|
+
finally:
|
|
234
|
+
if adapter is not None:
|
|
235
|
+
try:
|
|
236
|
+
adapter.close()
|
|
237
|
+
except Exception as close_exc: # never mask the primary outcome
|
|
238
|
+
logger.error("adapter close failed: %s", close_exc)
|
|
239
|
+
|
|
240
|
+
logger.info("ran op=%s on schema=%s (engine=%s)", op, schema, engine)
|
|
241
|
+
print(result.result_block("success", unique_id=unique_id), flush=True)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
if __name__ == "__main__":
|
|
245
|
+
main()
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""S3 helpers for the validation runner.
|
|
2
|
+
|
|
3
|
+
``parse_s3_uri`` and ``make_s3_client`` back the runner's candidate-SQL fetch
|
|
4
|
+
from ``CANDIDATE_SQL_URI``.
|
|
5
|
+
"""
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import boto3 # type: ignore[import-untyped]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def parse_s3_uri(uri: str) -> tuple[str, str]:
|
|
14
|
+
"""Parse an ``s3://bucket/key`` URI and return ``(bucket, key)``.
|
|
15
|
+
|
|
16
|
+
Raises :class:`ValueError` for non-s3:// URIs, a missing bucket, or a
|
|
17
|
+
missing/empty key (e.g. ``s3://bucket-only`` or ``s3://bucket/``).
|
|
18
|
+
"""
|
|
19
|
+
if not uri.startswith("s3://"):
|
|
20
|
+
raise ValueError(f"invalid S3 URI (must start with s3://): {uri!r}")
|
|
21
|
+
bucket, _, key = uri[len("s3://"):].partition("/")
|
|
22
|
+
if not bucket or not key:
|
|
23
|
+
raise ValueError(f"invalid S3 URI (missing bucket or key): {uri!r}")
|
|
24
|
+
return bucket, key
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def make_s3_client() -> Any:
|
|
28
|
+
"""Construct a boto3 S3 client, letting boto3 resolve credentials itself.
|
|
29
|
+
|
|
30
|
+
Only ``S3_ENDPOINT_URL`` (when set and non-empty, e.g. for MinIO) is passed
|
|
31
|
+
explicitly. Credentials and region are left for boto3's own chain, which
|
|
32
|
+
natively reads ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``,
|
|
33
|
+
``AWS_SESSION_TOKEN``, and ``AWS_DEFAULT_REGION`` from the environment, and
|
|
34
|
+
falls through to IAM roles / web identity / instance profiles when those are
|
|
35
|
+
absent. Passing raw (possibly blank or missing) env values as explicit
|
|
36
|
+
keyword args would instead select empty/explicit credentials or drop
|
|
37
|
+
temporary-credential session tokens, breaking that chain.
|
|
38
|
+
|
|
39
|
+
Tests patch ``s3.boto3`` to intercept the underlying ``client`` call.
|
|
40
|
+
"""
|
|
41
|
+
kwargs: dict[str, Any] = {}
|
|
42
|
+
endpoint_url = os.environ.get("S3_ENDPOINT_URL")
|
|
43
|
+
if endpoint_url:
|
|
44
|
+
kwargs["endpoint_url"] = endpoint_url
|
|
45
|
+
return boto3.client("s3", **kwargs)
|
{continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.3.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: continuo-python-runtime
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
Maintainer: Simone Carolini
|
|
@@ -8,9 +8,11 @@ Classifier: Development Status :: 4 - Beta
|
|
|
8
8
|
Classifier: Intended Audience :: Developers
|
|
9
9
|
Classifier: Programming Language :: Python :: 3.14
|
|
10
10
|
Requires-Python: >=3.14
|
|
11
|
-
Requires-Dist:
|
|
11
|
+
Requires-Dist: boto3==1.43.59
|
|
12
|
+
Requires-Dist: continuo-engine-contract==0.7.0
|
|
12
13
|
Requires-Dist: pyarrow==25.0.0
|
|
13
14
|
Requires-Dist: pyyaml==6.0.3
|
|
15
|
+
Requires-Dist: sqlglot==30.15.0
|
|
14
16
|
Description-Content-Type: text/markdown
|
|
15
17
|
|
|
16
18
|
# Continuo Python Runtime
|
|
@@ -23,44 +25,69 @@ and register the release with Continuo.
|
|
|
23
25
|
|
|
24
26
|
## What this repo is
|
|
25
27
|
|
|
26
|
-
|
|
28
|
+
Four artifacts come out of this repository:
|
|
27
29
|
|
|
28
30
|
- **The `continuo-python-runtime` PyPI package** — the `continuo-runtime` CLI
|
|
29
|
-
(`validate` / `merge` / `hash` / `lint` / `run`) and the
|
|
30
|
-
(`conform()`, `RunContext`, the error taxonomy) that domain
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
31
|
+
(`validate` / `merge` / `hash` / `lint` / `run` / `validation-op`) and the
|
|
32
|
+
harness library (`conform()`, `RunContext`, the error taxonomy) that domain
|
|
33
|
+
repos install.
|
|
34
|
+
- **The `continuo-engine-contract` PyPI package** — the `WarehouseAdapter`
|
|
35
|
+
port, the contract schema, the shared SQL/type/config guards, and the
|
|
36
|
+
sentinel result-block format. Adapter authors outside this repo pin it.
|
|
37
|
+
- **Per-engine base images**, one per warehouse engine
|
|
38
|
+
(`continuo-python-runtime-postgres`, `continuo-python-runtime-trino`), that
|
|
39
|
+
domain repos build `FROM`. Each image bakes in the runtime and a single
|
|
40
|
+
`WarehouseAdapter` for that engine, and serves both roles that adapter has:
|
|
41
|
+
the node harness (`ENTRYPOINT ["continuo-runtime"]`, `CMD ["run"]`) and the
|
|
42
|
+
validation runner (`continuo-runtime validation-op`).
|
|
35
43
|
- **`template/`** — a copy-ready domain repo: `Dockerfile`, `contracts/`,
|
|
36
44
|
`scripts/`, and the `release.yml` CI/CD workflow.
|
|
37
45
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
46
|
+
One `vX.Y.Z` git tag releases all of it: `publish-pypi.yml` builds both
|
|
47
|
+
distributions into a single `dist/` and publishes them together, and
|
|
48
|
+
`images.yml` builds and pushes both engine images multi-arch under the same
|
|
49
|
+
tag.
|
|
41
50
|
|
|
42
|
-
###
|
|
51
|
+
### What this repo owns
|
|
43
52
|
|
|
44
|
-
|
|
45
|
-
the
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
53
|
+
This repository owns the entire python-node surface: the engine contract, both
|
|
54
|
+
engine adapters, the validation runner, and the node harness. The former
|
|
55
|
+
`continuo-validation` repository was merged in — there is no longer a separate
|
|
56
|
+
validation-side port, adapter class, entry-point group, or image. One
|
|
57
|
+
`WarehouseAdapter` per engine serves both the data plane (`fetch` /
|
|
58
|
+
`ensure_table` / `load`) and validation (`ensure_schema` / `drop_schema` /
|
|
59
|
+
`build_empty_from_sql` / `build_empty_from_columns` / `clone_empty_from_prod` /
|
|
60
|
+
`check_binds`), and one image per engine runs both roles.
|
|
50
61
|
|
|
51
62
|
| Package (distribution name) | Module | Lives in | Role |
|
|
52
63
|
| --- | --- | --- | --- |
|
|
53
|
-
| `continuo-python-runtime` | `continuo_python_runtime` | this repo (root) | Harness
|
|
54
|
-
| `continuo-
|
|
55
|
-
| `continuo-python-runtime-
|
|
56
|
-
| `continuo-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
+
| `continuo-python-runtime` | `continuo_python_runtime` | this repo (root) | Harness (CLI, `conform()`, `RunContext`, error taxonomy) **and** the validation runner (`continuo-runtime validation-op`). Published to PyPI. |
|
|
65
|
+
| `continuo-engine-contract` | `continuo_engine_contract` | this repo, `contract/` | The `WarehouseAdapter` port, contract schema, the SQL/type/config guards adapters must run, and the result-block format. Published to PyPI. |
|
|
66
|
+
| `continuo-python-runtime-postgres` | `continuo_python_runtime_postgres` | this repo, `adapters/postgres/` | `PostgresAdapter` — one class, both roles. **Not published to PyPI** — built from source into the image. |
|
|
67
|
+
| `continuo-python-runtime-trino` | `continuo_python_runtime_trino` | this repo, `adapters/trino/` | `TrinoAdapter` — one class, both roles, for Trino/Iceberg. **Not published to PyPI** — built from source into the image. |
|
|
68
|
+
|
|
69
|
+
All four are uv workspace members (`[tool.uv.workspace]` in the root
|
|
70
|
+
`pyproject.toml`), so `uv sync --all-packages --all-groups` at the repo root
|
|
71
|
+
installs everything for local development.
|
|
72
|
+
|
|
73
|
+
**Only `continuo-python-runtime` and `continuo-engine-contract` are published
|
|
74
|
+
to PyPI.** The two engine adapters are built **from source into the engine
|
|
75
|
+
images**: `Dockerfile.postgres` and `Dockerfile.trino` install them out of the
|
|
76
|
+
build context, so each image ships exactly one adapter and the runtime
|
|
77
|
+
discovers it through the `continuo_engine.adapters` entry-point group at run
|
|
78
|
+
time. Nothing installs them from an index — the harness package does not
|
|
79
|
+
depend on them, and domain repos get their adapter by building `FROM` a
|
|
80
|
+
published base image. They are still built, type-checked, and tested by CI on
|
|
81
|
+
every change.
|
|
82
|
+
|
|
83
|
+
### The result block is a frozen wire contract
|
|
84
|
+
|
|
85
|
+
`continuo_engine_contract.result` writes a sentinel-framed JSON block as the
|
|
86
|
+
last line of stdout, and Continuo's Go side parses it byte-for-byte
|
|
87
|
+
(`pkg/validationresult` in the `continuo` repository). That wire format — the
|
|
88
|
+
sentinel markers, the framing, and the field names inside the block — is
|
|
89
|
+
**frozen**. Reuse it; never change it from this side alone. A change here that
|
|
90
|
+
the Go parser has not been taught is a production outage, not a refactor.
|
|
64
91
|
|
|
65
92
|
## Quickstart for domain teams
|
|
66
93
|
|
|
@@ -85,6 +112,28 @@ development.
|
|
|
85
112
|
and merges the contracts, builds and pushes the image, uploads the merged
|
|
86
113
|
contract to S3, and POSTs the release.
|
|
87
114
|
|
|
115
|
+
### Upgrading an existing domain repo
|
|
116
|
+
|
|
117
|
+
`validate` / `merge` / `hash` now hand every declared read to a real SQL
|
|
118
|
+
parser (sqlglot, via `continuo_engine_contract.sql.ensure_single_read`)
|
|
119
|
+
instead of scanning it for a leading `SELECT`/`WITH`. Two things follow for a
|
|
120
|
+
repo written before this, on its next release: SQL a driver would accept but
|
|
121
|
+
a parser will not — most commonly a driver-specific bind placeholder like
|
|
122
|
+
psycopg2's `%(name)s`, which `ctx.read(name)` could never have used anyway —
|
|
123
|
+
now fails validation, and engine-specific syntax (postgres `~`, `@>`, …)
|
|
124
|
+
needs `--dialect <engine>`, which a repo should be passing regardless since
|
|
125
|
+
Continuo bind-checks every read in the install's own warehouse dialect. Run
|
|
126
|
+
the pre-flight check once before your next release; it reports every affected
|
|
127
|
+
read at once:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
continuo-runtime validate contracts/ --dialect postgres # or trino
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
The runtime image does not re-run this gate, so a read that passes here is
|
|
134
|
+
not re-judged under a different grammar in production. See
|
|
135
|
+
`docs/boundary-contract.md` §13.1.
|
|
136
|
+
|
|
88
137
|
## The script API
|
|
89
138
|
|
|
90
139
|
A node script is a Python file with exactly one required entry point:
|
|
@@ -106,9 +155,13 @@ def run(ctx):
|
|
|
106
155
|
example a polars DataFrame. Returning anything else raises `ScriptError`.
|
|
107
156
|
- The dataframe library is your choice. No dataframe library is baked into
|
|
108
157
|
the base image — the package's own runtime dependencies are `pyarrow`,
|
|
109
|
-
`PyYAML`, and `continuo-
|
|
158
|
+
`PyYAML`, and `continuo-engine-contract`. Add whatever you script
|
|
110
159
|
against (pandas, polars, …) as a `RUN pip install` line in your own
|
|
111
|
-
`Dockerfile`, on top of the base image.
|
|
160
|
+
`Dockerfile`, on top of the base image. The base image runs as non-root
|
|
161
|
+
(uid 65532), which pip cannot install under, so switch to root for the
|
|
162
|
+
install and back afterward — `USER root`, then the `RUN pip install`
|
|
163
|
+
line, then `USER 65532:65532` — since the executor's pod spec expects
|
|
164
|
+
the image to end at uid 65532. See `template/Dockerfile` for the pattern.
|
|
112
165
|
- Scripts do not import warehouse drivers, write raw SQL literals, or call
|
|
113
166
|
data-access methods directly — `continuo-runtime lint` rejects those:
|
|
114
167
|
- forbidden driver imports (`psycopg2`/`sqlalchemy`/`trino`/etc.),
|
|
@@ -125,6 +178,17 @@ def run(ctx):
|
|
|
125
178
|
are the driver-import and data-access-call rules, together with the fact
|
|
126
179
|
that `RunContext` only exposes `ctx.read()` — all warehouse access goes
|
|
127
180
|
through it.
|
|
181
|
+
- Scripts may import shared in-repo helpers. Before executing a script the
|
|
182
|
+
harness puts the repo root (`APP_ROOT`) and the script's own directory on
|
|
183
|
+
`sys.path`, so both `import helpers` (a sibling of the script) and
|
|
184
|
+
`from lib.shared import ...` (anywhere under the repo root) work, including
|
|
185
|
+
from inside `run()`. Every helper a script reaches transitively is folded
|
|
186
|
+
into `shared_code_hash`, so editing one re-fingerprints the node — but the
|
|
187
|
+
hash does not put the file in the image: **`COPY` every directory your
|
|
188
|
+
scripts import from in your `Dockerfile`**, or the release is valid and the
|
|
189
|
+
node dies with `ModuleNotFoundError` on its first run. Because the repo root
|
|
190
|
+
precedes the standard library on `sys.path`, avoid naming a top-level module
|
|
191
|
+
after a stdlib one (`types.py`, `json.py`, `logging.py`, …).
|
|
128
192
|
- The harness — not the script — performs the write. It calls `conform()`
|
|
129
193
|
on whatever `run()` returned and issues the only INSERT; the script never
|
|
130
194
|
writes directly.
|
|
@@ -165,19 +229,20 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
165
229
|
`FROM`:
|
|
166
230
|
|
|
167
231
|
```dockerfile
|
|
168
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime:v0.
|
|
232
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.3.0
|
|
169
233
|
# or
|
|
170
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime:v0.
|
|
234
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.3.0
|
|
171
235
|
```
|
|
172
236
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
`
|
|
177
|
-
`
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
237
|
+
The engine is part of the image **name**; the tag is the bare version, so
|
|
238
|
+
Continuo's Helm chart can pin an image as `<name>:vX.Y.Z@sha256:<digest>`.
|
|
239
|
+
|
|
240
|
+
Each image bakes in exactly one `WarehouseAdapter` for that engine — installed
|
|
241
|
+
from this repo's `adapters/postgres/` or `adapters/trino/` package (see the
|
|
242
|
+
table above) — registered under the `continuo_engine.adapters` entry-point
|
|
243
|
+
group (entry names `postgres` / `trino`). The runtime discovers it via
|
|
244
|
+
`discover_adapter()` at run time, so a single image serves every node in the
|
|
245
|
+
service and the release-time validation Job for it. The executor injects the
|
|
181
246
|
warehouse connection as environment variables (engine-native, e.g.
|
|
182
247
|
`POSTGRES_HOST`/`POSTGRES_DB`/`POSTGRES_USER`) plus the node-selection
|
|
183
248
|
environment (`NODE_ID`, `TABLE_NAME`, `TARGET_SCHEMA`, and optionally
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
|
|
2
|
+
continuo_python_runtime/cli.py,sha256=X4JBC4FDpgFOeDdtBbeX9GTq1XTuJCghDpemlR_CPvE,5925
|
|
3
|
+
continuo_python_runtime/closure.py,sha256=6e-IAQp8Lieb0bGKP1F6JE4TgVBJ_COTGXRUult2xeI,12991
|
|
4
|
+
continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
|
|
5
|
+
continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7DLZfVg,1856
|
|
6
|
+
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
7
|
+
continuo_python_runtime/harness.py,sha256=1b_ze2_AJ9vncY-yj88TgJylVzKwGw1-idxHoaYAymE,12101
|
|
8
|
+
continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
|
|
9
|
+
continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
|
|
10
|
+
continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
|
|
11
|
+
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
12
|
+
continuo_python_runtime/contract/loader.py,sha256=WeKngAe6VH-WxgRTVdCERPMdsGBszJM9RkanUTTs9-w,15611
|
|
13
|
+
continuo_python_runtime/contract/merge.py,sha256=XCCeE2vJUm0KA60d2XsHpjRh5fIdVfFu_HsBzEyUWow,6505
|
|
14
|
+
continuo_python_runtime/contract/model.py,sha256=2hKI04M0WwvTkOy9eDGl2dRZgY3bAuJYktAG-ICI-AU,936
|
|
15
|
+
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
16
|
+
continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
|
|
17
|
+
continuo_python_runtime/validation/runner.py,sha256=zHs5CR3Wr9512sh6eY9hpN8NG3weJPYUFRDPJf2BNnc,10875
|
|
18
|
+
continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
|
|
19
|
+
continuo_python_runtime-0.3.0.dist-info/METADATA,sha256=SjKFljTkhZUCsswm3WWkstA2eOtrfdRonXKEJA4Sf-Q,14898
|
|
20
|
+
continuo_python_runtime-0.3.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
21
|
+
continuo_python_runtime-0.3.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
22
|
+
continuo_python_runtime-0.3.0.dist-info/RECORD,,
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
|
|
2
|
-
continuo_python_runtime/cli.py,sha256=Qki4Mfoab74D0DvguZd5yzSfI-FRQDmtvwC0OacxrAc,4860
|
|
3
|
-
continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
|
|
4
|
-
continuo_python_runtime/context.py,sha256=ZyeN_dA2DihQbegBiiqHj1_EKk79GsxS-tATPVymUPg,1856
|
|
5
|
-
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
6
|
-
continuo_python_runtime/harness.py,sha256=YcWy4ummKY3HQYQ4JQ3-UI5jxDgYwJDQlWT88_GRPH4,7504
|
|
7
|
-
continuo_python_runtime/hashing.py,sha256=6D4MntcjaHTEC55IQBBX2Pq9K59D0TWlxTSXGJ5Jsro,1159
|
|
8
|
-
continuo_python_runtime/lint.py,sha256=HgB0BAuiQuS3DgT6BsOoB2M5A-_OKcFsGbaTWhuM7XU,11897
|
|
9
|
-
continuo_python_runtime/types.py,sha256=z1qnBbbaP6nFN2mmRySXM4ZAesv5N6_4E9vZ4kv74HU,7138
|
|
10
|
-
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
11
|
-
continuo_python_runtime/contract/loader.py,sha256=wlxI6XZIoUZ4z9rDe1G8s26jbL0tGXnAHPGajTAM0ag,12828
|
|
12
|
-
continuo_python_runtime/contract/merge.py,sha256=WFde9lzrojlMNjuEU7YW8FWOs6VPSrWPY2_Sri0ljZE,2567
|
|
13
|
-
continuo_python_runtime/contract/model.py,sha256=6luq3SJcgshKoG0QsQhJPZ5fzLa8AcGAev9D4jiGgxA,849
|
|
14
|
-
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
15
|
-
continuo_python_runtime-0.1.0.dist-info/METADATA,sha256=YQWMZaYiS3gpRnIutzgKhT3KeB5BtUpeMrwi0qKLRUU,10953
|
|
16
|
-
continuo_python_runtime-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
17
|
-
continuo_python_runtime-0.1.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
18
|
-
continuo_python_runtime-0.1.0.dist-info/RECORD,,
|
{continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.3.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|