continuo-python-runtime 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/contract/loader.py +17 -1
- continuo_python_runtime/contract/merge.py +4 -0
- continuo_python_runtime/contract/model.py +1 -0
- continuo_python_runtime/harness.py +8 -0
- continuo_python_runtime/hashing.py +8 -3
- {continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/METADATA +21 -9
- {continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/RECORD +11 -11
- {continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/WHEEL +1 -1
- {continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/entry_points.txt +0 -0
- {continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/licenses/LICENSE +0 -0
- {continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -44,8 +44,13 @@ _ALLOWED_KEYS = {
|
|
|
44
44
|
"secret_ref",
|
|
45
45
|
"mode_type",
|
|
46
46
|
"primary_key",
|
|
47
|
+
"timeout_minutes",
|
|
47
48
|
}
|
|
48
49
|
|
|
50
|
+
# Bounds of a node's declared task timeout, in minutes (one minute to one week).
|
|
51
|
+
MIN_TIMEOUT_MINUTES = 1
|
|
52
|
+
MAX_TIMEOUT_MINUTES = 10_080
|
|
53
|
+
|
|
49
54
|
_REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule")
|
|
50
55
|
|
|
51
56
|
_ALLOWED_OUTPUT_COLUMN_KEYS = {"name", "type", "nullable"}
|
|
@@ -295,6 +300,16 @@ def parse_node(
|
|
|
295
300
|
raise ContractError(f"{label}: 'secret_ref' is not allowed for kind {kind}")
|
|
296
301
|
secret_ref = validate_secret_ref(raw["secret_ref"], label)
|
|
297
302
|
|
|
303
|
+
timeout_minutes = raw.get("timeout_minutes")
|
|
304
|
+
if "timeout_minutes" in raw and (
|
|
305
|
+
type(timeout_minutes) is not int
|
|
306
|
+
or not MIN_TIMEOUT_MINUTES <= timeout_minutes <= MAX_TIMEOUT_MINUTES
|
|
307
|
+
):
|
|
308
|
+
raise ContractError(
|
|
309
|
+
f"{label}: 'timeout_minutes' must be an integer between "
|
|
310
|
+
f"{MIN_TIMEOUT_MINUTES} and {MAX_TIMEOUT_MINUTES}, got {timeout_minutes!r}"
|
|
311
|
+
)
|
|
312
|
+
|
|
298
313
|
mode_type = raw.get("mode_type", DEFAULT_MODE_TYPE)
|
|
299
314
|
if not isinstance(mode_type, str) or mode_type not in MODE_TYPES:
|
|
300
315
|
raise ContractError(
|
|
@@ -302,7 +317,7 @@ def parse_node(
|
|
|
302
317
|
)
|
|
303
318
|
if mode_type not in rules.mode_types:
|
|
304
319
|
raise ContractError(
|
|
305
|
-
f"{label}: 'mode_type' {mode_type} is not allowed for kind {kind} "
|
|
320
|
+
f"{label}: 'mode_type' {mode_type!r} is not allowed for kind {kind!r} "
|
|
306
321
|
f"(allowed: {sorted(rules.mode_types)})"
|
|
307
322
|
)
|
|
308
323
|
|
|
@@ -375,6 +390,7 @@ def parse_node(
|
|
|
375
390
|
secret_ref=secret_ref,
|
|
376
391
|
mode_type=mode_type,
|
|
377
392
|
primary_key=primary_key,
|
|
393
|
+
timeout_minutes=timeout_minutes,
|
|
378
394
|
)
|
|
379
395
|
|
|
380
396
|
|
|
@@ -170,6 +170,10 @@ def build_wire_contract(
|
|
|
170
170
|
|
|
171
171
|
primary_bytes, member_bytes = _HASH_INPUTS[node.kind](node, repo_root)
|
|
172
172
|
entry.update(hash_parts(entry, primary_bytes, member_bytes))
|
|
173
|
+
# The one wire key outside config_hash: it is written after the hashes
|
|
174
|
+
# are computed, so changing it alone never changes content_hash.
|
|
175
|
+
if node.timeout_minutes is not None:
|
|
176
|
+
entry["timeout_minutes"] = node.timeout_minutes
|
|
173
177
|
|
|
174
178
|
wire_nodes.append(entry)
|
|
175
179
|
|
|
@@ -392,6 +392,14 @@ def run_node(
|
|
|
392
392
|
print(result_block("success", message=f"rows={conformed.num_rows}", unique_id=node_id))
|
|
393
393
|
return 0
|
|
394
394
|
except HarnessError as err:
|
|
395
|
+
# Logged as well as printed: the executor cuts the result block out of
|
|
396
|
+
# the pod log before storing it, so the log holds only what is logged.
|
|
397
|
+
# The traceback is kept when the error wraps another exception (a
|
|
398
|
+
# driver error, the script's own exception), whose frames locate it.
|
|
399
|
+
logger.error(
|
|
400
|
+
"node %s failed: %s", node_id, err.sentinel_message(),
|
|
401
|
+
exc_info=err.__cause__ is not None,
|
|
402
|
+
)
|
|
395
403
|
print(
|
|
396
404
|
result_block(
|
|
397
405
|
"error", message=err.sentinel_message(), failures=1, unique_id=node_id
|
|
@@ -21,6 +21,10 @@ from json import dumps
|
|
|
21
21
|
|
|
22
22
|
HASH_PART_FIELDS = ("source_hash", "shared_code_hash", "config_hash")
|
|
23
23
|
CONTENT_HASH_FIELD = "content_hash"
|
|
24
|
+
# A wire key that is not part of the config_hash basis: it tunes how the node
|
|
25
|
+
# runs (the task timeout) and says nothing about what the node computes, so
|
|
26
|
+
# changing it alone is not a code change.
|
|
27
|
+
NON_HASHED_FIELDS = ("timeout_minutes",)
|
|
24
28
|
|
|
25
29
|
|
|
26
30
|
def canonical_entry(entry: dict) -> dict:
|
|
@@ -28,13 +32,14 @@ def canonical_entry(entry: dict) -> dict:
|
|
|
28
32
|
|
|
29
33
|
Drops ``content_hash`` and all three of ``HASH_PART_FIELDS`` from the
|
|
30
34
|
basis, and whitespace-normalizes every ``str`` value in ``reads`` via
|
|
31
|
-
``" ".join(sql.split())``.
|
|
32
|
-
|
|
35
|
+
``" ".join(sql.split())``. ``timeout_minutes`` is the one other wire key
|
|
36
|
+
left out of the basis (``NON_HASHED_FIELDS``). Everything else is
|
|
37
|
+
preserved verbatim, including ``config``.
|
|
33
38
|
"""
|
|
34
39
|
canonical = deepcopy(entry)
|
|
35
40
|
|
|
36
41
|
canonical.pop(CONTENT_HASH_FIELD, None)
|
|
37
|
-
for field in HASH_PART_FIELDS:
|
|
42
|
+
for field in (*HASH_PART_FIELDS, *NON_HASHED_FIELDS):
|
|
38
43
|
canonical.pop(field, None)
|
|
39
44
|
|
|
40
45
|
reads = canonical.get("reads")
|
{continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: continuo-python-runtime
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
Maintainer: Simone Carolini
|
|
@@ -11,11 +11,11 @@ Classifier: Development Status :: 4 - Beta
|
|
|
11
11
|
Classifier: Intended Audience :: Developers
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.14
|
|
13
13
|
Requires-Python: >=3.14
|
|
14
|
-
Requires-Dist: boto3==1.43.
|
|
15
|
-
Requires-Dist: continuo-engine-contract==0.
|
|
16
|
-
Requires-Dist: pyarrow==
|
|
14
|
+
Requires-Dist: boto3==1.43.114
|
|
15
|
+
Requires-Dist: continuo-engine-contract==0.9.0
|
|
16
|
+
Requires-Dist: pyarrow==26.0.0
|
|
17
17
|
Requires-Dist: pyyaml==6.0.3
|
|
18
|
-
Requires-Dist: sqlglot==30.
|
|
18
|
+
Requires-Dist: sqlglot==30.23.0
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
|
|
21
21
|
# Continuo Python Runtime
|
|
@@ -196,6 +196,15 @@ produced, never how it is checked or written.
|
|
|
196
196
|
script reads with `os.environ[...]`. The contract never holds a secret value
|
|
197
197
|
or env var name; the Secret itself is created in the cluster by the operator.
|
|
198
198
|
|
|
199
|
+
### Task timeout
|
|
200
|
+
|
|
201
|
+
Any node may set `timeout_minutes:` — an integer from 1 to 10,080 (one week) —
|
|
202
|
+
to declare how long its task may run before continuo gives up on it. A value
|
|
203
|
+
outside that range, or one that is not an integer, fails
|
|
204
|
+
`continuo-runtime validate`/`merge`/`lint`. It is the one key that is not part
|
|
205
|
+
of `config_hash`, so changing only the timeout never changes the node's
|
|
206
|
+
`content_hash`.
|
|
207
|
+
|
|
199
208
|
## Write modes
|
|
200
209
|
|
|
201
210
|
`mode_type:` chooses how the harness writes a node's table on each run:
|
|
@@ -331,7 +340,10 @@ declared `output_columns` on the table `run()` returned, in this order:
|
|
|
331
340
|
Every runtime failure is one of five `HarnessError` subclasses
|
|
332
341
|
(`continuo_python_runtime/errors.py`). The sentinel result block's message
|
|
333
342
|
is prefixed `<ErrorClass>: ` so failures can be triaged without parsing free
|
|
334
|
-
text.
|
|
343
|
+
text. The harness also logs the same message at ERROR on stderr, with the
|
|
344
|
+
traceback of the exception it wraps (a driver error, the script's own
|
|
345
|
+
exception), so the stored log of a failed run names its failure: Continuo cuts
|
|
346
|
+
the result block out of the log before storing it.
|
|
335
347
|
|
|
336
348
|
| Class | Meaning | Typical fix target |
|
|
337
349
|
| --- | --- | --- |
|
|
@@ -347,11 +359,11 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
347
359
|
`FROM`:
|
|
348
360
|
|
|
349
361
|
```dockerfile
|
|
350
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.
|
|
362
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.11.0
|
|
351
363
|
# or
|
|
352
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.
|
|
364
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.11.0
|
|
353
365
|
# or
|
|
354
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-duckdb:v0.
|
|
366
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-duckdb:v0.11.0
|
|
355
367
|
```
|
|
356
368
|
|
|
357
369
|
The engine is part of the image **name**; the tag is the bare version, so
|
|
@@ -6,16 +6,16 @@ continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7D
|
|
|
6
6
|
continuo_python_runtime/csv_loader.py,sha256=a5Hwu-YIzIiko8X1_vqShsnzjs2-OsKEUSCHLXg_dio,3346
|
|
7
7
|
continuo_python_runtime/csv_source.py,sha256=7g8Tu6ieiJLvWH4Ep4nwYlOdGkjeePr12K5UHuO8TNc,4136
|
|
8
8
|
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
9
|
-
continuo_python_runtime/harness.py,sha256=
|
|
10
|
-
continuo_python_runtime/hashing.py,sha256=
|
|
9
|
+
continuo_python_runtime/harness.py,sha256=iRGfVXD4h57q2IBAcq3xyhcbmOrBbjDYchTeWL1y-eY,17254
|
|
10
|
+
continuo_python_runtime/hashing.py,sha256=7lIkWOM50xt9syFNPNSTLG3T5qxqGnA6doHXPAub0Bg,3711
|
|
11
11
|
continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
|
|
12
12
|
continuo_python_runtime/shape.py,sha256=C-5ClSxPYgoO9pUiGqw7pCnmlBbsEl-sIm7Twbf-H4o,3693
|
|
13
13
|
continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
|
|
14
14
|
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
15
15
|
continuo_python_runtime/contract/kinds.py,sha256=nirtluv9GU0sZzGF8_-qRIdbeCRtBul_ppNnr6CIEC0,5320
|
|
16
|
-
continuo_python_runtime/contract/loader.py,sha256=
|
|
17
|
-
continuo_python_runtime/contract/merge.py,sha256=
|
|
18
|
-
continuo_python_runtime/contract/model.py,sha256=
|
|
16
|
+
continuo_python_runtime/contract/loader.py,sha256=UqmN1RdtlebV3F-kwFcH5DDVbdxcKgnhGR7Sm6HCcMY,18999
|
|
17
|
+
continuo_python_runtime/contract/merge.py,sha256=DCyX9j2TIVgqPmKyFzU-dvy-VX526FZdfcS7-XWxO9c,8381
|
|
18
|
+
continuo_python_runtime/contract/model.py,sha256=uQs0hKkdu55XP6dcbxRFGrMsUe7lSV3QJk1UvOd_3Zo,1474
|
|
19
19
|
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
20
20
|
continuo_python_runtime/contract/secret_ref.py,sha256=96jh2G81clpabiEWGIsxbd9r5GKEGLb4Sa1-WJ9G59Q,1131
|
|
21
21
|
continuo_python_runtime/csv_readers/__init__.py,sha256=9TKvjC37C77uRaJQZuO0--zVj2OvuNwpJE6nWx7j2S8,806
|
|
@@ -24,9 +24,9 @@ continuo_python_runtime/csv_readers/s3.py,sha256=zmvHs3TxlN2alPKrdaHzPyREmkve2nB
|
|
|
24
24
|
continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
|
|
25
25
|
continuo_python_runtime/validation/runner.py,sha256=7pbksf7MXfTNcdklCOkOXg9qfRiJtyI_dbgbL3a9LWk,14072
|
|
26
26
|
continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
|
|
27
|
-
continuo_python_runtime-0.
|
|
28
|
-
continuo_python_runtime-0.
|
|
29
|
-
continuo_python_runtime-0.
|
|
30
|
-
continuo_python_runtime-0.
|
|
31
|
-
continuo_python_runtime-0.
|
|
32
|
-
continuo_python_runtime-0.
|
|
27
|
+
continuo_python_runtime-0.11.0.dist-info/METADATA,sha256=JAHvdx1U2JdSWc_yx5ygX3uGVSDfSpB7vBfWq0ndbmU,24065
|
|
28
|
+
continuo_python_runtime-0.11.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
29
|
+
continuo_python_runtime-0.11.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
30
|
+
continuo_python_runtime-0.11.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
|
|
31
|
+
continuo_python_runtime-0.11.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
|
|
32
|
+
continuo_python_runtime-0.11.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
{continuo_python_runtime-0.9.0.dist-info → continuo_python_runtime-0.11.0.dist-info}/licenses/NOTICE
RENAMED
|
File without changes
|