continuo-python-runtime 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/cli.py +27 -9
- continuo_python_runtime/closure.py +268 -0
- continuo_python_runtime/contract/loader.py +155 -129
- continuo_python_runtime/contract/merge.py +86 -9
- continuo_python_runtime/contract/model.py +3 -1
- continuo_python_runtime/harness.py +93 -5
- continuo_python_runtime/hashing.py +78 -23
- continuo_python_runtime/lint.py +40 -26
- continuo_python_runtime/types.py +32 -96
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/METADATA +50 -7
- continuo_python_runtime-0.2.0.dist-info/RECORD +19 -0
- continuo_python_runtime-0.1.0.dist-info/RECORD +0 -18
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/WHEEL +0 -0
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,34 +1,89 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
"""Content hashing: three independent parts folded into one content_hash.
|
|
2
|
+
|
|
3
|
+
The three parts (``source_hash``, ``shared_code_hash``, ``config_hash``) are
|
|
4
|
+
computed independently from already-gathered bytes/dicts and folded together
|
|
5
|
+
with ``content_hash_fold``. This module is pure computation — no filesystem
|
|
6
|
+
access, no knowledge of ``closure.py`` — so the known-vector tests can call
|
|
7
|
+
``content_hash_fold`` on bare strings with no entry or file involved.
|
|
8
|
+
|
|
9
|
+
The fold formula and the ``shared_code_hash`` sorted-digest fold are wire
|
|
10
|
+
contracts mirrored byte-for-byte from continuo's manifest-controller
|
|
11
|
+
(``content_hash_fold``) and its dbt-side shared-code hashing. Every byte here
|
|
12
|
+
is load-bearing: manifest-controller independently recomputes the fold from
|
|
13
|
+
the parts this repo emits and rejects the release artifact if the two
|
|
14
|
+
results differ.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from collections.abc import Iterable
|
|
18
|
+
from copy import deepcopy
|
|
19
|
+
from hashlib import sha256
|
|
20
|
+
from json import dumps
|
|
21
|
+
|
|
22
|
+
HASH_PART_FIELDS = ("source_hash", "shared_code_hash", "config_hash")
|
|
23
|
+
CONTENT_HASH_FIELD = "content_hash"
|
|
4
24
|
|
|
5
25
|
|
|
6
26
|
def canonical_entry(entry: dict) -> dict:
|
|
27
|
+
"""Deep-copy *entry*, drop the four hash fields, whitespace-normalize reads.
|
|
28
|
+
|
|
29
|
+
Drops ``content_hash`` and all three of ``HASH_PART_FIELDS`` from the
|
|
30
|
+
basis, and whitespace-normalizes every ``str`` value in ``reads`` via
|
|
31
|
+
``" ".join(sql.split())``. Everything else is preserved verbatim,
|
|
32
|
+
including ``config``.
|
|
7
33
|
"""
|
|
8
|
-
|
|
9
|
-
every value in reads via " ".join(sql.split()).
|
|
10
|
-
"""
|
|
11
|
-
canonical = copy.deepcopy(entry)
|
|
34
|
+
canonical = deepcopy(entry)
|
|
12
35
|
|
|
13
|
-
|
|
14
|
-
|
|
36
|
+
canonical.pop(CONTENT_HASH_FIELD, None)
|
|
37
|
+
for field in HASH_PART_FIELDS:
|
|
38
|
+
canonical.pop(field, None)
|
|
15
39
|
|
|
16
|
-
|
|
17
|
-
if
|
|
18
|
-
for key, value in
|
|
40
|
+
reads = canonical.get("reads")
|
|
41
|
+
if isinstance(reads, dict):
|
|
42
|
+
for key, value in reads.items():
|
|
19
43
|
if isinstance(value, str):
|
|
20
|
-
|
|
44
|
+
reads[key] = " ".join(value.split())
|
|
21
45
|
|
|
22
46
|
return canonical
|
|
23
47
|
|
|
24
48
|
|
|
25
|
-
def
|
|
26
|
-
"""
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
49
|
+
def canonical_json(entry: dict) -> str:
|
|
50
|
+
"""`json.dumps(canonical_entry(entry), sort_keys=True, separators=(",", ":"))`."""
|
|
51
|
+
return dumps(canonical_entry(entry), sort_keys=True, separators=(",", ":"))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def source_hash(script_bytes: bytes) -> str:
|
|
55
|
+
"""Bare hex `sha256(script_bytes)` — the node's own script, byte-for-byte."""
|
|
56
|
+
return sha256(script_bytes).hexdigest()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def shared_code_hash(member_bytes: Iterable[bytes]) -> str:
|
|
60
|
+
"""`""` when the closure is empty, else bare hex of the sorted-digest fold."""
|
|
61
|
+
unit_hashes = sorted(sha256(member).hexdigest() for member in member_bytes)
|
|
62
|
+
if not unit_hashes:
|
|
63
|
+
return ""
|
|
64
|
+
return sha256("".join(unit_hashes).encode()).hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def config_hash(entry: dict) -> str:
|
|
68
|
+
"""Bare hex `sha256(canonical_json(entry).encode())`."""
|
|
69
|
+
return sha256(canonical_json(entry).encode()).hexdigest()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def content_hash_fold(source: str, shared: str, config: str) -> str:
|
|
73
|
+
"""`"sha256:" + sha256(f"{source}|{shared}|{config}".encode()).hexdigest()`."""
|
|
74
|
+
return "sha256:" + sha256(f"{source}|{shared}|{config}".encode()).hexdigest()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def hash_parts(
|
|
78
|
+
entry: dict, script_bytes: bytes, member_bytes: Iterable[bytes]
|
|
79
|
+
) -> dict[str, str]:
|
|
80
|
+
"""Return all four fields: the three parts plus their fold."""
|
|
81
|
+
s = source_hash(script_bytes)
|
|
82
|
+
sh = shared_code_hash(member_bytes)
|
|
83
|
+
c = config_hash(entry)
|
|
84
|
+
return {
|
|
85
|
+
"source_hash": s,
|
|
86
|
+
"shared_code_hash": sh,
|
|
87
|
+
"config_hash": c,
|
|
88
|
+
"content_hash": content_hash_fold(s, sh, c),
|
|
89
|
+
}
|
continuo_python_runtime/lint.py
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
"""Script linting for forbidden imports, SQL literals,
|
|
1
|
+
"""Script linting for forbidden imports, SQL literals, data-access calls, and
|
|
2
|
+
dynamic-import constructs."""
|
|
2
3
|
|
|
3
4
|
import ast
|
|
4
5
|
import re
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
|
|
8
|
+
from continuo_python_runtime.closure import dynamic_import_violations
|
|
9
|
+
|
|
7
10
|
# Forbidden warehouse driver modules (check root module of imports)
|
|
8
11
|
FORBIDDEN_DRIVERS = {
|
|
9
12
|
"psycopg2",
|
|
@@ -127,14 +130,23 @@ def lint_source(source: str, filename: str) -> list[str]:
|
|
|
127
130
|
they are never matched and need no exemption. Exempted: attribute
|
|
128
131
|
access on ``self``/``cls`` (e.g. ``self._helper()``), so a script's own
|
|
129
132
|
class-private helpers aren't flagged.
|
|
133
|
+
- L5: dynamic-import construct (``importlib`` in any form, ``__import__``,
|
|
134
|
+
``exec``, ``eval``, or ``.import_module``). The content hash's import
|
|
135
|
+
closure is computed by static AST analysis (see ``closure.py``); a
|
|
136
|
+
construct that analysis cannot see must not exist, or a node could
|
|
137
|
+
import a module whose edits never re-fingerprint it. Detection is
|
|
138
|
+
delegated to ``continuo_python_runtime.closure.dynamic_import_violations``
|
|
139
|
+
so the predicate is defined in exactly one place.
|
|
130
140
|
|
|
131
141
|
Docstrings (the first statement of a Module/ClassDef/FunctionDef/
|
|
132
142
|
AsyncFunctionDef body, when it is a bare string Expr) are exempt from the
|
|
133
143
|
L2 constant pass, since prose commonly contains words like "select" and
|
|
134
144
|
"from". This is a best-effort, position-based exemption: the same prose
|
|
135
145
|
assigned to a variable is still flagged.
|
|
146
|
+
|
|
147
|
+
Violations are returned sorted by line number; ties keep discovery order.
|
|
136
148
|
"""
|
|
137
|
-
|
|
149
|
+
entries: list[tuple[int, str]] = []
|
|
138
150
|
|
|
139
151
|
# Try to parse the source code
|
|
140
152
|
try:
|
|
@@ -152,16 +164,16 @@ def lint_source(source: str, filename: str) -> list[str]:
|
|
|
152
164
|
for alias in node.names:
|
|
153
165
|
root_module = alias.name.split(".")[0]
|
|
154
166
|
if root_module in FORBIDDEN_DRIVERS:
|
|
155
|
-
|
|
156
|
-
|
|
167
|
+
entries.append(
|
|
168
|
+
(node.lineno, f"forbidden warehouse driver import '{root_module}'")
|
|
157
169
|
)
|
|
158
170
|
|
|
159
171
|
elif isinstance(node, ast.ImportFrom):
|
|
160
172
|
if node.module:
|
|
161
173
|
root_module = node.module.split(".")[0]
|
|
162
174
|
if root_module in FORBIDDEN_DRIVERS:
|
|
163
|
-
|
|
164
|
-
|
|
175
|
+
entries.append(
|
|
176
|
+
(node.lineno, f"forbidden warehouse driver import '{root_module}'")
|
|
165
177
|
)
|
|
166
178
|
|
|
167
179
|
# Track imports of forbidden data-access functions
|
|
@@ -171,8 +183,8 @@ def lint_source(source: str, filename: str) -> list[str]:
|
|
|
171
183
|
if alias.name in FORBIDDEN_CALLS:
|
|
172
184
|
local_name = alias.asname if alias.asname else alias.name
|
|
173
185
|
imported_forbidden_calls[local_name] = node.lineno
|
|
174
|
-
|
|
175
|
-
|
|
186
|
+
entries.append(
|
|
187
|
+
(node.lineno, f"forbidden data-access import '{alias.name}'")
|
|
176
188
|
)
|
|
177
189
|
|
|
178
190
|
# Track constants consumed in compound expressions (to avoid double-reporting)
|
|
@@ -190,9 +202,7 @@ def lint_source(source: str, filename: str) -> list[str]:
|
|
|
190
202
|
snippet = reconstructed[:40]
|
|
191
203
|
if len(reconstructed) > 40:
|
|
192
204
|
snippet += "..."
|
|
193
|
-
|
|
194
|
-
f"{filename}:{node.lineno}: SQL string literal '{snippet}'"
|
|
195
|
-
)
|
|
205
|
+
entries.append((node.lineno, f"SQL string literal '{snippet}'"))
|
|
196
206
|
# Only the literal fragments actually folded into the
|
|
197
207
|
# reconstruction are consumed; FormattedValue contents are not.
|
|
198
208
|
consumed_constants |= consumed
|
|
@@ -206,9 +216,7 @@ def lint_source(source: str, filename: str) -> list[str]:
|
|
|
206
216
|
snippet = reconstructed_text[:40]
|
|
207
217
|
if len(reconstructed_text) > 40:
|
|
208
218
|
snippet += "..."
|
|
209
|
-
|
|
210
|
-
f"{filename}:{node.lineno}: SQL string literal '{snippet}'"
|
|
211
|
-
)
|
|
219
|
+
entries.append((node.lineno, f"SQL string literal '{snippet}'"))
|
|
212
220
|
# Only the leaf Constants actually folded into the
|
|
213
221
|
# reconstruction are consumed - never descendants of a
|
|
214
222
|
# FormattedValue expression.
|
|
@@ -222,24 +230,22 @@ def lint_source(source: str, filename: str) -> list[str]:
|
|
|
222
230
|
snippet = node.value[:40]
|
|
223
231
|
if len(node.value) > 40:
|
|
224
232
|
snippet += "..."
|
|
225
|
-
|
|
226
|
-
f"{filename}:{node.lineno}: SQL string literal '{snippet}'"
|
|
227
|
-
)
|
|
233
|
+
entries.append((node.lineno, f"SQL string literal '{snippet}'"))
|
|
228
234
|
|
|
229
235
|
# L3: Check for forbidden data-access calls
|
|
230
236
|
elif isinstance(node, ast.Call):
|
|
231
237
|
# Attribute access: pd.read_sql(...)
|
|
232
238
|
if isinstance(node.func, ast.Attribute):
|
|
233
239
|
if node.func.attr in FORBIDDEN_CALLS:
|
|
234
|
-
|
|
235
|
-
|
|
240
|
+
entries.append(
|
|
241
|
+
(node.lineno, f"forbidden data-access call '{node.func.attr}'")
|
|
236
242
|
)
|
|
237
243
|
|
|
238
244
|
# Name reference: read_sql_table(...) or rs(...) where rs is an alias
|
|
239
245
|
elif isinstance(node.func, ast.Name):
|
|
240
246
|
if node.func.id in imported_forbidden_calls:
|
|
241
|
-
|
|
242
|
-
|
|
247
|
+
entries.append(
|
|
248
|
+
(node.lineno, f"forbidden data-access call '{node.func.id}'")
|
|
243
249
|
)
|
|
244
250
|
|
|
245
251
|
# L4: Check for private/protected attribute access
|
|
@@ -249,11 +255,19 @@ def lint_source(source: str, filename: str) -> list[str]:
|
|
|
249
255
|
"cls",
|
|
250
256
|
)
|
|
251
257
|
if node.attr.startswith("_") and not is_self_or_cls:
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
258
|
+
entries.append((node.lineno, f"private attribute access '{node.attr}'"))
|
|
259
|
+
|
|
260
|
+
# L5: dynamic-import constructs, reported via the shared closure.py
|
|
261
|
+
# predicate (which raises on only the first hit; here every hit counts).
|
|
262
|
+
for lineno, construct in dynamic_import_violations(tree):
|
|
263
|
+
entries.append((lineno, f"dynamic import construct '{construct}'"))
|
|
264
|
+
|
|
265
|
+
# Stable sort by line number: this is what puts L5 hits "alongside" the
|
|
266
|
+
# L1-L4 hits found above instead of trailing as a separate block, while
|
|
267
|
+
# violations on the same line keep the order the passes above found them
|
|
268
|
+
# (L1 first pass, then L2-L4 second pass, then L5).
|
|
269
|
+
entries.sort(key=lambda entry: entry[0])
|
|
270
|
+
return [f"{filename}:{lineno}: {message}" for lineno, message in entries]
|
|
257
271
|
|
|
258
272
|
|
|
259
273
|
def lint_paths(paths: list[Path]) -> list[str]:
|
continuo_python_runtime/types.py
CHANGED
|
@@ -10,6 +10,7 @@ import re
|
|
|
10
10
|
from dataclasses import dataclass
|
|
11
11
|
|
|
12
12
|
import pyarrow as pa # type: ignore[import-untyped]
|
|
13
|
+
from continuo_validation_contract.types import validate_column_type # type: ignore[import-untyped]
|
|
13
14
|
|
|
14
15
|
from continuo_python_runtime.errors import ContractError
|
|
15
16
|
|
|
@@ -36,6 +37,14 @@ class SqlType:
|
|
|
36
37
|
def parse_sql_type(raw: str) -> SqlType:
|
|
37
38
|
"""Parse a SQL type string into a canonical SqlType.
|
|
38
39
|
|
|
40
|
+
``continuo_validation_contract.types.validate_column_type`` is the single
|
|
41
|
+
acceptance authority for the grammar shape (case-insensitive, injection-
|
|
42
|
+
guarded): it runs first, and anything it rejects is rejected here too. The
|
|
43
|
+
logic below only extracts precision/scale/length and enforces the NUMERIC
|
|
44
|
+
range this repo's Arrow mapping additionally requires (1-38 precision,
|
|
45
|
+
0-precision scale) -- a semantic check the shared grammar deliberately
|
|
46
|
+
doesn't make, since it's specific to this repo's decimal128 mapping.
|
|
47
|
+
|
|
39
48
|
Args:
|
|
40
49
|
raw: A SQL type string (case-insensitive).
|
|
41
50
|
|
|
@@ -43,44 +52,39 @@ def parse_sql_type(raw: str) -> SqlType:
|
|
|
43
52
|
A SqlType with canonicalized base and parsed parameters.
|
|
44
53
|
|
|
45
54
|
Raises:
|
|
46
|
-
ContractError: If the type is unsupported or malformed
|
|
55
|
+
ContractError: If the type is unsupported or malformed, or a NUMERIC's
|
|
56
|
+
precision/scale is out of range.
|
|
47
57
|
"""
|
|
48
58
|
raw = raw.strip()
|
|
49
|
-
|
|
50
|
-
|
|
59
|
+
try:
|
|
60
|
+
validate_column_type(raw)
|
|
61
|
+
except ValueError as exc:
|
|
62
|
+
raise ContractError(str(exc)) from exc
|
|
51
63
|
|
|
52
64
|
# Normalize to uppercase for parsing
|
|
53
65
|
normalized = raw.upper()
|
|
54
66
|
|
|
55
|
-
# Handle DOUBLE PRECISION specially (has a space).
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
# though it is the SqlType.base value produced above.
|
|
67
|
+
# Handle DOUBLE PRECISION specially (has a space). It is the only grammar
|
|
68
|
+
# member whose canonical SqlType.base spelling (DOUBLE_PRECISION, with an
|
|
69
|
+
# underscore) differs from its authored one.
|
|
59
70
|
if normalized == "DOUBLE PRECISION":
|
|
60
71
|
return SqlType("DOUBLE_PRECISION")
|
|
61
72
|
|
|
62
|
-
#
|
|
73
|
+
# Parametrized types: TYPE(args). validate_column_type above has already
|
|
74
|
+
# rejected every shape but (NUMERIC|DECIMAL)(\d+,\s*\d+) and
|
|
75
|
+
# (VARCHAR|CHAR)(\d+), so no further shape checking is needed here.
|
|
63
76
|
match = re.match(r"^([A-Z_]+)\s*\((.*)\)$", normalized)
|
|
64
77
|
if match:
|
|
65
78
|
base_name = match.group(1).strip()
|
|
66
79
|
params_str = match.group(2).strip()
|
|
67
80
|
|
|
68
|
-
|
|
69
|
-
if base_name == "INT":
|
|
70
|
-
base_name = "INTEGER"
|
|
71
|
-
elif base_name == "DECIMAL":
|
|
81
|
+
if base_name == "DECIMAL":
|
|
72
82
|
base_name = "NUMERIC"
|
|
73
83
|
|
|
74
84
|
if base_name == "NUMERIC":
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
raise ContractError(
|
|
79
|
-
f"type: NUMERIC requires precision and scale as unsigned integers, "
|
|
80
|
-
f"got ({params_str})"
|
|
81
|
-
)
|
|
82
|
-
precision = int(numeric_match.group(1))
|
|
83
|
-
scale = int(numeric_match.group(2))
|
|
85
|
+
precision_str, scale_str = params_str.split(",")
|
|
86
|
+
precision = int(precision_str)
|
|
87
|
+
scale = int(scale_str)
|
|
84
88
|
if not (1 <= precision <= 38):
|
|
85
89
|
raise ContractError(
|
|
86
90
|
f"type: NUMERIC precision must be between 1 and 38, got {precision}"
|
|
@@ -92,82 +96,14 @@ def parse_sql_type(raw: str) -> SqlType:
|
|
|
92
96
|
)
|
|
93
97
|
return SqlType("NUMERIC", precision=precision, scale=scale)
|
|
94
98
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if "," in params_str:
|
|
98
|
-
raise ContractError(
|
|
99
|
-
f"type: VARCHAR takes a single parameter (length), "
|
|
100
|
-
f"got {params_str!r}"
|
|
101
|
-
)
|
|
102
|
-
varchar_match = re.match(r"^(\d+)$", params_str)
|
|
103
|
-
if not varchar_match:
|
|
104
|
-
raise ContractError(
|
|
105
|
-
f"type: VARCHAR length must be an unsigned integer, "
|
|
106
|
-
f"got {params_str!r}"
|
|
107
|
-
)
|
|
108
|
-
length = int(varchar_match.group(1))
|
|
109
|
-
return SqlType("VARCHAR", length=length)
|
|
99
|
+
# VARCHAR(length) or CHAR(length)
|
|
100
|
+
return SqlType(base_name, length=int(params_str))
|
|
110
101
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
f"got {params_str!r}"
|
|
117
|
-
)
|
|
118
|
-
char_match = re.match(r"^(\d+)$", params_str)
|
|
119
|
-
if not char_match:
|
|
120
|
-
raise ContractError(
|
|
121
|
-
f"type: CHAR length must be an unsigned integer, "
|
|
122
|
-
f"got {params_str!r}"
|
|
123
|
-
)
|
|
124
|
-
length = int(char_match.group(1))
|
|
125
|
-
return SqlType("CHAR", length=length)
|
|
126
|
-
|
|
127
|
-
else:
|
|
128
|
-
# No other types accept parameters
|
|
129
|
-
raise ContractError(
|
|
130
|
-
f"type: {base_name} does not accept parameters, got ({params_str})"
|
|
131
|
-
)
|
|
132
|
-
|
|
133
|
-
# No parameters - must be a bare base type
|
|
134
|
-
normalized_base = normalized
|
|
135
|
-
|
|
136
|
-
# Alias handling
|
|
137
|
-
if normalized_base == "INT":
|
|
138
|
-
normalized_base = "INTEGER"
|
|
139
|
-
elif normalized_base == "DECIMAL":
|
|
140
|
-
normalized_base = "NUMERIC"
|
|
141
|
-
|
|
142
|
-
# List of supported bare types
|
|
143
|
-
# Note: "DOUBLE_PRECISION" (underscored) is deliberately excluded here.
|
|
144
|
-
# It is the canonical SqlType.base value, but only the authored spelling
|
|
145
|
-
# "DOUBLE PRECISION" (with a space) is part of the contract grammar and
|
|
146
|
-
# is handled above.
|
|
147
|
-
supported_bare = {
|
|
148
|
-
"BIGINT",
|
|
149
|
-
"INTEGER",
|
|
150
|
-
"TEXT",
|
|
151
|
-
"TIMESTAMP",
|
|
152
|
-
"DATE",
|
|
153
|
-
"BOOLEAN",
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
if normalized_base in supported_bare:
|
|
157
|
-
return SqlType(normalized_base)
|
|
158
|
-
|
|
159
|
-
# Bare VARCHAR and CHAR require a length parameter
|
|
160
|
-
if normalized_base == "VARCHAR":
|
|
161
|
-
raise ContractError(f"type: VARCHAR requires a length parameter, got {raw!r}")
|
|
162
|
-
if normalized_base == "CHAR":
|
|
163
|
-
raise ContractError(f"type: CHAR requires a length parameter, got {raw!r}")
|
|
164
|
-
|
|
165
|
-
# Bare NUMERIC and DECIMAL require parameters
|
|
166
|
-
if normalized_base == "NUMERIC":
|
|
167
|
-
raise ContractError(f"type: NUMERIC requires precision and scale parameters, got {raw!r}")
|
|
168
|
-
|
|
169
|
-
# Unknown type
|
|
170
|
-
raise ContractError(f"type: unknown SQL type {raw!r}")
|
|
102
|
+
# Bare base type: BIGINT, INT, INTEGER, TEXT, TIMESTAMP, DATE, BOOLEAN --
|
|
103
|
+
# the only shapes left once DOUBLE PRECISION and the parametrized types
|
|
104
|
+
# above are ruled out.
|
|
105
|
+
normalized_base = "INTEGER" if normalized == "INT" else normalized
|
|
106
|
+
return SqlType(normalized_base)
|
|
171
107
|
|
|
172
108
|
|
|
173
109
|
def arrow_type(t: SqlType) -> pa.DataType:
|
{continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: continuo-python-runtime
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
Maintainer: Simone Carolini
|
|
@@ -8,9 +8,10 @@ Classifier: Development Status :: 4 - Beta
|
|
|
8
8
|
Classifier: Intended Audience :: Developers
|
|
9
9
|
Classifier: Programming Language :: Python :: 3.14
|
|
10
10
|
Requires-Python: >=3.14
|
|
11
|
-
Requires-Dist: continuo-validation-contract==0.
|
|
11
|
+
Requires-Dist: continuo-validation-contract==0.4.0
|
|
12
12
|
Requires-Dist: pyarrow==25.0.0
|
|
13
13
|
Requires-Dist: pyyaml==6.0.3
|
|
14
|
+
Requires-Dist: sqlglot==30.15.0
|
|
14
15
|
Description-Content-Type: text/markdown
|
|
15
16
|
|
|
16
17
|
# Continuo Python Runtime
|
|
@@ -51,17 +52,26 @@ data-plane adapters here.
|
|
|
51
52
|
| Package (distribution name) | Module | Lives in | Role |
|
|
52
53
|
| --- | --- | --- | --- |
|
|
53
54
|
| `continuo-python-runtime` | `continuo_python_runtime` | this repo (root) | Harness: CLI, `conform()`, `RunContext`, error taxonomy. |
|
|
54
|
-
| `continuo-python-runtime-postgres` | `continuo_python_runtime_postgres` | this repo, `python-runtime-postgres/` | Data-plane `RuntimeAdapter` for Postgres (`fetch`/`ensure_table`/`load`). |
|
|
55
|
-
| `continuo-python-runtime-trino` | `continuo_python_runtime_trino` | this repo, `python-runtime-trino/` | Data-plane `RuntimeAdapter` for Trino/Iceberg. |
|
|
55
|
+
| `continuo-python-runtime-postgres` | `continuo_python_runtime_postgres` | this repo, `python-runtime-postgres/` | Data-plane `RuntimeAdapter` for Postgres (`fetch`/`ensure_table`/`load`). **Not published to PyPI** — built from source into the image. |
|
|
56
|
+
| `continuo-python-runtime-trino` | `continuo_python_runtime_trino` | this repo, `python-runtime-trino/` | Data-plane `RuntimeAdapter` for Trino/Iceberg. **Not published to PyPI** — built from source into the image. |
|
|
56
57
|
| `continuo-validation-contract` | `continuo_validation_contract` | `continuo-validation-runners` | The published contract (schema, `RuntimeAdapter` port, result-block format) both sides depend on. |
|
|
57
58
|
| `continuo-validation-postgres` / `continuo-validation-trino` | `continuo_validation_postgres` / `continuo_validation_trino` | `continuo-validation-runners` | Validation-side (lint/merge, no live warehouse I/O) adapters — not to be confused with the data-plane adapters above. |
|
|
58
59
|
|
|
59
60
|
All three packages built in this repo resolve `continuo-validation-contract`
|
|
60
|
-
from PyPI (`==0.
|
|
61
|
+
from PyPI (`==0.4.0`); the two adapter packages are uv workspace members
|
|
61
62
|
(`[tool.uv.workspace]` in the root `pyproject.toml`), so `uv sync
|
|
62
63
|
--all-packages --all-groups` at the repo root installs everything for local
|
|
63
64
|
development.
|
|
64
65
|
|
|
66
|
+
**Only `continuo-python-runtime` is published to PyPI.** The two engine
|
|
67
|
+
adapters are built **from source into the runtime images**: `Dockerfile.postgres`
|
|
68
|
+
and `Dockerfile.trino` `pip install` them out of the build context, so each
|
|
69
|
+
image ships exactly one adapter and the harness discovers it through the
|
|
70
|
+
`continuo_runtime.adapters` entry-point group at run time. Nothing installs
|
|
71
|
+
them from an index — the harness package does not depend on them, and domain
|
|
72
|
+
repos get their adapter by building `FROM` a published base image. They are
|
|
73
|
+
still built, type-checked, and tested by CI on every change.
|
|
74
|
+
|
|
65
75
|
## Quickstart for domain teams
|
|
66
76
|
|
|
67
77
|
1. Copy `template/` into a new repository.
|
|
@@ -85,6 +95,28 @@ development.
|
|
|
85
95
|
and merges the contracts, builds and pushes the image, uploads the merged
|
|
86
96
|
contract to S3, and POSTs the release.
|
|
87
97
|
|
|
98
|
+
### Upgrading an existing domain repo
|
|
99
|
+
|
|
100
|
+
`validate` / `merge` / `hash` now hand every declared read to a real SQL
|
|
101
|
+
parser (sqlglot, via `continuo_validation_contract.sql.ensure_single_read`)
|
|
102
|
+
instead of scanning it for a leading `SELECT`/`WITH`. Two things follow for a
|
|
103
|
+
repo written before this, on its next release: SQL a driver would accept but
|
|
104
|
+
a parser will not — most commonly a driver-specific bind placeholder like
|
|
105
|
+
psycopg2's `%(name)s`, which `ctx.read(name)` could never have used anyway —
|
|
106
|
+
now fails validation, and engine-specific syntax (postgres `~`, `@>`, …)
|
|
107
|
+
needs `--dialect <engine>`, which a repo should be passing regardless since
|
|
108
|
+
Continuo bind-checks every read in the install's own warehouse dialect. Run
|
|
109
|
+
the pre-flight check once before your next release; it reports every affected
|
|
110
|
+
read at once:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
continuo-runtime validate contracts/ --dialect postgres # or trino
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The runtime image does not re-run this gate, so a read that passes here is
|
|
117
|
+
not re-judged under a different grammar in production. See
|
|
118
|
+
`docs/boundary-contract.md` §13.1.
|
|
119
|
+
|
|
88
120
|
## The script API
|
|
89
121
|
|
|
90
122
|
A node script is a Python file with exactly one required entry point:
|
|
@@ -125,6 +157,17 @@ def run(ctx):
|
|
|
125
157
|
are the driver-import and data-access-call rules, together with the fact
|
|
126
158
|
that `RunContext` only exposes `ctx.read()` — all warehouse access goes
|
|
127
159
|
through it.
|
|
160
|
+
- Scripts may import shared in-repo helpers. Before executing a script the
|
|
161
|
+
harness puts the repo root (`APP_ROOT`) and the script's own directory on
|
|
162
|
+
`sys.path`, so both `import helpers` (a sibling of the script) and
|
|
163
|
+
`from lib.shared import ...` (anywhere under the repo root) work, including
|
|
164
|
+
from inside `run()`. Every helper a script reaches transitively is folded
|
|
165
|
+
into `shared_code_hash`, so editing one re-fingerprints the node — but the
|
|
166
|
+
hash does not put the file in the image: **`COPY` every directory your
|
|
167
|
+
scripts import from in your `Dockerfile`**, or the release is valid and the
|
|
168
|
+
node dies with `ModuleNotFoundError` on its first run. Because the repo root
|
|
169
|
+
precedes the standard library on `sys.path`, avoid naming a top-level module
|
|
170
|
+
after a stdlib one (`types.py`, `json.py`, `logging.py`, …).
|
|
128
171
|
- The harness — not the script — performs the write. It calls `conform()`
|
|
129
172
|
on whatever `run()` returned and issues the only INSERT; the script never
|
|
130
173
|
writes directly.
|
|
@@ -165,9 +208,9 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
165
208
|
`FROM`:
|
|
166
209
|
|
|
167
210
|
```dockerfile
|
|
168
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime:v0.
|
|
211
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime:v0.2.0-postgres
|
|
169
212
|
# or
|
|
170
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime:v0.
|
|
213
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime:v0.2.0-trino
|
|
171
214
|
```
|
|
172
215
|
|
|
173
216
|
Each image bakes in exactly one `RuntimeAdapter` for that engine — installed
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
|
|
2
|
+
continuo_python_runtime/cli.py,sha256=FlY1GqEKrtx4O34zMzlFVBuM6wAPaFEyAHjhN0lDcdA,5495
|
|
3
|
+
continuo_python_runtime/closure.py,sha256=6e-IAQp8Lieb0bGKP1F6JE4TgVBJ_COTGXRUult2xeI,12991
|
|
4
|
+
continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
|
|
5
|
+
continuo_python_runtime/context.py,sha256=ZyeN_dA2DihQbegBiiqHj1_EKk79GsxS-tATPVymUPg,1856
|
|
6
|
+
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
7
|
+
continuo_python_runtime/harness.py,sha256=SF5ciKsc1RawAQ41h_TuYqqkOp5mf5SSeELv0_B2EvI,12211
|
|
8
|
+
continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
|
|
9
|
+
continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
|
|
10
|
+
continuo_python_runtime/types.py,sha256=kecwFUMkfglv0h6fkcXTlHkEHeLziWyzJvBsimy-Bl0,5065
|
|
11
|
+
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
12
|
+
continuo_python_runtime/contract/loader.py,sha256=6w8uC7kZzGKMDj45pXJtWVI7Pa675CsjJG3e37LcelI,15623
|
|
13
|
+
continuo_python_runtime/contract/merge.py,sha256=XCCeE2vJUm0KA60d2XsHpjRh5fIdVfFu_HsBzEyUWow,6505
|
|
14
|
+
continuo_python_runtime/contract/model.py,sha256=2hKI04M0WwvTkOy9eDGl2dRZgY3bAuJYktAG-ICI-AU,936
|
|
15
|
+
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
16
|
+
continuo_python_runtime-0.2.0.dist-info/METADATA,sha256=Dvmmo8AgQ3gg0jFhJbcSDC3tCagVG_dSy5gfUj6L_bk,13607
|
|
17
|
+
continuo_python_runtime-0.2.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
18
|
+
continuo_python_runtime-0.2.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
19
|
+
continuo_python_runtime-0.2.0.dist-info/RECORD,,
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
|
|
2
|
-
continuo_python_runtime/cli.py,sha256=Qki4Mfoab74D0DvguZd5yzSfI-FRQDmtvwC0OacxrAc,4860
|
|
3
|
-
continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
|
|
4
|
-
continuo_python_runtime/context.py,sha256=ZyeN_dA2DihQbegBiiqHj1_EKk79GsxS-tATPVymUPg,1856
|
|
5
|
-
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
6
|
-
continuo_python_runtime/harness.py,sha256=YcWy4ummKY3HQYQ4JQ3-UI5jxDgYwJDQlWT88_GRPH4,7504
|
|
7
|
-
continuo_python_runtime/hashing.py,sha256=6D4MntcjaHTEC55IQBBX2Pq9K59D0TWlxTSXGJ5Jsro,1159
|
|
8
|
-
continuo_python_runtime/lint.py,sha256=HgB0BAuiQuS3DgT6BsOoB2M5A-_OKcFsGbaTWhuM7XU,11897
|
|
9
|
-
continuo_python_runtime/types.py,sha256=z1qnBbbaP6nFN2mmRySXM4ZAesv5N6_4E9vZ4kv74HU,7138
|
|
10
|
-
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
11
|
-
continuo_python_runtime/contract/loader.py,sha256=wlxI6XZIoUZ4z9rDe1G8s26jbL0tGXnAHPGajTAM0ag,12828
|
|
12
|
-
continuo_python_runtime/contract/merge.py,sha256=WFde9lzrojlMNjuEU7YW8FWOs6VPSrWPY2_Sri0ljZE,2567
|
|
13
|
-
continuo_python_runtime/contract/model.py,sha256=6luq3SJcgshKoG0QsQhJPZ5fzLa8AcGAev9D4jiGgxA,849
|
|
14
|
-
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
15
|
-
continuo_python_runtime-0.1.0.dist-info/METADATA,sha256=YQWMZaYiS3gpRnIutzgKhT3KeB5BtUpeMrwi0qKLRUU,10953
|
|
16
|
-
continuo_python_runtime-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
17
|
-
continuo_python_runtime-0.1.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
18
|
-
continuo_python_runtime-0.1.0.dist-info/RECORD,,
|
|
File without changes
|
{continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|