continuo-python-runtime 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/contract/loader.py +66 -31
- continuo_python_runtime/contract/merge.py +14 -6
- continuo_python_runtime/contract/model.py +2 -0
- continuo_python_runtime/csv_loader.py +68 -0
- continuo_python_runtime/csv_readers/__init__.py +17 -0
- continuo_python_runtime/csv_readers/https.py +118 -0
- continuo_python_runtime/csv_readers/s3.py +57 -0
- continuo_python_runtime/csv_source.py +96 -0
- continuo_python_runtime/harness.py +18 -7
- continuo_python_runtime/validation/runner.py +39 -4
- {continuo_python_runtime-0.3.0.dist-info → continuo_python_runtime-0.4.0.dist-info}/METADATA +38 -5
- continuo_python_runtime-0.4.0.dist-info/RECORD +29 -0
- continuo_python_runtime-0.4.0.dist-info/licenses/LICENSE +201 -0
- continuo_python_runtime-0.4.0.dist-info/licenses/NOTICE +11 -0
- continuo_python_runtime-0.3.0.dist-info/RECORD +0 -22
- {continuo_python_runtime-0.3.0.dist-info → continuo_python_runtime-0.4.0.dist-info}/WHEEL +0 -0
- {continuo_python_runtime-0.3.0.dist-info → continuo_python_runtime-0.4.0.dist-info}/entry_points.txt +0 -0
|
@@ -18,9 +18,11 @@ from sqlglot.errors import TokenError
|
|
|
18
18
|
from continuo_python_runtime.contract.model import (
|
|
19
19
|
CRITICALITIES,
|
|
20
20
|
EXTRA_COLUMNS_POLICIES,
|
|
21
|
+
KINDS,
|
|
21
22
|
Column,
|
|
22
23
|
Node,
|
|
23
24
|
)
|
|
25
|
+
from continuo_python_runtime.csv_source import parse_csv_uri
|
|
24
26
|
from continuo_python_runtime.errors import ContractError
|
|
25
27
|
from continuo_python_runtime.types import parse_sql_type
|
|
26
28
|
|
|
@@ -31,6 +33,7 @@ _ALLOWED_KEYS = {
|
|
|
31
33
|
"owner",
|
|
32
34
|
"schedule",
|
|
33
35
|
"criticality",
|
|
36
|
+
"kind",
|
|
34
37
|
"script",
|
|
35
38
|
"extra_columns",
|
|
36
39
|
"reads",
|
|
@@ -39,7 +42,7 @@ _ALLOWED_KEYS = {
|
|
|
39
42
|
"content_hash",
|
|
40
43
|
}
|
|
41
44
|
|
|
42
|
-
_REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule"
|
|
45
|
+
_REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule")
|
|
43
46
|
|
|
44
47
|
_ALLOWED_OUTPUT_COLUMN_KEYS = {"name", "type", "nullable"}
|
|
45
48
|
|
|
@@ -170,6 +173,12 @@ def parse_node(
|
|
|
170
173
|
if unknown:
|
|
171
174
|
raise ContractError(f"{label}: unknown key(s) {sorted(unknown)}")
|
|
172
175
|
|
|
176
|
+
kind = raw.get("kind", "python-model")
|
|
177
|
+
if not isinstance(kind, str) or kind not in KINDS:
|
|
178
|
+
raise ContractError(
|
|
179
|
+
f"{label}: 'kind' must be one of {sorted(KINDS)}, got {kind!r}"
|
|
180
|
+
)
|
|
181
|
+
|
|
173
182
|
for field in _REQUIRED_STRING_FIELDS:
|
|
174
183
|
value = raw.get(field)
|
|
175
184
|
if not isinstance(value, str) or not value.strip():
|
|
@@ -181,7 +190,21 @@ def parse_node(
|
|
|
181
190
|
table = raw["table"]
|
|
182
191
|
owner = raw["owner"]
|
|
183
192
|
schedule = raw["schedule"]
|
|
184
|
-
|
|
193
|
+
|
|
194
|
+
if kind == "python-csv":
|
|
195
|
+
if "script" in raw:
|
|
196
|
+
raise ContractError(
|
|
197
|
+
f"{label}: 'script' is forbidden for kind python-csv "
|
|
198
|
+
"(csv nodes are contract-only)"
|
|
199
|
+
)
|
|
200
|
+
script = ""
|
|
201
|
+
else:
|
|
202
|
+
raw_script = raw.get("script")
|
|
203
|
+
if not isinstance(raw_script, str) or not raw_script.strip():
|
|
204
|
+
raise ContractError(
|
|
205
|
+
f"{label}: required field 'script' must be a non-empty string"
|
|
206
|
+
)
|
|
207
|
+
script = raw_script
|
|
185
208
|
|
|
186
209
|
criticality = raw.get("criticality")
|
|
187
210
|
if not isinstance(criticality, str) or criticality not in CRITICALITIES:
|
|
@@ -200,39 +223,50 @@ def parse_node(
|
|
|
200
223
|
)
|
|
201
224
|
|
|
202
225
|
reads = raw.get("reads")
|
|
203
|
-
if
|
|
204
|
-
|
|
205
|
-
f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
|
|
206
|
-
)
|
|
207
|
-
for name, sql in reads.items():
|
|
208
|
-
if not isinstance(name, str) or not name.strip():
|
|
209
|
-
raise ContractError(
|
|
210
|
-
f"{label}: 'reads' name {name!r} must be a non-empty string"
|
|
211
|
-
)
|
|
212
|
-
if not isinstance(sql, str) or not sql.strip():
|
|
226
|
+
if kind == "python-csv":
|
|
227
|
+
if not isinstance(reads, dict) or set(reads) != {"csv"}:
|
|
213
228
|
raise ContractError(
|
|
214
|
-
f"{label}: 'reads
|
|
229
|
+
f"{label}: a python-csv node's 'reads' must be exactly {{csv: <uri>}}"
|
|
215
230
|
)
|
|
216
|
-
if not check_reads:
|
|
217
|
-
continue
|
|
218
231
|
try:
|
|
219
|
-
|
|
220
|
-
except (ValueError,
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
# string literal or comment fails sqlglot's tokenizer with a
|
|
225
|
-
# TokenError, a SqlglotError sibling of ParseError and not a
|
|
226
|
-
# subclass of ValueError -- despite ensure_single_read's
|
|
227
|
-
# docstring promising every rejection is a ValueError. Only
|
|
228
|
-
# TokenError, not the broader SqlglotError, is caught here: by
|
|
229
|
-
# the time control reaches this point `dialect` has already been
|
|
230
|
-
# validated once in load_contract_dir, so any other SqlglotError
|
|
231
|
-
# a future sqlglot version might raise from this call should
|
|
232
|
-
# surface as itself, not get relabeled as a rejected read.
|
|
232
|
+
parse_csv_uri(reads["csv"])
|
|
233
|
+
except (ValueError, TypeError) as exc:
|
|
234
|
+
raise ContractError(f"{label}: invalid csv uri: {exc}") from exc
|
|
235
|
+
else:
|
|
236
|
+
if not isinstance(reads, dict) or not reads:
|
|
233
237
|
raise ContractError(
|
|
234
|
-
f"{label}: 'reads
|
|
235
|
-
)
|
|
238
|
+
f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
|
|
239
|
+
)
|
|
240
|
+
for name, sql in reads.items():
|
|
241
|
+
if not isinstance(name, str) or not name.strip():
|
|
242
|
+
raise ContractError(
|
|
243
|
+
f"{label}: 'reads' name {name!r} must be a non-empty string"
|
|
244
|
+
)
|
|
245
|
+
if not isinstance(sql, str) or not sql.strip():
|
|
246
|
+
raise ContractError(
|
|
247
|
+
f"{label}: 'reads.{name}' must be a non-empty SQL string"
|
|
248
|
+
)
|
|
249
|
+
if not check_reads:
|
|
250
|
+
continue
|
|
251
|
+
try:
|
|
252
|
+
ensure_single_read(sql, dialect)
|
|
253
|
+
except (ValueError, TokenError) as exc:
|
|
254
|
+
# ensure_single_read's own message is phrased for check_binds
|
|
255
|
+
# (its only other caller today), so it's wrapped rather than
|
|
256
|
+
# surfaced bare here. TokenError is also caught: an unterminated
|
|
257
|
+
# string literal or comment fails sqlglot's tokenizer with a
|
|
258
|
+
# TokenError, a SqlglotError sibling of ParseError and not a
|
|
259
|
+
# subclass of ValueError -- despite ensure_single_read's
|
|
260
|
+
# docstring promising every rejection is a ValueError. Only
|
|
261
|
+
# TokenError, not the broader SqlglotError, is caught here: by
|
|
262
|
+
# the time control reaches this point `dialect` has already
|
|
263
|
+
# been validated once in load_contract_dir, so any other
|
|
264
|
+
# SqlglotError a future sqlglot version might raise from this
|
|
265
|
+
# call should surface as itself, not get relabeled as a
|
|
266
|
+
# rejected read.
|
|
267
|
+
raise ContractError(
|
|
268
|
+
f"{label}: 'reads.{name}' must be a single read query ({exc})"
|
|
269
|
+
) from exc
|
|
236
270
|
|
|
237
271
|
raw_columns = raw.get("output_columns")
|
|
238
272
|
if not isinstance(raw_columns, list) or not raw_columns:
|
|
@@ -297,6 +331,7 @@ def parse_node(
|
|
|
297
331
|
extra_columns=extra_columns,
|
|
298
332
|
config=config,
|
|
299
333
|
content_hash=content_hash,
|
|
334
|
+
kind=kind,
|
|
300
335
|
)
|
|
301
336
|
|
|
302
337
|
|
|
@@ -27,6 +27,7 @@ def node_entry(node: Node) -> dict:
|
|
|
27
27
|
"owner": node.owner,
|
|
28
28
|
"schedule": node.schedule,
|
|
29
29
|
"criticality": node.criticality,
|
|
30
|
+
"kind": node.kind,
|
|
30
31
|
"script": node.script,
|
|
31
32
|
"reads": node.reads,
|
|
32
33
|
"output_columns": [
|
|
@@ -128,12 +129,19 @@ def build_wire_contract(
|
|
|
128
129
|
for node in nodes:
|
|
129
130
|
entry = node_entry(node)
|
|
130
131
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
132
|
+
if node.kind == "python-csv":
|
|
133
|
+
# A csv node has no script and no import closure: its source IS
|
|
134
|
+
# the uri -- new file content at the same uri is new data, not a
|
|
135
|
+
# new node version.
|
|
136
|
+
uri_bytes = node.reads["csv"].encode()
|
|
137
|
+
entry.update(hash_parts(entry, uri_bytes, []))
|
|
138
|
+
else:
|
|
139
|
+
script_path = resolve_script_path(node.script, repo_root, context=node.relation)
|
|
140
|
+
script_bytes = script_path.read_bytes()
|
|
141
|
+
closure = resolve_closure(script_path, repo_root)
|
|
142
|
+
member_bytes = [member.read_bytes() for member in closure]
|
|
143
|
+
_lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
|
|
144
|
+
entry.update(hash_parts(entry, script_bytes, member_bytes))
|
|
137
145
|
|
|
138
146
|
wire_nodes.append(entry)
|
|
139
147
|
|
|
@@ -6,6 +6,7 @@ from typing import Any
|
|
|
6
6
|
# Module-level constants
|
|
7
7
|
CRITICALITIES = frozenset({"REGULATORY", "CORE", "SECONDARY"})
|
|
8
8
|
EXTRA_COLUMNS_POLICIES = frozenset({"raise", "warn"})
|
|
9
|
+
KINDS = frozenset({"python-model", "python-csv"})
|
|
9
10
|
CONTRACT_VERSION = 1
|
|
10
11
|
|
|
11
12
|
|
|
@@ -34,6 +35,7 @@ class Node:
|
|
|
34
35
|
extra_columns: str = "raise"
|
|
35
36
|
config: dict[str, Any] = field(default_factory=dict)
|
|
36
37
|
content_hash: str | None = None
|
|
38
|
+
kind: str = "python-model"
|
|
37
39
|
|
|
38
40
|
@property
|
|
39
41
|
def relation(self) -> str:
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""continuo_python_runtime/csv_loader.py
|
|
2
|
+
|
|
3
|
+
Producer for python-csv nodes: materialize the declared table from the csv
|
|
4
|
+
source alone. Everything from conform() down (type coercion, extra_columns
|
|
5
|
+
policy, ensure_table, transactional load) is the existing harness path —
|
|
6
|
+
this module only turns the contract entry into a pyarrow Table.
|
|
7
|
+
"""
|
|
8
|
+
import logging
|
|
9
|
+
import tempfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import pyarrow.csv # type: ignore[import-untyped]
|
|
13
|
+
|
|
14
|
+
from continuo_python_runtime.contract.model import Node
|
|
15
|
+
from continuo_python_runtime.csv_readers import reader_for
|
|
16
|
+
from continuo_python_runtime.csv_source import CsvSourceReader, parse_csv_uri
|
|
17
|
+
from continuo_python_runtime.errors import LoadError
|
|
18
|
+
from continuo_python_runtime.types import arrow_type, parse_sql_type
|
|
19
|
+
|
|
20
|
+
logger = logging.getLogger("continuo_python_runtime.csv_loader")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def produce_csv(node: Node, reader: CsvSourceReader | None = None) -> "pyarrow.Table":
|
|
24
|
+
"""Fetch node.reads['csv'] and parse it (RFC4180 defaults) into a Table.
|
|
25
|
+
|
|
26
|
+
The caller conforms the result to output_columns exactly as for a script
|
|
27
|
+
node, so declared types — not csv inference — decide the warehouse schema.
|
|
28
|
+
|
|
29
|
+
``output_columns`` names/types are also passed to ``read_csv`` itself as
|
|
30
|
+
its convert schema (via ``ConvertOptions.column_types``): pyarrow's default
|
|
31
|
+
type inference is otherwise the *first* place a value gets interpreted,
|
|
32
|
+
and it can destroy the very lexical value ``conform()`` is supposed to
|
|
33
|
+
preserve -- a VARCHAR column holding ``00123`` infers as int64 and
|
|
34
|
+
conform() writes back ``"123"``, and a NUMERIC column holding a valid
|
|
35
|
+
decimal like ``10.50`` infers as float64, which conform()'s own
|
|
36
|
+
lossy-cast guard then rejects outright. Reading every declared column
|
|
37
|
+
directly as its target Arrow type sidesteps both: the value is parsed
|
|
38
|
+
once, as the type it is actually declared to be.
|
|
39
|
+
"""
|
|
40
|
+
uri = parse_csv_uri(node.reads["csv"])
|
|
41
|
+
active_reader = reader if reader is not None else reader_for(uri)
|
|
42
|
+
column_types = {
|
|
43
|
+
col.name: arrow_type(parse_sql_type(col.type)) for col in node.output_columns
|
|
44
|
+
}
|
|
45
|
+
convert_options = pyarrow.csv.ConvertOptions(column_types=column_types)
|
|
46
|
+
try:
|
|
47
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
48
|
+
dest = active_reader.fetch(uri, Path(tmp) / "source.csv")
|
|
49
|
+
table = pyarrow.csv.read_csv(dest, convert_options=convert_options)
|
|
50
|
+
except LoadError:
|
|
51
|
+
raise
|
|
52
|
+
except Exception as exc:
|
|
53
|
+
raise LoadError(f"csv fetch failed for {node.relation}: {exc}") from exc
|
|
54
|
+
declared = {col.name for col in node.output_columns}
|
|
55
|
+
extras = set(table.column_names) - declared
|
|
56
|
+
if extras:
|
|
57
|
+
# Spec parity with the validation runner's csv_source header check
|
|
58
|
+
# (continuo_python_runtime/validation/runner.py): extra_columns: drop
|
|
59
|
+
# silently discards these at conform() time, so this structured
|
|
60
|
+
# warning is the only place the RUN path surfaces which columns were
|
|
61
|
+
# dropped.
|
|
62
|
+
logger.warning(
|
|
63
|
+
"csv_header_extra_columns node=%s columns=%s — columns present in the "
|
|
64
|
+
"csv but not declared in output_columns; they will not be loaded",
|
|
65
|
+
node.relation, sorted(extras))
|
|
66
|
+
logger.info("csv source %s: %d rows, columns=%s",
|
|
67
|
+
uri.raw, table.num_rows, table.column_names)
|
|
68
|
+
return table
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""continuo_python_runtime/csv_readers/__init__.py"""
|
|
2
|
+
from continuo_python_runtime.csv_source import CsvSourceReader, CsvUri
|
|
3
|
+
from continuo_python_runtime.csv_readers.https import HttpsCsvSourceReader
|
|
4
|
+
from continuo_python_runtime.csv_readers.s3 import S3CsvSourceReader
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def reader_for(uri: CsvUri) -> CsvSourceReader:
|
|
8
|
+
"""Composition edge: pick the adapter for the parsed scheme.
|
|
9
|
+
|
|
10
|
+
parse_csv_uri already constrains uri.scheme to "s3" or "https", but the
|
|
11
|
+
dispatch stays explicit (rather than an s3/else fallback) so a scheme
|
|
12
|
+
added to the parser without a matching adapter fails loudly here too."""
|
|
13
|
+
if uri.scheme == "s3":
|
|
14
|
+
return S3CsvSourceReader()
|
|
15
|
+
if uri.scheme == "https":
|
|
16
|
+
return HttpsCsvSourceReader()
|
|
17
|
+
raise ValueError(f"unsupported csv scheme: {uri.scheme!r}")
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""continuo_python_runtime/csv_readers/https.py"""
|
|
2
|
+
import shutil
|
|
3
|
+
import urllib.error
|
|
4
|
+
import urllib.request
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from continuo_python_runtime.csv_source import (
|
|
8
|
+
HEADER_PROBE_BYTES,
|
|
9
|
+
MAX_HEADER_BYTES,
|
|
10
|
+
CsvSourceReader,
|
|
11
|
+
CsvUri,
|
|
12
|
+
extract_header_line,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
# Both urlopen calls below must never hang forever: a source that accepts the
|
|
16
|
+
# TCP connection but stalls on headers or body would otherwise wedge a
|
|
17
|
+
# validation run or a scheduled node run indefinitely.
|
|
18
|
+
_TIMEOUT_SECONDS = 30
|
|
19
|
+
|
|
20
|
+
# Chunk size for the bounded read used when a server ignores our Range header
|
|
21
|
+
# and answers 200: reading in chunks this small lets fetch_header_line stop
|
|
22
|
+
# at the first newline (or MAX_HEADER_BYTES) without ever buffering a
|
|
23
|
+
# multi-gigabyte body just to inspect its first line.
|
|
24
|
+
_READ_CHUNK_BYTES = 65_536
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _HttpsOnlyRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
28
|
+
"""Refuses to follow a redirect whose target is not itself https://.
|
|
29
|
+
|
|
30
|
+
urllib follows redirects automatically, and by default does not care
|
|
31
|
+
what scheme the target uses -- an https:// source that redirects to
|
|
32
|
+
http:// (or any other scheme) would otherwise silently downgrade both
|
|
33
|
+
the header probe and the full fetch to plaintext, defeating
|
|
34
|
+
parse_csv_uri's https-only restriction.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: D102
|
|
38
|
+
if not newurl.lower().startswith("https://"):
|
|
39
|
+
raise urllib.error.HTTPError(
|
|
40
|
+
newurl, code,
|
|
41
|
+
f"refusing to follow redirect to non-https URL: {newurl}",
|
|
42
|
+
headers, fp,
|
|
43
|
+
)
|
|
44
|
+
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
_opener = urllib.request.build_opener(_HttpsOnlyRedirectHandler)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _read_bounded(resp, limit: int) -> bytes:
|
|
51
|
+
"""Read ``resp`` in small chunks, stopping at the first newline or once
|
|
52
|
+
more than ``limit`` bytes have been buffered, whichever comes first.
|
|
53
|
+
|
|
54
|
+
Used for the range-ignored (status 200) path, where the server's
|
|
55
|
+
response is the whole object: an unbounded ``resp.read()`` there would
|
|
56
|
+
buffer a multi-gigabyte body in full just to read its header line.
|
|
57
|
+
"""
|
|
58
|
+
buf = b""
|
|
59
|
+
while True:
|
|
60
|
+
chunk = resp.read(_READ_CHUNK_BYTES)
|
|
61
|
+
if not chunk:
|
|
62
|
+
return buf
|
|
63
|
+
buf += chunk
|
|
64
|
+
if b"\n" in buf or len(buf) > limit:
|
|
65
|
+
return buf
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class HttpsCsvSourceReader(CsvSourceReader):
|
|
69
|
+
"""Reads a csv source over HTTPS (public URLs; no auth in v1). Mirrors
|
|
70
|
+
S3CsvSourceReader's probe-and-extend strategy: each ranged request is
|
|
71
|
+
independent, so a server that ignores Range and answers 200 (not 206)
|
|
72
|
+
is handled too -- its response is the whole object, so it is treated
|
|
73
|
+
as terminal on the first pass regardless of whether it contains a
|
|
74
|
+
newline or how large it is, rather than being re-fetched and
|
|
75
|
+
re-appended pass after pass."""
|
|
76
|
+
|
|
77
|
+
def fetch_header_line(self, uri: CsvUri) -> str:
|
|
78
|
+
start = 0
|
|
79
|
+
buf = b""
|
|
80
|
+
while True:
|
|
81
|
+
end = start + HEADER_PROBE_BYTES - 1
|
|
82
|
+
req = urllib.request.Request(
|
|
83
|
+
uri.raw, headers={"Range": f"bytes={start}-{end}"})
|
|
84
|
+
with _opener.open(req, timeout=_TIMEOUT_SECONDS) as resp: # noqa: S310 — scheme gated by parse_csv_uri and _HttpsOnlyRedirectHandler
|
|
85
|
+
range_honoured = resp.status == 206
|
|
86
|
+
if range_honoured:
|
|
87
|
+
body = resp.read()
|
|
88
|
+
else:
|
|
89
|
+
# The server ignored our Range header and returned the
|
|
90
|
+
# entire object (status 200): bound the read itself so a
|
|
91
|
+
# multi-gigabyte body is never buffered in full, and
|
|
92
|
+
# this response is terminal regardless of its size --
|
|
93
|
+
# every retry would re-fetch the identical full body, so
|
|
94
|
+
# looping would only re-append it pass after pass and
|
|
95
|
+
# eventually trip a false MAX_HEADER_BYTES overflow.
|
|
96
|
+
body = _read_bounded(resp, MAX_HEADER_BYTES)
|
|
97
|
+
if not range_honoured:
|
|
98
|
+
return extract_header_line(body, uri.raw)
|
|
99
|
+
buf += body
|
|
100
|
+
if b"\n" in buf:
|
|
101
|
+
return extract_header_line(buf, uri.raw)
|
|
102
|
+
if len(body) < HEADER_PROBE_BYTES: # whole object read, no newline
|
|
103
|
+
return buf.rstrip(b"\r").decode("utf-8-sig")
|
|
104
|
+
if len(buf) > MAX_HEADER_BYTES:
|
|
105
|
+
raise ValueError(
|
|
106
|
+
f"csv header line exceeds {MAX_HEADER_BYTES} bytes: {uri.raw}")
|
|
107
|
+
start += HEADER_PROBE_BYTES
|
|
108
|
+
|
|
109
|
+
def fetch(self, uri: CsvUri, dest: Path) -> Path:
|
|
110
|
+
with (
|
|
111
|
+
_opener.open(uri.raw, timeout=_TIMEOUT_SECONDS) as resp, # noqa: S310 — scheme gated by parse_csv_uri and _HttpsOnlyRedirectHandler
|
|
112
|
+
open(dest, "wb") as f,
|
|
113
|
+
):
|
|
114
|
+
shutil.copyfileobj(resp, f)
|
|
115
|
+
return dest
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
assert issubclass(HttpsCsvSourceReader, CsvSourceReader)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""continuo_python_runtime/csv_readers/s3.py"""
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from botocore.exceptions import ClientError # type: ignore[import-untyped]
|
|
5
|
+
|
|
6
|
+
from continuo_python_runtime.csv_source import (
|
|
7
|
+
HEADER_PROBE_BYTES,
|
|
8
|
+
MAX_HEADER_BYTES,
|
|
9
|
+
CsvSourceReader,
|
|
10
|
+
CsvUri,
|
|
11
|
+
extract_header_line,
|
|
12
|
+
)
|
|
13
|
+
from continuo_python_runtime.validation.s3 import make_s3_client
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class S3CsvSourceReader(CsvSourceReader):
|
|
17
|
+
"""Reads a csv source from S3. Reuses make_s3_client so S3_ENDPOINT_URL
|
|
18
|
+
(minio, localstack) and boto3's own credential chain behave identically
|
|
19
|
+
to the validation runner's existing S3 access."""
|
|
20
|
+
|
|
21
|
+
def fetch_header_line(self, uri: CsvUri) -> str:
|
|
22
|
+
client = make_s3_client()
|
|
23
|
+
start = 0
|
|
24
|
+
buf = b""
|
|
25
|
+
while True:
|
|
26
|
+
end = start + HEADER_PROBE_BYTES - 1
|
|
27
|
+
try:
|
|
28
|
+
body = client.get_object(
|
|
29
|
+
Bucket=uri.bucket, Key=uri.key, Range=f"bytes={start}-{end}"
|
|
30
|
+
)["Body"].read()
|
|
31
|
+
except ClientError as exc:
|
|
32
|
+
if start == 0 and exc.response.get("Error", {}).get("Code") == "InvalidRange":
|
|
33
|
+
# A Range request on byte 0 is unsatisfiable only when the
|
|
34
|
+
# object itself is 0 bytes long: treat a 0-byte csv source
|
|
35
|
+
# as an empty header line rather than a hard failure --
|
|
36
|
+
# the caller (validation/runner.py) raises a clear error
|
|
37
|
+
# for an empty header line.
|
|
38
|
+
return ""
|
|
39
|
+
raise
|
|
40
|
+
buf += body
|
|
41
|
+
if b"\n" in buf:
|
|
42
|
+
return extract_header_line(buf, uri.raw)
|
|
43
|
+
if len(body) < HEADER_PROBE_BYTES: # whole object read, no newline
|
|
44
|
+
return buf.rstrip(b"\r").decode("utf-8-sig")
|
|
45
|
+
if len(buf) > MAX_HEADER_BYTES:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"csv header line exceeds {MAX_HEADER_BYTES} bytes: {uri.raw}")
|
|
48
|
+
start += HEADER_PROBE_BYTES
|
|
49
|
+
|
|
50
|
+
def fetch(self, uri: CsvUri, dest: Path) -> Path:
|
|
51
|
+
client = make_s3_client()
|
|
52
|
+
with open(dest, "wb") as f:
|
|
53
|
+
client.download_fileobj(uri.bucket, uri.key, f)
|
|
54
|
+
return dest
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
assert issubclass(S3CsvSourceReader, CsvSourceReader)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""CSV-source rules shared by the run harness and the validation runner:
|
|
2
|
+
the URI grammar, the header-conformance rule, and the reader port both
|
|
3
|
+
consumers depend on. Dependency-free — adapters that do I/O live in
|
|
4
|
+
continuo_python_runtime/csv_readers/ and implement CsvSourceReader.
|
|
5
|
+
"""
|
|
6
|
+
from abc import ABC, abstractmethod
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
HEADER_PROBE_BYTES = 65_536 # first ranged fetch when probing for the header line
|
|
11
|
+
MAX_HEADER_BYTES = 1_048_576 # a header line longer than this is a failure
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class CsvUri:
|
|
16
|
+
scheme: str # "s3" | "https"
|
|
17
|
+
raw: str
|
|
18
|
+
bucket: str = "" # s3 only
|
|
19
|
+
key: str = "" # s3 only
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def parse_csv_uri(uri: str) -> CsvUri:
|
|
23
|
+
"""Parse a csv source URI. Accepts exactly s3://bucket/key and https://...
|
|
24
|
+
|
|
25
|
+
Raises ValueError for anything else (http:// included), and for a
|
|
26
|
+
non-string ``uri`` (e.g. a contract's `reads: {csv: 123}`) rather than
|
|
27
|
+
letting `.startswith()` raise a bare AttributeError/TypeError -- callers
|
|
28
|
+
(the contract loader) catch ValueError to turn this into a ContractError,
|
|
29
|
+
so a malformed uri fails at lint/parse time, never at run time.
|
|
30
|
+
"""
|
|
31
|
+
if not isinstance(uri, str):
|
|
32
|
+
raise ValueError(f"csv uri must be a string, got {type(uri).__name__}: {uri!r}")
|
|
33
|
+
if uri.startswith("s3://"):
|
|
34
|
+
bucket, _, key = uri[len("s3://"):].partition("/")
|
|
35
|
+
if not bucket or not key:
|
|
36
|
+
raise ValueError(f"invalid s3 csv uri (missing bucket or key): {uri!r}")
|
|
37
|
+
return CsvUri(scheme="s3", raw=uri, bucket=bucket, key=key)
|
|
38
|
+
if uri.startswith("https://"):
|
|
39
|
+
remainder = uri[len("https://"):]
|
|
40
|
+
host, _, _ = remainder.partition("/")
|
|
41
|
+
if host:
|
|
42
|
+
return CsvUri(scheme="https", raw=uri)
|
|
43
|
+
raise ValueError(
|
|
44
|
+
f"invalid csv uri {uri!r}: must be s3://bucket/key or https://..."
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def extract_header_line(data: bytes, source_desc: str) -> str:
|
|
49
|
+
"""Return the first line of ``data`` (no trailing newline), decoded as
|
|
50
|
+
utf-8-sig so a UTF-8 byte-order mark on the CSV's first byte does not end
|
|
51
|
+
up prepended to the first column name.
|
|
52
|
+
|
|
53
|
+
Shared by every :class:`CsvSourceReader` adapter's ``fetch_header_line``:
|
|
54
|
+
the check must run on the *resolved line itself* (the bytes up to, and
|
|
55
|
+
including the absence of, the first newline), not merely on an
|
|
56
|
+
intermediate buffer length -- a newline that only arrives after the
|
|
57
|
+
accumulated buffer has already grown past ``MAX_HEADER_BYTES`` must still
|
|
58
|
+
be rejected as oversized, not returned as a successful (if enormous)
|
|
59
|
+
header line.
|
|
60
|
+
|
|
61
|
+
Raises:
|
|
62
|
+
ValueError: If the line exceeds ``MAX_HEADER_BYTES``.
|
|
63
|
+
"""
|
|
64
|
+
line = data.split(b"\n", 1)[0] if b"\n" in data else data
|
|
65
|
+
if len(line) > MAX_HEADER_BYTES:
|
|
66
|
+
raise ValueError(f"csv header line exceeds {MAX_HEADER_BYTES} bytes: {source_desc}")
|
|
67
|
+
return line.rstrip(b"\r").decode("utf-8-sig")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def check_header(header_cols: list[str], declared_cols: list[str]) -> set[str]:
|
|
71
|
+
"""Presence-only header conformance: every declared column must appear in
|
|
72
|
+
the CSV header, in any order. Returns the set of header columns NOT
|
|
73
|
+
declared (extras) so callers can surface them as a warning.
|
|
74
|
+
|
|
75
|
+
Raises ValueError naming every missing declared column.
|
|
76
|
+
"""
|
|
77
|
+
header = set(header_cols)
|
|
78
|
+
missing = [c for c in declared_cols if c not in header]
|
|
79
|
+
if missing:
|
|
80
|
+
raise ValueError(f"csv header missing declared column(s): {sorted(missing)}")
|
|
81
|
+
return header - set(declared_cols)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class CsvSourceReader(ABC):
|
|
85
|
+
"""Port for reading a csv source. Implemented by csv_readers adapters;
|
|
86
|
+
consumed by the run harness (full fetch) and the validation runner
|
|
87
|
+
(header line only). The dependency arrow runs adapter -> this port."""
|
|
88
|
+
|
|
89
|
+
@abstractmethod
|
|
90
|
+
def fetch_header_line(self, uri: CsvUri) -> str:
|
|
91
|
+
"""Return the CSV's first line (no trailing newline). Raises on an
|
|
92
|
+
unreachable source or a header longer than MAX_HEADER_BYTES."""
|
|
93
|
+
|
|
94
|
+
@abstractmethod
|
|
95
|
+
def fetch(self, uri: CsvUri, dest: Path) -> Path:
|
|
96
|
+
"""Stream the full object to dest and return dest."""
|
|
@@ -31,6 +31,8 @@ from continuo_python_runtime.context import RunContext
|
|
|
31
31
|
from continuo_python_runtime.contract.loader import load_contract_dir
|
|
32
32
|
from continuo_python_runtime.contract.model import Node
|
|
33
33
|
from continuo_python_runtime.contract.paths import resolve_script_path
|
|
34
|
+
from continuo_python_runtime.csv_loader import produce_csv
|
|
35
|
+
from continuo_python_runtime.csv_source import CsvSourceReader
|
|
34
36
|
from continuo_python_runtime.errors import ContractError, HarnessError, LoadError, ScriptError
|
|
35
37
|
|
|
36
38
|
logger = logging.getLogger("continuo_python_runtime.harness")
|
|
@@ -206,9 +208,16 @@ def _validate_config_early(adapter: Any, node: Node) -> None:
|
|
|
206
208
|
) from exc
|
|
207
209
|
|
|
208
210
|
|
|
209
|
-
def run_node(
|
|
211
|
+
def run_node(
|
|
212
|
+
env: Mapping[str, str], adapter: Any = None, reader: CsvSourceReader | None = None
|
|
213
|
+
) -> int:
|
|
210
214
|
"""Run a single node end-to-end and print exactly one sentinel result block.
|
|
211
215
|
|
|
216
|
+
``reader`` mirrors the ``adapter`` injection seam: when given, it is
|
|
217
|
+
threaded into :func:`produce_csv` for a python-csv node instead of
|
|
218
|
+
letting that function pick a reader via ``reader_for``. Ignored for a
|
|
219
|
+
python-model node.
|
|
220
|
+
|
|
212
221
|
Returns 0 on success, 1 on any :class:`HarnessError`.
|
|
213
222
|
"""
|
|
214
223
|
node_id = env.get("NODE_ID") or ""
|
|
@@ -246,13 +255,15 @@ def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
|
|
|
246
255
|
|
|
247
256
|
_validate_config_early(active_adapter, node)
|
|
248
257
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
258
|
+
if node.kind == "python-csv":
|
|
259
|
+
table = produce_csv(node, reader=reader)
|
|
260
|
+
else:
|
|
261
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
262
|
+
module = load_script(node, app_root)
|
|
263
|
+
ctx = RunContext(node, active_adapter)
|
|
264
|
+
raw_result = _execute_script(module, ctx)
|
|
265
|
+
table = to_arrow(raw_result)
|
|
254
266
|
|
|
255
|
-
table = to_arrow(raw_result)
|
|
256
267
|
conformed = conform(table, node.output_columns, node.extra_columns)
|
|
257
268
|
|
|
258
269
|
columns = [
|
|
@@ -8,10 +8,16 @@ Dispatches on ``VALIDATION_OP`` env var (default ``build_from_sql``):
|
|
|
8
8
|
- ``build_from_columns``: for python nodes, which have no SELECT to shape their output
|
|
9
9
|
from. Fetch the node's validation spec JSON from S3 (``CANDIDATE_SPEC_URI`` —
|
|
10
10
|
``{"reads": [sql, ...], "output_columns": [{"name","type","nullable"}, ...],
|
|
11
|
-
"config": {...}}``; ``config``
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
output
|
|
11
|
+
"config": {...}, "csv_source": "s3://..." | "https://..."}``; ``config`` and
|
|
12
|
+
``csv_source`` are both optional. When ``csv_source`` is set, its header row is
|
|
13
|
+
fetched (without downloading the full object) and checked against the declared
|
|
14
|
+
output columns: a declared column missing from the header fails the release gate,
|
|
15
|
+
while a header column not declared only logs a ``csv_header_extra_columns``
|
|
16
|
+
warning (it is silently dropped at load time). ``config`` defaults to ``{}``.
|
|
17
|
+
Every declared read is bind-checked against the candidate schema so an upstream
|
|
18
|
+
that dropped a column the script reads fails the release gate, then the output
|
|
19
|
+
table is materialized empty from the declared typed columns and the declared
|
|
20
|
+
physical layout.
|
|
15
21
|
|
|
16
22
|
The engine adapter is discovered from the single installed
|
|
17
23
|
``continuo_engine.adapters`` entry point — each runner image installs exactly one.
|
|
@@ -19,6 +25,7 @@ stdout is reserved exclusively for the runner's one structured ``result_block``,
|
|
|
19
25
|
printed as its last line; all diagnostics go to stderr via the ``logging`` module.
|
|
20
26
|
A non-zero exit marks the node failed.
|
|
21
27
|
"""
|
|
28
|
+
import csv
|
|
22
29
|
import json
|
|
23
30
|
import logging
|
|
24
31
|
import os
|
|
@@ -29,6 +36,8 @@ from continuo_engine_contract.port import ( # type: ignore[import-untyped]
|
|
|
29
36
|
AdapterDiscoveryError,
|
|
30
37
|
discover_adapter,
|
|
31
38
|
)
|
|
39
|
+
from continuo_python_runtime.csv_readers import reader_for
|
|
40
|
+
from continuo_python_runtime.csv_source import check_header, parse_csv_uri
|
|
32
41
|
from continuo_python_runtime.validation import s3
|
|
33
42
|
|
|
34
43
|
logger = logging.getLogger("validation_runner")
|
|
@@ -126,6 +135,7 @@ def main() -> None:
|
|
|
126
135
|
prod_schema = None
|
|
127
136
|
spec: dict | None = None
|
|
128
137
|
config: dict = {}
|
|
138
|
+
csv_source: str = ""
|
|
129
139
|
if op in _NODE_OPS:
|
|
130
140
|
table = _require("TABLE_NAME")
|
|
131
141
|
unique_id = _node_id() or f"model.{table}"
|
|
@@ -175,6 +185,16 @@ def main() -> None:
|
|
|
175
185
|
print(result.result_block("error", msg, unique_id=unique_id), flush=True)
|
|
176
186
|
sys.exit(2)
|
|
177
187
|
config = raw_config or {}
|
|
188
|
+
csv_source = spec.get("csv_source", "")
|
|
189
|
+
if "csv_source" in spec and not isinstance(csv_source, str):
|
|
190
|
+
# Checked by presence, not truthiness: `csv_source: 0` / `false` /
|
|
191
|
+
# `[]` / `{}` are all non-string values a malformed spec could
|
|
192
|
+
# carry, and a truthiness guard would silently skip both this
|
|
193
|
+
# type check and the header check below for every one of them.
|
|
194
|
+
msg = f"candidate spec 'csv_source' must be a string, got {type(csv_source).__name__}"
|
|
195
|
+
logger.error("%s", msg)
|
|
196
|
+
print(result.result_block("error", msg, unique_id=unique_id), flush=True)
|
|
197
|
+
sys.exit(2)
|
|
178
198
|
else:
|
|
179
199
|
prod_schema = _require("PROD_SCHEMA")
|
|
180
200
|
elif op in _SCHEMA_OPS:
|
|
@@ -220,6 +240,21 @@ def main() -> None:
|
|
|
220
240
|
adapter.build_empty_from_sql(schema, table, candidate_sql)
|
|
221
241
|
elif op == "build_from_columns":
|
|
222
242
|
assert spec is not None, "spec must be set for build_from_columns"
|
|
243
|
+
if csv_source:
|
|
244
|
+
csv_uri = parse_csv_uri(csv_source)
|
|
245
|
+
header_line = reader_for(csv_uri).fetch_header_line(csv_uri)
|
|
246
|
+
if not header_line:
|
|
247
|
+
raise ValueError(
|
|
248
|
+
"csv source has no header line (empty or unreadable): "
|
|
249
|
+
f"{csv_source}"
|
|
250
|
+
)
|
|
251
|
+
declared = [c["name"] for c in spec["output_columns"]]
|
|
252
|
+
extras = check_header(next(csv.reader([header_line])), declared)
|
|
253
|
+
if extras:
|
|
254
|
+
logger.warning(
|
|
255
|
+
"csv_header_extra_columns node=%s columns=%s — columns present in the "
|
|
256
|
+
"csv but not declared in output_columns; they will not be loaded",
|
|
257
|
+
unique_id, sorted(extras))
|
|
223
258
|
for read_sql in spec.get("reads", []):
|
|
224
259
|
adapter.check_binds(read_sql)
|
|
225
260
|
adapter.build_empty_from_columns(schema, table, spec["output_columns"], config)
|
{continuo_python_runtime-0.3.0.dist-info → continuo_python_runtime-0.4.0.dist-info}/METADATA
RENAMED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: continuo-python-runtime
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
Maintainer: Simone Carolini
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
License-File: NOTICE
|
|
7
10
|
Classifier: Development Status :: 4 - Beta
|
|
8
11
|
Classifier: Intended Audience :: Developers
|
|
9
12
|
Classifier: Programming Language :: Python :: 3.14
|
|
10
13
|
Requires-Python: >=3.14
|
|
11
14
|
Requires-Dist: boto3==1.43.59
|
|
12
|
-
Requires-Dist: continuo-engine-contract==0.7.
|
|
15
|
+
Requires-Dist: continuo-engine-contract==0.7.2
|
|
13
16
|
Requires-Dist: pyarrow==25.0.0
|
|
14
17
|
Requires-Dist: pyyaml==6.0.3
|
|
15
18
|
Requires-Dist: sqlglot==30.15.0
|
|
@@ -17,6 +20,10 @@ Description-Content-Type: text/markdown
|
|
|
17
20
|
|
|
18
21
|
# Continuo Python Runtime
|
|
19
22
|
|
|
23
|
+
[](https://github.com/carolsimone/continuo-python-runtime/actions/workflows/ci.yml)
|
|
24
|
+
[](LICENSE)
|
|
25
|
+
[](https://github.com/carolsimone/continuo-python-runtime/releases)
|
|
26
|
+
|
|
20
27
|
Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
21
28
|
This repo is what domain data teams (marketing, finance, …) template from to
|
|
22
29
|
ship a python node into Continuo: write a contract + a `run(ctx)` script,
|
|
@@ -107,7 +114,9 @@ the Go parser has not been taught is a production outage, not a refactor.
|
|
|
107
114
|
login` step to `release.yml`.
|
|
108
115
|
5. Write a contract file under `contracts/` (see
|
|
109
116
|
`template/contracts/example.yml`) and a script under `scripts/` that
|
|
110
|
-
implements `run(ctx)` (see `template/scripts/example.py`).
|
|
117
|
+
implements `run(ctx)` (see `template/scripts/example.py`). A node that
|
|
118
|
+
only needs to land a csv file needs no script at all — see
|
|
119
|
+
`template/contracts/example_csv.yml` and "Node kinds" below.
|
|
111
120
|
6. Push to `main`. The `release.yml` workflow lints the scripts, validates
|
|
112
121
|
and merges the contracts, builds and pushes the image, uploads the merged
|
|
113
122
|
contract to S3, and POSTs the release.
|
|
@@ -134,6 +143,30 @@ The runtime image does not re-run this gate, so a read that passes here is
|
|
|
134
143
|
not re-judged under a different grammar in production. See
|
|
135
144
|
`docs/boundary-contract.md` §13.1.
|
|
136
145
|
|
|
146
|
+
## Node kinds
|
|
147
|
+
|
|
148
|
+
A contract node's `kind:` field selects how the node produces its rows.
|
|
149
|
+
Every rule below (`extra_columns`, `output_columns`, "Conform rules") applies
|
|
150
|
+
to both kinds identically — `kind` only changes how the pre-conform table is
|
|
151
|
+
produced, never how it is checked or written.
|
|
152
|
+
|
|
153
|
+
- **`python-model`** (the default; the field may be omitted) — a script node.
|
|
154
|
+
It requires `script:` and a `reads:` map of one or more named SQL queries,
|
|
155
|
+
as described in "The script API" below.
|
|
156
|
+
- **`python-csv`** — a contract-only node: it has no script and its `reads:`
|
|
157
|
+
map must be exactly `{csv: <uri>}`, where the uri is `s3://bucket/key` or
|
|
158
|
+
an `https://` url (`http://` is rejected at validate time, not run time).
|
|
159
|
+
The harness fetches the file, parses it with RFC 4180 defaults, and feeds
|
|
160
|
+
the result straight into `conform()` — declared `output_columns` types
|
|
161
|
+
decide the warehouse schema, not whatever pyarrow infers from the csv.
|
|
162
|
+
Because there is no script, `script:` is a forbidden key for this kind;
|
|
163
|
+
`continuo-runtime validate`/`merge`/`lint` reject one that sets it. The
|
|
164
|
+
csv's header row must contain every declared output column (checked again,
|
|
165
|
+
independently, at release time before promotion); columns present in the
|
|
166
|
+
header but not declared are governed by the same `extra_columns` policy as
|
|
167
|
+
a script node's output — `raise` (default) fails the run, `warn` drops
|
|
168
|
+
them and logs a warning. See `template/contracts/example_csv.yml`.
|
|
169
|
+
|
|
137
170
|
## The script API
|
|
138
171
|
|
|
139
172
|
A node script is a Python file with exactly one required entry point:
|
|
@@ -229,9 +262,9 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
229
262
|
`FROM`:
|
|
230
263
|
|
|
231
264
|
```dockerfile
|
|
232
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.
|
|
265
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.4.0
|
|
233
266
|
# or
|
|
234
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.
|
|
267
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.4.0
|
|
235
268
|
```
|
|
236
269
|
|
|
237
270
|
The engine is part of the image **name**; the tag is the bare version, so
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
|
|
2
|
+
continuo_python_runtime/cli.py,sha256=X4JBC4FDpgFOeDdtBbeX9GTq1XTuJCghDpemlR_CPvE,5925
|
|
3
|
+
continuo_python_runtime/closure.py,sha256=6e-IAQp8Lieb0bGKP1F6JE4TgVBJ_COTGXRUult2xeI,12991
|
|
4
|
+
continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
|
|
5
|
+
continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7DLZfVg,1856
|
|
6
|
+
continuo_python_runtime/csv_loader.py,sha256=a5Hwu-YIzIiko8X1_vqShsnzjs2-OsKEUSCHLXg_dio,3346
|
|
7
|
+
continuo_python_runtime/csv_source.py,sha256=7g8Tu6ieiJLvWH4Ep4nwYlOdGkjeePr12K5UHuO8TNc,4136
|
|
8
|
+
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
9
|
+
continuo_python_runtime/harness.py,sha256=BBJBmmiQ922WVmPi0Xh7CFnXjNxKefPey0YOEHercjs,12634
|
|
10
|
+
continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
|
|
11
|
+
continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
|
|
12
|
+
continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
|
|
13
|
+
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
14
|
+
continuo_python_runtime/contract/loader.py,sha256=oUer1sbaxhUMNQVUuqO5j5vbQCrgwuHOWJzGe0MHSLg,16973
|
|
15
|
+
continuo_python_runtime/contract/merge.py,sha256=36awQV8cHGcQJv6XJGC_mC5MuyRS5-7XTVTLw0sjMgE,6903
|
|
16
|
+
continuo_python_runtime/contract/model.py,sha256=L7GwSYKrh6Z0J1c1zWFHQw8GzYUUzzWycuBw5Qq5Dfg,1017
|
|
17
|
+
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
18
|
+
continuo_python_runtime/csv_readers/__init__.py,sha256=9TKvjC37C77uRaJQZuO0--zVj2OvuNwpJE6nWx7j2S8,806
|
|
19
|
+
continuo_python_runtime/csv_readers/https.py,sha256=tMJ9k5F0b9SAydpon0ZlOznfjvjhF2BBLltyVmvRneE,4998
|
|
20
|
+
continuo_python_runtime/csv_readers/s3.py,sha256=zmvHs3TxlN2alPKrdaHzPyREmkve2nBiAsHRXXe1mE0,2246
|
|
21
|
+
continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
|
|
22
|
+
continuo_python_runtime/validation/runner.py,sha256=AwYbiFrQubxSm1vfxJJpnp7xfekviUW2anw4LRQQ4Rc,13062
|
|
23
|
+
continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
|
|
24
|
+
continuo_python_runtime-0.4.0.dist-info/METADATA,sha256=JtdvcXJZtH27JKMGd5m1Rm6wvch7MzzrsvvAawOQkCU,17023
|
|
25
|
+
continuo_python_runtime-0.4.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
26
|
+
continuo_python_runtime-0.4.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
27
|
+
continuo_python_runtime-0.4.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
|
|
28
|
+
continuo_python_runtime-0.4.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
|
|
29
|
+
continuo_python_runtime-0.4.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the
|
|
44
|
+
purposes of this License, Derivative Works shall not include works
|
|
45
|
+
that remain separable from, or merely link (or bind by name) to the
|
|
46
|
+
interfaces of, the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2026 Simone Carolini
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Continuo Python Runtime
|
|
2
|
+
Copyright 2026 Simone Carolini
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
5
|
+
this product except in compliance with the License. You may obtain a copy of the
|
|
6
|
+
License in the LICENSE file at the root of this repository, or at:
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
This product includes third-party software. See `uv.lock` for the full
|
|
11
|
+
dependency inventory and each dependency's declared license.
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
continuo_python_runtime/__init__.py,sha256=otNaXEPsTtRXJV5FLtDUsFO8VD4OC-MLAzwhtimHfOE,70
|
|
2
|
-
continuo_python_runtime/cli.py,sha256=X4JBC4FDpgFOeDdtBbeX9GTq1XTuJCghDpemlR_CPvE,5925
|
|
3
|
-
continuo_python_runtime/closure.py,sha256=6e-IAQp8Lieb0bGKP1F6JE4TgVBJ_COTGXRUult2xeI,12991
|
|
4
|
-
continuo_python_runtime/conform.py,sha256=a9CKCA-gT9Zerzr6c5H8aByTFOUd6uBMo02sNKSDnXg,9199
|
|
5
|
-
continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7DLZfVg,1856
|
|
6
|
-
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
7
|
-
continuo_python_runtime/harness.py,sha256=1b_ze2_AJ9vncY-yj88TgJylVzKwGw1-idxHoaYAymE,12101
|
|
8
|
-
continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
|
|
9
|
-
continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
|
|
10
|
-
continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
|
|
11
|
-
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
12
|
-
continuo_python_runtime/contract/loader.py,sha256=WeKngAe6VH-WxgRTVdCERPMdsGBszJM9RkanUTTs9-w,15611
|
|
13
|
-
continuo_python_runtime/contract/merge.py,sha256=XCCeE2vJUm0KA60d2XsHpjRh5fIdVfFu_HsBzEyUWow,6505
|
|
14
|
-
continuo_python_runtime/contract/model.py,sha256=2hKI04M0WwvTkOy9eDGl2dRZgY3bAuJYktAG-ICI-AU,936
|
|
15
|
-
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
16
|
-
continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
|
|
17
|
-
continuo_python_runtime/validation/runner.py,sha256=zHs5CR3Wr9512sh6eY9hpN8NG3weJPYUFRDPJf2BNnc,10875
|
|
18
|
-
continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
|
|
19
|
-
continuo_python_runtime-0.3.0.dist-info/METADATA,sha256=SjKFljTkhZUCsswm3WWkstA2eOtrfdRonXKEJA4Sf-Q,14898
|
|
20
|
-
continuo_python_runtime-0.3.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
21
|
-
continuo_python_runtime-0.3.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
22
|
-
continuo_python_runtime-0.3.0.dist-info/RECORD,,
|
|
File without changes
|
{continuo_python_runtime-0.3.0.dist-info → continuo_python_runtime-0.4.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|