continuo-python-runtime 0.5.0__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/contract/kinds.py +124 -0
- continuo_python_runtime/contract/loader.py +25 -63
- continuo_python_runtime/contract/merge.py +33 -14
- continuo_python_runtime/contract/model.py +4 -2
- continuo_python_runtime/contract/secret_ref.py +31 -0
- continuo_python_runtime/harness.py +29 -14
- {continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/METADATA +10 -4
- {continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/RECORD +12 -10
- {continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/WHEEL +1 -1
- {continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/entry_points.txt +0 -0
- {continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/licenses/LICENSE +0 -0
- {continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Per-kind contract rules: which fields a node of each kind must carry.
|
|
2
|
+
|
|
3
|
+
Each kind is one ``KindRules`` entry composed of a script policy, a reads rule,
|
|
4
|
+
and whether it may name a Secret. Adding a kind means adding one entry to
|
|
5
|
+
``RULES``; nothing that parses a contract branches on the kind's name.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from continuo_engine_contract.sql import ensure_single_read # type: ignore[import-untyped]
|
|
15
|
+
from sqlglot.errors import TokenError
|
|
16
|
+
|
|
17
|
+
from continuo_python_runtime.contract.model import KINDS
|
|
18
|
+
from continuo_python_runtime.csv_source import parse_csv_uri
|
|
19
|
+
from continuo_python_runtime.errors import ContractError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ReadsRule(ABC):
|
|
23
|
+
"""Validates a node's ``reads`` mapping and returns it unchanged."""
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def validate(
|
|
27
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
28
|
+
) -> dict[str, str]: ...
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SqlReads(ReadsRule):
|
|
32
|
+
"""A non-empty mapping of read name -> single SELECT."""
|
|
33
|
+
|
|
34
|
+
def validate(
|
|
35
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
36
|
+
) -> dict[str, str]:
|
|
37
|
+
if not isinstance(reads, dict) or not reads:
|
|
38
|
+
raise ContractError(
|
|
39
|
+
f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
|
|
40
|
+
)
|
|
41
|
+
for name, sql in reads.items():
|
|
42
|
+
if not isinstance(name, str) or not name.strip():
|
|
43
|
+
raise ContractError(
|
|
44
|
+
f"{label}: 'reads' name {name!r} must be a non-empty string"
|
|
45
|
+
)
|
|
46
|
+
if not isinstance(sql, str) or not sql.strip():
|
|
47
|
+
raise ContractError(
|
|
48
|
+
f"{label}: 'reads.{name}' must be a non-empty SQL string"
|
|
49
|
+
)
|
|
50
|
+
if not check_reads:
|
|
51
|
+
continue
|
|
52
|
+
try:
|
|
53
|
+
ensure_single_read(sql, dialect)
|
|
54
|
+
except (ValueError, TokenError) as exc:
|
|
55
|
+
# ensure_single_read's own message is phrased for check_binds
|
|
56
|
+
# (its only other caller today), so it's wrapped rather than
|
|
57
|
+
# surfaced bare here. TokenError is also caught: an unterminated
|
|
58
|
+
# string literal or comment fails sqlglot's tokenizer with a
|
|
59
|
+
# TokenError, a SqlglotError sibling of ParseError and not a
|
|
60
|
+
# subclass of ValueError -- despite ensure_single_read's
|
|
61
|
+
# docstring promising every rejection is a ValueError. Only
|
|
62
|
+
# TokenError, not the broader SqlglotError, is caught here: by
|
|
63
|
+
# the time control reaches this point `dialect` has already
|
|
64
|
+
# been validated once in load_contract_dir, so any other
|
|
65
|
+
# SqlglotError a future sqlglot version might raise from this
|
|
66
|
+
# call should surface as itself, not get relabeled as a
|
|
67
|
+
# rejected read.
|
|
68
|
+
raise ContractError(
|
|
69
|
+
f"{label}: 'reads.{name}' must be a single read query ({exc})"
|
|
70
|
+
) from exc
|
|
71
|
+
return dict(reads)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class CsvRead(ReadsRule):
|
|
75
|
+
"""Exactly ``{csv: <uri>}``, the uri in the grammar ``parse_csv_uri`` accepts."""
|
|
76
|
+
|
|
77
|
+
def validate(
|
|
78
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
79
|
+
) -> dict[str, str]:
|
|
80
|
+
if not isinstance(reads, dict) or set(reads) != {"csv"}:
|
|
81
|
+
raise ContractError(
|
|
82
|
+
f"{label}: a python-csv node's 'reads' must be exactly {{csv: <uri>}}"
|
|
83
|
+
)
|
|
84
|
+
try:
|
|
85
|
+
parse_csv_uri(reads["csv"])
|
|
86
|
+
except (ValueError, TypeError) as exc:
|
|
87
|
+
raise ContractError(f"{label}: invalid csv uri: {exc}") from exc
|
|
88
|
+
return dict(reads)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class NoReads(ReadsRule):
|
|
92
|
+
"""No declared reads: the node's script fetches its own data."""
|
|
93
|
+
|
|
94
|
+
def validate(
|
|
95
|
+
self, reads: Any, label: str, *, dialect: str | None, check_reads: bool
|
|
96
|
+
) -> dict[str, str]:
|
|
97
|
+
if reads is None or (isinstance(reads, dict) and not reads):
|
|
98
|
+
return {}
|
|
99
|
+
raise ContractError(
|
|
100
|
+
f"{label}: a python-api node declares no 'reads' (its script fetches its own data)"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class KindRules:
|
|
106
|
+
"""What a node of one kind must declare."""
|
|
107
|
+
|
|
108
|
+
script_required: bool
|
|
109
|
+
reads: ReadsRule
|
|
110
|
+
secret_allowed: bool = False
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
RULES: dict[str, KindRules] = {
|
|
114
|
+
"python-node": KindRules(script_required=True, reads=SqlReads()),
|
|
115
|
+
"python-csv": KindRules(script_required=False, reads=CsvRead()),
|
|
116
|
+
"python-api": KindRules(script_required=True, reads=NoReads(), secret_allowed=True),
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def rules_for(kind: object, label: str) -> KindRules:
|
|
121
|
+
"""Return the rules for ``kind``; raise ``ContractError`` for any other value."""
|
|
122
|
+
if isinstance(kind, str) and kind in RULES:
|
|
123
|
+
return RULES[kind]
|
|
124
|
+
raise ContractError(f"{label}: 'kind' must be one of {sorted(KINDS)}, got {kind!r}")
|
|
@@ -11,18 +11,17 @@ from pathlib import Path
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
|
|
13
13
|
import yaml
|
|
14
|
-
from continuo_engine_contract.sql import ensure_single_read # type: ignore[import-untyped]
|
|
15
14
|
from sqlglot import Dialect
|
|
16
|
-
from sqlglot.errors import TokenError
|
|
17
15
|
|
|
16
|
+
from continuo_python_runtime.contract.kinds import rules_for
|
|
18
17
|
from continuo_python_runtime.contract.model import (
|
|
19
18
|
CRITICALITIES,
|
|
19
|
+
DEFAULT_KIND,
|
|
20
20
|
EXTRA_COLUMNS_POLICIES,
|
|
21
|
-
KINDS,
|
|
22
21
|
Column,
|
|
23
22
|
Node,
|
|
24
23
|
)
|
|
25
|
-
from continuo_python_runtime.
|
|
24
|
+
from continuo_python_runtime.contract.secret_ref import validate_secret_ref
|
|
26
25
|
from continuo_python_runtime.errors import ContractError
|
|
27
26
|
from continuo_python_runtime.types import parse_sql_type
|
|
28
27
|
|
|
@@ -40,6 +39,7 @@ _ALLOWED_KEYS = {
|
|
|
40
39
|
"output_columns",
|
|
41
40
|
"config",
|
|
42
41
|
"content_hash",
|
|
42
|
+
"secret_ref",
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
_REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule")
|
|
@@ -173,11 +173,8 @@ def parse_node(
|
|
|
173
173
|
if unknown:
|
|
174
174
|
raise ContractError(f"{label}: unknown key(s) {sorted(unknown)}")
|
|
175
175
|
|
|
176
|
-
kind = raw.get("kind",
|
|
177
|
-
|
|
178
|
-
raise ContractError(
|
|
179
|
-
f"{label}: 'kind' must be one of {sorted(KINDS)}, got {kind!r}"
|
|
180
|
-
)
|
|
176
|
+
kind = raw.get("kind", DEFAULT_KIND)
|
|
177
|
+
rules = rules_for(kind, label)
|
|
181
178
|
|
|
182
179
|
for field in _REQUIRED_STRING_FIELDS:
|
|
183
180
|
value = raw.get(field)
|
|
@@ -191,20 +188,20 @@ def parse_node(
|
|
|
191
188
|
owner = raw["owner"]
|
|
192
189
|
schedule = raw["schedule"]
|
|
193
190
|
|
|
194
|
-
if
|
|
195
|
-
if "script" in raw:
|
|
196
|
-
raise ContractError(
|
|
197
|
-
f"{label}: 'script' is forbidden for kind python-csv "
|
|
198
|
-
"(csv nodes are contract-only)"
|
|
199
|
-
)
|
|
200
|
-
script = ""
|
|
201
|
-
else:
|
|
191
|
+
if rules.script_required:
|
|
202
192
|
raw_script = raw.get("script")
|
|
203
193
|
if not isinstance(raw_script, str) or not raw_script.strip():
|
|
204
194
|
raise ContractError(
|
|
205
195
|
f"{label}: required field 'script' must be a non-empty string"
|
|
206
196
|
)
|
|
207
197
|
script = raw_script
|
|
198
|
+
else:
|
|
199
|
+
if "script" in raw:
|
|
200
|
+
raise ContractError(
|
|
201
|
+
f"{label}: 'script' is forbidden for kind {kind} "
|
|
202
|
+
"(it is a contract-only kind)"
|
|
203
|
+
)
|
|
204
|
+
script = ""
|
|
208
205
|
|
|
209
206
|
criticality = raw.get("criticality")
|
|
210
207
|
if not isinstance(criticality, str) or criticality not in CRITICALITIES:
|
|
@@ -222,51 +219,15 @@ def parse_node(
|
|
|
222
219
|
f"got {extra_columns!r}"
|
|
223
220
|
)
|
|
224
221
|
|
|
225
|
-
reads =
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
raise ContractError(f"{label}: invalid csv uri: {exc}") from exc
|
|
235
|
-
else:
|
|
236
|
-
if not isinstance(reads, dict) or not reads:
|
|
237
|
-
raise ContractError(
|
|
238
|
-
f"{label}: 'reads' must be a non-empty mapping of name -> SQL"
|
|
239
|
-
)
|
|
240
|
-
for name, sql in reads.items():
|
|
241
|
-
if not isinstance(name, str) or not name.strip():
|
|
242
|
-
raise ContractError(
|
|
243
|
-
f"{label}: 'reads' name {name!r} must be a non-empty string"
|
|
244
|
-
)
|
|
245
|
-
if not isinstance(sql, str) or not sql.strip():
|
|
246
|
-
raise ContractError(
|
|
247
|
-
f"{label}: 'reads.{name}' must be a non-empty SQL string"
|
|
248
|
-
)
|
|
249
|
-
if not check_reads:
|
|
250
|
-
continue
|
|
251
|
-
try:
|
|
252
|
-
ensure_single_read(sql, dialect)
|
|
253
|
-
except (ValueError, TokenError) as exc:
|
|
254
|
-
# ensure_single_read's own message is phrased for check_binds
|
|
255
|
-
# (its only other caller today), so it's wrapped rather than
|
|
256
|
-
# surfaced bare here. TokenError is also caught: an unterminated
|
|
257
|
-
# string literal or comment fails sqlglot's tokenizer with a
|
|
258
|
-
# TokenError, a SqlglotError sibling of ParseError and not a
|
|
259
|
-
# subclass of ValueError -- despite ensure_single_read's
|
|
260
|
-
# docstring promising every rejection is a ValueError. Only
|
|
261
|
-
# TokenError, not the broader SqlglotError, is caught here: by
|
|
262
|
-
# the time control reaches this point `dialect` has already
|
|
263
|
-
# been validated once in load_contract_dir, so any other
|
|
264
|
-
# SqlglotError a future sqlglot version might raise from this
|
|
265
|
-
# call should surface as itself, not get relabeled as a
|
|
266
|
-
# rejected read.
|
|
267
|
-
raise ContractError(
|
|
268
|
-
f"{label}: 'reads.{name}' must be a single read query ({exc})"
|
|
269
|
-
) from exc
|
|
222
|
+
reads = rules.reads.validate(
|
|
223
|
+
raw.get("reads"), label, dialect=dialect, check_reads=check_reads
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
secret_ref = ""
|
|
227
|
+
if "secret_ref" in raw:
|
|
228
|
+
if not rules.secret_allowed:
|
|
229
|
+
raise ContractError(f"{label}: 'secret_ref' is not allowed for kind {kind}")
|
|
230
|
+
secret_ref = validate_secret_ref(raw["secret_ref"], label)
|
|
270
231
|
|
|
271
232
|
raw_columns = raw.get("output_columns")
|
|
272
233
|
if not isinstance(raw_columns, list) or not raw_columns:
|
|
@@ -325,13 +286,14 @@ def parse_node(
|
|
|
325
286
|
schedule=schedule,
|
|
326
287
|
criticality=criticality,
|
|
327
288
|
script=script,
|
|
328
|
-
reads=
|
|
289
|
+
reads=reads,
|
|
329
290
|
output_columns=tuple(columns),
|
|
330
291
|
description=description,
|
|
331
292
|
extra_columns=extra_columns,
|
|
332
293
|
config=config,
|
|
333
294
|
content_hash=content_hash,
|
|
334
295
|
kind=kind,
|
|
296
|
+
secret_ref=secret_ref,
|
|
335
297
|
)
|
|
336
298
|
|
|
337
299
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"""Contract v1 merger: wire contract builder."""
|
|
2
2
|
|
|
3
|
+
from collections.abc import Callable
|
|
3
4
|
from pathlib import Path
|
|
4
5
|
|
|
5
6
|
import yaml
|
|
@@ -21,7 +22,7 @@ def node_entry(node: Node) -> dict:
|
|
|
21
22
|
four (source_hash, shared_code_hash, config_hash, content_hash). The
|
|
22
23
|
nullable field is always present in output_columns.
|
|
23
24
|
"""
|
|
24
|
-
|
|
25
|
+
entry = {
|
|
25
26
|
"schema": node.schema,
|
|
26
27
|
"table": node.table,
|
|
27
28
|
"owner": node.owner,
|
|
@@ -42,6 +43,11 @@ def node_entry(node: Node) -> dict:
|
|
|
42
43
|
"extra_columns": node.extra_columns,
|
|
43
44
|
"config": dict(node.config),
|
|
44
45
|
}
|
|
46
|
+
# Only a kind that may name a Secret ever carries one; emitting the key
|
|
47
|
+
# only when set keeps every other node's config_hash unchanged.
|
|
48
|
+
if node.secret_ref:
|
|
49
|
+
entry["secret_ref"] = node.secret_ref
|
|
50
|
+
return entry
|
|
45
51
|
|
|
46
52
|
|
|
47
53
|
def _lint_node_closure(
|
|
@@ -110,6 +116,30 @@ def _lint_node_closure(
|
|
|
110
116
|
)
|
|
111
117
|
|
|
112
118
|
|
|
119
|
+
def _script_hash_inputs(node: Node, repo_root: Path) -> tuple[bytes, list[bytes]]:
|
|
120
|
+
"""A script node's source is its script plus its in-repo import closure,
|
|
121
|
+
every file of which is held to the closure lint."""
|
|
122
|
+
script_path = resolve_script_path(node.script, repo_root, context=node.relation)
|
|
123
|
+
script_bytes = script_path.read_bytes()
|
|
124
|
+
closure = resolve_closure(script_path, repo_root)
|
|
125
|
+
member_bytes = [member.read_bytes() for member in closure]
|
|
126
|
+
_lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
|
|
127
|
+
return script_bytes, member_bytes
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _csv_hash_inputs(node: Node, repo_root: Path) -> tuple[bytes, list[bytes]]:
|
|
131
|
+
"""A csv node has no script and no import closure: its source is the uri.
|
|
132
|
+
New file content at the same uri is new data, not a new node version."""
|
|
133
|
+
return node.reads["csv"].encode(), []
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
_HASH_INPUTS: dict[str, Callable[[Node, Path], tuple[bytes, list[bytes]]]] = {
|
|
137
|
+
"python-node": _script_hash_inputs,
|
|
138
|
+
"python-csv": _csv_hash_inputs,
|
|
139
|
+
"python-api": _script_hash_inputs,
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
113
143
|
def build_wire_contract(
|
|
114
144
|
contract_dir: Path, repo_root: Path, service: str, *, dialect: str | None = None
|
|
115
145
|
) -> dict:
|
|
@@ -129,19 +159,8 @@ def build_wire_contract(
|
|
|
129
159
|
for node in nodes:
|
|
130
160
|
entry = node_entry(node)
|
|
131
161
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
# the uri -- new file content at the same uri is new data, not a
|
|
135
|
-
# new node version.
|
|
136
|
-
uri_bytes = node.reads["csv"].encode()
|
|
137
|
-
entry.update(hash_parts(entry, uri_bytes, []))
|
|
138
|
-
else:
|
|
139
|
-
script_path = resolve_script_path(node.script, repo_root, context=node.relation)
|
|
140
|
-
script_bytes = script_path.read_bytes()
|
|
141
|
-
closure = resolve_closure(script_path, repo_root)
|
|
142
|
-
member_bytes = [member.read_bytes() for member in closure]
|
|
143
|
-
_lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
|
|
144
|
-
entry.update(hash_parts(entry, script_bytes, member_bytes))
|
|
162
|
+
primary_bytes, member_bytes = _HASH_INPUTS[node.kind](node, repo_root)
|
|
163
|
+
entry.update(hash_parts(entry, primary_bytes, member_bytes))
|
|
145
164
|
|
|
146
165
|
wire_nodes.append(entry)
|
|
147
166
|
|
|
@@ -6,7 +6,8 @@ from typing import Any
|
|
|
6
6
|
# Module-level constants
|
|
7
7
|
CRITICALITIES = frozenset({"REGULATORY", "CORE", "SECONDARY"})
|
|
8
8
|
EXTRA_COLUMNS_POLICIES = frozenset({"raise", "warn"})
|
|
9
|
-
KINDS = frozenset({"python-
|
|
9
|
+
KINDS = frozenset({"python-node", "python-csv", "python-api"})
|
|
10
|
+
DEFAULT_KIND = "python-node"
|
|
10
11
|
CONTRACT_VERSION = 1
|
|
11
12
|
|
|
12
13
|
|
|
@@ -35,7 +36,8 @@ class Node:
|
|
|
35
36
|
extra_columns: str = "raise"
|
|
36
37
|
config: dict[str, Any] = field(default_factory=dict)
|
|
37
38
|
content_hash: str | None = None
|
|
38
|
-
kind: str =
|
|
39
|
+
kind: str = DEFAULT_KIND
|
|
40
|
+
secret_ref: str = ""
|
|
39
41
|
|
|
40
42
|
@property
|
|
41
43
|
def relation(self) -> str:
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""The name of the Kubernetes Secret a node's pod receives as env vars.
|
|
2
|
+
|
|
3
|
+
The ``continuo-api-`` prefix keeps a contract from naming any other Secret in
|
|
4
|
+
the namespace (the warehouse credentials, the platform's own credentials).
|
|
5
|
+
continuo's topology-controller and execution-controller enforce the same
|
|
6
|
+
pattern; this check gives the author the error in CI instead.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
|
|
13
|
+
from continuo_python_runtime.errors import ContractError
|
|
14
|
+
|
|
15
|
+
SECRET_REF_PATTERN = re.compile(r"^continuo-api-[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
|
|
16
|
+
SECRET_REF_MAX_LEN = 253
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def validate_secret_ref(value: object, label: str) -> str:
|
|
20
|
+
"""Return ``value`` if it is a valid ``continuo-api-*`` Secret name."""
|
|
21
|
+
if (
|
|
22
|
+
not isinstance(value, str)
|
|
23
|
+
or len(value) > SECRET_REF_MAX_LEN
|
|
24
|
+
or not SECRET_REF_PATTERN.fullmatch(value)
|
|
25
|
+
):
|
|
26
|
+
raise ContractError(
|
|
27
|
+
f"{label}: 'secret_ref' must be a Kubernetes Secret name starting with "
|
|
28
|
+
f"'continuo-api-' (lowercase letters, digits, '-'; at most "
|
|
29
|
+
f"{SECRET_REF_MAX_LEN} chars), got {value!r}"
|
|
30
|
+
)
|
|
31
|
+
return value
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Container entrypoint: dispatches a single node's script and writes its output.
|
|
2
2
|
|
|
3
|
-
``run_node`` is the sole write sink for a python
|
|
3
|
+
``run_node`` is the sole write sink for a python node. It resolves the
|
|
4
4
|
node from the contract, loads and executes its script inside a
|
|
5
5
|
:class:`~continuo_python_runtime.context.RunContext`, conforms the result to
|
|
6
6
|
the declared schema, and writes it through the runtime adapter. Exactly one
|
|
@@ -16,7 +16,7 @@ import logging
|
|
|
16
16
|
import os
|
|
17
17
|
import sys
|
|
18
18
|
import uuid
|
|
19
|
-
from collections.abc import Mapping
|
|
19
|
+
from collections.abc import Callable, Mapping
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
from types import ModuleType
|
|
22
22
|
from typing import Any
|
|
@@ -208,15 +208,37 @@ def _validate_config_early(adapter: Any, node: Node) -> None:
|
|
|
208
208
|
) from exc
|
|
209
209
|
|
|
210
210
|
|
|
211
|
+
def _produce_from_script(
|
|
212
|
+
node: Node, adapter: Any, app_root: Path, reader: CsvSourceReader | None
|
|
213
|
+
) -> Any:
|
|
214
|
+
"""Import the node's script and return its ``run(ctx)`` result as Arrow."""
|
|
215
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
216
|
+
module = load_script(node, app_root)
|
|
217
|
+
ctx = RunContext(node, adapter)
|
|
218
|
+
return to_arrow(_execute_script(module, ctx))
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _produce_from_csv(
|
|
222
|
+
node: Node, adapter: Any, app_root: Path, reader: CsvSourceReader | None
|
|
223
|
+
) -> Any:
|
|
224
|
+
"""Load the node's declared csv source."""
|
|
225
|
+
return produce_csv(node, reader=reader)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
_PRODUCERS: dict[str, Callable[[Node, Any, Path, CsvSourceReader | None], Any]] = {
|
|
229
|
+
"python-node": _produce_from_script,
|
|
230
|
+
"python-api": _produce_from_script,
|
|
231
|
+
"python-csv": _produce_from_csv,
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
|
|
211
235
|
def run_node(
|
|
212
236
|
env: Mapping[str, str], adapter: Any = None, reader: CsvSourceReader | None = None
|
|
213
237
|
) -> int:
|
|
214
238
|
"""Run a single node end-to-end and print exactly one sentinel result block.
|
|
215
239
|
|
|
216
|
-
``reader``
|
|
217
|
-
|
|
218
|
-
letting that function pick a reader via ``reader_for``. Ignored for a
|
|
219
|
-
python-model node.
|
|
240
|
+
``reader`` is threaded into the python-csv producer instead of letting
|
|
241
|
+
it pick a reader via ``reader_for``; the script producer ignores it.
|
|
220
242
|
|
|
221
243
|
Returns 0 on success, 1 on any :class:`HarnessError`.
|
|
222
244
|
"""
|
|
@@ -255,14 +277,7 @@ def run_node(
|
|
|
255
277
|
|
|
256
278
|
_validate_config_early(active_adapter, node)
|
|
257
279
|
|
|
258
|
-
|
|
259
|
-
table = produce_csv(node, reader=reader)
|
|
260
|
-
else:
|
|
261
|
-
with contextlib.redirect_stdout(sys.stderr):
|
|
262
|
-
module = load_script(node, app_root)
|
|
263
|
-
ctx = RunContext(node, active_adapter)
|
|
264
|
-
raw_result = _execute_script(module, ctx)
|
|
265
|
-
table = to_arrow(raw_result)
|
|
280
|
+
table = _PRODUCERS[node.kind](node, active_adapter, app_root, reader)
|
|
266
281
|
|
|
267
282
|
conformed = conform(table, node.output_columns, node.extra_columns)
|
|
268
283
|
|
{continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: continuo-python-runtime
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
Maintainer: Simone Carolini
|
|
@@ -163,7 +163,7 @@ Every rule below (`extra_columns`, `output_columns`, "Conform rules") applies
|
|
|
163
163
|
to both kinds identically — `kind` only changes how the pre-conform table is
|
|
164
164
|
produced, never how it is checked or written.
|
|
165
165
|
|
|
166
|
-
- **`python-
|
|
166
|
+
- **`python-node`** (the default; the field may be omitted) — a script node.
|
|
167
167
|
It requires `script:` and a `reads:` map of one or more named SQL queries,
|
|
168
168
|
as described in "The script API" below.
|
|
169
169
|
- **`python-csv`** — a contract-only node: it has no script and its `reads:`
|
|
@@ -179,6 +179,12 @@ produced, never how it is checked or written.
|
|
|
179
179
|
header but not declared are governed by the same `extra_columns` policy as
|
|
180
180
|
a script node's output — `raise` (default) fails the run, `warn` drops
|
|
181
181
|
them and logs a warning. See `template/contracts/example_csv.yml`.
|
|
182
|
+
- **`python-api`** — a script node that fetches its own data (typically over
|
|
183
|
+
HTTP) and declares no `reads:`. `ctx.read` has nothing to return. It may name
|
|
184
|
+
one Kubernetes Secret with `secret_ref: continuo-api-<name>`; continuo
|
|
185
|
+
attaches that Secret to the node's pod, so each of its keys is an env var the
|
|
186
|
+
script reads with `os.environ[...]`. The contract never holds a secret value
|
|
187
|
+
or env var name; the Secret itself is created in the cluster by the operator.
|
|
182
188
|
|
|
183
189
|
## The script API
|
|
184
190
|
|
|
@@ -275,9 +281,9 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
275
281
|
`FROM`:
|
|
276
282
|
|
|
277
283
|
```dockerfile
|
|
278
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.
|
|
284
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.7.0
|
|
279
285
|
# or
|
|
280
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.
|
|
286
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.7.0
|
|
281
287
|
```
|
|
282
288
|
|
|
283
289
|
The engine is part of the image **name**; the tag is the bare version, so
|
|
@@ -6,24 +6,26 @@ continuo_python_runtime/context.py,sha256=K-9XXYlQ4OZ250rCOq6EJNq7fg_D5sK6tOcu7D
|
|
|
6
6
|
continuo_python_runtime/csv_loader.py,sha256=a5Hwu-YIzIiko8X1_vqShsnzjs2-OsKEUSCHLXg_dio,3346
|
|
7
7
|
continuo_python_runtime/csv_source.py,sha256=7g8Tu6ieiJLvWH4Ep4nwYlOdGkjeePr12K5UHuO8TNc,4136
|
|
8
8
|
continuo_python_runtime/errors.py,sha256=nfZzfBhQ2ajm7USAWKKaGY2uUlh8eNjNPNYYuaOJgZE,1032
|
|
9
|
-
continuo_python_runtime/harness.py,sha256=
|
|
9
|
+
continuo_python_runtime/harness.py,sha256=vsoahI38OCaCEpg9Il7xcHYcsXp6A5cxY_fjCC8xGJk,13046
|
|
10
10
|
continuo_python_runtime/hashing.py,sha256=70iXQb0CmjGqniTPofXY6WapydQ7aK2XOk5dzpEf-zk,3354
|
|
11
11
|
continuo_python_runtime/lint.py,sha256=M59UtCHHVQnJKNolIkvkRk_AgGAFvxD9zkKAMMK-7GI,12984
|
|
12
12
|
continuo_python_runtime/types.py,sha256=_ssxNE2Lq5782n9_Ihs-u_6ybCCy8XaL6iYbE35tBug,5057
|
|
13
13
|
continuo_python_runtime/contract/__init__.py,sha256=S79x6j_P0CmaSuCGzeWoK1U0507KvD2MZXwQl9DFWMM,37
|
|
14
|
-
continuo_python_runtime/contract/
|
|
15
|
-
continuo_python_runtime/contract/
|
|
16
|
-
continuo_python_runtime/contract/
|
|
14
|
+
continuo_python_runtime/contract/kinds.py,sha256=Ht-hCT7R5A1tEnPlbGL-b9FyJQBvpDWrXK9PBt7yA28,4938
|
|
15
|
+
continuo_python_runtime/contract/loader.py,sha256=9HsV7dJhCwudjOPXBapYFKiZtslyXIkiZ_WfeuLh5-E,14858
|
|
16
|
+
continuo_python_runtime/contract/merge.py,sha256=Zw--_-v3ERE02Nu7qqLtbHa3Hr__0a1SF-2PRqUfXAY,7594
|
|
17
|
+
continuo_python_runtime/contract/model.py,sha256=WB_Qw4koP8BEwuiAZwIEsKFQRa_6QHHBK4xi4eGvPQo,1082
|
|
17
18
|
continuo_python_runtime/contract/paths.py,sha256=SRn70pJkDAIpXw0IcMU2iNKPKvct8w_SZP2ec-lV91M,1458
|
|
19
|
+
continuo_python_runtime/contract/secret_ref.py,sha256=96jh2G81clpabiEWGIsxbd9r5GKEGLb4Sa1-WJ9G59Q,1131
|
|
18
20
|
continuo_python_runtime/csv_readers/__init__.py,sha256=9TKvjC37C77uRaJQZuO0--zVj2OvuNwpJE6nWx7j2S8,806
|
|
19
21
|
continuo_python_runtime/csv_readers/https.py,sha256=tMJ9k5F0b9SAydpon0ZlOznfjvjhF2BBLltyVmvRneE,4998
|
|
20
22
|
continuo_python_runtime/csv_readers/s3.py,sha256=zmvHs3TxlN2alPKrdaHzPyREmkve2nBiAsHRXXe1mE0,2246
|
|
21
23
|
continuo_python_runtime/validation/__init__.py,sha256=hz5oGXsoaUoygR0_pBby9PS6cEfrre518EE0HGB58eE,79
|
|
22
24
|
continuo_python_runtime/validation/runner.py,sha256=7pbksf7MXfTNcdklCOkOXg9qfRiJtyI_dbgbL3a9LWk,14072
|
|
23
25
|
continuo_python_runtime/validation/s3.py,sha256=q8nRrT9R7L1-M4dSlDnkL7CUJMPmq86MHY_UKYFlEf8,1785
|
|
24
|
-
continuo_python_runtime-0.
|
|
25
|
-
continuo_python_runtime-0.
|
|
26
|
-
continuo_python_runtime-0.
|
|
27
|
-
continuo_python_runtime-0.
|
|
28
|
-
continuo_python_runtime-0.
|
|
29
|
-
continuo_python_runtime-0.
|
|
26
|
+
continuo_python_runtime-0.7.0.dist-info/METADATA,sha256=Uo2XMCZawYbKgpER3qDn-eF_0cKJUZTultaxxcBAhuQ,19281
|
|
27
|
+
continuo_python_runtime-0.7.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
28
|
+
continuo_python_runtime-0.7.0.dist-info/entry_points.txt,sha256=ZvIwSm9f9B9ZmZMf6CyF5Zt4QONn8rqaIk-u-B1CF2Y,70
|
|
29
|
+
continuo_python_runtime-0.7.0.dist-info/licenses/LICENSE,sha256=hfXfFCk-8gnps4YvHA6bjCNaSYqOrq2gAdqkXYycAQc,11345
|
|
30
|
+
continuo_python_runtime-0.7.0.dist-info/licenses/NOTICE,sha256=ykgYyQkAMx3uas90FZHzt0aTca9IgJQCLpZ4d74Pl3U,465
|
|
31
|
+
continuo_python_runtime-0.7.0.dist-info/RECORD,,
|
{continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{continuo_python_runtime-0.5.0.dist-info → continuo_python_runtime-0.7.0.dist-info}/licenses/NOTICE
RENAMED
|
File without changes
|