continuo-python-runtime 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/cli.py +27 -9
- continuo_python_runtime/closure.py +268 -0
- continuo_python_runtime/contract/loader.py +155 -129
- continuo_python_runtime/contract/merge.py +86 -9
- continuo_python_runtime/contract/model.py +3 -1
- continuo_python_runtime/harness.py +93 -5
- continuo_python_runtime/hashing.py +78 -23
- continuo_python_runtime/lint.py +40 -26
- continuo_python_runtime/types.py +32 -96
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/METADATA +50 -7
- continuo_python_runtime-0.2.0.dist-info/RECORD +19 -0
- continuo_python_runtime-0.1.0.dist-info/RECORD +0 -18
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/WHEEL +0 -0
- {continuo_python_runtime-0.1.0.dist-info → continuo_python_runtime-0.2.0.dist-info}/entry_points.txt +0 -0
|
@@ -11,6 +11,9 @@ from pathlib import Path
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
|
|
13
13
|
import yaml
|
|
14
|
+
from continuo_validation_contract.sql import ensure_single_read # type: ignore[import-untyped]
|
|
15
|
+
from sqlglot import Dialect
|
|
16
|
+
from sqlglot.errors import TokenError
|
|
14
17
|
|
|
15
18
|
from continuo_python_runtime.contract.model import (
|
|
16
19
|
CRITICALITIES,
|
|
@@ -32,6 +35,7 @@ _ALLOWED_KEYS = {
|
|
|
32
35
|
"extra_columns",
|
|
33
36
|
"reads",
|
|
34
37
|
"output_columns",
|
|
38
|
+
"config",
|
|
35
39
|
"content_hash",
|
|
36
40
|
}
|
|
37
41
|
|
|
@@ -40,129 +44,6 @@ _REQUIRED_STRING_FIELDS = ("schema", "table", "owner", "schedule", "script")
|
|
|
40
44
|
_ALLOWED_OUTPUT_COLUMN_KEYS = {"name", "type", "nullable"}
|
|
41
45
|
|
|
42
46
|
|
|
43
|
-
def _mask_string_literals(sql: str) -> str | None:
|
|
44
|
-
"""Return ``sql`` with the contents of single/double-quoted string
|
|
45
|
-
literals removed, so a ``;`` scan doesn't trip on one embedded in a
|
|
46
|
-
literal (e.g. ``split_part(tags, ';', 1)``).
|
|
47
|
-
|
|
48
|
-
Handles doubled ``''`` escapes inside single-quoted strings. The quote
|
|
49
|
-
delimiters themselves are kept; only the interior characters are
|
|
50
|
-
dropped. The result is not valid SQL - it exists solely for the
|
|
51
|
-
semicolon check below.
|
|
52
|
-
|
|
53
|
-
Returns ``None`` if a single- or double-quoted literal is left
|
|
54
|
-
unterminated (no closing quote before the end of the string): silently
|
|
55
|
-
truncating in that case would drop everything after the opening quote
|
|
56
|
-
-- including any ``;`` it was hiding -- and let a multi-statement read
|
|
57
|
-
through undetected.
|
|
58
|
-
"""
|
|
59
|
-
result = []
|
|
60
|
-
i = 0
|
|
61
|
-
n = len(sql)
|
|
62
|
-
while i < n:
|
|
63
|
-
ch = sql[i]
|
|
64
|
-
if ch == "'":
|
|
65
|
-
result.append(ch)
|
|
66
|
-
i += 1
|
|
67
|
-
closed = False
|
|
68
|
-
while i < n:
|
|
69
|
-
if sql[i] == "'":
|
|
70
|
-
if i + 1 < n and sql[i + 1] == "'":
|
|
71
|
-
# doubled '' escape: part of the literal, drop both
|
|
72
|
-
i += 2
|
|
73
|
-
continue
|
|
74
|
-
result.append(sql[i])
|
|
75
|
-
i += 1
|
|
76
|
-
closed = True
|
|
77
|
-
break
|
|
78
|
-
i += 1
|
|
79
|
-
if not closed:
|
|
80
|
-
return None
|
|
81
|
-
continue
|
|
82
|
-
if ch == '"':
|
|
83
|
-
result.append(ch)
|
|
84
|
-
i += 1
|
|
85
|
-
closed = False
|
|
86
|
-
while i < n:
|
|
87
|
-
if sql[i] == '"':
|
|
88
|
-
result.append(sql[i])
|
|
89
|
-
i += 1
|
|
90
|
-
closed = True
|
|
91
|
-
break
|
|
92
|
-
i += 1
|
|
93
|
-
if not closed:
|
|
94
|
-
return None
|
|
95
|
-
continue
|
|
96
|
-
result.append(ch)
|
|
97
|
-
i += 1
|
|
98
|
-
return "".join(result)
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
def _strip_leading_sql_comments(sql: str) -> str:
|
|
102
|
-
"""Strip leading whitespace, ``-- ...`` line comments, and ``/* ... */``
|
|
103
|
-
block comments from the start of ``sql``.
|
|
104
|
-
|
|
105
|
-
An unterminated ``/*`` block comment (no closing ``*/``) consumes the
|
|
106
|
-
rest of the string, so callers see an empty statement rather than a
|
|
107
|
-
crash. Bounded by ``len(sql) + 1`` iterations (each iteration strictly
|
|
108
|
-
shrinks ``text``) with an explicit trailing return, so every path
|
|
109
|
-
returns a definite ``str`` rather than relying on an unbounded
|
|
110
|
-
``while True`` loop that a type checker can't prove always exits via
|
|
111
|
-
``return``.
|
|
112
|
-
"""
|
|
113
|
-
text = sql
|
|
114
|
-
for _ in range(len(sql) + 1):
|
|
115
|
-
stripped = text.lstrip()
|
|
116
|
-
if stripped.startswith("--"):
|
|
117
|
-
newline = stripped.find("\n")
|
|
118
|
-
text = stripped[newline + 1 :] if newline != -1 else ""
|
|
119
|
-
continue
|
|
120
|
-
if stripped.startswith("/*"):
|
|
121
|
-
end = stripped.find("*/")
|
|
122
|
-
text = stripped[end + 2 :] if end != -1 else ""
|
|
123
|
-
continue
|
|
124
|
-
return stripped
|
|
125
|
-
return text.lstrip()
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
def _validate_read_shape(label: str, name: str, sql: str) -> None:
|
|
129
|
-
"""Enforce the §13.1 read shape: a single SELECT/WITH statement.
|
|
130
|
-
|
|
131
|
-
The read, after stripping whitespace and leading SQL comments, must
|
|
132
|
-
start with ``select``, ``with``, or a leading ``(`` (a parenthesized
|
|
133
|
-
SELECT), case-insensitive. It must contain no ``;`` except one optional
|
|
134
|
-
trailing semicolon; the semicolon scan is literal-aware, ignoring ``;``
|
|
135
|
-
characters inside single/double-quoted string literals. An unterminated
|
|
136
|
-
string literal is itself rejected -- it can otherwise hide arbitrary
|
|
137
|
-
trailing SQL (including a ``;``) from the scan.
|
|
138
|
-
Schema-qualification is left to the control plane's sqlglot validation.
|
|
139
|
-
|
|
140
|
-
Raises:
|
|
141
|
-
ContractError: If the shape is violated, naming the read.
|
|
142
|
-
"""
|
|
143
|
-
stripped = sql.strip()
|
|
144
|
-
body = stripped[:-1] if stripped.endswith(";") else stripped
|
|
145
|
-
masked = _mask_string_literals(body)
|
|
146
|
-
if masked is None:
|
|
147
|
-
raise ContractError(
|
|
148
|
-
f"{label}: read '{name}' has an unterminated string literal"
|
|
149
|
-
)
|
|
150
|
-
if ";" in masked:
|
|
151
|
-
raise ContractError(
|
|
152
|
-
f"{label}: 'reads.{name}' must be a single SQL statement "
|
|
153
|
-
f"(only one optional trailing semicolon allowed)"
|
|
154
|
-
)
|
|
155
|
-
lowered = _strip_leading_sql_comments(body).lower()
|
|
156
|
-
if not (
|
|
157
|
-
lowered.startswith("select")
|
|
158
|
-
or lowered.startswith("with")
|
|
159
|
-
or lowered.startswith("(")
|
|
160
|
-
):
|
|
161
|
-
raise ContractError(
|
|
162
|
-
f"{label}: 'reads.{name}' must start with SELECT or WITH"
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
|
|
166
47
|
class _StrictLoader(yaml.SafeLoader):
|
|
167
48
|
"""SafeLoader that rejects duplicate mapping keys instead of keeping the last."""
|
|
168
49
|
|
|
@@ -193,10 +74,90 @@ def _node_label(raw: dict[str, Any], source: str) -> str:
|
|
|
193
74
|
return source
|
|
194
75
|
|
|
195
76
|
|
|
196
|
-
|
|
77
|
+
_JSON_SCALARS = (str, int, float, bool, type(None))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _validate_config(raw: Any, label: str) -> dict[str, Any]:
|
|
81
|
+
"""Validate the node's physical-layout `config` as a JSON-shaped mapping.
|
|
82
|
+
|
|
83
|
+
The engine's adapter — not this loader — owns the vocabulary (§3.3), so the
|
|
84
|
+
only rules here are the ones the hash and the wire format need: it is a
|
|
85
|
+
mapping, every key at every level is a string, and every value is
|
|
86
|
+
JSON-serializable. Non-string keys would make `json.dumps(..., sort_keys=True)`
|
|
87
|
+
raise inside the hasher, and a non-serializable value would break the wire
|
|
88
|
+
artifact — both must surface here, naming the node, not deep in CI.
|
|
89
|
+
|
|
90
|
+
YAML aliases (``&anchor`` / ``*anchor``) can make PyYAML construct a
|
|
91
|
+
genuinely cyclic object graph — a mapping whose own value (however deeply
|
|
92
|
+
nested) is itself. Recursing into that without tracking what is already
|
|
93
|
+
on the call stack raises an uncaught ``RecursionError`` instead of this
|
|
94
|
+
function's promised ``ContractError``. ``active`` holds the ``id()`` of
|
|
95
|
+
every dict/list container currently being descended into; a container is
|
|
96
|
+
added just before recursing into its items and discarded right after, so
|
|
97
|
+
re-entering one still on the stack is a genuine back-edge (a cycle), while
|
|
98
|
+
the *same* container reached twice at sibling positions — a legal DAG,
|
|
99
|
+
e.g. one list aliased under two different keys — is not: its first visit
|
|
100
|
+
finishes (and is discarded) before the second begins.
|
|
101
|
+
"""
|
|
102
|
+
if raw is None:
|
|
103
|
+
return {}
|
|
104
|
+
if not isinstance(raw, dict):
|
|
105
|
+
raise ContractError(f"{label}: 'config' must be a mapping")
|
|
106
|
+
|
|
107
|
+
active: set[int] = set()
|
|
108
|
+
|
|
109
|
+
def _check(value: Any, key: Any) -> None:
|
|
110
|
+
"""Validate ``value`` (found under ``key`` in its enclosing mapping)."""
|
|
111
|
+
if isinstance(value, (dict, list)):
|
|
112
|
+
container_id = id(value)
|
|
113
|
+
if container_id in active:
|
|
114
|
+
raise ContractError(
|
|
115
|
+
f"{label}: 'config' contains a circular reference at {key!r}"
|
|
116
|
+
)
|
|
117
|
+
active.add(container_id)
|
|
118
|
+
try:
|
|
119
|
+
if isinstance(value, dict):
|
|
120
|
+
for sub_key, sub_value in value.items():
|
|
121
|
+
if not isinstance(sub_key, str):
|
|
122
|
+
raise ContractError(
|
|
123
|
+
f"{label}: 'config' keys must be strings, got {sub_key!r}"
|
|
124
|
+
)
|
|
125
|
+
_check(sub_value, sub_key)
|
|
126
|
+
else:
|
|
127
|
+
for item in value:
|
|
128
|
+
_check(item, key)
|
|
129
|
+
finally:
|
|
130
|
+
active.discard(container_id)
|
|
131
|
+
elif not isinstance(value, _JSON_SCALARS):
|
|
132
|
+
raise ContractError(
|
|
133
|
+
f"{label}: 'config' value for {key!r} is not JSON-serializable: {value!r}"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
_check(raw, None)
|
|
137
|
+
return raw
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def parse_node(
|
|
141
|
+
raw: dict[str, Any],
|
|
142
|
+
source: str,
|
|
143
|
+
*,
|
|
144
|
+
dialect: str | None = None,
|
|
145
|
+
check_reads: bool = True,
|
|
146
|
+
) -> Node:
|
|
197
147
|
"""Validate a single raw mapping and build a :class:`Node`.
|
|
198
148
|
|
|
199
149
|
``source`` is the originating filename; it appears in every error message.
|
|
150
|
+
``dialect`` is a sqlglot dialect name (e.g. ``"postgres"``, ``"trino"``)
|
|
151
|
+
each declared read is checked against via
|
|
152
|
+
:func:`~continuo_validation_contract.sql.ensure_single_read`; ``None``
|
|
153
|
+
(the default) uses sqlglot's dialect-neutral parser.
|
|
154
|
+
|
|
155
|
+
``check_reads=False`` skips *only* that :func:`ensure_single_read` call —
|
|
156
|
+
the read-shape gate — leaving every other rule here (required fields,
|
|
157
|
+
criticality, the ``reads`` map's own shape, output-column types and
|
|
158
|
+
uniqueness, ``config``, ``content_hash``) in force. It exists for
|
|
159
|
+
:func:`~continuo_python_runtime.harness.run_node`; see
|
|
160
|
+
:func:`load_contract_dir` for why the runtime opts out.
|
|
200
161
|
"""
|
|
201
162
|
if not isinstance(raw, dict):
|
|
202
163
|
raise ContractError(
|
|
@@ -252,7 +213,26 @@ def parse_node(raw: dict[str, Any], source: str) -> Node:
|
|
|
252
213
|
raise ContractError(
|
|
253
214
|
f"{label}: 'reads.{name}' must be a non-empty SQL string"
|
|
254
215
|
)
|
|
255
|
-
|
|
216
|
+
if not check_reads:
|
|
217
|
+
continue
|
|
218
|
+
try:
|
|
219
|
+
ensure_single_read(sql, dialect)
|
|
220
|
+
except (ValueError, TokenError) as exc:
|
|
221
|
+
# ensure_single_read's own message is phrased for check_binds
|
|
222
|
+
# (its only other caller today), so it's wrapped rather than
|
|
223
|
+
# surfaced bare here. TokenError is also caught: an unterminated
|
|
224
|
+
# string literal or comment fails sqlglot's tokenizer with a
|
|
225
|
+
# TokenError, a SqlglotError sibling of ParseError and not a
|
|
226
|
+
# subclass of ValueError -- despite ensure_single_read's
|
|
227
|
+
# docstring promising every rejection is a ValueError. Only
|
|
228
|
+
# TokenError, not the broader SqlglotError, is caught here: by
|
|
229
|
+
# the time control reaches this point `dialect` has already been
|
|
230
|
+
# validated once in load_contract_dir, so any other SqlglotError
|
|
231
|
+
# a future sqlglot version might raise from this call should
|
|
232
|
+
# surface as itself, not get relabeled as a rejected read.
|
|
233
|
+
raise ContractError(
|
|
234
|
+
f"{label}: 'reads.{name}' must be a single read query ({exc})"
|
|
235
|
+
) from exc
|
|
256
236
|
|
|
257
237
|
raw_columns = raw.get("output_columns")
|
|
258
238
|
if not isinstance(raw_columns, list) or not raw_columns:
|
|
@@ -298,6 +278,8 @@ def parse_node(raw: dict[str, Any], source: str) -> Node:
|
|
|
298
278
|
if not isinstance(description, str):
|
|
299
279
|
raise ContractError(f"{label}: 'description' must be a string")
|
|
300
280
|
|
|
281
|
+
config = _validate_config(raw.get("config"), label)
|
|
282
|
+
|
|
301
283
|
content_hash = raw.get("content_hash")
|
|
302
284
|
if content_hash is not None and not isinstance(content_hash, str):
|
|
303
285
|
raise ContractError(f"{label}: 'content_hash' must be a string")
|
|
@@ -313,16 +295,58 @@ def parse_node(raw: dict[str, Any], source: str) -> Node:
|
|
|
313
295
|
output_columns=tuple(columns),
|
|
314
296
|
description=description,
|
|
315
297
|
extra_columns=extra_columns,
|
|
298
|
+
config=config,
|
|
316
299
|
content_hash=content_hash,
|
|
317
300
|
)
|
|
318
301
|
|
|
319
302
|
|
|
320
|
-
def load_contract_dir(
|
|
303
|
+
def load_contract_dir(
|
|
304
|
+
path: Path, *, dialect: str | None = None, check_reads: bool = True
|
|
305
|
+
) -> list[Node]:
|
|
321
306
|
"""Load and validate every `*.yml`/`*.yaml` contract file under ``path``.
|
|
322
307
|
|
|
323
|
-
|
|
324
|
-
|
|
308
|
+
``check_reads=False`` skips the per-read
|
|
309
|
+
:func:`~continuo_validation_contract.sql.ensure_single_read` gate and
|
|
310
|
+
nothing else; every other rule in :func:`parse_node` and every rule here
|
|
311
|
+
(dialect validity, document shape, duplicate relations, "no contract
|
|
312
|
+
files found") still runs. The runtime
|
|
313
|
+
(:func:`~continuo_python_runtime.harness.run_node`) passes it, because
|
|
314
|
+
re-running the read-shape gate at container start can only introduce a
|
|
315
|
+
disagreement, never catch a new problem:
|
|
316
|
+
|
|
317
|
+
- CI already gated the reads (``continuo-runtime validate``), under the
|
|
318
|
+
repo's own ``--dialect``, and Continuo gated them again with its own
|
|
319
|
+
parser and bind-check before promoting the release.
|
|
320
|
+
- The runtime has no ``--dialect`` of its own, so it would re-parse with
|
|
321
|
+
the dialect-neutral grammar — a *different* and in places stricter
|
|
322
|
+
grammar than CI used. Ordinary postgres (``a ~ 'x'``, ``data @>
|
|
323
|
+
'{...}'``) parses under ``--dialect postgres`` and not under the
|
|
324
|
+
neutral parser, so a team following the docs would get a green
|
|
325
|
+
validate, a green merge, an accepted release, and a node that fails on
|
|
326
|
+
every single run.
|
|
327
|
+
- Nothing at run time consumes the parse: ``ctx.read`` resolves declared
|
|
328
|
+
reads by name and hands the SQL to the adapter verbatim.
|
|
329
|
+
|
|
330
|
+
``dialect`` is validated once, up front, then forwarded to
|
|
331
|
+
:func:`parse_node` for every node, so every declared read is checked
|
|
332
|
+
against that sqlglot dialect (``None`` -- the default -- uses sqlglot's
|
|
333
|
+
dialect-neutral parser). Validating it here rather than leaving it to
|
|
334
|
+
the per-read ``ensure_single_read`` call matters: an unrecognized
|
|
335
|
+
dialect name (e.g. a typo'd ``--dialect POSTGRES`` instead of
|
|
336
|
+
``postgres``) raises the same bare ``ValueError`` a genuinely
|
|
337
|
+
unparseable read would, and would otherwise be misreported as a
|
|
338
|
+
rejected read instead of a bad flag -- blaming an innocent, valid read.
|
|
339
|
+
|
|
340
|
+
Raises `ContractError` if ``dialect`` is not a sqlglot dialect name, if
|
|
341
|
+
no nodes are found, if any file's document is malformed, or if two nodes
|
|
342
|
+
across files share the same `(schema, table)`.
|
|
325
343
|
"""
|
|
344
|
+
if dialect is not None:
|
|
345
|
+
try:
|
|
346
|
+
Dialect.get_or_raise(dialect)
|
|
347
|
+
except ValueError as exc:
|
|
348
|
+
raise ContractError(f"unknown --dialect {dialect!r}: {exc}") from exc
|
|
349
|
+
|
|
326
350
|
files = sorted(path.glob("*.yml")) + sorted(path.glob("*.yaml"))
|
|
327
351
|
|
|
328
352
|
nodes: list[Node] = []
|
|
@@ -348,7 +372,9 @@ def load_contract_dir(path: Path) -> list[Node]:
|
|
|
348
372
|
raise ContractError(f"{file.name}: 'nodes' must be a list")
|
|
349
373
|
|
|
350
374
|
for raw_node in raw_nodes:
|
|
351
|
-
node = parse_node(
|
|
375
|
+
node = parse_node(
|
|
376
|
+
raw_node, file.name, dialect=dialect, check_reads=check_reads
|
|
377
|
+
)
|
|
352
378
|
existing_source = relation_sources.get(node.relation)
|
|
353
379
|
if existing_source is not None:
|
|
354
380
|
raise ContractError(
|
|
@@ -4,17 +4,22 @@ from pathlib import Path
|
|
|
4
4
|
|
|
5
5
|
import yaml
|
|
6
6
|
|
|
7
|
+
from continuo_python_runtime.closure import resolve_closure
|
|
7
8
|
from continuo_python_runtime.contract.loader import load_contract_dir
|
|
8
9
|
from continuo_python_runtime.contract.model import CONTRACT_VERSION, Node
|
|
9
10
|
from continuo_python_runtime.contract.paths import resolve_script_path
|
|
10
|
-
from continuo_python_runtime.
|
|
11
|
+
from continuo_python_runtime.errors import ContractError
|
|
12
|
+
from continuo_python_runtime.hashing import hash_parts
|
|
13
|
+
from continuo_python_runtime.lint import lint_source
|
|
11
14
|
|
|
12
15
|
|
|
13
16
|
def node_entry(node: Node) -> dict:
|
|
14
17
|
"""Convert a Node to its wire form dict.
|
|
15
18
|
|
|
16
|
-
Returns the node as a dict with all fields, output_columns as list of
|
|
17
|
-
|
|
19
|
+
Returns the node as a dict with all fields, output_columns as list of
|
|
20
|
+
dicts. The entry carries no hash fields: build_wire_contract adds all
|
|
21
|
+
four (source_hash, shared_code_hash, config_hash, content_hash). The
|
|
22
|
+
nullable field is always present in output_columns.
|
|
18
23
|
"""
|
|
19
24
|
return {
|
|
20
25
|
"schema": node.schema,
|
|
@@ -34,29 +39,101 @@ def node_entry(node: Node) -> dict:
|
|
|
34
39
|
],
|
|
35
40
|
"description": node.description,
|
|
36
41
|
"extra_columns": node.extra_columns,
|
|
42
|
+
"config": dict(node.config),
|
|
37
43
|
}
|
|
38
44
|
|
|
39
45
|
|
|
40
|
-
def
|
|
46
|
+
def _lint_node_closure(
|
|
47
|
+
node: Node,
|
|
48
|
+
repo_root: Path,
|
|
49
|
+
script_path: Path,
|
|
50
|
+
script_bytes: bytes,
|
|
51
|
+
closure: list[Path],
|
|
52
|
+
member_bytes: list[bytes],
|
|
53
|
+
) -> None:
|
|
54
|
+
"""Lint the node's script and every resolved closure member.
|
|
55
|
+
|
|
56
|
+
Every file that executes for this node — the script plus its transitive
|
|
57
|
+
in-repo import closure — is held to the same lint rules
|
|
58
|
+
(``continuo_python_runtime.lint.lint_source``: L1 forbidden driver
|
|
59
|
+
imports, L2 SQL string literals, L3 forbidden data-access calls, L4
|
|
60
|
+
private-attribute access, L5 dynamic-import constructs) that CI's
|
|
61
|
+
``continuo-runtime lint`` applies to ``scripts/``. Enforcing it again
|
|
62
|
+
here, at merge time, means a helper file outside ``scripts/`` — which
|
|
63
|
+
the CI lint step never looks at — cannot become a side channel around
|
|
64
|
+
the harness's sole write sink, and the check cannot be defeated by a
|
|
65
|
+
stale or hand-edited workflow file: whatever produces the release
|
|
66
|
+
artifact is the thing that checked.
|
|
67
|
+
|
|
68
|
+
Paths in violation messages are repo-root-relative (``str(path.relative_to
|
|
69
|
+
(repo_root))``), so a message reads ``lib/shared.py:1: ...`` identically
|
|
70
|
+
in CI and on a laptop, never an absolute path that differs between the
|
|
71
|
+
two.
|
|
72
|
+
|
|
73
|
+
Raises ``ContractError`` naming the node and listing every violation
|
|
74
|
+
from every offending file — not just the first file's — so an author
|
|
75
|
+
can act on the full report in one pass instead of an iterative guessing
|
|
76
|
+
game.
|
|
77
|
+
|
|
78
|
+
Every file is decoded as ``"utf-8-sig"`` (identical to plain ``"utf-8"``
|
|
79
|
+
except that a leading BOM, if present, is stripped rather than kept as a
|
|
80
|
+
literal ``U+FEFF`` character) — this matches what ``ast.parse(bytes)``
|
|
81
|
+
already does for the same file in ``resolve_closure`` (bytes-mode
|
|
82
|
+
parsing treats a UTF-8 BOM as an implicit encoding declaration and
|
|
83
|
+
strips it), so a BOM file that parses and runs fine is not rejected here
|
|
84
|
+
on a mismatch between the two decoders. A file that isn't valid UTF-8 at
|
|
85
|
+
all (e.g. a PEP 263 ``# -*- coding: latin-1 -*-`` cookie plus a
|
|
86
|
+
non-ASCII byte — valid input to ``ast.parse(bytes)``, which honors the
|
|
87
|
+
cookie) raises ``UnicodeDecodeError``; that is caught and re-raised as a
|
|
88
|
+
``ContractError`` naming the node and the offending repo-relative path,
|
|
89
|
+
so it joins the same error taxonomy as every other rejection here
|
|
90
|
+
instead of escaping as a bare stdlib exception past ``cli.main``, which
|
|
91
|
+
catches only ``HarnessError``/``OSError``.
|
|
92
|
+
"""
|
|
93
|
+
repo_root_resolved = repo_root.resolve()
|
|
94
|
+
files = [(script_path, script_bytes), *zip(closure, member_bytes, strict=True)]
|
|
95
|
+
|
|
96
|
+
violations: list[str] = []
|
|
97
|
+
for path, data in files:
|
|
98
|
+
rel = path.relative_to(repo_root_resolved)
|
|
99
|
+
try:
|
|
100
|
+
text = data.decode("utf-8-sig")
|
|
101
|
+
except UnicodeDecodeError as exc:
|
|
102
|
+
raise ContractError(f"{node.relation}: {rel}: not valid UTF-8: {exc}") from exc
|
|
103
|
+
violations.extend(lint_source(text, str(rel)))
|
|
104
|
+
|
|
105
|
+
if violations:
|
|
106
|
+
report = "\n".join(violations)
|
|
107
|
+
raise ContractError(
|
|
108
|
+
f"{node.relation}: lint violations in script or import closure:\n{report}"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def build_wire_contract(
|
|
113
|
+
contract_dir: Path, repo_root: Path, service: str, *, dialect: str | None = None
|
|
114
|
+
) -> dict:
|
|
41
115
|
"""Build and return a wire contract document.
|
|
42
116
|
|
|
43
117
|
Loads contracts from contract_dir, resolves script paths against repo_root,
|
|
44
118
|
computes content hashes, and returns a contract document sorted by relation.
|
|
119
|
+
``dialect`` is forwarded to :func:`~continuo_python_runtime.contract.loader
|
|
120
|
+
.load_contract_dir` so every declared read is checked against that sqlglot
|
|
121
|
+
dialect (``None`` -- the default -- uses sqlglot's dialect-neutral parser).
|
|
45
122
|
|
|
46
123
|
Raises ContractError if any script file is missing.
|
|
47
124
|
"""
|
|
48
|
-
nodes = load_contract_dir(contract_dir)
|
|
125
|
+
nodes = load_contract_dir(contract_dir, dialect=dialect)
|
|
49
126
|
|
|
50
127
|
wire_nodes = []
|
|
51
128
|
for node in nodes:
|
|
52
129
|
entry = node_entry(node)
|
|
53
130
|
|
|
54
131
|
script_path = resolve_script_path(node.script, repo_root, context=node.relation)
|
|
55
|
-
|
|
56
|
-
# Read script bytes and compute hash
|
|
57
132
|
script_bytes = script_path.read_bytes()
|
|
58
|
-
|
|
59
|
-
|
|
133
|
+
closure = resolve_closure(script_path, repo_root)
|
|
134
|
+
member_bytes = [member.read_bytes() for member in closure]
|
|
135
|
+
_lint_node_closure(node, repo_root, script_path, script_bytes, closure, member_bytes)
|
|
136
|
+
entry.update(hash_parts(entry, script_bytes, member_bytes))
|
|
60
137
|
|
|
61
138
|
wire_nodes.append(entry)
|
|
62
139
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Contract v1 model."""
|
|
2
2
|
|
|
3
|
-
from dataclasses import dataclass
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Any
|
|
4
5
|
|
|
5
6
|
# Module-level constants
|
|
6
7
|
CRITICALITIES = frozenset({"REGULATORY", "CORE", "SECONDARY"})
|
|
@@ -31,6 +32,7 @@ class Node:
|
|
|
31
32
|
output_columns: tuple[Column, ...]
|
|
32
33
|
description: str = ""
|
|
33
34
|
extra_columns: str = "raise"
|
|
35
|
+
config: dict[str, Any] = field(default_factory=dict)
|
|
34
36
|
content_hash: str | None = None
|
|
35
37
|
|
|
36
38
|
@property
|
|
@@ -71,15 +71,57 @@ def select_node(nodes: list[Node], node_id: str) -> Node:
|
|
|
71
71
|
)
|
|
72
72
|
|
|
73
73
|
|
|
74
|
+
def ensure_import_paths(repo_root: Path, script_dir: Path) -> None:
|
|
75
|
+
"""Prepend ``repo_root`` then ``script_dir`` to ``sys.path``, idempotently.
|
|
76
|
+
|
|
77
|
+
``importlib``'s ``spec_from_file_location`` + ``exec_module`` executes a
|
|
78
|
+
file without putting anything on ``sys.path``, and ``continuo-runtime`` is
|
|
79
|
+
an installed console script, so ``sys.path[0]`` is the venv's ``bin``
|
|
80
|
+
directory — ``/app`` is on no path. A node script importing its own
|
|
81
|
+
shared helpers (exactly what ``shared_code_hash`` folds into the content
|
|
82
|
+
hash) would therefore die with ``ModuleNotFoundError`` on its first
|
|
83
|
+
production run, having passed CI and the domain repo's own pytest, which
|
|
84
|
+
adds the rootdir to ``sys.path`` itself.
|
|
85
|
+
|
|
86
|
+
The two roots and their order mirror
|
|
87
|
+
:func:`continuo_python_runtime.closure._resolve_name`'s search roots, so a
|
|
88
|
+
name the closure resolver folded into the hash is a name the interpreter
|
|
89
|
+
can resolve: ``repo_root`` first (package-qualified ``import
|
|
90
|
+
scripts.helpers``, or a helper in a sibling ``lib/``), then the importing
|
|
91
|
+
script's own directory (sibling ``import helpers``).
|
|
92
|
+
|
|
93
|
+
Entries are left in place for the process lifetime rather than removed
|
|
94
|
+
after ``exec_module``: a script may import lazily inside ``run()``, and a
|
|
95
|
+
container runs exactly one node per process. Both paths are resolved
|
|
96
|
+
first, and each is unconditionally repositioned to the front rather than
|
|
97
|
+
left wherever it already was: the shipped images set ``PYTHONPATH=/app``,
|
|
98
|
+
so ``repo_root`` is *always* already on ``sys.path`` before this runs, and
|
|
99
|
+
merely skipping an already-present entry would leave it behind
|
|
100
|
+
``script_dir`` -- the reverse of the required order. An existing entry is
|
|
101
|
+
removed before being reinserted at index 0, so repeated calls still
|
|
102
|
+
cannot grow ``sys.path`` or duplicate an entry, however the caller spelled
|
|
103
|
+
the roots or whatever order they were already in.
|
|
104
|
+
"""
|
|
105
|
+
for entry in (str(script_dir.resolve()), str(repo_root.resolve())):
|
|
106
|
+
if entry in sys.path:
|
|
107
|
+
sys.path.remove(entry)
|
|
108
|
+
sys.path.insert(0, entry)
|
|
109
|
+
|
|
110
|
+
|
|
74
111
|
def load_script(node: Node, repo_root: Path) -> ModuleType:
|
|
75
112
|
"""Import ``node.script`` (relative to ``repo_root``) and return the module.
|
|
76
113
|
|
|
114
|
+
``repo_root`` and the script's own directory are put on ``sys.path`` first
|
|
115
|
+
(see :func:`ensure_import_paths`) so the script's in-repo import closure
|
|
116
|
+
is importable.
|
|
117
|
+
|
|
77
118
|
Raises:
|
|
78
119
|
ContractError: If the script path is absolute, escapes the
|
|
79
120
|
repository root, or does not exist.
|
|
80
121
|
ScriptError: If the module has no callable ``run``.
|
|
81
122
|
"""
|
|
82
123
|
script_path = resolve_script_path(node.script, repo_root, context=node.relation)
|
|
124
|
+
ensure_import_paths(repo_root, script_path.parent)
|
|
83
125
|
|
|
84
126
|
module_name = f"_continuo_node_{uuid.uuid4().hex}"
|
|
85
127
|
spec = importlib.util.spec_from_file_location(module_name, script_path)
|
|
@@ -129,6 +171,42 @@ def _execute_script(module: ModuleType, ctx: RunContext) -> Any:
|
|
|
129
171
|
raise ScriptError(f"run() raised {exc.__class__.__name__}: {exc}") from exc
|
|
130
172
|
|
|
131
173
|
|
|
174
|
+
def _validate_config_early(adapter: Any, node: Node) -> None:
|
|
175
|
+
"""Check ``node.config`` against the engine's vocabulary before anything runs.
|
|
176
|
+
|
|
177
|
+
``ensure_table`` validates ``config`` too and stays the enforcement point —
|
|
178
|
+
but it is called *after* the script has executed and its result has been
|
|
179
|
+
conformed, so a single typo (``config: {index: [...]}``) burned the whole
|
|
180
|
+
node run before failing. This is the tripwire that fails it in the first
|
|
181
|
+
second instead. Validation-time checking of the same vocabulary is a future
|
|
182
|
+
cross-repo dependency (continuo-validation step 3a), and `continuo-runtime
|
|
183
|
+
validate` runs on a CI runner where no engine adapter is installed at all,
|
|
184
|
+
so this is the earliest point the engine's own rules can be applied.
|
|
185
|
+
|
|
186
|
+
The call is skipped for an adapter that does not provide ``validate_config``:
|
|
187
|
+
the abstract ``RuntimeAdapter`` in the pinned ``continuo-validation-contract``
|
|
188
|
+
does not declare it (this repo ships the harness and both adapters as one
|
|
189
|
+
coordinated release), and an adapter without it loses only earliness —
|
|
190
|
+
``ensure_table`` still fails closed on the same config.
|
|
191
|
+
|
|
192
|
+
Raises:
|
|
193
|
+
LoadError: If the adapter rejects the config, matching how a rejection
|
|
194
|
+
from ``ensure_table`` surfaces.
|
|
195
|
+
"""
|
|
196
|
+
validate = getattr(adapter, "validate_config", None)
|
|
197
|
+
if validate is None:
|
|
198
|
+
return
|
|
199
|
+
column_names = [c.name for c in node.output_columns]
|
|
200
|
+
try:
|
|
201
|
+
validate(node.config, column_names)
|
|
202
|
+
except HarnessError:
|
|
203
|
+
raise
|
|
204
|
+
except Exception as exc:
|
|
205
|
+
raise LoadError(
|
|
206
|
+
f"invalid config for {node.relation}: {exc}"
|
|
207
|
+
) from exc
|
|
208
|
+
|
|
209
|
+
|
|
132
210
|
def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
|
|
133
211
|
"""Run a single node end-to-end and print exactly one sentinel result block.
|
|
134
212
|
|
|
@@ -146,12 +224,17 @@ def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
|
|
|
146
224
|
|
|
147
225
|
logger.info("running node %s -> %s.%s", node_id, target_schema, table_name)
|
|
148
226
|
|
|
149
|
-
|
|
227
|
+
# check_reads=False: the harness has no --dialect of its own, so
|
|
228
|
+
# re-running the read-shape gate here would judge the reads against
|
|
229
|
+
# sqlglot's dialect-neutral grammar -- a different, in places stricter
|
|
230
|
+
# grammar than the one CI used -- and could only disagree with the
|
|
231
|
+
# gates already passed (CI's `continuo-runtime validate`, then
|
|
232
|
+
# Continuo's own parser and bind-check), never catch a new problem.
|
|
233
|
+
# ctx.read resolves declared reads by name and never parses them.
|
|
234
|
+
# Every other loader validation still runs at container start.
|
|
235
|
+
nodes = load_contract_dir(contract_dir, check_reads=False)
|
|
150
236
|
node = select_node(nodes, node_id)
|
|
151
237
|
|
|
152
|
-
with contextlib.redirect_stdout(sys.stderr):
|
|
153
|
-
module = load_script(node, app_root)
|
|
154
|
-
|
|
155
238
|
if adapter is not None:
|
|
156
239
|
active_adapter = adapter
|
|
157
240
|
else:
|
|
@@ -162,6 +245,11 @@ def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
|
|
|
162
245
|
except Exception as exc:
|
|
163
246
|
raise LoadError(f"adapter construction failed: {exc}") from exc
|
|
164
247
|
|
|
248
|
+
_validate_config_early(active_adapter, node)
|
|
249
|
+
|
|
250
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
251
|
+
module = load_script(node, app_root)
|
|
252
|
+
|
|
165
253
|
ctx = RunContext(node, active_adapter)
|
|
166
254
|
raw_result = _execute_script(module, ctx)
|
|
167
255
|
|
|
@@ -173,7 +261,7 @@ def run_node(env: Mapping[str, str], adapter: Any = None) -> int:
|
|
|
173
261
|
for c in node.output_columns
|
|
174
262
|
]
|
|
175
263
|
try:
|
|
176
|
-
active_adapter.ensure_table(target_schema, table_name, columns)
|
|
264
|
+
active_adapter.ensure_table(target_schema, table_name, columns, config=node.config)
|
|
177
265
|
active_adapter.load(target_schema, table_name, conformed)
|
|
178
266
|
except HarnessError:
|
|
179
267
|
raise
|