continuo-python-runtime 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/cli.py +6 -0
- continuo_python_runtime/content_check.py +73 -0
- continuo_python_runtime/contract/kinds.py +10 -2
- continuo_python_runtime/contract/loader.py +52 -2
- continuo_python_runtime/contract/merge.py +26 -45
- continuo_python_runtime/contract/model.py +1 -0
- continuo_python_runtime/contract/paths.py +33 -0
- continuo_python_runtime/errors.py +13 -1
- continuo_python_runtime/harness.py +7 -1
- continuo_python_runtime/hash_inputs.py +63 -0
- continuo_python_runtime/hashing.py +69 -70
- continuo_python_runtime/lint.py +194 -6
- continuo_python_runtime/validation/preflight.py +410 -0
- continuo_python_runtime/validation/runner.py +20 -3
- {continuo_python_runtime-0.11.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/METADATA +129 -12
- continuo_python_runtime-0.12.0.dist-info/RECORD +35 -0
- continuo_python_runtime-0.11.0.dist-info/RECORD +0 -32
- {continuo_python_runtime-0.11.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/WHEEL +0 -0
- {continuo_python_runtime-0.11.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/entry_points.txt +0 -0
- {continuo_python_runtime-0.11.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/licenses/LICENSE +0 -0
- {continuo_python_runtime-0.11.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
"""The ``seed_preflight`` and ``probe_views`` validation ops.
|
|
2
|
+
|
|
3
|
+
``seed_preflight`` asks the warehouse which objects a reload of a seed's
|
|
4
|
+
production table would destroy or leave reading a missing table, and compares
|
|
5
|
+
them with the objects the operator has allowed or acknowledged. ``check`` stops
|
|
6
|
+
a reload that would hit an object outside that set; ``report`` prints the same
|
|
7
|
+
conclusion and never stops. ``probe_views`` tries each named view with a
|
|
8
|
+
zero-row read and reports which ones no longer bind, and never fails the op for
|
|
9
|
+
it.
|
|
10
|
+
|
|
11
|
+
Each op prints exactly one structured result block as the last line on stdout
|
|
12
|
+
(``continuo_engine_contract.result``) and returns the process exit code: 0 when
|
|
13
|
+
the op ran to its conclusion (or ``report`` mode), 1 when ``check`` stops, 2
|
|
14
|
+
when the op could not run (a bad input, no adapter, missing adapter env, an
|
|
15
|
+
unexpected error). The block's ``status`` is ``error`` for exit codes 1 and 2.
|
|
16
|
+
Diagnostics go to stderr through ``logging``; stdout carries only the block.
|
|
17
|
+
|
|
18
|
+
The decision of ``seed_preflight`` is the pure function :func:`decide`; the rest
|
|
19
|
+
of the module reads the inputs, drives the adapter and prints.
|
|
20
|
+
"""
|
|
21
|
+
import json
|
|
22
|
+
import logging
|
|
23
|
+
from collections.abc import Callable, Mapping
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from enum import Enum
|
|
26
|
+
from typing import Any, cast
|
|
27
|
+
|
|
28
|
+
from continuo_engine_contract import result # type: ignore[import-untyped]
|
|
29
|
+
from continuo_engine_contract.port import ( # type: ignore[import-untyped]
|
|
30
|
+
AdapterDiscoveryError,
|
|
31
|
+
DependentsIntrospection,
|
|
32
|
+
WarehouseAdapter,
|
|
33
|
+
discover_adapter,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger("validation_preflight")
|
|
37
|
+
|
|
38
|
+
_Discover = Callable[[], tuple[str, type[WarehouseAdapter]]]
|
|
39
|
+
|
|
40
|
+
# The adapter's ``drop_semantics`` mapped to the engine family the release plan declares.
|
|
41
|
+
_FAMILY_OF_DROP_SEMANTICS = {"cascade": "schema_bound", "late_binding": "late_binding"}
|
|
42
|
+
_MODES = ("check", "report")
|
|
43
|
+
_FAMILIES = ("schema_bound", "late_binding")
|
|
44
|
+
|
|
45
|
+
# Longest driver text kept in a block; the first line only, so a block stays small.
|
|
46
|
+
_MAX_TEXT = 500
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Situation(Enum):
|
|
50
|
+
"""How far ``seed_preflight`` got, which is what decides its verdict."""
|
|
51
|
+
|
|
52
|
+
BAD_INPUT = "bad_input"
|
|
53
|
+
ENGINE_MISMATCH = "engine_mismatch"
|
|
54
|
+
NO_CAPABILITY = "no_capability"
|
|
55
|
+
UNREACHABLE = "unreachable"
|
|
56
|
+
LISTING_FAILED = "listing_failed"
|
|
57
|
+
LISTED = "listed"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class Verdict:
|
|
62
|
+
"""What ``seed_preflight`` concludes: the block's status, reason, message and the exit code."""
|
|
63
|
+
|
|
64
|
+
status: str
|
|
65
|
+
reason: str
|
|
66
|
+
exit_code: int
|
|
67
|
+
checked: bool
|
|
68
|
+
completeness: str
|
|
69
|
+
message: str
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def decide(
|
|
73
|
+
mode: str,
|
|
74
|
+
family: str,
|
|
75
|
+
situation: Situation,
|
|
76
|
+
*,
|
|
77
|
+
detail: str = "",
|
|
78
|
+
completeness: str = "none",
|
|
79
|
+
has_unowned: bool = False,
|
|
80
|
+
) -> Verdict:
|
|
81
|
+
"""The decision table of ``seed_preflight``.
|
|
82
|
+
|
|
83
|
+
*family* is the engine family the release expects (``schema_bound`` or
|
|
84
|
+
``late_binding``), *detail* the text that explains the situation, *completeness*
|
|
85
|
+
the adapter's own and *has_unowned* whether a listed object is outside the
|
|
86
|
+
allowed set. A situation that a ``check`` stops on gives ``error`` and exit 1;
|
|
87
|
+
``report`` mode gives the same ``reason`` on a ``success`` block and exit 0. A
|
|
88
|
+
bad input is exit 2 in both modes. A ``reason`` prefixes the message.
|
|
89
|
+
"""
|
|
90
|
+
if situation is Situation.BAD_INPUT:
|
|
91
|
+
return Verdict("error", "", 2, False, "none", detail)
|
|
92
|
+
reason, stops = _conclusion(family, situation, has_unowned)
|
|
93
|
+
failed = stops and mode == "check"
|
|
94
|
+
return Verdict(
|
|
95
|
+
status="error" if failed else "success",
|
|
96
|
+
reason=reason,
|
|
97
|
+
exit_code=1 if failed else 0,
|
|
98
|
+
checked=situation is Situation.LISTED,
|
|
99
|
+
completeness=completeness if situation is Situation.LISTED else "none",
|
|
100
|
+
message=f"{reason}: {detail}" if reason else detail,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _conclusion(family: str, situation: Situation, has_unowned: bool) -> tuple[str, bool]:
|
|
105
|
+
"""The ``reason`` a situation carries and whether a ``check`` stops on it."""
|
|
106
|
+
schema_bound = family == "schema_bound"
|
|
107
|
+
if situation is Situation.ENGINE_MISMATCH:
|
|
108
|
+
return "engine_mismatch", True
|
|
109
|
+
if situation is Situation.UNREACHABLE:
|
|
110
|
+
return "warehouse_unreachable", True
|
|
111
|
+
if situation is Situation.NO_CAPABILITY:
|
|
112
|
+
return ("unowned_dependents", True) if schema_bound else ("", False)
|
|
113
|
+
if situation is Situation.LISTING_FAILED:
|
|
114
|
+
return "", schema_bound
|
|
115
|
+
if schema_bound and has_unowned:
|
|
116
|
+
return "unowned_dependents", True
|
|
117
|
+
return "", False
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _InputError(Exception):
|
|
121
|
+
"""An input of the op is missing or malformed."""
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _first_line(exc: BaseException) -> str:
|
|
125
|
+
"""The first line of an exception's text, at most ``_MAX_TEXT`` characters, never empty."""
|
|
126
|
+
text = str(exc).strip()
|
|
127
|
+
line = text.splitlines()[0] if text else ""
|
|
128
|
+
return (line or type(exc).__name__)[:_MAX_TEXT]
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _json_list(env: Mapping[str, str], name: str, *, required: bool) -> list[str]:
|
|
132
|
+
"""The env var *name* as a JSON array of strings; unset or empty is ``[]`` unless *required*."""
|
|
133
|
+
raw = env.get(name, "")
|
|
134
|
+
if not raw.strip():
|
|
135
|
+
if required:
|
|
136
|
+
raise _InputError(f"missing required env var {name}")
|
|
137
|
+
return []
|
|
138
|
+
try:
|
|
139
|
+
value = json.loads(raw)
|
|
140
|
+
except json.JSONDecodeError as exc:
|
|
141
|
+
raise _InputError(f"{name} is not valid JSON: {exc}") from exc
|
|
142
|
+
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
143
|
+
raise _InputError(f"{name} must be a JSON array of strings")
|
|
144
|
+
return cast(list[str], value)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _emit(
|
|
148
|
+
status: str, message: str, unique_id: str, reason: str, fields: dict[str, Any]
|
|
149
|
+
) -> None:
|
|
150
|
+
"""Print the op's one result block, the last line on stdout."""
|
|
151
|
+
print(
|
|
152
|
+
result.result_block(
|
|
153
|
+
status, message, 1 if status == "error" else 0, unique_id, reason=reason, fields=fields
|
|
154
|
+
),
|
|
155
|
+
flush=True,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _discover_and_check_env(
|
|
160
|
+
env: Mapping[str, str], discover: _Discover
|
|
161
|
+
) -> tuple[str, type[WarehouseAdapter]]:
|
|
162
|
+
"""The installed adapter, after checking its required env is present in *env*."""
|
|
163
|
+
try:
|
|
164
|
+
engine, adapter_cls = discover()
|
|
165
|
+
except AdapterDiscoveryError as exc:
|
|
166
|
+
raise _InputError(str(exc)) from exc
|
|
167
|
+
missing = [name for name in adapter_cls.required_env() if not env.get(name)]
|
|
168
|
+
if missing:
|
|
169
|
+
raise _InputError(f"missing required env for engine {engine!r}: {', '.join(missing)}")
|
|
170
|
+
return engine, adapter_cls
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _close(adapter: Any) -> None:
|
|
174
|
+
"""Close the adapter; a failure only logs, so it never masks the op's outcome."""
|
|
175
|
+
try:
|
|
176
|
+
adapter.close()
|
|
177
|
+
except Exception as exc:
|
|
178
|
+
logger.error("adapter close failed: %s", exc)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
# --- seed_preflight ---------------------------------------------------------------
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@dataclass(frozen=True)
|
|
185
|
+
class _SeedInputs:
|
|
186
|
+
mode: str
|
|
187
|
+
family: str
|
|
188
|
+
schema: str
|
|
189
|
+
table: str
|
|
190
|
+
allowed: frozenset[str]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _seed_node_id(env: Mapping[str, str]) -> str:
|
|
194
|
+
"""``NODE_ID``, else ``<PROD_SCHEMA>.<TABLE_NAME>`` when both are set, else ``""``."""
|
|
195
|
+
schema, table = env.get("PROD_SCHEMA", ""), env.get("TABLE_NAME", "")
|
|
196
|
+
return env.get("NODE_ID") or (f"{schema}.{table}" if schema and table else "")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _seed_inputs(env: Mapping[str, str]) -> _SeedInputs:
|
|
200
|
+
mode = env.get("SEED_PREFLIGHT_MODE", "")
|
|
201
|
+
if mode not in _MODES:
|
|
202
|
+
raise _InputError(f"SEED_PREFLIGHT_MODE must be one of {', '.join(_MODES)}, got {mode!r}")
|
|
203
|
+
family = env.get("SEED_PREFLIGHT_ENGINE_FAMILY", "")
|
|
204
|
+
if family not in _FAMILIES:
|
|
205
|
+
raise _InputError(
|
|
206
|
+
f"SEED_PREFLIGHT_ENGINE_FAMILY must be one of {', '.join(_FAMILIES)}, got {family!r}"
|
|
207
|
+
)
|
|
208
|
+
for name in ("PROD_SCHEMA", "TABLE_NAME"):
|
|
209
|
+
if not env.get(name):
|
|
210
|
+
raise _InputError(f"missing required env var {name}")
|
|
211
|
+
allowed = _json_list(env, "SEED_PREFLIGHT_ALLOWED", required=False)
|
|
212
|
+
acknowledged = _json_list(env, "SEED_PREFLIGHT_ACKNOWLEDGED", required=False)
|
|
213
|
+
return _SeedInputs(
|
|
214
|
+
mode=mode,
|
|
215
|
+
family=family,
|
|
216
|
+
schema=env["PROD_SCHEMA"],
|
|
217
|
+
table=env["TABLE_NAME"],
|
|
218
|
+
allowed=frozenset(entry.strip().lower() for entry in [*allowed, *acknowledged]),
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _seed_fields(
|
|
223
|
+
env: Mapping[str, str], drop_semantics: str, verdict: Verdict, dependents: list[dict[str, Any]]
|
|
224
|
+
) -> dict[str, Any]:
|
|
225
|
+
return {
|
|
226
|
+
"op": "seed_preflight",
|
|
227
|
+
"mode": env.get("SEED_PREFLIGHT_MODE", ""),
|
|
228
|
+
"checked": verdict.checked,
|
|
229
|
+
"engine_family": env.get("SEED_PREFLIGHT_ENGINE_FAMILY", ""),
|
|
230
|
+
"drop_semantics": drop_semantics,
|
|
231
|
+
"completeness": verdict.completeness,
|
|
232
|
+
"dependents": dependents,
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _finish_seed(
|
|
237
|
+
env: Mapping[str, str], drop_semantics: str, verdict: Verdict,
|
|
238
|
+
dependents: list[dict[str, Any]] | None = None,
|
|
239
|
+
) -> int:
|
|
240
|
+
node_id = _seed_node_id(env)
|
|
241
|
+
if verdict.exit_code == 2:
|
|
242
|
+
logger.error("seed_preflight %s: %s", node_id, verdict.message)
|
|
243
|
+
elif verdict.reason or verdict.exit_code:
|
|
244
|
+
logger.warning("seed_preflight %s: %s", node_id, verdict.message)
|
|
245
|
+
_emit(
|
|
246
|
+
verdict.status, verdict.message, node_id, verdict.reason,
|
|
247
|
+
_seed_fields(env, drop_semantics, verdict, dependents or []),
|
|
248
|
+
)
|
|
249
|
+
return verdict.exit_code
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def run_seed_preflight(
|
|
253
|
+
env: Mapping[str, str], discover: _Discover = discover_adapter
|
|
254
|
+
) -> int:
|
|
255
|
+
"""Run ``seed_preflight`` against *env*; print its one block and return the exit code.
|
|
256
|
+
|
|
257
|
+
Reads the op's inputs and the adapter's ``required_env()`` from *env* (adapters
|
|
258
|
+
connect from ``os.environ``, so the runner passes that). An unexpected error
|
|
259
|
+
still prints one ``error`` block and returns 2.
|
|
260
|
+
"""
|
|
261
|
+
try:
|
|
262
|
+
return _seed_preflight(env, discover)
|
|
263
|
+
except Exception as exc:
|
|
264
|
+
logger.exception("seed_preflight %s failed unexpectedly", _seed_node_id(env))
|
|
265
|
+
verdict = Verdict("error", "", 2, False, "none", f"unexpected error: {_first_line(exc)}")
|
|
266
|
+
return _finish_seed(env, "", verdict)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _seed_preflight(env: Mapping[str, str], discover: _Discover) -> int:
|
|
270
|
+
try:
|
|
271
|
+
inputs = _seed_inputs(env)
|
|
272
|
+
engine, adapter_cls = _discover_and_check_env(env, discover)
|
|
273
|
+
except _InputError as exc:
|
|
274
|
+
return _finish_seed(env, "", decide("", "", Situation.BAD_INPUT, detail=str(exc)))
|
|
275
|
+
|
|
276
|
+
declared = getattr(adapter_cls, "drop_semantics", "")
|
|
277
|
+
drop_semantics = declared if isinstance(declared, str) else ""
|
|
278
|
+
adapter_family = _FAMILY_OF_DROP_SEMANTICS.get(drop_semantics)
|
|
279
|
+
|
|
280
|
+
def finish(
|
|
281
|
+
situation: Situation, detail: str = "", *, completeness: str = "none",
|
|
282
|
+
has_unowned: bool = False, dependents: list[dict[str, Any]] | None = None,
|
|
283
|
+
) -> int:
|
|
284
|
+
verdict = decide(
|
|
285
|
+
inputs.mode, inputs.family, situation,
|
|
286
|
+
detail=detail, completeness=completeness, has_unowned=has_unowned,
|
|
287
|
+
)
|
|
288
|
+
return _finish_seed(env, drop_semantics, verdict, dependents)
|
|
289
|
+
|
|
290
|
+
if adapter_family != inputs.family:
|
|
291
|
+
return finish(
|
|
292
|
+
Situation.ENGINE_MISMATCH,
|
|
293
|
+
f"engine {engine!r} is {adapter_family or 'of no known family'}, "
|
|
294
|
+
f"but the release expects {inputs.family}",
|
|
295
|
+
)
|
|
296
|
+
if not issubclass(adapter_cls, DependentsIntrospection):
|
|
297
|
+
return finish(Situation.NO_CAPABILITY, f"engine {engine!r} cannot list the objects that read a table")
|
|
298
|
+
|
|
299
|
+
try:
|
|
300
|
+
adapter = adapter_cls.from_env()
|
|
301
|
+
except Exception as exc:
|
|
302
|
+
return finish(Situation.UNREACHABLE, _first_line(exc))
|
|
303
|
+
try:
|
|
304
|
+
try:
|
|
305
|
+
listing = cast(DependentsIntrospection, adapter).dependents(inputs.schema, inputs.table)
|
|
306
|
+
except Exception as exc:
|
|
307
|
+
return finish(
|
|
308
|
+
Situation.LISTING_FAILED,
|
|
309
|
+
f"listing the objects that read {inputs.schema}.{inputs.table} failed: {_first_line(exc)}",
|
|
310
|
+
)
|
|
311
|
+
dependents = [
|
|
312
|
+
{
|
|
313
|
+
"schema": found.schema, "name": found.name, "kind": found.kind, "owner": found.owner,
|
|
314
|
+
"depth": found.depth,
|
|
315
|
+
"allowed": f"{found.schema}.{found.name}".lower() in inputs.allowed,
|
|
316
|
+
}
|
|
317
|
+
for found in sorted(listing.objects, key=lambda d: (d.depth, d.schema, d.name))
|
|
318
|
+
]
|
|
319
|
+
finally:
|
|
320
|
+
_close(adapter)
|
|
321
|
+
outside = [d for d in dependents if not d["allowed"]]
|
|
322
|
+
if inputs.family == "schema_bound":
|
|
323
|
+
detail = "the reload would drop objects continuo does not own: " + ", ".join(
|
|
324
|
+
f"{d['schema']}.{d['name']} ({d['kind']}, owner {d['owner']})" for d in outside
|
|
325
|
+
)
|
|
326
|
+
else:
|
|
327
|
+
detail = (
|
|
328
|
+
f"{len(outside)} external view(s) may go stale; recreate them after the reload: "
|
|
329
|
+
+ ", ".join(f"{d['schema']}.{d['name']}" for d in outside)
|
|
330
|
+
)
|
|
331
|
+
return finish(
|
|
332
|
+
Situation.LISTED, detail if outside else "",
|
|
333
|
+
completeness=listing.completeness, has_unowned=bool(outside), dependents=dependents,
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# --- probe_views ------------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _probe_views_input(env: Mapping[str, str]) -> list[tuple[str, str]]:
|
|
341
|
+
"""``PROBE_VIEWS`` as ``(schema, name)`` pairs, split on the first dot, each once, in order."""
|
|
342
|
+
pairs: list[tuple[str, str]] = []
|
|
343
|
+
for entry in _json_list(env, "PROBE_VIEWS", required=True):
|
|
344
|
+
schema, dot, name = entry.partition(".")
|
|
345
|
+
if not dot or not schema or not name:
|
|
346
|
+
raise _InputError(f"PROBE_VIEWS entry {entry!r} is not <schema>.<name>")
|
|
347
|
+
if (schema, name) not in pairs:
|
|
348
|
+
pairs.append((schema, name))
|
|
349
|
+
return pairs
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _quote(identifier: str) -> str:
|
|
353
|
+
return '"' + identifier.replace('"', '""') + '"'
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _finish_probe(
|
|
357
|
+
env: Mapping[str, str], status: str, message: str, reason: str, checked: bool,
|
|
358
|
+
views: list[dict[str, Any]],
|
|
359
|
+
) -> int:
|
|
360
|
+
node_id = env.get("NODE_ID") or "probe_views"
|
|
361
|
+
if status == "error":
|
|
362
|
+
logger.error("probe_views %s: %s", node_id, message)
|
|
363
|
+
elif reason:
|
|
364
|
+
logger.warning("probe_views %s: %s", node_id, message)
|
|
365
|
+
_emit(status, message, node_id, reason, {"op": "probe_views", "checked": checked, "views": views})
|
|
366
|
+
return 2 if status == "error" else 0
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def run_probe_views(env: Mapping[str, str], discover: _Discover = discover_adapter) -> int:
|
|
370
|
+
"""Run ``probe_views`` against *env*; print its one block and return the exit code.
|
|
371
|
+
|
|
372
|
+
Reads ``PROBE_VIEWS`` and the adapter's ``required_env()`` from *env*. A view
|
|
373
|
+
that fails its probe, or a warehouse that cannot be reached, still ends with
|
|
374
|
+
exit 0; only an input or adapter problem or an unexpected error is exit 2.
|
|
375
|
+
"""
|
|
376
|
+
try:
|
|
377
|
+
return _probe_views(env, discover)
|
|
378
|
+
except Exception as exc:
|
|
379
|
+
logger.exception("probe_views failed unexpectedly")
|
|
380
|
+
return _finish_probe(env, "error", f"unexpected error: {_first_line(exc)}", "", False, [])
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _probe_views(env: Mapping[str, str], discover: _Discover) -> int:
|
|
384
|
+
try:
|
|
385
|
+
pairs = _probe_views_input(env)
|
|
386
|
+
_, adapter_cls = _discover_and_check_env(env, discover)
|
|
387
|
+
except _InputError as exc:
|
|
388
|
+
return _finish_probe(env, "error", str(exc), "", False, [])
|
|
389
|
+
try:
|
|
390
|
+
adapter = adapter_cls.from_env()
|
|
391
|
+
except Exception as exc:
|
|
392
|
+
return _finish_probe(
|
|
393
|
+
env, "success", f"warehouse_unreachable: {_first_line(exc)}", "warehouse_unreachable", False, []
|
|
394
|
+
)
|
|
395
|
+
views: list[dict[str, Any]] = []
|
|
396
|
+
try:
|
|
397
|
+
for schema, name in pairs:
|
|
398
|
+
error = ""
|
|
399
|
+
try:
|
|
400
|
+
adapter.fetch(f"SELECT * FROM {_quote(schema)}.{_quote(name)} LIMIT 0")
|
|
401
|
+
except Exception as exc:
|
|
402
|
+
error = _first_line(exc)
|
|
403
|
+
views.append({"schema": schema, "name": name, "ok": not error, "error": error})
|
|
404
|
+
finally:
|
|
405
|
+
_close(adapter)
|
|
406
|
+
broken = [f"{v['schema']}.{v['name']}" for v in views if not v["ok"]]
|
|
407
|
+
for name in broken:
|
|
408
|
+
logger.warning("probe_views: %s does not bind", name)
|
|
409
|
+
message = f"{len(broken)} of {len(views)} views failed: {', '.join(broken)}" if broken else ""
|
|
410
|
+
return _finish_probe(env, "success", message, "", True, views)
|
|
@@ -23,6 +23,12 @@ Dispatches on ``VALIDATION_OP`` env var (default ``build_from_sql``):
|
|
|
23
23
|
via the engine adapter and creates nothing — a test whose compiled SQL names
|
|
24
24
|
a column a candidate change dropped or renamed fails the release at its
|
|
25
25
|
source, before any table is built.
|
|
26
|
+
- ``seed_preflight``: before a seed's production table is reloaded, list the objects
|
|
27
|
+
(``PROD_SCHEMA``.``TABLE_NAME``) that read it and compare them with the operator's
|
|
28
|
+
allowed and acknowledged lists. Needs no ``DBT_TARGET_SCHEMA`` and creates nothing.
|
|
29
|
+
- ``probe_views``: try each view of ``PROBE_VIEWS`` with a zero-row read and report
|
|
30
|
+
which ones no longer bind. Needs no ``DBT_TARGET_SCHEMA`` and creates nothing.
|
|
31
|
+
The inputs and the result block of both ops are in ``preflight``.
|
|
26
32
|
|
|
27
33
|
The engine adapter is discovered from the single installed
|
|
28
34
|
``continuo_engine.adapters`` entry point — each runner image installs exactly one.
|
|
@@ -43,7 +49,7 @@ from continuo_engine_contract.port import ( # type: ignore[import-untyped]
|
|
|
43
49
|
)
|
|
44
50
|
from continuo_python_runtime.csv_readers import reader_for
|
|
45
51
|
from continuo_python_runtime.csv_source import check_header, parse_csv_uri
|
|
46
|
-
from continuo_python_runtime.validation import s3
|
|
52
|
+
from continuo_python_runtime.validation import preflight, s3
|
|
47
53
|
|
|
48
54
|
logger = logging.getLogger("validation_runner")
|
|
49
55
|
|
|
@@ -115,6 +121,7 @@ def load_candidate_spec() -> dict:
|
|
|
115
121
|
|
|
116
122
|
_NODE_OPS = ("build_from_sql", "clone_from_prod", "build_from_columns", "check_binds")
|
|
117
123
|
_SCHEMA_OPS = ("ensure_schema", "drop_schema")
|
|
124
|
+
_PREFLIGHT_OPS = ("seed_preflight", "probe_views")
|
|
118
125
|
|
|
119
126
|
|
|
120
127
|
def main() -> None:
|
|
@@ -128,13 +135,23 @@ def main() -> None:
|
|
|
128
135
|
Schema ops (``ensure_schema``/``drop_schema``) act on the whole candidate
|
|
129
136
|
schema — the executor schedules them as one-shot engine-image Jobs to own
|
|
130
137
|
the candidate-schema lifecycle without connecting to the warehouse itself —
|
|
131
|
-
and take no table.
|
|
138
|
+
and take no table. The preflight ops (``seed_preflight``/``probe_views``) read
|
|
139
|
+
their own inputs, need no candidate schema and exit with the code ``preflight``
|
|
140
|
+
returns.
|
|
132
141
|
"""
|
|
133
142
|
logging.basicConfig(
|
|
134
143
|
level=logging.INFO, stream=sys.stderr, format="%(levelname)s %(name)s: %(message)s"
|
|
135
144
|
)
|
|
136
|
-
schema = _require("DBT_TARGET_SCHEMA")
|
|
137
145
|
op = os.environ.get("VALIDATION_OP", "build_from_sql")
|
|
146
|
+
if op in _PREFLIGHT_OPS:
|
|
147
|
+
if op == "seed_preflight":
|
|
148
|
+
code = preflight.run_seed_preflight(os.environ)
|
|
149
|
+
else:
|
|
150
|
+
code = preflight.run_probe_views(os.environ)
|
|
151
|
+
if code:
|
|
152
|
+
sys.exit(code)
|
|
153
|
+
return
|
|
154
|
+
schema = _require("DBT_TARGET_SCHEMA")
|
|
138
155
|
|
|
139
156
|
# Gather op-specific inputs and identity BEFORE touching the adapter, surfacing
|
|
140
157
|
# input errors as a structured block (preserves the prior contract + exit codes).
|
{continuo_python_runtime-0.11.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: continuo-python-runtime
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.12.0
|
|
4
4
|
Summary: Runtime harness, contract tooling, and CI lint for Continuo python nodes.
|
|
5
5
|
Author: Simone Carolini
|
|
6
6
|
Maintainer: Simone Carolini
|
|
@@ -12,7 +12,8 @@ Classifier: Intended Audience :: Developers
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.14
|
|
13
13
|
Requires-Python: >=3.14
|
|
14
14
|
Requires-Dist: boto3==1.43.114
|
|
15
|
-
Requires-Dist: continuo-
|
|
15
|
+
Requires-Dist: continuo-contracts==0.1.0
|
|
16
|
+
Requires-Dist: continuo-engine-contract==0.10.0
|
|
16
17
|
Requires-Dist: pyarrow==26.0.0
|
|
17
18
|
Requires-Dist: pyyaml==6.0.3
|
|
18
19
|
Requires-Dist: sqlglot==30.23.0
|
|
@@ -112,7 +113,12 @@ on every change.
|
|
|
112
113
|
last line of stdout, and Continuo's Go side parses it byte-for-byte
|
|
113
114
|
(`pkg/validationresult` in the `continuo` repository). That wire format — the
|
|
114
115
|
sentinel markers, the framing, and the field names inside the block — is
|
|
115
|
-
**frozen**.
|
|
116
|
+
**frozen**. The additions are an optional `reason` key, written after
|
|
117
|
+
`unique_id` only when an outcome has a machine-readable code (for example
|
|
118
|
+
`image_contract_mismatch`), and the op's own keys that the `seed_preflight` and
|
|
119
|
+
`probe_views` ops write after `reason`; a block without either keeps its bytes,
|
|
120
|
+
and the sentinels, `schema_version` and the five standard keys do not change. Reuse
|
|
121
|
+
it; never change it from this side alone. A change here that
|
|
116
122
|
the Go parser has not been taught is a production outage, not a refactor.
|
|
117
123
|
|
|
118
124
|
## Quickstart for domain teams
|
|
@@ -196,14 +202,37 @@ produced, never how it is checked or written.
|
|
|
196
202
|
script reads with `os.environ[...]`. The contract never holds a secret value
|
|
197
203
|
or env var name; the Secret itself is created in the cluster by the operator.
|
|
198
204
|
|
|
205
|
+
### Files a script reads (`assets`)
|
|
206
|
+
|
|
207
|
+
A `python-node` or `python-api` script that reads a file from the repository
|
|
208
|
+
(a model, a lookup table, a JSON config) lists it under `assets:`, one path
|
|
209
|
+
relative to the repository root per entry, written with forward slashes and
|
|
210
|
+
without a `..` segment:
|
|
211
|
+
|
|
212
|
+
```yaml
|
|
213
|
+
- schema: analytics
|
|
214
|
+
table: scored_orders
|
|
215
|
+
kind: python-node
|
|
216
|
+
script: scripts/scored_orders.py
|
|
217
|
+
assets: [data/rates.json, models/scorer.bin]
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
`merge` reads each listed file and folds its bytes into `shared_code_hash`
|
|
221
|
+
under its path, so editing one changes the node's `content_hash` and starts a
|
|
222
|
+
validation. A path that is also an import of the script is one unit. `merge`
|
|
223
|
+
fails with a `ContractError` for a path that is missing, is not a file or
|
|
224
|
+
resolves outside the repository, and a path listed twice is refused. The image
|
|
225
|
+
must `COPY` the files, because `assets` only tells the hash about them. A
|
|
226
|
+
`python-csv` node has no script, so `assets` is refused on it.
|
|
227
|
+
|
|
199
228
|
### Task timeout
|
|
200
229
|
|
|
201
230
|
Any node may set `timeout_minutes:` — an integer from 1 to 10,080 (one week) —
|
|
202
231
|
to declare how long its task may run before continuo gives up on it. A value
|
|
203
232
|
outside that range, or one that is not an integer, fails
|
|
204
|
-
`continuo-runtime validate`/`merge`/`lint`.
|
|
205
|
-
|
|
206
|
-
`content_hash`.
|
|
233
|
+
`continuo-runtime validate`/`merge`/`lint`. Like `owner`, `schedule`,
|
|
234
|
+
`criticality` and `description`, it is outside the code projection that
|
|
235
|
+
`config_hash` hashes, so changing it never changes the node's `content_hash`.
|
|
207
236
|
|
|
208
237
|
## Write modes
|
|
209
238
|
|
|
@@ -282,7 +311,7 @@ def run(ctx):
|
|
|
282
311
|
example a polars DataFrame. Returning anything else raises `ScriptError`.
|
|
283
312
|
- The dataframe library is your choice. No dataframe library is baked into
|
|
284
313
|
the base image — the package's own runtime dependencies are `pyarrow`,
|
|
285
|
-
`PyYAML`,
|
|
314
|
+
`PyYAML`, `continuo-engine-contract` and `continuo-contracts`. Add whatever you script
|
|
286
315
|
against (pandas, polars, …) as a `RUN pip install` line in your own
|
|
287
316
|
`Dockerfile`, on top of the base image. The base image runs as non-root
|
|
288
317
|
(uid 65532), which pip cannot install under, so switch to root for the
|
|
@@ -298,7 +327,18 @@ def run(ctx):
|
|
|
298
327
|
read_sql as rs` then calling `rs(...)`),
|
|
299
328
|
- private/protected attribute access (`obj._x`) — except on `self`/`cls`,
|
|
300
329
|
so a script's own class-private helpers (`self._helper()`) aren't
|
|
301
|
-
flagged
|
|
330
|
+
flagged,
|
|
331
|
+
- code the content hash cannot see. The hash folds the script and the files
|
|
332
|
+
it imports, found by reading imports, so a node that runs anything else
|
|
333
|
+
can change without its `content_hash` changing. Lint refuses `runpy`, a
|
|
334
|
+
change to `sys.path` (`append`, `insert`, assignment, `del`, and
|
|
335
|
+
`site.addsitedir`), a `sitecustomize` or `usercustomize` module (and a file
|
|
336
|
+
of that name), `compile()` and `types.FunctionType`, and `pickle.load`,
|
|
337
|
+
`pickle.loads` and `pickle.Unpickler`. `merge` applies the same rules to
|
|
338
|
+
every file in a node's import closure, and refuses a `sitecustomize.py` or
|
|
339
|
+
`usercustomize.py` at the repository root, which Python imports before the
|
|
340
|
+
harness starts. A file a script reads, such as a model or a lookup table,
|
|
341
|
+
belongs in the node's `assets`, where its bytes are hashed.
|
|
302
342
|
|
|
303
343
|
The SQL-literal rule is best-effort: docstrings are exempt, but other
|
|
304
344
|
prose may still occasionally match. The hard guarantees enforced by lint
|
|
@@ -337,7 +377,7 @@ declared `output_columns` on the table `run()` returned, in this order:
|
|
|
337
377
|
|
|
338
378
|
## Error taxonomy
|
|
339
379
|
|
|
340
|
-
Every runtime failure is one of
|
|
380
|
+
Every runtime failure is one of six `HarnessError` subclasses
|
|
341
381
|
(`continuo_python_runtime/errors.py`). The sentinel result block's message
|
|
342
382
|
is prefixed `<ErrorClass>: ` so failures can be triaged without parsing free
|
|
343
383
|
text. The harness also logs the same message at ERROR on stderr, with the
|
|
@@ -348,6 +388,7 @@ the result block out of the log before storing it.
|
|
|
348
388
|
| Class | Meaning | Typical fix target |
|
|
349
389
|
| --- | --- | --- |
|
|
350
390
|
| `ContractError` | Contract missing or invalid, node not found for `NODE_ID`, or the declared script is missing/unreachable. | The contract yaml or the `script:` path. |
|
|
391
|
+
| `ImageContractMismatch` (a `ContractError`) | The image does not hold the contract, script, imports or assets the release hashed: the `content_hash` in `CONTENT_HASH` differs from the one the image's files give, or a file the release hashed is missing from the image. The block also carries `"reason":"image_contract_mismatch"`. | Rebuild the image from the released commit, with CI and the image on the same runtime. |
|
|
351
392
|
| `ReadError` | `ctx.read()` was called with an undeclared name, or a declared read failed at the warehouse. | The `reads:` map, or the upstream query/warehouse access. |
|
|
352
393
|
| `ScriptError` | `run()` raised, has no callable `run`, or returned a value that isn't Arrow-convertible. | The node script. |
|
|
353
394
|
| `ConformError` | Structural mismatch (extra/missing/duplicate columns), a strict-cast failure, a not-null violation, a VARCHAR/CHAR overflow, or an `upsert` run whose rows repeat a primary key. | The script's output shape, or the `output_columns` declaration. |
|
|
@@ -359,11 +400,11 @@ A domain repo picks its warehouse engine by which base image it builds
|
|
|
359
400
|
`FROM`:
|
|
360
401
|
|
|
361
402
|
```dockerfile
|
|
362
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.
|
|
403
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-postgres:v0.12.0
|
|
363
404
|
# or
|
|
364
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.
|
|
405
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-trino:v0.12.0
|
|
365
406
|
# or
|
|
366
|
-
FROM ghcr.io/carolsimone/continuo-python-runtime-duckdb:v0.
|
|
407
|
+
FROM ghcr.io/carolsimone/continuo-python-runtime-duckdb:v0.12.0
|
|
367
408
|
```
|
|
368
409
|
|
|
369
410
|
The engine is part of the image **name**; the tag is the bare version, so
|
|
@@ -402,6 +443,82 @@ Both shapes end up running the same runtime and the same adapter version;
|
|
|
402
443
|
which one you pick only changes who controls the base OS layer underneath
|
|
403
444
|
them.
|
|
404
445
|
|
|
446
|
+
## Seed preflight and view probe
|
|
447
|
+
|
|
448
|
+
Two validation ops read the warehouse and change nothing. They run through the
|
|
449
|
+
same runner image as the other validation ops (`VALIDATION_OP`), need no
|
|
450
|
+
`DBT_TARGET_SCHEMA`, and print one result block as the last line of stdout.
|
|
451
|
+
|
|
452
|
+
A seed reload drops and recreates the seed's production table. On an engine where
|
|
453
|
+
a drop destroys the objects that read the table (`drop_semantics` `cascade`,
|
|
454
|
+
postgres) that takes every view, function and constraint built on the seed with
|
|
455
|
+
it, including objects continuo does not own. On an engine where those objects
|
|
456
|
+
survive and fail on their next read (`late_binding`, trino and duckdb) they
|
|
457
|
+
may go stale. `seed_preflight` lists what a reload would touch and compares it
|
|
458
|
+
with what the operator has allowed or acknowledged; `probe_views` tries named
|
|
459
|
+
views after a reload and says which no longer bind.
|
|
460
|
+
|
|
461
|
+
`seed_preflight` runs in two places. `check` runs in the seed task's own pod just
|
|
462
|
+
before dbt and stops the reload with exit 1 when the check concludes it must.
|
|
463
|
+
`report` runs in the release's seed-build Job and only reports what a `check`
|
|
464
|
+
would conclude; it exits 0 whatever it finds.
|
|
465
|
+
|
|
466
|
+
### `seed_preflight` inputs
|
|
467
|
+
|
|
468
|
+
| Variable | Meaning |
|
|
469
|
+
|---|---|
|
|
470
|
+
| `SEED_PREFLIGHT_MODE` | `check` or `report`. |
|
|
471
|
+
| `SEED_PREFLIGHT_ENGINE_FAMILY` | `schema_bound` or `late_binding`, the family the release plan expects of the engine. |
|
|
472
|
+
| `PROD_SCHEMA`, `TABLE_NAME` | The seed's production relation. |
|
|
473
|
+
| `SEED_PREFLIGHT_ALLOWED`, `SEED_PREFLIGHT_ACKNOWLEDGED` | JSON arrays of `"<schema>.<name>"`, default `[]`. An object is allowed when `<schema>.<name>`, in lower case, is in either list (entries are trimmed and lower-cased). |
|
|
474
|
+
| `NODE_ID` | Optional; the block's `unique_id`. Default `<PROD_SCHEMA>.<TABLE_NAME>`. |
|
|
475
|
+
|
|
476
|
+
The adapter's own variables (`POSTGRES_*`, `TRINO_*`, ...) are needed as for any
|
|
477
|
+
other op. An adapter's family is `schema_bound` when its `drop_semantics` is
|
|
478
|
+
`cascade`, and `late_binding` when it is `late_binding`.
|
|
479
|
+
|
|
480
|
+
### `seed_preflight` block
|
|
481
|
+
|
|
482
|
+
After the standard keys (and `reason` when one applies): `op` (`seed_preflight`),
|
|
483
|
+
`mode`, `checked` (the warehouse was asked and its listing came back), `engine_family` (as given),
|
|
484
|
+
`drop_semantics` (the adapter's), `completeness` (`exact`, `best_effort`, or
|
|
485
|
+
`none` when nothing was listed) and `dependents`, a list of
|
|
486
|
+
`{schema, name, kind, owner, depth, allowed}` sorted by depth, schema and name.
|
|
487
|
+
`status` is `error` for exit codes 1 and 2, and `failures` is 1 then.
|
|
488
|
+
|
|
489
|
+
`reason` is what a `check` concludes. `report` prints the same `reason` on a
|
|
490
|
+
`success` block.
|
|
491
|
+
|
|
492
|
+
| Situation | `reason` | `check` | `report` |
|
|
493
|
+
|---|---|---|---|
|
|
494
|
+
| a missing or malformed input, no adapter, missing adapter env | none | exit 2 | exit 2 |
|
|
495
|
+
| the adapter's family differs from `SEED_PREFLIGHT_ENGINE_FAMILY`, or it declares none | `engine_mismatch` | exit 1 | exit 0 |
|
|
496
|
+
| the adapter cannot list dependents, `schema_bound` | `unowned_dependents` | exit 1 | exit 0 |
|
|
497
|
+
| the adapter cannot list dependents, `late_binding` | none | exit 0 | exit 0 |
|
|
498
|
+
| the connection fails | `warehouse_unreachable` | exit 1 | exit 0 |
|
|
499
|
+
| the listing fails, `schema_bound` | none | exit 1 | exit 0 |
|
|
500
|
+
| the listing fails, `late_binding` | none | exit 0 | exit 0 |
|
|
501
|
+
| listed, `schema_bound`, an object is not allowed | `unowned_dependents` | exit 1 | exit 0 |
|
|
502
|
+
| anything else listed (on `late_binding` the objects not allowed are external views that may go stale, named in `message`) | none | exit 0 | exit 0 |
|
|
503
|
+
|
|
504
|
+
A seed table that does not exist yet has no dependents and passes. A stop's
|
|
505
|
+
`message` starts with its `reason` and names each object outside the set as
|
|
506
|
+
`<schema>.<name> (<kind>, owner <owner>)`.
|
|
507
|
+
|
|
508
|
+
### `probe_views`
|
|
509
|
+
|
|
510
|
+
`PROBE_VIEWS` is a JSON array of `"<schema>.<name>"`, split on the first dot;
|
|
511
|
+
`NODE_ID` is optional (default `probe_views`). Each view is read with
|
|
512
|
+
`SELECT * FROM "<schema>"."<name>" LIMIT 0`. The block has `op`
|
|
513
|
+
(`probe_views`), `checked` (the warehouse was reachable) and `views`, a list of
|
|
514
|
+
`{schema, name, ok, error}` in the order given, where `error` is the first line
|
|
515
|
+
of the failure, at most 500 characters, and empty when `ok`. A view that fails
|
|
516
|
+
its probe never fails the op: the exit code is 0, and `message` reads
|
|
517
|
+
`<n> of <total> views failed: <names>`. A warehouse that cannot be reached gives
|
|
518
|
+
exit 0, `checked` false, `views` empty and `reason` `warehouse_unreachable`.
|
|
519
|
+
Only a missing or malformed input, no adapter, missing adapter env or an
|
|
520
|
+
unexpected error is exit 2.
|
|
521
|
+
|
|
405
522
|
## Further reading
|
|
406
523
|
|
|
407
524
|
- `docs/superpowers/specs/2026-07-31-python-runtime-design.md` — this
|