continuo-python-runtime 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- continuo_python_runtime/cli.py +6 -0
- continuo_python_runtime/content_check.py +73 -0
- continuo_python_runtime/contract/kinds.py +10 -2
- continuo_python_runtime/contract/loader.py +52 -2
- continuo_python_runtime/contract/merge.py +26 -45
- continuo_python_runtime/contract/model.py +1 -0
- continuo_python_runtime/contract/paths.py +33 -0
- continuo_python_runtime/errors.py +13 -1
- continuo_python_runtime/harness.py +15 -1
- continuo_python_runtime/hash_inputs.py +63 -0
- continuo_python_runtime/hashing.py +69 -70
- continuo_python_runtime/lint.py +194 -6
- continuo_python_runtime/validation/preflight.py +410 -0
- continuo_python_runtime/validation/runner.py +20 -3
- {continuo_python_runtime-0.10.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/METADATA +136 -16
- continuo_python_runtime-0.12.0.dist-info/RECORD +35 -0
- {continuo_python_runtime-0.10.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/WHEEL +1 -1
- continuo_python_runtime-0.10.0.dist-info/RECORD +0 -32
- {continuo_python_runtime-0.10.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/entry_points.txt +0 -0
- {continuo_python_runtime-0.10.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/licenses/LICENSE +0 -0
- {continuo_python_runtime-0.10.0.dist-info → continuo_python_runtime-0.12.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
"""The ``seed_preflight`` and ``probe_views`` validation ops.
|
|
2
|
+
|
|
3
|
+
``seed_preflight`` asks the warehouse which objects a reload of a seed's
|
|
4
|
+
production table would destroy or leave reading a missing table, and compares
|
|
5
|
+
them with the objects the operator has allowed or acknowledged. ``check`` stops
|
|
6
|
+
a reload that would hit an object outside that set; ``report`` prints the same
|
|
7
|
+
conclusion and never stops. ``probe_views`` tries each named view with a
|
|
8
|
+
zero-row read and reports which ones no longer bind, and never fails the op for
|
|
9
|
+
it.
|
|
10
|
+
|
|
11
|
+
Each op prints exactly one structured result block as the last line on stdout
|
|
12
|
+
(``continuo_engine_contract.result``) and returns the process exit code: 0 when
|
|
13
|
+
the op ran to its conclusion (or ``report`` mode), 1 when ``check`` stops, 2
|
|
14
|
+
when the op could not run (a bad input, no adapter, missing adapter env, an
|
|
15
|
+
unexpected error). The block's ``status`` is ``error`` for exit codes 1 and 2.
|
|
16
|
+
Diagnostics go to stderr through ``logging``; stdout carries only the block.
|
|
17
|
+
|
|
18
|
+
The decision of ``seed_preflight`` is the pure function :func:`decide`; the rest
|
|
19
|
+
of the module reads the inputs, drives the adapter and prints.
|
|
20
|
+
"""
|
|
21
|
+
import json
|
|
22
|
+
import logging
|
|
23
|
+
from collections.abc import Callable, Mapping
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from enum import Enum
|
|
26
|
+
from typing import Any, cast
|
|
27
|
+
|
|
28
|
+
from continuo_engine_contract import result # type: ignore[import-untyped]
|
|
29
|
+
from continuo_engine_contract.port import ( # type: ignore[import-untyped]
|
|
30
|
+
AdapterDiscoveryError,
|
|
31
|
+
DependentsIntrospection,
|
|
32
|
+
WarehouseAdapter,
|
|
33
|
+
discover_adapter,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger("validation_preflight")
|
|
37
|
+
|
|
38
|
+
_Discover = Callable[[], tuple[str, type[WarehouseAdapter]]]
|
|
39
|
+
|
|
40
|
+
# The adapter's ``drop_semantics`` mapped to the engine family the release plan declares.
|
|
41
|
+
_FAMILY_OF_DROP_SEMANTICS = {"cascade": "schema_bound", "late_binding": "late_binding"}
|
|
42
|
+
_MODES = ("check", "report")
|
|
43
|
+
_FAMILIES = ("schema_bound", "late_binding")
|
|
44
|
+
|
|
45
|
+
# Longest driver text kept in a block; the first line only, so a block stays small.
|
|
46
|
+
_MAX_TEXT = 500
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Situation(Enum):
|
|
50
|
+
"""How far ``seed_preflight`` got, which is what decides its verdict."""
|
|
51
|
+
|
|
52
|
+
BAD_INPUT = "bad_input"
|
|
53
|
+
ENGINE_MISMATCH = "engine_mismatch"
|
|
54
|
+
NO_CAPABILITY = "no_capability"
|
|
55
|
+
UNREACHABLE = "unreachable"
|
|
56
|
+
LISTING_FAILED = "listing_failed"
|
|
57
|
+
LISTED = "listed"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class Verdict:
|
|
62
|
+
"""What ``seed_preflight`` concludes: the block's status, reason, message and the exit code."""
|
|
63
|
+
|
|
64
|
+
status: str
|
|
65
|
+
reason: str
|
|
66
|
+
exit_code: int
|
|
67
|
+
checked: bool
|
|
68
|
+
completeness: str
|
|
69
|
+
message: str
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def decide(
|
|
73
|
+
mode: str,
|
|
74
|
+
family: str,
|
|
75
|
+
situation: Situation,
|
|
76
|
+
*,
|
|
77
|
+
detail: str = "",
|
|
78
|
+
completeness: str = "none",
|
|
79
|
+
has_unowned: bool = False,
|
|
80
|
+
) -> Verdict:
|
|
81
|
+
"""The decision table of ``seed_preflight``.
|
|
82
|
+
|
|
83
|
+
*family* is the engine family the release expects (``schema_bound`` or
|
|
84
|
+
``late_binding``), *detail* the text that explains the situation, *completeness*
|
|
85
|
+
the adapter's own and *has_unowned* whether a listed object is outside the
|
|
86
|
+
allowed set. A situation that a ``check`` stops on gives ``error`` and exit 1;
|
|
87
|
+
``report`` mode gives the same ``reason`` on a ``success`` block and exit 0. A
|
|
88
|
+
bad input is exit 2 in both modes. A ``reason`` prefixes the message.
|
|
89
|
+
"""
|
|
90
|
+
if situation is Situation.BAD_INPUT:
|
|
91
|
+
return Verdict("error", "", 2, False, "none", detail)
|
|
92
|
+
reason, stops = _conclusion(family, situation, has_unowned)
|
|
93
|
+
failed = stops and mode == "check"
|
|
94
|
+
return Verdict(
|
|
95
|
+
status="error" if failed else "success",
|
|
96
|
+
reason=reason,
|
|
97
|
+
exit_code=1 if failed else 0,
|
|
98
|
+
checked=situation is Situation.LISTED,
|
|
99
|
+
completeness=completeness if situation is Situation.LISTED else "none",
|
|
100
|
+
message=f"{reason}: {detail}" if reason else detail,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _conclusion(family: str, situation: Situation, has_unowned: bool) -> tuple[str, bool]:
|
|
105
|
+
"""The ``reason`` a situation carries and whether a ``check`` stops on it."""
|
|
106
|
+
schema_bound = family == "schema_bound"
|
|
107
|
+
if situation is Situation.ENGINE_MISMATCH:
|
|
108
|
+
return "engine_mismatch", True
|
|
109
|
+
if situation is Situation.UNREACHABLE:
|
|
110
|
+
return "warehouse_unreachable", True
|
|
111
|
+
if situation is Situation.NO_CAPABILITY:
|
|
112
|
+
return ("unowned_dependents", True) if schema_bound else ("", False)
|
|
113
|
+
if situation is Situation.LISTING_FAILED:
|
|
114
|
+
return "", schema_bound
|
|
115
|
+
if schema_bound and has_unowned:
|
|
116
|
+
return "unowned_dependents", True
|
|
117
|
+
return "", False
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class _InputError(Exception):
|
|
121
|
+
"""An input of the op is missing or malformed."""
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _first_line(exc: BaseException) -> str:
|
|
125
|
+
"""The first line of an exception's text, at most ``_MAX_TEXT`` characters, never empty."""
|
|
126
|
+
text = str(exc).strip()
|
|
127
|
+
line = text.splitlines()[0] if text else ""
|
|
128
|
+
return (line or type(exc).__name__)[:_MAX_TEXT]
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _json_list(env: Mapping[str, str], name: str, *, required: bool) -> list[str]:
|
|
132
|
+
"""The env var *name* as a JSON array of strings; unset or empty is ``[]`` unless *required*."""
|
|
133
|
+
raw = env.get(name, "")
|
|
134
|
+
if not raw.strip():
|
|
135
|
+
if required:
|
|
136
|
+
raise _InputError(f"missing required env var {name}")
|
|
137
|
+
return []
|
|
138
|
+
try:
|
|
139
|
+
value = json.loads(raw)
|
|
140
|
+
except json.JSONDecodeError as exc:
|
|
141
|
+
raise _InputError(f"{name} is not valid JSON: {exc}") from exc
|
|
142
|
+
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
143
|
+
raise _InputError(f"{name} must be a JSON array of strings")
|
|
144
|
+
return cast(list[str], value)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _emit(
|
|
148
|
+
status: str, message: str, unique_id: str, reason: str, fields: dict[str, Any]
|
|
149
|
+
) -> None:
|
|
150
|
+
"""Print the op's one result block, the last line on stdout."""
|
|
151
|
+
print(
|
|
152
|
+
result.result_block(
|
|
153
|
+
status, message, 1 if status == "error" else 0, unique_id, reason=reason, fields=fields
|
|
154
|
+
),
|
|
155
|
+
flush=True,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _discover_and_check_env(
|
|
160
|
+
env: Mapping[str, str], discover: _Discover
|
|
161
|
+
) -> tuple[str, type[WarehouseAdapter]]:
|
|
162
|
+
"""The installed adapter, after checking its required env is present in *env*."""
|
|
163
|
+
try:
|
|
164
|
+
engine, adapter_cls = discover()
|
|
165
|
+
except AdapterDiscoveryError as exc:
|
|
166
|
+
raise _InputError(str(exc)) from exc
|
|
167
|
+
missing = [name for name in adapter_cls.required_env() if not env.get(name)]
|
|
168
|
+
if missing:
|
|
169
|
+
raise _InputError(f"missing required env for engine {engine!r}: {', '.join(missing)}")
|
|
170
|
+
return engine, adapter_cls
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _close(adapter: Any) -> None:
|
|
174
|
+
"""Close the adapter; a failure only logs, so it never masks the op's outcome."""
|
|
175
|
+
try:
|
|
176
|
+
adapter.close()
|
|
177
|
+
except Exception as exc:
|
|
178
|
+
logger.error("adapter close failed: %s", exc)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
# --- seed_preflight ---------------------------------------------------------------
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@dataclass(frozen=True)
|
|
185
|
+
class _SeedInputs:
|
|
186
|
+
mode: str
|
|
187
|
+
family: str
|
|
188
|
+
schema: str
|
|
189
|
+
table: str
|
|
190
|
+
allowed: frozenset[str]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _seed_node_id(env: Mapping[str, str]) -> str:
|
|
194
|
+
"""``NODE_ID``, else ``<PROD_SCHEMA>.<TABLE_NAME>`` when both are set, else ``""``."""
|
|
195
|
+
schema, table = env.get("PROD_SCHEMA", ""), env.get("TABLE_NAME", "")
|
|
196
|
+
return env.get("NODE_ID") or (f"{schema}.{table}" if schema and table else "")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _seed_inputs(env: Mapping[str, str]) -> _SeedInputs:
|
|
200
|
+
mode = env.get("SEED_PREFLIGHT_MODE", "")
|
|
201
|
+
if mode not in _MODES:
|
|
202
|
+
raise _InputError(f"SEED_PREFLIGHT_MODE must be one of {', '.join(_MODES)}, got {mode!r}")
|
|
203
|
+
family = env.get("SEED_PREFLIGHT_ENGINE_FAMILY", "")
|
|
204
|
+
if family not in _FAMILIES:
|
|
205
|
+
raise _InputError(
|
|
206
|
+
f"SEED_PREFLIGHT_ENGINE_FAMILY must be one of {', '.join(_FAMILIES)}, got {family!r}"
|
|
207
|
+
)
|
|
208
|
+
for name in ("PROD_SCHEMA", "TABLE_NAME"):
|
|
209
|
+
if not env.get(name):
|
|
210
|
+
raise _InputError(f"missing required env var {name}")
|
|
211
|
+
allowed = _json_list(env, "SEED_PREFLIGHT_ALLOWED", required=False)
|
|
212
|
+
acknowledged = _json_list(env, "SEED_PREFLIGHT_ACKNOWLEDGED", required=False)
|
|
213
|
+
return _SeedInputs(
|
|
214
|
+
mode=mode,
|
|
215
|
+
family=family,
|
|
216
|
+
schema=env["PROD_SCHEMA"],
|
|
217
|
+
table=env["TABLE_NAME"],
|
|
218
|
+
allowed=frozenset(entry.strip().lower() for entry in [*allowed, *acknowledged]),
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _seed_fields(
|
|
223
|
+
env: Mapping[str, str], drop_semantics: str, verdict: Verdict, dependents: list[dict[str, Any]]
|
|
224
|
+
) -> dict[str, Any]:
|
|
225
|
+
return {
|
|
226
|
+
"op": "seed_preflight",
|
|
227
|
+
"mode": env.get("SEED_PREFLIGHT_MODE", ""),
|
|
228
|
+
"checked": verdict.checked,
|
|
229
|
+
"engine_family": env.get("SEED_PREFLIGHT_ENGINE_FAMILY", ""),
|
|
230
|
+
"drop_semantics": drop_semantics,
|
|
231
|
+
"completeness": verdict.completeness,
|
|
232
|
+
"dependents": dependents,
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _finish_seed(
|
|
237
|
+
env: Mapping[str, str], drop_semantics: str, verdict: Verdict,
|
|
238
|
+
dependents: list[dict[str, Any]] | None = None,
|
|
239
|
+
) -> int:
|
|
240
|
+
node_id = _seed_node_id(env)
|
|
241
|
+
if verdict.exit_code == 2:
|
|
242
|
+
logger.error("seed_preflight %s: %s", node_id, verdict.message)
|
|
243
|
+
elif verdict.reason or verdict.exit_code:
|
|
244
|
+
logger.warning("seed_preflight %s: %s", node_id, verdict.message)
|
|
245
|
+
_emit(
|
|
246
|
+
verdict.status, verdict.message, node_id, verdict.reason,
|
|
247
|
+
_seed_fields(env, drop_semantics, verdict, dependents or []),
|
|
248
|
+
)
|
|
249
|
+
return verdict.exit_code
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def run_seed_preflight(
|
|
253
|
+
env: Mapping[str, str], discover: _Discover = discover_adapter
|
|
254
|
+
) -> int:
|
|
255
|
+
"""Run ``seed_preflight`` against *env*; print its one block and return the exit code.
|
|
256
|
+
|
|
257
|
+
Reads the op's inputs and the adapter's ``required_env()`` from *env* (adapters
|
|
258
|
+
connect from ``os.environ``, so the runner passes that). An unexpected error
|
|
259
|
+
still prints one ``error`` block and returns 2.
|
|
260
|
+
"""
|
|
261
|
+
try:
|
|
262
|
+
return _seed_preflight(env, discover)
|
|
263
|
+
except Exception as exc:
|
|
264
|
+
logger.exception("seed_preflight %s failed unexpectedly", _seed_node_id(env))
|
|
265
|
+
verdict = Verdict("error", "", 2, False, "none", f"unexpected error: {_first_line(exc)}")
|
|
266
|
+
return _finish_seed(env, "", verdict)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _seed_preflight(env: Mapping[str, str], discover: _Discover) -> int:
|
|
270
|
+
try:
|
|
271
|
+
inputs = _seed_inputs(env)
|
|
272
|
+
engine, adapter_cls = _discover_and_check_env(env, discover)
|
|
273
|
+
except _InputError as exc:
|
|
274
|
+
return _finish_seed(env, "", decide("", "", Situation.BAD_INPUT, detail=str(exc)))
|
|
275
|
+
|
|
276
|
+
declared = getattr(adapter_cls, "drop_semantics", "")
|
|
277
|
+
drop_semantics = declared if isinstance(declared, str) else ""
|
|
278
|
+
adapter_family = _FAMILY_OF_DROP_SEMANTICS.get(drop_semantics)
|
|
279
|
+
|
|
280
|
+
def finish(
|
|
281
|
+
situation: Situation, detail: str = "", *, completeness: str = "none",
|
|
282
|
+
has_unowned: bool = False, dependents: list[dict[str, Any]] | None = None,
|
|
283
|
+
) -> int:
|
|
284
|
+
verdict = decide(
|
|
285
|
+
inputs.mode, inputs.family, situation,
|
|
286
|
+
detail=detail, completeness=completeness, has_unowned=has_unowned,
|
|
287
|
+
)
|
|
288
|
+
return _finish_seed(env, drop_semantics, verdict, dependents)
|
|
289
|
+
|
|
290
|
+
if adapter_family != inputs.family:
|
|
291
|
+
return finish(
|
|
292
|
+
Situation.ENGINE_MISMATCH,
|
|
293
|
+
f"engine {engine!r} is {adapter_family or 'of no known family'}, "
|
|
294
|
+
f"but the release expects {inputs.family}",
|
|
295
|
+
)
|
|
296
|
+
if not issubclass(adapter_cls, DependentsIntrospection):
|
|
297
|
+
return finish(Situation.NO_CAPABILITY, f"engine {engine!r} cannot list the objects that read a table")
|
|
298
|
+
|
|
299
|
+
try:
|
|
300
|
+
adapter = adapter_cls.from_env()
|
|
301
|
+
except Exception as exc:
|
|
302
|
+
return finish(Situation.UNREACHABLE, _first_line(exc))
|
|
303
|
+
try:
|
|
304
|
+
try:
|
|
305
|
+
listing = cast(DependentsIntrospection, adapter).dependents(inputs.schema, inputs.table)
|
|
306
|
+
except Exception as exc:
|
|
307
|
+
return finish(
|
|
308
|
+
Situation.LISTING_FAILED,
|
|
309
|
+
f"listing the objects that read {inputs.schema}.{inputs.table} failed: {_first_line(exc)}",
|
|
310
|
+
)
|
|
311
|
+
dependents = [
|
|
312
|
+
{
|
|
313
|
+
"schema": found.schema, "name": found.name, "kind": found.kind, "owner": found.owner,
|
|
314
|
+
"depth": found.depth,
|
|
315
|
+
"allowed": f"{found.schema}.{found.name}".lower() in inputs.allowed,
|
|
316
|
+
}
|
|
317
|
+
for found in sorted(listing.objects, key=lambda d: (d.depth, d.schema, d.name))
|
|
318
|
+
]
|
|
319
|
+
finally:
|
|
320
|
+
_close(adapter)
|
|
321
|
+
outside = [d for d in dependents if not d["allowed"]]
|
|
322
|
+
if inputs.family == "schema_bound":
|
|
323
|
+
detail = "the reload would drop objects continuo does not own: " + ", ".join(
|
|
324
|
+
f"{d['schema']}.{d['name']} ({d['kind']}, owner {d['owner']})" for d in outside
|
|
325
|
+
)
|
|
326
|
+
else:
|
|
327
|
+
detail = (
|
|
328
|
+
f"{len(outside)} external view(s) may go stale; recreate them after the reload: "
|
|
329
|
+
+ ", ".join(f"{d['schema']}.{d['name']}" for d in outside)
|
|
330
|
+
)
|
|
331
|
+
return finish(
|
|
332
|
+
Situation.LISTED, detail if outside else "",
|
|
333
|
+
completeness=listing.completeness, has_unowned=bool(outside), dependents=dependents,
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# --- probe_views ------------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _probe_views_input(env: Mapping[str, str]) -> list[tuple[str, str]]:
|
|
341
|
+
"""``PROBE_VIEWS`` as ``(schema, name)`` pairs, split on the first dot, each once, in order."""
|
|
342
|
+
pairs: list[tuple[str, str]] = []
|
|
343
|
+
for entry in _json_list(env, "PROBE_VIEWS", required=True):
|
|
344
|
+
schema, dot, name = entry.partition(".")
|
|
345
|
+
if not dot or not schema or not name:
|
|
346
|
+
raise _InputError(f"PROBE_VIEWS entry {entry!r} is not <schema>.<name>")
|
|
347
|
+
if (schema, name) not in pairs:
|
|
348
|
+
pairs.append((schema, name))
|
|
349
|
+
return pairs
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _quote(identifier: str) -> str:
|
|
353
|
+
return '"' + identifier.replace('"', '""') + '"'
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def _finish_probe(
|
|
357
|
+
env: Mapping[str, str], status: str, message: str, reason: str, checked: bool,
|
|
358
|
+
views: list[dict[str, Any]],
|
|
359
|
+
) -> int:
|
|
360
|
+
node_id = env.get("NODE_ID") or "probe_views"
|
|
361
|
+
if status == "error":
|
|
362
|
+
logger.error("probe_views %s: %s", node_id, message)
|
|
363
|
+
elif reason:
|
|
364
|
+
logger.warning("probe_views %s: %s", node_id, message)
|
|
365
|
+
_emit(status, message, node_id, reason, {"op": "probe_views", "checked": checked, "views": views})
|
|
366
|
+
return 2 if status == "error" else 0
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def run_probe_views(env: Mapping[str, str], discover: _Discover = discover_adapter) -> int:
|
|
370
|
+
"""Run ``probe_views`` against *env*; print its one block and return the exit code.
|
|
371
|
+
|
|
372
|
+
Reads ``PROBE_VIEWS`` and the adapter's ``required_env()`` from *env*. A view
|
|
373
|
+
that fails its probe, or a warehouse that cannot be reached, still ends with
|
|
374
|
+
exit 0; only an input or adapter problem or an unexpected error is exit 2.
|
|
375
|
+
"""
|
|
376
|
+
try:
|
|
377
|
+
return _probe_views(env, discover)
|
|
378
|
+
except Exception as exc:
|
|
379
|
+
logger.exception("probe_views failed unexpectedly")
|
|
380
|
+
return _finish_probe(env, "error", f"unexpected error: {_first_line(exc)}", "", False, [])
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _probe_views(env: Mapping[str, str], discover: _Discover) -> int:
|
|
384
|
+
try:
|
|
385
|
+
pairs = _probe_views_input(env)
|
|
386
|
+
_, adapter_cls = _discover_and_check_env(env, discover)
|
|
387
|
+
except _InputError as exc:
|
|
388
|
+
return _finish_probe(env, "error", str(exc), "", False, [])
|
|
389
|
+
try:
|
|
390
|
+
adapter = adapter_cls.from_env()
|
|
391
|
+
except Exception as exc:
|
|
392
|
+
return _finish_probe(
|
|
393
|
+
env, "success", f"warehouse_unreachable: {_first_line(exc)}", "warehouse_unreachable", False, []
|
|
394
|
+
)
|
|
395
|
+
views: list[dict[str, Any]] = []
|
|
396
|
+
try:
|
|
397
|
+
for schema, name in pairs:
|
|
398
|
+
error = ""
|
|
399
|
+
try:
|
|
400
|
+
adapter.fetch(f"SELECT * FROM {_quote(schema)}.{_quote(name)} LIMIT 0")
|
|
401
|
+
except Exception as exc:
|
|
402
|
+
error = _first_line(exc)
|
|
403
|
+
views.append({"schema": schema, "name": name, "ok": not error, "error": error})
|
|
404
|
+
finally:
|
|
405
|
+
_close(adapter)
|
|
406
|
+
broken = [f"{v['schema']}.{v['name']}" for v in views if not v["ok"]]
|
|
407
|
+
for name in broken:
|
|
408
|
+
logger.warning("probe_views: %s does not bind", name)
|
|
409
|
+
message = f"{len(broken)} of {len(views)} views failed: {', '.join(broken)}" if broken else ""
|
|
410
|
+
return _finish_probe(env, "success", message, "", True, views)
|
|
@@ -23,6 +23,12 @@ Dispatches on ``VALIDATION_OP`` env var (default ``build_from_sql``):
|
|
|
23
23
|
via the engine adapter and creates nothing — a test whose compiled SQL names
|
|
24
24
|
a column a candidate change dropped or renamed fails the release at its
|
|
25
25
|
source, before any table is built.
|
|
26
|
+
- ``seed_preflight``: before a seed's production table is reloaded, list the objects
|
|
27
|
+
(``PROD_SCHEMA``.``TABLE_NAME``) that read it and compare them with the operator's
|
|
28
|
+
allowed and acknowledged lists. Needs no ``DBT_TARGET_SCHEMA`` and creates nothing.
|
|
29
|
+
- ``probe_views``: try each view of ``PROBE_VIEWS`` with a zero-row read and report
|
|
30
|
+
which ones no longer bind. Needs no ``DBT_TARGET_SCHEMA`` and creates nothing.
|
|
31
|
+
The inputs and the result block of both ops are in ``preflight``.
|
|
26
32
|
|
|
27
33
|
The engine adapter is discovered from the single installed
|
|
28
34
|
``continuo_engine.adapters`` entry point — each runner image installs exactly one.
|
|
@@ -43,7 +49,7 @@ from continuo_engine_contract.port import ( # type: ignore[import-untyped]
|
|
|
43
49
|
)
|
|
44
50
|
from continuo_python_runtime.csv_readers import reader_for
|
|
45
51
|
from continuo_python_runtime.csv_source import check_header, parse_csv_uri
|
|
46
|
-
from continuo_python_runtime.validation import s3
|
|
52
|
+
from continuo_python_runtime.validation import preflight, s3
|
|
47
53
|
|
|
48
54
|
logger = logging.getLogger("validation_runner")
|
|
49
55
|
|
|
@@ -115,6 +121,7 @@ def load_candidate_spec() -> dict:
|
|
|
115
121
|
|
|
116
122
|
_NODE_OPS = ("build_from_sql", "clone_from_prod", "build_from_columns", "check_binds")
|
|
117
123
|
_SCHEMA_OPS = ("ensure_schema", "drop_schema")
|
|
124
|
+
_PREFLIGHT_OPS = ("seed_preflight", "probe_views")
|
|
118
125
|
|
|
119
126
|
|
|
120
127
|
def main() -> None:
|
|
@@ -128,13 +135,23 @@ def main() -> None:
|
|
|
128
135
|
Schema ops (``ensure_schema``/``drop_schema``) act on the whole candidate
|
|
129
136
|
schema — the executor schedules them as one-shot engine-image Jobs to own
|
|
130
137
|
the candidate-schema lifecycle without connecting to the warehouse itself —
|
|
131
|
-
and take no table.
|
|
138
|
+
and take no table. The preflight ops (``seed_preflight``/``probe_views``) read
|
|
139
|
+
their own inputs, need no candidate schema and exit with the code ``preflight``
|
|
140
|
+
returns.
|
|
132
141
|
"""
|
|
133
142
|
logging.basicConfig(
|
|
134
143
|
level=logging.INFO, stream=sys.stderr, format="%(levelname)s %(name)s: %(message)s"
|
|
135
144
|
)
|
|
136
|
-
schema = _require("DBT_TARGET_SCHEMA")
|
|
137
145
|
op = os.environ.get("VALIDATION_OP", "build_from_sql")
|
|
146
|
+
if op in _PREFLIGHT_OPS:
|
|
147
|
+
if op == "seed_preflight":
|
|
148
|
+
code = preflight.run_seed_preflight(os.environ)
|
|
149
|
+
else:
|
|
150
|
+
code = preflight.run_probe_views(os.environ)
|
|
151
|
+
if code:
|
|
152
|
+
sys.exit(code)
|
|
153
|
+
return
|
|
154
|
+
schema = _require("DBT_TARGET_SCHEMA")
|
|
138
155
|
|
|
139
156
|
# Gather op-specific inputs and identity BEFORE touching the adapter, surfacing
|
|
140
157
|
# input errors as a structured block (preserves the prior contract + exit codes).
|