continuo-python-runtime 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,410 @@
1
+ """The ``seed_preflight`` and ``probe_views`` validation ops.
2
+
3
+ ``seed_preflight`` asks the warehouse which objects a reload of a seed's
4
+ production table would destroy or leave reading a missing table, and compares
5
+ them with the objects the operator has allowed or acknowledged. ``check`` stops
6
+ a reload that would hit an object outside that set; ``report`` prints the same
7
+ conclusion and never stops. ``probe_views`` tries each named view with a
8
+ zero-row read and reports which ones no longer bind, and never fails the op for
9
+ it.
10
+
11
+ Each op prints exactly one structured result block as the last line on stdout
12
+ (``continuo_engine_contract.result``) and returns the process exit code: 0 when
13
+ the op ran to its conclusion (or ``report`` mode), 1 when ``check`` stops, 2
14
+ when the op could not run (a bad input, no adapter, missing adapter env, an
15
+ unexpected error). The block's ``status`` is ``error`` for exit codes 1 and 2.
16
+ Diagnostics go to stderr through ``logging``; stdout carries only the block.
17
+
18
+ The decision of ``seed_preflight`` is the pure function :func:`decide`; the rest
19
+ of the module reads the inputs, drives the adapter and prints.
20
+ """
21
+ import json
22
+ import logging
23
+ from collections.abc import Callable, Mapping
24
+ from dataclasses import dataclass
25
+ from enum import Enum
26
+ from typing import Any, cast
27
+
28
+ from continuo_engine_contract import result # type: ignore[import-untyped]
29
+ from continuo_engine_contract.port import ( # type: ignore[import-untyped]
30
+ AdapterDiscoveryError,
31
+ DependentsIntrospection,
32
+ WarehouseAdapter,
33
+ discover_adapter,
34
+ )
35
+
36
+ logger = logging.getLogger("validation_preflight")
37
+
38
+ _Discover = Callable[[], tuple[str, type[WarehouseAdapter]]]
39
+
40
+ # The adapter's ``drop_semantics`` mapped to the engine family the release plan declares.
41
+ _FAMILY_OF_DROP_SEMANTICS = {"cascade": "schema_bound", "late_binding": "late_binding"}
42
+ _MODES = ("check", "report")
43
+ _FAMILIES = ("schema_bound", "late_binding")
44
+
45
+ # Longest driver text kept in a block; the first line only, so a block stays small.
46
+ _MAX_TEXT = 500
47
+
48
+
49
+ class Situation(Enum):
50
+ """How far ``seed_preflight`` got, which is what decides its verdict."""
51
+
52
+ BAD_INPUT = "bad_input"
53
+ ENGINE_MISMATCH = "engine_mismatch"
54
+ NO_CAPABILITY = "no_capability"
55
+ UNREACHABLE = "unreachable"
56
+ LISTING_FAILED = "listing_failed"
57
+ LISTED = "listed"
58
+
59
+
60
+ @dataclass(frozen=True)
61
+ class Verdict:
62
+ """What ``seed_preflight`` concludes: the block's status, reason, message and the exit code."""
63
+
64
+ status: str
65
+ reason: str
66
+ exit_code: int
67
+ checked: bool
68
+ completeness: str
69
+ message: str
70
+
71
+
72
+ def decide(
73
+ mode: str,
74
+ family: str,
75
+ situation: Situation,
76
+ *,
77
+ detail: str = "",
78
+ completeness: str = "none",
79
+ has_unowned: bool = False,
80
+ ) -> Verdict:
81
+ """The decision table of ``seed_preflight``.
82
+
83
+ *family* is the engine family the release expects (``schema_bound`` or
84
+ ``late_binding``), *detail* the text that explains the situation, *completeness*
85
+ the adapter's own and *has_unowned* whether a listed object is outside the
86
+ allowed set. A situation that a ``check`` stops on gives ``error`` and exit 1;
87
+ ``report`` mode gives the same ``reason`` on a ``success`` block and exit 0. A
88
+ bad input is exit 2 in both modes. A ``reason`` prefixes the message.
89
+ """
90
+ if situation is Situation.BAD_INPUT:
91
+ return Verdict("error", "", 2, False, "none", detail)
92
+ reason, stops = _conclusion(family, situation, has_unowned)
93
+ failed = stops and mode == "check"
94
+ return Verdict(
95
+ status="error" if failed else "success",
96
+ reason=reason,
97
+ exit_code=1 if failed else 0,
98
+ checked=situation is Situation.LISTED,
99
+ completeness=completeness if situation is Situation.LISTED else "none",
100
+ message=f"{reason}: {detail}" if reason else detail,
101
+ )
102
+
103
+
104
+ def _conclusion(family: str, situation: Situation, has_unowned: bool) -> tuple[str, bool]:
105
+ """The ``reason`` a situation carries and whether a ``check`` stops on it."""
106
+ schema_bound = family == "schema_bound"
107
+ if situation is Situation.ENGINE_MISMATCH:
108
+ return "engine_mismatch", True
109
+ if situation is Situation.UNREACHABLE:
110
+ return "warehouse_unreachable", True
111
+ if situation is Situation.NO_CAPABILITY:
112
+ return ("unowned_dependents", True) if schema_bound else ("", False)
113
+ if situation is Situation.LISTING_FAILED:
114
+ return "", schema_bound
115
+ if schema_bound and has_unowned:
116
+ return "unowned_dependents", True
117
+ return "", False
118
+
119
+
120
+ class _InputError(Exception):
121
+ """An input of the op is missing or malformed."""
122
+
123
+
124
+ def _first_line(exc: BaseException) -> str:
125
+ """The first line of an exception's text, at most ``_MAX_TEXT`` characters, never empty."""
126
+ text = str(exc).strip()
127
+ line = text.splitlines()[0] if text else ""
128
+ return (line or type(exc).__name__)[:_MAX_TEXT]
129
+
130
+
131
+ def _json_list(env: Mapping[str, str], name: str, *, required: bool) -> list[str]:
132
+ """The env var *name* as a JSON array of strings; unset or empty is ``[]`` unless *required*."""
133
+ raw = env.get(name, "")
134
+ if not raw.strip():
135
+ if required:
136
+ raise _InputError(f"missing required env var {name}")
137
+ return []
138
+ try:
139
+ value = json.loads(raw)
140
+ except json.JSONDecodeError as exc:
141
+ raise _InputError(f"{name} is not valid JSON: {exc}") from exc
142
+ if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
143
+ raise _InputError(f"{name} must be a JSON array of strings")
144
+ return cast(list[str], value)
145
+
146
+
147
+ def _emit(
148
+ status: str, message: str, unique_id: str, reason: str, fields: dict[str, Any]
149
+ ) -> None:
150
+ """Print the op's one result block, the last line on stdout."""
151
+ print(
152
+ result.result_block(
153
+ status, message, 1 if status == "error" else 0, unique_id, reason=reason, fields=fields
154
+ ),
155
+ flush=True,
156
+ )
157
+
158
+
159
+ def _discover_and_check_env(
160
+ env: Mapping[str, str], discover: _Discover
161
+ ) -> tuple[str, type[WarehouseAdapter]]:
162
+ """The installed adapter, after checking its required env is present in *env*."""
163
+ try:
164
+ engine, adapter_cls = discover()
165
+ except AdapterDiscoveryError as exc:
166
+ raise _InputError(str(exc)) from exc
167
+ missing = [name for name in adapter_cls.required_env() if not env.get(name)]
168
+ if missing:
169
+ raise _InputError(f"missing required env for engine {engine!r}: {', '.join(missing)}")
170
+ return engine, adapter_cls
171
+
172
+
173
+ def _close(adapter: Any) -> None:
174
+ """Close the adapter; a failure only logs, so it never masks the op's outcome."""
175
+ try:
176
+ adapter.close()
177
+ except Exception as exc:
178
+ logger.error("adapter close failed: %s", exc)
179
+
180
+
181
+ # --- seed_preflight ---------------------------------------------------------------
182
+
183
+
184
+ @dataclass(frozen=True)
185
+ class _SeedInputs:
186
+ mode: str
187
+ family: str
188
+ schema: str
189
+ table: str
190
+ allowed: frozenset[str]
191
+
192
+
193
+ def _seed_node_id(env: Mapping[str, str]) -> str:
194
+ """``NODE_ID``, else ``<PROD_SCHEMA>.<TABLE_NAME>`` when both are set, else ``""``."""
195
+ schema, table = env.get("PROD_SCHEMA", ""), env.get("TABLE_NAME", "")
196
+ return env.get("NODE_ID") or (f"{schema}.{table}" if schema and table else "")
197
+
198
+
199
+ def _seed_inputs(env: Mapping[str, str]) -> _SeedInputs:
200
+ mode = env.get("SEED_PREFLIGHT_MODE", "")
201
+ if mode not in _MODES:
202
+ raise _InputError(f"SEED_PREFLIGHT_MODE must be one of {', '.join(_MODES)}, got {mode!r}")
203
+ family = env.get("SEED_PREFLIGHT_ENGINE_FAMILY", "")
204
+ if family not in _FAMILIES:
205
+ raise _InputError(
206
+ f"SEED_PREFLIGHT_ENGINE_FAMILY must be one of {', '.join(_FAMILIES)}, got {family!r}"
207
+ )
208
+ for name in ("PROD_SCHEMA", "TABLE_NAME"):
209
+ if not env.get(name):
210
+ raise _InputError(f"missing required env var {name}")
211
+ allowed = _json_list(env, "SEED_PREFLIGHT_ALLOWED", required=False)
212
+ acknowledged = _json_list(env, "SEED_PREFLIGHT_ACKNOWLEDGED", required=False)
213
+ return _SeedInputs(
214
+ mode=mode,
215
+ family=family,
216
+ schema=env["PROD_SCHEMA"],
217
+ table=env["TABLE_NAME"],
218
+ allowed=frozenset(entry.strip().lower() for entry in [*allowed, *acknowledged]),
219
+ )
220
+
221
+
222
+ def _seed_fields(
223
+ env: Mapping[str, str], drop_semantics: str, verdict: Verdict, dependents: list[dict[str, Any]]
224
+ ) -> dict[str, Any]:
225
+ return {
226
+ "op": "seed_preflight",
227
+ "mode": env.get("SEED_PREFLIGHT_MODE", ""),
228
+ "checked": verdict.checked,
229
+ "engine_family": env.get("SEED_PREFLIGHT_ENGINE_FAMILY", ""),
230
+ "drop_semantics": drop_semantics,
231
+ "completeness": verdict.completeness,
232
+ "dependents": dependents,
233
+ }
234
+
235
+
236
+ def _finish_seed(
237
+ env: Mapping[str, str], drop_semantics: str, verdict: Verdict,
238
+ dependents: list[dict[str, Any]] | None = None,
239
+ ) -> int:
240
+ node_id = _seed_node_id(env)
241
+ if verdict.exit_code == 2:
242
+ logger.error("seed_preflight %s: %s", node_id, verdict.message)
243
+ elif verdict.reason or verdict.exit_code:
244
+ logger.warning("seed_preflight %s: %s", node_id, verdict.message)
245
+ _emit(
246
+ verdict.status, verdict.message, node_id, verdict.reason,
247
+ _seed_fields(env, drop_semantics, verdict, dependents or []),
248
+ )
249
+ return verdict.exit_code
250
+
251
+
252
+ def run_seed_preflight(
253
+ env: Mapping[str, str], discover: _Discover = discover_adapter
254
+ ) -> int:
255
+ """Run ``seed_preflight`` against *env*; print its one block and return the exit code.
256
+
257
+ Reads the op's inputs and the adapter's ``required_env()`` from *env* (adapters
258
+ connect from ``os.environ``, so the runner passes that). An unexpected error
259
+ still prints one ``error`` block and returns 2.
260
+ """
261
+ try:
262
+ return _seed_preflight(env, discover)
263
+ except Exception as exc:
264
+ logger.exception("seed_preflight %s failed unexpectedly", _seed_node_id(env))
265
+ verdict = Verdict("error", "", 2, False, "none", f"unexpected error: {_first_line(exc)}")
266
+ return _finish_seed(env, "", verdict)
267
+
268
+
269
+ def _seed_preflight(env: Mapping[str, str], discover: _Discover) -> int:
270
+ try:
271
+ inputs = _seed_inputs(env)
272
+ engine, adapter_cls = _discover_and_check_env(env, discover)
273
+ except _InputError as exc:
274
+ return _finish_seed(env, "", decide("", "", Situation.BAD_INPUT, detail=str(exc)))
275
+
276
+ declared = getattr(adapter_cls, "drop_semantics", "")
277
+ drop_semantics = declared if isinstance(declared, str) else ""
278
+ adapter_family = _FAMILY_OF_DROP_SEMANTICS.get(drop_semantics)
279
+
280
+ def finish(
281
+ situation: Situation, detail: str = "", *, completeness: str = "none",
282
+ has_unowned: bool = False, dependents: list[dict[str, Any]] | None = None,
283
+ ) -> int:
284
+ verdict = decide(
285
+ inputs.mode, inputs.family, situation,
286
+ detail=detail, completeness=completeness, has_unowned=has_unowned,
287
+ )
288
+ return _finish_seed(env, drop_semantics, verdict, dependents)
289
+
290
+ if adapter_family != inputs.family:
291
+ return finish(
292
+ Situation.ENGINE_MISMATCH,
293
+ f"engine {engine!r} is {adapter_family or 'of no known family'}, "
294
+ f"but the release expects {inputs.family}",
295
+ )
296
+ if not issubclass(adapter_cls, DependentsIntrospection):
297
+ return finish(Situation.NO_CAPABILITY, f"engine {engine!r} cannot list the objects that read a table")
298
+
299
+ try:
300
+ adapter = adapter_cls.from_env()
301
+ except Exception as exc:
302
+ return finish(Situation.UNREACHABLE, _first_line(exc))
303
+ try:
304
+ try:
305
+ listing = cast(DependentsIntrospection, adapter).dependents(inputs.schema, inputs.table)
306
+ except Exception as exc:
307
+ return finish(
308
+ Situation.LISTING_FAILED,
309
+ f"listing the objects that read {inputs.schema}.{inputs.table} failed: {_first_line(exc)}",
310
+ )
311
+ dependents = [
312
+ {
313
+ "schema": found.schema, "name": found.name, "kind": found.kind, "owner": found.owner,
314
+ "depth": found.depth,
315
+ "allowed": f"{found.schema}.{found.name}".lower() in inputs.allowed,
316
+ }
317
+ for found in sorted(listing.objects, key=lambda d: (d.depth, d.schema, d.name))
318
+ ]
319
+ finally:
320
+ _close(adapter)
321
+ outside = [d for d in dependents if not d["allowed"]]
322
+ if inputs.family == "schema_bound":
323
+ detail = "the reload would drop objects continuo does not own: " + ", ".join(
324
+ f"{d['schema']}.{d['name']} ({d['kind']}, owner {d['owner']})" for d in outside
325
+ )
326
+ else:
327
+ detail = (
328
+ f"{len(outside)} external view(s) may go stale; recreate them after the reload: "
329
+ + ", ".join(f"{d['schema']}.{d['name']}" for d in outside)
330
+ )
331
+ return finish(
332
+ Situation.LISTED, detail if outside else "",
333
+ completeness=listing.completeness, has_unowned=bool(outside), dependents=dependents,
334
+ )
335
+
336
+
337
+ # --- probe_views ------------------------------------------------------------------
338
+
339
+
340
+ def _probe_views_input(env: Mapping[str, str]) -> list[tuple[str, str]]:
341
+ """``PROBE_VIEWS`` as ``(schema, name)`` pairs, split on the first dot, each once, in order."""
342
+ pairs: list[tuple[str, str]] = []
343
+ for entry in _json_list(env, "PROBE_VIEWS", required=True):
344
+ schema, dot, name = entry.partition(".")
345
+ if not dot or not schema or not name:
346
+ raise _InputError(f"PROBE_VIEWS entry {entry!r} is not <schema>.<name>")
347
+ if (schema, name) not in pairs:
348
+ pairs.append((schema, name))
349
+ return pairs
350
+
351
+
352
+ def _quote(identifier: str) -> str:
353
+ return '"' + identifier.replace('"', '""') + '"'
354
+
355
+
356
+ def _finish_probe(
357
+ env: Mapping[str, str], status: str, message: str, reason: str, checked: bool,
358
+ views: list[dict[str, Any]],
359
+ ) -> int:
360
+ node_id = env.get("NODE_ID") or "probe_views"
361
+ if status == "error":
362
+ logger.error("probe_views %s: %s", node_id, message)
363
+ elif reason:
364
+ logger.warning("probe_views %s: %s", node_id, message)
365
+ _emit(status, message, node_id, reason, {"op": "probe_views", "checked": checked, "views": views})
366
+ return 2 if status == "error" else 0
367
+
368
+
369
+ def run_probe_views(env: Mapping[str, str], discover: _Discover = discover_adapter) -> int:
370
+ """Run ``probe_views`` against *env*; print its one block and return the exit code.
371
+
372
+ Reads ``PROBE_VIEWS`` and the adapter's ``required_env()`` from *env*. A view
373
+ that fails its probe, or a warehouse that cannot be reached, still ends with
374
+ exit 0; only an input or adapter problem or an unexpected error is exit 2.
375
+ """
376
+ try:
377
+ return _probe_views(env, discover)
378
+ except Exception as exc:
379
+ logger.exception("probe_views failed unexpectedly")
380
+ return _finish_probe(env, "error", f"unexpected error: {_first_line(exc)}", "", False, [])
381
+
382
+
383
+ def _probe_views(env: Mapping[str, str], discover: _Discover) -> int:
384
+ try:
385
+ pairs = _probe_views_input(env)
386
+ _, adapter_cls = _discover_and_check_env(env, discover)
387
+ except _InputError as exc:
388
+ return _finish_probe(env, "error", str(exc), "", False, [])
389
+ try:
390
+ adapter = adapter_cls.from_env()
391
+ except Exception as exc:
392
+ return _finish_probe(
393
+ env, "success", f"warehouse_unreachable: {_first_line(exc)}", "warehouse_unreachable", False, []
394
+ )
395
+ views: list[dict[str, Any]] = []
396
+ try:
397
+ for schema, name in pairs:
398
+ error = ""
399
+ try:
400
+ adapter.fetch(f"SELECT * FROM {_quote(schema)}.{_quote(name)} LIMIT 0")
401
+ except Exception as exc:
402
+ error = _first_line(exc)
403
+ views.append({"schema": schema, "name": name, "ok": not error, "error": error})
404
+ finally:
405
+ _close(adapter)
406
+ broken = [f"{v['schema']}.{v['name']}" for v in views if not v["ok"]]
407
+ for name in broken:
408
+ logger.warning("probe_views: %s does not bind", name)
409
+ message = f"{len(broken)} of {len(views)} views failed: {', '.join(broken)}" if broken else ""
410
+ return _finish_probe(env, "success", message, "", True, views)
@@ -23,6 +23,12 @@ Dispatches on ``VALIDATION_OP`` env var (default ``build_from_sql``):
23
23
  via the engine adapter and creates nothing — a test whose compiled SQL names
24
24
  a column a candidate change dropped or renamed fails the release at its
25
25
  source, before any table is built.
26
+ - ``seed_preflight``: before a seed's production table is reloaded, list the objects
27
+ (``PROD_SCHEMA``.``TABLE_NAME``) that read it and compare them with the operator's
28
+ allowed and acknowledged lists. Needs no ``DBT_TARGET_SCHEMA`` and creates nothing.
29
+ - ``probe_views``: try each view of ``PROBE_VIEWS`` with a zero-row read and report
30
+ which ones no longer bind. Needs no ``DBT_TARGET_SCHEMA`` and creates nothing.
31
+ The inputs and the result block of both ops are in ``preflight``.
26
32
 
27
33
  The engine adapter is discovered from the single installed
28
34
  ``continuo_engine.adapters`` entry point — each runner image installs exactly one.
@@ -43,7 +49,7 @@ from continuo_engine_contract.port import ( # type: ignore[import-untyped]
43
49
  )
44
50
  from continuo_python_runtime.csv_readers import reader_for
45
51
  from continuo_python_runtime.csv_source import check_header, parse_csv_uri
46
- from continuo_python_runtime.validation import s3
52
+ from continuo_python_runtime.validation import preflight, s3
47
53
 
48
54
  logger = logging.getLogger("validation_runner")
49
55
 
@@ -115,6 +121,7 @@ def load_candidate_spec() -> dict:
115
121
 
116
122
  _NODE_OPS = ("build_from_sql", "clone_from_prod", "build_from_columns", "check_binds")
117
123
  _SCHEMA_OPS = ("ensure_schema", "drop_schema")
124
+ _PREFLIGHT_OPS = ("seed_preflight", "probe_views")
118
125
 
119
126
 
120
127
  def main() -> None:
@@ -128,13 +135,23 @@ def main() -> None:
128
135
  Schema ops (``ensure_schema``/``drop_schema``) act on the whole candidate
129
136
  schema — the executor schedules them as one-shot engine-image Jobs to own
130
137
  the candidate-schema lifecycle without connecting to the warehouse itself —
131
- and take no table.
138
+ and take no table. The preflight ops (``seed_preflight``/``probe_views``) read
139
+ their own inputs, need no candidate schema and exit with the code ``preflight``
140
+ returns.
132
141
  """
133
142
  logging.basicConfig(
134
143
  level=logging.INFO, stream=sys.stderr, format="%(levelname)s %(name)s: %(message)s"
135
144
  )
136
- schema = _require("DBT_TARGET_SCHEMA")
137
145
  op = os.environ.get("VALIDATION_OP", "build_from_sql")
146
+ if op in _PREFLIGHT_OPS:
147
+ if op == "seed_preflight":
148
+ code = preflight.run_seed_preflight(os.environ)
149
+ else:
150
+ code = preflight.run_probe_views(os.environ)
151
+ if code:
152
+ sys.exit(code)
153
+ return
154
+ schema = _require("DBT_TARGET_SCHEMA")
138
155
 
139
156
  # Gather op-specific inputs and identity BEFORE touching the adapter, surfacing
140
157
  # input errors as a structured block (preserves the prior contract + exit codes).