interlaced 2.0.0a2__py3-none-any.whl

interlace/__init__.py

@@ -0,0 +1,26 @@
+"""Interlace v2 — Python/SQL-first data platform.
+
+Transformation (sqlmesh-grade snapshots, virtual environments, plan/apply),
+built-in orchestration (durable work queue + unified triggers), and durable
+streaming ingestion — in one process. See docs/architecture/v2-design.md.
+
+This package is under active greenfield construction; the public surface is the
+``@model`` / ``@stream`` / ``@check`` decorators plus the core IR types.
+"""
+
+from __future__ import annotations
+
+from interlace.dsl.decorators import check, model, stream
+from interlace.ir.relation import EngineRef, SqlRelation, TableRef
+
+__version__ = "2.0.0a2"
+
+__all__ = [
+    "EngineRef",
+    "SqlRelation",
+    "TableRef",
+    "__version__",
+    "check",
+    "model",
+    "stream",
+]

interlace/checks/__init__.py

@@ -0,0 +1,10 @@
+"""Data-quality checks. Results gate promotion: an ``error``-severity failure
+aborts the apply before the environment is promoted.
+
+Import :mod:`interlace.checks.runner` for execution — kept out of this package
+init so declaring checks (spec) never drags in the runtime.
+"""
+
+from interlace.checks.spec import CheckSpec, parse_checks
+
+__all__ = ["CheckSpec", "parse_checks"]

interlace/checks/builtin.py

@@ -0,0 +1,152 @@
+"""Built-in checks as AST builders.
+
+Every check compiles to one SELECT returning a single ``failures`` bigint —
+0 means pass. Row-predicated checks count violating rows; table-level checks
+(row_count, freshness) collapse their condition to 0/1. Queries are sqlglot
+expressions end to end, so they transpile with the engine dialect like any
+strategy output.
+
+NULL semantics (deliberate change from v0.x): ``pattern``, ``range``,
+``accepted_values``, and ``relationships`` ignore NULLs — ``not_null`` is the
+dedicated null check, so combine them when NULLs should also fail (v0.x counted
+NULLs as pattern/range failures, which conflated the two).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from typing import Any
+
+from sqlglot import exp, parse_one
+
+from interlace.checks.spec import CheckSpec
+from interlace.exceptions import DefinitionError
+from interlace.ir.relation import TableRef
+
+_FAILURES = "failures"
+_AGE_RE = re.compile(r"^\s*(\d+)\s*([smhdw])\s*$")
+_AGE_UNITS = {"s": "SECOND", "m": "MINUTE", "h": "HOUR", "d": "DAY", "w": "WEEK"}
+
+ResolveTable = Callable[[str], TableRef]
+"""Maps an upstream model name to its physical table (for ``relationships``)."""
+
+
+def _table(ref: TableRef) -> exp.Table:
+    return exp.table_(ref.name, db=ref.schema or None, catalog=ref.catalog)
+
+
+def _count_where(table: TableRef, condition: exp.Expression) -> exp.Select:
+    return exp.select(exp.alias_(exp.Count(this=exp.Star()), _FAILURES)).from_(_table(table)).where(condition)
+
+
+def _flag(table: TableRef, condition: exp.Expression) -> exp.Select:
+    """1 if ``condition`` holds over the whole table, else 0."""
+    flag = exp.Case().when(condition, exp.Literal.number(1)).else_(exp.Literal.number(0))
+    return exp.select(exp.alias_(flag, _FAILURES)).from_(_table(table))
+
+
+def _require(spec: CheckSpec, model: str, *, columns: int | None = None, params: tuple[str, ...] = ()) -> None:
+    if columns is not None and len(spec.columns) < columns:
+        raise DefinitionError(f"check {spec.type!r} on {model!r} needs a column")
+    for name in params:
+        if name not in spec.params:
+            raise DefinitionError(f"check {spec.type!r} on {model!r} needs {name!r}")
+
+
+def _parse_expr(sql: str, dialect: str) -> exp.Expression:
+    return parse_one(sql, read=dialect)
+
+
+def _interval(max_age: str) -> exp.Interval:
+    match = _AGE_RE.fullmatch(str(max_age))
+    if match is None:
+        raise DefinitionError(f"invalid max_age {max_age!r}; expected like '2h', '30m', '1d'")
+    return exp.Interval(this=exp.Literal.string(match.group(1)), unit=exp.Var(this=_AGE_UNITS[match.group(2)]))
+
+
+def build_check_query(
+    spec: CheckSpec, table: TableRef, model: str, dialect: str, resolve: ResolveTable
+) -> exp.Expression:
+    """Compile ``spec`` against ``table`` into a query returning ``failures``."""
+    cols = [exp.column(c) for c in spec.columns]
+    params: dict[str, Any] = spec.params
+
+    if spec.type == "not_null":
+        _require(spec, model, columns=1)
+        condition: exp.Expression = exp.or_(*[col.is_(exp.null()) for col in cols])
+        return _count_where(table, condition)
+
+    if spec.type == "unique":
+        _require(spec, model, columns=1)
+        grouped = (
+            exp.select(*cols)
+            .from_(_table(table))
+            .group_by(*cols)
+            .having(exp.GT(this=exp.Count(this=exp.Star()), expression=exp.Literal.number(1)))
+        )
+        return exp.select(exp.alias_(exp.Count(this=exp.Star()), _FAILURES)).from_(grouped.subquery("dup"))
+
+    if spec.type == "accepted_values":
+        _require(spec, model, columns=1, params=("values",))
+        values = [exp.Literal.string(v) if isinstance(v, str) else exp.Literal.number(v) for v in params["values"]]
+        col = cols[0]
+        return _count_where(table, exp.and_(col.is_(exp.null()).not_(), exp.In(this=col, expressions=values).not_()))
+
+    if spec.type == "range":
+        _require(spec, model, columns=1)
+        if "min" not in params and "max" not in params:
+            raise DefinitionError(f"check 'range' on {model!r} needs min and/or max")
+        col = cols[0]
+        bounds: list[exp.Expression] = []
+        if "min" in params:
+            bounds.append(exp.LT(this=col, expression=exp.Literal.number(params["min"])))
+        if "max" in params:
+            bounds.append(exp.GT(this=col, expression=exp.Literal.number(params["max"])))
+        return _count_where(table, exp.or_(*bounds))
+
+    if spec.type == "pattern":
+        _require(spec, model, columns=1, params=("regex",))
+        col = cols[0]
+        matches = exp.RegexpLike(this=col, expression=exp.Literal.string(str(params["regex"])))
+        return _count_where(table, exp.and_(col.is_(exp.null()).not_(), matches.not_()))
+
+    if spec.type == "expression":
+        _require(spec, model, params=("expression",))
+        predicate = _parse_expr(str(params["expression"]), dialect)
+        return _count_where(table, exp.paren(predicate).not_())
+
+    if spec.type == "relationships":
+        _require(spec, model, columns=1, params=("to", "field"))
+        col = cols[0]
+        parent = exp.select(exp.column(str(params["field"]))).from_(_table(resolve(str(params["to"]))))
+        orphan = exp.In(this=col, query=exp.Subquery(this=parent)).not_()
+        return _count_where(table, exp.and_(col.is_(exp.null()).not_(), orphan))
+
+    if spec.type == "row_count":
+        if "min" not in params and "max" not in params:
+            raise DefinitionError(f"check 'row_count' on {model!r} needs min and/or max")
+        count = exp.Count(this=exp.Star())
+        counts: list[exp.Expression] = []
+        if "min" in params:
+            counts.append(exp.LT(this=count, expression=exp.Literal.number(params["min"])))
+        if "max" in params:
+            counts.append(exp.GT(this=count.copy(), expression=exp.Literal.number(params["max"])))
+        return _flag(table, exp.or_(*counts))
+
+    if spec.type == "freshness":
+        _require(spec, model, columns=1, params=("max_age",))
+        threshold = exp.Sub(this=exp.CurrentTimestamp(), expression=_interval(str(params["max_age"])))
+        newest = exp.Max(this=cols[0])
+        stale = exp.or_(exp.LT(this=newest, expression=threshold), newest.copy().is_(exp.null()))  # no rows = stale
+        return _flag(table, stale)
+
+    if spec.type == "sql":
+        _require(spec, model, params=("query",))
+        sql = str(params["query"]).replace("{table}", _table(table).sql(dialect=dialect))
+        inner = _parse_expr(sql, dialect)
+        return exp.select(exp.alias_(exp.Count(this=exp.Star()), _FAILURES)).from_(
+            exp.Subquery(this=inner, alias=exp.TableAlias(this=exp.to_identifier("q")))
+        )
+
+    raise DefinitionError(f"unknown check type {spec.type!r} on {model!r}")

interlace/checks/runner.py

@@ -0,0 +1,103 @@
+"""Run a model's checks against its freshly built physical table.
+
+Declarative checks compile to one ``failures``-count query each and run on the
+engine. Python ``@check`` functions receive a :class:`RelationHandle` over the
+built table and pass by returning a truthy success (``True``/``0``) — return
+``False`` or a failure count to fail. A check that itself crashes is recorded
+as ``error`` status (an engine problem, not a data-quality verdict).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+import pyarrow as pa
+from sqlglot import exp
+
+from interlace.checks.builtin import build_check_query
+from interlace.checks.spec import CheckSpec
+from interlace.dsl.decorators import CheckDef
+from interlace.engines.base import EngineAdapter
+from interlace.exceptions import DefinitionError
+from interlace.graph.project import CompiledModel, CompiledProject
+from interlace.ir.relation import TableRef
+from interlace.runtime.handles import RelationHandle
+
+
+@dataclass(frozen=True)
+class CheckOutcome:
+    """The result of one check run."""
+
+    model: str
+    name: str
+    type: str
+    severity: str
+    status: str  # "passed" | "failed" | "error"
+    failures: int = 0
+    message: str | None = None
+
+    @property
+    def blocking(self) -> bool:
+        return self.status != "passed" and self.severity == "error"
+
+
+async def _run_declared(
+    spec: CheckSpec,
+    model: CompiledModel,
+    compiled: CompiledProject,
+    engine: EngineAdapter,
+    table: TableRef,
+    physical: Mapping[str, TableRef] | None,
+) -> CheckOutcome:
+    def resolve(name: str) -> TableRef:
+        upstream = compiled.models.get(name)
+        if upstream is None:
+            raise DefinitionError(f"check on {model.name!r} references unknown model {name!r}")
+        return (physical or {}).get(name, upstream.physical_table)
+
+    try:
+        query = build_check_query(spec, table, model.name, model.dialect, resolve)
+        reader = await engine.fetch(query)
+        row = reader.read_all().to_pylist()[0]
+        failures = int(row["failures"] or 0)
+    except DefinitionError:
+        raise  # a misdeclared check is a definition problem, not a data problem
+    except Exception as error:
+        return CheckOutcome(model.name, spec.name, spec.type, spec.severity, "error", message=str(error))
+    status = "passed" if failures == 0 else "failed"
+    return CheckOutcome(model.name, spec.name, spec.type, spec.severity, status, failures=failures)
+
+
+async def _run_python(check: CheckDef, engine: EngineAdapter, table: TableRef, model: str) -> CheckOutcome:
+    try:
+        query = exp.select("*").from_(table.to_expr())
+        handle = RelationHandle(model, await engine.fetch(query))
+        if inspect.iscoroutinefunction(check.fn):
+            result = await check.fn(handle)
+        else:
+            result = await asyncio.to_thread(check.fn, handle)
+    except Exception as error:
+        return CheckOutcome(model, check.name, "python", check.severity, "error", message=str(error))
+    if isinstance(result, pa.Table):  # returned failing rows: empty = pass
+        result = result.num_rows
+    if result is True or result is None or result == 0:
+        return CheckOutcome(model, check.name, "python", check.severity, "passed")
+    failures = int(result) if isinstance(result, int) else 1
+    return CheckOutcome(model, check.name, "python", check.severity, "failed", failures=failures)
+
+
+async def run_checks(
+    model: CompiledModel,
+    compiled: CompiledProject,
+    engine: EngineAdapter,
+    table: TableRef,
+    python_checks: tuple[CheckDef, ...] = (),
+    physical: Mapping[str, TableRef] | None = None,
+) -> list[CheckOutcome]:
+    """Run all of ``model``'s checks against ``table``; returns every outcome."""
+    outcomes = [await _run_declared(spec, model, compiled, engine, table, physical) for spec in model.checks]
+    outcomes += [await _run_python(check, engine, table, model.name) for check in python_checks]
+    return outcomes

interlace/checks/spec.py

@@ -0,0 +1,113 @@
+"""Check declarations.
+
+A model declares checks in its config (SQL block or ``@model(checks=[...])``)::
+
+    checks:
+      - not_null: order_id                 # shorthand: {type: column}
+      - unique: [order_id, customer_id]
+      - accepted_values: {column: status, values: [open, closed]}
+      - relationships: {column: customer_id, to: customers, field: customer_id}
+      - expression: {expression: "amount >= 0", severity: warn}
+      - row_count: {min: 1}
+      - range: {column: amount, min: 0}
+      - pattern: {column: email, regex: ".+@.+"}
+      - freshness: {column: updated_at, max_age: 2h}
+      - sql: {query: "SELECT * FROM {table} WHERE total < 0"}
+
+Each entry normalises to a :class:`CheckSpec`. Severity: ``error`` (default)
+fails the apply before promotion; ``warn``/``info`` record and continue.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from interlace.exceptions import DefinitionError
+
+SEVERITIES = frozenset({"error", "warn", "info"})
+CHECK_TYPES = frozenset(
+    {
+        "not_null",
+        "unique",
+        "accepted_values",
+        "row_count",
+        "freshness",
+        "expression",
+        "relationships",
+        "pattern",
+        "range",
+        "sql",
+    }
+)
+_COLUMN_SHORTHAND = frozenset({"not_null", "unique", "range", "pattern", "freshness", "accepted_values"})
+
+
+@dataclass(frozen=True)
+class CheckSpec:
+    """One declared check on a model."""
+
+    type: str
+    columns: tuple[str, ...] = ()  # subject column(s); empty for table-level checks
+    severity: str = "error"
+    params: dict[str, Any] = field(default_factory=dict)  # type-specific settings
+
+    @property
+    def name(self) -> str:
+        return "_".join([self.type, *self.columns]) if self.columns else self.type
+
+
+def _as_columns(value: Any) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    if isinstance(value, str):
+        return (value,)
+    if isinstance(value, Sequence):
+        return tuple(str(v) for v in value)
+    raise DefinitionError(f"check column must be a name or list of names, got {type(value).__name__}")
+
+
+def _parse_one(entry: Any, model: str) -> CheckSpec:
+    if isinstance(entry, str):  # bare "not_null" is meaningless without a column
+        raise DefinitionError(f"check {entry!r} on {model!r} needs a column or config mapping")
+    if not isinstance(entry, dict) or not entry:
+        raise DefinitionError(f"invalid check entry on {model!r}: {entry!r}")
+
+    if "type" in entry:  # explicit form: {type: not_null, column: id, ...}
+        params = dict(entry)
+        check_type = str(params.pop("type"))
+        value: Any = params.pop("column", None) or params.pop("columns", None)
+    else:  # shorthand form: {not_null: id} or {row_count: {min: 1}}
+        if len(entry) != 1:
+            raise DefinitionError(f"ambiguous check entry on {model!r}: {entry!r}; use the {{type: ...}} form")
+        check_type, value = next(iter(entry.items()))
+        check_type = str(check_type)
+        if check_type not in CHECK_TYPES:
+            raise DefinitionError(
+                f"unknown check type {check_type!r} on {model!r}; expected one of {sorted(CHECK_TYPES)}"
+            )
+        params = {}
+        if isinstance(value, dict):  # {accepted_values: {column: status, values: [...]}}
+            params = dict(value)
+            value = params.pop("column", None) or params.pop("columns", None)
+        elif check_type not in _COLUMN_SHORTHAND:
+            raise DefinitionError(f"check {check_type!r} on {model!r} needs a config mapping")
+
+    if check_type not in CHECK_TYPES:
+        raise DefinitionError(f"unknown check type {check_type!r} on {model!r}; expected one of {sorted(CHECK_TYPES)}")
+    severity = str(params.pop("severity", "error"))
+    if severity not in SEVERITIES:
+        raise DefinitionError(f"unknown check severity {severity!r} on {model!r}; expected one of {sorted(SEVERITIES)}")
+    return CheckSpec(type=check_type, columns=_as_columns(value), severity=severity, params=params)
+
+
+def parse_checks(value: Any, model: str) -> tuple[CheckSpec, ...]:
+    """Normalise a model's ``checks:`` config into :class:`CheckSpec` tuples."""
+    if value is None:
+        return ()
+    if isinstance(value, CheckSpec):
+        return (value,)
+    if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
+        raise DefinitionError(f"checks on {model!r} must be a list")
+    return tuple(entry if isinstance(entry, CheckSpec) else _parse_one(entry, model) for entry in value)

interlace/cli/__init__.py

@@ -0,0 +1,7 @@
+"""Command-line interface."""
+
+from __future__ import annotations
+
+from interlace.cli.main import app, main
+
+__all__ = ["app", "main"]