interlaced 2.8.3__tar.gz → 2.8.4__tar.gz

{interlaced-2.8.3/src/interlaced.egg-info → interlaced-2.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.8.3
+Version: 2.8.4
 Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT

{interlaced-2.8.3 → interlaced-2.8.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.8.3"
+version = "2.8.4"
 description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -61,7 +61,8 @@ service = [
     "msgspec>=0.18,<1.0",  # the wire types (litestar serialises msgspec structs natively)
 ]
 # Remote engines via Arrow-native ADBC transfer. postgres (+ redshift, same wire)
-# are the tested transport; snowflake/bigquery are ALPHA (wired, not run live).
+# are the tested transport. snowflake and bigquery are ALPHA: snowflake has been
+# exercised against one account, outside CI; bigquery has not been run live.
 adbc = [
     "adbc-driver-manager>=1.2,<2.0",
     "adbc-driver-postgresql>=1.2,<2.0",

{interlaced-2.8.3 → interlaced-2.8.4}/src/interlace/checks/builtin.py

@@ -14,7 +14,6 @@ NULLs as pattern/range failures, which conflated the two).
 
 from __future__ import annotations
 
-import re
 from collections.abc import Callable
 from typing import Any, cast
 
@@ -24,10 +23,9 @@ from interlace.checks.spec import CheckSpec
 from interlace.exceptions import DefinitionError
 from interlace.ir.canonicalize import as_query
 from interlace.ir.relation import TableRef
+from interlace.state.interval import as_grain
 
 _FAILURES = "failures"
-_AGE_RE = re.compile(r"^\s*(\d+)\s*([smhdw])\s*$")
-_AGE_UNITS = {"s": "SECOND", "m": "MINUTE", "h": "HOUR", "d": "DAY", "w": "WEEK"}
 
 ResolveTable = Callable[[str], TableRef]
 """Maps an upstream model name to its physical table (for ``relationships``)."""
@@ -59,11 +57,24 @@ def _parse_expr(sql: str, dialect: str) -> exp.Expr:
     return parse_one(sql, read=dialect)
 
 
+def sql_interval(amount: int, unit: str) -> exp.Interval:
+    """``amount`` of a SQL unit (``HOUR``, ``DAY``, …) as an interval expression."""
+    return exp.Interval(this=exp.Literal.string(str(amount)), unit=exp.Var(this=unit))
+
+
+def max_is_stale(column: exp.Expr, interval: exp.Expr) -> exp.Expr:
+    """``max(column)`` is null or older than ``interval``. An empty table counts."""
+    newest = exp.Max(this=column)
+    threshold = exp.Sub(this=exp.CurrentTimestamp(), expression=interval)
+    return exp.or_(exp.LT(this=newest, expression=threshold), newest.copy().is_(exp.null()))
+
+
 def _interval(max_age: str) -> exp.Interval:
-    match = _AGE_RE.fullmatch(str(max_age))
-    if match is None:
-        raise DefinitionError(f"invalid max_age {max_age!r}; expected like '2h', '30m', '1d'")
-    return exp.Interval(this=exp.Literal.string(match.group(1)), unit=exp.Var(this=_AGE_UNITS[match.group(2)]))
+    try:
+        parsed = as_grain(str(max_age))
+    except ValueError as exc:
+        raise DefinitionError(f"invalid max_age {max_age!r}; expected like '2h', '30m', '1d'") from exc
+    return sql_interval(parsed.amount, parsed.sql_unit)
 
 
 def _row_predicate(spec: CheckSpec, model: str, dialect: str, resolve: ResolveTable) -> exp.Expr | None:
@@ -147,10 +158,7 @@ def build_check_query(spec: CheckSpec, table: TableRef, model: str, dialect: str
 
     if spec.type == "freshness":
         _require(spec, model, columns=1, params=("max_age",))
-        threshold = exp.Sub(this=exp.CurrentTimestamp(), expression=_interval(str(params["max_age"])))
-        newest = exp.Max(this=cols[0])
-        stale = exp.or_(exp.LT(this=newest, expression=threshold), newest.copy().is_(exp.null()))  # no rows = stale
-        return _flag(table, stale)
+        return _flag(table, max_is_stale(cols[0], _interval(str(params["max_age"]))))
 
     if spec.type == "sql":
         _require(spec, model, params=("query",))

{interlaced-2.8.3 → interlaced-2.8.4}/src/interlace/cli/operate.py

@@ -285,6 +285,11 @@ def serve(
         True, "--scheduler/--no-scheduler", help="Run the scheduler loop in this process (combined daemon)."
     ),
     interval: float = typer.Option(60.0, "--interval", help="Seconds between scheduler ticks."),
+    apply_on_start: bool = typer.Option(
+        True,
+        "--apply/--no-apply",
+        help="Apply the project once at startup. --no-apply serves the warehouse as it is.",
+    ),
     allow_open: bool = typer.Option(
         False,
         "--allow-open",
@@ -294,6 +299,7 @@ def serve(
     """Run the interlace daemon: HTTP API + scheduler in one process (requires the `service` extra).
 
     Use --no-scheduler for an API-only process (run `interlace scheduler` separately).
+    The project is applied once before the loops start; pass --no-apply to skip that.
     A `cdc:` block in the project config is read here: each Postgres slot appends into its stream.
     """
     try:
@@ -339,6 +345,7 @@ def serve(
         quack_token=token or None,
         scheduler=scheduler,
         scheduler_interval=interval,
+        apply_on_start=apply_on_start,
     )
     config = uvicorn.Config(
         app,

{interlaced-2.8.3 → interlaced-2.8.4}/src/interlace/dsl/decorators.py

@@ -9,7 +9,7 @@ Compilation, planning, and execution happen later over the registry. UK spelling
 from __future__ import annotations
 
 import re
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from contextvars import ContextVar
 from dataclasses import dataclass, field
 from typing import Any
@@ -30,6 +30,28 @@ from interlace.strategies import named_strategy
 
 ModelFn = Callable[..., Any]
 
+
+def normalize_schedule(schedule: Mapping[str, Any] | None, *, model: str) -> dict[str, str] | None:
+    """Schedules are string values. ``fresh: {column, within}`` becomes ``"column within"``."""
+    if not schedule:
+        return None
+    normalised: dict[str, str] = {}
+    for key, value in schedule.items():
+        if key == "fresh" and isinstance(value, dict):
+            column = value.get("column")
+            window = value.get("within")
+            if not isinstance(column, str) or not column.strip():
+                raise DefinitionError(f"model {model!r}: fresh.column must name a timestamp column")
+            if not isinstance(window, str) or not window.strip():
+                raise DefinitionError(f"model {model!r}: fresh.within must be a window like '2h'")
+            normalised[key] = f"{column.strip()} {window.strip()}"
+            continue
+        if not isinstance(value, str):
+            raise DefinitionError(f"model {model!r}: schedule {key!r} must be a string")
+        normalised[key] = value
+    return normalised
+
+
 # Set while a Python model function is running. Registrations then replace a
 # model this run already created, and are recorded for the apply that builds them.
 dynamic_owner: ContextVar[str | None] = ContextVar("interlace_dynamic_owner", default=None)
@@ -130,7 +152,9 @@ class ModelDef:
     path: str | None = None  # output path for materialise: file
     format: str | None = None  # csv | parquet | json for materialise: file
     environments: tuple[str, ...] = ("prod",)  # which environments actually deliver a terminal model
-    schedule: dict[str, str] | None = None  # cron, every, watch (glob), or webhook name
+    # cron, every, watch, on_change, fresh, or webhook. ``fresh`` may be the
+    # mapping ``{column, within}`` at construction; __post_init__ stores "column within".
+    schedule: dict[str, Any] | None = None
     checks: tuple[CheckSpec, ...] = ()  # data-quality checks; error severity gates promotion
     indexes: tuple[IndexSpec, ...] = ()  # physical indexes; not part of the data fingerprint
     constraints: tuple[ConstraintSpec, ...] = ()  # physical constraints; engine-enforced, not checks
@@ -151,6 +175,7 @@ class ModelDef:
         self.constraints = parse_constraints(self.constraints, self.name)
         self.schema_policy = parse_schema_policy(self.schema_policy, self.name)
         validate_physical_allowed(self.name, self.materialise, self.indexes, self.constraints, self.schema_policy)
+        self.schedule = normalize_schedule(self.schedule, model=self.name)
 
     @property
     def is_terminal(self) -> bool:
@@ -242,7 +267,7 @@ def model(
     owner: str | None = None,
     description: str | None = None,
     columns: dict[str, str | None] | Sequence[str] | None = None,
-    schedule: dict[str, str] | None = None,
+    schedule: dict[str, Any] | None = None,
     checks: Sequence[dict[str, Any] | CheckSpec] | None = None,
     indexes: Sequence[dict[str, Any] | IndexSpec] | None = None,
     constraints: Sequence[dict[str, Any] | ConstraintSpec] | None = None,

{interlaced-2.8.3 → interlaced-2.8.4}/src/interlace/graph/project.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 import inspect
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, cast
 
 from sqlglot import exp
 
@@ -308,7 +308,7 @@ def compile_models(  # noqa: C901
             cursor=definition.cursor,
             interval=definition.interval,
             tags=definition.tags,
-            schedule=definition.schedule,
+            schedule=cast(dict[str, str] | None, definition.schedule),  # __post_init__ stores strings
             columns=definition.columns,
             target=definition.target,
             path=definition.path,

{interlaced-2.8.3 → interlaced-2.8.4}/src/interlace/scheduler/daemon.py

@@ -205,6 +205,35 @@ async def flusher_loop(state: Any, *, flush_interval: float) -> None:
             await asyncio.sleep(1.0)
 
 
+async def startup_apply(state: Any) -> None:
+    """Apply the project once, before the scheduler starts.
+
+    A breaking or blocked plan is logged and the process still serves. Any other
+    failure propagates: a warehouse that cannot be opened should not pretend to
+    be up.
+    """
+    from interlace.exceptions import PlanError
+    from interlace.plan.orchestrate import plan_and_apply
+
+    try:
+        _plan, result = await plan_and_apply(
+            state.compiled,
+            environment=state.environment,
+            project=state.project,
+            engines=state.engines,
+            state=state.store,
+            lock_owner=state.lock_owner,
+            connections=state.connections,
+            on_compiled=lambda fresh: publish_compiled(state, fresh),
+        )
+    except PlanError as exc:
+        logger.error("startup apply refused: %s", exc.message)
+        return
+    if result is None:
+        return
+    logger.info("startup apply built %s", ", ".join(result.built) or "nothing")
+
+
 async def scheduler_loop(
     state: Any,
     *,
@@ -220,9 +249,12 @@ async def scheduler_loop(
         try:
             await reload_if_stale(state)
             compiled = state.compiled
-            await TriggerEngine(build_triggers(compiled, root=state.project.root), state.store, compiled).tick(
-                datetime.now()
-            )
+            await TriggerEngine(
+                build_triggers(compiled, root=state.project.root, environment=state.environment),
+                state.store,
+                compiled,
+                engines=state.engines,
+            ).tick(datetime.now())
             if asyncio.get_running_loop().time() >= next_trim:
                 await state.store.trim_logs()
                 next_trim = asyncio.get_running_loop().time() + 6 * 3600

interlaced-2.8.4/src/interlace/scheduler/engine.py

@@ -0,0 +1,254 @@
+"""The trigger engine — we own the scheduling loop.
+
+Each tick, every trigger is asked what's due given when it last fired; due runs
+are enqueued onto the durable work queue and the trigger's last-fired time is
+persisted. State lives in our state DB (not an external scheduler's jobstore), so
+it survives restarts and is unified with runs and snapshots.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import cast
+
+from sqlglot import exp
+
+from interlace.engines.base import EngineAdapter, relation_is_absent
+from interlace.engines.registry import EngineRegistry
+from interlace.exceptions import DefinitionError
+from interlace.graph.project import CompiledModel, CompiledProject
+from interlace.ir.canonicalize import table_references
+from interlace.ir.layout import env_view
+from interlace.ir.relation import TableRef
+from interlace.scheduler.triggers import (
+    ABSENT,
+    CronTrigger,
+    FreshTrigger,
+    IntervalTrigger,
+    OnChangeTrigger,
+    Trigger,
+    WatchTrigger,
+)
+from interlace.state.interval import Grain, as_grain, parse_grain
+from interlace.state.store import SqliteStateStore
+
+_SCHEDULE_KINDS = ("cron", "every", "watch", "on_change", "fresh", "webhook")
+_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+def schedule_kind(model: str, schedule: dict[str, str]) -> tuple[str, str]:
+    """The one schedule key and its value. Cron, interval, file watch, table change, freshness, or webhook."""
+    present = [key for key in _SCHEDULE_KINDS if key in schedule]
+    if len(present) != 1:
+        raise DefinitionError(
+            f"model {model!r}: schedule needs exactly one of {', '.join(_SCHEDULE_KINDS)}",
+            details={"schedule": schedule},
+        )
+    return present[0], schedule[present[0]]
+
+
+def webhook_targets(project: CompiledProject) -> dict[str, str]:
+    """Map a webhook name to the model ``POST /hooks/{name}`` enqueues."""
+    found: dict[str, str] = {}
+    for model in project.models.values():
+        if not model.schedule or "webhook" not in model.schedule:
+            continue
+        kind, name = schedule_kind(model.name, model.schedule)
+        if kind != "webhook":
+            continue
+        other = found.get(name)
+        if other is not None:
+            raise DefinitionError(f"webhook {name!r} is declared by both {other!r} and {model.name!r}")
+        found[name] = model.name
+    return found
+
+
+def build_triggers(project: CompiledProject, *, root: Path | None = None, environment: str = "prod") -> list[Trigger]:
+    """Construct triggers from each model's ``schedule`` config.
+
+    ``root`` is the project directory a ``watch:`` glob is relative to.
+    ``environment`` names the view an ``on_change`` or ``fresh`` schedule reads when its
+    source is another model. Webhook schedules are validated here and enqueued
+    by ``POST /hooks/{name}``, so they do not become ticking triggers.
+    """
+    triggers: list[Trigger] = []
+    for model in project.models.values():
+        trigger = _trigger_for(model, project, root, environment)
+        if trigger is not None:
+            triggers.append(trigger)
+    webhook_targets(project)  # reject two models sharing one hook name
+    return triggers
+
+
+def _trigger_for(model: CompiledModel, project: CompiledProject, root: Path | None, environment: str) -> Trigger | None:
+    schedule = model.schedule
+    if not schedule:
+        return None
+    kind, value = schedule_kind(model.name, schedule)
+    if kind == "cron":
+        return CronTrigger(model.name, value)
+    if kind == "every":
+        return IntervalTrigger(model.name, parse_grain(value))
+    if kind == "watch":
+        if root is None:
+            raise DefinitionError(f"model {model.name!r}: a watch schedule needs the project root")
+        return WatchTrigger(model.name, value, root)
+    if kind == "on_change":
+        table, column, engine = resolve_column(model, project, environment, value)
+        return OnChangeTrigger(model.name, column, table, engine)
+    if kind == "fresh":
+        column_spec, window = _fresh_parts(model.name, value)
+        table, column, engine = resolve_column(model, project, environment, column_spec, label="fresh")
+        return FreshTrigger(model.name, column, table, engine, _fresh_grain(model.name, window))
+    return None
+
+
+def _fresh_parts(model: str, value: str) -> tuple[str, str]:
+    column_spec, _, window = value.strip().rpartition(" ")
+    if not column_spec.strip() or not window.strip():
+        raise DefinitionError(f"model {model!r}: fresh {value!r} must look like 'updated_at 2h'")
+    return column_spec.strip(), window.strip()
+
+
+def _fresh_grain(model: str, window: str) -> Grain:
+    try:
+        return as_grain(window)
+    except ValueError as exc:
+        raise DefinitionError(f"model {model!r}: {exc}") from exc
+
+
+def resolve_column(
+    model: CompiledModel, project: CompiledProject, environment: str, value: str, *, label: str = "on_change"
+) -> tuple[TableRef, str, str]:
+    """The table, column, and engine a column schedule probes.
+
+    A bare column is read from the one table the model reads. ``table.column``
+    and ``schema.table.column`` name it. A name that is a model is that model's
+    environment view; anything else is a table on the model's own engine.
+    """
+    parts = value.split(".")
+    if not parts or any(not _IDENT.match(part) for part in parts):
+        raise DefinitionError(
+            f"model {model.name!r}: {label} {value!r} must be a column, table.column, or schema.table.column"
+        )
+    column = parts[-1]
+    if len(parts) == 1:
+        table, engine = _infer_change_table(model, project, environment, label=label)
+        return table, column, engine
+    if len(parts) == 2:
+        schema = None
+        relation = parts[0]
+    elif len(parts) == 3:
+        schema, relation = parts[0], parts[1]
+    else:
+        raise DefinitionError(
+            f"model {model.name!r}: {label} {value!r} must be a column, table.column, or schema.table.column"
+        )
+    qualified = f"{schema}.{relation}" if schema else relation
+    if qualified in project.models:
+        upstream = project.models[qualified]
+        return env_view(environment, qualified), column, upstream.engine
+    return TableRef(schema=schema or "main", name=relation), column, model.engine
+
+
+def _infer_change_table(
+    model: CompiledModel, project: CompiledProject, environment: str, *, label: str = "on_change"
+) -> tuple[TableRef, str]:
+    refs = table_references(model.ast) if model.ast is not None else []
+    external = [ref for ref in refs if ref not in project.models]
+    models = [ref for ref in refs if ref in project.models]
+    if model.ast is None:
+        external = []
+        models = list(model.dependencies)
+    if len(external) == 1 and not models:
+        ref = external[0]
+        schema, _, name = ref.rpartition(".")
+        return TableRef(schema=schema or "main", name=name), model.engine
+    if not external and len(models) == 1:
+        upstream = project.models[models[0]]
+        return env_view(environment, models[0]), upstream.engine
+    raise DefinitionError(
+        f"model {model.name!r}: {label} needs one source table; write table.column or schema.table.column",
+        details={"reads": refs or list(model.dependencies)},
+    )
+
+
+def scheduled_closure(project: CompiledProject, models: list[str]) -> list[str]:
+    """The triggered models plus every model downstream of them.
+
+    A cron, interval, file watch, table change, freshness, or webhook means that model's inputs changed.
+    Downstream snapshots would otherwise keep the previous build. An explicit
+    ``run --select`` is not expanded here: ``model``, ``model+``, and ``+model``
+    stay exactly what was asked for.
+    """
+    chosen: set[str] = set()
+    for name in models:
+        chosen.add(name)
+        if name in project.models:
+            chosen |= project.graph.descendants(name)
+    return sorted(chosen)
+
+
+class TriggerEngine:
+    """Evaluates triggers on each tick and enqueues due runs."""
+
+    def __init__(
+        self,
+        triggers: list[Trigger],
+        store: SqliteStateStore,
+        project: CompiledProject,
+        *,
+        engines: EngineRegistry | EngineAdapter | None = None,
+    ) -> None:
+        self.triggers = triggers
+        self.store = store
+        self.project = project
+        self.engines = engines
+
+    async def tick(self, now: datetime) -> int:
+        """Enqueue all runs due at ``now``; returns how many were newly enqueued."""
+        enqueued = 0
+        for trigger in self.triggers:
+            last_fired = await self.store.get_trigger_last_fired(trigger.id)
+            requests = await trigger.due(now, last_fired, self)
+            for request in requests:
+                selector = scheduled_closure(self.project, request.flow_selector)
+                partition = (
+                    (request.partition.start.isoformat(), request.partition.end.isoformat())
+                    if request.partition is not None
+                    else None
+                )
+                if await self.store.enqueue_run(request.idempotency_key, selector, partition, request.priority):
+                    enqueued += 1
+                    await self.store.append_event(
+                        "run.enqueued", entity=request.idempotency_key, payload={"models": selector}
+                    )
+            if requests:
+                await self.store.set_trigger_last_fired(trigger.id, now)
+        return enqueued
+
+    async def scalar(self, engine: str, table: TableRef, query: exp.Expression) -> object | None:
+        """First cell of ``query``. ``ABSENT`` when the table is not there yet; ``None`` when the cell is NULL."""
+        adapter = self._engine(engine)
+        if not await adapter.table_exists(table):
+            return ABSENT
+        try:
+            reader = await adapter.fetch(query)
+        except Exception as exc:
+            if relation_is_absent(exc):
+                return ABSENT
+            raise
+        rows = reader.read_all()
+        if rows.num_rows == 0:
+            return ABSENT
+        return cast(object | None, rows.column(0)[0].as_py())
+
+    def _engine(self, name: str) -> EngineAdapter:
+        engines = self.engines
+        if isinstance(engines, EngineRegistry):
+            return engines.get(name)
+        if isinstance(engines, EngineAdapter):
+            return engines
+        raise DefinitionError("a column schedule needs the project's engines")