interlaced 2.8.3__tar.gz → 2.8.6__tar.gz

{interlaced-2.8.3/src/interlaced.egg-info → interlaced-2.8.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.8.3
+Version: 2.8.6
 Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT

{interlaced-2.8.3 → interlaced-2.8.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.8.3"
+version = "2.8.6"
 description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -61,7 +61,8 @@ service = [
     "msgspec>=0.18,<1.0",  # the wire types (litestar serialises msgspec structs natively)
 ]
 # Remote engines via Arrow-native ADBC transfer. postgres (+ redshift, same wire)
-# are the tested transport; snowflake/bigquery are ALPHA (wired, not run live).
+# are the tested transport. snowflake and bigquery are ALPHA: snowflake has been
+# exercised against one account, outside CI; bigquery has not been run live.
 adbc = [
     "adbc-driver-manager>=1.2,<2.0",
     "adbc-driver-postgresql>=1.2,<2.0",

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/checks/builtin.py

@@ -14,7 +14,6 @@ NULLs as pattern/range failures, which conflated the two).
 
 from __future__ import annotations
 
-import re
 from collections.abc import Callable
 from typing import Any, cast
 
@@ -24,10 +23,9 @@ from interlace.checks.spec import CheckSpec
 from interlace.exceptions import DefinitionError
 from interlace.ir.canonicalize import as_query
 from interlace.ir.relation import TableRef
+from interlace.state.interval import as_grain
 
 _FAILURES = "failures"
-_AGE_RE = re.compile(r"^\s*(\d+)\s*([smhdw])\s*$")
-_AGE_UNITS = {"s": "SECOND", "m": "MINUTE", "h": "HOUR", "d": "DAY", "w": "WEEK"}
 
 ResolveTable = Callable[[str], TableRef]
 """Maps an upstream model name to its physical table (for ``relationships``)."""
@@ -59,11 +57,24 @@ def _parse_expr(sql: str, dialect: str) -> exp.Expr:
     return parse_one(sql, read=dialect)
 
 
+def sql_interval(amount: int, unit: str) -> exp.Interval:
+    """``amount`` of a SQL unit (``HOUR``, ``DAY``, …) as an interval expression."""
+    return exp.Interval(this=exp.Literal.string(str(amount)), unit=exp.Var(this=unit))
+
+
+def max_is_stale(column: exp.Expr, interval: exp.Expr) -> exp.Expr:
+    """``max(column)`` is null or older than ``interval``. An empty table counts."""
+    newest = exp.Max(this=column)
+    threshold = exp.Sub(this=exp.CurrentTimestamp(), expression=interval)
+    return exp.or_(exp.LT(this=newest, expression=threshold), newest.copy().is_(exp.null()))
+
+
 def _interval(max_age: str) -> exp.Interval:
-    match = _AGE_RE.fullmatch(str(max_age))
-    if match is None:
-        raise DefinitionError(f"invalid max_age {max_age!r}; expected like '2h', '30m', '1d'")
-    return exp.Interval(this=exp.Literal.string(match.group(1)), unit=exp.Var(this=_AGE_UNITS[match.group(2)]))
+    try:
+        parsed = as_grain(str(max_age))
+    except ValueError as exc:
+        raise DefinitionError(f"invalid max_age {max_age!r}; expected like '2h', '30m', '1d'") from exc
+    return sql_interval(parsed.amount, parsed.sql_unit)
 
 
 def _row_predicate(spec: CheckSpec, model: str, dialect: str, resolve: ResolveTable) -> exp.Expr | None:
@@ -147,10 +158,7 @@ def build_check_query(spec: CheckSpec, table: TableRef, model: str, dialect: str
 
     if spec.type == "freshness":
         _require(spec, model, columns=1, params=("max_age",))
-        threshold = exp.Sub(this=exp.CurrentTimestamp(), expression=_interval(str(params["max_age"])))
-        newest = exp.Max(this=cols[0])
-        stale = exp.or_(exp.LT(this=newest, expression=threshold), newest.copy().is_(exp.null()))  # no rows = stale
-        return _flag(table, stale)
+        return _flag(table, max_is_stale(cols[0], _interval(str(params["max_age"]))))
 
     if spec.type == "sql":
         _require(spec, model, params=("query",))

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/cli/operate.py

@@ -285,6 +285,11 @@ def serve(
         True, "--scheduler/--no-scheduler", help="Run the scheduler loop in this process (combined daemon)."
     ),
     interval: float = typer.Option(60.0, "--interval", help="Seconds between scheduler ticks."),
+    apply_on_start: bool = typer.Option(
+        True,
+        "--apply/--no-apply",
+        help="Apply the project once at startup. --no-apply serves the warehouse as it is.",
+    ),
     allow_open: bool = typer.Option(
         False,
         "--allow-open",
@@ -294,6 +299,7 @@ def serve(
     """Run the interlace daemon: HTTP API + scheduler in one process (requires the `service` extra).
 
     Use --no-scheduler for an API-only process (run `interlace scheduler` separately).
+    The project is applied once before the loops start; pass --no-apply to skip that.
     A `cdc:` block in the project config is read here: each Postgres slot appends into its stream.
     """
     try:
@@ -339,6 +345,7 @@ def serve(
         quack_token=token or None,
         scheduler=scheduler,
         scheduler_interval=interval,
+        apply_on_start=apply_on_start,
     )
     config = uvicorn.Config(
         app,

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/config/config.py

@@ -250,6 +250,9 @@ class ProjectConfig(BaseModel):
     default_engine: str = "default"
     engines: dict[str, EngineConfig] = Field(default_factory=dict)
     state_path: str = ".interlace/state.db"  # SQLite control-plane database
+    # When set, snapshots, the queue, and the event log live in this Postgres
+    # database (schema ``interlace``) instead of ``state_path``. Not the warehouse.
+    state_url: str | None = None
     # Optional NDJSON mirror of the operator event log. Unset writes nowhere.
     # One JSON line is appended after each SQLite commit (CLI and daemon share it).
     event_log_path: str | None = None
@@ -275,6 +278,11 @@ class ProjectConfig(BaseModel):
     secrets: dict[str, SecretConfig] = Field(default_factory=dict)
     quack_token: str | None = None  # token for quack: databases (or INTERLACE_QUACK_TOKEN)
     stream_path: str = ".interlace/streams.db"  # durable stream log (SQLite WAL)
+    # When set, the stream log lives in this Postgres database (schema
+    # ``interlace_streams``) instead of ``stream_path``. A commit is durable
+    # under Postgres ``synchronous_commit`` (the default), the same promise as
+    # the SQLite WAL fsync.
+    stream_url: str | None = None
     # Databases to ATTACH to the warehouse engine at open: alias -> DuckDB attach
     # URI/path (a .duckdb file, "postgres:...", "sqlite:...", ...). Models can read
     # them and table exports can write to them as <alias>.<schema>.<table>.

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/dsl/decorators.py

@@ -9,7 +9,7 @@ Compilation, planning, and execution happen later over the registry. UK spelling
 from __future__ import annotations
 
 import re
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from contextvars import ContextVar
 from dataclasses import dataclass, field
 from typing import Any
@@ -30,6 +30,37 @@ from interlace.strategies import named_strategy
 
 ModelFn = Callable[..., Any]
 
+
+def normalize_schedule(schedule: Mapping[str, Any] | None, *, model: str) -> dict[str, str] | None:
+    """Schedules are string values. ``fresh: {column, within}`` becomes ``"column within"``.
+    ``after: [a, b]`` becomes ``"a,b"``."""
+    if not schedule:
+        return None
+    normalised: dict[str, str] = {}
+    for key, value in schedule.items():
+        if key == "fresh" and isinstance(value, dict):
+            column = value.get("column")
+            window = value.get("within")
+            if not isinstance(column, str) or not column.strip():
+                raise DefinitionError(f"model {model!r}: fresh.column must name a timestamp column")
+            if not isinstance(window, str) or not window.strip():
+                raise DefinitionError(f"model {model!r}: fresh.within must be a window like '2h'")
+            normalised[key] = f"{column.strip()} {window.strip()}"
+            continue
+        if key == "after" and isinstance(value, (list, tuple)):
+            names = [str(item).strip() for item in value]
+            if not names or any(not name for name in names):
+                raise DefinitionError(f"model {model!r}: after must name a model, or a list of models")
+            normalised[key] = ",".join(names)
+            continue
+        if key == "after" and not isinstance(value, str):
+            raise DefinitionError(f"model {model!r}: after must name a model, or a list of models")
+        if not isinstance(value, str):
+            raise DefinitionError(f"model {model!r}: schedule {key!r} must be a string")
+        normalised[key] = value
+    return normalised
+
+
 # Set while a Python model function is running. Registrations then replace a
 # model this run already created, and are recorded for the apply that builds them.
 dynamic_owner: ContextVar[str | None] = ContextVar("interlace_dynamic_owner", default=None)
@@ -130,7 +161,9 @@ class ModelDef:
     path: str | None = None  # output path for materialise: file
     format: str | None = None  # csv | parquet | json for materialise: file
     environments: tuple[str, ...] = ("prod",)  # which environments actually deliver a terminal model
-    schedule: dict[str, str] | None = None  # cron, every, watch (glob), or webhook name
+    # cron, every, watch, on_change, fresh, or webhook. ``fresh`` may be the
+    # mapping ``{column, within}`` at construction; __post_init__ stores "column within".
+    schedule: dict[str, Any] | None = None
     checks: tuple[CheckSpec, ...] = ()  # data-quality checks; error severity gates promotion
     indexes: tuple[IndexSpec, ...] = ()  # physical indexes; not part of the data fingerprint
     constraints: tuple[ConstraintSpec, ...] = ()  # physical constraints; engine-enforced, not checks
@@ -151,6 +184,7 @@ class ModelDef:
         self.constraints = parse_constraints(self.constraints, self.name)
         self.schema_policy = parse_schema_policy(self.schema_policy, self.name)
         validate_physical_allowed(self.name, self.materialise, self.indexes, self.constraints, self.schema_policy)
+        self.schedule = normalize_schedule(self.schedule, model=self.name)
 
     @property
     def is_terminal(self) -> bool:
@@ -242,7 +276,7 @@ def model(
     owner: str | None = None,
     description: str | None = None,
     columns: dict[str, str | None] | Sequence[str] | None = None,
-    schedule: dict[str, str] | None = None,
+    schedule: dict[str, Any] | None = None,
     checks: Sequence[dict[str, Any] | CheckSpec] | None = None,
     indexes: Sequence[dict[str, Any] | IndexSpec] | None = None,
     constraints: Sequence[dict[str, Any] | ConstraintSpec] | None = None,

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/graph/project.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 import inspect
 from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, cast
 
 from sqlglot import exp
 
@@ -308,7 +308,7 @@ def compile_models(  # noqa: C901
             cursor=definition.cursor,
             interval=definition.interval,
             tags=definition.tags,
-            schedule=definition.schedule,
+            schedule=cast(dict[str, str] | None, definition.schedule),  # __post_init__ stores strings
             columns=definition.columns,
             target=definition.target,
             path=definition.path,

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/plan/differ.py

@@ -338,6 +338,8 @@ async def diff(  # noqa: C901
     plan = Plan(environment=environment)
     impact: dict[str, str] = {}  # changed models only: "semantic" | "additive" | "clean"
     touched: dict[str, frozenset[str] | None] = {}  # semantic models: provably-changed columns (None = all)
+    # downstream models: the upstream columns that forced the rebuild
+    indirect: dict[str, frozenset[str]] = {}
 
     previous_snapshots = await state.get_snapshots(
         (name, fingerprint)
@@ -410,6 +412,8 @@ async def diff(  # noqa: C901
                     consumed = _consumed_columns(model.ast, dep)
                     if dep_touched is not None and consumed is not None and not (consumed & dep_touched):
                         continue  # column-pruned: reads only provably-unchanged columns
+                    if dep_touched is not None and consumed is not None:
+                        indirect[model.name] = consumed & dep_touched
                     verdict = "semantic"
                     break
                 if dep_impact == "additive" and _selects_star(model.ast):
@@ -436,8 +440,15 @@ async def diff(  # noqa: C901
         )
         if inherit:
             category = ChangeCategory.FORWARD_ONLY
+        shown = added
+        if impact.get(model.name) == "semantic":
+            own = touched.get(model.name)
+            if own:
+                shown = tuple(sorted(own))
+            elif model.name in indirect:
+                shown = tuple(sorted(indirect[model.name]))
         plan.changes.append(
-            ModelChange(model.name, ChangeType.MODIFIED, category, previous_fingerprint, model.fingerprint, added)
+            ModelChange(model.name, ChangeType.MODIFIED, category, previous_fingerprint, model.fingerprint, shown)
         )
         if inherit:  # copy-on-write: history seeds the NEW table; checks gate before views move
             snapshot = replace(

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/plan/schedule.py

@@ -25,7 +25,8 @@ from interlace.ir.relation import TableRef
 from interlace.plan.backfill import run_backfill
 from interlace.plan.plan import BackfillTask, Plan
 from interlace.plan.result import ApplyResult, ProgressCallback, build_detail, failure_detail
-from interlace.state.store import StateStore
+from interlace.scheduler.completion import after_waiters, wake_after
+from interlace.state.store import SqliteStateStore, StateStore
 
 logger = logging.getLogger("interlace.apply")
 
@@ -123,6 +124,7 @@ async def schedule_builds(  # noqa: C901
                 deps.add(ref)
         blocking[name] = deps
 
+    waiting = after_waiters(compiled)
     finished = {name: asyncio.Event() for name in per_model}
     build_slots = asyncio.Semaphore(max(1, parallelism))
 
@@ -163,6 +165,8 @@ async def schedule_builds(  # noqa: C901
         finished[name].set()
         if on_progress is not None:
             on_progress(name, "done", build_detail(result, name))
+        if isinstance(state, SqliteStateStore):
+            await wake_after(state, compiled, name, building=set(per_model), waiters=waiting)
 
     # Copied into each build task at creation, so a Python model that registers
     # models (including from a worker thread) appends to this same list.

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/project.py

@@ -375,15 +375,20 @@ class Project:
         )
 
     async def open_state(self) -> SqliteStateStore:
-        path = self.root / self.config.state_path
-        path.parent.mkdir(parents=True, exist_ok=True)
-        store = await SqliteStateStore.open(path)
+        if self.config.state_url:
+            store = await SqliteStateStore.open_postgres(self.config.state_url)
+        else:
+            path = self.root / self.config.state_path
+            path.parent.mkdir(parents=True, exist_ok=True)
+            store = await SqliteStateStore.open(path)
         if self.config.event_log_path:
             target = Path(self.config.event_log_path)
             store.event_log_path = str(target if target.is_absolute() else self.root / target)
         return store
 
     async def open_stream_log(self) -> SqliteStreamLog:
+        if self.config.stream_url:
+            return await SqliteStreamLog.open_postgres(self.config.stream_url)
         path = self.root / self.config.stream_path
         path.parent.mkdir(parents=True, exist_ok=True)
         return await SqliteStreamLog.open(path)

interlaced-2.8.6/src/interlace/scheduler/__init__.py

@@ -0,0 +1,39 @@
+"""Orchestration: triggers, the trigger engine, and the worker (queue = state store).
+
+Submodules import each other. This package does not load them up front: importing
+``interlace.scheduler.completion`` must not pull in the worker, which imports the
+apply path that imports completion.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+__all__ = [
+    "CronTrigger",
+    "IntervalTrigger",
+    "RunRequest",
+    "Trigger",
+    "TriggerEngine",
+    "build_triggers",
+    "drain",
+]
+
+_EXPORTS: dict[str, tuple[str, str]] = {
+    "CronTrigger": ("interlace.scheduler.triggers", "CronTrigger"),
+    "IntervalTrigger": ("interlace.scheduler.triggers", "IntervalTrigger"),
+    "RunRequest": ("interlace.scheduler.triggers", "RunRequest"),
+    "Trigger": ("interlace.scheduler.triggers", "Trigger"),
+    "TriggerEngine": ("interlace.scheduler.engine", "TriggerEngine"),
+    "build_triggers": ("interlace.scheduler.engine", "build_triggers"),
+    "drain": ("interlace.scheduler.worker", "drain"),
+}
+
+
+def __getattr__(name: str) -> Any:
+    found = _EXPORTS.get(name)
+    if found is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module_name, attr = found
+    return getattr(importlib.import_module(module_name), attr)

interlaced-2.8.6/src/interlace/scheduler/completion.py

@@ -0,0 +1,132 @@
+"""Run a model when another model finishes.
+
+``schedule: {after: …}`` is not a clock tick. Apply calls :func:`wake_after` once
+a build reaches ``model.done``. The trigger engine only validates the graph.
+"""
+
+from __future__ import annotations
+
+import time
+
+from interlace.exceptions import DefinitionError
+from interlace.graph.project import CompiledProject
+from interlace.state.store import SqliteStateStore
+
+_SCHEDULE_KINDS = ("cron", "every", "watch", "on_change", "fresh", "after", "webhook")
+
+
+def schedule_kind(model: str, schedule: dict[str, str]) -> tuple[str, str]:
+    """The one schedule key and its value.
+
+    Cron, interval, file watch, table change, freshness, upstream completion, or webhook.
+    """
+    present = [key for key in _SCHEDULE_KINDS if key in schedule]
+    if len(present) != 1:
+        raise DefinitionError(
+            f"model {model!r}: schedule needs exactly one of {', '.join(_SCHEDULE_KINDS)}",
+            details={"schedule": schedule},
+        )
+    return present[0], schedule[present[0]]
+
+
+def scheduled_closure(project: CompiledProject, models: list[str]) -> list[str]:
+    """The triggered models plus every model downstream of them.
+
+    A cron, interval, file watch, table change, freshness, upstream completion, or webhook
+    means that model's inputs changed. Downstream snapshots would otherwise keep the
+    previous build. An explicit ``run --select`` is not expanded here: ``model``,
+    ``model+``, and ``+model`` stay exactly what was asked for.
+    """
+    chosen: set[str] = set()
+    for name in models:
+        chosen.add(name)
+        if name in project.models:
+            chosen |= project.graph.descendants(name)
+    return sorted(chosen)
+
+
+def after_waiters(project: CompiledProject) -> dict[str, list[str]]:
+    """Map an upstream model to the models that run when it finishes.
+
+    ``schedule: {after: raw}`` or ``{after: [raw, staging]}`` (stored as
+    ``"raw,staging"``). A model cannot name itself, and the ``after`` edges
+    cannot cycle — otherwise two models would enqueue each other forever.
+    """
+    found: dict[str, list[str]] = {}
+    for model in project.models.values():
+        if not model.schedule or "after" not in model.schedule:
+            continue
+        kind, value = schedule_kind(model.name, model.schedule)
+        if kind != "after":
+            continue
+        names = list(dict.fromkeys(part.strip() for part in value.split(",") if part.strip()))
+        if not names:
+            raise DefinitionError(f"model {model.name!r}: after must name a model")
+        if model.name in names:
+            raise DefinitionError(f"model {model.name!r}: after cannot name itself")
+        missing = [name for name in names if name not in project.models]
+        if missing:
+            raise DefinitionError(
+                f"model {model.name!r}: after names unknown models: {', '.join(missing)}",
+                details={"missing": missing},
+            )
+        for name in names:
+            found.setdefault(name, []).append(model.name)
+    _reject_after_cycles(found)
+    return found
+
+
+def _reject_after_cycles(waiters: dict[str, list[str]]) -> None:
+    visiting: set[str] = set()
+    visited: set[str] = set()
+
+    def walk(node: str) -> None:
+        if node in visited:
+            return
+        if node in visiting:
+            raise DefinitionError(f"schedule after has a cycle through {node!r}")
+        visiting.add(node)
+        for nxt in waiters.get(node, ()):
+            walk(nxt)
+        visiting.remove(node)
+        visited.add(node)
+
+    for node in list(waiters):
+        walk(node)
+
+
+async def wake_after(
+    store: SqliteStateStore,
+    project: CompiledProject,
+    upstream: str,
+    *,
+    building: set[str],
+    waiters: dict[str, list[str]] | None = None,
+    stamp: str | None = None,
+) -> int:
+    """Enqueue models waiting on ``upstream`` after it reaches ``model.done``.
+
+    ``building`` is the set already in this apply, so a model the plan is
+    constructing is not queued again. One stamp shared by the waiters of a
+    single finish: a retry of the same finish dedupes, a later finish does not.
+    Returns how many runs were newly enqueued.
+    """
+    table = after_waiters(project) if waiters is None else waiters
+    pending = [name for name in table.get(upstream, ()) if name not in building]
+    if not pending:
+        return 0
+    order = {model.name: index for index, model in enumerate(project.ordered())}
+    pending.sort(key=lambda name: order.get(name, 0))
+    token = stamp if stamp is not None else str(time.time_ns())
+    covered: set[str] = set(building)
+    enqueued = 0
+    for name in pending:
+        if name in covered:
+            continue
+        selector = scheduled_closure(project, [name])
+        key = f"after:{name}:{upstream}:{token}"
+        if await store.enqueue_run(key, selector, None, 0):
+            enqueued += 1
+            await store.append_event("run.enqueued", entity=key, payload={"models": selector, "after": upstream})
+        covered.update(selector)
+    return enqueued

{interlaced-2.8.3 → interlaced-2.8.6}/src/interlace/scheduler/daemon.py

@@ -205,6 +205,35 @@ async def flusher_loop(state: Any, *, flush_interval: float) -> None:
             await asyncio.sleep(1.0)
 
 
+async def startup_apply(state: Any) -> None:
+    """Apply the project once, before the scheduler starts.
+
+    A breaking or blocked plan is logged and the process still serves. Any other
+    failure propagates: a warehouse that cannot be opened should not pretend to
+    be up.
+    """
+    from interlace.exceptions import PlanError
+    from interlace.plan.orchestrate import plan_and_apply
+
+    try:
+        _plan, result = await plan_and_apply(
+            state.compiled,
+            environment=state.environment,
+            project=state.project,
+            engines=state.engines,
+            state=state.store,
+            lock_owner=state.lock_owner,
+            connections=state.connections,
+            on_compiled=lambda fresh: publish_compiled(state, fresh),
+        )
+    except PlanError as exc:
+        logger.error("startup apply refused: %s", exc.message)
+        return
+    if result is None:
+        return
+    logger.info("startup apply built %s", ", ".join(result.built) or "nothing")
+
+
 async def scheduler_loop(
     state: Any,
     *,
@@ -220,9 +249,12 @@ async def scheduler_loop(
         try:
             await reload_if_stale(state)
             compiled = state.compiled
-            await TriggerEngine(build_triggers(compiled, root=state.project.root), state.store, compiled).tick(
-                datetime.now()
-            )
+            await TriggerEngine(
+                build_triggers(compiled, root=state.project.root, environment=state.environment),
+                state.store,
+                compiled,
+                engines=state.engines,
+            ).tick(datetime.now())
             if asyncio.get_running_loop().time() >= next_trim:
                 await state.store.trim_logs()
                 next_trim = asyncio.get_running_loop().time() + 6 * 3600