interlaced 2.0.0a2__py3-none-any.whl

interlace/plan/differ.py

@@ -0,0 +1,211 @@
+"""Compute a plan by diffing a compiled project against an environment.
+
+For each model, compares the desired (compiled) fingerprint against the
+fingerprint currently promoted in the target environment, classifying it as
+ADDED / MODIFIED / REMOVED / UNCHANGED. A MODIFIED change is further classified
+BREAKING vs NON_BREAKING, and each changed model gets an *impact* that drives
+what actually gets rebuilt:
+
+- ``semantic`` — the data of pre-existing columns may differ (changed
+  expressions, filters, strategy config, Python source, or any semantic
+  upstream). Rebuilt; downstream inherits BREAKING.
+- ``additive`` — existing columns are provably identical, new columns appeared
+  (strictly additive projections). Rebuilt; downstream inherits NON_BREAKING.
+- ``clean`` — output is provably identical (only upstream fingerprints moved,
+  and no additive upstream leaks in). **Not rebuilt**: the new snapshot reuses
+  the previous physical table and the environment view repoints to it.
+
+The clean case is the improvement over sqlmesh's model-granular invalidation.
+It needs no column lineage: an indirectly-changed model's SQL is unchanged and
+was previously valid, so it cannot reference newly-added upstream columns —
+the only leak is a projection ``*`` (which inherits new columns), and Python
+models, which see whole upstream tables and are treated as ``*``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from sqlglot import exp
+
+from interlace.graph.project import CompiledModel, CompiledProject
+from interlace.ir.canonicalize import parse
+from interlace.ir.fingerprint import canonical_sql
+from interlace.plan.plan import ChangeType, ModelChange, Plan, ViewSwap, collect_transfers, env_view, schedule_build
+from interlace.state.snapshot import ChangeCategory, Snapshot
+from interlace.state.store import StateStore
+
+
+def snapshot_of(model: CompiledModel, category: ChangeCategory) -> Snapshot:
+    """Build the persistable Snapshot for a compiled model's new version."""
+    return Snapshot(
+        name=model.name,
+        fingerprint=model.fingerprint,
+        metadata_hash=model.metadata_hash,
+        physical_table=model.physical_table,
+        change_category=category,
+        local_fingerprint=model.local_fingerprint,
+        definition_sql=model.definition_sql,
+        engine=model.engine,
+    )
+
+
+def _projection_map(ast: exp.Expression | None) -> dict[str, str] | None:
+    """Map output column name -> expression SQL for a simple SELECT, or None if undeterminable."""
+    if not isinstance(ast, exp.Select):
+        return None
+    columns: dict[str, str] = {}
+    for projection in ast.selects:
+        if projection.find(exp.Star) is not None:
+            return None  # star -> output columns unknown without a schema
+        expr = projection.this if isinstance(projection, exp.Alias) else projection
+        columns[projection.alias_or_name] = expr.sql()
+    return columns
+
+
+def _added_columns(previous_sql: str | None, ast: exp.Expression | None) -> tuple[str, ...] | None:
+    """The strictly-added output columns of a direct change, or None if the change
+    is not provably additive (i.e. pre-existing column data may differ)."""
+    if previous_sql is None or not isinstance(ast, exp.Select):
+        return None
+    previous = parse(previous_sql)
+    previous_map = _projection_map(previous)
+    current_map = _projection_map(ast)
+    if previous_map is None or current_map is None:
+        return None
+    added = [name for name in current_map if name not in previous_map]
+    if not added or any(current_map.get(name) != expr for name, expr in previous_map.items()):
+        return None  # nothing added, or an existing projection changed/was removed
+    # strict: with the added projections removed, everything else (FROM, WHERE,
+    # GROUP BY, ...) must be identical — a filter change is never additive
+    stripped = ast.copy()
+    stripped.set("expressions", [p for p in stripped.selects if p.alias_or_name not in set(added)])
+    if canonical_sql(stripped) != canonical_sql(previous):
+        return None
+    return tuple(added)
+
+
+def _selects_star(ast: exp.Expression | None) -> bool:
+    """Whether the model's output can inherit new upstream columns."""
+    if ast is None:
+        return True  # Python models read whole upstream tables
+    for select in ast.find_all(exp.Select):
+        for projection in select.selects:
+            if isinstance(projection, exp.Star):
+                return True
+            if isinstance(projection, exp.Column) and isinstance(projection.this, exp.Star):
+                return True  # qualified: t.*
+    return False
+
+
+def _schedule_reuse(plan: Plan, model: CompiledModel, previous: Snapshot, environment: str) -> None:
+    """Record the new fingerprint over the previous physical table; build nothing."""
+    if model.materialise == "ephemeral":
+        return  # never physical; promotion alone carries it
+    snapshot = replace(
+        snapshot_of(model, ChangeCategory.NON_BREAKING),
+        physical_table=previous.physical_table,
+        intervals=previous.intervals,
+    )
+    plan.reuses.append(snapshot)
+    if model.export is None and model.materialise in ("table", "view"):
+        plan.virtual_updates.append(
+            ViewSwap(env_view(environment, model.name), previous.physical_table, engine=model.engine)
+        )
+
+
+_HISTORY_STRATEGIES = frozenset({"merge_by_key", "full_merge", "scd_type_2", "scd2", "incremental_by_time"})
+"""Strategies whose targets accumulate state a rebuild would destroy."""
+
+
+async def diff(
+    compiled: CompiledProject,
+    environment: str,
+    state: StateStore,
+    *,
+    select: set[str] | None = None,
+    forward_only: bool = False,
+) -> Plan:
+    """Diff the compiled project against ``environment`` and return the plan.
+
+    ``select`` limits which models are scheduled and promoted (None = all). Impact
+    classification still runs over the whole graph so downstream categories are correct.
+
+    ``forward_only``: modified models whose strategy accumulates history
+    (merge_by_key / full_merge / scd_type_2 / incremental_by_time) inherit their
+    previous physical table and interval ledger instead of starting fresh — the
+    new logic applies going forward, history survives. Requires the new query to
+    stay shape-compatible with the existing table.
+    """
+    selected = set(compiled.models) if select is None else select
+    current = await state.get_environment(environment)
+    plan = Plan(environment=environment)
+    impact: dict[str, str] = {}  # changed models only: "semantic" | "additive" | "clean"
+
+    for model in compiled.ordered():  # topo order: upstream impact known before downstream
+        previous_fingerprint = current.get(model.name)
+
+        if previous_fingerprint is None:
+            impact[model.name] = "semantic"
+            if model.name in selected:
+                plan.changes.append(ModelChange(model.name, ChangeType.ADDED, None, None, model.fingerprint))
+                schedule_build(plan, model, snapshot_of(model, ChangeCategory.BREAKING), environment)
+            continue
+
+        if previous_fingerprint == model.fingerprint:
+            continue  # unchanged
+
+        previous = await state.get_snapshot(model.name, previous_fingerprint)
+        added: tuple[str, ...] = ()
+        if previous is None or previous.local_fingerprint != model.local_fingerprint:
+            # direct change: always rebuilds itself; additive-only narrows downstream impact
+            columns = _added_columns(previous.definition_sql if previous else None, model.ast)
+            semantic = columns is None
+            added = columns or ()
+            category = ChangeCategory.BREAKING if semantic else ChangeCategory.NON_BREAKING
+            impact[model.name] = "semantic" if semantic else "additive"
+            rebuild = True
+        else:  # indirect: only upstream fingerprints moved
+            upstream = [impact.get(dep, "clean") for dep in model.dependencies]
+            if "semantic" in upstream:
+                category, rebuild = ChangeCategory.BREAKING, True
+                impact[model.name] = "semantic"
+            elif "additive" in upstream and _selects_star(model.ast):
+                category, rebuild = ChangeCategory.NON_BREAKING, True
+                impact[model.name] = "additive"  # a * projection inherits the new columns
+            else:
+                category, rebuild = ChangeCategory.NON_BREAKING, False
+                impact[model.name] = "clean"  # provably identical output
+
+        if model.name not in selected:
+            continue
+        inherit = (
+            forward_only
+            and rebuild
+            and model.strategy in _HISTORY_STRATEGIES
+            and previous is not None
+            and previous.engine == model.engine  # history can't be copied across engines
+        )
+        if inherit:
+            category = ChangeCategory.FORWARD_ONLY
+        plan.changes.append(
+            ModelChange(model.name, ChangeType.MODIFIED, category, previous_fingerprint, model.fingerprint, added)
+        )
+        if inherit:  # copy-on-write: history seeds the NEW table; checks gate before views move
+            snapshot = replace(
+                snapshot_of(model, ChangeCategory.FORWARD_ONLY),
+                intervals=previous.intervals,  # type: ignore[union-attr]
+            )
+            schedule_build(plan, model, snapshot, environment, seed_from=previous.physical_table)  # type: ignore[union-attr]
+        elif rebuild:
+            schedule_build(plan, model, snapshot_of(model, category), environment)
+        else:
+            _schedule_reuse(plan, model, previous, environment)  # type: ignore[arg-type]  # previous is not None here
+
+    if select is None:
+        for removed in sorted(set(current) - set(compiled.models)):
+            plan.changes.append(ModelChange(removed, ChangeType.REMOVED, None, current[removed], None))
+
+    plan.promote = sorted(selected)
+    plan.transfers = collect_transfers(compiled, [task.snapshot.name for task in plan.backfills])
+    return plan

interlace/plan/plan.py

@@ -0,0 +1,177 @@
+"""The plan/apply data model.
+
+``interlace plan`` produces a :class:`Plan`: the classified set of model changes,
+the exact ``(snapshot, interval)`` work needed to back them, any explicit
+cross-engine transfers, and the virtual-view repointing that promotes the result.
+Apply executes it; nothing here runs SQL.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from interlace.ir.relation import EngineRef, TableRef
+from interlace.state.interval import Interval
+from interlace.state.snapshot import ChangeCategory, Snapshot
+
+if TYPE_CHECKING:
+    from interlace.graph.project import CompiledModel, CompiledProject
+
+
+class ChangeType(Enum):
+    """A model's status relative to the target environment."""
+
+    ADDED = "added"
+    MODIFIED = "modified"
+    REMOVED = "removed"
+    UNCHANGED = "unchanged"
+
+
+@dataclass(frozen=True)
+class ModelChange:
+    """A model added, modified, or removed relative to the target environment."""
+
+    name: str
+    change_type: ChangeType
+    category: ChangeCategory | None  # severity of a MODIFIED change; None for added/removed
+    previous_fingerprint: str | None
+    new_fingerprint: str | None
+    impacted_columns: tuple[str, ...] = ()  # columns whose lineage changed (drives narrowing)
+
+
+@dataclass(frozen=True)
+class BackfillTask:
+    """One unit of physical work: build ``snapshot`` (optionally for one ``interval``)."""
+
+    snapshot: Snapshot
+    interval: Interval | None = None  # None = full refresh (non-incremental models)
+    # Forward-only: copy this table into the snapshot's (new) physical table before
+    # the strategy runs — history moves to the new fingerprint, the old table stays
+    # as the rollback until gc.
+    seed_from: TableRef | None = None
+
+
+@dataclass(frozen=True)
+class TransferEdge:
+    """Explicit cross-engine data movement, surfaced in plan output (never silent)."""
+
+    source: EngineRef
+    target: EngineRef
+    table: TableRef  # the staging table on the target engine
+    via: str  # "arrow" (generic fetch->load) | "attach" (future optimisation)
+    model: str = ""  # the upstream model being moved
+
+
+XFER_SCHEMA = "interlace__xfer"
+
+
+def staging_table(upstream: str) -> TableRef:
+    """Where a transferred upstream lands on the consumer's engine (replaced on every transfer)."""
+    return TableRef(schema=XFER_SCHEMA, name=upstream.replace(".", "__"))
+
+
+def collect_transfers(compiled: CompiledProject, build_names: Iterable[str]) -> list[TransferEdge]:
+    """One edge per (upstream, target engine) needed by the scheduled builds."""
+    edges: dict[tuple[str, str], TransferEdge] = {}
+    for name in build_names:
+        model = compiled.models[name]
+        for dep in model.dependencies:
+            upstream = compiled.models[dep]
+            if upstream.engine == model.engine or upstream.materialise == "ephemeral":
+                continue
+            key = (dep, model.engine)
+            if key not in edges:
+                edges[key] = TransferEdge(
+                    source=EngineRef(name=upstream.engine, dialect=upstream.dialect),
+                    target=EngineRef(name=model.engine, dialect=model.dialect),
+                    table=staging_table(dep),
+                    via="arrow",
+                    model=dep,
+                )
+    return list(edges.values())
+
+
+@dataclass(frozen=True)
+class ViewSwap:
+    """Repoint an environment view at a (new) physical snapshot table."""
+
+    view: TableRef
+    target: TableRef
+    engine: str = "default"  # named engine that hosts the view
+
+
+PRODUCTION_ENV = "prod"
+"""The production environment lives at the *unprefixed* schema (``main.orders``):
+that's what BI tools and consumers connect to. Every other environment is a
+prefixed sandbox (``dev__main.orders``) over the same physical snapshots."""
+
+
+def env_view(environment: str, model_name: str) -> TableRef:
+    """The virtual-environment view for a model: ``<schema>.<model>`` in
+    production, ``<env>__<schema>.<model>`` everywhere else."""
+    schema, _, base = model_name.rpartition(".")
+    prefix = "" if environment == PRODUCTION_ENV else f"{environment}__"
+    return TableRef(schema=f"{prefix}{schema or 'main'}", name=base)
+
+
+def schedule_build(
+    plan: Plan, model: CompiledModel, snapshot: Snapshot, environment: str, *, seed_from: TableRef | None = None
+) -> None:
+    """Add the right tasks for a model: ephemeral builds nothing; a sink builds but
+    gets no view; a table/view builds and is repointed by an environment view.
+
+    An incremental_by_time model cannot build without a window, so an apply fills
+    the latest grain interval — the same default as ``interlace run`` — leaving
+    history to ``run --start/--end``.
+    """
+    if model.materialise == "ephemeral":  # inlined into consumers, never built
+        return
+    if model.strategy == "incremental_by_time" and model.export is None:
+        from datetime import datetime
+
+        from interlace.state.interval import parse_grain
+
+        grain = parse_grain(model.interval or "1d")
+        now = datetime.now()
+        window = Interval(now - grain, now)
+        # scheduled even when an inherited ledger covers the window: the task is what
+        # seeds forward-only history, creates the table, and records the snapshot —
+        # and forward-only means the new logic owns the latest window anyway
+        plan.backfills.append(BackfillTask(snapshot=snapshot, interval=window, seed_from=seed_from))
+        plan.virtual_updates.append(
+            ViewSwap(env_view(environment, model.name), snapshot.physical_table, engine=model.engine)
+        )
+        return
+    plan.backfills.append(BackfillTask(snapshot=snapshot, seed_from=seed_from))
+    if model.export is None and model.materialise in ("table", "view"):  # sinks have no view
+        # the snapshot's table, not the fingerprint-derived one: a forward-only
+        # snapshot builds into (and the view must point at) its inherited table
+        plan.virtual_updates.append(
+            ViewSwap(env_view(environment, model.name), snapshot.physical_table, engine=model.engine)
+        )
+
+
+@dataclass
+class Plan:
+    """The full preview of an apply against one environment."""
+
+    environment: str
+    changes: list[ModelChange] = field(default_factory=list)
+    backfills: list[BackfillTask] = field(default_factory=list)
+    transfers: list[TransferEdge] = field(default_factory=list)
+    virtual_updates: list[ViewSwap] = field(default_factory=list)
+    promote: list[str] = field(default_factory=list)  # model names whose fingerprints to promote
+    # Indirectly-changed models whose output is provably identical: their new
+    # snapshot points at the previous physical table — recorded, never rebuilt.
+    reuses: list[Snapshot] = field(default_factory=list)
+
+    @property
+    def is_empty(self) -> bool:
+        return not (self.changes or self.backfills or self.virtual_updates)
+
+    @property
+    def has_breaking_changes(self) -> bool:
+        return any(c.category is ChangeCategory.BREAKING for c in self.changes)

interlace/plan/resolve.py

@@ -0,0 +1,74 @@
+"""Resolve a model's query for execution.
+
+Direct upstream references are rewritten to their physical tables; references to
+*ephemeral* upstreams are instead inlined as CTEs (the ephemeral model produces
+no table). Ephemeral chains inline recursively, in dependency order, so a model
+that reads an ephemeral staging model gets its logic spliced in at compile time.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import cast
+
+from sqlglot import exp
+
+from interlace.exceptions import PlanError
+from interlace.graph.project import CompiledModel, CompiledProject
+from interlace.ir.canonicalize import resolve_references
+from interlace.ir.relation import TableRef
+
+
+def _cte_name(model_name: str) -> str:
+    return "_eph_" + model_name.replace(".", "_")
+
+
+def _ref_mapping(
+    model: CompiledModel, project: CompiledProject, physical: Mapping[str, TableRef] | None
+) -> dict[str, TableRef]:
+    """Map each dependency to a physical table, or to a (schema-less) CTE name if ephemeral.
+
+    ``physical`` overrides the fingerprint-derived table names — a reused snapshot
+    lives at its *previous* physical table, and only the caller (apply) knows that.
+    """
+    mapping: dict[str, TableRef] = {}
+    for dep in model.dependencies:
+        upstream = project.models[dep]
+        if upstream.materialise == "ephemeral":
+            mapping[dep] = TableRef(schema="", name=_cte_name(dep))
+        else:
+            mapping[dep] = (physical or {}).get(dep, upstream.physical_table)
+    return mapping
+
+
+def _ephemeral_ancestors(model: CompiledModel, project: CompiledProject) -> list[str]:
+    """Ephemeral models reachable through ephemeral chains, in dependency order."""
+    order: list[str] = []
+    seen: set[str] = set()
+
+    def visit(name: str) -> None:
+        for dep in project.models[name].dependencies:
+            if project.models[dep].materialise == "ephemeral" and dep not in seen:
+                seen.add(dep)
+                visit(dep)  # inline an ephemeral's own ephemeral deps first
+                order.append(dep)
+
+    visit(model.name)
+    return order
+
+
+def resolve_model_query(
+    model: CompiledModel, project: CompiledProject, physical: Mapping[str, TableRef] | None = None
+) -> exp.Expression:
+    """Rewrite a model's query for execution, inlining ephemeral upstreams as CTEs."""
+    if model.ast is None:
+        raise PlanError(f"cannot resolve a Python model query: {model.name!r}")
+
+    body: exp.Expression = resolve_references(model.ast, _ref_mapping(model, project, physical))
+    for ancestor_name in _ephemeral_ancestors(model, project):
+        ancestor = project.models[ancestor_name]
+        if ancestor.ast is None:
+            raise PlanError(f"ephemeral model {ancestor_name!r} must be SQL (cannot inline a Python model)")
+        cte_body = resolve_references(ancestor.ast, _ref_mapping(ancestor, project, physical))
+        body = cast("exp.Query", body).with_(_cte_name(ancestor_name), as_=cte_body)
+    return body

interlace/plan/run.py

@@ -0,0 +1,79 @@
+"""Forced / windowed execution — the plan behind ``interlace run``.
+
+Unlike ``diff`` (which schedules only models whose fingerprint changed), a run
+rebuilds every model regardless of change detection. Incremental_by_time models
+are expanded over a time window into one task per grain interval, skipping
+intervals already recorded in the ledger (catchup); non-incremental models get a
+single full task. Because unchanged SQL keeps the same fingerprint (and physical
+table), a full model is replaced and a merge model upserts — both pick up new
+source data.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from interlace.graph.project import CompiledProject
+from interlace.plan.differ import snapshot_of
+from interlace.plan.plan import (
+    BackfillTask,
+    ChangeType,
+    ModelChange,
+    Plan,
+    ViewSwap,
+    collect_transfers,
+    env_view,
+    schedule_build,
+)
+from interlace.state.interval import Interval, parse_grain, slice_interval
+from interlace.state.snapshot import ChangeCategory
+from interlace.state.store import StateStore
+
+
+def _window(start: datetime | None, end: datetime | None, grain: timedelta) -> Interval:
+    finish = end or datetime.now()
+    begin = start or (finish - grain)
+    return Interval(begin, finish)
+
+
+async def run_plan(
+    compiled: CompiledProject,
+    environment: str,
+    state: StateStore,
+    *,
+    start: datetime | None = None,
+    end: datetime | None = None,
+    select: set[str] | None = None,
+    restate: bool = False,
+) -> Plan:
+    """Build a forced plan; incremental models are expanded over ``[start, end)``.
+
+    ``select`` limits which models run (None = all). ``restate`` reprocesses every
+    interval in the window instead of skipping the ones already filled (catchup).
+    """
+    selected = set(compiled.models) if select is None else select
+    plan = Plan(environment=environment)
+    for model in compiled.ordered():
+        if model.name not in selected:
+            continue
+        plan.changes.append(ModelChange(model.name, ChangeType.MODIFIED, None, None, model.fingerprint))
+
+        is_incremental = (
+            model.strategy == "incremental_by_time" and model.export is None and model.materialise != "ephemeral"
+        )
+        if is_incremental:
+            grain = parse_grain(model.interval or "1d")
+            filled = await state.get_intervals(model.name, model.fingerprint)
+            snapshot = snapshot_of(model, ChangeCategory.BREAKING)
+            for window in slice_interval(_window(start, end, grain), grain):
+                if restate or not filled.covers(window):  # restate reprocesses; otherwise catch up
+                    plan.backfills.append(BackfillTask(snapshot=snapshot, interval=window))
+            plan.virtual_updates.append(
+                ViewSwap(env_view(environment, model.name), model.physical_table, engine=model.engine)
+            )
+        else:
+            schedule_build(plan, model, snapshot_of(model, ChangeCategory.BREAKING), environment)
+
+    plan.promote = sorted(selected)
+    plan.transfers = collect_transfers(compiled, [task.snapshot.name for task in plan.backfills])
+    return plan