interlaced 2.6.0__tar.gz → 2.7.0__tar.gz

{interlaced-2.6.0/src/interlaced.egg-info → interlaced-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.6.0
+Version: 2.7.0
 Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT

{interlaced-2.6.0 → interlaced-2.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.6.0"
+version = "2.7.0"
 description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
 readme = "README.md"
 requires-python = ">=3.12"

{interlaced-2.6.0 → interlaced-2.7.0}/src/interlace/cli/main.py

@@ -17,16 +17,23 @@ from rich.markup import escape
 from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeElapsedColumn
 from rich.table import Table
 
-from interlace.dsl.dynamic import apply_with_registrations
-from interlace.exceptions import CheckError, ConfigurationError, InterlaceError, LockError, QueryError, SelectionError
+from interlace.exceptions import (
+    CheckError,
+    ConfigurationError,
+    InterlaceError,
+    LockError,
+    PlanError,
+    QueryError,
+    SelectionError,
+)
 from interlace.graph.column_lineage import column_impact, column_lineage, split_target
 from interlace.graph.project import CompiledProject
-from interlace.graph.selectors import select_models, wants_state
-from interlace.physical.annotate import annotate_plan
+from interlace.graph.selectors import select_models
 from interlace.plan.apply import ApplyResult
-from interlace.plan.differ import diff
+from interlace.plan.comment import plan_markdown
+from interlace.plan.orchestrate import compute_plan, plan_and_apply, resolve_selection, run_and_apply
 from interlace.plan.plan import ChangeType, Plan
-from interlace.plan.run import run_plan
+from interlace.plan.table_diff import TableDiff
 from interlace.project import Project
 from interlace.scaffold import list_templates, scaffold_project
 from interlace.scheduler.engine import TriggerEngine, build_triggers
@@ -243,11 +250,6 @@ def _selection(
         raise typer.Exit(1) from exc
 
 
-async def _promoted_if_needed(state: Any, environment: str, selectors: list[str]) -> dict[str, str] | None:
-    """state:modified compares against the target environment's fingerprints."""
-    return await state.get_environment(environment) if wants_state(selectors) else None
-
-
 @app.command()
 def init(
     path: Path = typer.Argument(Path("."), help="Directory to initialise."),
@@ -287,9 +289,10 @@ def plan(
     select: list[str] = _SELECT,
     forward_only: bool = _FORWARD_ONLY,
     as_json: bool = _JSON,
+    markdown: bool = typer.Option(False, "--markdown", help="Emit a GitHub-flavoured plan comment (CI)."),
 ) -> None:
     """Show what apply would change in an environment."""
-    asyncio.run(_plan(environment, path, select, forward_only, as_json))
+    asyncio.run(_plan(environment, path, select, forward_only, as_json, markdown))
 
 
 @app.command()
@@ -305,6 +308,22 @@ def apply(
     asyncio.run(_apply(environment, path, select, forward_only, force, parallelism))
 
 
+@app.command("diff")
+def table_diff_cmd(
+    environment: str = _ENV,
+    path: Path = _PATH,
+    select: list[str] = _SELECT,
+    against: str = typer.Option("", "--against", help="Other environment (env mode)."),
+    source: str = typer.Option("", "--source", help="Left table, schema.table (table mode)."),
+    target: str = typer.Option("", "--target", help="Right table, schema.table (table mode)."),
+    on: list[str] = typer.Option([], "--on", help="Join key columns. Default: the model's key, else common columns."),
+    limit: int = typer.Option(20, "--limit", "-n", help="Sample rows per category."),
+    as_json: bool = _JSON,
+) -> None:
+    """Compare a model across two environments, or two tables on the warehouse."""
+    asyncio.run(_table_diff(environment, path, select, against, source, target, on, limit, as_json))
+
+
 @app.command()
 def test(
     path: Path = _PATH,
@@ -333,24 +352,31 @@ def test(
 
 
 async def _plan(
-    environment: str, path: Path, select: list[str], forward_only: bool = False, as_json: bool = False
+    environment: str,
+    path: Path,
+    select: list[str],
+    forward_only: bool = False,
+    as_json: bool = False,
+    markdown: bool = False,
 ) -> None:
     project = Project.load(path)
     compiled = project.compile()
     state = await project.open_state()
     engines = project.open_engines()
     try:
-        promoted = await _promoted_if_needed(state, environment, select)
-        result = await diff(
-            compiled, environment, state, select=_selection(compiled, select, promoted), forward_only=forward_only
-        )
-        await annotate_plan(result, compiled, engines)
-        if as_json:
+        selected = await resolve_selection(compiled, state, environment, select)
+        result = await compute_plan(compiled, environment, state, engines, select=selected, forward_only=forward_only)
+        if markdown:
+            typer.echo(plan_markdown(result, environment))
+        elif as_json:
             _emit_json(_plan_dict(result, environment))
         else:
             _render(result, environment)
         if result.blocking:
             raise typer.Exit(1)
+    except SelectionError as exc:
+        console.print(f"[red]{escape(exc.message)}[/red]")
+        raise typer.Exit(1) from exc
     finally:
         await state.close()
         engines.close()
@@ -385,6 +411,132 @@ def _plan_dict(plan: Plan, environment: str) -> dict:
     }
 
 
+def _diff_dict(result: TableDiff) -> dict[str, object]:
+    schema = {
+        "added": [{"name": name, "type": dtype} for name, dtype in result.schema.added],
+        "removed": [{"name": name, "type": dtype} for name, dtype in result.schema.removed],
+        "type_changed": [
+            {"name": name, "left": left, "right": right} for name, left, right in result.schema.type_changed
+        ],
+    }
+    rows = None
+    if result.rows is not None:
+        rows = {
+            "left_count": result.rows.left_count,
+            "right_count": result.rows.right_count,
+            "left_only": result.rows.left_only,
+            "right_only": result.rows.right_only,
+            "changed": result.rows.changed,
+            "matched": result.rows.matched,
+            "keys": result.rows.keys,
+            "left_only_sample": result.rows.left_only_sample,
+            "right_only_sample": result.rows.right_only_sample,
+            "changed_sample": result.rows.changed_sample,
+        }
+    return {
+        "left": result.left,
+        "right": result.right,
+        "model": result.model,
+        "schema": schema,
+        "rows": rows,
+        "message": result.message,
+        "differs": result.differs,
+    }
+
+
+def _render_diffs(results: list[TableDiff]) -> None:
+    for result in results:
+        title = result.model or f"{result.left} vs {result.right}"
+        if result.message and result.rows is None and result.schema.empty:
+            console.print(f"[yellow]{title}[/yellow]  {result.message}")
+            continue
+        table = _table(title)
+        table.add_column("Side")
+        table.add_column("Relation", style="dim")
+        table.add_row("left", result.left)
+        table.add_row("right", result.right)
+        console.print(table)
+        if result.schema.added or result.schema.removed or result.schema.type_changed:
+            for name, dtype in result.schema.added:
+                console.print(f"  [green]+[/] column {name} {dtype}")
+            for name, dtype in result.schema.removed:
+                console.print(f"  [red]-[/] column {name} {dtype}")
+            for name, left, right in result.schema.type_changed:
+                console.print(f"  [yellow]~[/] column {name} {left} → {right}")
+        if result.message:
+            console.print(f"  [yellow]{result.message}[/yellow]")
+        if result.rows is None:
+            continue
+        rows = result.rows
+        console.print(
+            f"  rows  left={rows.left_count} right={rows.right_count}  "
+            f"only-left={rows.left_only} only-right={rows.right_only} changed={rows.changed} "
+            f"matched={rows.matched}  keys={','.join(rows.keys) or '—'}"
+        )
+
+
+async def _table_diff(
+    environment: str,
+    path: Path,
+    select: list[str],
+    against: str,
+    source: str,
+    target: str,
+    on: list[str],
+    limit: int,
+    as_json: bool,
+) -> None:
+    from interlace.plan.table_diff import diff_environments, diff_tables, parse_table_ref
+
+    if bool(source) != bool(target):
+        console.print("[red]table mode needs both --source and --target[/red]")
+        raise typer.Exit(2)
+    if not source and not against:
+        console.print("[red]env mode needs --against ENV (or pass --source/--target)[/red]")
+        raise typer.Exit(2)
+    project = Project.load(path)
+    compiled = project.compile()
+    engines = project.open_engines()
+    state = await project.open_state()
+    try:
+        if source:
+            result = await diff_tables(
+                engines.get(),
+                parse_table_ref(source),
+                parse_table_ref(target),
+                keys=on or None,
+                sample=limit,
+            )
+            results = [result]
+        else:
+            selected = await resolve_selection(compiled, state, environment, select)
+            results = await diff_environments(
+                compiled,
+                left_env=environment,
+                right_env=against,
+                store=state,
+                engines=engines,
+                select=selected,
+                keys=on or None,
+                sample=limit,
+            )
+        if as_json:
+            _emit_json([_diff_dict(item) for item in results])
+        else:
+            if not results:
+                console.print("No comparable models (ephemeral/file outputs are skipped).")
+            else:
+                _render_diffs(results)
+        if any(item.differs for item in results):
+            raise typer.Exit(1)
+    except PlanError as exc:
+        console.print(f"[red]{escape(exc.message)}[/red]")
+        raise typer.Exit(1) from exc
+    finally:
+        await state.close()
+        engines.close()
+
+
 async def _apply(
     environment: str,
     path: Path,
@@ -393,50 +545,64 @@ async def _apply(
     force: bool = False,
     parallelism: int = 0,
 ) -> None:
+    from interlace.exceptions import BreakingPlanError
+
     project = Project.load(path)
     compiled = project.compile()
     engines = project.open_engines()
     state = await project.open_state()
+    tracker: _BuildProgress | None = _BuildProgress() if console.is_terminal else None
+    started = False
     try:
-        # Stream-fed projects must build without the daemon ever having run:
-        # declared stream tables are ensured (empty) so models reading them work.
-        if project.streams:
-            await ensure_stream_tables(project.streams, engines.get())
-        promoted = await _promoted_if_needed(state, environment, select)
-        plan_result = await diff(
-            compiled, environment, state, select=_selection(compiled, select, promoted), forward_only=forward_only
-        )
-        await annotate_plan(plan_result, compiled, engines)
-        _render(plan_result, environment)
-        if plan_result.is_empty:
-            return
-        if plan_result.has_breaking_changes and not force:  # same guard the HTTP API enforces
-            breaking = ", ".join(
-                c.name for c in plan_result.changes if c.category is not None and c.category.value == "breaking"
-            )
-            console.print(f"[red]plan has breaking changes ({breaking}); re-run with --force to proceed[/red]")
-            raise typer.Exit(1)
-        progress = _build_progress(plan_result)
+
+        async def prepare() -> None:
+            # Stream-fed projects must build without the daemon ever having run:
+            # declared stream tables are ensured (empty) so models reading them work.
+            if project.streams:
+                await ensure_stream_tables(project.streams, engines.get())
+
+        def on_plan(plan: Plan) -> None:
+            nonlocal started, tracker
+            _render(plan, environment)
+            if tracker is not None and not plan.backfills:
+                tracker = None
+            if tracker is not None:
+                tracker.progress.start()
+                started = True
+
+        def on_progress(model: str, event: str, detail: dict[str, Any]) -> None:
+            if tracker is not None:
+                tracker(model, event, detail)
+
         try:
-            with progress.progress if progress else contextlib.nullcontext():
-                async with hold_apply_lock(state, owner=f"cli:{os.getpid()}:apply"):
-                    result = await apply_with_registrations(
-                        plan_result,
-                        compiled=compiled,
-                        engines=engines,
-                        state=state,
-                        base_path=project.root,
-                        on_progress=progress,
-                        connections=project.config.connections,
-                        parallelism=parallelism or project.config.parallelism,  # --parallelism wins over config
-                        project=project,
-                    )
+            _plan_result, result = await plan_and_apply(
+                compiled,
+                environment=environment,
+                project=project,
+                engines=engines,
+                state=state,
+                lock_owner=f"cli:{os.getpid()}:apply",
+                selectors=select,
+                forward_only=forward_only,
+                force=force,
+                parallelism=parallelism or None,
+                on_progress=on_progress,
+                on_plan=on_plan,
+                prepare=prepare,
+            )
+        except BreakingPlanError as exc:
+            console.print(
+                f"[red]plan has breaking changes ({', '.join(exc.names)}); re-run with --force to proceed[/red]"
+            )
+            raise typer.Exit(1) from exc
         except CheckError as exc:
             console.print(f"[red]{escape(exc.message)}[/red]")
             raise typer.Exit(1) from exc
         except LockError as exc:
             console.print(f"[red]{escape(exc.message)}[/red]")
             raise typer.Exit(1) from exc
+        if result is None:
+            return
         _render_build_results(result, compiled)
         _render_checks(result)
         await _render_empty_incrementals(result, compiled, engines)
@@ -444,6 +610,8 @@ async def _apply(
             f"[green]Built {len(set(result.built))} model(s); promoted {result.promoted} to '{environment}'.[/green]"
         )
     finally:
+        if started and tracker is not None:
+            tracker.progress.stop()
         await state.close()
         engines.close()
 
@@ -509,34 +677,44 @@ async def _execute(
     compiled = project.compile()
     engines = project.open_engines()
     state = await project.open_state()
+    tracker: _BuildProgress | None = _BuildProgress() if console.is_terminal else None
+    started_bar = False
     try:
-        if project.streams:  # as in _apply: stream tables must exist daemon or not
-            await ensure_stream_tables(project.streams, engines.get())
-        plan_result = await run_plan(
-            compiled,
-            environment,
-            state,
-            start=window_start,
-            end=window_end,
-            select=_selection(compiled, select, await _promoted_if_needed(state, environment, select)),
-            restate=restate,
-        )
-        progress = _build_progress(plan_result)
+
+        async def prepare() -> None:
+            if project.streams:  # as in _apply: stream tables must exist daemon or not
+                await ensure_stream_tables(project.streams, engines.get())
+
+        def on_plan(plan: Plan) -> None:
+            nonlocal started_bar, tracker
+            if tracker is not None and not plan.backfills:
+                tracker = None
+            if tracker is not None:
+                tracker.progress.start()
+                started_bar = True
+
+        def on_progress(model: str, event: str, detail: dict[str, Any]) -> None:
+            if tracker is not None:
+                tracker(model, event, detail)
+
         started = time.perf_counter()
         try:
-            with progress.progress if progress else contextlib.nullcontext():
-                async with hold_apply_lock(state, owner=f"cli:{os.getpid()}:run"):
-                    result = await apply_with_registrations(
-                        plan_result,
-                        compiled=compiled,
-                        engines=engines,
-                        state=state,
-                        base_path=project.root,
-                        on_progress=progress,
-                        connections=project.config.connections,
-                        parallelism=parallelism or project.config.parallelism,  # --parallelism wins over config
-                        project=project,
-                    )
+            plan_result, result = await run_and_apply(
+                compiled,
+                environment=environment,
+                project=project,
+                engines=engines,
+                state=state,
+                lock_owner=f"cli:{os.getpid()}:run",
+                selectors=select,
+                start=window_start,
+                end=window_end,
+                restate=restate,
+                parallelism=parallelism or None,
+                on_progress=on_progress,
+                on_plan=on_plan,
+                prepare=prepare,
+            )
         except CheckError as exc:
             console.print(f"[red]{escape(exc.message)}[/red]")
             raise typer.Exit(1) from exc
@@ -553,6 +731,8 @@ async def _execute(
             f"promoted {result.promoted} to '{environment}'.[/green]"
         )
     finally:
+        if started_bar and tracker is not None:
+            tracker.progress.stop()
         await state.close()
         engines.close()
 

{interlaced-2.6.0 → interlaced-2.7.0}/src/interlace/exceptions.py

@@ -42,6 +42,18 @@ class PlanError(InterlaceError):
     """A plan could not be computed or applied."""
 
 
+class BreakingPlanError(PlanError):
+    """The plan has breaking changes and ``force`` was not set."""
+
+    def __init__(self, names: list[str]) -> None:
+        listed = ", ".join(names)
+        super().__init__(
+            f"plan has breaking changes ({listed}); pass force to proceed",
+            details={"kind": "breaking", "names": names},
+        )
+        self.names = names
+
+
 class ExecutionError(InterlaceError):
     """A model failed while building (its query, its Python function, or the load)."""
 

{interlaced-2.6.0 → interlaced-2.7.0}/src/interlace/inspect.py

@@ -23,7 +23,12 @@ from interlace.sinks import target_ref
 from interlace.state.snapshot import Snapshot
 
 
-class _InspectStore(Protocol):
+class _RelationStore(Protocol):
+    async def get_environment(self, environment: str) -> dict[str, str]: ...
+    async def get_snapshot(self, name: str, fingerprint: str) -> Snapshot | None: ...
+
+
+class _InspectStore(_RelationStore, Protocol):
     """The control-plane reads a preview needs, beyond the plan/apply store slice."""
 
     async def latest_model_build(self, model: str) -> dict[str, object] | None: ...
@@ -118,7 +123,7 @@ async def last_build(store: _InspectStore, name: str) -> LastBuild | None:
 
 
 async def _built_table(
-    model: CompiledModel, store: _InspectStore, engine: EngineAdapter, environment: str
+    model: CompiledModel, store: _RelationStore, engine: EngineAdapter, environment: str
 ) -> tuple[TableRef, str] | None:
     """The relation a preview should read, and why (environment view or snapshot)."""
     if model.materialise == "ephemeral" or model.materialise == "file":
@@ -144,6 +149,14 @@ async def _built_table(
     return None
 
 
+async def model_relation(
+    model: CompiledModel, store: _RelationStore, engine: EngineAdapter, environment: str
+) -> TableRef | None:
+    """The table a preview or table-diff should read for ``model`` in ``environment``."""
+    found = await _built_table(model, store, engine, environment)
+    return None if found is None else found[0]
+
+
 def _unavailable(model: CompiledModel) -> str:
     if model.materialise == "ephemeral":
         return "ephemeral models are inlined into the models that read them"

{interlaced-2.6.0 → interlaced-2.7.0}/src/interlace/mcp_server.py

@@ -17,18 +17,13 @@ from pathlib import Path
 from typing import Any, cast
 
 from interlace import __version__
-from interlace.dsl.dynamic import apply_with_registrations
 from interlace.exceptions import InterlaceError
 from interlace.graph.column_lineage import column_lineage
 from interlace.graph.project import CompiledProject
-from interlace.graph.selectors import select_models, wants_state
 from interlace.inspect import failing_rows, preview_model
-from interlace.physical.annotate import annotate_plan
-from interlace.plan.differ import diff
+from interlace.plan.orchestrate import compute_plan, plan_and_apply, resolve_selection
 from interlace.project import Project
 from interlace.query import prepare_readonly
-from interlace.state.locks import hold_apply_lock
-from interlace.state.snapshot import ChangeCategory
 from interlace.streaming import ensure_stream_tables
 
 _PROTOCOL = "2024-11-05"
@@ -158,10 +153,10 @@ async def _plan(path: Path, args: dict[str, Any]) -> Any:
     try:
         environment = _env(project, args)
         selectors = [str(item) for item in args.get("selectors") or []]
-        promoted = await state.get_environment(environment) if wants_state(selectors) else None
-        selected = select_models(selectors, compiled, promoted=promoted) if selectors else None
-        plan = await diff(compiled, environment, state, select=selected, forward_only=bool(args.get("forward_only")))
-        await annotate_plan(plan, compiled, engines)
+        selected = await resolve_selection(compiled, state, environment, selectors)
+        plan = await compute_plan(
+            compiled, environment, state, engines, select=selected, forward_only=bool(args.get("forward_only"))
+        )
     finally:
         await state.close()
         engines.close()
@@ -187,34 +182,28 @@ async def _apply(path: Path, args: dict[str, Any]) -> Any:
     project, compiled, state, engines = await _open(path)
     try:
         environment = _env(project, args)
-        selectors = [str(item) for item in args.get("selectors") or []]
-        if project.streams:
-            await ensure_stream_tables(project.streams, engines.get())
-        promoted = await state.get_environment(environment) if wants_state(selectors) else None
-        selected = select_models(selectors, compiled, promoted=promoted) if selectors else None
-        plan = await diff(compiled, environment, state, select=selected, forward_only=bool(args.get("forward_only")))
-        await annotate_plan(plan, compiled, engines)
-        if plan.blocking:
-            raise InterlaceError("schema drift blocks apply: " + "; ".join(plan.blocking))
-        if plan.is_empty:
-            return {"environment": environment, "built": [], "promoted": 0}
-        if plan.has_breaking_changes and not args.get("force"):
-            names = ", ".join(change.name for change in plan.changes if change.category is ChangeCategory.BREAKING)
-            raise InterlaceError(f"plan has breaking changes ({names}); resubmit with force true")
-        async with hold_apply_lock(state, owner=f"mcp:{os.getpid()}"):
-            result = await apply_with_registrations(
-                plan,
-                compiled=compiled,
-                engines=engines,
-                state=state,
-                base_path=project.root,
-                parallelism=project.config.parallelism,
-                connections=project.config.connections,
-                project=project,
-            )
+
+        async def prepare() -> None:
+            if project.streams:
+                await ensure_stream_tables(project.streams, engines.get())
+
+        plan, result = await plan_and_apply(
+            compiled,
+            environment=environment,
+            project=project,
+            engines=engines,
+            state=state,
+            lock_owner=f"mcp:{os.getpid()}",
+            selectors=[str(item) for item in args.get("selectors") or []],
+            forward_only=bool(args.get("forward_only")),
+            force=bool(args.get("force")),
+            prepare=prepare,
+        )
     finally:
         await state.close()
         engines.close()
+    if result is None:
+        return {"environment": environment, "built": [], "promoted": 0}
     return {
         "environment": environment,
         "built": result.built,

{interlaced-2.6.0 → interlaced-2.7.0}/src/interlace/plan/__init__.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from interlace.plan.apply import ApplyResult, apply
 from interlace.plan.differ import diff, snapshot_of
+from interlace.plan.orchestrate import compute_plan, plan_and_apply, run_and_apply
 from interlace.plan.plan import BackfillTask, ChangeType, ModelChange, Plan, TransferEdge, ViewSwap, env_view
 from interlace.plan.run import run_plan
 
@@ -16,8 +17,11 @@ __all__ = [
     "TransferEdge",
     "ViewSwap",
     "apply",
+    "compute_plan",
     "diff",
     "env_view",
+    "plan_and_apply",
+    "run_and_apply",
     "run_plan",
     "snapshot_of",
 ]