interlaced 2.1.0__tar.gz → 2.3.0__tar.gz

{interlaced-2.1.0/src/interlaced.egg-info → interlaced-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.1.0
+Version: 2.3.0
 Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT
@@ -18,12 +18,12 @@ Classifier: Programming Language :: SQL
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: sqlglot<29.0,>=25.0
+Requires-Dist: sqlglot<30.0,>=25.0
 Requires-Dist: duckdb>=1.5.3
 Requires-Dist: pyarrow>=17.0
 Requires-Dist: pydantic<3.0,>=2.5
 Requires-Dist: typer<1.0,>=0.12
-Requires-Dist: rich<15.0,>=13.0
+Requires-Dist: rich<16.0,>=13.0
 Requires-Dist: cronsim<3.0,>=2.5
 Requires-Dist: tenacity<10.0,>=8.2
 Requires-Dist: pyyaml<7.0,>=6.0
@@ -59,7 +59,7 @@ Requires-Dist: httpx<1.0,>=0.27; extra == "dev"
 Requires-Dist: pytest-asyncio<2.0,>=1.0; extra == "dev"
 Requires-Dist: ruff<1.0,>=0.6; extra == "dev"
 Requires-Dist: black<27.0,>=24.0; extra == "dev"
-Requires-Dist: mypy<2.0,>=1.11; extra == "dev"
+Requires-Dist: mypy<3.0,>=1.11; extra == "dev"
 Dynamic: license-file
 
 # interlace
@@ -127,8 +127,9 @@ def orders(cursor, this):
 ```
 
 **Strategies:** `replace`, `view`, `ephemeral` (CTE-inlined), `merge` (upsert),
-`full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
-(windowed, interval-ledger backfill/catchup), `scd` (history with validity windows).
+`full_merge` (full-state source applied as a minimal diff), `incremental` (one time window
+at a time — rewrites the window, or upserts within it if you give it a `key`), `scd`
+(history with validity windows).
 
 ## Plan / apply
 

{interlaced-2.1.0 → interlaced-2.3.0}/README.md

@@ -63,8 +63,9 @@ def orders(cursor, this):
 ```
 
 **Strategies:** `replace`, `view`, `ephemeral` (CTE-inlined), `merge` (upsert),
-`full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
-(windowed, interval-ledger backfill/catchup), `scd` (history with validity windows).
+`full_merge` (full-state source applied as a minimal diff), `incremental` (one time window
+at a time — rewrites the window, or upserts within it if you give it a `key`), `scd`
+(history with validity windows).
 
 ## Plan / apply
 

{interlaced-2.1.0 → interlaced-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.1.0"
+version = "2.3.0"
 description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -24,12 +24,12 @@ classifiers = [
 # Core: the IR + engine + state + plan spine (Phase 1). Everything is Arrow- and
 # sqlglot-native. See docs/architecture/architecture.md.
 dependencies = [
-    "sqlglot>=25.0,<29.0",        # canonical IR, transpilation, semantic diff, column lineage
+    "sqlglot>=25.0,<30.0",        # canonical IR, transpilation, semantic diff, column lineage
     "duckdb>=1.5.3",              # default engine + federation hub; DuckLake storage + quack serving
     "pyarrow>=17.0",              # the wire format: RecordBatchReader everywhere
     "pydantic>=2.5,<3.0",         # config + manifest validation (cold paths only)
     "typer>=0.12,<1.0",           # CLI
-    "rich>=13.0,<15.0",           # display, strictly an event subscriber
+    "rich>=13.0,<16.0",           # display, strictly an event subscriber
     "cronsim>=2.5,<3.0",          # cron parsing for the trigger engine
     "tenacity>=8.2,<10.0",        # retries (tasks, DuckLake commit conflicts, transfers)
     "pyyaml>=6.0,<7.0",           # project config (config + env overlays)
@@ -78,7 +78,7 @@ dev = [
     "pytest-asyncio>=1.0,<2.0",
     "ruff>=0.6,<1.0",
     "black>=24.0,<27.0",
-    "mypy>=1.11,<2.0",
+    "mypy>=1.11,<3.0",
 ]
 
 [tool.setuptools.packages.find]

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/cli/main.py

@@ -107,7 +107,7 @@ _END = typer.Option("", "--end", help="Window end (ISO), for incremental models.
 _FORWARD_ONLY = typer.Option(
     False,
     "--forward-only",
-    help="Modified history-keeping models (merge/full_merge/scd/incremental_by_time) carry their history "
+    help="Modified history-keeping models (merge/full_merge/scd/incremental) carry their history "
     "forward: it is copied to the new version, the new logic applies to the copy, and checks gate "
     "before views move. Requires a shape-compatible change.",
 )
@@ -205,7 +205,7 @@ async def _render_empty_incrementals(result: ApplyResult, compiled: CompiledProj
 
     for name in result.built:
         model = compiled.models[name]
-        if model.strategy != "incremental_by_time" or model.is_terminal:
+        if model.strategy != "incremental" or model.is_terminal:
             continue
         counts = result.rows.get(name)
         if counts is not None and (counts.inserted or counts.updated):
@@ -252,7 +252,8 @@ def init(
         table.add_column("Description", style="dim")
         table.add_column("Needs", style="dim")
         for info in list_templates():
-            table.add_row(info.name, info.description, ", ".join(info.requires_env) or "—")
+            # escape: a description may contain [sources]-style brackets Rich would eat as markup
+            table.add_row(info.name, escape(info.description), ", ".join(info.requires_env) or "—")
         console.print(table)
         return
     try:
@@ -403,7 +404,7 @@ def run(
 ) -> None:
     """Force-build models and promote, ignoring change detection.
 
-    For incremental_by_time models, --start/--end set the catchup window
+    For incremental models, --start/--end set the catchup window
     (default: the latest grain interval).
     """
     asyncio.run(_execute(environment, path, select, start, end, restate=False, parallelism=parallelism))

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/dsl/decorators.py

@@ -22,7 +22,7 @@ ModelFn = Callable[..., Any]
 # (a fingerprinted snapshot read through an environment view); `table`/`file` are
 # terminal deliveries into a destination interlace does not own.
 _MATERIALISATIONS = frozenset({"virtual", "view", "ephemeral", "table", "file"})
-_KEYED_STRATEGIES = frozenset({"merge", "full_merge", "scd"})
+_KEYED_STRATEGIES = frozenset({"merge", "full_merge", "hash_merge", "scd"})
 _DRIFT_MODES = frozenset({"evolve", "reject", "quarantine"})
 
 
@@ -94,9 +94,9 @@ class ModelDef:
     dialect: str | None = None
     engine: str | None = None  # named engine from config (None → project default_engine)
     depends_on: tuple[str, ...] = ()
-    interval: str | None = None  # grain for incremental_by_time (e.g. "1d")
-    time_column: str | None = None  # partition column for incremental_by_time
-    # First-build window for incremental_by_time: "auto" derives [min, max] of the
+    interval: str | None = None  # grain for incremental (e.g. "1d")
+    time_column: str | None = None  # partition column for incremental
+    # First-build window for incremental: "auto" derives [min, max] of the
     # time column from the source at apply time and fills it as ONE interval;
     # "none" keeps only the latest grain window; an ISO date pins the start.
     backfill: str = "auto"

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/dsl/discovery.py

@@ -64,7 +64,7 @@ def _sql_model(default_name: str, sql: str, config: dict[str, Any], default_dial
         depends_on=_as_tuple(config.get("depends_on") or ()),
         interval=config.get("interval"),
         time_column=config.get("time_column"),
-        backfill=config.get("backfill", "auto"),  # first-build window for incremental_by_time
+        backfill=config.get("backfill", "auto"),  # first-build window for incremental
         tags=_as_tuple(config.get("tags") or ()),
         owner=config.get("owner"),
         description=config.get("description"),

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/engines/spark.py

@@ -7,7 +7,7 @@ come back as Arrow via ``DataFrame.toArrow()``, and Arrow loads go in through
 (``local[*]``, for tests) or a remote one (Spark Connect / a shared session).
 
 **Strategy support.** ``replace``, ``append`` and ``view`` run on any Spark
-catalog. ``merge`` (native ``MERGE``) and ``incremental_by_time`` (windowed
+catalog. ``merge`` (native ``MERGE``) and ``incremental`` (windowed
 ``DELETE`` by literal predicate + ``INSERT``) need a catalog with row-level
 mutations — Delta Lake or Iceberg — configured on the session you hand the
 adapter (the tests use a Delta-backed local session). ``scd`` and ``full_merge``

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/graph/project.py

@@ -44,9 +44,9 @@ class CompiledModel:
     materialise: str
     strategy: str
     key: tuple[str, ...]  # business key for keyed strategies (merge)
-    time_column: str | None  # partition column for incremental_by_time
+    time_column: str | None  # partition column for incremental
     cursor: str | None  # column whose max is injected into a Python model's `cursor` param
-    interval: str | None  # grain for incremental_by_time (e.g. "1d")
+    interval: str | None  # grain for incremental (e.g. "1d")
     tags: tuple[str, ...]  # for tag: selection
     schedule: dict[str, str] | None  # cron/interval schedule for the trigger engine
     columns: dict[str, str | None] | None  # output contract validated at apply time

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/ir/canonicalize.py

@@ -8,6 +8,8 @@ pruning and the ``impact`` command).
 
 from __future__ import annotations
 
+from typing import cast
+
 import sqlglot
 from sqlglot import exp
 
@@ -86,4 +88,5 @@ def resolve_references(ast: exp.Expression, mapping: dict[str, TableRef]) -> exp
                 node.set("catalog", exp.to_identifier(target.catalog) if target.catalog else None)
         return node
 
-    return ast.transform(rewrite)
+    # sqlglot 29 loosened transform()'s return annotation; it is an Expression.
+    return cast("exp.Expression", ast.transform(rewrite))

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/plan/apply.py

@@ -36,8 +36,9 @@ from interlace.runtime.python_model import build_python_model, run_python_model
 from interlace.sinks import file_statements, target_ref
 from interlace.state.interval import Interval
 from interlace.state.store import StateStore
-from interlace.strategies import Strategy, resolve_strategy
+from interlace.strategies import Incremental, Strategy, resolve_strategy
 from interlace.strategies.base import RowCounts
+from interlace.strategies.hash_merge import HashMerge
 
 
 @dataclass
@@ -86,7 +87,9 @@ async def _merge_python_output(
     reader: pa.RecordBatchReader,
     *,
     exists: bool,
-) -> RowCounts:
+    interval: Interval | None = None,
+    bootstrap: bool = False,
+) -> tuple[RowCounts, Interval | None]:
     """Stage a Python model's Arrow output and apply its keyed strategy in SQL.
 
     The output lands in a stage table (CREATE OR REPLACE, so a crashed run's
@@ -107,12 +110,21 @@ async def _merge_python_output(
         pre_statements, source, columns = await _align_stage_to_target(
             engine, stage, target, exclude=strategy.managed_columns
         )
+    elif isinstance(strategy, HashMerge):
+        # hash_merge builds its _hash from the column list; on the first build the target
+        # doesn't exist yet (so no align pass ran), so take the columns from the staged output
+        columns = [c for c in await engine.describe(stage) if c not in strategy.managed_columns]
 
     relation = SqlRelation(ast=source)
-    statements = strategy.plan_statements(relation, target, engine.caps, None, columns)
+    if isinstance(strategy, Incremental) and bootstrap:
+        # First build of a keyed incremental Python model: the range comes from the
+        # staged output, since there is no query to probe the way a SQL model has.
+        interval = await _bootstrap_window(model, exp.select("*").from_(stage_table.copy()), engine)
+    statements = strategy.plan_statements(relation, target, engine.caps, interval, columns)
     drop_stage = exp.Drop(this=stage_table.copy(), kind="TABLE", exists=True)
     counts = await engine.execute_all([*pre_statements, *statements, drop_stage])
-    return strategy.row_counts(counts[len(pre_statements) : len(pre_statements) + len(statements)])
+    written = strategy.row_counts(counts[len(pre_statements) : len(pre_statements) + len(statements)])
+    return written, interval
 
 
 async def _align_stage_to_target(
@@ -178,7 +190,7 @@ async def _deliver_table(
     interval: Interval | None,
 ) -> RowCounts:
     """Deliver ``resolved`` into an external table (``materialise: table``) via
-    ``strategy`` (replace / append / merge / full_merge / incremental_by_time).
+    ``strategy`` (replace / append / merge / full_merge / incremental).
 
     The external target is never dropped (grants and readers survive). When it already
     exists the source is staged in the warehouse and aligned to the target (additive
@@ -188,7 +200,7 @@ async def _deliver_table(
     order exactly.
 
     Two cases skip staging and run the strategy directly against the target: the first
-    delivery (the ensure-create matches the source), and any windowed ``incremental_by_time``
+    delivery (the ensure-create matches the source), and any windowed ``incremental``
     delivery (``interval`` set). An incremental window is grain-scoped and stays
     schema-stable within a fingerprint, so staging the *whole* source once per window
     would make a wide backfill/restate O(windows × source) — the pathological case."""
@@ -409,15 +421,38 @@ async def _run_backfill(
     target_engine = registry.require(model.engine, model=model.name)
     resolution = await _stage_cross_engine_inputs(model, compiled, registry, physical, staged, stage_lock, result)
 
+    if task.reuse_existing and await target_engine.table_exists(snapshot.physical_table):
+        # fingerprint already materialised (e.g. by another environment): the content-
+        # addressed table exists, so skip the (re)build compute — record the snapshot for
+        # this env's promotion, gate on checks against the existing table, and let the
+        # caller swap this environment's view onto the shared table. The table_exists guard
+        # is what makes the differ's optimistic reuse safe: a snapshot row can outlive its
+        # table (an in-memory warehouse across processes, a gc'd table) — then we fall
+        # through and build for real.
+        await state.add_snapshot(snapshot)  # idempotent — the row may already exist
+        if snapshot.name not in result.built and snapshot.name not in result.reused:
+            result.reused.append(snapshot.name)
+        await _gate_checks(model, compiled, target_engine, state, plan.environment, result, resolution)
+        result.timings[snapshot.name] = result.timings.get(snapshot.name, 0.0) + (time.perf_counter() - task_started)
+        return
+
     if model.ast is None:  # Python model: run the function, load Arrow into the snapshot table
         if model.materialise != "virtual":
             raise PlanError(
                 f"Python model {snapshot.name!r} must materialise as virtual; table/file (write a SQL model "
                 f"over its output), view and ephemeral are not supported for Python models"
             )
-        if model.strategy == "incremental_by_time":
+        if model.strategy == "incremental" and not model.key:
+            # Keyed is supported: the window bounds which staged rows are upserted.
+            # Unkeyed is not, and deliberately. For a SQL model the window predicate
+            # is pushed into the query so the engine only computes the window; a
+            # Python function has already computed everything by the time we could
+            # filter it, so an unkeyed windowed rewrite would look incremental while
+            # doing the full work every run. Bound the fetch with cursor= instead.
             raise PlanError(
-                f"Python model {snapshot.name!r} cannot use incremental_by_time; " f"use cursor= with merge instead"
+                f"Python model {snapshot.name!r} cannot use incremental without a key: the function "
+                f"runs in full before the window can be applied, so the window would not save any work. "
+                f"Add key= to upsert the window's rows, or use cursor= with merge to bound the fetch"
             )
         recorded_self = await state.get_snapshot(snapshot.name, snapshot.fingerprint)
         previous = recorded_self.physical_table if recorded_self is not None else None
@@ -432,10 +467,21 @@ async def _run_backfill(
             result.record_rows(snapshot.name, RowCounts(inserted=loaded))
         else:  # keyed strategy: stage the Arrow output, then merge it in SQL
             reader = await run_python_model(model, compiled, target_engine, resolution, previous)
-            merged = await _merge_python_output(
-                model, target_engine, snapshot.physical_table, reader, exists=previous is not None
+            merged, filled_window = await _merge_python_output(
+                model,
+                target_engine,
+                snapshot.physical_table,
+                reader,
+                exists=previous is not None,
+                interval=task.interval,
+                bootstrap=task.bootstrap,
             )
             result.record_rows(snapshot.name, merged)
+            if filled_window is not None:  # incremental: accumulate the window in the ledger
+                filled = await state.get_intervals(snapshot.name, snapshot.fingerprint)
+                for carried in snapshot.intervals:
+                    filled = filled.add(carried)
+                snapshot = replace(snapshot, intervals=filled.add(filled_window))
         if model.columns:
             validate_contract(model.name, await target_engine.describe(snapshot.physical_table), model.columns)
         await state.add_snapshot(snapshot)
@@ -669,7 +715,10 @@ async def apply(
 
     mapping = {name: compiled.models[name].fingerprint for name in plan.promote}
     await state.promote(plan.environment, mapping)
-    result.promoted = len(mapping)
+    # ephemeral models are tracked in the mapping (so re-plans stay clean) but are inlined
+    # into consumers — they have no promotable table/view, so the user-facing count omits
+    # them, keeping "promoted N" consistent with the N build rows shown
+    result.promoted = sum(1 for name in mapping if compiled.models[name].materialise != "ephemeral")
 
     # deleted models: drop their env view and demote them, or the view serves the
     # last snapshot forever and pins it against gc

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/plan/differ.py

@@ -243,7 +243,7 @@ def _schedule_reuse(plan: Plan, model: CompiledModel, previous: Snapshot, enviro
         )
 
 
-_HISTORY_STRATEGIES = frozenset({"merge", "full_merge", "scd", "incremental_by_time"})
+_HISTORY_STRATEGIES = frozenset({"merge", "full_merge", "hash_merge", "scd", "incremental"})
 """Strategies whose targets accumulate state a rebuild would destroy."""
 
 
@@ -283,7 +283,7 @@ async def diff(
     classification still runs over the whole graph so downstream categories are correct.
 
     ``forward_only``: modified models whose strategy accumulates history
-    (merge / full_merge / scd / incremental_by_time) inherit their
+    (merge / full_merge / scd / incremental) inherit their
     previous physical table and interval ledger instead of starting fresh — the
     new logic applies going forward, history survives. Requires the new query to
     stay shape-compatible with the existing table.
@@ -303,6 +303,13 @@ async def diff(
         for name, fingerprint in current.items()
         if name in compiled.models and fingerprint != compiled.models[name].fingerprint
     )
+    # Fingerprints already materialised (by any prior apply — most usefully another
+    # environment): building these again would recompute an identical, content-addressed
+    # table, so schedule a reuse (record + view-swap) instead of a rebuild.
+    already_built = await state.get_snapshots((name, compiled.models[name].fingerprint) for name in selected)
+
+    def is_materialised(model: CompiledModel) -> bool:
+        return (model.name, model.fingerprint) in already_built
 
     for model in compiled.ordered():  # topo order: upstream impact known before downstream
         previous_fingerprint = current.get(model.name)
@@ -311,7 +318,13 @@ async def diff(
             impact[model.name] = "semantic"
             if model.name in selected:
                 plan.changes.append(ModelChange(model.name, ChangeType.ADDED, None, None, model.fingerprint))
-                schedule_build(plan, model, snapshot_of(model, ChangeCategory.BREAKING), environment)
+                schedule_build(
+                    plan,
+                    model,
+                    snapshot_of(model, ChangeCategory.BREAKING),
+                    environment,
+                    reuse_existing=is_materialised(model),
+                )
             continue
 
         if previous_fingerprint == model.fingerprint:
@@ -373,7 +386,9 @@ async def diff(
             )
             schedule_build(plan, model, snapshot, environment, seed_from=previous.physical_table)  # type: ignore[union-attr]
         elif rebuild:
-            schedule_build(plan, model, snapshot_of(model, category), environment)
+            schedule_build(
+                plan, model, snapshot_of(model, category), environment, reuse_existing=is_materialised(model)
+            )
         else:
             _schedule_reuse(plan, model, previous, environment)  # type: ignore[arg-type]  # previous is not None here
 

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/plan/plan.py

@@ -55,6 +55,10 @@ class BackfillTask:
     # the strategy runs — history moves to the new fingerprint, the old table stays
     # as the rollback until gc.
     seed_from: TableRef | None = None
+    # This exact fingerprint is already materialised (a prior apply, often in another
+    # environment): the physical table exists and its content is fingerprint-pinned, so
+    # apply skips the build compute and only gates checks + swaps this env's view.
+    reuse_existing: bool = False
 
 
 @dataclass(frozen=True)
@@ -121,16 +125,29 @@ def env_view(environment: str, model_name: str) -> TableRef:
 
 
 def schedule_build(
-    plan: Plan, model: CompiledModel, snapshot: Snapshot, environment: str, *, seed_from: TableRef | None = None
+    plan: Plan,
+    model: CompiledModel,
+    snapshot: Snapshot,
+    environment: str,
+    *,
+    seed_from: TableRef | None = None,
+    reuse_existing: bool = False,
 ) -> None:
     """Add the right tasks for a model: ephemeral builds nothing; a terminal
     table/file builds (delivers) but gets no environment view; a virtual/view model
     builds and is repointed by an environment view.
 
-    An incremental_by_time model (virtual, or a terminal ``table``) cannot build
+    An incremental model (virtual, or a terminal ``table``) cannot build
     without a window, so an apply fills the latest grain interval — the same default
     as ``interlace run`` — leaving history to ``run --start/--end``.
+
+    ``reuse_existing`` (the fingerprint is already materialised) skips the compute for a
+    plain virtual/view build — never for a terminal delivery, a forward-only seed, or an
+    incremental window, which must always run.
     """
+    # only a plain (non-seeded, non-windowed) virtual/view build can skip its compute:
+    # a terminal always delivers, a seed must copy history, an interval must fill
+    reuse = reuse_existing and seed_from is None and model.materialise in ("virtual", "view")
     if model.materialise == "ephemeral":  # inlined into consumers, never built
         return
     wants_view = model.materialise in ("virtual", "view")  # terminal table/file has no env view
@@ -143,7 +160,7 @@ def schedule_build(
                 ViewSwap(env_view(environment, model.name), snapshot.physical_table, engine=model.engine)
             )
 
-    if model.strategy == "incremental_by_time":  # virtual or terminal table: windowed delete+insert
+    if model.strategy == "incremental":  # virtual or terminal table: windowed delete+insert
         from datetime import datetime
 
         from interlace.state.interval import latest_complete_window, parse_grain
@@ -164,7 +181,7 @@ def schedule_build(
         plan.backfills.append(BackfillTask(snapshot=snapshot, interval=window, seed_from=seed_from))
         add_view()
         return
-    plan.backfills.append(BackfillTask(snapshot=snapshot, seed_from=seed_from))
+    plan.backfills.append(BackfillTask(snapshot=snapshot, seed_from=seed_from, reuse_existing=reuse))
     add_view()
 
 

{interlaced-2.1.0 → interlaced-2.3.0}/src/interlace/plan/run.py

@@ -70,7 +70,7 @@ async def run_plan(
 
         # incremental into the interlace-owned virtual plane, or into a terminal
         # `table` (windowed delete+insert against the external target)
-        is_incremental = model.strategy == "incremental_by_time" and model.materialise != "ephemeral"
+        is_incremental = model.strategy == "incremental" and model.materialise != "ephemeral"
         wants_view = model.materialise in ("virtual", "view")  # terminal table has no env view
         if is_incremental:
             grain = parse_grain(model.interval or "1d")