interlaced 2.4.0__tar.gz → 2.4.1__tar.gz

{interlaced-2.4.0/src/interlaced.egg-info → interlaced-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.4.0
+Version: 2.4.1
 Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT

{interlaced-2.4.0 → interlaced-2.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.4.0"
+version = "2.4.1"
 description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -180,6 +180,7 @@ markers = [
     "integration: Integration tests (slower, require database)",
     "slow: Slow tests (may take a long time)",
     "requires_db: Needs a reachable external database (e.g. Postgres)",
+    "requires_network: Needs the public internet (e.g. an example reading a remote CSV)",
 ]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"

{interlaced-2.4.0 → interlaced-2.4.1}/src/interlace/cli/main.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import asyncio
 import contextlib
+import time
 from datetime import datetime
 from pathlib import Path
 from typing import Any
@@ -467,6 +468,7 @@ async def _execute(
             restate=restate,
         )
         progress = _build_progress(plan_result)
+        started = time.perf_counter()
         try:
             with progress.progress if progress else contextlib.nullcontext():
                 result = await apply_plan(
@@ -487,7 +489,7 @@ async def _execute(
         await _render_empty_incrementals(result, compiled, engines)
         verb = "Restated" if restate else "Ran"
         console.print(
-            f"[green]{verb} {len(set(result.built))} model(s) ({len(result.built)} task(s)); "
+            f"[green]{verb} {len(set(result.built))} model(s) in {time.perf_counter() - started:.2f}s; "
             f"promoted {result.promoted} to '{environment}'.[/green]"
         )
     finally:

{interlaced-2.4.0 → interlaced-2.4.1}/src/interlace/config/config.py

@@ -145,6 +145,9 @@ class ProjectConfig(BaseModel):
     # This is T0 federation (all SQL still runs in DuckDB) — see docs/architecture/MULTI_ENGINE.md.
     attach: dict[str, str] = Field(default_factory=dict)
     model_paths: list[str] = Field(default_factory=lambda: ["models"])
+    # Directories of CREATE MACRO definitions, expanded into every model at compile
+    # time (see interlace.ir.macros). Missing directories are ignored.
+    macro_paths: list[str] = Field(default_factory=lambda: ["macros"])
     # How many models build concurrently. Lower it to serialise a build end-to-end
     # when diagnosing a crash or a warehouse that dislikes parallel writers; `run`
     # and `apply` also take --parallelism, which wins over this.

{interlaced-2.4.0 → interlaced-2.4.1}/src/interlace/dsl/discovery.py

@@ -9,8 +9,10 @@ cleared first for a clean load.
 
 from __future__ import annotations
 
+import contextlib
 import importlib.util
 import sys
+from collections.abc import Iterator
 from pathlib import Path
 from typing import Any
 
@@ -18,22 +20,61 @@ from interlace.checks.spec import parse_checks
 from interlace.dsl.decorators import REGISTRY, ModelDef, _as_columns, _as_tuple, validate_materialise
 from interlace.dsl.sql_config import extract_sql_config
 from interlace.exceptions import DefinitionError, InterlaceError
+from interlace.ir.macros import Macro, parse_macros
 
 
 def discover_models(root: Path, model_paths: list[str], default_dialect: str) -> list[ModelDef]:
     REGISTRY.clear()
-    for relative in model_paths:
+    imported = set(sys.modules)
+    try:
+        for relative in model_paths:
+            base = root / relative
+            if not base.is_dir():
+                continue
+            for sql_file in sorted(base.rglob("*.sql")):
+                config, sql = extract_sql_config(sql_file.read_text())
+                REGISTRY.register_model(_sql_model(_model_name(base, sql_file), sql, config, default_dialect))
+            for py_file in sorted(base.rglob("*.py")):
+                if py_file.name.startswith("_"):
+                    continue
+                _import_module(base, py_file)
+    finally:
+        _forget_project_modules(root, imported)
+    return list(REGISTRY.models.values())
+
+
+def _forget_project_modules(root: Path, before: set[str]) -> None:
+    """Drop the project's own helper modules from the import cache.
+
+    A helper is imported under whatever plain name it has (``_macros``), so leaving it
+    cached would hand the *next* project — a reload under ``interlace serve``, another
+    project in the same process — the first one's version of a file with the same name.
+    Only modules loaded from inside this project during this pass are dropped."""
+    for name in set(sys.modules) - before:
+        module = sys.modules.get(name)
+        origin = getattr(module, "__file__", None)
+        if origin and Path(origin).is_relative_to(root):
+            del sys.modules[name]
+
+
+def discover_macros(root: Path, macro_paths: list[str], default_dialect: str) -> dict[str, Macro]:
+    """Parse every ``CREATE MACRO`` under the macro paths, keyed by casefolded name."""
+    macros: dict[str, Macro] = {}
+    for relative in macro_paths:
         base = root / relative
         if not base.is_dir():
             continue
-        for sql_file in sorted(base.rglob("*.sql")):
-            config, sql = extract_sql_config(sql_file.read_text())
-            REGISTRY.register_model(_sql_model(_model_name(base, sql_file), sql, config, default_dialect))
-        for py_file in sorted(base.rglob("*.py")):
-            if py_file.name.startswith("_"):
-                continue
-            _import_module(base, py_file)
-    return list(REGISTRY.models.values())
+        for macro_file in sorted(base.rglob("*.sql")):
+            source = str(macro_file.relative_to(root))
+            for macro in parse_macros(macro_file.read_text(), default_dialect, source):
+                key = macro.name.casefold()
+                if key in macros:
+                    raise DefinitionError(
+                        f"macro {macro.name!r} is defined twice: {macros[key].source} and {source}",
+                        details={"macro": macro.name},
+                    )
+                macros[key] = macro
+    return macros
 
 
 def _sql_model(default_name: str, sql: str, config: dict[str, Any], default_dialect: str) -> ModelDef:
@@ -89,6 +130,18 @@ def _relative_to_cwd(file: Path) -> str:
         return str(file)
 
 
+@contextlib.contextmanager
+def _importable(directory: Path) -> Iterator[None]:
+    """Put ``directory`` at the front of ``sys.path``, and take it back off."""
+    entry = str(directory)
+    sys.path.insert(0, entry)
+    try:
+        yield
+    finally:
+        with contextlib.suppress(ValueError):
+            sys.path.remove(entry)
+
+
 def _import_module(base: Path, file: Path) -> None:
     module_name = "interlace_model_" + "_".join(file.relative_to(base).with_suffix("").parts)
     spec = importlib.util.spec_from_file_location(module_name, file)
@@ -97,7 +150,13 @@ def _import_module(base: Path, file: Path) -> None:
     module = importlib.util.module_from_spec(spec)
     sys.modules[module_name] = module
     try:
-        spec.loader.exec_module(module)
+        # The model's own directory goes on the path for the duration of the import, so a
+        # model can `from _shared import ...` a helper sitting next to it — the closest
+        # thing to a macro. Files starting with `_` are already skipped as models, which
+        # is only useful if they can be imported. Removed again straight after: a project's
+        # helpers must not leak into the import path of everything that follows.
+        with _importable(file.parent):
+            spec.loader.exec_module(module)
     except InterlaceError:
         raise  # a bad @model config is already a clean, actionable error — don't rewrap it
     except Exception as exc:

{interlaced-2.4.0 → interlaced-2.4.1}/src/interlace/graph/project.py

@@ -12,7 +12,7 @@ from __future__ import annotations
 
 import inspect
 import textwrap
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field
 
 from sqlglot import exp
@@ -23,6 +23,7 @@ from interlace.exceptions import CompilationError, DefinitionError
 from interlace.graph.dag import DependencyGraph
 from interlace.ir.canonicalize import parse, table_references
 from interlace.ir.fingerprint import canonical_sql, data_fingerprint, metadata_fingerprint
+from interlace.ir.macros import Macro, expand_macros
 from interlace.ir.relation import TableRef
 
 _PHYSICAL_PREFIX = "interlace__"
@@ -93,7 +94,7 @@ def _physical_table(name: str, fingerprint: str, catalog: str | None) -> TableRe
 
 
 def _resolve_dependencies(
-    model: ModelDef, names: set[str], default_dialect: str
+    model: ModelDef, names: set[str], default_dialect: str, macros: Mapping[str, Macro]
 ) -> tuple[tuple[str, ...], exp.Expression | None, str]:
     dialect = model.dialect or default_dialect
     deps: list[str] = []
@@ -115,6 +116,10 @@ def _resolve_dependencies(
             raise CompilationError(
                 f"model {model.name!r}: {exc.message}", details={**exc.details, "model": model.name}
             ) from exc
+        # Before anything reads the AST: the fingerprint is canonical SQL, so expanding
+        # here is what makes a macro edit rebuild its callers, and what lets a macro body
+        # reference a model and have that count as a dependency.
+        ast = expand_macros(ast, macros, model.name)
         for ref in table_references(ast):
             if ref in names:
                 add(ref)
@@ -149,12 +154,14 @@ def compile_models(
     known_engines: set[str] | None = None,
     catalog: str | None = None,
     checks: Iterable[CheckDef] = (),
+    macros: Mapping[str, Macro] | None = None,
 ) -> CompiledProject:
     """Compile models into a fingerprinted, topologically-ordered project.
 
     ``checks`` are ``@check``-decorated Python functions, attached by model name.
     ``engine_dialects`` maps engine name → sqlglot dialect (used when a model
     omits ``dialect``). ``known_engines`` validates model ``engine`` pins.
+    ``macros`` are expanded into every model's AST before it is fingerprinted.
     """
     definitions = {m.name: m for m in models}
     names = set(definitions)
@@ -178,7 +185,7 @@ def compile_models(
             )
         # Authoring dialect: explicit model dialect, else the engine's, else project default.
         model_default_dialect = dialects_by_engine.get(engine, default_dialect)
-        deps, ast, dialect = _resolve_dependencies(definition, names, model_default_dialect)
+        deps, ast, dialect = _resolve_dependencies(definition, names, model_default_dialect, macros or {})
         resolved[name] = (deps, ast, dialect, engine)
         graph.add_node(name)
         for dep in deps:

interlaced-2.4.1/src/interlace/ir/macros.py

@@ -0,0 +1,154 @@
+"""SQL macros: one definition, expanded into the IR before anything else sees it.
+
+A macro is written in SQL, in the engine's own ``CREATE MACRO`` syntax::
+
+    CREATE MACRO cents_to_dollars(amount) AS (amount / 100)::numeric(16, 2);
+
+and called like a function from any model::
+
+    SELECT cents_to_dollars(subtotal) AS subtotal FROM raw_orders
+
+The call is substituted for the macro's body while the model is compiled — before the
+fingerprint, before lineage, before transpilation. That ordering is the whole design:
+
+- **The fingerprint covers the macro.** A model's fingerprint is its canonical SQL, and
+  the expansion is part of it, so editing a macro rebuilds every model that calls it.
+  A macro created in the warehouse instead would be invisible to the fingerprint, and a
+  changed one would leave stale tables behind with nothing to notice.
+- **Lineage sees through it.** Column lineage reads the AST, and the AST no longer has
+  an opaque function call in it.
+- **One definition, every engine.** The expansion happens in dialect-agnostic AST, so
+  the transpiler renders it per engine. This is the job dbt does with adapter dispatch —
+  ``postgres__cents_to_dollars``, ``bigquery__cents_to_dollars`` — and it is not needed
+  here: ``(amount / 100)`` becomes integer division on Postgres by itself, so sqlglot
+  emits ``CAST(amount AS DOUBLE PRECISION) / NULLIF(100, 0)`` and the macro stays one
+  line.
+
+The cost, and it is a real one: the macro exists at build time, not in the warehouse.
+Someone querying the tables by hand cannot call it.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+import sqlglot
+from sqlglot import exp
+
+from interlace.exceptions import DefinitionError
+
+_MAX_DEPTH = 10  # macros may call macros; this is the runaway/recursion stop
+
+
+@dataclass(frozen=True)
+class Macro:
+    """A parsed ``CREATE MACRO``: its name, parameter names, and body expression."""
+
+    name: str
+    params: tuple[str, ...]
+    body: exp.Expression
+    source: str  # the file it came from, for error messages
+
+
+def parse_macros(sql: str, dialect: str, source: str) -> list[Macro]:
+    """Parse the ``CREATE MACRO`` statements in one file."""
+    try:
+        statements = sqlglot.parse(sql, read=dialect)
+    except Exception as exc:  # noqa: BLE001 — sqlglot raises a family of parse errors
+        raise DefinitionError(f"could not parse macros in {source}: {exc}", details={"source": source}) from exc
+
+    macros: list[Macro] = []
+    for statement in statements:
+        if statement is None:
+            continue
+        if isinstance(statement, exp.Command):
+            # sqlglot could not parse it — a table macro (AS TABLE SELECT ...) lands here
+            raise DefinitionError(
+                f"{source}: could not read this as a macro definition: {statement.sql()[:80]!r}. A macro is a "
+                f"scalar expression — CREATE MACRO name(args) AS <expression>. A table macro has no call site "
+                f"to expand into; write it as a model.",
+                details={"source": source},
+            )
+        udf = statement.this if isinstance(statement, exp.Create) else None
+        if not isinstance(udf, exp.UserDefinedFunction):
+            raise DefinitionError(
+                f"{source}: expected only CREATE MACRO statements, found {type(statement).__name__.upper()}",
+                details={"source": source},
+            )
+        body = statement.expression
+        if body is None or isinstance(body, exp.Query):
+            raise DefinitionError(
+                f"{source}: macro {udf.this.name!r} must be a scalar expression — a table macro "
+                f"(AS TABLE SELECT ...) has no call site to expand into; write it as a model",
+                details={"source": source, "macro": udf.this.name},
+            )
+        # the UDF's `this` is a Table node wrapping the identifier, so the name is a
+        # level down from where UserDefinedFunction.name looks
+        name = udf.this.name if isinstance(udf.this, exp.Expression) else str(udf.this)
+        macros.append(
+            Macro(
+                name=name,
+                params=tuple(param.name for param in udf.expressions),
+                body=body,
+                source=source,
+            )
+        )
+    return macros
+
+
+def expand_macros(ast: exp.Expression, macros: Mapping[str, Macro], model: str) -> exp.Expression:
+    """Substitute every macro call in ``ast`` for its body. Returns a new expression."""
+    if not macros:
+        return ast
+    expanded = ast
+    for _ in range(_MAX_DEPTH):
+        expanded, hits = _expand_once(expanded, macros, model)
+        if not hits:
+            return expanded
+    raise DefinitionError(
+        f"model {model!r}: macro expansion did not settle after {_MAX_DEPTH} passes — a macro calls itself, "
+        f"directly or through another",
+        details={"model": model},
+    )
+
+
+def _expand_once(ast: exp.Expression, macros: Mapping[str, Macro], model: str) -> tuple[exp.Expression, int]:
+    hits = 0
+
+    def substitute(node: exp.Expression) -> exp.Expression:
+        nonlocal hits
+        if not isinstance(node, exp.Anonymous):
+            return node
+        macro = macros.get(node.name.casefold())
+        if macro is None:
+            return node  # an ordinary engine function
+        args = list(node.expressions)
+        if len(args) != len(macro.params):
+            raise DefinitionError(
+                f"model {model!r}: macro {macro.name!r} takes {len(macro.params)} argument(s) "
+                f"({', '.join(macro.params) or 'none'}), called with {len(args)}",
+                details={"model": model, "macro": macro.name, "source": macro.source},
+            )
+        hits += 1
+        return exp.paren(_bind(macro, args))
+
+    return ast.transform(substitute, copy=True), hits
+
+
+def _bind(macro: Macro, args: list[exp.Expression]) -> exp.Expression:
+    """The macro's body with each parameter reference replaced by its argument."""
+    body = macro.body.copy()
+    if not macro.params:
+        return body
+    by_name = {param.casefold(): arg for param, arg in zip(macro.params, args, strict=True)}
+
+    def replace(node: exp.Expression) -> exp.Expression:
+        if isinstance(node, exp.Column) and not node.table:
+            argument = by_name.get(node.name.casefold())
+            if argument is not None:
+                return argument.copy()
+        return node
+
+    bound: exp.Expression = body.transform(replace, copy=False)
+    return bound

{interlaced-2.4.0 → interlaced-2.4.1}/src/interlace/plan/apply.py

@@ -733,16 +733,19 @@ async def apply(
         return found
 
     # Checks that read *other* models (relationships, sql) add scheduling edges the
-    # data DAG doesn't have — and may point "backwards", which would deadlock the
-    # per-model events. Keep a check edge only when it agrees with the topological
-    # order (dep built before the reader); a back edge is dropped (its check may run
-    # a beat early, far better than a hang). Data edges always agree, so they all stay.
-    topo = {name: index for index, name in enumerate(compiled.graph.topological_sort())}
+    # data DAG doesn't have. Keep one unless it would close a cycle — which it does
+    # exactly when the reader is already an ancestor of what its check reads (a check
+    # pointing downstream, e.g. order_items testing against orders, which is built
+    # from order_items). Those are dropped: the check runs before its target exists
+    # and fails, but a cycle would hang the whole apply, and the model can point its
+    # check at an upstream instead. Everything else is a plain sibling reference and
+    # is enforced — dropping those on a topological-order heuristic failed checks
+    # whose target simply happened to sort later.
     blocking: dict[str, set[str]] = {}
     for name in per_model:
         deps = in_plan_ancestors(name)
         for ref in _check_references(compiled.models[name], compiled):
-            if ref in per_model and topo.get(ref, -1) < topo.get(name, 0):
+            if ref in per_model and name not in compiled.graph.ancestors(ref):
                 deps.add(ref)
         blocking[name] = deps
 

{interlaced-2.4.0 → interlaced-2.4.1}/src/interlace/project.py

@@ -9,17 +9,18 @@ from __future__ import annotations
 
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 
 from interlace.config.config import CONFIG_FILE, EngineConfig, ProjectConfig, SecretConfig, load_config
 from interlace.dsl.decorators import REGISTRY, CheckDef, ModelDef, StreamDef
-from interlace.dsl.discovery import discover_models
+from interlace.dsl.discovery import discover_macros, discover_models
 from interlace.engines.base import EngineAdapter
 from interlace.engines.duckdb import DuckDBAdapter
 from interlace.engines.registry import EngineRegistry
 from interlace.exceptions import ConfigurationError
 from interlace.graph.project import CompiledProject, compile_models
+from interlace.ir.macros import Macro
 from interlace.state.store import SqliteStateStore
 from interlace.streaming.log import SqliteStreamLog
 
@@ -86,6 +87,7 @@ class Project:
     models: list[ModelDef]
     checks: list[CheckDef]
     streams: list[StreamDef]
+    macros: dict[str, Macro] = field(default_factory=dict)
 
     @classmethod
     def load(cls, root: Path | str) -> Project:
@@ -98,6 +100,7 @@ class Project:
             models=models,
             checks=list(REGISTRY.checks),
             streams=list(REGISTRY.streams.values()),
+            macros=discover_macros(root, config.macro_paths, config.default_dialect),
         )
 
     def compile(self) -> CompiledProject:
@@ -109,6 +112,7 @@ class Project:
             engine_dialects={name: cfg.resolved_dialect() for name, cfg in engine_cfgs.items()},
             known_engines=set(engine_cfgs),
             checks=self.checks,
+            macros=self.macros,
         )
 
     def open_engine(self, name: str | None = None) -> EngineAdapter:

{interlaced-2.4.0 → interlaced-2.4.1/src/interlaced.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.4.0
+Version: 2.4.1
 Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT

{interlaced-2.4.0 → interlaced-2.4.1}/src/interlaced.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ src/interlace/graph/selectors.py
 src/interlace/ir/__init__.py
 src/interlace/ir/canonicalize.py
 src/interlace/ir/fingerprint.py
+src/interlace/ir/macros.py
 src/interlace/ir/relation.py
 src/interlace/plan/__init__.py
 src/interlace/plan/apply.py