interlaced 2.0.3__tar.gz → 2.1.0__tar.gz

{interlaced-2.0.3/src/interlaced.egg-info → interlaced-2.1.0}/PKG-INFO

@@ -1,9 +1,14 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.0.3
+Version: 2.1.0
 Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT
+Project-URL: Homepage, https://interlace.sh
+Project-URL: Documentation, https://interlace.sh/docs
+Project-URL: Source, https://github.com/interlace-sh/interlace
+Project-URL: Changelog, https://github.com/interlace-sh/interlace/blob/master/CHANGELOG.md
+Project-URL: Issues, https://github.com/interlace-sh/interlace/issues
 Keywords: data,pipeline,orchestration,transformation,etl,streaming,dbt,sqlmesh,duckdb
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
@@ -44,8 +49,10 @@ Provides-Extra: polars
 Requires-Dist: polars<2.0,>=1.0; extra == "polars"
 Provides-Extra: pandas
 Requires-Dist: pandas<4.0,>=2.0; extra == "pandas"
+Provides-Extra: sources
+Requires-Dist: httpx<1.0,>=0.27; extra == "sources"
 Provides-Extra: all
-Requires-Dist: interlaced[adbc,polars,postgres,service]; extra == "all"
+Requires-Dist: interlaced[adbc,polars,postgres,service,sources]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest<10.0,>=8.0; extra == "dev"
 Requires-Dist: httpx<1.0,>=0.27; extra == "dev"
@@ -65,7 +72,7 @@ Models are `.sql` files or Python functions; state is versioned snapshots with v
 environments and a terraform-style plan/apply; everything runs in a single daemon on
 DuckDB + DuckLake by default.
 
-> **Status: 1.0.** Requires Python 3.12+.
+> **Status: 2.0.** Requires Python 3.12+.
 > The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
 
 ```bash
@@ -186,13 +193,19 @@ attach:
 ```
 
 ```sql
-/* interlace: {export: {to: table, target: crm.public.accounts, mode: merge, key: id}} */
+/* interlace:
+  materialise: table
+  target: crm.public.accounts
+  strategy: merge
+  key: id
+*/
 SELECT id, tier, lifetime_value FROM account_summary
 ```
 
-File exports (`to: parquet|csv|json`) work the same way. Sinks are **environment-gated**: by
-default the side effect fires only from prod — a dev apply never writes to a live external
-table (opt in with `environments: [dev, prod]`).
+Files work the same way — `materialise: file` with `format: parquet | csv | json` and a
+`path`. Terminal models are **environment-gated**: by default the side effect fires only from
+prod, so a dev apply never writes to a live external table (opt in with
+`environments: [dev, prod]`).
 
 ## Multi-engine
 

{interlaced-2.0.3 → interlaced-2.1.0}/README.md

@@ -8,7 +8,7 @@ Models are `.sql` files or Python functions; state is versioned snapshots with v
 environments and a terraform-style plan/apply; everything runs in a single daemon on
 DuckDB + DuckLake by default.
 
-> **Status: 1.0.** Requires Python 3.12+.
+> **Status: 2.0.** Requires Python 3.12+.
 > The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
 
 ```bash
@@ -129,13 +129,19 @@ attach:
 ```
 
 ```sql
-/* interlace: {export: {to: table, target: crm.public.accounts, mode: merge, key: id}} */
+/* interlace:
+  materialise: table
+  target: crm.public.accounts
+  strategy: merge
+  key: id
+*/
 SELECT id, tier, lifetime_value FROM account_summary
 ```
 
-File exports (`to: parquet|csv|json`) work the same way. Sinks are **environment-gated**: by
-default the side effect fires only from prod — a dev apply never writes to a live external
-table (opt in with `environments: [dev, prod]`).
+Files work the same way — `materialise: file` with `format: parquet | csv | json` and a
+`path`. Terminal models are **environment-gated**: by default the side effect fires only from
+prod, so a dev apply never writes to a live external table (opt in with
+`environments: [dev, prod]`).
 
 ## Multi-engine
 

{interlaced-2.0.3 → interlaced-2.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.0.3"
+version = "2.1.0"
 description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -35,6 +35,15 @@ dependencies = [
     "pyyaml>=6.0,<7.0",           # project config (config + env overlays)
 ]
 
+# PyPI renders these as the project-page sidebar links. Without them the page is
+# a dead end — no route from a search result to the docs or the source.
+[project.urls]
+Homepage = "https://interlace.sh"
+Documentation = "https://interlace.sh/docs"
+Source = "https://github.com/interlace-sh/interlace"
+Changelog = "https://github.com/interlace-sh/interlace/blob/master/CHANGELOG.md"
+Issues = "https://github.com/interlace-sh/interlace/issues"
+
 [project.scripts]
 interlace = "interlace.cli.main:main"
 
@@ -59,7 +68,10 @@ spark = ["pyspark>=4.0,<4.1", "delta-spark>=4.0,<4.1"]
 postgres = ["psycopg[binary]>=3.1,<4.0"]
 polars = ["polars>=1.0,<2.0"]
 pandas = ["pandas>=2.0,<4.0"]
-all = ["interlaced[service,adbc,postgres,polars]"]
+# Building source models (interlace.sources): a sync REST client — auth, paging,
+# retry, rate limiting — that yields Arrow. httpx is the transport.
+sources = ["httpx>=0.27,<1.0"]
+all = ["interlaced[service,adbc,postgres,polars,sources]"]
 dev = [
     "pytest>=8.0,<10.0",
     "httpx>=0.27,<1.0",  # litestar's TestClient transport
@@ -73,15 +85,18 @@ dev = [
 where = ["src"]
 
 [tool.setuptools.package-data]
-interlace = ["py.typed", "service/ui/*", "service/ui/**/*"]
+interlace = ["py.typed", "service/ui/*", "service/ui/**/*", "templates/*", "templates/**/*"]
 
 [tool.black]
 line-length = 120
 target-version = ['py312']
+# init templates are user-facing example projects, not library code
+extend-exclude = 'src/interlace/templates/'
 
 [tool.ruff]
 line-length = 120
 target-version = "py312"
+extend-exclude = ["src/interlace/templates"]
 
 [tool.ruff.lint]
 select = [
@@ -104,6 +119,7 @@ ignore = [
 
 [tool.mypy]
 python_version = "3.12"
+exclude = ['src/interlace/templates/']  # example project code, not library code
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{interlaced-2.0.3 → interlaced-2.1.0}/src/interlace/__init__.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 from importlib.metadata import PackageNotFoundError, version
 
+from interlace.checks.spec import CheckSpec
 from interlace.dsl.decorators import check, model, stream
 
 try:
@@ -20,6 +21,7 @@ except PackageNotFoundError:  # running from a source tree without an install
 
 __all__ = [
     "__version__",
+    "CheckSpec",
     "check",
     "model",
     "stream",

{interlaced-2.0.3 → interlaced-2.1.0}/src/interlace/cli/main.py

@@ -11,10 +11,11 @@ from typing import Any
 import typer
 from rich import box
 from rich.console import Console
+from rich.markup import escape
 from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeElapsedColumn
 from rich.table import Table
 
-from interlace.exceptions import CheckError, ConfigurationError, InterlaceError, SelectionError
+from interlace.exceptions import CheckError, ConfigurationError, InterlaceError, QueryError, SelectionError
 from interlace.graph.column_lineage import column_impact, column_lineage, split_target
 from interlace.graph.project import CompiledProject
 from interlace.graph.selectors import select_models, wants_state
@@ -24,7 +25,7 @@ from interlace.plan.differ import diff
 from interlace.plan.plan import ChangeType, Plan
 from interlace.plan.run import run_plan
 from interlace.project import Project
-from interlace.scaffold import scaffold_project
+from interlace.scaffold import list_templates, scaffold_project
 from interlace.scheduler.engine import TriggerEngine, build_triggers
 from interlace.scheduler.worker import drain
 from interlace.sinks import target_ref
@@ -58,12 +59,16 @@ class _BuildProgress:
     def __call__(self, model: str, event: str) -> None:
         if event == "start":
             self._rows[model] = self.progress.add_task(model, total=1, status="")
-        elif event == "done":
-            self.progress.update(self._rows[model], completed=1, status="[green]✓[/green]")
+            return
+        task = self._rows.get(model)
+        if task is None:  # a model cancelled by a sibling's failure before it ever started has no row
+            return
+        if event == "done":
+            self.progress.update(task, completed=1, status="[green]✓[/green]")
         elif event == "cancelled":  # collateral of a sibling's failure, not a failure itself
-            self.progress.update(self._rows[model], completed=1, status="[dim]⊘[/dim]")
+            self.progress.update(task, completed=1, status="[dim]⊘[/dim]")
         else:  # failed
-            self.progress.update(self._rows[model], completed=1, status="[red]✗[/red]")
+            self.progress.update(task, completed=1, status="[red]✗[/red]")
 
 
 def _build_progress(plan_result: Plan) -> _BuildProgress | None:
@@ -224,7 +229,7 @@ def _selection(
     try:
         return select_models(selectors, compiled, promoted=promoted)
     except SelectionError as exc:
-        console.print(f"[red]{exc.message}[/red]")
+        console.print(f"[red]{escape(exc.message)}[/red]")
         raise typer.Exit(1) from exc
 
 
@@ -237,16 +242,30 @@ async def _promoted_if_needed(state: Any, environment: str, selectors: list[str]
 def init(
     path: Path = typer.Argument(Path("."), help="Directory to initialise."),
     name: str = typer.Option("", "--name", "-n", help="Project name (defaults to the directory name)."),
+    template: str = typer.Option("quickstart", "--template", "-t", help="Which starter to scaffold (see --list)."),
+    show_list: bool = typer.Option(False, "--list", help="List available templates and exit."),
 ) -> None:
-    """Scaffold a new interlace project."""
+    """Scaffold a new interlace project from a template."""
+    if show_list:
+        table = _table("Templates")
+        table.add_column("Template")
+        table.add_column("Description", style="dim")
+        table.add_column("Needs", style="dim")
+        for info in list_templates():
+            table.add_row(info.name, info.description, ", ".join(info.requires_env) or "—")
+        console.print(table)
+        return
     try:
-        written = scaffold_project(path, name or None)
+        written = scaffold_project(path, name or None, template)
     except ConfigurationError as exc:
-        console.print(f"[red]{exc.message}[/red] ({exc.details.get('path', '')})")
+        console.print(f"[red]{escape(exc.message)}[/red] ({exc.details.get('path', '')})")
         raise typer.Exit(1) from exc
-    console.print(f"[green]Initialised interlace project in {path}[/green]")
+    console.print(f"[green]Initialised interlace project in {path}[/green] [dim](template: {template})[/dim]")
     for written_path in written:
         console.print(f"  + {written_path}")
+    needs = next((t.requires_env for t in list_templates() if t.name == template), ())
+    if needs:
+        console.print(f"\n[yellow]Set before applying:[/yellow] {', '.join(needs)}")
     console.print("\nNext: [bold]interlace apply[/bold] (or --env dev for a sandbox)")
 
 
@@ -360,7 +379,7 @@ async def _apply(
                     parallelism=parallelism or project.config.parallelism,  # --parallelism wins over config
                 )
         except CheckError as exc:
-            console.print(f"[red]{exc.message}[/red]")
+            console.print(f"[red]{escape(exc.message)}[/red]")
             raise typer.Exit(1) from exc
         _render_build_results(result, compiled)
         _render_checks(result)
@@ -459,7 +478,7 @@ async def _execute(
                     parallelism=parallelism or project.config.parallelism,  # --parallelism wins over config
                 )
         except CheckError as exc:
-            console.print(f"[red]{exc.message}[/red]")
+            console.print(f"[red]{escape(exc.message)}[/red]")
             raise typer.Exit(1) from exc
         _render_build_results(result, compiled)
         _render_checks(result)
@@ -631,7 +650,7 @@ def serve(
 
         from interlace.service.app import create_app
     except ImportError as exc:
-        console.print("[red]The HTTP API needs the 'service' extra: pip install 'interlaced[service]'[/red]")
+        console.print(r"[red]The HTTP API needs the 'service' extra: pip install 'interlaced\[service]'[/red]")
         raise typer.Exit(1) from exc
     bound = _free_port(host, port)
     if bound != port:
@@ -679,6 +698,56 @@ def serve(
     server.run()
 
 
+@app.command()
+def query(
+    sql: str = typer.Argument(..., help="A read-only SELECT to run against the warehouse."),
+    path: Path = _PATH,
+    limit: int = typer.Option(100, "--limit", "-n", help="Maximum rows to display (max 10,000)."),
+) -> None:
+    """Run a read-only SELECT against the warehouse and print the result.
+
+    SELECT only — real tables and views, not table functions or files (the same fence
+    as the web console). Unqualified names resolve to the promoted (prod) views:
+
+        interlace query "SELECT * FROM raw_events"
+    """
+    asyncio.run(_query(sql, path, limit))
+
+
+async def _query(sql: str, path: Path, limit: int) -> None:
+    from interlace.query import prepare_readonly
+
+    project = Project.load(path)
+    engines = project.open_engines()
+    try:
+        engine = engines.get()
+        bounded, cap = prepare_readonly(sql, engine.dialect, limit)
+        try:
+            reader = await engine.fetch(bounded)
+            columns = list(reader.schema.names)
+            records = await asyncio.to_thread(lambda: reader.read_all().to_pylist())
+        except QueryError:
+            raise
+        except Exception as exc:  # engine errors (missing table, bad column) are the user's feedback
+            raise QueryError(str(exc)) from exc
+    finally:
+        engines.close()
+    _render_query(columns, records, cap)
+
+
+def _render_query(columns: list[str], records: list[dict], cap: int) -> None:
+    truncated = len(records) > cap
+    shown = records[:cap]
+    table = _table("")
+    for name in columns:
+        table.add_column(name)
+    for record in shown:
+        table.add_row(*("[dim]NULL[/dim]" if record[name] is None else escape(str(record[name])) for name in columns))
+    console.print(table)
+    note = f"{len(shown)} row(s)" + (" — truncated; raise --limit for more" if truncated else "")
+    console.print(f"[dim]{note}[/dim]")
+
+
 @app.command("models")
 def list_models(path: Path = _PATH, select: list[str] = _SELECT, as_json: bool = _JSON) -> None:
     """List models with their materialisation, strategy, engine, and dependencies."""
@@ -799,7 +868,7 @@ async def _env_rollback(name: str, path: Path, to: int | None, history: bool, as
         try:
             result = await rollback_environment(state, engines=engines, environment=name, to_generation=to)
         except PlanError as exc:
-            console.print(f"[red]{exc.message}[/red]")
+            console.print(f"[red]{escape(exc.message)}[/red]")
             raise typer.Exit(1) from exc
         if as_json:
             _emit_json(result)
@@ -1128,6 +1197,28 @@ def engines(path: Path = _PATH, as_json: bool = _JSON) -> None:
     console.print(table)
 
 
+async def _warehouse_columns(project: Project, compiled: CompiledProject) -> dict[str, list[str]]:
+    """Best-effort real output columns per model, probed from the built warehouse so
+    column lineage traces precisely through Python models (a Python model's true
+    columns can't be known statically). Empty on any failure — an unbuilt project or
+    a warehouse held by a running ``serve`` — and lineage falls back to static analysis."""
+    from sqlglot import exp
+
+    try:
+        engines = project.open_engines()
+    except Exception:
+        return {}
+    described: dict[str, list[str]] = {}
+    try:
+        for name, model in compiled.models.items():
+            with contextlib.suppress(Exception):  # unbuilt / non-queryable (file target): names-only is fine
+                reader = await engines.get(model.engine).fetch(exp.select("*").from_(exp.to_table(name)).limit(0))
+                described[name] = list(reader.schema.names)
+    finally:
+        engines.close()
+    return described
+
+
 @app.command()
 def impact(
     target: str = typer.Argument(..., help="model.column — what would changing this column touch?"),
@@ -1143,7 +1234,8 @@ def impact(
         console.print(f"[red]expected <model>.<column> with a known model; got {target!r}[/red]")
         raise typer.Exit(1)
     model, column = parsed
-    result = column_impact(compiled, model, column)
+    described = asyncio.run(_warehouse_columns(project, compiled))
+    result = column_impact(compiled, model, column, known_columns=described)
     impacted = result["impacted"]
     opaque = result["opaque_consumers"]
 
@@ -1186,7 +1278,8 @@ def lineage(
 
     upstream = sorted(compiled.graph.ancestors(model))
     downstream = sorted(compiled.graph.descendants(model))
-    sources = column_lineage(compiled).get(model, {}) if columns else {}
+    described = asyncio.run(_warehouse_columns(project, compiled)) if columns else {}
+    sources = column_lineage(compiled, known_columns=described).get(model, {}) if columns else {}
 
     if fmt == "dot":
         typer.echo(_lineage_dot(compiled, model, upstream, downstream, sources))
@@ -1348,9 +1441,28 @@ def _render(plan: Plan, environment: str) -> None:
         )
 
 
+def _flatten_exceptions(exc: BaseException) -> list[BaseException]:
+    """Leaf exceptions of a (possibly nested) ExceptionGroup — a parallel apply
+    reports its failures as one."""
+    if isinstance(exc, BaseExceptionGroup):
+        return [leaf for sub in exc.exceptions for leaf in _flatten_exceptions(sub)]
+    return [exc]
+
+
 def main() -> None:
     try:
         app()
     except InterlaceError as exc:  # expected, user-facing errors: one clean line, no traceback
-        err_console.print(f"[red]error:[/red] {exc.message}")
+        err_console.print(f"[red]error:[/red] {escape(exc.message)}")
         raise SystemExit(1) from None
+    except BaseExceptionGroup as group:
+        # A parallel apply surfaces failures as an ExceptionGroup. When every leaf is a
+        # user-facing InterlaceError (e.g. several models failed their definition/checks),
+        # print one clean line each instead of dumping the group traceback; a genuine
+        # internal error in the mix still propagates with its trace.
+        leaves = _flatten_exceptions(group)
+        if leaves and all(isinstance(leaf, InterlaceError) for leaf in leaves):
+            for message in dict.fromkeys(leaf.message for leaf in leaves if isinstance(leaf, InterlaceError)):
+                err_console.print(f"[red]error:[/red] {escape(message)}")
+            raise SystemExit(1) from None
+        raise

{interlaced-2.0.3 → interlaced-2.1.0}/src/interlace/dsl/decorators.py

@@ -201,6 +201,11 @@ def model(
 
     ``engine`` pins the model to a named engine from ``interlace.yaml`` (defaults
     to the project's ``default_engine``).
+
+    ``checks`` takes the *same* shorthand mappings as a SQL config block — one
+    spelling for both surfaces — e.g. ``checks=[{"not_null": "customer_id"},
+    {"row_count": {"min": 1}}]``. A :class:`~interlace.CheckSpec` also works if you
+    prefer the typed form.
     """
     model_name = name or "<model>"
     if export is not None:

{interlaced-2.0.3 → interlaced-2.1.0}/src/interlace/dsl/discovery.py

@@ -17,7 +17,7 @@ from typing import Any
 from interlace.checks.spec import parse_checks
 from interlace.dsl.decorators import REGISTRY, ModelDef, _as_columns, _as_tuple, validate_materialise
 from interlace.dsl.sql_config import extract_sql_config
-from interlace.exceptions import DefinitionError
+from interlace.exceptions import DefinitionError, InterlaceError
 
 
 def discover_models(root: Path, model_paths: list[str], default_dialect: str) -> list[ModelDef]:
@@ -82,6 +82,13 @@ def _model_name(base: Path, file: Path) -> str:
     return ".".join(file.relative_to(base).with_suffix("").parts)
 
 
+def _relative_to_cwd(file: Path) -> str:
+    try:
+        return str(file.relative_to(Path.cwd()))
+    except ValueError:
+        return str(file)
+
+
 def _import_module(base: Path, file: Path) -> None:
     module_name = "interlace_model_" + "_".join(file.relative_to(base).with_suffix("").parts)
     spec = importlib.util.spec_from_file_location(module_name, file)
@@ -89,4 +96,15 @@ def _import_module(base: Path, file: Path) -> None:
         raise DefinitionError("could not import model module", details={"path": str(file)})
     module = importlib.util.module_from_spec(spec)
     sys.modules[module_name] = module
-    spec.loader.exec_module(module)
+    try:
+        spec.loader.exec_module(module)
+    except InterlaceError:
+        raise  # a bad @model config is already a clean, actionable error — don't rewrap it
+    except Exception as exc:
+        # a typo/import error in user model code should read like a user error — one
+        # line naming the file — not a dozen frames of interlace's import machinery
+        sys.modules.pop(module_name, None)  # don't leave a half-initialised module behind
+        raise DefinitionError(
+            f"could not load {_relative_to_cwd(file)}: {type(exc).__name__}: {exc}",
+            details={"path": str(file)},
+        ) from exc

{interlaced-2.0.3 → interlaced-2.1.0}/src/interlace/exceptions.py

@@ -42,6 +42,14 @@ class PlanError(InterlaceError):
     """A plan could not be computed or applied."""
 
 
+class ExecutionError(InterlaceError):
+    """A model failed while building (its query, its Python function, or the load)."""
+
+
+class QueryError(InterlaceError):
+    """A read-only query (the console / `interlace query`) was rejected or failed."""
+
+
 class SchemaError(InterlaceError):
     """A model's built schema violates its declared column contract."""