interlaced 2.0.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. interlace/__init__.py +26 -0
  2. interlace/checks/__init__.py +10 -0
  3. interlace/checks/builtin.py +152 -0
  4. interlace/checks/runner.py +103 -0
  5. interlace/checks/spec.py +113 -0
  6. interlace/cli/__init__.py +7 -0
  7. interlace/cli/main.py +1072 -0
  8. interlace/config/__init__.py +7 -0
  9. interlace/config/config.py +172 -0
  10. interlace/contracts.py +35 -0
  11. interlace/dsl/__init__.py +16 -0
  12. interlace/dsl/decorators.py +222 -0
  13. interlace/dsl/discovery.py +75 -0
  14. interlace/dsl/sql_config.py +47 -0
  15. interlace/engines/__init__.py +16 -0
  16. interlace/engines/base.py +86 -0
  17. interlace/engines/duckdb.py +249 -0
  18. interlace/engines/postgres.py +157 -0
  19. interlace/engines/quack.py +113 -0
  20. interlace/engines/registry.py +125 -0
  21. interlace/exceptions.py +66 -0
  22. interlace/exports.py +145 -0
  23. interlace/graph/__init__.py +17 -0
  24. interlace/graph/column_lineage.py +69 -0
  25. interlace/graph/dag.py +81 -0
  26. interlace/graph/project.py +242 -0
  27. interlace/graph/selectors.py +45 -0
  28. interlace/ir/__init__.py +24 -0
  29. interlace/ir/canonicalize.py +71 -0
  30. interlace/ir/fingerprint.py +55 -0
  31. interlace/ir/relation.py +71 -0
  32. interlace/ir/schema.py +23 -0
  33. interlace/plan/__init__.py +23 -0
  34. interlace/plan/apply.py +527 -0
  35. interlace/plan/differ.py +211 -0
  36. interlace/plan/plan.py +177 -0
  37. interlace/plan/resolve.py +74 -0
  38. interlace/plan/run.py +79 -0
  39. interlace/project.py +278 -0
  40. interlace/py.typed +0 -0
  41. interlace/runtime/__init__.py +6 -0
  42. interlace/runtime/handles.py +48 -0
  43. interlace/runtime/python_model.py +175 -0
  44. interlace/scaffold.py +83 -0
  45. interlace/scheduler/__init__.py +17 -0
  46. interlace/scheduler/engine.py +66 -0
  47. interlace/scheduler/triggers.py +84 -0
  48. interlace/scheduler/worker.py +154 -0
  49. interlace/service/__init__.py +7 -0
  50. interlace/service/app.py +836 -0
  51. interlace/service/auth.py +41 -0
  52. interlace/state/__init__.py +18 -0
  53. interlace/state/interval.py +134 -0
  54. interlace/state/janitor.py +142 -0
  55. interlace/state/snapshot.py +47 -0
  56. interlace/state/store.py +906 -0
  57. interlace/strategies/__init__.py +64 -0
  58. interlace/strategies/base.py +71 -0
  59. interlace/strategies/full.py +38 -0
  60. interlace/strategies/full_merge.py +98 -0
  61. interlace/strategies/incremental_by_time.py +65 -0
  62. interlace/strategies/merge_by_key.py +69 -0
  63. interlace/strategies/scd_type_2.py +122 -0
  64. interlace/strategies/view.py +33 -0
  65. interlace/streaming/__init__.py +18 -0
  66. interlace/streaming/log.py +298 -0
  67. interlace/streaming/materializer.py +166 -0
  68. interlace/streaming/schema.py +218 -0
  69. interlaced-2.0.0a2.dist-info/METADATA +216 -0
  70. interlaced-2.0.0a2.dist-info/RECORD +74 -0
  71. interlaced-2.0.0a2.dist-info/WHEEL +5 -0
  72. interlaced-2.0.0a2.dist-info/entry_points.txt +2 -0
  73. interlaced-2.0.0a2.dist-info/licenses/LICENSE +21 -0
  74. interlaced-2.0.0a2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,66 @@
1
+ """Exception hierarchy. Everything inherits from :class:`InterlaceError`.
2
+
3
+ Ported in spirit from v0.x: a single root with an optional structured ``details``
4
+ payload so errors carry machine-readable context to the CLI, API, and event log.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any
10
+
11
+
12
+ class InterlaceError(Exception):
13
+ """Base class for all Interlace errors."""
14
+
15
+ def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
16
+ super().__init__(message)
17
+ self.message = message
18
+ self.details = details or {}
19
+
20
+
21
+ class ConfigurationError(InterlaceError):
22
+ """Invalid or missing project/model configuration."""
23
+
24
+
25
+ class DefinitionError(InterlaceError):
26
+ """A model, stream, or check is declared incorrectly."""
27
+
28
+
29
+ class CompilationError(InterlaceError):
30
+ """SQL could not be parsed, qualified, or transpiled."""
31
+
32
+
33
+ class DependencyError(InterlaceError):
34
+ """The dependency graph is invalid (e.g. a cycle)."""
35
+
36
+
37
+ class SelectionError(InterlaceError):
38
+ """A model selector could not be resolved."""
39
+
40
+
41
+ class EngineError(InterlaceError):
42
+ """An engine adapter failed to execute, fetch, or load."""
43
+
44
+
45
+ class StateError(InterlaceError):
46
+ """The state store could not be read or written consistently."""
47
+
48
+
49
+ class PlanError(InterlaceError):
50
+ """A plan could not be computed or applied."""
51
+
52
+
53
+ class SchemaError(InterlaceError):
54
+ """A model's built schema violates its declared column contract."""
55
+
56
+
57
+ class StreamError(InterlaceError):
58
+ """Stream ingestion, the durable log, or a consumer failed."""
59
+
60
+
61
+ class Backpressure(StreamError):
62
+ """The durable log's bounded commit queue is full; callers should retry (HTTP 429)."""
63
+
64
+
65
+ class CheckError(InterlaceError):
66
+ """A data check failed with error severity."""
interlace/exports.py ADDED
@@ -0,0 +1,145 @@
1
+ """Sinks / exports — pushing a model's result to an external destination.
2
+
3
+ A model with an ``export`` block is a *sink*: it produces no managed table and no
4
+ environment view; instead its (resolved) query result is written to a
5
+ destination. Two families:
6
+
7
+ - **files** — ``to: parquet|csv|json`` + ``path``, via DuckDB ``COPY``.
8
+ - **tables** (reverse ETL) — ``to: table`` + ``target: <alias>.<schema>.<table>``
9
+ where ``alias`` is a database attached via the project's ``attach:`` config
10
+ (Postgres, SQLite, another DuckDB, ...). ``mode`` picks the delivery:
11
+ ``replace`` (DELETE all + INSERT — the live table is never dropped, so grants
12
+ and readers survive), ``append``, or the keyed ``merge_by_key`` /
13
+ ``full_merge`` — which reuse the *same strategy AST builders* as managed
14
+ models, pointed at the external catalog.
15
+
16
+ Exports are side-effecting — no view-swap rollback (see v2-design §6).
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from collections.abc import Sequence
22
+ from dataclasses import dataclass
23
+ from typing import Any
24
+
25
+ import sqlglot
26
+ from sqlglot import exp
27
+
28
+ from interlace.engines.base import EngineCaps
29
+ from interlace.exceptions import ConfigurationError, PlanError
30
+ from interlace.ir.relation import EngineRef, SqlRelation, TableRef
31
+ from interlace.ir.schema import empty_schema
32
+ from interlace.strategies.base import RowCounts
33
+
34
+ _FILE_FORMATS = frozenset({"parquet", "csv", "json"})
35
+ _TABLE_MODES = frozenset({"replace", "append", "merge_by_key", "full_merge"})
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class ExportConfig:
40
+ """Where a sink writes. ``to`` is the destination type; ``path`` (files) or
41
+ ``target`` + ``mode`` (+ ``key`` for keyed modes) for tables."""
42
+
43
+ to: str
44
+ path: str = ""
45
+ target: str = ""
46
+ mode: str = "replace"
47
+ key: tuple[str, ...] = ()
48
+
49
+ @classmethod
50
+ def from_dict(cls, data: Any) -> ExportConfig:
51
+ if not isinstance(data, dict) or "to" not in data:
52
+ raise ConfigurationError("export requires 'to'", details={"got": data})
53
+ to = str(data["to"])
54
+ key = data.get("key") or ()
55
+ config = cls(
56
+ to=to,
57
+ path=str(data.get("path", "")),
58
+ target=str(data.get("target", "")),
59
+ mode=str(data.get("mode", "replace")),
60
+ key=(key,) if isinstance(key, str) else tuple(key),
61
+ )
62
+ if to in _FILE_FORMATS and not config.path:
63
+ raise ConfigurationError(f"export to {to!r} requires 'path'", details={"got": data})
64
+ if to == "table":
65
+ if not config.target:
66
+ raise ConfigurationError("export to table requires 'target'", details={"got": data})
67
+ if config.mode not in _TABLE_MODES:
68
+ raise ConfigurationError(f"unknown export mode {config.mode!r}; expected one of {sorted(_TABLE_MODES)}")
69
+ if config.mode in ("merge_by_key", "full_merge") and not config.key:
70
+ raise ConfigurationError(f"export mode {config.mode!r} requires 'key'", details={"got": data})
71
+ return config
72
+
73
+
74
+ def export_target_ref(target: str) -> TableRef:
75
+ parts = target.split(".")
76
+ if len(parts) == 3:
77
+ return TableRef(catalog=parts[0], schema=parts[1], name=parts[2])
78
+ if len(parts) == 2:
79
+ return TableRef(catalog=parts[0], schema="main", name=parts[1])
80
+ raise PlanError(
81
+ f"export target {target!r} must be <alias>.<schema>.<table> (or <alias>.<table> for the main schema)"
82
+ )
83
+
84
+
85
+ def table_export_statements(
86
+ export: ExportConfig,
87
+ query: exp.Expression,
88
+ dialect: str,
89
+ engine: str = "default",
90
+ columns: Sequence[str] | None = None,
91
+ ) -> list[exp.Expression]:
92
+ """Deliver ``query`` into the external table — never DROP it (grants/readers survive).
93
+
94
+ ``columns`` names the target's column order when the source has been aligned to an
95
+ existing target (see ``plan.apply._deliver_table_export``): replace/append inserts
96
+ then carry an explicit column list. The keyed modes reuse the strategy builders,
97
+ whose inserts bind positionally — safe because the aligned projection reproduces
98
+ the target's column order exactly."""
99
+ from interlace.strategies import FullMerge, MergeByKey # runtime import: strategies build on ir like this module
100
+
101
+ target = export_target_ref(export.target)
102
+ table = target.to_expr()
103
+ relation = SqlRelation(ast=query, engine=EngineRef(name=engine, dialect=dialect), schema=empty_schema())
104
+
105
+ if export.mode == "merge_by_key":
106
+ return MergeByKey(export.key).plan_statements(relation, target, EngineCaps())
107
+ if export.mode == "full_merge":
108
+ return FullMerge(export.key).plan_statements(relation, target, EngineCaps())
109
+
110
+ derived = exp.select("*").from_(exp.Subquery(this=query.copy(), alias=exp.TableAlias(this="_s")))
111
+ ensure = exp.Create(this=table.copy(), kind="TABLE", exists=True, expression=derived.copy().limit(0))
112
+ insert_this: exp.Expression = table.copy()
113
+ if columns:
114
+ insert_this = exp.Schema(this=table.copy(), expressions=[exp.to_identifier(c) for c in columns])
115
+ insert = exp.Insert(this=insert_this, expression=query.copy())
116
+ if export.mode == "append":
117
+ return [ensure, insert]
118
+ wipe = exp.Delete(this=table.copy()) # replace: empty in place, never drop
119
+ return [ensure, wipe, insert]
120
+
121
+
122
+ def export_row_counts(export: ExportConfig, counts: Sequence[int]) -> RowCounts:
123
+ """Interpret a table delivery's per-statement counts for its mode."""
124
+ from interlace.strategies import FullMerge, MergeByKey
125
+
126
+ if export.mode == "merge_by_key":
127
+ return MergeByKey(export.key).row_counts(counts)
128
+ if export.mode == "full_merge":
129
+ return FullMerge(export.key).row_counts(counts)
130
+ if export.mode == "append": # [ensure, insert]
131
+ return RowCounts(inserted=counts[1] if len(counts) > 1 else 0)
132
+ # replace: [ensure, wipe, insert] — the wipe clears the previous delivery
133
+ return RowCounts(inserted=counts[2] if len(counts) > 2 else 0, deleted=counts[1] if len(counts) > 1 else 0)
134
+
135
+
136
+ def export_statements(
137
+ export: ExportConfig, query: exp.Expression, resolved_path: str, dialect: str
138
+ ) -> list[exp.Expression]:
139
+ """Build the statements that write ``query`` to a file export destination."""
140
+ if export.to not in _FILE_FORMATS:
141
+ raise PlanError(f"unsupported export destination: {export.to!r}", details={"to": export.to})
142
+ options = "FORMAT csv, HEADER" if export.to == "csv" else f"FORMAT {export.to}"
143
+ query_sql = query.sql(dialect=dialect)
144
+ escaped = resolved_path.replace("'", "''")
145
+ return [sqlglot.parse_one(f"COPY ({query_sql}) TO '{escaped}' ({options})", read=dialect)]
@@ -0,0 +1,17 @@
1
+ """Dependency graph, compilation, lineage, and selectors."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from interlace.graph.column_lineage import column_lineage
6
+ from interlace.graph.dag import DependencyGraph
7
+ from interlace.graph.project import CompiledModel, CompiledProject, compile_models
8
+ from interlace.graph.selectors import select_models
9
+
10
+ __all__ = [
11
+ "CompiledModel",
12
+ "CompiledProject",
13
+ "DependencyGraph",
14
+ "column_lineage",
15
+ "compile_models",
16
+ "select_models",
17
+ ]
@@ -0,0 +1,69 @@
1
+ """Column-level lineage.
2
+
3
+ For each model output column, which upstream ``(table, column)`` it derives from.
4
+ Built in topological order: each model's SQL is qualified against a schema graph
5
+ accumulated from upstream models, ``*`` and unqualified columns are resolved, and
6
+ the columns feeding each projection are extracted. Best-effort — a model whose
7
+ SQL can't be qualified (e.g. a Python model, or an unresolvable query) yields no
8
+ column lineage rather than failing the whole project.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from typing import Any
14
+
15
+ from sqlglot import exp
16
+ from sqlglot.optimizer.qualify import qualify
17
+
18
+ from interlace.graph.project import CompiledProject
19
+
20
+ ColumnSources = dict[str, list[tuple[str, str]]] # output column -> [(table, column), ...]
21
+
22
+
23
+ def _table_name(table: exp.Table) -> str:
24
+ return f"{table.db}.{table.name}" if table.db else table.name
25
+
26
+
27
+ def _insert_schema(schema: dict[str, Any], model_name: str, columns: dict[str, str]) -> None:
28
+ """Insert a model's columns into the (possibly nested) schema graph by name parts."""
29
+ *namespaces, leaf = model_name.split(".")
30
+ node = schema
31
+ for namespace in namespaces:
32
+ node = node.setdefault(namespace, {})
33
+ node[leaf] = columns
34
+
35
+
36
+ def _model_lineage(ast: exp.Expression, schema: dict[str, Any], dialect: str) -> ColumnSources | None:
37
+ try:
38
+ qualified = qualify(ast.copy(), schema=schema, dialect=dialect)
39
+ except Exception:
40
+ return None
41
+ if not isinstance(qualified, exp.Select):
42
+ return None
43
+
44
+ aliases = {table.alias_or_name: _table_name(table) for table in qualified.find_all(exp.Table)}
45
+ lineage: ColumnSources = {}
46
+ for projection in qualified.selects:
47
+ sources: list[tuple[str, str]] = []
48
+ for column in projection.find_all(exp.Column):
49
+ ref = (aliases.get(column.table, column.table), column.name)
50
+ if ref not in sources:
51
+ sources.append(ref)
52
+ lineage[projection.alias_or_name] = sources
53
+ return lineage
54
+
55
+
56
+ def column_lineage(project: CompiledProject) -> dict[str, ColumnSources]:
57
+ """Compute per-model column lineage for the whole project (topological order)."""
58
+ schema: dict[str, Any] = {}
59
+ result: dict[str, ColumnSources] = {}
60
+ for name in project.graph.topological_sort():
61
+ model = project.models[name]
62
+ if model.ast is None:
63
+ result[name] = {}
64
+ continue
65
+ lineage = _model_lineage(model.ast, schema, model.dialect)
66
+ result[name] = lineage or {}
67
+ if lineage is not None:
68
+ _insert_schema(schema, name, dict.fromkeys(lineage, "UNKNOWN"))
69
+ return result
interlace/graph/dag.py ADDED
@@ -0,0 +1,81 @@
1
+ """The dependency DAG.
2
+
3
+ A directed acyclic graph of model names with edges pointing from a node to its
4
+ upstreams. Kahn's algorithm gives a deterministic topological order (upstreams
5
+ before downstreams); a short order means a cycle. No third-party graph library —
6
+ this is ~40 lines of stdlib.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections import deque
12
+ from collections.abc import Callable
13
+
14
+ from interlace.exceptions import DependencyError
15
+
16
+
17
+ class DependencyGraph:
18
+ """Model names and their upstream dependencies."""
19
+
20
+ def __init__(self) -> None:
21
+ self._upstreams: dict[str, set[str]] = {}
22
+
23
+ def add_node(self, name: str) -> None:
24
+ self._upstreams.setdefault(name, set())
25
+
26
+ def add_dependency(self, node: str, upstream: str) -> None:
27
+ """Record that ``node`` depends on ``upstream``. Self-edges are ignored."""
28
+ self.add_node(node)
29
+ self.add_node(upstream)
30
+ if node != upstream:
31
+ self._upstreams[node].add(upstream)
32
+
33
+ def nodes(self) -> list[str]:
34
+ return list(self._upstreams)
35
+
36
+ def upstreams(self, node: str) -> set[str]:
37
+ return set(self._upstreams.get(node, set()))
38
+
39
+ def downstreams(self, node: str) -> set[str]:
40
+ return {n for n, ups in self._upstreams.items() if node in ups}
41
+
42
+ def ancestors(self, node: str) -> set[str]:
43
+ """All transitive upstreams of ``node``."""
44
+ return self._reach(node, self.upstreams)
45
+
46
+ def descendants(self, node: str) -> set[str]:
47
+ """All transitive downstreams of ``node``."""
48
+ return self._reach(node, self.downstreams)
49
+
50
+ def _reach(self, node: str, step: Callable[[str], set[str]]) -> set[str]:
51
+ result: set[str] = set()
52
+ stack = list(step(node))
53
+ while stack:
54
+ current = stack.pop()
55
+ if current not in result:
56
+ result.add(current)
57
+ stack.extend(step(current))
58
+ return result
59
+
60
+ def topological_sort(self) -> list[str]:
61
+ """Return nodes ordered upstreams-first. Raises ``DependencyError`` on a cycle."""
62
+ indegree = {n: len(ups) for n, ups in self._upstreams.items()}
63
+ adjacency: dict[str, list[str]] = {n: [] for n in self._upstreams}
64
+ for node, ups in self._upstreams.items():
65
+ for upstream in ups:
66
+ adjacency[upstream].append(node)
67
+
68
+ queue = deque(sorted(n for n, deg in indegree.items() if deg == 0))
69
+ order: list[str] = []
70
+ while queue:
71
+ node = queue.popleft()
72
+ order.append(node)
73
+ for downstream in sorted(adjacency[node]):
74
+ indegree[downstream] -= 1
75
+ if indegree[downstream] == 0:
76
+ queue.append(downstream)
77
+
78
+ if len(order) != len(self._upstreams):
79
+ cyclic = sorted(set(self._upstreams) - set(order))
80
+ raise DependencyError("dependency cycle detected", details={"nodes": cyclic})
81
+ return order
@@ -0,0 +1,242 @@
1
+ """Compile declared models into a fingerprinted dependency graph.
2
+
3
+ Takes the registry's :class:`ModelDef`s and produces a :class:`CompiledProject`:
4
+ each model parsed (SQL) or source-hashed (Python), its dependencies resolved
5
+ (explicit ``depends_on`` plus implicit table references that match a known
6
+ model), topologically ordered, and fingerprinted upstream-first. The plan step
7
+ consumes this — diffing fingerprints against the state store to classify changes
8
+ and build the snapshots it persists.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import inspect
14
+ import textwrap
15
+ from collections.abc import Iterable
16
+ from dataclasses import dataclass, field
17
+
18
+ from sqlglot import exp
19
+
20
+ from interlace.checks.spec import CheckSpec
21
+ from interlace.dsl.decorators import CheckDef, ModelDef, ModelFn
22
+ from interlace.exceptions import DefinitionError
23
+ from interlace.exports import ExportConfig
24
+ from interlace.graph.dag import DependencyGraph
25
+ from interlace.ir.canonicalize import parse, table_references
26
+ from interlace.ir.fingerprint import canonical_sql, data_fingerprint, metadata_fingerprint
27
+ from interlace.ir.relation import TableRef
28
+
29
+ _PHYSICAL_PREFIX = "interlace__"
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class CompiledModel:
34
+ """A model resolved to a fingerprint, physical table, and dependency set."""
35
+
36
+ name: str
37
+ dialect: str
38
+ engine: str # named engine that executes and stores this model
39
+ dependencies: tuple[str, ...]
40
+ fingerprint: str # full: SQL + config + upstream fingerprints
41
+ local_fingerprint: str # SQL + config only — separates direct from indirect changes
42
+ metadata_hash: str
43
+ definition_sql: str | None # canonical SQL, for change classification (None for Python models)
44
+ physical_table: TableRef
45
+ materialise: str
46
+ strategy: str
47
+ key: tuple[str, ...] # business key for keyed strategies (merge_by_key)
48
+ time_column: str | None # partition column for incremental_by_time
49
+ cursor: str | None # column whose max is injected into a Python model's `cursor` param
50
+ interval: str | None # grain for incremental_by_time (e.g. "1d")
51
+ tags: tuple[str, ...] # for tag: selection
52
+ schedule: dict[str, str] | None # cron/interval schedule for the trigger engine
53
+ columns: dict[str, str | None] | None # output contract validated at apply time
54
+ export: ExportConfig | None # presence makes this a sink (no physical table/view)
55
+ ast: exp.Expression | None # parsed SQL, or None for Python models
56
+ owner: str | None = None # surfaced in the catalog/API (metadata, not fingerprinted into data)
57
+ description: str | None = None
58
+ fn: ModelFn | None = None # the Python model function (source is fingerprinted; None for SQL)
59
+ checks: tuple[CheckSpec, ...] = () # metadata-fingerprinted: changing a check never rebuilds data
60
+
61
+
62
+ @dataclass
63
+ class CompiledProject:
64
+ """All compiled models plus the dependency graph that orders them."""
65
+
66
+ models: dict[str, CompiledModel]
67
+ graph: DependencyGraph
68
+ python_checks: dict[str, tuple[CheckDef, ...]] = field(default_factory=dict) # @check fns by model
69
+
70
+ def ordered(self) -> list[CompiledModel]:
71
+ return [self.models[name] for name in self.graph.topological_sort()]
72
+
73
+
74
+ def _split_name(name: str) -> tuple[str, str]:
75
+ schema, _, base = name.rpartition(".")
76
+ return (schema or "main"), base
77
+
78
+
79
+ def _physical_table(name: str, fingerprint: str, catalog: str | None) -> TableRef:
80
+ schema, base = _split_name(name)
81
+ return TableRef(schema=f"{_PHYSICAL_PREFIX}{schema}", name=f"{base}__{fingerprint}", catalog=catalog)
82
+
83
+
84
+ def _resolve_dependencies(
85
+ model: ModelDef, names: set[str], default_dialect: str
86
+ ) -> tuple[tuple[str, ...], exp.Expression | None, str]:
87
+ dialect = model.dialect or default_dialect
88
+ deps: list[str] = []
89
+ seen: set[str] = set()
90
+
91
+ def add(candidate: str) -> None:
92
+ if candidate != model.name and candidate in names and candidate not in seen:
93
+ seen.add(candidate)
94
+ deps.append(candidate)
95
+
96
+ for explicit in model.depends_on:
97
+ add(explicit)
98
+
99
+ ast: exp.Expression | None = None
100
+ if model.sql is not None:
101
+ ast = parse(model.sql, dialect)
102
+ for ref in table_references(ast):
103
+ if ref in names:
104
+ add(ref)
105
+ else:
106
+ add(ref.rsplit(".", 1)[-1]) # match a qualified ref to a model by its tail
107
+
108
+ return tuple(deps), ast, dialect
109
+
110
+
111
+ def _fingerprint_query(model: ModelDef, ast: exp.Expression | None) -> str | exp.Expression:
112
+ if ast is not None:
113
+ return ast
114
+ if model.fn is not None:
115
+ return textwrap.dedent(inspect.getsource(model.fn))
116
+ raise DefinitionError(f"model {model.name!r} has neither SQL nor a function body")
117
+
118
+
119
+ def compile_models(
120
+ models: Iterable[ModelDef],
121
+ *,
122
+ default_dialect: str = "duckdb",
123
+ default_engine: str = "default",
124
+ engine_dialects: dict[str, str] | None = None,
125
+ known_engines: set[str] | None = None,
126
+ catalog: str | None = None,
127
+ checks: Iterable[CheckDef] = (),
128
+ ) -> CompiledProject:
129
+ """Compile models into a fingerprinted, topologically-ordered project.
130
+
131
+ ``checks`` are ``@check``-decorated Python functions, attached by model name.
132
+ ``engine_dialects`` maps engine name → sqlglot dialect (used when a model
133
+ omits ``dialect``). ``known_engines`` validates model ``engine`` pins.
134
+ """
135
+ definitions = {m.name: m for m in models}
136
+ names = set(definitions)
137
+ dialects_by_engine = engine_dialects or {}
138
+ engines = known_engines
139
+
140
+ resolved: dict[str, tuple[tuple[str, ...], exp.Expression | None, str, str]] = {}
141
+ graph = DependencyGraph()
142
+ for name, definition in definitions.items():
143
+ engine = definition.engine or default_engine
144
+ if engines is not None and engine not in engines:
145
+ raise DefinitionError(
146
+ f"model {name!r} references unknown engine {engine!r}",
147
+ details={"engines": sorted(engines)},
148
+ )
149
+ # Authoring dialect: explicit model dialect, else the engine's, else project default.
150
+ model_default_dialect = dialects_by_engine.get(engine, default_dialect)
151
+ deps, ast, dialect = _resolve_dependencies(definition, names, model_default_dialect)
152
+ resolved[name] = (deps, ast, dialect, engine)
153
+ graph.add_node(name)
154
+ for dep in deps:
155
+ graph.add_dependency(name, dep)
156
+
157
+ compiled: dict[str, CompiledModel] = {}
158
+ for name in graph.topological_sort(): # upstreams first; raises on cycle
159
+ definition = definitions[name]
160
+ deps, ast, dialect, engine = resolved[name]
161
+ for dep in deps: # topo order: deps already compiled
162
+ if compiled[dep].engine != engine and compiled[dep].materialise == "ephemeral":
163
+ raise DefinitionError(
164
+ f"model {name!r} on engine {engine!r} inlines ephemeral {dep!r} declared on engine "
165
+ f"{compiled[dep].engine!r}; an ephemeral model must share its consumers' engine "
166
+ f"(see docs/architecture/MULTI_ENGINE.md)"
167
+ )
168
+ strategy_config = {
169
+ "materialise": definition.materialise,
170
+ "strategy": definition.strategy,
171
+ "key": list(definition.key),
172
+ "interval": definition.interval,
173
+ "time_column": definition.time_column,
174
+ "cursor": definition.cursor,
175
+ "engine": engine,
176
+ "export": (
177
+ {
178
+ "to": definition.export.to,
179
+ "path": definition.export.path,
180
+ "target": definition.export.target,
181
+ "mode": definition.export.mode,
182
+ "key": list(definition.export.key),
183
+ }
184
+ if definition.export
185
+ else None
186
+ ),
187
+ "dialect": dialect,
188
+ }
189
+ query = _fingerprint_query(definition, ast)
190
+ # render the canonical SQL once: it feeds both fingerprints and definition_sql
191
+ canonical = canonical_sql(query) if isinstance(query, exp.Expression) else query
192
+ local_fingerprint = data_fingerprint(query=canonical, strategy_config=strategy_config, upstream_fingerprints=[])
193
+ fingerprint = data_fingerprint(
194
+ query=canonical,
195
+ strategy_config=strategy_config,
196
+ upstream_fingerprints=[compiled[dep].fingerprint for dep in deps],
197
+ )
198
+ metadata_hash = metadata_fingerprint(
199
+ {
200
+ "owner": definition.owner,
201
+ "tags": list(definition.tags),
202
+ "description": definition.description,
203
+ "checks": [
204
+ {"type": c.type, "columns": list(c.columns), "severity": c.severity, "params": c.params}
205
+ for c in definition.checks
206
+ ],
207
+ }
208
+ )
209
+ compiled[name] = CompiledModel(
210
+ name=name,
211
+ dialect=dialect,
212
+ engine=engine,
213
+ dependencies=deps,
214
+ fingerprint=fingerprint,
215
+ local_fingerprint=local_fingerprint,
216
+ metadata_hash=metadata_hash,
217
+ definition_sql=canonical if ast is not None else None,
218
+ physical_table=_physical_table(name, fingerprint, catalog),
219
+ materialise=definition.materialise,
220
+ strategy=definition.strategy,
221
+ key=definition.key,
222
+ time_column=definition.time_column,
223
+ cursor=definition.cursor,
224
+ interval=definition.interval,
225
+ tags=definition.tags,
226
+ schedule=definition.schedule,
227
+ columns=definition.columns,
228
+ export=definition.export,
229
+ ast=ast,
230
+ owner=definition.owner,
231
+ description=definition.description,
232
+ fn=definition.fn,
233
+ checks=definition.checks,
234
+ )
235
+
236
+ python_checks: dict[str, tuple[CheckDef, ...]] = {}
237
+ for check in checks:
238
+ if check.model not in compiled:
239
+ raise DefinitionError(f"@check {check.name!r} references unknown model {check.model!r}")
240
+ python_checks[check.model] = (*python_checks.get(check.model, ()), check)
241
+
242
+ return CompiledProject(models=compiled, graph=graph, python_checks=python_checks)
@@ -0,0 +1,45 @@
1
+ """dbt-style model selection.
2
+
3
+ A selector is one of: ``model`` (exact), ``+model`` (model + ancestors),
4
+ ``model+`` (model + descendants), ``+model+`` (both), or ``tag:<name>``. Multiple
5
+ selectors (space- or comma-separated, or repeated) union together. An empty
6
+ selector list means "all models".
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from interlace.exceptions import SelectionError
12
+ from interlace.graph.project import CompiledProject
13
+
14
+
15
+ def select_models(selectors: list[str], project: CompiledProject) -> set[str]:
16
+ if not selectors:
17
+ return set(project.models)
18
+ chosen: set[str] = set()
19
+ for raw in selectors:
20
+ for token in _tokens(raw):
21
+ chosen |= _resolve(token, project)
22
+ return chosen
23
+
24
+
25
+ def _tokens(raw: str) -> list[str]:
26
+ return [token for token in raw.replace(",", " ").split() if token]
27
+
28
+
29
+ def _resolve(token: str, project: CompiledProject) -> set[str]:
30
+ if token.startswith("tag:"):
31
+ tag = token[4:]
32
+ return {name for name, model in project.models.items() if tag in model.tags}
33
+
34
+ include_ancestors = token.startswith("+")
35
+ include_descendants = token.endswith("+")
36
+ name = token.strip("+")
37
+ if name not in project.models:
38
+ raise SelectionError(f"unknown model in selector: {name!r}", details={"selector": token})
39
+
40
+ chosen = {name}
41
+ if include_ancestors:
42
+ chosen |= project.graph.ancestors(name)
43
+ if include_descendants:
44
+ chosen |= project.graph.descendants(name)
45
+ return chosen