interlaced 2.0.0a2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interlace/__init__.py +26 -0
- interlace/checks/__init__.py +10 -0
- interlace/checks/builtin.py +152 -0
- interlace/checks/runner.py +103 -0
- interlace/checks/spec.py +113 -0
- interlace/cli/__init__.py +7 -0
- interlace/cli/main.py +1072 -0
- interlace/config/__init__.py +7 -0
- interlace/config/config.py +172 -0
- interlace/contracts.py +35 -0
- interlace/dsl/__init__.py +16 -0
- interlace/dsl/decorators.py +222 -0
- interlace/dsl/discovery.py +75 -0
- interlace/dsl/sql_config.py +47 -0
- interlace/engines/__init__.py +16 -0
- interlace/engines/base.py +86 -0
- interlace/engines/duckdb.py +249 -0
- interlace/engines/postgres.py +157 -0
- interlace/engines/quack.py +113 -0
- interlace/engines/registry.py +125 -0
- interlace/exceptions.py +66 -0
- interlace/exports.py +145 -0
- interlace/graph/__init__.py +17 -0
- interlace/graph/column_lineage.py +69 -0
- interlace/graph/dag.py +81 -0
- interlace/graph/project.py +242 -0
- interlace/graph/selectors.py +45 -0
- interlace/ir/__init__.py +24 -0
- interlace/ir/canonicalize.py +71 -0
- interlace/ir/fingerprint.py +55 -0
- interlace/ir/relation.py +71 -0
- interlace/ir/schema.py +23 -0
- interlace/plan/__init__.py +23 -0
- interlace/plan/apply.py +527 -0
- interlace/plan/differ.py +211 -0
- interlace/plan/plan.py +177 -0
- interlace/plan/resolve.py +74 -0
- interlace/plan/run.py +79 -0
- interlace/project.py +278 -0
- interlace/py.typed +0 -0
- interlace/runtime/__init__.py +6 -0
- interlace/runtime/handles.py +48 -0
- interlace/runtime/python_model.py +175 -0
- interlace/scaffold.py +83 -0
- interlace/scheduler/__init__.py +17 -0
- interlace/scheduler/engine.py +66 -0
- interlace/scheduler/triggers.py +84 -0
- interlace/scheduler/worker.py +154 -0
- interlace/service/__init__.py +7 -0
- interlace/service/app.py +836 -0
- interlace/service/auth.py +41 -0
- interlace/state/__init__.py +18 -0
- interlace/state/interval.py +134 -0
- interlace/state/janitor.py +142 -0
- interlace/state/snapshot.py +47 -0
- interlace/state/store.py +906 -0
- interlace/strategies/__init__.py +64 -0
- interlace/strategies/base.py +71 -0
- interlace/strategies/full.py +38 -0
- interlace/strategies/full_merge.py +98 -0
- interlace/strategies/incremental_by_time.py +65 -0
- interlace/strategies/merge_by_key.py +69 -0
- interlace/strategies/scd_type_2.py +122 -0
- interlace/strategies/view.py +33 -0
- interlace/streaming/__init__.py +18 -0
- interlace/streaming/log.py +298 -0
- interlace/streaming/materializer.py +166 -0
- interlace/streaming/schema.py +218 -0
- interlaced-2.0.0a2.dist-info/METADATA +216 -0
- interlaced-2.0.0a2.dist-info/RECORD +74 -0
- interlaced-2.0.0a2.dist-info/WHEEL +5 -0
- interlaced-2.0.0a2.dist-info/entry_points.txt +2 -0
- interlaced-2.0.0a2.dist-info/licenses/LICENSE +21 -0
- interlaced-2.0.0a2.dist-info/top_level.txt +1 -0
interlace/exceptions.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Exception hierarchy. Everything inherits from :class:`InterlaceError`.
|
|
2
|
+
|
|
3
|
+
Ported in spirit from v0.x: a single root with an optional structured ``details``
|
|
4
|
+
payload so errors carry machine-readable context to the CLI, API, and event log.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class InterlaceError(Exception):
|
|
13
|
+
"""Base class for all Interlace errors."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, message: str, details: dict[str, Any] | None = None) -> None:
|
|
16
|
+
super().__init__(message)
|
|
17
|
+
self.message = message
|
|
18
|
+
self.details = details or {}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ConfigurationError(InterlaceError):
|
|
22
|
+
"""Invalid or missing project/model configuration."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DefinitionError(InterlaceError):
|
|
26
|
+
"""A model, stream, or check is declared incorrectly."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class CompilationError(InterlaceError):
|
|
30
|
+
"""SQL could not be parsed, qualified, or transpiled."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class DependencyError(InterlaceError):
|
|
34
|
+
"""The dependency graph is invalid (e.g. a cycle)."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SelectionError(InterlaceError):
|
|
38
|
+
"""A model selector could not be resolved."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class EngineError(InterlaceError):
|
|
42
|
+
"""An engine adapter failed to execute, fetch, or load."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class StateError(InterlaceError):
|
|
46
|
+
"""The state store could not be read or written consistently."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class PlanError(InterlaceError):
|
|
50
|
+
"""A plan could not be computed or applied."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class SchemaError(InterlaceError):
|
|
54
|
+
"""A model's built schema violates its declared column contract."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class StreamError(InterlaceError):
|
|
58
|
+
"""Stream ingestion, the durable log, or a consumer failed."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Backpressure(StreamError):
|
|
62
|
+
"""The durable log's bounded commit queue is full; callers should retry (HTTP 429)."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class CheckError(InterlaceError):
|
|
66
|
+
"""A data check failed with error severity."""
|
interlace/exports.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Sinks / exports — pushing a model's result to an external destination.
|
|
2
|
+
|
|
3
|
+
A model with an ``export`` block is a *sink*: it produces no managed table and no
|
|
4
|
+
environment view; instead its (resolved) query result is written to a
|
|
5
|
+
destination. Two families:
|
|
6
|
+
|
|
7
|
+
- **files** — ``to: parquet|csv|json`` + ``path``, via DuckDB ``COPY``.
|
|
8
|
+
- **tables** (reverse ETL) — ``to: table`` + ``target: <alias>.<schema>.<table>``
|
|
9
|
+
where ``alias`` is a database attached via the project's ``attach:`` config
|
|
10
|
+
(Postgres, SQLite, another DuckDB, ...). ``mode`` picks the delivery:
|
|
11
|
+
``replace`` (DELETE all + INSERT — the live table is never dropped, so grants
|
|
12
|
+
and readers survive), ``append``, or the keyed ``merge_by_key`` /
|
|
13
|
+
``full_merge`` — which reuse the *same strategy AST builders* as managed
|
|
14
|
+
models, pointed at the external catalog.
|
|
15
|
+
|
|
16
|
+
Exports are side-effecting — no view-swap rollback (see v2-design §6).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from collections.abc import Sequence
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
import sqlglot
|
|
26
|
+
from sqlglot import exp
|
|
27
|
+
|
|
28
|
+
from interlace.engines.base import EngineCaps
|
|
29
|
+
from interlace.exceptions import ConfigurationError, PlanError
|
|
30
|
+
from interlace.ir.relation import EngineRef, SqlRelation, TableRef
|
|
31
|
+
from interlace.ir.schema import empty_schema
|
|
32
|
+
from interlace.strategies.base import RowCounts
|
|
33
|
+
|
|
34
|
+
_FILE_FORMATS = frozenset({"parquet", "csv", "json"})
|
|
35
|
+
_TABLE_MODES = frozenset({"replace", "append", "merge_by_key", "full_merge"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class ExportConfig:
|
|
40
|
+
"""Where a sink writes. ``to`` is the destination type; ``path`` (files) or
|
|
41
|
+
``target`` + ``mode`` (+ ``key`` for keyed modes) for tables."""
|
|
42
|
+
|
|
43
|
+
to: str
|
|
44
|
+
path: str = ""
|
|
45
|
+
target: str = ""
|
|
46
|
+
mode: str = "replace"
|
|
47
|
+
key: tuple[str, ...] = ()
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def from_dict(cls, data: Any) -> ExportConfig:
|
|
51
|
+
if not isinstance(data, dict) or "to" not in data:
|
|
52
|
+
raise ConfigurationError("export requires 'to'", details={"got": data})
|
|
53
|
+
to = str(data["to"])
|
|
54
|
+
key = data.get("key") or ()
|
|
55
|
+
config = cls(
|
|
56
|
+
to=to,
|
|
57
|
+
path=str(data.get("path", "")),
|
|
58
|
+
target=str(data.get("target", "")),
|
|
59
|
+
mode=str(data.get("mode", "replace")),
|
|
60
|
+
key=(key,) if isinstance(key, str) else tuple(key),
|
|
61
|
+
)
|
|
62
|
+
if to in _FILE_FORMATS and not config.path:
|
|
63
|
+
raise ConfigurationError(f"export to {to!r} requires 'path'", details={"got": data})
|
|
64
|
+
if to == "table":
|
|
65
|
+
if not config.target:
|
|
66
|
+
raise ConfigurationError("export to table requires 'target'", details={"got": data})
|
|
67
|
+
if config.mode not in _TABLE_MODES:
|
|
68
|
+
raise ConfigurationError(f"unknown export mode {config.mode!r}; expected one of {sorted(_TABLE_MODES)}")
|
|
69
|
+
if config.mode in ("merge_by_key", "full_merge") and not config.key:
|
|
70
|
+
raise ConfigurationError(f"export mode {config.mode!r} requires 'key'", details={"got": data})
|
|
71
|
+
return config
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def export_target_ref(target: str) -> TableRef:
|
|
75
|
+
parts = target.split(".")
|
|
76
|
+
if len(parts) == 3:
|
|
77
|
+
return TableRef(catalog=parts[0], schema=parts[1], name=parts[2])
|
|
78
|
+
if len(parts) == 2:
|
|
79
|
+
return TableRef(catalog=parts[0], schema="main", name=parts[1])
|
|
80
|
+
raise PlanError(
|
|
81
|
+
f"export target {target!r} must be <alias>.<schema>.<table> (or <alias>.<table> for the main schema)"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def table_export_statements(
|
|
86
|
+
export: ExportConfig,
|
|
87
|
+
query: exp.Expression,
|
|
88
|
+
dialect: str,
|
|
89
|
+
engine: str = "default",
|
|
90
|
+
columns: Sequence[str] | None = None,
|
|
91
|
+
) -> list[exp.Expression]:
|
|
92
|
+
"""Deliver ``query`` into the external table — never DROP it (grants/readers survive).
|
|
93
|
+
|
|
94
|
+
``columns`` names the target's column order when the source has been aligned to an
|
|
95
|
+
existing target (see ``plan.apply._deliver_table_export``): replace/append inserts
|
|
96
|
+
then carry an explicit column list. The keyed modes reuse the strategy builders,
|
|
97
|
+
whose inserts bind positionally — safe because the aligned projection reproduces
|
|
98
|
+
the target's column order exactly."""
|
|
99
|
+
from interlace.strategies import FullMerge, MergeByKey # runtime import: strategies build on ir like this module
|
|
100
|
+
|
|
101
|
+
target = export_target_ref(export.target)
|
|
102
|
+
table = target.to_expr()
|
|
103
|
+
relation = SqlRelation(ast=query, engine=EngineRef(name=engine, dialect=dialect), schema=empty_schema())
|
|
104
|
+
|
|
105
|
+
if export.mode == "merge_by_key":
|
|
106
|
+
return MergeByKey(export.key).plan_statements(relation, target, EngineCaps())
|
|
107
|
+
if export.mode == "full_merge":
|
|
108
|
+
return FullMerge(export.key).plan_statements(relation, target, EngineCaps())
|
|
109
|
+
|
|
110
|
+
derived = exp.select("*").from_(exp.Subquery(this=query.copy(), alias=exp.TableAlias(this="_s")))
|
|
111
|
+
ensure = exp.Create(this=table.copy(), kind="TABLE", exists=True, expression=derived.copy().limit(0))
|
|
112
|
+
insert_this: exp.Expression = table.copy()
|
|
113
|
+
if columns:
|
|
114
|
+
insert_this = exp.Schema(this=table.copy(), expressions=[exp.to_identifier(c) for c in columns])
|
|
115
|
+
insert = exp.Insert(this=insert_this, expression=query.copy())
|
|
116
|
+
if export.mode == "append":
|
|
117
|
+
return [ensure, insert]
|
|
118
|
+
wipe = exp.Delete(this=table.copy()) # replace: empty in place, never drop
|
|
119
|
+
return [ensure, wipe, insert]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def export_row_counts(export: ExportConfig, counts: Sequence[int]) -> RowCounts:
|
|
123
|
+
"""Interpret a table delivery's per-statement counts for its mode."""
|
|
124
|
+
from interlace.strategies import FullMerge, MergeByKey
|
|
125
|
+
|
|
126
|
+
if export.mode == "merge_by_key":
|
|
127
|
+
return MergeByKey(export.key).row_counts(counts)
|
|
128
|
+
if export.mode == "full_merge":
|
|
129
|
+
return FullMerge(export.key).row_counts(counts)
|
|
130
|
+
if export.mode == "append": # [ensure, insert]
|
|
131
|
+
return RowCounts(inserted=counts[1] if len(counts) > 1 else 0)
|
|
132
|
+
# replace: [ensure, wipe, insert] — the wipe clears the previous delivery
|
|
133
|
+
return RowCounts(inserted=counts[2] if len(counts) > 2 else 0, deleted=counts[1] if len(counts) > 1 else 0)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def export_statements(
|
|
137
|
+
export: ExportConfig, query: exp.Expression, resolved_path: str, dialect: str
|
|
138
|
+
) -> list[exp.Expression]:
|
|
139
|
+
"""Build the statements that write ``query`` to a file export destination."""
|
|
140
|
+
if export.to not in _FILE_FORMATS:
|
|
141
|
+
raise PlanError(f"unsupported export destination: {export.to!r}", details={"to": export.to})
|
|
142
|
+
options = "FORMAT csv, HEADER" if export.to == "csv" else f"FORMAT {export.to}"
|
|
143
|
+
query_sql = query.sql(dialect=dialect)
|
|
144
|
+
escaped = resolved_path.replace("'", "''")
|
|
145
|
+
return [sqlglot.parse_one(f"COPY ({query_sql}) TO '{escaped}' ({options})", read=dialect)]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Dependency graph, compilation, lineage, and selectors."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from interlace.graph.column_lineage import column_lineage
|
|
6
|
+
from interlace.graph.dag import DependencyGraph
|
|
7
|
+
from interlace.graph.project import CompiledModel, CompiledProject, compile_models
|
|
8
|
+
from interlace.graph.selectors import select_models
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"CompiledModel",
|
|
12
|
+
"CompiledProject",
|
|
13
|
+
"DependencyGraph",
|
|
14
|
+
"column_lineage",
|
|
15
|
+
"compile_models",
|
|
16
|
+
"select_models",
|
|
17
|
+
]
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Column-level lineage.
|
|
2
|
+
|
|
3
|
+
For each model output column, which upstream ``(table, column)`` it derives from.
|
|
4
|
+
Built in topological order: each model's SQL is qualified against a schema graph
|
|
5
|
+
accumulated from upstream models, ``*`` and unqualified columns are resolved, and
|
|
6
|
+
the columns feeding each projection are extracted. Best-effort — a model whose
|
|
7
|
+
SQL can't be qualified (e.g. a Python model, or an unresolvable query) yields no
|
|
8
|
+
column lineage rather than failing the whole project.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from sqlglot import exp
|
|
16
|
+
from sqlglot.optimizer.qualify import qualify
|
|
17
|
+
|
|
18
|
+
from interlace.graph.project import CompiledProject
|
|
19
|
+
|
|
20
|
+
ColumnSources = dict[str, list[tuple[str, str]]] # output column -> [(table, column), ...]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _table_name(table: exp.Table) -> str:
|
|
24
|
+
return f"{table.db}.{table.name}" if table.db else table.name
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _insert_schema(schema: dict[str, Any], model_name: str, columns: dict[str, str]) -> None:
|
|
28
|
+
"""Insert a model's columns into the (possibly nested) schema graph by name parts."""
|
|
29
|
+
*namespaces, leaf = model_name.split(".")
|
|
30
|
+
node = schema
|
|
31
|
+
for namespace in namespaces:
|
|
32
|
+
node = node.setdefault(namespace, {})
|
|
33
|
+
node[leaf] = columns
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _model_lineage(ast: exp.Expression, schema: dict[str, Any], dialect: str) -> ColumnSources | None:
|
|
37
|
+
try:
|
|
38
|
+
qualified = qualify(ast.copy(), schema=schema, dialect=dialect)
|
|
39
|
+
except Exception:
|
|
40
|
+
return None
|
|
41
|
+
if not isinstance(qualified, exp.Select):
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
aliases = {table.alias_or_name: _table_name(table) for table in qualified.find_all(exp.Table)}
|
|
45
|
+
lineage: ColumnSources = {}
|
|
46
|
+
for projection in qualified.selects:
|
|
47
|
+
sources: list[tuple[str, str]] = []
|
|
48
|
+
for column in projection.find_all(exp.Column):
|
|
49
|
+
ref = (aliases.get(column.table, column.table), column.name)
|
|
50
|
+
if ref not in sources:
|
|
51
|
+
sources.append(ref)
|
|
52
|
+
lineage[projection.alias_or_name] = sources
|
|
53
|
+
return lineage
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def column_lineage(project: CompiledProject) -> dict[str, ColumnSources]:
|
|
57
|
+
"""Compute per-model column lineage for the whole project (topological order)."""
|
|
58
|
+
schema: dict[str, Any] = {}
|
|
59
|
+
result: dict[str, ColumnSources] = {}
|
|
60
|
+
for name in project.graph.topological_sort():
|
|
61
|
+
model = project.models[name]
|
|
62
|
+
if model.ast is None:
|
|
63
|
+
result[name] = {}
|
|
64
|
+
continue
|
|
65
|
+
lineage = _model_lineage(model.ast, schema, model.dialect)
|
|
66
|
+
result[name] = lineage or {}
|
|
67
|
+
if lineage is not None:
|
|
68
|
+
_insert_schema(schema, name, dict.fromkeys(lineage, "UNKNOWN"))
|
|
69
|
+
return result
|
interlace/graph/dag.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""The dependency DAG.
|
|
2
|
+
|
|
3
|
+
A directed acyclic graph of model names with edges pointing from a node to its
|
|
4
|
+
upstreams. Kahn's algorithm gives a deterministic topological order (upstreams
|
|
5
|
+
before downstreams); a short order means a cycle. No third-party graph library —
|
|
6
|
+
this is ~40 lines of stdlib.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections import deque
|
|
12
|
+
from collections.abc import Callable
|
|
13
|
+
|
|
14
|
+
from interlace.exceptions import DependencyError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class DependencyGraph:
|
|
18
|
+
"""Model names and their upstream dependencies."""
|
|
19
|
+
|
|
20
|
+
def __init__(self) -> None:
|
|
21
|
+
self._upstreams: dict[str, set[str]] = {}
|
|
22
|
+
|
|
23
|
+
def add_node(self, name: str) -> None:
|
|
24
|
+
self._upstreams.setdefault(name, set())
|
|
25
|
+
|
|
26
|
+
def add_dependency(self, node: str, upstream: str) -> None:
|
|
27
|
+
"""Record that ``node`` depends on ``upstream``. Self-edges are ignored."""
|
|
28
|
+
self.add_node(node)
|
|
29
|
+
self.add_node(upstream)
|
|
30
|
+
if node != upstream:
|
|
31
|
+
self._upstreams[node].add(upstream)
|
|
32
|
+
|
|
33
|
+
def nodes(self) -> list[str]:
|
|
34
|
+
return list(self._upstreams)
|
|
35
|
+
|
|
36
|
+
def upstreams(self, node: str) -> set[str]:
|
|
37
|
+
return set(self._upstreams.get(node, set()))
|
|
38
|
+
|
|
39
|
+
def downstreams(self, node: str) -> set[str]:
|
|
40
|
+
return {n for n, ups in self._upstreams.items() if node in ups}
|
|
41
|
+
|
|
42
|
+
def ancestors(self, node: str) -> set[str]:
|
|
43
|
+
"""All transitive upstreams of ``node``."""
|
|
44
|
+
return self._reach(node, self.upstreams)
|
|
45
|
+
|
|
46
|
+
def descendants(self, node: str) -> set[str]:
|
|
47
|
+
"""All transitive downstreams of ``node``."""
|
|
48
|
+
return self._reach(node, self.downstreams)
|
|
49
|
+
|
|
50
|
+
def _reach(self, node: str, step: Callable[[str], set[str]]) -> set[str]:
|
|
51
|
+
result: set[str] = set()
|
|
52
|
+
stack = list(step(node))
|
|
53
|
+
while stack:
|
|
54
|
+
current = stack.pop()
|
|
55
|
+
if current not in result:
|
|
56
|
+
result.add(current)
|
|
57
|
+
stack.extend(step(current))
|
|
58
|
+
return result
|
|
59
|
+
|
|
60
|
+
def topological_sort(self) -> list[str]:
|
|
61
|
+
"""Return nodes ordered upstreams-first. Raises ``DependencyError`` on a cycle."""
|
|
62
|
+
indegree = {n: len(ups) for n, ups in self._upstreams.items()}
|
|
63
|
+
adjacency: dict[str, list[str]] = {n: [] for n in self._upstreams}
|
|
64
|
+
for node, ups in self._upstreams.items():
|
|
65
|
+
for upstream in ups:
|
|
66
|
+
adjacency[upstream].append(node)
|
|
67
|
+
|
|
68
|
+
queue = deque(sorted(n for n, deg in indegree.items() if deg == 0))
|
|
69
|
+
order: list[str] = []
|
|
70
|
+
while queue:
|
|
71
|
+
node = queue.popleft()
|
|
72
|
+
order.append(node)
|
|
73
|
+
for downstream in sorted(adjacency[node]):
|
|
74
|
+
indegree[downstream] -= 1
|
|
75
|
+
if indegree[downstream] == 0:
|
|
76
|
+
queue.append(downstream)
|
|
77
|
+
|
|
78
|
+
if len(order) != len(self._upstreams):
|
|
79
|
+
cyclic = sorted(set(self._upstreams) - set(order))
|
|
80
|
+
raise DependencyError("dependency cycle detected", details={"nodes": cyclic})
|
|
81
|
+
return order
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Compile declared models into a fingerprinted dependency graph.
|
|
2
|
+
|
|
3
|
+
Takes the registry's :class:`ModelDef`s and produces a :class:`CompiledProject`:
|
|
4
|
+
each model parsed (SQL) or source-hashed (Python), its dependencies resolved
|
|
5
|
+
(explicit ``depends_on`` plus implicit table references that match a known
|
|
6
|
+
model), topologically ordered, and fingerprinted upstream-first. The plan step
|
|
7
|
+
consumes this — diffing fingerprints against the state store to classify changes
|
|
8
|
+
and build the snapshots it persists.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import inspect
|
|
14
|
+
import textwrap
|
|
15
|
+
from collections.abc import Iterable
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
|
|
18
|
+
from sqlglot import exp
|
|
19
|
+
|
|
20
|
+
from interlace.checks.spec import CheckSpec
|
|
21
|
+
from interlace.dsl.decorators import CheckDef, ModelDef, ModelFn
|
|
22
|
+
from interlace.exceptions import DefinitionError
|
|
23
|
+
from interlace.exports import ExportConfig
|
|
24
|
+
from interlace.graph.dag import DependencyGraph
|
|
25
|
+
from interlace.ir.canonicalize import parse, table_references
|
|
26
|
+
from interlace.ir.fingerprint import canonical_sql, data_fingerprint, metadata_fingerprint
|
|
27
|
+
from interlace.ir.relation import TableRef
|
|
28
|
+
|
|
29
|
+
_PHYSICAL_PREFIX = "interlace__"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class CompiledModel:
|
|
34
|
+
"""A model resolved to a fingerprint, physical table, and dependency set."""
|
|
35
|
+
|
|
36
|
+
name: str
|
|
37
|
+
dialect: str
|
|
38
|
+
engine: str # named engine that executes and stores this model
|
|
39
|
+
dependencies: tuple[str, ...]
|
|
40
|
+
fingerprint: str # full: SQL + config + upstream fingerprints
|
|
41
|
+
local_fingerprint: str # SQL + config only — separates direct from indirect changes
|
|
42
|
+
metadata_hash: str
|
|
43
|
+
definition_sql: str | None # canonical SQL, for change classification (None for Python models)
|
|
44
|
+
physical_table: TableRef
|
|
45
|
+
materialise: str
|
|
46
|
+
strategy: str
|
|
47
|
+
key: tuple[str, ...] # business key for keyed strategies (merge_by_key)
|
|
48
|
+
time_column: str | None # partition column for incremental_by_time
|
|
49
|
+
cursor: str | None # column whose max is injected into a Python model's `cursor` param
|
|
50
|
+
interval: str | None # grain for incremental_by_time (e.g. "1d")
|
|
51
|
+
tags: tuple[str, ...] # for tag: selection
|
|
52
|
+
schedule: dict[str, str] | None # cron/interval schedule for the trigger engine
|
|
53
|
+
columns: dict[str, str | None] | None # output contract validated at apply time
|
|
54
|
+
export: ExportConfig | None # presence makes this a sink (no physical table/view)
|
|
55
|
+
ast: exp.Expression | None # parsed SQL, or None for Python models
|
|
56
|
+
owner: str | None = None # surfaced in the catalog/API (metadata, not fingerprinted into data)
|
|
57
|
+
description: str | None = None
|
|
58
|
+
fn: ModelFn | None = None # the Python model function (source is fingerprinted; None for SQL)
|
|
59
|
+
checks: tuple[CheckSpec, ...] = () # metadata-fingerprinted: changing a check never rebuilds data
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class CompiledProject:
|
|
64
|
+
"""All compiled models plus the dependency graph that orders them."""
|
|
65
|
+
|
|
66
|
+
models: dict[str, CompiledModel]
|
|
67
|
+
graph: DependencyGraph
|
|
68
|
+
python_checks: dict[str, tuple[CheckDef, ...]] = field(default_factory=dict) # @check fns by model
|
|
69
|
+
|
|
70
|
+
def ordered(self) -> list[CompiledModel]:
|
|
71
|
+
return [self.models[name] for name in self.graph.topological_sort()]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _split_name(name: str) -> tuple[str, str]:
|
|
75
|
+
schema, _, base = name.rpartition(".")
|
|
76
|
+
return (schema or "main"), base
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _physical_table(name: str, fingerprint: str, catalog: str | None) -> TableRef:
|
|
80
|
+
schema, base = _split_name(name)
|
|
81
|
+
return TableRef(schema=f"{_PHYSICAL_PREFIX}{schema}", name=f"{base}__{fingerprint}", catalog=catalog)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _resolve_dependencies(
|
|
85
|
+
model: ModelDef, names: set[str], default_dialect: str
|
|
86
|
+
) -> tuple[tuple[str, ...], exp.Expression | None, str]:
|
|
87
|
+
dialect = model.dialect or default_dialect
|
|
88
|
+
deps: list[str] = []
|
|
89
|
+
seen: set[str] = set()
|
|
90
|
+
|
|
91
|
+
def add(candidate: str) -> None:
|
|
92
|
+
if candidate != model.name and candidate in names and candidate not in seen:
|
|
93
|
+
seen.add(candidate)
|
|
94
|
+
deps.append(candidate)
|
|
95
|
+
|
|
96
|
+
for explicit in model.depends_on:
|
|
97
|
+
add(explicit)
|
|
98
|
+
|
|
99
|
+
ast: exp.Expression | None = None
|
|
100
|
+
if model.sql is not None:
|
|
101
|
+
ast = parse(model.sql, dialect)
|
|
102
|
+
for ref in table_references(ast):
|
|
103
|
+
if ref in names:
|
|
104
|
+
add(ref)
|
|
105
|
+
else:
|
|
106
|
+
add(ref.rsplit(".", 1)[-1]) # match a qualified ref to a model by its tail
|
|
107
|
+
|
|
108
|
+
return tuple(deps), ast, dialect
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _fingerprint_query(model: ModelDef, ast: exp.Expression | None) -> str | exp.Expression:
|
|
112
|
+
if ast is not None:
|
|
113
|
+
return ast
|
|
114
|
+
if model.fn is not None:
|
|
115
|
+
return textwrap.dedent(inspect.getsource(model.fn))
|
|
116
|
+
raise DefinitionError(f"model {model.name!r} has neither SQL nor a function body")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def compile_models(
|
|
120
|
+
models: Iterable[ModelDef],
|
|
121
|
+
*,
|
|
122
|
+
default_dialect: str = "duckdb",
|
|
123
|
+
default_engine: str = "default",
|
|
124
|
+
engine_dialects: dict[str, str] | None = None,
|
|
125
|
+
known_engines: set[str] | None = None,
|
|
126
|
+
catalog: str | None = None,
|
|
127
|
+
checks: Iterable[CheckDef] = (),
|
|
128
|
+
) -> CompiledProject:
|
|
129
|
+
"""Compile models into a fingerprinted, topologically-ordered project.
|
|
130
|
+
|
|
131
|
+
``checks`` are ``@check``-decorated Python functions, attached by model name.
|
|
132
|
+
``engine_dialects`` maps engine name → sqlglot dialect (used when a model
|
|
133
|
+
omits ``dialect``). ``known_engines`` validates model ``engine`` pins.
|
|
134
|
+
"""
|
|
135
|
+
definitions = {m.name: m for m in models}
|
|
136
|
+
names = set(definitions)
|
|
137
|
+
dialects_by_engine = engine_dialects or {}
|
|
138
|
+
engines = known_engines
|
|
139
|
+
|
|
140
|
+
resolved: dict[str, tuple[tuple[str, ...], exp.Expression | None, str, str]] = {}
|
|
141
|
+
graph = DependencyGraph()
|
|
142
|
+
for name, definition in definitions.items():
|
|
143
|
+
engine = definition.engine or default_engine
|
|
144
|
+
if engines is not None and engine not in engines:
|
|
145
|
+
raise DefinitionError(
|
|
146
|
+
f"model {name!r} references unknown engine {engine!r}",
|
|
147
|
+
details={"engines": sorted(engines)},
|
|
148
|
+
)
|
|
149
|
+
# Authoring dialect: explicit model dialect, else the engine's, else project default.
|
|
150
|
+
model_default_dialect = dialects_by_engine.get(engine, default_dialect)
|
|
151
|
+
deps, ast, dialect = _resolve_dependencies(definition, names, model_default_dialect)
|
|
152
|
+
resolved[name] = (deps, ast, dialect, engine)
|
|
153
|
+
graph.add_node(name)
|
|
154
|
+
for dep in deps:
|
|
155
|
+
graph.add_dependency(name, dep)
|
|
156
|
+
|
|
157
|
+
compiled: dict[str, CompiledModel] = {}
|
|
158
|
+
for name in graph.topological_sort(): # upstreams first; raises on cycle
|
|
159
|
+
definition = definitions[name]
|
|
160
|
+
deps, ast, dialect, engine = resolved[name]
|
|
161
|
+
for dep in deps: # topo order: deps already compiled
|
|
162
|
+
if compiled[dep].engine != engine and compiled[dep].materialise == "ephemeral":
|
|
163
|
+
raise DefinitionError(
|
|
164
|
+
f"model {name!r} on engine {engine!r} inlines ephemeral {dep!r} declared on engine "
|
|
165
|
+
f"{compiled[dep].engine!r}; an ephemeral model must share its consumers' engine "
|
|
166
|
+
f"(see docs/architecture/MULTI_ENGINE.md)"
|
|
167
|
+
)
|
|
168
|
+
strategy_config = {
|
|
169
|
+
"materialise": definition.materialise,
|
|
170
|
+
"strategy": definition.strategy,
|
|
171
|
+
"key": list(definition.key),
|
|
172
|
+
"interval": definition.interval,
|
|
173
|
+
"time_column": definition.time_column,
|
|
174
|
+
"cursor": definition.cursor,
|
|
175
|
+
"engine": engine,
|
|
176
|
+
"export": (
|
|
177
|
+
{
|
|
178
|
+
"to": definition.export.to,
|
|
179
|
+
"path": definition.export.path,
|
|
180
|
+
"target": definition.export.target,
|
|
181
|
+
"mode": definition.export.mode,
|
|
182
|
+
"key": list(definition.export.key),
|
|
183
|
+
}
|
|
184
|
+
if definition.export
|
|
185
|
+
else None
|
|
186
|
+
),
|
|
187
|
+
"dialect": dialect,
|
|
188
|
+
}
|
|
189
|
+
query = _fingerprint_query(definition, ast)
|
|
190
|
+
# render the canonical SQL once: it feeds both fingerprints and definition_sql
|
|
191
|
+
canonical = canonical_sql(query) if isinstance(query, exp.Expression) else query
|
|
192
|
+
local_fingerprint = data_fingerprint(query=canonical, strategy_config=strategy_config, upstream_fingerprints=[])
|
|
193
|
+
fingerprint = data_fingerprint(
|
|
194
|
+
query=canonical,
|
|
195
|
+
strategy_config=strategy_config,
|
|
196
|
+
upstream_fingerprints=[compiled[dep].fingerprint for dep in deps],
|
|
197
|
+
)
|
|
198
|
+
metadata_hash = metadata_fingerprint(
|
|
199
|
+
{
|
|
200
|
+
"owner": definition.owner,
|
|
201
|
+
"tags": list(definition.tags),
|
|
202
|
+
"description": definition.description,
|
|
203
|
+
"checks": [
|
|
204
|
+
{"type": c.type, "columns": list(c.columns), "severity": c.severity, "params": c.params}
|
|
205
|
+
for c in definition.checks
|
|
206
|
+
],
|
|
207
|
+
}
|
|
208
|
+
)
|
|
209
|
+
compiled[name] = CompiledModel(
|
|
210
|
+
name=name,
|
|
211
|
+
dialect=dialect,
|
|
212
|
+
engine=engine,
|
|
213
|
+
dependencies=deps,
|
|
214
|
+
fingerprint=fingerprint,
|
|
215
|
+
local_fingerprint=local_fingerprint,
|
|
216
|
+
metadata_hash=metadata_hash,
|
|
217
|
+
definition_sql=canonical if ast is not None else None,
|
|
218
|
+
physical_table=_physical_table(name, fingerprint, catalog),
|
|
219
|
+
materialise=definition.materialise,
|
|
220
|
+
strategy=definition.strategy,
|
|
221
|
+
key=definition.key,
|
|
222
|
+
time_column=definition.time_column,
|
|
223
|
+
cursor=definition.cursor,
|
|
224
|
+
interval=definition.interval,
|
|
225
|
+
tags=definition.tags,
|
|
226
|
+
schedule=definition.schedule,
|
|
227
|
+
columns=definition.columns,
|
|
228
|
+
export=definition.export,
|
|
229
|
+
ast=ast,
|
|
230
|
+
owner=definition.owner,
|
|
231
|
+
description=definition.description,
|
|
232
|
+
fn=definition.fn,
|
|
233
|
+
checks=definition.checks,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
python_checks: dict[str, tuple[CheckDef, ...]] = {}
|
|
237
|
+
for check in checks:
|
|
238
|
+
if check.model not in compiled:
|
|
239
|
+
raise DefinitionError(f"@check {check.name!r} references unknown model {check.model!r}")
|
|
240
|
+
python_checks[check.model] = (*python_checks.get(check.model, ()), check)
|
|
241
|
+
|
|
242
|
+
return CompiledProject(models=compiled, graph=graph, python_checks=python_checks)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""dbt-style model selection.
|
|
2
|
+
|
|
3
|
+
A selector is one of: ``model`` (exact), ``+model`` (model + ancestors),
|
|
4
|
+
``model+`` (model + descendants), ``+model+`` (both), or ``tag:<name>``. Multiple
|
|
5
|
+
selectors (space- or comma-separated, or repeated) union together. An empty
|
|
6
|
+
selector list means "all models".
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from interlace.exceptions import SelectionError
|
|
12
|
+
from interlace.graph.project import CompiledProject
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def select_models(selectors: list[str], project: CompiledProject) -> set[str]:
|
|
16
|
+
if not selectors:
|
|
17
|
+
return set(project.models)
|
|
18
|
+
chosen: set[str] = set()
|
|
19
|
+
for raw in selectors:
|
|
20
|
+
for token in _tokens(raw):
|
|
21
|
+
chosen |= _resolve(token, project)
|
|
22
|
+
return chosen
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _tokens(raw: str) -> list[str]:
|
|
26
|
+
return [token for token in raw.replace(",", " ").split() if token]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _resolve(token: str, project: CompiledProject) -> set[str]:
|
|
30
|
+
if token.startswith("tag:"):
|
|
31
|
+
tag = token[4:]
|
|
32
|
+
return {name for name, model in project.models.items() if tag in model.tags}
|
|
33
|
+
|
|
34
|
+
include_ancestors = token.startswith("+")
|
|
35
|
+
include_descendants = token.endswith("+")
|
|
36
|
+
name = token.strip("+")
|
|
37
|
+
if name not in project.models:
|
|
38
|
+
raise SelectionError(f"unknown model in selector: {name!r}", details={"selector": token})
|
|
39
|
+
|
|
40
|
+
chosen = {name}
|
|
41
|
+
if include_ancestors:
|
|
42
|
+
chosen |= project.graph.ancestors(name)
|
|
43
|
+
if include_descendants:
|
|
44
|
+
chosen |= project.graph.descendants(name)
|
|
45
|
+
return chosen
|