interlaced 2.2.0__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {interlaced-2.2.0/src/interlaced.egg-info → interlaced-2.4.0}/PKG-INFO +53 -16
- {interlaced-2.2.0 → interlaced-2.4.0}/README.md +46 -11
- {interlaced-2.2.0 → interlaced-2.4.0}/pyproject.toml +9 -5
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/cli/main.py +3 -3
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/config/config.py +8 -7
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/decorators.py +15 -3
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/discovery.py +1 -1
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/duckdb.py +54 -3
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/spark.py +1 -1
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/project.py +2 -2
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/canonicalize.py +4 -1
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/apply.py +153 -34
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/differ.py +2 -2
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/plan.py +2 -2
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/run.py +1 -1
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/project.py +6 -1
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scaffold.py +19 -2
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/app.py +8 -4
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/__init__.py +10 -4
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/append.py +12 -4
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/base.py +13 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/hash_merge.py +40 -18
- interlaced-2.4.0/src/interlace/strategies/incremental.py +95 -0
- interlaced-2.4.0/src/interlace/strategies/merge.py +188 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/scd.py +9 -1
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/interlace.yaml +2 -1
- interlaced-2.4.0/src/interlace/templates/quickstart/interlace.yaml +6 -0
- {interlaced-2.2.0 → interlaced-2.4.0/src/interlaced.egg-info}/PKG-INFO +53 -16
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/SOURCES.txt +1 -1
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/requires.txt +3 -3
- interlaced-2.2.0/src/interlace/strategies/incremental_by_time.py +0 -64
- interlaced-2.2.0/src/interlace/strategies/merge.py +0 -115
- interlaced-2.2.0/src/interlace/templates/quickstart/interlace.yaml +0 -6
- {interlaced-2.2.0 → interlaced-2.4.0}/LICENSE +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/MANIFEST.in +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/setup.cfg +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/builtin.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/runner.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/spec.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/cli/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/config/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/contracts.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/sql_config.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/adbc.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/base.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/bigquery.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/postgres.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/quack.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/redshift.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/registry.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/snowflake.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/exceptions.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/column_lineage.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/dag.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/selectors.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/fingerprint.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/relation.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/resolve.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/py.typed +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/query.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/runtime/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/runtime/handles.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/runtime/python_model.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/engine.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/triggers.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/worker.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/auth.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/app.css +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/favicon.svg +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/index.html +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/api.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/app.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/dag.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/timeline.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/ui.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/checks.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/environments.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/lineage.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/models.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/overview.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/plan.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/query.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/runs.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/streams.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/system.js +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sinks.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sources/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sources/auth.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sources/rest.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/interval.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/janitor.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/snapshot.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/store.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/full_merge.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/replace.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/replace_in_place.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/view.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/__init__.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/log.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/materializer.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/schema.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/README.md +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/generate.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/events.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/events_by_minute.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/events_by_type.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/top_users.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/user_spend.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/template.yaml +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/README.md +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/interlace.yaml +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/models/github_issues.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/models/issues_by_state.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/template.yaml +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/README.md +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/docker-compose.yml +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/init/seed.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/interlace.yaml +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/models/orders.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/models/orders_by_status.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/template.yaml +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/README.md +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/models/enriched_events.py +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/models/event_summary.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/models/raw_events.sql +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/template.yaml +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/dependency_links.txt +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/entry_points.txt +0 -0
- {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: interlaced
|
|
3
|
-
Version: 2.
|
|
4
|
-
Summary: Python
|
|
3
|
+
Version: 2.4.0
|
|
4
|
+
Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
|
|
5
5
|
Author-email: Mark <mark@interlace.sh>
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://interlace.sh
|
|
@@ -13,17 +13,19 @@ Keywords: data,pipeline,orchestration,transformation,etl,streaming,dbt,sqlmesh,d
|
|
|
13
13
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
14
|
Classifier: Intended Audience :: Developers
|
|
15
15
|
Classifier: Topic :: Software Development :: Build Tools
|
|
16
|
+
Classifier: Topic :: Database
|
|
16
17
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
18
|
Classifier: Programming Language :: SQL
|
|
19
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
20
|
Requires-Python: >=3.12
|
|
19
21
|
Description-Content-Type: text/markdown
|
|
20
22
|
License-File: LICENSE
|
|
21
|
-
Requires-Dist: sqlglot<
|
|
23
|
+
Requires-Dist: sqlglot<30.0,>=25.0
|
|
22
24
|
Requires-Dist: duckdb>=1.5.3
|
|
23
25
|
Requires-Dist: pyarrow>=17.0
|
|
24
26
|
Requires-Dist: pydantic<3.0,>=2.5
|
|
25
27
|
Requires-Dist: typer<1.0,>=0.12
|
|
26
|
-
Requires-Dist: rich<
|
|
28
|
+
Requires-Dist: rich<16.0,>=13.0
|
|
27
29
|
Requires-Dist: cronsim<3.0,>=2.5
|
|
28
30
|
Requires-Dist: tenacity<10.0,>=8.2
|
|
29
31
|
Requires-Dist: pyyaml<7.0,>=6.0
|
|
@@ -59,20 +61,46 @@ Requires-Dist: httpx<1.0,>=0.27; extra == "dev"
|
|
|
59
61
|
Requires-Dist: pytest-asyncio<2.0,>=1.0; extra == "dev"
|
|
60
62
|
Requires-Dist: ruff<1.0,>=0.6; extra == "dev"
|
|
61
63
|
Requires-Dist: black<27.0,>=24.0; extra == "dev"
|
|
62
|
-
Requires-Dist: mypy<
|
|
64
|
+
Requires-Dist: mypy<3.0,>=1.11; extra == "dev"
|
|
63
65
|
Dynamic: license-file
|
|
64
66
|
|
|
65
67
|
# interlace
|
|
66
68
|
|
|
67
|
-
**Python
|
|
69
|
+
**Python and SQL models are the same kind of node in one DAG.**
|
|
68
70
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
71
|
+
A `.py` model sits mid-graph with SQL either side, in both directions, with no bridge and no
|
|
72
|
+
separate runtime — running in-process on DuckDB and Postgres, not only on a cloud warehouse.
|
|
73
|
+
The Python model stays a plain function: call it in a test with no warehouse and no session.
|
|
74
|
+
|
|
75
|
+
How that compares with dbt and SQLMesh, including where they are ahead, is on
|
|
76
|
+
[interlace.sh/why](https://interlace.sh/why).
|
|
77
|
+
|
|
78
|
+
```sql
|
|
79
|
+
-- models/raw_events.sql SQL
|
|
80
|
+
SELECT event_id, user_id, kind, amount, country, ts FROM read_parquet('events/*.parquet')
|
|
81
|
+
```
|
|
82
|
+
```python
|
|
83
|
+
# models/enriched_events.py Python, mid-DAG
|
|
84
|
+
@model() # the parameter name IS the dependency — no depends_on
|
|
85
|
+
def enriched_events(raw_events):
|
|
86
|
+
for batch in raw_events.reader(): # Arrow in, Arrow out, bounded memory
|
|
87
|
+
yield add_revenue(batch)
|
|
88
|
+
```
|
|
89
|
+
```sql
|
|
90
|
+
-- models/event_summary.sql SQL again, straight over the Python
|
|
91
|
+
SELECT country, count(*) FILTER (WHERE is_conversion) AS conversions
|
|
92
|
+
FROM enriched_events GROUP BY country
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
`interlace init` scaffolds exactly this shape, runnable, with no external source.
|
|
96
|
+
|
|
97
|
+
That is the wedge. The rest is the reveal: interlace is an independent, MIT-licensed alternative
|
|
98
|
+
to dbt/SQLMesh that also replaces the orchestrator (no Airflow) and the ingestion layer
|
|
99
|
+
(Cloudflare-Pipelines-style durable streams). State is versioned snapshots with virtual
|
|
72
100
|
environments and a terraform-style plan/apply; everything runs in a single daemon on
|
|
73
|
-
DuckDB
|
|
101
|
+
DuckDB by default (DuckLake one config line away).
|
|
74
102
|
|
|
75
|
-
> **
|
|
103
|
+
> **2.x — see [releases](https://github.com/interlace-sh/interlace/releases).** Requires Python 3.12+.
|
|
76
104
|
> The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
|
|
77
105
|
|
|
78
106
|
```bash
|
|
@@ -80,6 +108,13 @@ pip install 'interlaced[service]' # the CLI + daemon; core CLI only: pip insta
|
|
|
80
108
|
# more extras: [adbc] postgres/redshift · [spark] · [polars] · [all]
|
|
81
109
|
```
|
|
82
110
|
|
|
111
|
+
> **Platforms.** Developed on Linux; CI runs Linux only. Nothing in the codebase is
|
|
112
|
+
> platform-specific — no `fork`, no signal handling, no POSIX-only calls, no shelling out — and
|
|
113
|
+
> every dependency ships macOS and Windows wheels, so both are expected to work. But
|
|
114
|
+
> **neither is tested**, so treat them as unverified rather than supported. If you run interlace
|
|
115
|
+
> on macOS or Windows, please open an issue either way; that is the fastest route to changing
|
|
116
|
+
> this paragraph.
|
|
117
|
+
|
|
83
118
|
## Sixty seconds
|
|
84
119
|
|
|
85
120
|
```bash
|
|
@@ -127,8 +162,9 @@ def orders(cursor, this):
|
|
|
127
162
|
```
|
|
128
163
|
|
|
129
164
|
**Strategies:** `replace`, `view`, `ephemeral` (CTE-inlined), `merge` (upsert),
|
|
130
|
-
`full_merge` (full-state source applied as a minimal diff), `
|
|
131
|
-
|
|
165
|
+
`full_merge` (full-state source applied as a minimal diff), `incremental` (one time window
|
|
166
|
+
at a time — rewrites the window, or upserts within it if you give it a `key`), `scd`
|
|
167
|
+
(history with validity windows).
|
|
132
168
|
|
|
133
169
|
## Plan / apply
|
|
134
170
|
|
|
@@ -209,7 +245,7 @@ prod, so a dev apply never writes to a live external table (opt in with
|
|
|
209
245
|
|
|
210
246
|
## Multi-engine
|
|
211
247
|
|
|
212
|
-
Models run on **named engines**: DuckDB
|
|
248
|
+
Models run on **named engines**: DuckDB by default (DuckLake opt-in), Postgres natively over ADBC
|
|
213
249
|
(`pip install 'interlaced[adbc]'`), Spark (beta, `[spark]` extra), plus alpha adapters for
|
|
214
250
|
MotherDuck, Redshift, Snowflake and BigQuery (wired and dialect-correct, not yet run against a
|
|
215
251
|
live account), with per-model pinning:
|
|
@@ -251,7 +287,8 @@ other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by
|
|
|
251
287
|
|
|
252
288
|
- The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
|
|
253
289
|
are AST builders and dialect appears only at `transpile()`.
|
|
254
|
-
- Storage defaults to **DuckLake** (Parquet + SQL catalog
|
|
290
|
+
- Storage defaults to a plain **DuckDB** file; **DuckLake** (Parquet + SQL catalog, and
|
|
291
|
+
concurrent writers) is one `database: ducklake:…` line away.
|
|
255
292
|
- Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
|
|
256
293
|
scale-out swap.
|
|
257
294
|
- Streams live in their own durable log; the materializer commits data + watermark in one
|
|
@@ -268,7 +305,7 @@ Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
|
|
|
268
305
|
```bash
|
|
269
306
|
proto install
|
|
270
307
|
moon run interlace:sync # install deps
|
|
271
|
-
moon run interlace:test #
|
|
308
|
+
moon run interlace:test # 500+ tests
|
|
272
309
|
moon run interlace:check # black + ruff (CI equivalent)
|
|
273
310
|
moon run interlace:typecheck # mypy
|
|
274
311
|
```
|
|
@@ -1,14 +1,40 @@
|
|
|
1
1
|
# interlace
|
|
2
2
|
|
|
3
|
-
**Python
|
|
3
|
+
**Python and SQL models are the same kind of node in one DAG.**
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
A `.py` model sits mid-graph with SQL either side, in both directions, with no bridge and no
|
|
6
|
+
separate runtime — running in-process on DuckDB and Postgres, not only on a cloud warehouse.
|
|
7
|
+
The Python model stays a plain function: call it in a test with no warehouse and no session.
|
|
8
|
+
|
|
9
|
+
How that compares with dbt and SQLMesh, including where they are ahead, is on
|
|
10
|
+
[interlace.sh/why](https://interlace.sh/why).
|
|
11
|
+
|
|
12
|
+
```sql
|
|
13
|
+
-- models/raw_events.sql SQL
|
|
14
|
+
SELECT event_id, user_id, kind, amount, country, ts FROM read_parquet('events/*.parquet')
|
|
15
|
+
```
|
|
16
|
+
```python
|
|
17
|
+
# models/enriched_events.py Python, mid-DAG
|
|
18
|
+
@model() # the parameter name IS the dependency — no depends_on
|
|
19
|
+
def enriched_events(raw_events):
|
|
20
|
+
for batch in raw_events.reader(): # Arrow in, Arrow out, bounded memory
|
|
21
|
+
yield add_revenue(batch)
|
|
22
|
+
```
|
|
23
|
+
```sql
|
|
24
|
+
-- models/event_summary.sql SQL again, straight over the Python
|
|
25
|
+
SELECT country, count(*) FILTER (WHERE is_conversion) AS conversions
|
|
26
|
+
FROM enriched_events GROUP BY country
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`interlace init` scaffolds exactly this shape, runnable, with no external source.
|
|
30
|
+
|
|
31
|
+
That is the wedge. The rest is the reveal: interlace is an independent, MIT-licensed alternative
|
|
32
|
+
to dbt/SQLMesh that also replaces the orchestrator (no Airflow) and the ingestion layer
|
|
33
|
+
(Cloudflare-Pipelines-style durable streams). State is versioned snapshots with virtual
|
|
8
34
|
environments and a terraform-style plan/apply; everything runs in a single daemon on
|
|
9
|
-
DuckDB
|
|
35
|
+
DuckDB by default (DuckLake one config line away).
|
|
10
36
|
|
|
11
|
-
> **
|
|
37
|
+
> **2.x — see [releases](https://github.com/interlace-sh/interlace/releases).** Requires Python 3.12+.
|
|
12
38
|
> The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
|
|
13
39
|
|
|
14
40
|
```bash
|
|
@@ -16,6 +42,13 @@ pip install 'interlaced[service]' # the CLI + daemon; core CLI only: pip insta
|
|
|
16
42
|
# more extras: [adbc] postgres/redshift · [spark] · [polars] · [all]
|
|
17
43
|
```
|
|
18
44
|
|
|
45
|
+
> **Platforms.** Developed on Linux; CI runs Linux only. Nothing in the codebase is
|
|
46
|
+
> platform-specific — no `fork`, no signal handling, no POSIX-only calls, no shelling out — and
|
|
47
|
+
> every dependency ships macOS and Windows wheels, so both are expected to work. But
|
|
48
|
+
> **neither is tested**, so treat them as unverified rather than supported. If you run interlace
|
|
49
|
+
> on macOS or Windows, please open an issue either way; that is the fastest route to changing
|
|
50
|
+
> this paragraph.
|
|
51
|
+
|
|
19
52
|
## Sixty seconds
|
|
20
53
|
|
|
21
54
|
```bash
|
|
@@ -63,8 +96,9 @@ def orders(cursor, this):
|
|
|
63
96
|
```
|
|
64
97
|
|
|
65
98
|
**Strategies:** `replace`, `view`, `ephemeral` (CTE-inlined), `merge` (upsert),
|
|
66
|
-
`full_merge` (full-state source applied as a minimal diff), `
|
|
67
|
-
|
|
99
|
+
`full_merge` (full-state source applied as a minimal diff), `incremental` (one time window
|
|
100
|
+
at a time — rewrites the window, or upserts within it if you give it a `key`), `scd`
|
|
101
|
+
(history with validity windows).
|
|
68
102
|
|
|
69
103
|
## Plan / apply
|
|
70
104
|
|
|
@@ -145,7 +179,7 @@ prod, so a dev apply never writes to a live external table (opt in with
|
|
|
145
179
|
|
|
146
180
|
## Multi-engine
|
|
147
181
|
|
|
148
|
-
Models run on **named engines**: DuckDB
|
|
182
|
+
Models run on **named engines**: DuckDB by default (DuckLake opt-in), Postgres natively over ADBC
|
|
149
183
|
(`pip install 'interlaced[adbc]'`), Spark (beta, `[spark]` extra), plus alpha adapters for
|
|
150
184
|
MotherDuck, Redshift, Snowflake and BigQuery (wired and dialect-correct, not yet run against a
|
|
151
185
|
live account), with per-model pinning:
|
|
@@ -187,7 +221,8 @@ other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by
|
|
|
187
221
|
|
|
188
222
|
- The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
|
|
189
223
|
are AST builders and dialect appears only at `transpile()`.
|
|
190
|
-
- Storage defaults to **DuckLake** (Parquet + SQL catalog
|
|
224
|
+
- Storage defaults to a plain **DuckDB** file; **DuckLake** (Parquet + SQL catalog, and
|
|
225
|
+
concurrent writers) is one `database: ducklake:…` line away.
|
|
191
226
|
- Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
|
|
192
227
|
scale-out swap.
|
|
193
228
|
- Streams live in their own durable log; the materializer commits data + watermark in one
|
|
@@ -204,7 +239,7 @@ Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
|
|
|
204
239
|
```bash
|
|
205
240
|
proto install
|
|
206
241
|
moon run interlace:sync # install deps
|
|
207
|
-
moon run interlace:test #
|
|
242
|
+
moon run interlace:test # 500+ tests
|
|
208
243
|
moon run interlace:check # black + ruff (CI equivalent)
|
|
209
244
|
moon run interlace:typecheck # mypy
|
|
210
245
|
```
|
|
@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "interlaced"
|
|
7
|
-
version = "2.
|
|
8
|
-
description = "Python
|
|
7
|
+
version = "2.4.0"
|
|
8
|
+
description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.12"
|
|
11
11
|
license = "MIT"
|
|
@@ -17,19 +17,23 @@ classifiers = [
|
|
|
17
17
|
"Development Status :: 5 - Production/Stable",
|
|
18
18
|
"Intended Audience :: Developers",
|
|
19
19
|
"Topic :: Software Development :: Build Tools",
|
|
20
|
+
"Topic :: Database",
|
|
20
21
|
"Programming Language :: Python :: 3.12",
|
|
21
22
|
"Programming Language :: SQL",
|
|
23
|
+
# Linux is what CI runs and what this is developed on. Nothing in the codebase
|
|
24
|
+
# is platform-specific, but macOS and Windows are untested — see the README.
|
|
25
|
+
"Operating System :: POSIX :: Linux",
|
|
22
26
|
]
|
|
23
27
|
|
|
24
28
|
# Core: the IR + engine + state + plan spine (Phase 1). Everything is Arrow- and
|
|
25
29
|
# sqlglot-native. See docs/architecture/architecture.md.
|
|
26
30
|
dependencies = [
|
|
27
|
-
"sqlglot>=25.0,<
|
|
31
|
+
"sqlglot>=25.0,<30.0", # canonical IR, transpilation, semantic diff, column lineage
|
|
28
32
|
"duckdb>=1.5.3", # default engine + federation hub; DuckLake storage + quack serving
|
|
29
33
|
"pyarrow>=17.0", # the wire format: RecordBatchReader everywhere
|
|
30
34
|
"pydantic>=2.5,<3.0", # config + manifest validation (cold paths only)
|
|
31
35
|
"typer>=0.12,<1.0", # CLI
|
|
32
|
-
"rich>=13.0,<
|
|
36
|
+
"rich>=13.0,<16.0", # display, strictly an event subscriber
|
|
33
37
|
"cronsim>=2.5,<3.0", # cron parsing for the trigger engine
|
|
34
38
|
"tenacity>=8.2,<10.0", # retries (tasks, DuckLake commit conflicts, transfers)
|
|
35
39
|
"pyyaml>=6.0,<7.0", # project config (config + env overlays)
|
|
@@ -78,7 +82,7 @@ dev = [
|
|
|
78
82
|
"pytest-asyncio>=1.0,<2.0",
|
|
79
83
|
"ruff>=0.6,<1.0",
|
|
80
84
|
"black>=24.0,<27.0",
|
|
81
|
-
"mypy>=1.11,<
|
|
85
|
+
"mypy>=1.11,<3.0",
|
|
82
86
|
]
|
|
83
87
|
|
|
84
88
|
[tool.setuptools.packages.find]
|
|
@@ -107,7 +107,7 @@ _END = typer.Option("", "--end", help="Window end (ISO), for incremental models.
|
|
|
107
107
|
_FORWARD_ONLY = typer.Option(
|
|
108
108
|
False,
|
|
109
109
|
"--forward-only",
|
|
110
|
-
help="Modified history-keeping models (merge/full_merge/scd/
|
|
110
|
+
help="Modified history-keeping models (merge/full_merge/scd/incremental) carry their history "
|
|
111
111
|
"forward: it is copied to the new version, the new logic applies to the copy, and checks gate "
|
|
112
112
|
"before views move. Requires a shape-compatible change.",
|
|
113
113
|
)
|
|
@@ -205,7 +205,7 @@ async def _render_empty_incrementals(result: ApplyResult, compiled: CompiledProj
|
|
|
205
205
|
|
|
206
206
|
for name in result.built:
|
|
207
207
|
model = compiled.models[name]
|
|
208
|
-
if model.strategy != "
|
|
208
|
+
if model.strategy != "incremental" or model.is_terminal:
|
|
209
209
|
continue
|
|
210
210
|
counts = result.rows.get(name)
|
|
211
211
|
if counts is not None and (counts.inserted or counts.updated):
|
|
@@ -404,7 +404,7 @@ def run(
|
|
|
404
404
|
) -> None:
|
|
405
405
|
"""Force-build models and promote, ignoring change detection.
|
|
406
406
|
|
|
407
|
-
For
|
|
407
|
+
For incremental models, --start/--end set the catchup window
|
|
408
408
|
(default: the latest grain interval).
|
|
409
409
|
"""
|
|
410
410
|
asyncio.run(_execute(environment, path, select, start, end, restate=False, parallelism=parallelism))
|
|
@@ -84,7 +84,7 @@ class EngineConfig(BaseModel):
|
|
|
84
84
|
declaring them fails at open until an adapter ships.
|
|
85
85
|
"""
|
|
86
86
|
|
|
87
|
-
type: str = "
|
|
87
|
+
type: str = "duckdb"
|
|
88
88
|
# Path / URI for DuckDB-family engines. Also accepted on the project top level
|
|
89
89
|
# as ``database:`` (synthesised into the ``default`` engine).
|
|
90
90
|
database: str | None = None
|
|
@@ -117,13 +117,14 @@ class ProjectConfig(BaseModel):
|
|
|
117
117
|
default_engine: str = "default"
|
|
118
118
|
engines: dict[str, EngineConfig] = Field(default_factory=dict)
|
|
119
119
|
state_path: str = ".interlace/state.db" # SQLite control-plane database
|
|
120
|
-
# The warehouse. Default is
|
|
121
|
-
# Also accepted: a DuckLake catalog
|
|
122
|
-
#
|
|
123
|
-
#
|
|
124
|
-
#
|
|
120
|
+
# The warehouse. Default is a plain DuckDB file — simplest, single-process.
|
|
121
|
+
# Also accepted: a DuckLake catalog (``ducklake:.interlace/warehouse.ducklake``, or
|
|
122
|
+
# hosted in a SQL DB: ``ducklake:postgres:dbname=... host=...`` — pair with
|
|
123
|
+
# data_path/metadata_schema) which serialises catalog writes so `interlace serve`
|
|
124
|
+
# and a separate CLI can share the warehouse concurrently; ":memory:"; or
|
|
125
|
+
# "quack:<host>:<port>" to connect to a warehouse served by `interlace serve --quack`.
|
|
125
126
|
# When ``engines.default`` is not set, these top-level fields synthesise it.
|
|
126
|
-
database: str = "
|
|
127
|
+
database: str = ".interlace/warehouse.duckdb"
|
|
127
128
|
# The warehouse catalog's ATTACH alias (defaults to ``name``). Set it when a
|
|
128
129
|
# schema inside the warehouse shares the project name — see EngineConfig.alias.
|
|
129
130
|
alias: str | None = None
|
|
@@ -94,9 +94,9 @@ class ModelDef:
|
|
|
94
94
|
dialect: str | None = None
|
|
95
95
|
engine: str | None = None # named engine from config (None → project default_engine)
|
|
96
96
|
depends_on: tuple[str, ...] = ()
|
|
97
|
-
interval: str | None = None # grain for
|
|
98
|
-
time_column: str | None = None # partition column for
|
|
99
|
-
# First-build window for
|
|
97
|
+
interval: str | None = None # grain for incremental (e.g. "1d")
|
|
98
|
+
time_column: str | None = None # partition column for incremental
|
|
99
|
+
# First-build window for incremental: "auto" derives [min, max] of the
|
|
100
100
|
# time column from the source at apply time and fills it as ONE interval;
|
|
101
101
|
# "none" keeps only the latest grain window; an ISO date pins the start.
|
|
102
102
|
backfill: str = "auto"
|
|
@@ -114,6 +114,18 @@ class ModelDef:
|
|
|
114
114
|
schedule: dict[str, str] | None = None # {"cron": "0 * * * *"} or {"every": "5m"} for `interlace serve`
|
|
115
115
|
checks: tuple[CheckSpec, ...] = () # data-quality checks; error severity gates promotion
|
|
116
116
|
|
|
117
|
+
def __post_init__(self) -> None:
|
|
118
|
+
# `@model(checks=…)` normalises through parse_checks, but a ModelDef built
|
|
119
|
+
# directly — the dynamic-model path, which is what generated models and dbt
|
|
120
|
+
# migrations use — stored the dicts raw and only failed at compile time with
|
|
121
|
+
# `AttributeError: 'dict' object has no attribute 'type'`. Normalise here too,
|
|
122
|
+
# so one spelling works on both surfaces and a bad check fails at declaration.
|
|
123
|
+
# Passed through as-is, not as list(...): parse_checks already handles a bare
|
|
124
|
+
# CheckSpec and reports a non-list clearly, both of which list() would mangle
|
|
125
|
+
# (TypeError on a CheckSpec; a dict silently degraded to its keys). Always run,
|
|
126
|
+
# so `checks=[]` normalises to the declared tuple rather than staying a list.
|
|
127
|
+
self.checks = parse_checks(self.checks, self.name)
|
|
128
|
+
|
|
117
129
|
@property
|
|
118
130
|
def is_terminal(self) -> bool:
|
|
119
131
|
"""A terminal model delivers into an external destination (table/file):
|
|
@@ -64,7 +64,7 @@ def _sql_model(default_name: str, sql: str, config: dict[str, Any], default_dial
|
|
|
64
64
|
depends_on=_as_tuple(config.get("depends_on") or ()),
|
|
65
65
|
interval=config.get("interval"),
|
|
66
66
|
time_column=config.get("time_column"),
|
|
67
|
-
backfill=config.get("backfill", "auto"), # first-build window for
|
|
67
|
+
backfill=config.get("backfill", "auto"), # first-build window for incremental
|
|
68
68
|
tags=_as_tuple(config.get("tags") or ()),
|
|
69
69
|
owner=config.get("owner"),
|
|
70
70
|
description=config.get("description"),
|
|
@@ -28,6 +28,7 @@ from __future__ import annotations
|
|
|
28
28
|
|
|
29
29
|
import asyncio
|
|
30
30
|
import contextlib
|
|
31
|
+
import re
|
|
31
32
|
import threading
|
|
32
33
|
from collections.abc import Iterator, Sequence
|
|
33
34
|
from uuid import uuid4
|
|
@@ -38,6 +39,7 @@ import tenacity
|
|
|
38
39
|
from sqlglot import exp
|
|
39
40
|
|
|
40
41
|
from interlace.engines.base import EngineAdapter, EngineCaps, LoadMode
|
|
42
|
+
from interlace.exceptions import ConfigurationError
|
|
41
43
|
from interlace.ir.relation import TableRef
|
|
42
44
|
|
|
43
45
|
_DUCKDB_CAPS = EngineCaps(
|
|
@@ -58,6 +60,39 @@ _commit_retry = tenacity.retry(
|
|
|
58
60
|
)
|
|
59
61
|
|
|
60
62
|
|
|
63
|
+
@contextlib.contextmanager
|
|
64
|
+
def _clean_lock_error(database: str) -> Iterator[None]:
|
|
65
|
+
"""Translate DuckDB's file-lock conflict into one actionable line.
|
|
66
|
+
|
|
67
|
+
A DuckDB/DuckLake database is held by a single process. The common way to hit
|
|
68
|
+
that is running a CLI command (`interlace query`, `plan`, `apply`) while
|
|
69
|
+
`interlace serve` is up — an obvious thing to do, since serve is the daemon
|
|
70
|
+
and query is the console's CLI counterpart. Raw, that surfaces as a dozen
|
|
71
|
+
frames ending in `duckdb.IOException`, which names neither the cause nor the
|
|
72
|
+
fix. The fix is `--quack`, and it is already documented; the error just never
|
|
73
|
+
said so.
|
|
74
|
+
"""
|
|
75
|
+
try:
|
|
76
|
+
yield
|
|
77
|
+
except duckdb.IOException as exc:
|
|
78
|
+
message = str(exc)
|
|
79
|
+
# "Conflicting lock is held in <exe> (PID n)" is the Linux rendering — DuckDB
|
|
80
|
+
# names the holder from /proc/locks. Elsewhere the message is the bare "Could
|
|
81
|
+
# not set lock on file", and a same-process conflict says "already held", so
|
|
82
|
+
# match all three or macOS/Windows keep the raw traceback.
|
|
83
|
+
if not any(marker in message for marker in ("Conflicting lock", "Could not set lock", "already held")):
|
|
84
|
+
raise
|
|
85
|
+
holder = re.search(r"\(PID (\d+)\)", message)
|
|
86
|
+
held_by = f" (PID {holder.group(1)})" if holder else ""
|
|
87
|
+
raise ConfigurationError(
|
|
88
|
+
f"the warehouse {database!r} is already open in another process{held_by}. "
|
|
89
|
+
"DuckDB allows one process at a time — stop `interlace serve`, or serve the "
|
|
90
|
+
"warehouse over the quack protocol (`interlace serve --quack quack:localhost:4213`) "
|
|
91
|
+
"and point this process at `database: quack:localhost:4213` to share it.",
|
|
92
|
+
details={"database": database},
|
|
93
|
+
) from None
|
|
94
|
+
|
|
95
|
+
|
|
61
96
|
def _affected(cur: duckdb.DuckDBPyConnection) -> int:
|
|
62
97
|
"""DML/CTAS/COPY return their affected-row count as a one-cell result; DDL returns
|
|
63
98
|
nothing. Never raises — row stats are best-effort decoration, not correctness."""
|
|
@@ -108,7 +143,9 @@ class DuckDBAdapter(EngineAdapter):
|
|
|
108
143
|
|
|
109
144
|
@classmethod
|
|
110
145
|
def connect(cls, path: str) -> DuckDBAdapter:
|
|
111
|
-
|
|
146
|
+
with _clean_lock_error(path):
|
|
147
|
+
conn = duckdb.connect(path)
|
|
148
|
+
return cls(conn, serialise_writes=path.startswith("ducklake:"))
|
|
112
149
|
|
|
113
150
|
@classmethod
|
|
114
151
|
def connect_ducklake(
|
|
@@ -139,7 +176,8 @@ class DuckDBAdapter(EngineAdapter):
|
|
|
139
176
|
options_sql = f" ({', '.join(options)})" if options else ""
|
|
140
177
|
escaped = catalog.replace("'", "''")
|
|
141
178
|
alias_sql = exp.to_identifier(alias).sql("duckdb")
|
|
142
|
-
|
|
179
|
+
with _clean_lock_error(catalog):
|
|
180
|
+
conn.execute(f"ATTACH IF NOT EXISTS '{escaped}' AS {alias_sql}{options_sql}")
|
|
143
181
|
conn.execute(f"USE {alias_sql}")
|
|
144
182
|
# LOAD, secrets, and ATTACH are all instance-wide — they carry into every
|
|
145
183
|
# cursor and must run ONCE (re-running CREATE OR REPLACE SECRET per cursor
|
|
@@ -162,10 +200,23 @@ class DuckDBAdapter(EngineAdapter):
|
|
|
162
200
|
with contextlib.suppress(Exception):
|
|
163
201
|
self._conn.interrupt()
|
|
164
202
|
|
|
203
|
+
def search_files_from(self, directory: str) -> None:
|
|
204
|
+
"""Resolve relative read paths (``read_csv_auto('seeds/x.csv')``) against
|
|
205
|
+
``directory`` — the project root — as well as the process CWD.
|
|
206
|
+
|
|
207
|
+
Additive: a CWD-relative path still resolves, so this only ever widens what a
|
|
208
|
+
model can find. GLOBAL scope because a plain ``SET`` is session-scoped and would
|
|
209
|
+
not reach the per-task cursors that actually run the queries. Reads only —
|
|
210
|
+
``COPY`` targets stay CWD-relative, which is why exports resolve their own paths
|
|
211
|
+
against the root (``plan.apply._resolve_export_path``)."""
|
|
212
|
+
escaped = directory.replace("'", "''")
|
|
213
|
+
self._conn.execute(f"SET GLOBAL file_search_path='{escaped}'")
|
|
214
|
+
|
|
165
215
|
def attach(self, alias: str, uri: str) -> None:
|
|
166
216
|
"""ATTACH another database (duckdb/sqlite/postgres/... URI) under ``alias``."""
|
|
167
217
|
escaped = uri.replace("'", "''")
|
|
168
|
-
|
|
218
|
+
with _clean_lock_error(uri): # attaching a held duckdb/ducklake file conflicts just like opening one
|
|
219
|
+
self._conn.execute(f"ATTACH IF NOT EXISTS '{escaped}' AS {exp.to_identifier(alias).sql('duckdb')}")
|
|
169
220
|
self._attached.append(alias)
|
|
170
221
|
if uri.startswith("ducklake:"): # writes may now reach a DuckLake catalog (e.g. table sinks)
|
|
171
222
|
if isinstance(self._write_lock, contextlib.nullcontext):
|
|
@@ -7,7 +7,7 @@ come back as Arrow via ``DataFrame.toArrow()``, and Arrow loads go in through
|
|
|
7
7
|
(``local[*]``, for tests) or a remote one (Spark Connect / a shared session).
|
|
8
8
|
|
|
9
9
|
**Strategy support.** ``replace``, ``append`` and ``view`` run on any Spark
|
|
10
|
-
catalog. ``merge`` (native ``MERGE``) and ``
|
|
10
|
+
catalog. ``merge`` (native ``MERGE``) and ``incremental`` (windowed
|
|
11
11
|
``DELETE`` by literal predicate + ``INSERT``) need a catalog with row-level
|
|
12
12
|
mutations — Delta Lake or Iceberg — configured on the session you hand the
|
|
13
13
|
adapter (the tests use a Delta-backed local session). ``scd`` and ``full_merge``
|
|
@@ -44,9 +44,9 @@ class CompiledModel:
|
|
|
44
44
|
materialise: str
|
|
45
45
|
strategy: str
|
|
46
46
|
key: tuple[str, ...] # business key for keyed strategies (merge)
|
|
47
|
-
time_column: str | None # partition column for
|
|
47
|
+
time_column: str | None # partition column for incremental
|
|
48
48
|
cursor: str | None # column whose max is injected into a Python model's `cursor` param
|
|
49
|
-
interval: str | None # grain for
|
|
49
|
+
interval: str | None # grain for incremental (e.g. "1d")
|
|
50
50
|
tags: tuple[str, ...] # for tag: selection
|
|
51
51
|
schedule: dict[str, str] | None # cron/interval schedule for the trigger engine
|
|
52
52
|
columns: dict[str, str | None] | None # output contract validated at apply time
|
|
@@ -8,6 +8,8 @@ pruning and the ``impact`` command).
|
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
|
+
from typing import cast
|
|
12
|
+
|
|
11
13
|
import sqlglot
|
|
12
14
|
from sqlglot import exp
|
|
13
15
|
|
|
@@ -86,4 +88,5 @@ def resolve_references(ast: exp.Expression, mapping: dict[str, TableRef]) -> exp
|
|
|
86
88
|
node.set("catalog", exp.to_identifier(target.catalog) if target.catalog else None)
|
|
87
89
|
return node
|
|
88
90
|
|
|
89
|
-
|
|
91
|
+
# sqlglot 29 loosened transform()'s return annotation; it is an Expression.
|
|
92
|
+
return cast("exp.Expression", ast.transform(rewrite))
|