interlaced 2.0.0a2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interlaced-2.0.0a2/LICENSE +21 -0
- interlaced-2.0.0a2/PKG-INFO +216 -0
- interlaced-2.0.0a2/README.md +164 -0
- interlaced-2.0.0a2/pyproject.toml +159 -0
- interlaced-2.0.0a2/setup.cfg +4 -0
- interlaced-2.0.0a2/src/interlace/__init__.py +26 -0
- interlaced-2.0.0a2/src/interlace/checks/__init__.py +10 -0
- interlaced-2.0.0a2/src/interlace/checks/builtin.py +152 -0
- interlaced-2.0.0a2/src/interlace/checks/runner.py +103 -0
- interlaced-2.0.0a2/src/interlace/checks/spec.py +113 -0
- interlaced-2.0.0a2/src/interlace/cli/__init__.py +7 -0
- interlaced-2.0.0a2/src/interlace/cli/main.py +1072 -0
- interlaced-2.0.0a2/src/interlace/config/__init__.py +7 -0
- interlaced-2.0.0a2/src/interlace/config/config.py +172 -0
- interlaced-2.0.0a2/src/interlace/contracts.py +35 -0
- interlaced-2.0.0a2/src/interlace/dsl/__init__.py +16 -0
- interlaced-2.0.0a2/src/interlace/dsl/decorators.py +222 -0
- interlaced-2.0.0a2/src/interlace/dsl/discovery.py +75 -0
- interlaced-2.0.0a2/src/interlace/dsl/sql_config.py +47 -0
- interlaced-2.0.0a2/src/interlace/engines/__init__.py +16 -0
- interlaced-2.0.0a2/src/interlace/engines/base.py +86 -0
- interlaced-2.0.0a2/src/interlace/engines/duckdb.py +249 -0
- interlaced-2.0.0a2/src/interlace/engines/postgres.py +157 -0
- interlaced-2.0.0a2/src/interlace/engines/quack.py +113 -0
- interlaced-2.0.0a2/src/interlace/engines/registry.py +125 -0
- interlaced-2.0.0a2/src/interlace/exceptions.py +66 -0
- interlaced-2.0.0a2/src/interlace/exports.py +145 -0
- interlaced-2.0.0a2/src/interlace/graph/__init__.py +17 -0
- interlaced-2.0.0a2/src/interlace/graph/column_lineage.py +69 -0
- interlaced-2.0.0a2/src/interlace/graph/dag.py +81 -0
- interlaced-2.0.0a2/src/interlace/graph/project.py +242 -0
- interlaced-2.0.0a2/src/interlace/graph/selectors.py +45 -0
- interlaced-2.0.0a2/src/interlace/ir/__init__.py +24 -0
- interlaced-2.0.0a2/src/interlace/ir/canonicalize.py +71 -0
- interlaced-2.0.0a2/src/interlace/ir/fingerprint.py +55 -0
- interlaced-2.0.0a2/src/interlace/ir/relation.py +71 -0
- interlaced-2.0.0a2/src/interlace/ir/schema.py +23 -0
- interlaced-2.0.0a2/src/interlace/plan/__init__.py +23 -0
- interlaced-2.0.0a2/src/interlace/plan/apply.py +527 -0
- interlaced-2.0.0a2/src/interlace/plan/differ.py +211 -0
- interlaced-2.0.0a2/src/interlace/plan/plan.py +177 -0
- interlaced-2.0.0a2/src/interlace/plan/resolve.py +74 -0
- interlaced-2.0.0a2/src/interlace/plan/run.py +79 -0
- interlaced-2.0.0a2/src/interlace/project.py +278 -0
- interlaced-2.0.0a2/src/interlace/py.typed +0 -0
- interlaced-2.0.0a2/src/interlace/runtime/__init__.py +6 -0
- interlaced-2.0.0a2/src/interlace/runtime/handles.py +48 -0
- interlaced-2.0.0a2/src/interlace/runtime/python_model.py +175 -0
- interlaced-2.0.0a2/src/interlace/scaffold.py +83 -0
- interlaced-2.0.0a2/src/interlace/scheduler/__init__.py +17 -0
- interlaced-2.0.0a2/src/interlace/scheduler/engine.py +66 -0
- interlaced-2.0.0a2/src/interlace/scheduler/triggers.py +84 -0
- interlaced-2.0.0a2/src/interlace/scheduler/worker.py +154 -0
- interlaced-2.0.0a2/src/interlace/service/__init__.py +7 -0
- interlaced-2.0.0a2/src/interlace/service/app.py +836 -0
- interlaced-2.0.0a2/src/interlace/service/auth.py +41 -0
- interlaced-2.0.0a2/src/interlace/state/__init__.py +18 -0
- interlaced-2.0.0a2/src/interlace/state/interval.py +134 -0
- interlaced-2.0.0a2/src/interlace/state/janitor.py +142 -0
- interlaced-2.0.0a2/src/interlace/state/snapshot.py +47 -0
- interlaced-2.0.0a2/src/interlace/state/store.py +906 -0
- interlaced-2.0.0a2/src/interlace/strategies/__init__.py +64 -0
- interlaced-2.0.0a2/src/interlace/strategies/base.py +71 -0
- interlaced-2.0.0a2/src/interlace/strategies/full.py +38 -0
- interlaced-2.0.0a2/src/interlace/strategies/full_merge.py +98 -0
- interlaced-2.0.0a2/src/interlace/strategies/incremental_by_time.py +65 -0
- interlaced-2.0.0a2/src/interlace/strategies/merge_by_key.py +69 -0
- interlaced-2.0.0a2/src/interlace/strategies/scd_type_2.py +122 -0
- interlaced-2.0.0a2/src/interlace/strategies/view.py +33 -0
- interlaced-2.0.0a2/src/interlace/streaming/__init__.py +18 -0
- interlaced-2.0.0a2/src/interlace/streaming/log.py +298 -0
- interlaced-2.0.0a2/src/interlace/streaming/materializer.py +166 -0
- interlaced-2.0.0a2/src/interlace/streaming/schema.py +218 -0
- interlaced-2.0.0a2/src/interlaced.egg-info/PKG-INFO +216 -0
- interlaced-2.0.0a2/src/interlaced.egg-info/SOURCES.txt +120 -0
- interlaced-2.0.0a2/src/interlaced.egg-info/dependency_links.txt +1 -0
- interlaced-2.0.0a2/src/interlaced.egg-info/entry_points.txt +2 -0
- interlaced-2.0.0a2/src/interlaced.egg-info/requires.txt +42 -0
- interlaced-2.0.0a2/src/interlaced.egg-info/top_level.txt +1 -0
- interlaced-2.0.0a2/tests/test_apply.py +199 -0
- interlaced-2.0.0a2/tests/test_checks.py +189 -0
- interlaced-2.0.0a2/tests/test_cli.py +102 -0
- interlaced-2.0.0a2/tests/test_cli_inspect.py +163 -0
- interlaced-2.0.0a2/tests/test_column_lineage.py +73 -0
- interlaced-2.0.0a2/tests/test_compile.py +74 -0
- interlaced-2.0.0a2/tests/test_contracts.py +35 -0
- interlaced-2.0.0a2/tests/test_dag.py +48 -0
- interlaced-2.0.0a2/tests/test_differ.py +112 -0
- interlaced-2.0.0a2/tests/test_discovery.py +63 -0
- interlaced-2.0.0a2/tests/test_dsl.py +73 -0
- interlaced-2.0.0a2/tests/test_duckdb_adapter.py +119 -0
- interlaced-2.0.0a2/tests/test_ephemeral.py +81 -0
- interlaced-2.0.0a2/tests/test_example.py +46 -0
- interlaced-2.0.0a2/tests/test_example_benchmark.py +44 -0
- interlaced-2.0.0a2/tests/test_example_platform_tour.py +64 -0
- interlaced-2.0.0a2/tests/test_exports.py +91 -0
- interlaced-2.0.0a2/tests/test_fingerprint.py +42 -0
- interlaced-2.0.0a2/tests/test_forward_only.py +207 -0
- interlaced-2.0.0a2/tests/test_incremental.py +170 -0
- interlaced-2.0.0a2/tests/test_incremental_python.py +225 -0
- interlaced-2.0.0a2/tests/test_init.py +52 -0
- interlaced-2.0.0a2/tests/test_interval.py +74 -0
- interlaced-2.0.0a2/tests/test_janitor.py +154 -0
- interlaced-2.0.0a2/tests/test_multi_engine.py +203 -0
- interlaced-2.0.0a2/tests/test_postgres_engine.py +198 -0
- interlaced-2.0.0a2/tests/test_python_models.py +197 -0
- interlaced-2.0.0a2/tests/test_rebuild_skip.py +160 -0
- interlaced-2.0.0a2/tests/test_run.py +108 -0
- interlaced-2.0.0a2/tests/test_scd2.py +109 -0
- interlaced-2.0.0a2/tests/test_scheduler.py +120 -0
- interlaced-2.0.0a2/tests/test_schema_evolution.py +122 -0
- interlaced-2.0.0a2/tests/test_selectors.py +53 -0
- interlaced-2.0.0a2/tests/test_service.py +346 -0
- interlaced-2.0.0a2/tests/test_sql_config.py +66 -0
- interlaced-2.0.0a2/tests/test_state_store.py +97 -0
- interlaced-2.0.0a2/tests/test_storage.py +145 -0
- interlaced-2.0.0a2/tests/test_strategies.py +107 -0
- interlaced-2.0.0a2/tests/test_streaming.py +230 -0
- interlaced-2.0.0a2/tests/test_table_sinks.py +169 -0
- interlaced-2.0.0a2/tests/test_transfers.py +201 -0
- interlaced-2.0.0a2/tests/test_warehouse_config.py +238 -0
- interlaced-2.0.0a2/tests/test_worker_leases.py +175 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025-2026 Interlace Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: interlaced
|
|
3
|
+
Version: 2.0.0a2
|
|
4
|
+
Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
|
|
5
|
+
Author-email: Mark <mark@interlace.sh>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: data,pipeline,orchestration,transformation,etl,streaming,dbt,sqlmesh,duckdb
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Topic :: Software Development :: Build Tools
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: SQL
|
|
13
|
+
Requires-Python: >=3.12
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: sqlglot<31.0,>=25.0
|
|
17
|
+
Requires-Dist: duckdb>=1.5.3
|
|
18
|
+
Requires-Dist: pyarrow>=17.0
|
|
19
|
+
Requires-Dist: msgspec<1.0,>=0.18
|
|
20
|
+
Requires-Dist: pydantic<3.0,>=2.5
|
|
21
|
+
Requires-Dist: typer<1.0,>=0.12
|
|
22
|
+
Requires-Dist: rich<15.0,>=13.0
|
|
23
|
+
Requires-Dist: cronsim<3.0,>=2.5
|
|
24
|
+
Requires-Dist: tenacity<10.0,>=8.2
|
|
25
|
+
Requires-Dist: structlog<26.0,>=24.0
|
|
26
|
+
Requires-Dist: pyyaml<7.0,>=6.0
|
|
27
|
+
Provides-Extra: service
|
|
28
|
+
Requires-Dist: litestar<3.0,>=2.12; extra == "service"
|
|
29
|
+
Requires-Dist: uvicorn<1.0,>=0.30; extra == "service"
|
|
30
|
+
Requires-Dist: httpx<1.0,>=0.27; extra == "service"
|
|
31
|
+
Requires-Dist: argon2-cffi<26.0,>=23.1; extra == "service"
|
|
32
|
+
Requires-Dist: joserfc<2.0,>=1.0; extra == "service"
|
|
33
|
+
Requires-Dist: watchfiles<2.0,>=0.22; extra == "service"
|
|
34
|
+
Provides-Extra: adbc
|
|
35
|
+
Requires-Dist: adbc-driver-manager<2.0,>=1.2; extra == "adbc"
|
|
36
|
+
Requires-Dist: adbc-driver-postgresql<2.0,>=1.2; extra == "adbc"
|
|
37
|
+
Provides-Extra: postgres
|
|
38
|
+
Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "postgres"
|
|
39
|
+
Provides-Extra: polars
|
|
40
|
+
Requires-Dist: polars<2.0,>=1.0; extra == "polars"
|
|
41
|
+
Provides-Extra: pandas
|
|
42
|
+
Requires-Dist: pandas<4.0,>=2.0; extra == "pandas"
|
|
43
|
+
Provides-Extra: all
|
|
44
|
+
Requires-Dist: interlaced[adbc,polars,postgres,service]; extra == "all"
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: pytest<10.0,>=8.0; extra == "dev"
|
|
47
|
+
Requires-Dist: pytest-asyncio<2.0,>=1.0; extra == "dev"
|
|
48
|
+
Requires-Dist: ruff<1.0,>=0.6; extra == "dev"
|
|
49
|
+
Requires-Dist: black<27.0,>=24.0; extra == "dev"
|
|
50
|
+
Requires-Dist: mypy<2.0,>=1.11; extra == "dev"
|
|
51
|
+
Dynamic: license-file
|
|
52
|
+
|
|
53
|
+
# interlace
|
|
54
|
+
|
|
55
|
+
**Python/SQL-first data platform: transformation, orchestration, and durable streaming — one process.**
|
|
56
|
+
|
|
57
|
+
interlace is an independent, MIT-licensed alternative to dbt/SQLMesh that also replaces the
|
|
58
|
+
orchestrator (no Airflow) and the ingestion layer (Cloudflare-Pipelines-style durable streams).
|
|
59
|
+
Models are `.sql` files or Python functions; state is versioned snapshots with virtual
|
|
60
|
+
environments and a terraform-style plan/apply; everything runs in a single daemon on
|
|
61
|
+
DuckDB + DuckLake by default.
|
|
62
|
+
|
|
63
|
+
> **Status: v2 pre-release.** This branch is a ground-up rebuild (the `interlace` package on
|
|
64
|
+
> PyPI is the older 0.x line). APIs may still move. Requires Python 3.12+.
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv pip install "interlaced[service]" # or from source: "interlaced[service] @ git+https://github.com/interlace-sh/interlace@v2"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Sixty seconds
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
interlace init my-project && cd my-project
|
|
74
|
+
interlace plan # terraform-style preview: added / breaking / non-breaking / reuse
|
|
75
|
+
interlace apply # build changed models, run checks, promote the environment
|
|
76
|
+
interlace serve # the daemon: HTTP API + scheduler + stream ingestion, one process
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Every model builds into a fingerprinted physical table (`interlace__main.orders__a1b2c3`);
|
|
80
|
+
environments are views over those tables, so promotion and rollback are atomic view swaps and a
|
|
81
|
+
dev environment reuses prod's tables for free. **Production is the unprefixed namespace** —
|
|
82
|
+
consumers query `main.orders`; sandboxes are prefixed (`dev__main.orders`). Commands default to
|
|
83
|
+
prod; pass `--env dev` while developing.
|
|
84
|
+
|
|
85
|
+
## Models
|
|
86
|
+
|
|
87
|
+
**SQL** — a file per model; upstreams referenced by model name, dependencies inferred by parsing
|
|
88
|
+
(sqlglot), config in a leading comment block:
|
|
89
|
+
|
|
90
|
+
```sql
|
|
91
|
+
/* interlace:
|
|
92
|
+
strategy: scd_type_2
|
|
93
|
+
key: customer_id
|
|
94
|
+
schedule: {cron: "0 * * * *"}
|
|
95
|
+
checks:
|
|
96
|
+
- not_null: customer_id
|
|
97
|
+
- unique: customer_id
|
|
98
|
+
*/
|
|
99
|
+
SELECT customer_id, name, tier FROM raw_customers
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
**Python** — functions whose parameters name their upstreams; data crosses as Arrow
|
|
103
|
+
(never pandas), streamed with bounded memory:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from interlace import model
|
|
107
|
+
|
|
108
|
+
@model(strategy="merge_by_key", key="order_id", cursor="updated_at")
|
|
109
|
+
def orders(cursor, this):
|
|
110
|
+
"""Incremental API extract: `cursor` is max(updated_at) already in the
|
|
111
|
+
warehouse (None on first run); `this` is the previous materialisation."""
|
|
112
|
+
rows = fetch_orders(since=cursor) # your code
|
|
113
|
+
return pyarrow.Table.from_pylist(rows) # or RecordBatchReader / generator of batches
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
**Strategies:** `full`, `view`, `ephemeral` (CTE-inlined), `merge_by_key` (upsert),
|
|
117
|
+
`full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
|
|
118
|
+
(windowed, interval-ledger backfill/catchup), `scd_type_2` (history with validity windows).
|
|
119
|
+
|
|
120
|
+
## Plan / apply
|
|
121
|
+
|
|
122
|
+
```
|
|
123
|
+
$ interlace plan
|
|
124
|
+
Model Change Category Build
|
|
125
|
+
orders modified non_breaking rebuild
|
|
126
|
+
order_stats modified non_breaking reuse <- output provably identical: not rebuilt
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
- Changes classify **breaking / non-breaking / forward-only**; downstream models whose output
|
|
130
|
+
is provably identical **reuse their existing tables** instead of rebuilding (an improvement
|
|
131
|
+
over model-granular invalidation).
|
|
132
|
+
- `apply --forward-only` lets history-keeping models (scd2/merge/incremental) survive a
|
|
133
|
+
definition change: the new logic inherits the existing table and applies going forward.
|
|
134
|
+
- **Checks gate promotion**: 10 built-in types (not_null, unique, accepted_values, row_count,
|
|
135
|
+
freshness, expression, relationships, pattern, range, sql) plus `@check` Python functions —
|
|
136
|
+
an error-severity failure blocks before the environment view moves.
|
|
137
|
+
- `interlace gc` removes snapshots no environment references (reference-aware: tables shared
|
|
138
|
+
through reuse survive).
|
|
139
|
+
|
|
140
|
+
## Streaming
|
|
141
|
+
|
|
142
|
+
Declare a stream; POST to it; rows are durable before the 200, deduplicated by idempotency
|
|
143
|
+
key, and materialized exactly-once into `streams.<name>` — where SQL models just read them.
|
|
144
|
+
A flush triggers the models that consume the stream.
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from interlace import stream
|
|
148
|
+
|
|
149
|
+
@stream("orders", schema={"order_id": "string", "total": "double"},
|
|
150
|
+
idempotency_key="order_id", retention="7d", on_schema_drift="evolve")
|
|
151
|
+
def orders(event): ...
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
curl -X POST localhost:8000/streams/orders -d '{"order_id": "o1", "total": 49.5}'
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Schema drift is yours to choose: `reject` (400), `evolve` (new columns appear), or
|
|
159
|
+
`quarantine` (bad events divert to `<stream>__quarantine`).
|
|
160
|
+
|
|
161
|
+
## Reverse ETL
|
|
162
|
+
|
|
163
|
+
Attach external databases and deliver model results into them — the live table is never
|
|
164
|
+
dropped, keyed modes reuse the same merge strategies:
|
|
165
|
+
|
|
166
|
+
```yaml
|
|
167
|
+
# interlace.yaml
|
|
168
|
+
attach:
|
|
169
|
+
crm: "postgres:host=... dbname=crm"
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
```sql
|
|
173
|
+
/* interlace: {export: {to: table, target: crm.public.accounts, mode: merge_by_key, key: id}} */
|
|
174
|
+
SELECT id, tier, lifetime_value FROM account_summary
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
File exports (`to: parquet|csv|json`) work the same way.
|
|
178
|
+
|
|
179
|
+
## The daemon
|
|
180
|
+
|
|
181
|
+
`interlace serve` runs the HTTP API (Litestar + msgspec, OpenAPI at `/schema/scalar`), the
|
|
182
|
+
scheduler (cron/interval triggers → durable run queue), stream ingestion, and retention in one
|
|
183
|
+
process. Scoped API keys (`interlace apikey create ci --scope read`) lock it down; a durable
|
|
184
|
+
event log backs `GET /events/stream` (SSE with replay).
|
|
185
|
+
|
|
186
|
+
Add `--quack quack:localhost:4213` to serve the warehouse itself over DuckDB's quack protocol —
|
|
187
|
+
other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by setting
|
|
188
|
+
`database: quack:localhost:4213`.
|
|
189
|
+
|
|
190
|
+
## Architecture in five lines
|
|
191
|
+
|
|
192
|
+
- The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
|
|
193
|
+
are AST builders and dialect appears only at `transpile()`.
|
|
194
|
+
- Storage defaults to **DuckLake** (Parquet + SQL catalog) opened as DuckDB's primary database.
|
|
195
|
+
- Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
|
|
196
|
+
scale-out swap.
|
|
197
|
+
- Streams live in their own durable log; the materializer commits data + watermark in one
|
|
198
|
+
warehouse transaction — exactly-once without distributed coordination.
|
|
199
|
+
- No Jinja, no pandas in core, no external orchestrator.
|
|
200
|
+
|
|
201
|
+
The full design rationale lives in `docs/architecture/v2-design.md`.
|
|
202
|
+
|
|
203
|
+
## Development
|
|
204
|
+
|
|
205
|
+
Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
|
|
206
|
+
[moon](https://moonrepo.dev/moon), `uv` owns the virtualenv:
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
proto install
|
|
210
|
+
moon run interlace:sync # install deps
|
|
211
|
+
moon run interlace:test # 240+ tests
|
|
212
|
+
moon run interlace:check # black + ruff (CI equivalent)
|
|
213
|
+
moon run interlace:typecheck # mypy --strict
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
MIT licensed.
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# interlace
|
|
2
|
+
|
|
3
|
+
**Python/SQL-first data platform: transformation, orchestration, and durable streaming — one process.**
|
|
4
|
+
|
|
5
|
+
interlace is an independent, MIT-licensed alternative to dbt/SQLMesh that also replaces the
|
|
6
|
+
orchestrator (no Airflow) and the ingestion layer (Cloudflare-Pipelines-style durable streams).
|
|
7
|
+
Models are `.sql` files or Python functions; state is versioned snapshots with virtual
|
|
8
|
+
environments and a terraform-style plan/apply; everything runs in a single daemon on
|
|
9
|
+
DuckDB + DuckLake by default.
|
|
10
|
+
|
|
11
|
+
> **Status: v2 pre-release.** This branch is a ground-up rebuild (the `interlace` package on
|
|
12
|
+
> PyPI is the older 0.x line). APIs may still move. Requires Python 3.12+.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
uv pip install "interlaced[service]" # or from source: "interlaced[service] @ git+https://github.com/interlace-sh/interlace@v2"
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Sixty seconds
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
interlace init my-project && cd my-project
|
|
22
|
+
interlace plan # terraform-style preview: added / breaking / non-breaking / reuse
|
|
23
|
+
interlace apply # build changed models, run checks, promote the environment
|
|
24
|
+
interlace serve # the daemon: HTTP API + scheduler + stream ingestion, one process
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Every model builds into a fingerprinted physical table (`interlace__main.orders__a1b2c3`);
|
|
28
|
+
environments are views over those tables, so promotion and rollback are atomic view swaps and a
|
|
29
|
+
dev environment reuses prod's tables for free. **Production is the unprefixed namespace** —
|
|
30
|
+
consumers query `main.orders`; sandboxes are prefixed (`dev__main.orders`). Commands default to
|
|
31
|
+
prod; pass `--env dev` while developing.
|
|
32
|
+
|
|
33
|
+
## Models
|
|
34
|
+
|
|
35
|
+
**SQL** — a file per model; upstreams referenced by model name, dependencies inferred by parsing
|
|
36
|
+
(sqlglot), config in a leading comment block:
|
|
37
|
+
|
|
38
|
+
```sql
|
|
39
|
+
/* interlace:
|
|
40
|
+
strategy: scd_type_2
|
|
41
|
+
key: customer_id
|
|
42
|
+
schedule: {cron: "0 * * * *"}
|
|
43
|
+
checks:
|
|
44
|
+
- not_null: customer_id
|
|
45
|
+
- unique: customer_id
|
|
46
|
+
*/
|
|
47
|
+
SELECT customer_id, name, tier FROM raw_customers
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
**Python** — functions whose parameters name their upstreams; data crosses as Arrow
|
|
51
|
+
(never pandas), streamed with bounded memory:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from interlace import model
|
|
55
|
+
|
|
56
|
+
@model(strategy="merge_by_key", key="order_id", cursor="updated_at")
|
|
57
|
+
def orders(cursor, this):
|
|
58
|
+
"""Incremental API extract: `cursor` is max(updated_at) already in the
|
|
59
|
+
warehouse (None on first run); `this` is the previous materialisation."""
|
|
60
|
+
rows = fetch_orders(since=cursor) # your code
|
|
61
|
+
return pyarrow.Table.from_pylist(rows) # or RecordBatchReader / generator of batches
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
**Strategies:** `full`, `view`, `ephemeral` (CTE-inlined), `merge_by_key` (upsert),
|
|
65
|
+
`full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
|
|
66
|
+
(windowed, interval-ledger backfill/catchup), `scd_type_2` (history with validity windows).
|
|
67
|
+
|
|
68
|
+
## Plan / apply
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
$ interlace plan
|
|
72
|
+
Model Change Category Build
|
|
73
|
+
orders modified non_breaking rebuild
|
|
74
|
+
order_stats modified non_breaking reuse <- output provably identical: not rebuilt
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
- Changes classify **breaking / non-breaking / forward-only**; downstream models whose output
|
|
78
|
+
is provably identical **reuse their existing tables** instead of rebuilding (an improvement
|
|
79
|
+
over model-granular invalidation).
|
|
80
|
+
- `apply --forward-only` lets history-keeping models (scd2/merge/incremental) survive a
|
|
81
|
+
definition change: the new logic inherits the existing table and applies going forward.
|
|
82
|
+
- **Checks gate promotion**: 10 built-in types (not_null, unique, accepted_values, row_count,
|
|
83
|
+
freshness, expression, relationships, pattern, range, sql) plus `@check` Python functions —
|
|
84
|
+
an error-severity failure blocks before the environment view moves.
|
|
85
|
+
- `interlace gc` removes snapshots no environment references (reference-aware: tables shared
|
|
86
|
+
through reuse survive).
|
|
87
|
+
|
|
88
|
+
## Streaming
|
|
89
|
+
|
|
90
|
+
Declare a stream; POST to it; rows are durable before the 200, deduplicated by idempotency
|
|
91
|
+
key, and materialized exactly-once into `streams.<name>` — where SQL models just read them.
|
|
92
|
+
A flush triggers the models that consume the stream.
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from interlace import stream
|
|
96
|
+
|
|
97
|
+
@stream("orders", schema={"order_id": "string", "total": "double"},
|
|
98
|
+
idempotency_key="order_id", retention="7d", on_schema_drift="evolve")
|
|
99
|
+
def orders(event): ...
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
curl -X POST localhost:8000/streams/orders -d '{"order_id": "o1", "total": 49.5}'
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Schema drift is yours to choose: `reject` (400), `evolve` (new columns appear), or
|
|
107
|
+
`quarantine` (bad events divert to `<stream>__quarantine`).
|
|
108
|
+
|
|
109
|
+
## Reverse ETL
|
|
110
|
+
|
|
111
|
+
Attach external databases and deliver model results into them — the live table is never
|
|
112
|
+
dropped, keyed modes reuse the same merge strategies:
|
|
113
|
+
|
|
114
|
+
```yaml
|
|
115
|
+
# interlace.yaml
|
|
116
|
+
attach:
|
|
117
|
+
crm: "postgres:host=... dbname=crm"
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
```sql
|
|
121
|
+
/* interlace: {export: {to: table, target: crm.public.accounts, mode: merge_by_key, key: id}} */
|
|
122
|
+
SELECT id, tier, lifetime_value FROM account_summary
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
File exports (`to: parquet|csv|json`) work the same way.
|
|
126
|
+
|
|
127
|
+
## The daemon
|
|
128
|
+
|
|
129
|
+
`interlace serve` runs the HTTP API (Litestar + msgspec, OpenAPI at `/schema/scalar`), the
|
|
130
|
+
scheduler (cron/interval triggers → durable run queue), stream ingestion, and retention in one
|
|
131
|
+
process. Scoped API keys (`interlace apikey create ci --scope read`) lock it down; a durable
|
|
132
|
+
event log backs `GET /events/stream` (SSE with replay).
|
|
133
|
+
|
|
134
|
+
Add `--quack quack:localhost:4213` to serve the warehouse itself over DuckDB's quack protocol —
|
|
135
|
+
other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by setting
|
|
136
|
+
`database: quack:localhost:4213`.
|
|
137
|
+
|
|
138
|
+
## Architecture in five lines
|
|
139
|
+
|
|
140
|
+
- The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
|
|
141
|
+
are AST builders and dialect appears only at `transpile()`.
|
|
142
|
+
- Storage defaults to **DuckLake** (Parquet + SQL catalog) opened as DuckDB's primary database.
|
|
143
|
+
- Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
|
|
144
|
+
scale-out swap.
|
|
145
|
+
- Streams live in their own durable log; the materializer commits data + watermark in one
|
|
146
|
+
warehouse transaction — exactly-once without distributed coordination.
|
|
147
|
+
- No Jinja, no pandas in core, no external orchestrator.
|
|
148
|
+
|
|
149
|
+
The full design rationale lives in `docs/architecture/v2-design.md`.
|
|
150
|
+
|
|
151
|
+
## Development
|
|
152
|
+
|
|
153
|
+
Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
|
|
154
|
+
[moon](https://moonrepo.dev/moon), `uv` owns the virtualenv:
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
proto install
|
|
158
|
+
moon run interlace:sync # install deps
|
|
159
|
+
moon run interlace:test # 240+ tests
|
|
160
|
+
moon run interlace:check # black + ruff (CI equivalent)
|
|
161
|
+
moon run interlace:typecheck # mypy --strict
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
MIT licensed.
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "interlaced"
|
|
7
|
+
version = "2.0.0a2"
|
|
8
|
+
description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Mark", email = "mark@interlace.sh"}
|
|
14
|
+
]
|
|
15
|
+
keywords = ["data", "pipeline", "orchestration", "transformation", "etl", "streaming", "dbt", "sqlmesh", "duckdb"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Topic :: Software Development :: Build Tools",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: SQL",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
# Core: the IR + engine + state + plan spine (Phase 1). Everything is Arrow- and
|
|
25
|
+
# sqlglot-native. See docs/architecture/v2-design.md.
|
|
26
|
+
dependencies = [
|
|
27
|
+
"sqlglot>=25.0,<31.0", # canonical IR, transpilation, semantic diff, column lineage
|
|
28
|
+
"duckdb>=1.5.3", # default engine + federation hub; DuckLake storage + quack serving
|
|
29
|
+
"pyarrow>=17.0", # the wire format: RecordBatchReader everywhere
|
|
30
|
+
"msgspec>=0.18,<1.0", # ingest validation + fast (de)serialisation
|
|
31
|
+
"pydantic>=2.5,<3.0", # config + manifest validation (cold paths only)
|
|
32
|
+
"typer>=0.12,<1.0", # CLI
|
|
33
|
+
"rich>=13.0,<15.0", # display, strictly an event subscriber
|
|
34
|
+
"cronsim>=2.5,<3.0", # cron parsing for the trigger engine
|
|
35
|
+
"tenacity>=8.2,<10.0", # retries (tasks, DuckLake commit conflicts, transfers)
|
|
36
|
+
"structlog>=24.0,<26.0", # structured logging
|
|
37
|
+
"pyyaml>=6.0,<7.0", # project config (config + env overlays)
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
interlace = "interlace.cli.main:main"
|
|
42
|
+
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
# Service + orchestration daemon (Phase 2).
|
|
45
|
+
service = [
|
|
46
|
+
"litestar>=2.12,<3.0",
|
|
47
|
+
"uvicorn>=0.30,<1.0",
|
|
48
|
+
"httpx>=0.27,<1.0",
|
|
49
|
+
"argon2-cffi>=23.1,<26.0",
|
|
50
|
+
"joserfc>=1.0,<2.0",
|
|
51
|
+
"watchfiles>=0.22,<2.0",
|
|
52
|
+
]
|
|
53
|
+
# Remote engines via Arrow-native transfer (Phase 4).
|
|
54
|
+
adbc = [
|
|
55
|
+
"adbc-driver-manager>=1.2,<2.0",
|
|
56
|
+
"adbc-driver-postgresql>=1.2,<2.0",
|
|
57
|
+
]
|
|
58
|
+
postgres = ["psycopg[binary]>=3.1,<4.0"]
|
|
59
|
+
polars = ["polars>=1.0,<2.0"]
|
|
60
|
+
pandas = ["pandas>=2.0,<4.0"]
|
|
61
|
+
all = ["interlaced[service,adbc,postgres,polars]"]
|
|
62
|
+
dev = [
|
|
63
|
+
"pytest>=8.0,<10.0",
|
|
64
|
+
"pytest-asyncio>=1.0,<2.0",
|
|
65
|
+
"ruff>=0.6,<1.0",
|
|
66
|
+
"black>=24.0,<27.0",
|
|
67
|
+
"mypy>=1.11,<2.0",
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
[tool.setuptools.packages.find]
|
|
71
|
+
where = ["src"]
|
|
72
|
+
|
|
73
|
+
[tool.setuptools.package-data]
|
|
74
|
+
interlace = ["py.typed"]
|
|
75
|
+
|
|
76
|
+
[tool.black]
|
|
77
|
+
line-length = 120
|
|
78
|
+
target-version = ['py312']
|
|
79
|
+
|
|
80
|
+
[tool.ruff]
|
|
81
|
+
line-length = 120
|
|
82
|
+
target-version = "py312"
|
|
83
|
+
|
|
84
|
+
[tool.ruff.lint]
|
|
85
|
+
select = [
|
|
86
|
+
"E", # pycodestyle errors
|
|
87
|
+
"W", # pycodestyle warnings
|
|
88
|
+
"F", # pyflakes
|
|
89
|
+
"I", # isort
|
|
90
|
+
"B", # flake8-bugbear
|
|
91
|
+
"C4", # flake8-comprehensions
|
|
92
|
+
"UP", # pyupgrade
|
|
93
|
+
]
|
|
94
|
+
ignore = [
|
|
95
|
+
"E501", # line too long (handled by formatter)
|
|
96
|
+
"B008", # function calls in argument defaults
|
|
97
|
+
"C901", # too complex
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
[tool.ruff.lint.per-file-ignores]
|
|
101
|
+
"__init__.py" = ["F401"] # Allow unused imports in __init__.py
|
|
102
|
+
|
|
103
|
+
[tool.mypy]
|
|
104
|
+
python_version = "3.12"
|
|
105
|
+
warn_return_any = true
|
|
106
|
+
warn_unused_configs = true
|
|
107
|
+
disallow_untyped_defs = true
|
|
108
|
+
disallow_incomplete_defs = true
|
|
109
|
+
check_untyped_defs = true
|
|
110
|
+
no_implicit_optional = true
|
|
111
|
+
warn_redundant_casts = true
|
|
112
|
+
warn_unused_ignores = true
|
|
113
|
+
|
|
114
|
+
[[tool.mypy.overrides]]
|
|
115
|
+
module = [
|
|
116
|
+
"sqlglot.*",
|
|
117
|
+
"pyarrow.*",
|
|
118
|
+
"yaml",
|
|
119
|
+
]
|
|
120
|
+
ignore_missing_imports = true
|
|
121
|
+
|
|
122
|
+
[tool.coverage.run]
|
|
123
|
+
source = ["src/interlace"]
|
|
124
|
+
omit = [
|
|
125
|
+
"*/tests/*",
|
|
126
|
+
"*/test_*.py",
|
|
127
|
+
"*/__pycache__/*",
|
|
128
|
+
"*/__init__.py",
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
[tool.coverage.report]
|
|
132
|
+
exclude_lines = [
|
|
133
|
+
"pragma: no cover",
|
|
134
|
+
"def __repr__",
|
|
135
|
+
"raise AssertionError",
|
|
136
|
+
"raise NotImplementedError",
|
|
137
|
+
"if __name__ == .__main__.:",
|
|
138
|
+
"if TYPE_CHECKING:",
|
|
139
|
+
"@abstractmethod",
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
[tool.pytest.ini_options]
|
|
143
|
+
testpaths = ["tests"]
|
|
144
|
+
python_files = ["test_*.py", "*_test.py"]
|
|
145
|
+
python_classes = ["Test*"]
|
|
146
|
+
python_functions = ["test_*"]
|
|
147
|
+
addopts = [
|
|
148
|
+
"-v",
|
|
149
|
+
"--strict-markers",
|
|
150
|
+
"--tb=short",
|
|
151
|
+
]
|
|
152
|
+
markers = [
|
|
153
|
+
"unit: Unit tests (fast, isolated)",
|
|
154
|
+
"integration: Integration tests (slower, require database)",
|
|
155
|
+
"slow: Slow tests (may take a long time)",
|
|
156
|
+
"requires_db: Needs a reachable external database (e.g. Postgres)",
|
|
157
|
+
]
|
|
158
|
+
asyncio_mode = "auto"
|
|
159
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Interlace v2 — Python/SQL-first data platform.
|
|
2
|
+
|
|
3
|
+
Transformation (sqlmesh-grade snapshots, virtual environments, plan/apply),
|
|
4
|
+
built-in orchestration (durable work queue + unified triggers), and durable
|
|
5
|
+
streaming ingestion — in one process. See docs/architecture/v2-design.md.
|
|
6
|
+
|
|
7
|
+
This package is under active greenfield construction; the public surface is the
|
|
8
|
+
``@model`` / ``@stream`` / ``@check`` decorators plus the core IR types.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from interlace.dsl.decorators import check, model, stream
|
|
14
|
+
from interlace.ir.relation import EngineRef, SqlRelation, TableRef
|
|
15
|
+
|
|
16
|
+
__version__ = "2.0.0a2"
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"EngineRef",
|
|
20
|
+
"SqlRelation",
|
|
21
|
+
"TableRef",
|
|
22
|
+
"__version__",
|
|
23
|
+
"check",
|
|
24
|
+
"model",
|
|
25
|
+
"stream",
|
|
26
|
+
]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Data-quality checks. Results gate promotion: an ``error``-severity failure
|
|
2
|
+
aborts the apply before the environment is promoted.
|
|
3
|
+
|
|
4
|
+
Import :mod:`interlace.checks.runner` for execution — kept out of this package
|
|
5
|
+
init so declaring checks (spec) never drags in the runtime.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from interlace.checks.spec import CheckSpec, parse_checks
|
|
9
|
+
|
|
10
|
+
__all__ = ["CheckSpec", "parse_checks"]
|