interlaced 2.0.0a2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. interlaced-2.0.0a2/LICENSE +21 -0
  2. interlaced-2.0.0a2/PKG-INFO +216 -0
  3. interlaced-2.0.0a2/README.md +164 -0
  4. interlaced-2.0.0a2/pyproject.toml +159 -0
  5. interlaced-2.0.0a2/setup.cfg +4 -0
  6. interlaced-2.0.0a2/src/interlace/__init__.py +26 -0
  7. interlaced-2.0.0a2/src/interlace/checks/__init__.py +10 -0
  8. interlaced-2.0.0a2/src/interlace/checks/builtin.py +152 -0
  9. interlaced-2.0.0a2/src/interlace/checks/runner.py +103 -0
  10. interlaced-2.0.0a2/src/interlace/checks/spec.py +113 -0
  11. interlaced-2.0.0a2/src/interlace/cli/__init__.py +7 -0
  12. interlaced-2.0.0a2/src/interlace/cli/main.py +1072 -0
  13. interlaced-2.0.0a2/src/interlace/config/__init__.py +7 -0
  14. interlaced-2.0.0a2/src/interlace/config/config.py +172 -0
  15. interlaced-2.0.0a2/src/interlace/contracts.py +35 -0
  16. interlaced-2.0.0a2/src/interlace/dsl/__init__.py +16 -0
  17. interlaced-2.0.0a2/src/interlace/dsl/decorators.py +222 -0
  18. interlaced-2.0.0a2/src/interlace/dsl/discovery.py +75 -0
  19. interlaced-2.0.0a2/src/interlace/dsl/sql_config.py +47 -0
  20. interlaced-2.0.0a2/src/interlace/engines/__init__.py +16 -0
  21. interlaced-2.0.0a2/src/interlace/engines/base.py +86 -0
  22. interlaced-2.0.0a2/src/interlace/engines/duckdb.py +249 -0
  23. interlaced-2.0.0a2/src/interlace/engines/postgres.py +157 -0
  24. interlaced-2.0.0a2/src/interlace/engines/quack.py +113 -0
  25. interlaced-2.0.0a2/src/interlace/engines/registry.py +125 -0
  26. interlaced-2.0.0a2/src/interlace/exceptions.py +66 -0
  27. interlaced-2.0.0a2/src/interlace/exports.py +145 -0
  28. interlaced-2.0.0a2/src/interlace/graph/__init__.py +17 -0
  29. interlaced-2.0.0a2/src/interlace/graph/column_lineage.py +69 -0
  30. interlaced-2.0.0a2/src/interlace/graph/dag.py +81 -0
  31. interlaced-2.0.0a2/src/interlace/graph/project.py +242 -0
  32. interlaced-2.0.0a2/src/interlace/graph/selectors.py +45 -0
  33. interlaced-2.0.0a2/src/interlace/ir/__init__.py +24 -0
  34. interlaced-2.0.0a2/src/interlace/ir/canonicalize.py +71 -0
  35. interlaced-2.0.0a2/src/interlace/ir/fingerprint.py +55 -0
  36. interlaced-2.0.0a2/src/interlace/ir/relation.py +71 -0
  37. interlaced-2.0.0a2/src/interlace/ir/schema.py +23 -0
  38. interlaced-2.0.0a2/src/interlace/plan/__init__.py +23 -0
  39. interlaced-2.0.0a2/src/interlace/plan/apply.py +527 -0
  40. interlaced-2.0.0a2/src/interlace/plan/differ.py +211 -0
  41. interlaced-2.0.0a2/src/interlace/plan/plan.py +177 -0
  42. interlaced-2.0.0a2/src/interlace/plan/resolve.py +74 -0
  43. interlaced-2.0.0a2/src/interlace/plan/run.py +79 -0
  44. interlaced-2.0.0a2/src/interlace/project.py +278 -0
  45. interlaced-2.0.0a2/src/interlace/py.typed +0 -0
  46. interlaced-2.0.0a2/src/interlace/runtime/__init__.py +6 -0
  47. interlaced-2.0.0a2/src/interlace/runtime/handles.py +48 -0
  48. interlaced-2.0.0a2/src/interlace/runtime/python_model.py +175 -0
  49. interlaced-2.0.0a2/src/interlace/scaffold.py +83 -0
  50. interlaced-2.0.0a2/src/interlace/scheduler/__init__.py +17 -0
  51. interlaced-2.0.0a2/src/interlace/scheduler/engine.py +66 -0
  52. interlaced-2.0.0a2/src/interlace/scheduler/triggers.py +84 -0
  53. interlaced-2.0.0a2/src/interlace/scheduler/worker.py +154 -0
  54. interlaced-2.0.0a2/src/interlace/service/__init__.py +7 -0
  55. interlaced-2.0.0a2/src/interlace/service/app.py +836 -0
  56. interlaced-2.0.0a2/src/interlace/service/auth.py +41 -0
  57. interlaced-2.0.0a2/src/interlace/state/__init__.py +18 -0
  58. interlaced-2.0.0a2/src/interlace/state/interval.py +134 -0
  59. interlaced-2.0.0a2/src/interlace/state/janitor.py +142 -0
  60. interlaced-2.0.0a2/src/interlace/state/snapshot.py +47 -0
  61. interlaced-2.0.0a2/src/interlace/state/store.py +906 -0
  62. interlaced-2.0.0a2/src/interlace/strategies/__init__.py +64 -0
  63. interlaced-2.0.0a2/src/interlace/strategies/base.py +71 -0
  64. interlaced-2.0.0a2/src/interlace/strategies/full.py +38 -0
  65. interlaced-2.0.0a2/src/interlace/strategies/full_merge.py +98 -0
  66. interlaced-2.0.0a2/src/interlace/strategies/incremental_by_time.py +65 -0
  67. interlaced-2.0.0a2/src/interlace/strategies/merge_by_key.py +69 -0
  68. interlaced-2.0.0a2/src/interlace/strategies/scd_type_2.py +122 -0
  69. interlaced-2.0.0a2/src/interlace/strategies/view.py +33 -0
  70. interlaced-2.0.0a2/src/interlace/streaming/__init__.py +18 -0
  71. interlaced-2.0.0a2/src/interlace/streaming/log.py +298 -0
  72. interlaced-2.0.0a2/src/interlace/streaming/materializer.py +166 -0
  73. interlaced-2.0.0a2/src/interlace/streaming/schema.py +218 -0
  74. interlaced-2.0.0a2/src/interlaced.egg-info/PKG-INFO +216 -0
  75. interlaced-2.0.0a2/src/interlaced.egg-info/SOURCES.txt +120 -0
  76. interlaced-2.0.0a2/src/interlaced.egg-info/dependency_links.txt +1 -0
  77. interlaced-2.0.0a2/src/interlaced.egg-info/entry_points.txt +2 -0
  78. interlaced-2.0.0a2/src/interlaced.egg-info/requires.txt +42 -0
  79. interlaced-2.0.0a2/src/interlaced.egg-info/top_level.txt +1 -0
  80. interlaced-2.0.0a2/tests/test_apply.py +199 -0
  81. interlaced-2.0.0a2/tests/test_checks.py +189 -0
  82. interlaced-2.0.0a2/tests/test_cli.py +102 -0
  83. interlaced-2.0.0a2/tests/test_cli_inspect.py +163 -0
  84. interlaced-2.0.0a2/tests/test_column_lineage.py +73 -0
  85. interlaced-2.0.0a2/tests/test_compile.py +74 -0
  86. interlaced-2.0.0a2/tests/test_contracts.py +35 -0
  87. interlaced-2.0.0a2/tests/test_dag.py +48 -0
  88. interlaced-2.0.0a2/tests/test_differ.py +112 -0
  89. interlaced-2.0.0a2/tests/test_discovery.py +63 -0
  90. interlaced-2.0.0a2/tests/test_dsl.py +73 -0
  91. interlaced-2.0.0a2/tests/test_duckdb_adapter.py +119 -0
  92. interlaced-2.0.0a2/tests/test_ephemeral.py +81 -0
  93. interlaced-2.0.0a2/tests/test_example.py +46 -0
  94. interlaced-2.0.0a2/tests/test_example_benchmark.py +44 -0
  95. interlaced-2.0.0a2/tests/test_example_platform_tour.py +64 -0
  96. interlaced-2.0.0a2/tests/test_exports.py +91 -0
  97. interlaced-2.0.0a2/tests/test_fingerprint.py +42 -0
  98. interlaced-2.0.0a2/tests/test_forward_only.py +207 -0
  99. interlaced-2.0.0a2/tests/test_incremental.py +170 -0
  100. interlaced-2.0.0a2/tests/test_incremental_python.py +225 -0
  101. interlaced-2.0.0a2/tests/test_init.py +52 -0
  102. interlaced-2.0.0a2/tests/test_interval.py +74 -0
  103. interlaced-2.0.0a2/tests/test_janitor.py +154 -0
  104. interlaced-2.0.0a2/tests/test_multi_engine.py +203 -0
  105. interlaced-2.0.0a2/tests/test_postgres_engine.py +198 -0
  106. interlaced-2.0.0a2/tests/test_python_models.py +197 -0
  107. interlaced-2.0.0a2/tests/test_rebuild_skip.py +160 -0
  108. interlaced-2.0.0a2/tests/test_run.py +108 -0
  109. interlaced-2.0.0a2/tests/test_scd2.py +109 -0
  110. interlaced-2.0.0a2/tests/test_scheduler.py +120 -0
  111. interlaced-2.0.0a2/tests/test_schema_evolution.py +122 -0
  112. interlaced-2.0.0a2/tests/test_selectors.py +53 -0
  113. interlaced-2.0.0a2/tests/test_service.py +346 -0
  114. interlaced-2.0.0a2/tests/test_sql_config.py +66 -0
  115. interlaced-2.0.0a2/tests/test_state_store.py +97 -0
  116. interlaced-2.0.0a2/tests/test_storage.py +145 -0
  117. interlaced-2.0.0a2/tests/test_strategies.py +107 -0
  118. interlaced-2.0.0a2/tests/test_streaming.py +230 -0
  119. interlaced-2.0.0a2/tests/test_table_sinks.py +169 -0
  120. interlaced-2.0.0a2/tests/test_transfers.py +201 -0
  121. interlaced-2.0.0a2/tests/test_warehouse_config.py +238 -0
  122. interlaced-2.0.0a2/tests/test_worker_leases.py +175 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025-2026 Interlace Contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,216 @@
1
+ Metadata-Version: 2.4
2
+ Name: interlaced
3
+ Version: 2.0.0a2
4
+ Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
5
+ Author-email: Mark <mark@interlace.sh>
6
+ License-Expression: MIT
7
+ Keywords: data,pipeline,orchestration,transformation,etl,streaming,dbt,sqlmesh,duckdb
8
+ Classifier: Development Status :: 3 - Alpha
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Topic :: Software Development :: Build Tools
11
+ Classifier: Programming Language :: Python :: 3.12
12
+ Classifier: Programming Language :: SQL
13
+ Requires-Python: >=3.12
14
+ Description-Content-Type: text/markdown
15
+ License-File: LICENSE
16
+ Requires-Dist: sqlglot<31.0,>=25.0
17
+ Requires-Dist: duckdb>=1.5.3
18
+ Requires-Dist: pyarrow>=17.0
19
+ Requires-Dist: msgspec<1.0,>=0.18
20
+ Requires-Dist: pydantic<3.0,>=2.5
21
+ Requires-Dist: typer<1.0,>=0.12
22
+ Requires-Dist: rich<15.0,>=13.0
23
+ Requires-Dist: cronsim<3.0,>=2.5
24
+ Requires-Dist: tenacity<10.0,>=8.2
25
+ Requires-Dist: structlog<26.0,>=24.0
26
+ Requires-Dist: pyyaml<7.0,>=6.0
27
+ Provides-Extra: service
28
+ Requires-Dist: litestar<3.0,>=2.12; extra == "service"
29
+ Requires-Dist: uvicorn<1.0,>=0.30; extra == "service"
30
+ Requires-Dist: httpx<1.0,>=0.27; extra == "service"
31
+ Requires-Dist: argon2-cffi<26.0,>=23.1; extra == "service"
32
+ Requires-Dist: joserfc<2.0,>=1.0; extra == "service"
33
+ Requires-Dist: watchfiles<2.0,>=0.22; extra == "service"
34
+ Provides-Extra: adbc
35
+ Requires-Dist: adbc-driver-manager<2.0,>=1.2; extra == "adbc"
36
+ Requires-Dist: adbc-driver-postgresql<2.0,>=1.2; extra == "adbc"
37
+ Provides-Extra: postgres
38
+ Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "postgres"
39
+ Provides-Extra: polars
40
+ Requires-Dist: polars<2.0,>=1.0; extra == "polars"
41
+ Provides-Extra: pandas
42
+ Requires-Dist: pandas<4.0,>=2.0; extra == "pandas"
43
+ Provides-Extra: all
44
+ Requires-Dist: interlaced[adbc,polars,postgres,service]; extra == "all"
45
+ Provides-Extra: dev
46
+ Requires-Dist: pytest<10.0,>=8.0; extra == "dev"
47
+ Requires-Dist: pytest-asyncio<2.0,>=1.0; extra == "dev"
48
+ Requires-Dist: ruff<1.0,>=0.6; extra == "dev"
49
+ Requires-Dist: black<27.0,>=24.0; extra == "dev"
50
+ Requires-Dist: mypy<2.0,>=1.11; extra == "dev"
51
+ Dynamic: license-file
52
+
53
+ # interlace
54
+
55
+ **Python/SQL-first data platform: transformation, orchestration, and durable streaming — one process.**
56
+
57
+ interlace is an independent, MIT-licensed alternative to dbt/SQLMesh that also replaces the
58
+ orchestrator (no Airflow) and the ingestion layer (Cloudflare-Pipelines-style durable streams).
59
+ Models are `.sql` files or Python functions; state is versioned snapshots with virtual
60
+ environments and a terraform-style plan/apply; everything runs in a single daemon on
61
+ DuckDB + DuckLake by default.
62
+
63
+ > **Status: v2 pre-release.** This branch is a ground-up rebuild (the `interlace` package on
64
+ > PyPI is the older 0.x line). APIs may still move. Requires Python 3.12+.
65
+
66
+ ```bash
67
+ uv pip install "interlaced[service]" # or from source: "interlaced[service] @ git+https://github.com/interlace-sh/interlace@v2"
68
+ ```
69
+
70
+ ## Sixty seconds
71
+
72
+ ```bash
73
+ interlace init my-project && cd my-project
74
+ interlace plan # terraform-style preview: added / breaking / non-breaking / reuse
75
+ interlace apply # build changed models, run checks, promote the environment
76
+ interlace serve # the daemon: HTTP API + scheduler + stream ingestion, one process
77
+ ```
78
+
79
+ Every model builds into a fingerprinted physical table (`interlace__main.orders__a1b2c3`);
80
+ environments are views over those tables, so promotion and rollback are atomic view swaps and a
81
+ dev environment reuses prod's tables for free. **Production is the unprefixed namespace** —
82
+ consumers query `main.orders`; sandboxes are prefixed (`dev__main.orders`). Commands default to
83
+ prod; pass `--env dev` while developing.
84
+
85
+ ## Models
86
+
87
+ **SQL** — a file per model; upstreams referenced by model name, dependencies inferred by parsing
88
+ (sqlglot), config in a leading comment block:
89
+
90
+ ```sql
91
+ /* interlace:
92
+ strategy: scd_type_2
93
+ key: customer_id
94
+ schedule: {cron: "0 * * * *"}
95
+ checks:
96
+ - not_null: customer_id
97
+ - unique: customer_id
98
+ */
99
+ SELECT customer_id, name, tier FROM raw_customers
100
+ ```
101
+
102
+ **Python** — functions whose parameters name their upstreams; data crosses as Arrow
103
+ (never pandas), streamed with bounded memory:
104
+
105
+ ```python
106
+ from interlace import model
107
+
108
+ @model(strategy="merge_by_key", key="order_id", cursor="updated_at")
109
+ def orders(cursor, this):
110
+ """Incremental API extract: `cursor` is max(updated_at) already in the
111
+ warehouse (None on first run); `this` is the previous materialisation."""
112
+ rows = fetch_orders(since=cursor) # your code
113
+ return pyarrow.Table.from_pylist(rows) # or RecordBatchReader / generator of batches
114
+ ```
115
+
116
+ **Strategies:** `full`, `view`, `ephemeral` (CTE-inlined), `merge_by_key` (upsert),
117
+ `full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
118
+ (windowed, interval-ledger backfill/catchup), `scd_type_2` (history with validity windows).
119
+
120
+ ## Plan / apply
121
+
122
+ ```
123
+ $ interlace plan
124
+ Model Change Category Build
125
+ orders modified non_breaking rebuild
126
+ order_stats modified non_breaking reuse <- output provably identical: not rebuilt
127
+ ```
128
+
129
+ - Changes classify **breaking / non-breaking / forward-only**; downstream models whose output
130
+ is provably identical **reuse their existing tables** instead of rebuilding (an improvement
131
+ over model-granular invalidation).
132
+ - `apply --forward-only` lets history-keeping models (scd2/merge/incremental) survive a
133
+ definition change: the new logic inherits the existing table and applies going forward.
134
+ - **Checks gate promotion**: 10 built-in types (not_null, unique, accepted_values, row_count,
135
+ freshness, expression, relationships, pattern, range, sql) plus `@check` Python functions —
136
+ an error-severity failure blocks before the environment view moves.
137
+ - `interlace gc` removes snapshots no environment references (reference-aware: tables shared
138
+ through reuse survive).
139
+
140
+ ## Streaming
141
+
142
+ Declare a stream; POST to it; rows are durable before the 200, deduplicated by idempotency
143
+ key, and materialized exactly-once into `streams.<name>` — where SQL models just read them.
144
+ A flush triggers the models that consume the stream.
145
+
146
+ ```python
147
+ from interlace import stream
148
+
149
+ @stream("orders", schema={"order_id": "string", "total": "double"},
150
+ idempotency_key="order_id", retention="7d", on_schema_drift="evolve")
151
+ def orders(event): ...
152
+ ```
153
+
154
+ ```bash
155
+ curl -X POST localhost:8000/streams/orders -d '{"order_id": "o1", "total": 49.5}'
156
+ ```
157
+
158
+ Schema drift is yours to choose: `reject` (400), `evolve` (new columns appear), or
159
+ `quarantine` (bad events divert to `<stream>__quarantine`).
160
+
161
+ ## Reverse ETL
162
+
163
+ Attach external databases and deliver model results into them — the live table is never
164
+ dropped, keyed modes reuse the same merge strategies:
165
+
166
+ ```yaml
167
+ # interlace.yaml
168
+ attach:
169
+ crm: "postgres:host=... dbname=crm"
170
+ ```
171
+
172
+ ```sql
173
+ /* interlace: {export: {to: table, target: crm.public.accounts, mode: merge_by_key, key: id}} */
174
+ SELECT id, tier, lifetime_value FROM account_summary
175
+ ```
176
+
177
+ File exports (`to: parquet|csv|json`) work the same way.
178
+
179
+ ## The daemon
180
+
181
+ `interlace serve` runs the HTTP API (Litestar + msgspec, OpenAPI at `/schema/scalar`), the
182
+ scheduler (cron/interval triggers → durable run queue), stream ingestion, and retention in one
183
+ process. Scoped API keys (`interlace apikey create ci --scope read`) lock it down; a durable
184
+ event log backs `GET /events/stream` (SSE with replay).
185
+
186
+ Add `--quack quack:localhost:4213` to serve the warehouse itself over DuckDB's quack protocol —
187
+ other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by setting
188
+ `database: quack:localhost:4213`.
189
+
190
+ ## Architecture in five lines
191
+
192
+ - The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
193
+ are AST builders and dialect appears only at `transpile()`.
194
+ - Storage defaults to **DuckLake** (Parquet + SQL catalog) opened as DuckDB's primary database.
195
+ - Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
196
+ scale-out swap.
197
+ - Streams live in their own durable log; the materializer commits data + watermark in one
198
+ warehouse transaction — exactly-once without distributed coordination.
199
+ - No Jinja, no pandas in core, no external orchestrator.
200
+
201
+ The full design rationale lives in `docs/architecture/v2-design.md`.
202
+
203
+ ## Development
204
+
205
+ Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
206
+ [moon](https://moonrepo.dev/moon), `uv` owns the virtualenv:
207
+
208
+ ```bash
209
+ proto install
210
+ moon run interlace:sync # install deps
211
+ moon run interlace:test # 240+ tests
212
+ moon run interlace:check # black + ruff (CI equivalent)
213
+ moon run interlace:typecheck # mypy --strict
214
+ ```
215
+
216
+ MIT licensed.
@@ -0,0 +1,164 @@
1
+ # interlace
2
+
3
+ **Python/SQL-first data platform: transformation, orchestration, and durable streaming — one process.**
4
+
5
+ interlace is an independent, MIT-licensed alternative to dbt/SQLMesh that also replaces the
6
+ orchestrator (no Airflow) and the ingestion layer (Cloudflare-Pipelines-style durable streams).
7
+ Models are `.sql` files or Python functions; state is versioned snapshots with virtual
8
+ environments and a terraform-style plan/apply; everything runs in a single daemon on
9
+ DuckDB + DuckLake by default.
10
+
11
+ > **Status: v2 pre-release.** This branch is a ground-up rebuild (the `interlace` package on
12
+ > PyPI is the older 0.x line). APIs may still move. Requires Python 3.12+.
13
+
14
+ ```bash
15
+ uv pip install "interlaced[service]" # or from source: "interlaced[service] @ git+https://github.com/interlace-sh/interlace@v2"
16
+ ```
17
+
18
+ ## Sixty seconds
19
+
20
+ ```bash
21
+ interlace init my-project && cd my-project
22
+ interlace plan # terraform-style preview: added / breaking / non-breaking / reuse
23
+ interlace apply # build changed models, run checks, promote the environment
24
+ interlace serve # the daemon: HTTP API + scheduler + stream ingestion, one process
25
+ ```
26
+
27
+ Every model builds into a fingerprinted physical table (`interlace__main.orders__a1b2c3`);
28
+ environments are views over those tables, so promotion and rollback are atomic view swaps and a
29
+ dev environment reuses prod's tables for free. **Production is the unprefixed namespace** —
30
+ consumers query `main.orders`; sandboxes are prefixed (`dev__main.orders`). Commands default to
31
+ prod; pass `--env dev` while developing.
32
+
33
+ ## Models
34
+
35
+ **SQL** — a file per model; upstreams referenced by model name, dependencies inferred by parsing
36
+ (sqlglot), config in a leading comment block:
37
+
38
+ ```sql
39
+ /* interlace:
40
+ strategy: scd_type_2
41
+ key: customer_id
42
+ schedule: {cron: "0 * * * *"}
43
+ checks:
44
+ - not_null: customer_id
45
+ - unique: customer_id
46
+ */
47
+ SELECT customer_id, name, tier FROM raw_customers
48
+ ```
49
+
50
+ **Python** — functions whose parameters name their upstreams; data crosses as Arrow
51
+ (never pandas), streamed with bounded memory:
52
+
53
+ ```python
54
+ from interlace import model
55
+
56
+ @model(strategy="merge_by_key", key="order_id", cursor="updated_at")
57
+ def orders(cursor, this):
58
+ """Incremental API extract: `cursor` is max(updated_at) already in the
59
+ warehouse (None on first run); `this` is the previous materialisation."""
60
+ rows = fetch_orders(since=cursor) # your code
61
+ return pyarrow.Table.from_pylist(rows) # or RecordBatchReader / generator of batches
62
+ ```
63
+
64
+ **Strategies:** `full`, `view`, `ephemeral` (CTE-inlined), `merge_by_key` (upsert),
65
+ `full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
66
+ (windowed, interval-ledger backfill/catchup), `scd_type_2` (history with validity windows).
67
+
68
+ ## Plan / apply
69
+
70
+ ```
71
+ $ interlace plan
72
+ Model Change Category Build
73
+ orders modified non_breaking rebuild
74
+ order_stats modified non_breaking reuse <- output provably identical: not rebuilt
75
+ ```
76
+
77
+ - Changes classify **breaking / non-breaking / forward-only**; downstream models whose output
78
+ is provably identical **reuse their existing tables** instead of rebuilding (an improvement
79
+ over model-granular invalidation).
80
+ - `apply --forward-only` lets history-keeping models (scd2/merge/incremental) survive a
81
+ definition change: the new logic inherits the existing table and applies going forward.
82
+ - **Checks gate promotion**: 10 built-in types (not_null, unique, accepted_values, row_count,
83
+ freshness, expression, relationships, pattern, range, sql) plus `@check` Python functions —
84
+ an error-severity failure blocks before the environment view moves.
85
+ - `interlace gc` removes snapshots no environment references (reference-aware: tables shared
86
+ through reuse survive).
87
+
88
+ ## Streaming
89
+
90
+ Declare a stream; POST to it; rows are durable before the 200, deduplicated by idempotency
91
+ key, and materialized exactly-once into `streams.<name>` — where SQL models just read them.
92
+ A flush triggers the models that consume the stream.
93
+
94
+ ```python
95
+ from interlace import stream
96
+
97
+ @stream("orders", schema={"order_id": "string", "total": "double"},
98
+ idempotency_key="order_id", retention="7d", on_schema_drift="evolve")
99
+ def orders(event): ...
100
+ ```
101
+
102
+ ```bash
103
+ curl -X POST localhost:8000/streams/orders -d '{"order_id": "o1", "total": 49.5}'
104
+ ```
105
+
106
+ Schema drift is yours to choose: `reject` (400), `evolve` (new columns appear), or
107
+ `quarantine` (bad events divert to `<stream>__quarantine`).
108
+
109
+ ## Reverse ETL
110
+
111
+ Attach external databases and deliver model results into them — the live table is never
112
+ dropped, keyed modes reuse the same merge strategies:
113
+
114
+ ```yaml
115
+ # interlace.yaml
116
+ attach:
117
+ crm: "postgres:host=... dbname=crm"
118
+ ```
119
+
120
+ ```sql
121
+ /* interlace: {export: {to: table, target: crm.public.accounts, mode: merge_by_key, key: id}} */
122
+ SELECT id, tier, lifetime_value FROM account_summary
123
+ ```
124
+
125
+ File exports (`to: parquet|csv|json`) work the same way.
126
+
127
+ ## The daemon
128
+
129
+ `interlace serve` runs the HTTP API (Litestar + msgspec, OpenAPI at `/schema/scalar`), the
130
+ scheduler (cron/interval triggers → durable run queue), stream ingestion, and retention in one
131
+ process. Scoped API keys (`interlace apikey create ci --scope read`) lock it down; a durable
132
+ event log backs `GET /events/stream` (SSE with replay).
133
+
134
+ Add `--quack quack:localhost:4213` to serve the warehouse itself over DuckDB's quack protocol —
135
+ other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by setting
136
+ `database: quack:localhost:4213`.
137
+
138
+ ## Architecture in five lines
139
+
140
+ - The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
141
+ are AST builders and dialect appears only at `transpile()`.
142
+ - Storage defaults to **DuckLake** (Parquet + SQL catalog) opened as DuckDB's primary database.
143
+ - Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
144
+ scale-out swap.
145
+ - Streams live in their own durable log; the materializer commits data + watermark in one
146
+ warehouse transaction — exactly-once without distributed coordination.
147
+ - No Jinja, no pandas in core, no external orchestrator.
148
+
149
+ The full design rationale lives in `docs/architecture/v2-design.md`.
150
+
151
+ ## Development
152
+
153
+ Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
154
+ [moon](https://moonrepo.dev/moon), `uv` owns the virtualenv:
155
+
156
+ ```bash
157
+ proto install
158
+ moon run interlace:sync # install deps
159
+ moon run interlace:test # 240+ tests
160
+ moon run interlace:check # black + ruff (CI equivalent)
161
+ moon run interlace:typecheck # mypy --strict
162
+ ```
163
+
164
+ MIT licensed.
@@ -0,0 +1,159 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "interlaced"
7
+ version = "2.0.0a2"
8
+ description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
9
+ readme = "README.md"
10
+ requires-python = ">=3.12"
11
+ license = "MIT"
12
+ authors = [
13
+ {name = "Mark", email = "mark@interlace.sh"}
14
+ ]
15
+ keywords = ["data", "pipeline", "orchestration", "transformation", "etl", "streaming", "dbt", "sqlmesh", "duckdb"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Intended Audience :: Developers",
19
+ "Topic :: Software Development :: Build Tools",
20
+ "Programming Language :: Python :: 3.12",
21
+ "Programming Language :: SQL",
22
+ ]
23
+
24
+ # Core: the IR + engine + state + plan spine (Phase 1). Everything is Arrow- and
25
+ # sqlglot-native. See docs/architecture/v2-design.md.
26
+ dependencies = [
27
+ "sqlglot>=25.0,<31.0", # canonical IR, transpilation, semantic diff, column lineage
28
+ "duckdb>=1.5.3", # default engine + federation hub; DuckLake storage + quack serving
29
+ "pyarrow>=17.0", # the wire format: RecordBatchReader everywhere
30
+ "msgspec>=0.18,<1.0", # ingest validation + fast (de)serialisation
31
+ "pydantic>=2.5,<3.0", # config + manifest validation (cold paths only)
32
+ "typer>=0.12,<1.0", # CLI
33
+ "rich>=13.0,<15.0", # display, strictly an event subscriber
34
+ "cronsim>=2.5,<3.0", # cron parsing for the trigger engine
35
+ "tenacity>=8.2,<10.0", # retries (tasks, DuckLake commit conflicts, transfers)
36
+ "structlog>=24.0,<26.0", # structured logging
37
+ "pyyaml>=6.0,<7.0", # project config (config + env overlays)
38
+ ]
39
+
40
+ [project.scripts]
41
+ interlace = "interlace.cli.main:main"
42
+
43
+ [project.optional-dependencies]
44
+ # Service + orchestration daemon (Phase 2).
45
+ service = [
46
+ "litestar>=2.12,<3.0",
47
+ "uvicorn>=0.30,<1.0",
48
+ "httpx>=0.27,<1.0",
49
+ "argon2-cffi>=23.1,<26.0",
50
+ "joserfc>=1.0,<2.0",
51
+ "watchfiles>=0.22,<2.0",
52
+ ]
53
+ # Remote engines via Arrow-native transfer (Phase 4).
54
+ adbc = [
55
+ "adbc-driver-manager>=1.2,<2.0",
56
+ "adbc-driver-postgresql>=1.2,<2.0",
57
+ ]
58
+ postgres = ["psycopg[binary]>=3.1,<4.0"]
59
+ polars = ["polars>=1.0,<2.0"]
60
+ pandas = ["pandas>=2.0,<4.0"]
61
+ all = ["interlaced[service,adbc,postgres,polars]"]
62
+ dev = [
63
+ "pytest>=8.0,<10.0",
64
+ "pytest-asyncio>=1.0,<2.0",
65
+ "ruff>=0.6,<1.0",
66
+ "black>=24.0,<27.0",
67
+ "mypy>=1.11,<2.0",
68
+ ]
69
+
70
+ [tool.setuptools.packages.find]
71
+ where = ["src"]
72
+
73
+ [tool.setuptools.package-data]
74
+ interlace = ["py.typed"]
75
+
76
+ [tool.black]
77
+ line-length = 120
78
+ target-version = ['py312']
79
+
80
+ [tool.ruff]
81
+ line-length = 120
82
+ target-version = "py312"
83
+
84
+ [tool.ruff.lint]
85
+ select = [
86
+ "E", # pycodestyle errors
87
+ "W", # pycodestyle warnings
88
+ "F", # pyflakes
89
+ "I", # isort
90
+ "B", # flake8-bugbear
91
+ "C4", # flake8-comprehensions
92
+ "UP", # pyupgrade
93
+ ]
94
+ ignore = [
95
+ "E501", # line too long (handled by formatter)
96
+ "B008", # function calls in argument defaults
97
+ "C901", # too complex
98
+ ]
99
+
100
+ [tool.ruff.lint.per-file-ignores]
101
+ "__init__.py" = ["F401"] # Allow unused imports in __init__.py
102
+
103
+ [tool.mypy]
104
+ python_version = "3.12"
105
+ warn_return_any = true
106
+ warn_unused_configs = true
107
+ disallow_untyped_defs = true
108
+ disallow_incomplete_defs = true
109
+ check_untyped_defs = true
110
+ no_implicit_optional = true
111
+ warn_redundant_casts = true
112
+ warn_unused_ignores = true
113
+
114
+ [[tool.mypy.overrides]]
115
+ module = [
116
+ "sqlglot.*",
117
+ "pyarrow.*",
118
+ "yaml",
119
+ ]
120
+ ignore_missing_imports = true
121
+
122
+ [tool.coverage.run]
123
+ source = ["src/interlace"]
124
+ omit = [
125
+ "*/tests/*",
126
+ "*/test_*.py",
127
+ "*/__pycache__/*",
128
+ "*/__init__.py",
129
+ ]
130
+
131
+ [tool.coverage.report]
132
+ exclude_lines = [
133
+ "pragma: no cover",
134
+ "def __repr__",
135
+ "raise AssertionError",
136
+ "raise NotImplementedError",
137
+ "if __name__ == .__main__.:",
138
+ "if TYPE_CHECKING:",
139
+ "@abstractmethod",
140
+ ]
141
+
142
+ [tool.pytest.ini_options]
143
+ testpaths = ["tests"]
144
+ python_files = ["test_*.py", "*_test.py"]
145
+ python_classes = ["Test*"]
146
+ python_functions = ["test_*"]
147
+ addopts = [
148
+ "-v",
149
+ "--strict-markers",
150
+ "--tb=short",
151
+ ]
152
+ markers = [
153
+ "unit: Unit tests (fast, isolated)",
154
+ "integration: Integration tests (slower, require database)",
155
+ "slow: Slow tests (may take a long time)",
156
+ "requires_db: Needs a reachable external database (e.g. Postgres)",
157
+ ]
158
+ asyncio_mode = "auto"
159
+ asyncio_default_fixture_loop_scope = "function"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,26 @@
1
+ """Interlace v2 — Python/SQL-first data platform.
2
+
3
+ Transformation (sqlmesh-grade snapshots, virtual environments, plan/apply),
4
+ built-in orchestration (durable work queue + unified triggers), and durable
5
+ streaming ingestion — in one process. See docs/architecture/v2-design.md.
6
+
7
+ This package is under active greenfield construction; the public surface is the
8
+ ``@model`` / ``@stream`` / ``@check`` decorators plus the core IR types.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from interlace.dsl.decorators import check, model, stream
14
+ from interlace.ir.relation import EngineRef, SqlRelation, TableRef
15
+
16
+ __version__ = "2.0.0a2"
17
+
18
+ __all__ = [
19
+ "EngineRef",
20
+ "SqlRelation",
21
+ "TableRef",
22
+ "__version__",
23
+ "check",
24
+ "model",
25
+ "stream",
26
+ ]
@@ -0,0 +1,10 @@
1
+ """Data-quality checks. Results gate promotion: an ``error``-severity failure
2
+ aborts the apply before the environment is promoted.
3
+
4
+ Import :mod:`interlace.checks.runner` for execution — kept out of this package
5
+ init so declaring checks (spec) never drags in the runtime.
6
+ """
7
+
8
+ from interlace.checks.spec import CheckSpec, parse_checks
9
+
10
+ __all__ = ["CheckSpec", "parse_checks"]