interlaced 2.2.0__tar.gz → 2.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. {interlaced-2.2.0/src/interlaced.egg-info → interlaced-2.4.0}/PKG-INFO +53 -16
  2. {interlaced-2.2.0 → interlaced-2.4.0}/README.md +46 -11
  3. {interlaced-2.2.0 → interlaced-2.4.0}/pyproject.toml +9 -5
  4. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/cli/main.py +3 -3
  5. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/config/config.py +8 -7
  6. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/decorators.py +15 -3
  7. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/discovery.py +1 -1
  8. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/duckdb.py +54 -3
  9. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/spark.py +1 -1
  10. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/project.py +2 -2
  11. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/canonicalize.py +4 -1
  12. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/apply.py +153 -34
  13. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/differ.py +2 -2
  14. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/plan.py +2 -2
  15. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/run.py +1 -1
  16. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/project.py +6 -1
  17. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scaffold.py +19 -2
  18. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/app.py +8 -4
  19. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/__init__.py +10 -4
  20. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/append.py +12 -4
  21. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/base.py +13 -0
  22. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/hash_merge.py +40 -18
  23. interlaced-2.4.0/src/interlace/strategies/incremental.py +95 -0
  24. interlaced-2.4.0/src/interlace/strategies/merge.py +188 -0
  25. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/scd.py +9 -1
  26. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/interlace.yaml +2 -1
  27. interlaced-2.4.0/src/interlace/templates/quickstart/interlace.yaml +6 -0
  28. {interlaced-2.2.0 → interlaced-2.4.0/src/interlaced.egg-info}/PKG-INFO +53 -16
  29. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/SOURCES.txt +1 -1
  30. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/requires.txt +3 -3
  31. interlaced-2.2.0/src/interlace/strategies/incremental_by_time.py +0 -64
  32. interlaced-2.2.0/src/interlace/strategies/merge.py +0 -115
  33. interlaced-2.2.0/src/interlace/templates/quickstart/interlace.yaml +0 -6
  34. {interlaced-2.2.0 → interlaced-2.4.0}/LICENSE +0 -0
  35. {interlaced-2.2.0 → interlaced-2.4.0}/MANIFEST.in +0 -0
  36. {interlaced-2.2.0 → interlaced-2.4.0}/setup.cfg +0 -0
  37. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/__init__.py +0 -0
  38. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/__init__.py +0 -0
  39. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/builtin.py +0 -0
  40. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/runner.py +0 -0
  41. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/checks/spec.py +0 -0
  42. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/cli/__init__.py +0 -0
  43. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/config/__init__.py +0 -0
  44. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/contracts.py +0 -0
  45. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/__init__.py +0 -0
  46. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/dsl/sql_config.py +0 -0
  47. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/__init__.py +0 -0
  48. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/adbc.py +0 -0
  49. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/base.py +0 -0
  50. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/bigquery.py +0 -0
  51. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/postgres.py +0 -0
  52. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/quack.py +0 -0
  53. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/redshift.py +0 -0
  54. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/registry.py +0 -0
  55. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/engines/snowflake.py +0 -0
  56. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/exceptions.py +0 -0
  57. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/__init__.py +0 -0
  58. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/column_lineage.py +0 -0
  59. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/dag.py +0 -0
  60. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/graph/selectors.py +0 -0
  61. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/__init__.py +0 -0
  62. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/fingerprint.py +0 -0
  63. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/ir/relation.py +0 -0
  64. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/__init__.py +0 -0
  65. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/plan/resolve.py +0 -0
  66. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/py.typed +0 -0
  67. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/query.py +0 -0
  68. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/runtime/__init__.py +0 -0
  69. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/runtime/handles.py +0 -0
  70. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/runtime/python_model.py +0 -0
  71. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/__init__.py +0 -0
  72. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/engine.py +0 -0
  73. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/triggers.py +0 -0
  74. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/scheduler/worker.py +0 -0
  75. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/__init__.py +0 -0
  76. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/auth.py +0 -0
  77. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/app.css +0 -0
  78. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/favicon.svg +0 -0
  79. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/index.html +0 -0
  80. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/api.js +0 -0
  81. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/app.js +0 -0
  82. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/dag.js +0 -0
  83. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/timeline.js +0 -0
  84. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/ui.js +0 -0
  85. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/checks.js +0 -0
  86. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/environments.js +0 -0
  87. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/lineage.js +0 -0
  88. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/models.js +0 -0
  89. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/overview.js +0 -0
  90. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/plan.js +0 -0
  91. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/query.js +0 -0
  92. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/runs.js +0 -0
  93. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/streams.js +0 -0
  94. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/service/ui/js/views/system.js +0 -0
  95. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sinks.py +0 -0
  96. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sources/__init__.py +0 -0
  97. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sources/auth.py +0 -0
  98. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/sources/rest.py +0 -0
  99. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/__init__.py +0 -0
  100. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/interval.py +0 -0
  101. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/janitor.py +0 -0
  102. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/snapshot.py +0 -0
  103. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/state/store.py +0 -0
  104. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/full_merge.py +0 -0
  105. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/replace.py +0 -0
  106. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/replace_in_place.py +0 -0
  107. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/strategies/view.py +0 -0
  108. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/__init__.py +0 -0
  109. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/log.py +0 -0
  110. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/materializer.py +0 -0
  111. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/streaming/schema.py +0 -0
  112. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/README.md +0 -0
  113. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/generate.py +0 -0
  114. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/events.py +0 -0
  115. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/events_by_minute.sql +0 -0
  116. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/events_by_type.sql +0 -0
  117. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/top_users.sql +0 -0
  118. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/models/user_spend.sql +0 -0
  119. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/events/template.yaml +0 -0
  120. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/README.md +0 -0
  121. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/interlace.yaml +0 -0
  122. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/models/github_issues.py +0 -0
  123. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/models/issues_by_state.sql +0 -0
  124. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/github/template.yaml +0 -0
  125. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/README.md +0 -0
  126. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/docker-compose.yml +0 -0
  127. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/init/seed.sql +0 -0
  128. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/interlace.yaml +0 -0
  129. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/models/orders.py +0 -0
  130. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/models/orders_by_status.sql +0 -0
  131. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/postgres/template.yaml +0 -0
  132. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/README.md +0 -0
  133. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/models/enriched_events.py +0 -0
  134. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/models/event_summary.sql +0 -0
  135. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/models/raw_events.sql +0 -0
  136. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlace/templates/quickstart/template.yaml +0 -0
  137. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/dependency_links.txt +0 -0
  138. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/entry_points.txt +0 -0
  139. {interlaced-2.2.0 → interlaced-2.4.0}/src/interlaced.egg-info/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: interlaced
3
- Version: 2.2.0
4
- Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
3
+ Version: 2.4.0
4
+ Summary: Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres.
5
5
  Author-email: Mark <mark@interlace.sh>
6
6
  License-Expression: MIT
7
7
  Project-URL: Homepage, https://interlace.sh
@@ -13,17 +13,19 @@ Keywords: data,pipeline,orchestration,transformation,etl,streaming,dbt,sqlmesh,d
13
13
  Classifier: Development Status :: 5 - Production/Stable
14
14
  Classifier: Intended Audience :: Developers
15
15
  Classifier: Topic :: Software Development :: Build Tools
16
+ Classifier: Topic :: Database
16
17
  Classifier: Programming Language :: Python :: 3.12
17
18
  Classifier: Programming Language :: SQL
19
+ Classifier: Operating System :: POSIX :: Linux
18
20
  Requires-Python: >=3.12
19
21
  Description-Content-Type: text/markdown
20
22
  License-File: LICENSE
21
- Requires-Dist: sqlglot<29.0,>=25.0
23
+ Requires-Dist: sqlglot<30.0,>=25.0
22
24
  Requires-Dist: duckdb>=1.5.3
23
25
  Requires-Dist: pyarrow>=17.0
24
26
  Requires-Dist: pydantic<3.0,>=2.5
25
27
  Requires-Dist: typer<1.0,>=0.12
26
- Requires-Dist: rich<15.0,>=13.0
28
+ Requires-Dist: rich<16.0,>=13.0
27
29
  Requires-Dist: cronsim<3.0,>=2.5
28
30
  Requires-Dist: tenacity<10.0,>=8.2
29
31
  Requires-Dist: pyyaml<7.0,>=6.0
@@ -59,20 +61,46 @@ Requires-Dist: httpx<1.0,>=0.27; extra == "dev"
59
61
  Requires-Dist: pytest-asyncio<2.0,>=1.0; extra == "dev"
60
62
  Requires-Dist: ruff<1.0,>=0.6; extra == "dev"
61
63
  Requires-Dist: black<27.0,>=24.0; extra == "dev"
62
- Requires-Dist: mypy<2.0,>=1.11; extra == "dev"
64
+ Requires-Dist: mypy<3.0,>=1.11; extra == "dev"
63
65
  Dynamic: license-file
64
66
 
65
67
  # interlace
66
68
 
67
- **Python/SQL-first data platform: transformation, orchestration, and durable streaming — one process.**
69
+ **Python and SQL models are the same kind of node in one DAG.**
68
70
 
69
- interlace is an independent, MIT-licensed alternative to dbt/SQLMesh that also replaces the
70
- orchestrator (no Airflow) and the ingestion layer (Cloudflare-Pipelines-style durable streams).
71
- Models are `.sql` files or Python functions; state is versioned snapshots with virtual
71
+ A `.py` model sits mid-graph with SQL either side, in both directions, with no bridge and no
72
+ separate runtime — running in-process on DuckDB and Postgres, not only on a cloud warehouse.
73
+ The Python model stays a plain function: call it in a test with no warehouse and no session.
74
+
75
+ How that compares with dbt and SQLMesh, including where they are ahead, is on
76
+ [interlace.sh/why](https://interlace.sh/why).
77
+
78
+ ```sql
79
+ -- models/raw_events.sql SQL
80
+ SELECT event_id, user_id, kind, amount, country, ts FROM read_parquet('events/*.parquet')
81
+ ```
82
+ ```python
83
+ # models/enriched_events.py Python, mid-DAG
84
+ @model() # the parameter name IS the dependency — no depends_on
85
+ def enriched_events(raw_events):
86
+ for batch in raw_events.reader(): # Arrow in, Arrow out, bounded memory
87
+ yield add_revenue(batch)
88
+ ```
89
+ ```sql
90
+ -- models/event_summary.sql SQL again, straight over the Python
91
+ SELECT country, count(*) FILTER (WHERE is_conversion) AS conversions
92
+ FROM enriched_events GROUP BY country
93
+ ```
94
+
95
+ `interlace init` scaffolds exactly this shape, runnable, with no external source.
96
+
97
+ That is the wedge. The rest is the reveal: interlace is an independent, MIT-licensed alternative
98
+ to dbt/SQLMesh that also replaces the orchestrator (no Airflow) and the ingestion layer
99
+ (Cloudflare-Pipelines-style durable streams). State is versioned snapshots with virtual
72
100
  environments and a terraform-style plan/apply; everything runs in a single daemon on
73
- DuckDB + DuckLake by default.
101
+ DuckDB by default (DuckLake one config line away).
74
102
 
75
- > **Status: 2.0.** Requires Python 3.12+.
103
+ > **2.x — see [releases](https://github.com/interlace-sh/interlace/releases).** Requires Python 3.12+.
76
104
  > The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
77
105
 
78
106
  ```bash
@@ -80,6 +108,13 @@ pip install 'interlaced[service]' # the CLI + daemon; core CLI only: pip insta
80
108
  # more extras: [adbc] postgres/redshift · [spark] · [polars] · [all]
81
109
  ```
82
110
 
111
+ > **Platforms.** Developed on Linux; CI runs Linux only. Nothing in the codebase is
112
+ > platform-specific — no `fork`, no signal handling, no POSIX-only calls, no shelling out — and
113
+ > every dependency ships macOS and Windows wheels, so both are expected to work. But
114
+ > **neither is tested**, so treat them as unverified rather than supported. If you run interlace
115
+ > on macOS or Windows, please open an issue either way; that is the fastest route to changing
116
+ > this paragraph.
117
+
83
118
  ## Sixty seconds
84
119
 
85
120
  ```bash
@@ -127,8 +162,9 @@ def orders(cursor, this):
127
162
  ```
128
163
 
129
164
  **Strategies:** `replace`, `view`, `ephemeral` (CTE-inlined), `merge` (upsert),
130
- `full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
131
- (windowed, interval-ledger backfill/catchup), `scd` (history with validity windows).
165
+ `full_merge` (full-state source applied as a minimal diff), `incremental` (one time window
166
+ at a time — rewrites the window, or upserts within it if you give it a `key`), `scd`
167
+ (history with validity windows).
132
168
 
133
169
  ## Plan / apply
134
170
 
@@ -209,7 +245,7 @@ prod, so a dev apply never writes to a live external table (opt in with
209
245
 
210
246
  ## Multi-engine
211
247
 
212
- Models run on **named engines**: DuckDB/DuckLake by default, Postgres natively over ADBC
248
+ Models run on **named engines**: DuckDB by default (DuckLake opt-in), Postgres natively over ADBC
213
249
  (`pip install 'interlaced[adbc]'`), Spark (beta, `[spark]` extra), plus alpha adapters for
214
250
  MotherDuck, Redshift, Snowflake and BigQuery (wired and dialect-correct, not yet run against a
215
251
  live account), with per-model pinning:
@@ -251,7 +287,8 @@ other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by
251
287
 
252
288
  - The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
253
289
  are AST builders and dialect appears only at `transpile()`.
254
- - Storage defaults to **DuckLake** (Parquet + SQL catalog) opened as DuckDB's primary database.
290
+ - Storage defaults to a plain **DuckDB** file; **DuckLake** (Parquet + SQL catalog, and
291
+ concurrent writers) is one `database: ducklake:…` line away.
255
292
  - Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
256
293
  scale-out swap.
257
294
  - Streams live in their own durable log; the materializer commits data + watermark in one
@@ -268,7 +305,7 @@ Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
268
305
  ```bash
269
306
  proto install
270
307
  moon run interlace:sync # install deps
271
- moon run interlace:test # 350+ tests
308
+ moon run interlace:test # 500+ tests
272
309
  moon run interlace:check # black + ruff (CI equivalent)
273
310
  moon run interlace:typecheck # mypy
274
311
  ```
@@ -1,14 +1,40 @@
1
1
  # interlace
2
2
 
3
- **Python/SQL-first data platform: transformation, orchestration, and durable streaming — one process.**
3
+ **Python and SQL models are the same kind of node in one DAG.**
4
4
 
5
- interlace is an independent, MIT-licensed alternative to dbt/SQLMesh that also replaces the
6
- orchestrator (no Airflow) and the ingestion layer (Cloudflare-Pipelines-style durable streams).
7
- Models are `.sql` files or Python functions; state is versioned snapshots with virtual
5
+ A `.py` model sits mid-graph with SQL either side, in both directions, with no bridge and no
6
+ separate runtime — running in-process on DuckDB and Postgres, not only on a cloud warehouse.
7
+ The Python model stays a plain function: call it in a test with no warehouse and no session.
8
+
9
+ How that compares with dbt and SQLMesh, including where they are ahead, is on
10
+ [interlace.sh/why](https://interlace.sh/why).
11
+
12
+ ```sql
13
+ -- models/raw_events.sql SQL
14
+ SELECT event_id, user_id, kind, amount, country, ts FROM read_parquet('events/*.parquet')
15
+ ```
16
+ ```python
17
+ # models/enriched_events.py Python, mid-DAG
18
+ @model() # the parameter name IS the dependency — no depends_on
19
+ def enriched_events(raw_events):
20
+ for batch in raw_events.reader(): # Arrow in, Arrow out, bounded memory
21
+ yield add_revenue(batch)
22
+ ```
23
+ ```sql
24
+ -- models/event_summary.sql SQL again, straight over the Python
25
+ SELECT country, count(*) FILTER (WHERE is_conversion) AS conversions
26
+ FROM enriched_events GROUP BY country
27
+ ```
28
+
29
+ `interlace init` scaffolds exactly this shape, runnable, with no external source.
30
+
31
+ That is the wedge. The rest is the reveal: interlace is an independent, MIT-licensed alternative
32
+ to dbt/SQLMesh that also replaces the orchestrator (no Airflow) and the ingestion layer
33
+ (Cloudflare-Pipelines-style durable streams). State is versioned snapshots with virtual
8
34
  environments and a terraform-style plan/apply; everything runs in a single daemon on
9
- DuckDB + DuckLake by default.
35
+ DuckDB by default (DuckLake one config line away).
10
36
 
11
- > **Status: 2.0.** Requires Python 3.12+.
37
+ > **2.x — see [releases](https://github.com/interlace-sh/interlace/releases).** Requires Python 3.12+.
12
38
  > The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
13
39
 
14
40
  ```bash
@@ -16,6 +42,13 @@ pip install 'interlaced[service]' # the CLI + daemon; core CLI only: pip insta
16
42
  # more extras: [adbc] postgres/redshift · [spark] · [polars] · [all]
17
43
  ```
18
44
 
45
+ > **Platforms.** Developed on Linux; CI runs Linux only. Nothing in the codebase is
46
+ > platform-specific — no `fork`, no signal handling, no POSIX-only calls, no shelling out — and
47
+ > every dependency ships macOS and Windows wheels, so both are expected to work. But
48
+ > **neither is tested**, so treat them as unverified rather than supported. If you run interlace
49
+ > on macOS or Windows, please open an issue either way; that is the fastest route to changing
50
+ > this paragraph.
51
+
19
52
  ## Sixty seconds
20
53
 
21
54
  ```bash
@@ -63,8 +96,9 @@ def orders(cursor, this):
63
96
  ```
64
97
 
65
98
  **Strategies:** `replace`, `view`, `ephemeral` (CTE-inlined), `merge` (upsert),
66
- `full_merge` (full-state source applied as a minimal diff), `incremental_by_time`
67
- (windowed, interval-ledger backfill/catchup), `scd` (history with validity windows).
99
+ `full_merge` (full-state source applied as a minimal diff), `incremental` (one time window
100
+ at a time — rewrites the window, or upserts within it if you give it a `key`), `scd`
101
+ (history with validity windows).
68
102
 
69
103
  ## Plan / apply
70
104
 
@@ -145,7 +179,7 @@ prod, so a dev apply never writes to a live external table (opt in with
145
179
 
146
180
  ## Multi-engine
147
181
 
148
- Models run on **named engines**: DuckDB/DuckLake by default, Postgres natively over ADBC
182
+ Models run on **named engines**: DuckDB by default (DuckLake opt-in), Postgres natively over ADBC
149
183
  (`pip install 'interlaced[adbc]'`), Spark (beta, `[spark]` extra), plus alpha adapters for
150
184
  MotherDuck, Redshift, Snowflake and BigQuery (wired and dialect-correct, not yet run against a
151
185
  live account), with per-model pinning:
@@ -187,7 +221,8 @@ other processes (CLI runs, ad-hoc DuckDB clients) then share it concurrently by
187
221
 
188
222
  - The IR is a **sqlglot AST**; the wire format is an **Arrow RecordBatchReader**; strategies
189
223
  are AST builders and dialect appears only at `transpile()`.
190
- - Storage defaults to **DuckLake** (Parquet + SQL catalog) opened as DuckDB's primary database.
224
+ - Storage defaults to a plain **DuckDB** file; **DuckLake** (Parquet + SQL catalog, and
225
+ concurrent writers) is one `database: ducklake:…` line away.
191
226
  - Control plane (snapshots, intervals, queue, events, keys) is **SQLite WAL**; Postgres is the
192
227
  scale-out swap.
193
228
  - Streams live in their own durable log; the materializer commits data + watermark in one
@@ -204,7 +239,7 @@ Toolchain is pinned with [proto](https://moonrepo.dev/proto), tasks run via
204
239
  ```bash
205
240
  proto install
206
241
  moon run interlace:sync # install deps
207
- moon run interlace:test # 350+ tests
242
+ moon run interlace:test # 500+ tests
208
243
  moon run interlace:check # black + ruff (CI equivalent)
209
244
  moon run interlace:typecheck # mypy
210
245
  ```
@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "interlaced"
7
- version = "2.2.0"
8
- description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
7
+ version = "2.4.0"
8
+ description = "Python and SQL models in one DAG. Transformation, orchestration and durable streaming in one process, on DuckDB and Postgres."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.12"
11
11
  license = "MIT"
@@ -17,19 +17,23 @@ classifiers = [
17
17
  "Development Status :: 5 - Production/Stable",
18
18
  "Intended Audience :: Developers",
19
19
  "Topic :: Software Development :: Build Tools",
20
+ "Topic :: Database",
20
21
  "Programming Language :: Python :: 3.12",
21
22
  "Programming Language :: SQL",
23
+ # Linux is what CI runs and what this is developed on. Nothing in the codebase
24
+ # is platform-specific, but macOS and Windows are untested — see the README.
25
+ "Operating System :: POSIX :: Linux",
22
26
  ]
23
27
 
24
28
  # Core: the IR + engine + state + plan spine (Phase 1). Everything is Arrow- and
25
29
  # sqlglot-native. See docs/architecture/architecture.md.
26
30
  dependencies = [
27
- "sqlglot>=25.0,<29.0", # canonical IR, transpilation, semantic diff, column lineage
31
+ "sqlglot>=25.0,<30.0", # canonical IR, transpilation, semantic diff, column lineage
28
32
  "duckdb>=1.5.3", # default engine + federation hub; DuckLake storage + quack serving
29
33
  "pyarrow>=17.0", # the wire format: RecordBatchReader everywhere
30
34
  "pydantic>=2.5,<3.0", # config + manifest validation (cold paths only)
31
35
  "typer>=0.12,<1.0", # CLI
32
- "rich>=13.0,<15.0", # display, strictly an event subscriber
36
+ "rich>=13.0,<16.0", # display, strictly an event subscriber
33
37
  "cronsim>=2.5,<3.0", # cron parsing for the trigger engine
34
38
  "tenacity>=8.2,<10.0", # retries (tasks, DuckLake commit conflicts, transfers)
35
39
  "pyyaml>=6.0,<7.0", # project config (config + env overlays)
@@ -78,7 +82,7 @@ dev = [
78
82
  "pytest-asyncio>=1.0,<2.0",
79
83
  "ruff>=0.6,<1.0",
80
84
  "black>=24.0,<27.0",
81
- "mypy>=1.11,<2.0",
85
+ "mypy>=1.11,<3.0",
82
86
  ]
83
87
 
84
88
  [tool.setuptools.packages.find]
@@ -107,7 +107,7 @@ _END = typer.Option("", "--end", help="Window end (ISO), for incremental models.
107
107
  _FORWARD_ONLY = typer.Option(
108
108
  False,
109
109
  "--forward-only",
110
- help="Modified history-keeping models (merge/full_merge/scd/incremental_by_time) carry their history "
110
+ help="Modified history-keeping models (merge/full_merge/scd/incremental) carry their history "
111
111
  "forward: it is copied to the new version, the new logic applies to the copy, and checks gate "
112
112
  "before views move. Requires a shape-compatible change.",
113
113
  )
@@ -205,7 +205,7 @@ async def _render_empty_incrementals(result: ApplyResult, compiled: CompiledProj
205
205
 
206
206
  for name in result.built:
207
207
  model = compiled.models[name]
208
- if model.strategy != "incremental_by_time" or model.is_terminal:
208
+ if model.strategy != "incremental" or model.is_terminal:
209
209
  continue
210
210
  counts = result.rows.get(name)
211
211
  if counts is not None and (counts.inserted or counts.updated):
@@ -404,7 +404,7 @@ def run(
404
404
  ) -> None:
405
405
  """Force-build models and promote, ignoring change detection.
406
406
 
407
- For incremental_by_time models, --start/--end set the catchup window
407
+ For incremental models, --start/--end set the catchup window
408
408
  (default: the latest grain interval).
409
409
  """
410
410
  asyncio.run(_execute(environment, path, select, start, end, restate=False, parallelism=parallelism))
@@ -84,7 +84,7 @@ class EngineConfig(BaseModel):
84
84
  declaring them fails at open until an adapter ships.
85
85
  """
86
86
 
87
- type: str = "ducklake"
87
+ type: str = "duckdb"
88
88
  # Path / URI for DuckDB-family engines. Also accepted on the project top level
89
89
  # as ``database:`` (synthesised into the ``default`` engine).
90
90
  database: str | None = None
@@ -117,13 +117,14 @@ class ProjectConfig(BaseModel):
117
117
  default_engine: str = "default"
118
118
  engines: dict[str, EngineConfig] = Field(default_factory=dict)
119
119
  state_path: str = ".interlace/state.db" # SQLite control-plane database
120
- # The warehouse. Default is DuckLake (Parquet data + SQL catalog) via DuckDB.
121
- # Also accepted: a DuckLake catalog hosted in a SQL database
122
- # ("ducklake:postgres:dbname=... host=..." — pair with data_path/metadata_schema),
123
- # a plain DuckDB file path, ":memory:", or "quack:<host>:<port>" to connect to a
124
- # warehouse served by `interlace serve --quack`.
120
+ # The warehouse. Default is a plain DuckDB file — simplest, single-process.
121
+ # Also accepted: a DuckLake catalog (``ducklake:.interlace/warehouse.ducklake``, or
122
+ # hosted in a SQL DB: ``ducklake:postgres:dbname=... host=...`` — pair with
123
+ # data_path/metadata_schema) which serialises catalog writes so `interlace serve`
124
+ # and a separate CLI can share the warehouse concurrently; ":memory:"; or
125
+ # "quack:<host>:<port>" to connect to a warehouse served by `interlace serve --quack`.
125
126
  # When ``engines.default`` is not set, these top-level fields synthesise it.
126
- database: str = "ducklake:.interlace/warehouse.ducklake"
127
+ database: str = ".interlace/warehouse.duckdb"
127
128
  # The warehouse catalog's ATTACH alias (defaults to ``name``). Set it when a
128
129
  # schema inside the warehouse shares the project name — see EngineConfig.alias.
129
130
  alias: str | None = None
@@ -94,9 +94,9 @@ class ModelDef:
94
94
  dialect: str | None = None
95
95
  engine: str | None = None # named engine from config (None → project default_engine)
96
96
  depends_on: tuple[str, ...] = ()
97
- interval: str | None = None # grain for incremental_by_time (e.g. "1d")
98
- time_column: str | None = None # partition column for incremental_by_time
99
- # First-build window for incremental_by_time: "auto" derives [min, max] of the
97
+ interval: str | None = None # grain for incremental (e.g. "1d")
98
+ time_column: str | None = None # partition column for incremental
99
+ # First-build window for incremental: "auto" derives [min, max] of the
100
100
  # time column from the source at apply time and fills it as ONE interval;
101
101
  # "none" keeps only the latest grain window; an ISO date pins the start.
102
102
  backfill: str = "auto"
@@ -114,6 +114,18 @@ class ModelDef:
114
114
  schedule: dict[str, str] | None = None # {"cron": "0 * * * *"} or {"every": "5m"} for `interlace serve`
115
115
  checks: tuple[CheckSpec, ...] = () # data-quality checks; error severity gates promotion
116
116
 
117
+ def __post_init__(self) -> None:
118
+ # `@model(checks=…)` normalises through parse_checks, but a ModelDef built
119
+ # directly — the dynamic-model path, which is what generated models and dbt
120
+ # migrations use — stored the dicts raw and only failed at compile time with
121
+ # `AttributeError: 'dict' object has no attribute 'type'`. Normalise here too,
122
+ # so one spelling works on both surfaces and a bad check fails at declaration.
123
+ # Passed through as-is, not as list(...): parse_checks already handles a bare
124
+ # CheckSpec and reports a non-list clearly, both of which list() would mangle
125
+ # (TypeError on a CheckSpec; a dict silently degraded to its keys). Always run,
126
+ # so `checks=[]` normalises to the declared tuple rather than staying a list.
127
+ self.checks = parse_checks(self.checks, self.name)
128
+
117
129
  @property
118
130
  def is_terminal(self) -> bool:
119
131
  """A terminal model delivers into an external destination (table/file):
@@ -64,7 +64,7 @@ def _sql_model(default_name: str, sql: str, config: dict[str, Any], default_dial
64
64
  depends_on=_as_tuple(config.get("depends_on") or ()),
65
65
  interval=config.get("interval"),
66
66
  time_column=config.get("time_column"),
67
- backfill=config.get("backfill", "auto"), # first-build window for incremental_by_time
67
+ backfill=config.get("backfill", "auto"), # first-build window for incremental
68
68
  tags=_as_tuple(config.get("tags") or ()),
69
69
  owner=config.get("owner"),
70
70
  description=config.get("description"),
@@ -28,6 +28,7 @@ from __future__ import annotations
28
28
 
29
29
  import asyncio
30
30
  import contextlib
31
+ import re
31
32
  import threading
32
33
  from collections.abc import Iterator, Sequence
33
34
  from uuid import uuid4
@@ -38,6 +39,7 @@ import tenacity
38
39
  from sqlglot import exp
39
40
 
40
41
  from interlace.engines.base import EngineAdapter, EngineCaps, LoadMode
42
+ from interlace.exceptions import ConfigurationError
41
43
  from interlace.ir.relation import TableRef
42
44
 
43
45
  _DUCKDB_CAPS = EngineCaps(
@@ -58,6 +60,39 @@ _commit_retry = tenacity.retry(
58
60
  )
59
61
 
60
62
 
63
+ @contextlib.contextmanager
64
+ def _clean_lock_error(database: str) -> Iterator[None]:
65
+ """Translate DuckDB's file-lock conflict into one actionable line.
66
+
67
+ A DuckDB/DuckLake database is held by a single process. The common way to hit
68
+ that is running a CLI command (`interlace query`, `plan`, `apply`) while
69
+ `interlace serve` is up — an obvious thing to do, since serve is the daemon
70
+ and query is the console's CLI counterpart. Raw, that surfaces as a dozen
71
+ frames ending in `duckdb.IOException`, which names neither the cause nor the
72
+ fix. The fix is `--quack`, and it is already documented; the error just never
73
+ said so.
74
+ """
75
+ try:
76
+ yield
77
+ except duckdb.IOException as exc:
78
+ message = str(exc)
79
+ # "Conflicting lock is held in <exe> (PID n)" is the Linux rendering — DuckDB
80
+ # names the holder from /proc/locks. Elsewhere the message is the bare "Could
81
+ # not set lock on file", and a same-process conflict says "already held", so
82
+ # match all three or macOS/Windows keep the raw traceback.
83
+ if not any(marker in message for marker in ("Conflicting lock", "Could not set lock", "already held")):
84
+ raise
85
+ holder = re.search(r"\(PID (\d+)\)", message)
86
+ held_by = f" (PID {holder.group(1)})" if holder else ""
87
+ raise ConfigurationError(
88
+ f"the warehouse {database!r} is already open in another process{held_by}. "
89
+ "DuckDB allows one process at a time — stop `interlace serve`, or serve the "
90
+ "warehouse over the quack protocol (`interlace serve --quack quack:localhost:4213`) "
91
+ "and point this process at `database: quack:localhost:4213` to share it.",
92
+ details={"database": database},
93
+ ) from None
94
+
95
+
61
96
  def _affected(cur: duckdb.DuckDBPyConnection) -> int:
62
97
  """DML/CTAS/COPY return their affected-row count as a one-cell result; DDL returns
63
98
  nothing. Never raises — row stats are best-effort decoration, not correctness."""
@@ -108,7 +143,9 @@ class DuckDBAdapter(EngineAdapter):
108
143
 
109
144
  @classmethod
110
145
  def connect(cls, path: str) -> DuckDBAdapter:
111
- return cls(duckdb.connect(path), serialise_writes=path.startswith("ducklake:"))
146
+ with _clean_lock_error(path):
147
+ conn = duckdb.connect(path)
148
+ return cls(conn, serialise_writes=path.startswith("ducklake:"))
112
149
 
113
150
  @classmethod
114
151
  def connect_ducklake(
@@ -139,7 +176,8 @@ class DuckDBAdapter(EngineAdapter):
139
176
  options_sql = f" ({', '.join(options)})" if options else ""
140
177
  escaped = catalog.replace("'", "''")
141
178
  alias_sql = exp.to_identifier(alias).sql("duckdb")
142
- conn.execute(f"ATTACH IF NOT EXISTS '{escaped}' AS {alias_sql}{options_sql}")
179
+ with _clean_lock_error(catalog):
180
+ conn.execute(f"ATTACH IF NOT EXISTS '{escaped}' AS {alias_sql}{options_sql}")
143
181
  conn.execute(f"USE {alias_sql}")
144
182
  # LOAD, secrets, and ATTACH are all instance-wide — they carry into every
145
183
  # cursor and must run ONCE (re-running CREATE OR REPLACE SECRET per cursor
@@ -162,10 +200,23 @@ class DuckDBAdapter(EngineAdapter):
162
200
  with contextlib.suppress(Exception):
163
201
  self._conn.interrupt()
164
202
 
203
+ def search_files_from(self, directory: str) -> None:
204
+ """Resolve relative read paths (``read_csv_auto('seeds/x.csv')``) against
205
+ ``directory`` — the project root — as well as the process CWD.
206
+
207
+ Additive: a CWD-relative path still resolves, so this only ever widens what a
208
+ model can find. GLOBAL scope because a plain ``SET`` is session-scoped and would
209
+ not reach the per-task cursors that actually run the queries. Reads only —
210
+ ``COPY`` targets stay CWD-relative, which is why exports resolve their own paths
211
+ against the root (``plan.apply._resolve_export_path``)."""
212
+ escaped = directory.replace("'", "''")
213
+ self._conn.execute(f"SET GLOBAL file_search_path='{escaped}'")
214
+
165
215
  def attach(self, alias: str, uri: str) -> None:
166
216
  """ATTACH another database (duckdb/sqlite/postgres/... URI) under ``alias``."""
167
217
  escaped = uri.replace("'", "''")
168
- self._conn.execute(f"ATTACH IF NOT EXISTS '{escaped}' AS {exp.to_identifier(alias).sql('duckdb')}")
218
+ with _clean_lock_error(uri): # attaching a held duckdb/ducklake file conflicts just like opening one
219
+ self._conn.execute(f"ATTACH IF NOT EXISTS '{escaped}' AS {exp.to_identifier(alias).sql('duckdb')}")
169
220
  self._attached.append(alias)
170
221
  if uri.startswith("ducklake:"): # writes may now reach a DuckLake catalog (e.g. table sinks)
171
222
  if isinstance(self._write_lock, contextlib.nullcontext):
@@ -7,7 +7,7 @@ come back as Arrow via ``DataFrame.toArrow()``, and Arrow loads go in through
7
7
  (``local[*]``, for tests) or a remote one (Spark Connect / a shared session).
8
8
 
9
9
  **Strategy support.** ``replace``, ``append`` and ``view`` run on any Spark
10
- catalog. ``merge`` (native ``MERGE``) and ``incremental_by_time`` (windowed
10
+ catalog. ``merge`` (native ``MERGE``) and ``incremental`` (windowed
11
11
  ``DELETE`` by literal predicate + ``INSERT``) need a catalog with row-level
12
12
  mutations — Delta Lake or Iceberg — configured on the session you hand the
13
13
  adapter (the tests use a Delta-backed local session). ``scd`` and ``full_merge``
@@ -44,9 +44,9 @@ class CompiledModel:
44
44
  materialise: str
45
45
  strategy: str
46
46
  key: tuple[str, ...] # business key for keyed strategies (merge)
47
- time_column: str | None # partition column for incremental_by_time
47
+ time_column: str | None # partition column for incremental
48
48
  cursor: str | None # column whose max is injected into a Python model's `cursor` param
49
- interval: str | None # grain for incremental_by_time (e.g. "1d")
49
+ interval: str | None # grain for incremental (e.g. "1d")
50
50
  tags: tuple[str, ...] # for tag: selection
51
51
  schedule: dict[str, str] | None # cron/interval schedule for the trigger engine
52
52
  columns: dict[str, str | None] | None # output contract validated at apply time
@@ -8,6 +8,8 @@ pruning and the ``impact`` command).
8
8
 
9
9
  from __future__ import annotations
10
10
 
11
+ from typing import cast
12
+
11
13
  import sqlglot
12
14
  from sqlglot import exp
13
15
 
@@ -86,4 +88,5 @@ def resolve_references(ast: exp.Expression, mapping: dict[str, TableRef]) -> exp
86
88
  node.set("catalog", exp.to_identifier(target.catalog) if target.catalog else None)
87
89
  return node
88
90
 
89
- return ast.transform(rewrite)
91
+ # sqlglot 29 loosened transform()'s return annotation; it is an Expression.
92
+ return cast("exp.Expression", ast.transform(rewrite))