interlaced 2.0.2__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. {interlaced-2.0.2/src/interlaced.egg-info → interlaced-2.1.0}/PKG-INFO +20 -7
  2. {interlaced-2.0.2 → interlaced-2.1.0}/README.md +11 -5
  3. {interlaced-2.0.2 → interlaced-2.1.0}/pyproject.toml +19 -3
  4. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/__init__.py +2 -0
  5. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/cli/main.py +148 -31
  6. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/dsl/decorators.py +5 -0
  7. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/dsl/discovery.py +20 -2
  8. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/base.py +7 -3
  9. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/duckdb.py +6 -10
  10. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/exceptions.py +8 -0
  11. interlaced-2.1.0/src/interlace/graph/column_lineage.py +197 -0
  12. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/graph/project.py +8 -0
  13. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/ir/canonicalize.py +12 -0
  14. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/plan/apply.py +15 -3
  15. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/plan/differ.py +3 -3
  16. interlaced-2.1.0/src/interlace/query.py +78 -0
  17. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/runtime/python_model.py +10 -4
  18. interlaced-2.1.0/src/interlace/scaffold.py +81 -0
  19. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/app.py +88 -60
  20. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/app.css +8 -9
  21. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/favicon.svg +1 -1
  22. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/index.html +13 -11
  23. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/api.js +23 -5
  24. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/app.js +70 -17
  25. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/ui.js +60 -7
  26. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/checks.js +0 -0
  27. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/models.js +36 -11
  28. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/overview.js +19 -13
  29. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/plan.js +12 -4
  30. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/query.js +33 -11
  31. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/runs.js +20 -12
  32. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/system.js +16 -3
  33. interlaced-2.1.0/src/interlace/sources/__init__.py +52 -0
  34. interlaced-2.1.0/src/interlace/sources/auth.py +97 -0
  35. interlaced-2.1.0/src/interlace/sources/rest.py +305 -0
  36. interlaced-2.1.0/src/interlace/templates/events/README.md +74 -0
  37. interlaced-2.1.0/src/interlace/templates/events/generate.py +125 -0
  38. interlaced-2.1.0/src/interlace/templates/events/interlace.yaml +6 -0
  39. interlaced-2.1.0/src/interlace/templates/events/models/events.py +26 -0
  40. interlaced-2.1.0/src/interlace/templates/events/models/events_by_minute.sql +14 -0
  41. interlaced-2.1.0/src/interlace/templates/events/models/events_by_type.sql +9 -0
  42. interlaced-2.1.0/src/interlace/templates/events/models/top_users.sql +9 -0
  43. interlaced-2.1.0/src/interlace/templates/events/models/user_spend.sql +7 -0
  44. interlaced-2.1.0/src/interlace/templates/events/template.yaml +3 -0
  45. interlaced-2.1.0/src/interlace/templates/github/README.md +38 -0
  46. interlaced-2.1.0/src/interlace/templates/github/interlace.yaml +5 -0
  47. interlaced-2.1.0/src/interlace/templates/github/models/github_issues.py +84 -0
  48. interlaced-2.1.0/src/interlace/templates/github/models/issues_by_state.sql +15 -0
  49. interlaced-2.1.0/src/interlace/templates/github/template.yaml +3 -0
  50. interlaced-2.1.0/src/interlace/templates/postgres/README.md +62 -0
  51. interlaced-2.1.0/src/interlace/templates/postgres/docker-compose.yml +20 -0
  52. interlaced-2.1.0/src/interlace/templates/postgres/init/seed.sql +20 -0
  53. interlaced-2.1.0/src/interlace/templates/postgres/interlace.yaml +4 -0
  54. interlaced-2.1.0/src/interlace/templates/postgres/models/orders.py +55 -0
  55. interlaced-2.1.0/src/interlace/templates/postgres/models/orders_by_status.sql +14 -0
  56. interlaced-2.1.0/src/interlace/templates/postgres/template.yaml +3 -0
  57. interlaced-2.1.0/src/interlace/templates/quickstart/README.md +20 -0
  58. interlaced-2.1.0/src/interlace/templates/quickstart/interlace.yaml +6 -0
  59. interlaced-2.1.0/src/interlace/templates/quickstart/models/enriched_events.py +23 -0
  60. interlaced-2.1.0/src/interlace/templates/quickstart/models/event_summary.sql +15 -0
  61. interlaced-2.1.0/src/interlace/templates/quickstart/models/raw_events.sql +19 -0
  62. interlaced-2.1.0/src/interlace/templates/quickstart/template.yaml +3 -0
  63. {interlaced-2.0.2 → interlaced-2.1.0/src/interlaced.egg-info}/PKG-INFO +20 -7
  64. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlaced.egg-info/SOURCES.txt +31 -0
  65. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlaced.egg-info/requires.txt +4 -1
  66. interlaced-2.0.2/src/interlace/graph/column_lineage.py +0 -112
  67. interlaced-2.0.2/src/interlace/scaffold.py +0 -83
  68. {interlaced-2.0.2 → interlaced-2.1.0}/LICENSE +0 -0
  69. {interlaced-2.0.2 → interlaced-2.1.0}/MANIFEST.in +0 -0
  70. {interlaced-2.0.2 → interlaced-2.1.0}/setup.cfg +0 -0
  71. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/checks/__init__.py +0 -0
  72. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/checks/builtin.py +0 -0
  73. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/checks/runner.py +0 -0
  74. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/checks/spec.py +0 -0
  75. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/cli/__init__.py +0 -0
  76. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/config/__init__.py +0 -0
  77. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/config/config.py +0 -0
  78. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/contracts.py +0 -0
  79. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/dsl/__init__.py +0 -0
  80. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/dsl/sql_config.py +0 -0
  81. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/__init__.py +0 -0
  82. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/adbc.py +0 -0
  83. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/bigquery.py +0 -0
  84. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/postgres.py +0 -0
  85. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/quack.py +0 -0
  86. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/redshift.py +0 -0
  87. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/registry.py +0 -0
  88. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/snowflake.py +0 -0
  89. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/engines/spark.py +0 -0
  90. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/graph/__init__.py +0 -0
  91. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/graph/dag.py +0 -0
  92. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/graph/selectors.py +0 -0
  93. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/ir/__init__.py +0 -0
  94. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/ir/fingerprint.py +0 -0
  95. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/ir/relation.py +0 -0
  96. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/plan/__init__.py +0 -0
  97. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/plan/plan.py +0 -0
  98. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/plan/resolve.py +0 -0
  99. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/plan/run.py +0 -0
  100. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/project.py +0 -0
  101. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/py.typed +0 -0
  102. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/runtime/__init__.py +0 -0
  103. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/runtime/handles.py +0 -0
  104. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/scheduler/__init__.py +0 -0
  105. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/scheduler/engine.py +0 -0
  106. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/scheduler/triggers.py +0 -0
  107. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/scheduler/worker.py +0 -0
  108. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/__init__.py +0 -0
  109. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/auth.py +0 -0
  110. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/dag.js +0 -0
  111. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/environments.js +0 -0
  112. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/lineage.js +0 -0
  113. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/service/ui/js/views/streams.js +0 -0
  114. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/sinks.py +0 -0
  115. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/state/__init__.py +0 -0
  116. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/state/interval.py +0 -0
  117. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/state/janitor.py +0 -0
  118. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/state/snapshot.py +0 -0
  119. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/state/store.py +0 -0
  120. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/__init__.py +0 -0
  121. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/append.py +0 -0
  122. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/base.py +0 -0
  123. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/full_merge.py +0 -0
  124. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/incremental_by_time.py +0 -0
  125. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/merge.py +0 -0
  126. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/replace.py +0 -0
  127. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/replace_in_place.py +0 -0
  128. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/scd.py +0 -0
  129. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/strategies/view.py +0 -0
  130. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/streaming/__init__.py +0 -0
  131. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/streaming/log.py +0 -0
  132. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/streaming/materializer.py +0 -0
  133. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlace/streaming/schema.py +0 -0
  134. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlaced.egg-info/dependency_links.txt +0 -0
  135. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlaced.egg-info/entry_points.txt +0 -0
  136. {interlaced-2.0.2 → interlaced-2.1.0}/src/interlaced.egg-info/top_level.txt +0 -0
@@ -1,9 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: interlaced
3
- Version: 2.0.2
3
+ Version: 2.1.0
4
4
  Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
5
5
  Author-email: Mark <mark@interlace.sh>
6
6
  License-Expression: MIT
7
+ Project-URL: Homepage, https://interlace.sh
8
+ Project-URL: Documentation, https://interlace.sh/docs
9
+ Project-URL: Source, https://github.com/interlace-sh/interlace
10
+ Project-URL: Changelog, https://github.com/interlace-sh/interlace/blob/master/CHANGELOG.md
11
+ Project-URL: Issues, https://github.com/interlace-sh/interlace/issues
7
12
  Keywords: data,pipeline,orchestration,transformation,etl,streaming,dbt,sqlmesh,duckdb
8
13
  Classifier: Development Status :: 5 - Production/Stable
9
14
  Classifier: Intended Audience :: Developers
@@ -44,8 +49,10 @@ Provides-Extra: polars
44
49
  Requires-Dist: polars<2.0,>=1.0; extra == "polars"
45
50
  Provides-Extra: pandas
46
51
  Requires-Dist: pandas<4.0,>=2.0; extra == "pandas"
52
+ Provides-Extra: sources
53
+ Requires-Dist: httpx<1.0,>=0.27; extra == "sources"
47
54
  Provides-Extra: all
48
- Requires-Dist: interlaced[adbc,polars,postgres,service]; extra == "all"
55
+ Requires-Dist: interlaced[adbc,polars,postgres,service,sources]; extra == "all"
49
56
  Provides-Extra: dev
50
57
  Requires-Dist: pytest<10.0,>=8.0; extra == "dev"
51
58
  Requires-Dist: httpx<1.0,>=0.27; extra == "dev"
@@ -65,7 +72,7 @@ Models are `.sql` files or Python functions; state is versioned snapshots with v
65
72
  environments and a terraform-style plan/apply; everything runs in a single daemon on
66
73
  DuckDB + DuckLake by default.
67
74
 
68
- > **Status: 1.0.** Requires Python 3.12+.
75
+ > **Status: 2.0.** Requires Python 3.12+.
69
76
  > The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
70
77
 
71
78
  ```bash
@@ -186,13 +193,19 @@ attach:
186
193
  ```
187
194
 
188
195
  ```sql
189
- /* interlace: {export: {to: table, target: crm.public.accounts, mode: merge, key: id}} */
196
+ /* interlace:
197
+ materialise: table
198
+ target: crm.public.accounts
199
+ strategy: merge
200
+ key: id
201
+ */
190
202
  SELECT id, tier, lifetime_value FROM account_summary
191
203
  ```
192
204
 
193
- File exports (`to: parquet|csv|json`) work the same way. Sinks are **environment-gated**: by
194
- default the side effect fires only from prod — a dev apply never writes to a live external
195
- table (opt in with `environments: [dev, prod]`).
205
+ Files work the same way — `materialise: file` with `format: parquet | csv | json` and a
206
+ `path`. Terminal models are **environment-gated**: by default the side effect fires only from
207
+ prod, so a dev apply never writes to a live external table (opt in with
208
+ `environments: [dev, prod]`).
196
209
 
197
210
  ## Multi-engine
198
211
 
@@ -8,7 +8,7 @@ Models are `.sql` files or Python functions; state is versioned snapshots with v
8
8
  environments and a terraform-style plan/apply; everything runs in a single daemon on
9
9
  DuckDB + DuckLake by default.
10
10
 
11
- > **Status: 1.0.** Requires Python 3.12+.
11
+ > **Status: 2.0.** Requires Python 3.12+.
12
12
  > The package is published to PyPI as **`interlaced`**; the import name and CLI are `interlace`.
13
13
 
14
14
  ```bash
@@ -129,13 +129,19 @@ attach:
129
129
  ```
130
130
 
131
131
  ```sql
132
- /* interlace: {export: {to: table, target: crm.public.accounts, mode: merge, key: id}} */
132
+ /* interlace:
133
+ materialise: table
134
+ target: crm.public.accounts
135
+ strategy: merge
136
+ key: id
137
+ */
133
138
  SELECT id, tier, lifetime_value FROM account_summary
134
139
  ```
135
140
 
136
- File exports (`to: parquet|csv|json`) work the same way. Sinks are **environment-gated**: by
137
- default the side effect fires only from prod — a dev apply never writes to a live external
138
- table (opt in with `environments: [dev, prod]`).
141
+ Files work the same way — `materialise: file` with `format: parquet | csv | json` and a
142
+ `path`. Terminal models are **environment-gated**: by default the side effect fires only from
143
+ prod, so a dev apply never writes to a live external table (opt in with
144
+ `environments: [dev, prod]`).
139
145
 
140
146
  ## Multi-engine
141
147
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "interlaced"
7
- version = "2.0.2"
7
+ version = "2.1.0"
8
8
  description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.12"
@@ -35,6 +35,15 @@ dependencies = [
35
35
  "pyyaml>=6.0,<7.0", # project config (config + env overlays)
36
36
  ]
37
37
 
38
+ # PyPI renders these as the project-page sidebar links. Without them the page is
39
+ # a dead end — no route from a search result to the docs or the source.
40
+ [project.urls]
41
+ Homepage = "https://interlace.sh"
42
+ Documentation = "https://interlace.sh/docs"
43
+ Source = "https://github.com/interlace-sh/interlace"
44
+ Changelog = "https://github.com/interlace-sh/interlace/blob/master/CHANGELOG.md"
45
+ Issues = "https://github.com/interlace-sh/interlace/issues"
46
+
38
47
  [project.scripts]
39
48
  interlace = "interlace.cli.main:main"
40
49
 
@@ -59,7 +68,10 @@ spark = ["pyspark>=4.0,<4.1", "delta-spark>=4.0,<4.1"]
59
68
  postgres = ["psycopg[binary]>=3.1,<4.0"]
60
69
  polars = ["polars>=1.0,<2.0"]
61
70
  pandas = ["pandas>=2.0,<4.0"]
62
- all = ["interlaced[service,adbc,postgres,polars]"]
71
+ # Building source models (interlace.sources): a sync REST client — auth, paging,
72
+ # retry, rate limiting — that yields Arrow. httpx is the transport.
73
+ sources = ["httpx>=0.27,<1.0"]
74
+ all = ["interlaced[service,adbc,postgres,polars,sources]"]
63
75
  dev = [
64
76
  "pytest>=8.0,<10.0",
65
77
  "httpx>=0.27,<1.0", # litestar's TestClient transport
@@ -73,15 +85,18 @@ dev = [
73
85
  where = ["src"]
74
86
 
75
87
  [tool.setuptools.package-data]
76
- interlace = ["py.typed", "service/ui/*", "service/ui/**/*"]
88
+ interlace = ["py.typed", "service/ui/*", "service/ui/**/*", "templates/*", "templates/**/*"]
77
89
 
78
90
  [tool.black]
79
91
  line-length = 120
80
92
  target-version = ['py312']
93
+ # init templates are user-facing example projects, not library code
94
+ extend-exclude = 'src/interlace/templates/'
81
95
 
82
96
  [tool.ruff]
83
97
  line-length = 120
84
98
  target-version = "py312"
99
+ extend-exclude = ["src/interlace/templates"]
85
100
 
86
101
  [tool.ruff.lint]
87
102
  select = [
@@ -104,6 +119,7 @@ ignore = [
104
119
 
105
120
  [tool.mypy]
106
121
  python_version = "3.12"
122
+ exclude = ['src/interlace/templates/'] # example project code, not library code
107
123
  warn_return_any = true
108
124
  warn_unused_configs = true
109
125
  disallow_untyped_defs = true
@@ -11,6 +11,7 @@ from __future__ import annotations
11
11
 
12
12
  from importlib.metadata import PackageNotFoundError, version
13
13
 
14
+ from interlace.checks.spec import CheckSpec
14
15
  from interlace.dsl.decorators import check, model, stream
15
16
 
16
17
  try:
@@ -20,6 +21,7 @@ except PackageNotFoundError: # running from a source tree without an install
20
21
 
21
22
  __all__ = [
22
23
  "__version__",
24
+ "CheckSpec",
23
25
  "check",
24
26
  "model",
25
27
  "stream",
@@ -11,10 +11,11 @@ from typing import Any
11
11
  import typer
12
12
  from rich import box
13
13
  from rich.console import Console
14
+ from rich.markup import escape
14
15
  from rich.progress import Progress, SpinnerColumn, TaskID, TextColumn, TimeElapsedColumn
15
16
  from rich.table import Table
16
17
 
17
- from interlace.exceptions import CheckError, ConfigurationError, InterlaceError, SelectionError
18
+ from interlace.exceptions import CheckError, ConfigurationError, InterlaceError, QueryError, SelectionError
18
19
  from interlace.graph.column_lineage import column_impact, column_lineage, split_target
19
20
  from interlace.graph.project import CompiledProject
20
21
  from interlace.graph.selectors import select_models, wants_state
@@ -24,7 +25,7 @@ from interlace.plan.differ import diff
24
25
  from interlace.plan.plan import ChangeType, Plan
25
26
  from interlace.plan.run import run_plan
26
27
  from interlace.project import Project
27
- from interlace.scaffold import scaffold_project
28
+ from interlace.scaffold import list_templates, scaffold_project
28
29
  from interlace.scheduler.engine import TriggerEngine, build_triggers
29
30
  from interlace.scheduler.worker import drain
30
31
  from interlace.sinks import target_ref
@@ -58,12 +59,16 @@ class _BuildProgress:
58
59
  def __call__(self, model: str, event: str) -> None:
59
60
  if event == "start":
60
61
  self._rows[model] = self.progress.add_task(model, total=1, status="")
61
- elif event == "done":
62
- self.progress.update(self._rows[model], completed=1, status="[green]✓[/green]")
62
+ return
63
+ task = self._rows.get(model)
64
+ if task is None: # a model cancelled by a sibling's failure before it ever started has no row
65
+ return
66
+ if event == "done":
67
+ self.progress.update(task, completed=1, status="[green]✓[/green]")
63
68
  elif event == "cancelled": # collateral of a sibling's failure, not a failure itself
64
- self.progress.update(self._rows[model], completed=1, status="[dim]⊘[/dim]")
69
+ self.progress.update(task, completed=1, status="[dim]⊘[/dim]")
65
70
  else: # failed
66
- self.progress.update(self._rows[model], completed=1, status="[red]✗[/red]")
71
+ self.progress.update(task, completed=1, status="[red]✗[/red]")
67
72
 
68
73
 
69
74
  def _build_progress(plan_result: Plan) -> _BuildProgress | None:
@@ -224,7 +229,7 @@ def _selection(
224
229
  try:
225
230
  return select_models(selectors, compiled, promoted=promoted)
226
231
  except SelectionError as exc:
227
- console.print(f"[red]{exc.message}[/red]")
232
+ console.print(f"[red]{escape(exc.message)}[/red]")
228
233
  raise typer.Exit(1) from exc
229
234
 
230
235
 
@@ -237,16 +242,30 @@ async def _promoted_if_needed(state: Any, environment: str, selectors: list[str]
237
242
  def init(
238
243
  path: Path = typer.Argument(Path("."), help="Directory to initialise."),
239
244
  name: str = typer.Option("", "--name", "-n", help="Project name (defaults to the directory name)."),
245
+ template: str = typer.Option("quickstart", "--template", "-t", help="Which starter to scaffold (see --list)."),
246
+ show_list: bool = typer.Option(False, "--list", help="List available templates and exit."),
240
247
  ) -> None:
241
- """Scaffold a new interlace project."""
248
+ """Scaffold a new interlace project from a template."""
249
+ if show_list:
250
+ table = _table("Templates")
251
+ table.add_column("Template")
252
+ table.add_column("Description", style="dim")
253
+ table.add_column("Needs", style="dim")
254
+ for info in list_templates():
255
+ table.add_row(info.name, info.description, ", ".join(info.requires_env) or "—")
256
+ console.print(table)
257
+ return
242
258
  try:
243
- written = scaffold_project(path, name or None)
259
+ written = scaffold_project(path, name or None, template)
244
260
  except ConfigurationError as exc:
245
- console.print(f"[red]{exc.message}[/red] ({exc.details.get('path', '')})")
261
+ console.print(f"[red]{escape(exc.message)}[/red] ({exc.details.get('path', '')})")
246
262
  raise typer.Exit(1) from exc
247
- console.print(f"[green]Initialised interlace project in {path}[/green]")
263
+ console.print(f"[green]Initialised interlace project in {path}[/green] [dim](template: {template})[/dim]")
248
264
  for written_path in written:
249
265
  console.print(f" + {written_path}")
266
+ needs = next((t.requires_env for t in list_templates() if t.name == template), ())
267
+ if needs:
268
+ console.print(f"\n[yellow]Set before applying:[/yellow] {', '.join(needs)}")
250
269
  console.print("\nNext: [bold]interlace apply[/bold] (or --env dev for a sandbox)")
251
270
 
252
271
 
@@ -360,7 +379,7 @@ async def _apply(
360
379
  parallelism=parallelism or project.config.parallelism, # --parallelism wins over config
361
380
  )
362
381
  except CheckError as exc:
363
- console.print(f"[red]{exc.message}[/red]")
382
+ console.print(f"[red]{escape(exc.message)}[/red]")
364
383
  raise typer.Exit(1) from exc
365
384
  _render_build_results(result, compiled)
366
385
  _render_checks(result)
@@ -459,7 +478,7 @@ async def _execute(
459
478
  parallelism=parallelism or project.config.parallelism, # --parallelism wins over config
460
479
  )
461
480
  except CheckError as exc:
462
- console.print(f"[red]{exc.message}[/red]")
481
+ console.print(f"[red]{escape(exc.message)}[/red]")
463
482
  raise typer.Exit(1) from exc
464
483
  _render_build_results(result, compiled)
465
484
  _render_checks(result)
@@ -631,7 +650,7 @@ def serve(
631
650
 
632
651
  from interlace.service.app import create_app
633
652
  except ImportError as exc:
634
- console.print("[red]The HTTP API needs the 'service' extra: pip install 'interlaced[service]'[/red]")
653
+ console.print(r"[red]The HTTP API needs the 'service' extra: pip install 'interlaced\[service]'[/red]")
635
654
  raise typer.Exit(1) from exc
636
655
  bound = _free_port(host, port)
637
656
  if bound != port:
@@ -655,23 +674,78 @@ def serve(
655
674
  token = secrets.token_hex(8)
656
675
  console.print(f"[bold]quack[/bold] warehouse at [cyan]{quack}[/cyan] · token [yellow]{token}[/yellow]")
657
676
  console.print("Clients: set [bold]database: quack:...[/bold] and INTERLACE_QUACK_TOKEN in the environment.")
658
- uvicorn.run(
659
- create_app(
660
- path,
661
- environment,
662
- quack=quack or None,
663
- quack_token=token or None,
664
- scheduler=scheduler,
665
- scheduler_interval=interval,
666
- ),
677
+ app = create_app(
678
+ path,
679
+ environment,
680
+ quack=quack or None,
681
+ quack_token=token or None,
682
+ scheduler=scheduler,
683
+ scheduler_interval=interval,
684
+ )
685
+ config = uvicorn.Config(
686
+ app,
667
687
  host=host,
668
688
  port=port,
669
- # SSE clients (/events/stream) hold their response open forever; without a
670
- # bound, uvicorn's graceful shutdown waits on them indefinitely ("Waiting
671
- # for connections to close") and Ctrl+C appears to hang. Lifespan cleanup
672
- # (flush, store/engine close) runs only after this drain completes.
689
+ # SSE clients (/events/stream) hold their response open forever. The app's
690
+ # shutdown watcher ends them the instant this server flips `should_exit`, so
691
+ # the graceful-shutdown drain finds the connections already closed instead of
692
+ # force-cancelling them (which used to dump a CancelledError traceback on
693
+ # Ctrl+C). The bound is a backstop; lifespan cleanup runs after the drain.
673
694
  timeout_graceful_shutdown=3,
674
695
  )
696
+ server = uvicorn.Server(config)
697
+ app.state.uvicorn_server = server # let the app release SSE streams as shutdown begins
698
+ server.run()
699
+
700
+
701
+ @app.command()
702
+ def query(
703
+ sql: str = typer.Argument(..., help="A read-only SELECT to run against the warehouse."),
704
+ path: Path = _PATH,
705
+ limit: int = typer.Option(100, "--limit", "-n", help="Maximum rows to display (max 10,000)."),
706
+ ) -> None:
707
+ """Run a read-only SELECT against the warehouse and print the result.
708
+
709
+ SELECT only — real tables and views, not table functions or files (the same fence
710
+ as the web console). Unqualified names resolve to the promoted (prod) views:
711
+
712
+ interlace query "SELECT * FROM raw_events"
713
+ """
714
+ asyncio.run(_query(sql, path, limit))
715
+
716
+
717
+ async def _query(sql: str, path: Path, limit: int) -> None:
718
+ from interlace.query import prepare_readonly
719
+
720
+ project = Project.load(path)
721
+ engines = project.open_engines()
722
+ try:
723
+ engine = engines.get()
724
+ bounded, cap = prepare_readonly(sql, engine.dialect, limit)
725
+ try:
726
+ reader = await engine.fetch(bounded)
727
+ columns = list(reader.schema.names)
728
+ records = await asyncio.to_thread(lambda: reader.read_all().to_pylist())
729
+ except QueryError:
730
+ raise
731
+ except Exception as exc: # engine errors (missing table, bad column) are the user's feedback
732
+ raise QueryError(str(exc)) from exc
733
+ finally:
734
+ engines.close()
735
+ _render_query(columns, records, cap)
736
+
737
+
738
+ def _render_query(columns: list[str], records: list[dict], cap: int) -> None:
739
+ truncated = len(records) > cap
740
+ shown = records[:cap]
741
+ table = _table("")
742
+ for name in columns:
743
+ table.add_column(name)
744
+ for record in shown:
745
+ table.add_row(*("[dim]NULL[/dim]" if record[name] is None else escape(str(record[name])) for name in columns))
746
+ console.print(table)
747
+ note = f"{len(shown)} row(s)" + (" — truncated; raise --limit for more" if truncated else "")
748
+ console.print(f"[dim]{note}[/dim]")
675
749
 
676
750
 
677
751
  @app.command("models")
@@ -794,7 +868,7 @@ async def _env_rollback(name: str, path: Path, to: int | None, history: bool, as
794
868
  try:
795
869
  result = await rollback_environment(state, engines=engines, environment=name, to_generation=to)
796
870
  except PlanError as exc:
797
- console.print(f"[red]{exc.message}[/red]")
871
+ console.print(f"[red]{escape(exc.message)}[/red]")
798
872
  raise typer.Exit(1) from exc
799
873
  if as_json:
800
874
  _emit_json(result)
@@ -1123,6 +1197,28 @@ def engines(path: Path = _PATH, as_json: bool = _JSON) -> None:
1123
1197
  console.print(table)
1124
1198
 
1125
1199
 
1200
+ async def _warehouse_columns(project: Project, compiled: CompiledProject) -> dict[str, list[str]]:
1201
+ """Best-effort real output columns per model, probed from the built warehouse so
1202
+ column lineage traces precisely through Python models (a Python model's true
1203
+ columns can't be known statically). Empty on any failure — an unbuilt project or
1204
+ a warehouse held by a running ``serve`` — and lineage falls back to static analysis."""
1205
+ from sqlglot import exp
1206
+
1207
+ try:
1208
+ engines = project.open_engines()
1209
+ except Exception:
1210
+ return {}
1211
+ described: dict[str, list[str]] = {}
1212
+ try:
1213
+ for name, model in compiled.models.items():
1214
+ with contextlib.suppress(Exception): # unbuilt / non-queryable (file target): names-only is fine
1215
+ reader = await engines.get(model.engine).fetch(exp.select("*").from_(exp.to_table(name)).limit(0))
1216
+ described[name] = list(reader.schema.names)
1217
+ finally:
1218
+ engines.close()
1219
+ return described
1220
+
1221
+
1126
1222
  @app.command()
1127
1223
  def impact(
1128
1224
  target: str = typer.Argument(..., help="model.column — what would changing this column touch?"),
@@ -1138,7 +1234,8 @@ def impact(
1138
1234
  console.print(f"[red]expected <model>.<column> with a known model; got {target!r}[/red]")
1139
1235
  raise typer.Exit(1)
1140
1236
  model, column = parsed
1141
- result = column_impact(compiled, model, column)
1237
+ described = asyncio.run(_warehouse_columns(project, compiled))
1238
+ result = column_impact(compiled, model, column, known_columns=described)
1142
1239
  impacted = result["impacted"]
1143
1240
  opaque = result["opaque_consumers"]
1144
1241
 
@@ -1181,7 +1278,8 @@ def lineage(
1181
1278
 
1182
1279
  upstream = sorted(compiled.graph.ancestors(model))
1183
1280
  downstream = sorted(compiled.graph.descendants(model))
1184
- sources = column_lineage(compiled).get(model, {}) if columns else {}
1281
+ described = asyncio.run(_warehouse_columns(project, compiled)) if columns else {}
1282
+ sources = column_lineage(compiled, known_columns=described).get(model, {}) if columns else {}
1185
1283
 
1186
1284
  if fmt == "dot":
1187
1285
  typer.echo(_lineage_dot(compiled, model, upstream, downstream, sources))
@@ -1343,9 +1441,28 @@ def _render(plan: Plan, environment: str) -> None:
1343
1441
  )
1344
1442
 
1345
1443
 
1444
+ def _flatten_exceptions(exc: BaseException) -> list[BaseException]:
1445
+ """Leaf exceptions of a (possibly nested) ExceptionGroup — a parallel apply
1446
+ reports its failures as one."""
1447
+ if isinstance(exc, BaseExceptionGroup):
1448
+ return [leaf for sub in exc.exceptions for leaf in _flatten_exceptions(sub)]
1449
+ return [exc]
1450
+
1451
+
1346
1452
  def main() -> None:
1347
1453
  try:
1348
1454
  app()
1349
1455
  except InterlaceError as exc: # expected, user-facing errors: one clean line, no traceback
1350
- err_console.print(f"[red]error:[/red] {exc.message}")
1456
+ err_console.print(f"[red]error:[/red] {escape(exc.message)}")
1351
1457
  raise SystemExit(1) from None
1458
+ except BaseExceptionGroup as group:
1459
+ # A parallel apply surfaces failures as an ExceptionGroup. When every leaf is a
1460
+ # user-facing InterlaceError (e.g. several models failed their definition/checks),
1461
+ # print one clean line each instead of dumping the group traceback; a genuine
1462
+ # internal error in the mix still propagates with its trace.
1463
+ leaves = _flatten_exceptions(group)
1464
+ if leaves and all(isinstance(leaf, InterlaceError) for leaf in leaves):
1465
+ for message in dict.fromkeys(leaf.message for leaf in leaves if isinstance(leaf, InterlaceError)):
1466
+ err_console.print(f"[red]error:[/red] {escape(message)}")
1467
+ raise SystemExit(1) from None
1468
+ raise
@@ -201,6 +201,11 @@ def model(
201
201
 
202
202
  ``engine`` pins the model to a named engine from ``interlace.yaml`` (defaults
203
203
  to the project's ``default_engine``).
204
+
205
+ ``checks`` takes the *same* shorthand mappings as a SQL config block — one
206
+ spelling for both surfaces — e.g. ``checks=[{"not_null": "customer_id"},
207
+ {"row_count": {"min": 1}}]``. A :class:`~interlace.CheckSpec` also works if you
208
+ prefer the typed form.
204
209
  """
205
210
  model_name = name or "<model>"
206
211
  if export is not None:
@@ -17,7 +17,7 @@ from typing import Any
17
17
  from interlace.checks.spec import parse_checks
18
18
  from interlace.dsl.decorators import REGISTRY, ModelDef, _as_columns, _as_tuple, validate_materialise
19
19
  from interlace.dsl.sql_config import extract_sql_config
20
- from interlace.exceptions import DefinitionError
20
+ from interlace.exceptions import DefinitionError, InterlaceError
21
21
 
22
22
 
23
23
  def discover_models(root: Path, model_paths: list[str], default_dialect: str) -> list[ModelDef]:
@@ -82,6 +82,13 @@ def _model_name(base: Path, file: Path) -> str:
82
82
  return ".".join(file.relative_to(base).with_suffix("").parts)
83
83
 
84
84
 
85
+ def _relative_to_cwd(file: Path) -> str:
86
+ try:
87
+ return str(file.relative_to(Path.cwd()))
88
+ except ValueError:
89
+ return str(file)
90
+
91
+
85
92
  def _import_module(base: Path, file: Path) -> None:
86
93
  module_name = "interlace_model_" + "_".join(file.relative_to(base).with_suffix("").parts)
87
94
  spec = importlib.util.spec_from_file_location(module_name, file)
@@ -89,4 +96,15 @@ def _import_module(base: Path, file: Path) -> None:
89
96
  raise DefinitionError("could not import model module", details={"path": str(file)})
90
97
  module = importlib.util.module_from_spec(spec)
91
98
  sys.modules[module_name] = module
92
- spec.loader.exec_module(module)
99
+ try:
100
+ spec.loader.exec_module(module)
101
+ except InterlaceError:
102
+ raise # a bad @model config is already a clean, actionable error — don't rewrap it
103
+ except Exception as exc:
104
+ # a typo/import error in user model code should read like a user error — one
105
+ # line naming the file — not a dozen frames of interlace's import machinery
106
+ sys.modules.pop(module_name, None) # don't leave a half-initialised module behind
107
+ raise DefinitionError(
108
+ f"could not load {_relative_to_cwd(file)}: {type(exc).__name__}: {exc}",
109
+ details={"path": str(file)},
110
+ ) from exc
@@ -85,9 +85,13 @@ class EngineAdapter(ABC):
85
85
  async def fetch_sandboxed(self, ast: exp.Expression) -> pa.RecordBatchReader:
86
86
  """Like :meth:`fetch`, but for untrusted queries (the HTTP query console):
87
87
  the engine must not touch anything outside the warehouse — no local files,
88
- no network. The default cannot enforce that, so it is the caller's job to
89
- gate this path; adapters over an embedded engine that CAN read the host
90
- filesystem (DuckDB) MUST override to lock the connection down."""
88
+ no network. The engine cannot always enforce that itself (DuckDB's
89
+ ``enable_external_access`` is instance-wide and one-way, so disabling it on
90
+ the shared warehouse connection would permanently break the writer's own
91
+ file writes), so **the caller MUST gate this path** — the service validates
92
+ the SELECT's AST (real tables only, no table functions, no file paths) before
93
+ calling here. An engine with a genuinely isolated read-only mode may override
94
+ to add defence in depth."""
91
95
  return await self.fetch(ast)
92
96
 
93
97
  def transpile(self, ast: exp.Expression) -> str:
@@ -203,8 +203,11 @@ class DuckDBAdapter(EngineAdapter):
203
203
  async def fetch_sql(self, sql: str) -> pa.RecordBatchReader:
204
204
  return await asyncio.to_thread(self._fetch_sync, sql)
205
205
 
206
- async def fetch_sandboxed(self, ast: exp.Expression) -> pa.RecordBatchReader:
207
- return await asyncio.to_thread(self._fetch_sync, self.transpile(ast), sandboxed=True)
206
+ # No fetch_sandboxed override: DuckDB cannot sandbox one query on the shared
207
+ # warehouse connection without poisoning the writer (enable_external_access is
208
+ # instance-wide and one-way, and a DuckLake catalog can be held by only one
209
+ # connection per process). The untrusted /query path is fenced at parse time by
210
+ # the service's _guard_console_query instead — as the base contract expects.
208
211
 
209
212
  async def create_schema(self, name: str) -> None:
210
213
  await self.execute_sql(f"CREATE SCHEMA IF NOT EXISTS {exp.to_identifier(name).sql(dialect=self.dialect)}")
@@ -244,19 +247,12 @@ class DuckDBAdapter(EngineAdapter):
244
247
  cur.close()
245
248
  return counts
246
249
 
247
- def _fetch_sync(self, sql: str, *, sandboxed: bool = False) -> pa.RecordBatchReader:
250
+ def _fetch_sync(self, sql: str) -> pa.RecordBatchReader:
248
251
  # Read-only: deliberately not locked, so scans run concurrently (DuckDB MVCC).
249
252
  # The cursor must outlive the stream that reads from it, so it is closed when
250
253
  # that stream ends rather than left for the garbage collector to reclaim on
251
254
  # whatever thread happens to drop the last reference.
252
255
  cur = self._cursor()
253
- if sandboxed:
254
- # Untrusted query (the HTTP console). enable_external_access is
255
- # connection-local, so this cursor can neither read host files nor
256
- # open network connections — closing the query()/read_csv()/httpfs
257
- # escape hatches at the engine level, not by name-matching. The build
258
- # path (other cursors) keeps external access for exports and ATTACH.
259
- cur.execute("SET enable_external_access = false")
260
256
  cur.execute(sql)
261
257
  reader = cur.to_arrow_reader()
262
258
 
@@ -42,6 +42,14 @@ class PlanError(InterlaceError):
42
42
  """A plan could not be computed or applied."""
43
43
 
44
44
 
45
+ class ExecutionError(InterlaceError):
46
+ """A model failed while building (its query, its Python function, or the load)."""
47
+
48
+
49
+ class QueryError(InterlaceError):
50
+ """A read-only query (the console / `interlace query`) was rejected or failed."""
51
+
52
+
45
53
  class SchemaError(InterlaceError):
46
54
  """A model's built schema violates its declared column contract."""
47
55