duckstring 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. {duckstring-0.1.0 → duckstring-0.2.0}/PKG-INFO +61 -199
  2. duckstring-0.2.0/README.md +85 -0
  3. duckstring-0.2.0/pyproject.toml +65 -0
  4. duckstring-0.2.0/src/duckstring/__init__.py +28 -0
  5. duckstring-0.2.0/src/duckstring/catchment/app.py +110 -0
  6. duckstring-0.2.0/src/duckstring/catchment/asgi.py +30 -0
  7. duckstring-0.2.0/src/duckstring/catchment/dag.py +84 -0
  8. duckstring-0.2.0/src/duckstring/catchment/db.py +39 -0
  9. duckstring-0.2.0/src/duckstring/catchment/driver.py +836 -0
  10. duckstring-0.2.0/src/duckstring/catchment/launcher.py +94 -0
  11. duckstring-0.2.0/src/duckstring/catchment/registry.py +27 -0
  12. duckstring-0.2.0/src/duckstring/catchment/routes/__init__.py +16 -0
  13. duckstring-0.2.0/src/duckstring/catchment/routes/catchment.py +121 -0
  14. duckstring-0.2.0/src/duckstring/catchment/routes/data.py +143 -0
  15. duckstring-0.2.0/src/duckstring/catchment/routes/deploy.py +212 -0
  16. duckstring-0.2.0/src/duckstring/catchment/routes/duck.py +31 -0
  17. duckstring-0.2.0/src/duckstring/catchment/routes/orchestrate.py +209 -0
  18. duckstring-0.2.0/src/duckstring/catchment/schema/001_init.sql +158 -0
  19. duckstring-0.2.0/src/duckstring/catchment/static/404.html +1 -0
  20. duckstring-0.2.0/src/duckstring/catchment/static/__next.__PAGE__.txt +7 -0
  21. duckstring-0.2.0/src/duckstring/catchment/static/__next._full.txt +16 -0
  22. duckstring-0.2.0/src/duckstring/catchment/static/__next._head.txt +6 -0
  23. duckstring-0.2.0/src/duckstring/catchment/static/__next._index.txt +5 -0
  24. duckstring-0.2.0/src/duckstring/catchment/static/__next._tree.txt +3 -0
  25. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/10u3y4bw1ayzs.js → duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/07lhk_q6pmm3r.js +1 -1
  26. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0dbhjjzl8qfwv.js +1 -0
  27. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0jvmviuftg5e2.css +1 -0
  28. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0kczw6usu-y-f.js +5 -0
  29. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0pgf-_4l3d1gl.css +1 -0
  30. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0qh0m742pxib~.js +2 -0
  31. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/11kjahy2ntf0n.js +31 -0
  32. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/turbopack-03~mbvk_uplk_.js +1 -0
  33. duckstring-0.2.0/src/duckstring/catchment/static/_next/static/uFzbxKIUibAOY83mw7FJ7/_buildManifest.js +16 -0
  34. duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._full.txt +16 -0
  35. duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._head.txt +6 -0
  36. duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._index.txt +5 -0
  37. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/_not-found/__next._not-found.__PAGE__.txt +2 -2
  38. duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._not-found.txt +5 -0
  39. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/_not-found/__next._tree.txt +2 -2
  40. duckstring-0.2.0/src/duckstring/catchment/static/_not-found.html +1 -0
  41. duckstring-0.2.0/src/duckstring/catchment/static/_not-found.txt +16 -0
  42. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/favicon.svg +1 -1
  43. duckstring-0.2.0/src/duckstring/catchment/static/index.html +1 -0
  44. duckstring-0.2.0/src/duckstring/catchment/static/index.txt +16 -0
  45. duckstring-0.2.0/src/duckstring/catchment/static/logo-mark.svg +15 -0
  46. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/logo.svg +1 -1
  47. duckstring-0.2.0/src/duckstring/cli/__init__.py +73 -0
  48. duckstring-0.2.0/src/duckstring/cli/_http.py +79 -0
  49. duckstring-0.2.0/src/duckstring/cli/catchment.py +368 -0
  50. duckstring-0.2.0/src/duckstring/cli/config.py +122 -0
  51. duckstring-0.2.0/src/duckstring/cli/control.py +126 -0
  52. duckstring-0.2.0/src/duckstring/cli/data.py +133 -0
  53. duckstring-0.2.0/src/duckstring/cli/deploy.py +150 -0
  54. duckstring-0.2.0/src/duckstring/cli/pond.py +212 -0
  55. duckstring-0.2.0/src/duckstring/cli/puddle.py +141 -0
  56. duckstring-0.2.0/src/duckstring/cli/status.py +290 -0
  57. duckstring-0.2.0/src/duckstring/cli/trigger.py +110 -0
  58. duckstring-0.2.0/src/duckstring/cli/window.py +178 -0
  59. duckstring-0.2.0/src/duckstring/core.py +368 -0
  60. duckstring-0.2.0/src/duckstring/demo/products/.gitignore +8 -0
  61. duckstring-0.2.0/src/duckstring/demo/products/README.md +9 -0
  62. duckstring-0.2.0/src/duckstring/demo/products/pond.toml +4 -0
  63. duckstring-0.2.0/src/duckstring/demo/products/src/pond.py +66 -0
  64. duckstring-0.2.0/src/duckstring/demo/reports/.gitignore +8 -0
  65. duckstring-0.2.0/src/duckstring/demo/reports/README.md +11 -0
  66. duckstring-0.2.0/src/duckstring/demo/reports/pond.toml +7 -0
  67. duckstring-0.2.0/src/duckstring/demo/reports/src/pond.py +24 -0
  68. duckstring-0.2.0/src/duckstring/demo/sales/.gitignore +8 -0
  69. duckstring-0.2.0/src/duckstring/demo/sales/README.md +11 -0
  70. duckstring-0.2.0/src/duckstring/demo/sales/pond.toml +7 -0
  71. duckstring-0.2.0/src/duckstring/demo/sales/src/pond.py +67 -0
  72. duckstring-0.2.0/src/duckstring/demo/sales/src/puddles.py +47 -0
  73. duckstring-0.2.0/src/duckstring/demo/transactions/.gitignore +8 -0
  74. duckstring-0.2.0/src/duckstring/demo/transactions/README.md +9 -0
  75. duckstring-0.2.0/src/duckstring/demo/transactions/pond.toml +4 -0
  76. duckstring-0.2.0/src/duckstring/demo/transactions/src/pond.py +46 -0
  77. duckstring-0.2.0/src/duckstring/duck/__init__.py +8 -0
  78. duckstring-0.2.0/src/duckstring/duck/__main__.py +169 -0
  79. duckstring-0.2.0/src/duckstring/duck/client.py +46 -0
  80. duckstring-0.2.0/src/duckstring/duck/core.py +144 -0
  81. duckstring-0.2.0/src/duckstring/duck/executor.py +142 -0
  82. duckstring-0.2.0/src/duckstring/engine/__init__.py +95 -0
  83. duckstring-0.2.0/src/duckstring/engine/catchment.py +573 -0
  84. duckstring-0.2.0/src/duckstring/engine/core.py +219 -0
  85. duckstring-0.2.0/src/duckstring/engine/pond.py +122 -0
  86. duckstring-0.2.0/src/duckstring/engine/worker.py +177 -0
  87. duckstring-0.2.0/src/duckstring/keys.py +22 -0
  88. duckstring-0.2.0/src/duckstring/local/__init__.py +12 -0
  89. duckstring-0.2.0/src/duckstring/local/hydrate.py +114 -0
  90. duckstring-0.2.0/src/duckstring/local/project.py +55 -0
  91. duckstring-0.2.0/src/duckstring/local/runner.py +147 -0
  92. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/PKG-INFO +61 -199
  93. duckstring-0.2.0/src/duckstring.egg-info/SOURCES.txt +125 -0
  94. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/requires.txt +7 -0
  95. duckstring-0.2.0/tests/test_auth.py +258 -0
  96. duckstring-0.2.0/tests/test_catchment.py +289 -0
  97. duckstring-0.2.0/tests/test_catchment_app.py +674 -0
  98. duckstring-0.2.0/tests/test_config.py +70 -0
  99. duckstring-0.2.0/tests/test_core.py +94 -0
  100. duckstring-0.2.0/tests/test_data.py +148 -0
  101. duckstring-0.2.0/tests/test_deploy.py +146 -0
  102. duckstring-0.2.0/tests/test_download.py +77 -0
  103. duckstring-0.2.0/tests/test_duck.py +175 -0
  104. duckstring-0.2.0/tests/test_engine.py +591 -0
  105. duckstring-0.2.0/tests/test_engine_split.py +228 -0
  106. duckstring-0.2.0/tests/test_execute.py +149 -0
  107. duckstring-0.2.0/tests/test_multimajor.py +188 -0
  108. duckstring-0.2.0/tests/test_platform.py +155 -0
  109. duckstring-0.2.0/tests/test_pond.py +156 -0
  110. duckstring-0.2.0/tests/test_puddle.py +392 -0
  111. duckstring-0.2.0/tests/test_restart.py +98 -0
  112. duckstring-0.2.0/tests/test_runtime.py +233 -0
  113. duckstring-0.2.0/tests/test_static.py +23 -0
  114. duckstring-0.2.0/tests/test_status.py +198 -0
  115. duckstring-0.2.0/tests/test_status_api.py +233 -0
  116. duckstring-0.2.0/tests/test_window.py +193 -0
  117. duckstring-0.1.0/README.md +0 -232
  118. duckstring-0.1.0/pyproject.toml +0 -46
  119. duckstring-0.1.0/src/duckstring/__init__.py +0 -13
  120. duckstring-0.1.0/src/duckstring/catchment/app.py +0 -9
  121. duckstring-0.1.0/src/duckstring/catchment/routes/__init__.py +0 -0
  122. duckstring-0.1.0/src/duckstring/catchment/routes/catchment.py +0 -0
  123. duckstring-0.1.0/src/duckstring/catchment/static/404.html +0 -1
  124. duckstring-0.1.0/src/duckstring/catchment/static/__next.__PAGE__.txt +0 -6
  125. duckstring-0.1.0/src/duckstring/catchment/static/__next._full.txt +0 -15
  126. duckstring-0.1.0/src/duckstring/catchment/static/__next._head.txt +0 -6
  127. duckstring-0.1.0/src/duckstring/catchment/static/__next._index.txt +0 -5
  128. duckstring-0.1.0/src/duckstring/catchment/static/__next._tree.txt +0 -2
  129. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/07mne3zcixcz2.js +0 -3
  130. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0d-5~.f~0wtco.js +0 -5
  131. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0d3tkj33oycf2.js +0 -1
  132. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0fsi8lw5utejl.js +0 -1
  133. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0j18t3ydystkv.css +0 -1
  134. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/11qk_vpw_zwkf.js +0 -31
  135. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/turbopack-0e47rfx~.~3-8.js +0 -1
  136. duckstring-0.1.0/src/duckstring/catchment/static/_next/static/v34k52t11dNGCtVyPJ3io/_buildManifest.js +0 -11
  137. duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._full.txt +0 -16
  138. duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._head.txt +0 -6
  139. duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._index.txt +0 -5
  140. duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._not-found.txt +0 -5
  141. duckstring-0.1.0/src/duckstring/catchment/static/_not-found.html +0 -1
  142. duckstring-0.1.0/src/duckstring/catchment/static/_not-found.txt +0 -16
  143. duckstring-0.1.0/src/duckstring/catchment/static/dev/cache/turbopack/ee6e79b1/CURRENT +0 -0
  144. duckstring-0.1.0/src/duckstring/catchment/static/dev/logs/next-development.log +0 -0
  145. duckstring-0.1.0/src/duckstring/catchment/static/index.html +0 -1
  146. duckstring-0.1.0/src/duckstring/catchment/static/index.txt +0 -15
  147. duckstring-0.1.0/src/duckstring/cli/__init__.py +0 -16
  148. duckstring-0.1.0/src/duckstring/cli/catchment.py +0 -33
  149. duckstring-0.1.0/src/duckstring/core.py +0 -17
  150. duckstring-0.1.0/src/duckstring.egg-info/SOURCES.txt +0 -51
  151. {duckstring-0.1.0 → duckstring-0.2.0}/LICENSE +0 -0
  152. {duckstring-0.1.0 → duckstring-0.2.0}/setup.cfg +0 -0
  153. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/__main__.py +0 -0
  154. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/__init__.py +0 -0
  155. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/_next/static/chunks/03~yq9q893hmn.js +0 -0
  156. {duckstring-0.1.0/src/duckstring/catchment/static/_next/static/v34k52t11dNGCtVyPJ3io → duckstring-0.2.0/src/duckstring/catchment/static/_next/static/uFzbxKIUibAOY83mw7FJ7}/_clientMiddlewareManifest.js +0 -0
  157. {duckstring-0.1.0/src/duckstring/catchment/static/_next/static/v34k52t11dNGCtVyPJ3io → duckstring-0.2.0/src/duckstring/catchment/static/_next/static/uFzbxKIUibAOY83mw7FJ7}/_ssgManifest.js +0 -0
  158. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/utils.py +0 -0
  159. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/dependency_links.txt +0 -0
  160. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/entry_points.txt +0 -0
  161. {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: duckstring
3
- Version: 0.1.0
4
- Summary: Duckstring is a Python-native data-mesh orchestration framework designed to be as naturally extensible as installing packages.
3
+ Version: 0.2.0
4
+ Summary: Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically.
5
5
  Author-email: Duckstring <dev@duckstring.com>
6
6
  License: Apache License
7
7
  Version 2.0, January 2004
@@ -205,6 +205,10 @@ License: Apache License
205
205
  See the License for the specific language governing permissions and
206
206
  limitations under the License.
207
207
 
208
+ Project-URL: Homepage, https://duckstring.com
209
+ Project-URL: Documentation, https://docs.duckstring.com
210
+ Project-URL: Repository, https://github.com/duckstring-dev/duckstring
211
+ Project-URL: Playground, https://playground.duckstring.com
208
212
  Requires-Python: >=3.10
209
213
  Description-Content-Type: text/markdown
210
214
  License-File: LICENSE
@@ -212,242 +216,100 @@ Requires-Dist: duckdb>=0.10
212
216
  Requires-Dist: rich>=13.7
213
217
  Requires-Dist: typer>=0.12
214
218
  Requires-Dist: fastapi>=0.111
219
+ Requires-Dist: python-multipart>=0.0.9
215
220
  Requires-Dist: uvicorn[standard]>=0.29
221
+ Requires-Dist: httpx>=0.25
222
+ Requires-Dist: tomli>=2.0; python_version < "3.11"
223
+ Requires-Dist: tomli-w>=1.0
216
224
  Provides-Extra: dev
217
225
  Requires-Dist: pytest>=8.0; extra == "dev"
226
+ Requires-Dist: pytest-timeout>=0.5; extra == "dev"
218
227
  Requires-Dist: ruff>=0.3; extra == "dev"
219
228
  Requires-Dist: mypy>=1.8; extra == "dev"
220
229
  Dynamic: license-file
221
230
 
222
231
  # Duckstring
232
+ *There is no DAG.*
223
233
 
224
- Duckstring is a data pipeline framework built around modular, versioned nodes called **Ponds**. Each Pond specifies its immediate parents (with version), allowing for the formation of a DAG much like one would install packages.
234
+ Duckstring treats data transformations as software packages. Upstream dependencies are declared per Pond (unit operation), defining the DAG without the need for its direct management.
225
235
 
226
- Pond execution is orchestrated within an environment - a **Catchment** - that controls storage and other global settings. It uses a pull-based system modelled after Kanban, with **Outlets** (terminal Ponds) sending demand upstream. This allows each Pond to be modified and deployed independently, with any paths in the DAG that are not attached to any Outlet automatically skipped.
236
+ Ponds are upgraded and deployed to Duckstring's pull-based Catchment (orchestrator) atomically - like upgrading a package - with the earlier version continuing to execute until there are no consumers dependent on it. Upstream defines constraints on what it can consume, downstream defines when it's needed, and the Catchment optimally executes the sequence of Ponds supplying it with the best currency and frequency as possible.
227
237
 
228
- Duckstring is built on the philosophy that most data pipelines are not truly "big data" and with good design can execute on a single compute node. It is primarily designed for batch and incremental workloads for tables on the order of tens of millions of rows (e.g. <50M).
229
-
230
- The default engine is DuckDB, though this is configurable. Duckstring is however an independent project and is not affiliated with, endorsed by, or maintained by the DuckDB project.
231
-
232
- -- **Note**: As the project is in development, most of the notes below should be read as *indended functionality*, and most features are not yet implemented.
238
+ You should not need to manage the DAG. You should not need global governance. You should know yourself and your suppliers and trust that you'll get what you need when you need it.
233
239
 
234
240
  ## Core Concepts
235
241
 
236
- - **Catchment**: Control environment - a FastAPI application
237
- - **Pond**: Versioned transformation unit with declared upstream dependencies - the main element of version control
238
- - **Inlet**: Pond with external dependencies and no upstream Ponds
239
- - **Outlet**: Pond with no downstream Ponds (e.g. outputs final data products)
240
- - **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
241
-
242
- ## Installation
243
-
244
- ```bash
245
- pip install duckstring
246
- ```
247
-
248
- ## Quickstart
249
-
250
- ### 1) Connect to a Catchment
251
-
252
- A Catchment is the execution environment, receiving Ponds and managing runs. It runs either as a local daemon or as a remote server, allowing you to start locally and seamlessly upgrade to a hosted/cloud server if you need to later.
253
-
254
- #### Start a Catchment Server
255
-
256
- To run a Catchment locally, run:
257
-
258
- ```bash
259
- duckstring catchment start --name dev --port 5000 --root ~/.duckstring/dev
260
- ```
261
-
262
- This will start a server with name 'dev' at port 5000 (the default, if none specified) and store Catchment details at `~/.duckstring/dev` (default is `~/.duckstring/{name}`). If any of these options are omitted you will be prompted on start.
263
-
264
- #### Connect to a Remote Server
265
-
266
- Alternatively, you can connect to a server running a Catchment:
267
-
268
- ```bash
269
- duckstring catchment connect --name dev --path https://path.to.catchment
270
- ```
271
-
272
- This will prompt for any necessary auth, and will add the Catchment under the specified name.
273
-
274
- #### Connect to *duckstring.com*
275
-
276
- There are future plans for a dedicated Catchment service at https://duckstring.com. If you're interested, please [contact me](mailto:dev@duckstring.com).
277
-
278
- ### 2) Define Pond(s)
279
-
280
- #### Demo Ponds
281
-
282
- If you want to see an example sequence of Ponds in action immediately, create three project directories and run one of these commands in each:
283
-
284
- ```bash
285
- duckstring pond demo inlet
286
- duckstring pond demo pond
287
- duckstring pond demo outlet
288
- ```
289
-
290
- It's recommended to do this before attempting to make your own so that you can get a feel for the structure.
291
-
292
- #### Custom Pond
293
-
294
- Create a project directory and run:
295
-
296
- ```bash
297
- duckstring pond init example_pond
298
- ```
299
-
300
- This will create a duckstring pond structure:
301
-
302
- ```text
303
- root/
304
- |-- src/
305
- | |-- pond.py
306
- |-- pond.toml
307
- |-- __main__.py
308
- |-- .gitignore
309
- |-- README.md
310
- ```
311
-
312
- Here `pond.py` contains the code for a single Ripple operation (currently blank), and `pond.toml` specifies the Pond name "example_pond" and version (defaulting to "0.1.0").
313
-
314
- ### 3) Deploy to Catchment
315
-
316
- #### From Local
317
-
318
- From a Pond's project root run:
319
-
320
- ```bash
321
- duckstring deploy dev
322
- ```
323
-
324
- This will read the pond name, version and type (Inlet, Pond, Outlet) from `pond.toml` and deploy the project contents to the Catchment specified by name (here `dev`).
325
-
326
- Alternatively, you can import the Pond using the Catchment UI.
327
-
328
- #### From Git
242
+ The main elements:
329
243
 
330
- If you are using git with a remote, you can deploy with:
331
-
332
- ```bash
333
- duckstring deploy dev --git {branch|commit|tag}
334
- ```
335
-
336
- This will use the current branch/commit/tag to define the Pond. Upon each execution the Catchment will clone the repository and run it.
337
-
338
- This can also be specified using the Catchment UI.
339
-
340
- ### 3) Execute
341
-
342
- Ponds are executed by sending a Demand signal from an Outlet. This propagates backwards through the DAG until it reaches each upstream Inlet, causing them to execute, with children beginning upon completion of all of their parents.
343
-
344
- These examples will use the Pond `outlet`, version `1.0.0`, as the execution reference. All examples may also be alternatively executed using the Catchment UI.
345
-
346
- #### Pulse
347
-
348
- To initiate a single run:
349
-
350
- ```bash
351
- duckstring pulse dev outlet
352
- ```
353
-
354
- The `pulse` mode emits a Demand signal from `outlet`, and when it begins execution, sends a Stop signal. This causes it to execute exactly once.
355
-
356
- This will automatically run against the maximum version available for that Pond. To use a specific version:
357
-
358
- ```bash
359
- duckstring pulse dev outlet --version 1
360
- ```
361
-
362
- #### Wave
363
-
364
- To continuously run:
365
-
366
- ```bash
367
- duckstring wave dev outlet
368
- ```
369
-
370
- The `wave` mode emits a Demand signal from `outlet`, and when it begins execution, sends *another* Demand signal. This causes it to execute continuously, as frequently as the DAG allows (i.e. at a period equal to the execution time of the slowest Ripple in any Pond).
371
-
372
- #### Tide
373
-
374
- To run at a scheduled frequency:
375
-
376
- ```bash
377
- duckstring tide dev outlet 15 2 * * * --local
378
- ```
244
+ - **Catchment**: Control environment (FastAPI + UI + CLI)
245
+ - **Pond**: Versioned container with declared upstream dependencies
246
+ - **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
379
247
 
380
- This would run at 2:15am every day local time, using cron syntax. Omitting the `--local` flag defaults to UTC.
248
+ Ponds are typed or referred to in context:
381
249
 
382
- ### 4) Monitor
250
+ - **Source**: A parent Pond
251
+ - **Sink**: A child Pond
252
+ - **Inlet**: A Pond with external dependencies and no Sources
253
+ - **Outlet**: A Pond with no Sinks (e.g. outputs final data products)
383
254
 
384
- To print out a summary of current processes in the Catchment:
255
+ To see the orchestration model in action without installing anything, try the [Duckstring Playground](https://playground.duckstring.com).
385
256
 
386
- ```bash
387
- duckstring status dev
388
- ```
389
-
390
- This will print to CLI a summary for each Pond that is either currently executing or has Demand.
391
-
392
- To include all Ponds:
257
+ ## Quickstart
393
258
 
394
259
  ```bash
395
- duckstring status dev --all
396
- ```
397
-
398
- ### 5) Retrieve Data
399
-
400
- #### Get
401
-
402
- The simplest way to retrieve data is to load by the Ripple name. This returns the entire contents of the directory, and does not require that the data be in a tabular format (e.g. SQL-compatible).
260
+ pip install duckstring
403
261
 
404
- ```bash
405
- duckstring get dev outlet daily
262
+ # Start a local Catchment (the runtime + web UI) — leave it running
263
+ duckstring catchment init --name dev
406
264
  ```
407
265
 
408
- This writes a directory `./ponds/outlet/daily` with the 'daily' Ripple's contents. You may also override the default location:
266
+ Then, in another terminal:
409
267
 
410
268
  ```bash
411
- duckstring get dev outlet daily --path ./daily_output
412
- ```
413
-
414
- #### SQL Query
269
+ # Create the demo pipeline (transactions, products → sales → reports) and deploy it
270
+ mkdir demo && cd demo
271
+ duckstring pond demo
272
+ duckstring pond deploy --all -y
415
273
 
416
- If the target is an SQL-compatible table (e.g. DuckDB or Parquet), an SQL statement may be sent directly, outputting the result to the command line:
274
+ # Run it end to end, once
275
+ duckstring trigger pulse reports
417
276
 
418
- ```bash
419
- duckstring query dev outlet --sql "SELECT * FROM daily WHERE id=1;"
277
+ # Look around
278
+ duckstring status # live state of every Pond
279
+ duckstring query reports monthly_summary # peek at an output table
420
280
  ```
421
281
 
422
- Alternatively, include a file path:
282
+ The Catchment also serves a live web UI at `http://127.0.0.1:7474` — the Pond graph, freshness, run history, and the full trigger/control surface.
423
283
 
424
- ```bash
425
- duckstring query dev outlet --sql @path/to/query.sql
426
- ```
284
+ ## Execution
427
285
 
428
- Omitting the `--sql` statement queries with a default SELECT * LIMIT 10 on the specified table:
286
+ Ponds execute on demand signals sent to an Outlet, in two flavours — **push** runs the lineage forward to a target freshness; **pull** propagates demand upstream so every Pond re-runs as its Sources update, naturally throttled to the bottleneck. Each comes as a one-shot or a standing trigger:
429
287
 
430
- ```bash
431
- duckstring query dev outlet daily
432
- ```
288
+ | | Once | Continuously |
289
+ |---|---|---|
290
+ | **Push** | Pulse | Tide |
291
+ | **Pull** | Tap | Wave |
433
292
 
434
- ##### Write to file
293
+ A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
435
294
 
436
- To output to a file, include a flag for the file format, followed by the file name:
295
+ ## Scope & maturity
437
296
 
438
- `--csv`: Comma-separated values
439
- `--json`: JSON records
440
- `--parquet`: Parquet file
297
+ Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
441
298
 
442
- This writes by default to `./ponds/outlet/daily/{filename}`. To overrite the default location you may use the `--path` flag.
299
+ ## Going further
443
300
 
444
- For example, to execute an sql statement from file `query.sql` and write the result to CSV at the current directory:
301
+ Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:
445
302
 
446
- ```bash
447
- duckstring query dev outlet --sql @query.sql --csv daily.csv --path .
448
- ```
303
+ - [Quickstart](https://docs.duckstring.com/getting-started/quickstart) — the path above, with explanations
304
+ - [Theory](https://docs.duckstring.com/theory) the freshness-based orchestration model in depth
305
+ - [Versioning](https://docs.duckstring.com/concepts/versioning) — SemVer on Ponds, concurrent major versions, atomic upgrades
306
+ - [Local testing](https://docs.duckstring.com/guides/local-testing) — Puddles: test a Pond before deploying it
307
+ - [Fault tolerance](https://docs.duckstring.com/guides/fault-tolerance) — retry budgets, failure states, recovery
308
+ - [Running a Catchment](https://docs.duckstring.com/guides/running-a-catchment) — hosting, authentication, platform deployment
309
+ - [CLI](https://docs.duckstring.com/reference/cli) / [HTTP API](https://docs.duckstring.com/reference/http-api) — full references
449
310
 
450
- ## Further Reading
311
+ There are future plans for a hosted Catchment service at [duckstring.com](https://duckstring.com). If you're interested, please [get in touch](mailto:dev@duckstring.com).
451
312
 
452
- For more detail on each component, please read the corresponding documentation in `docs/`.
313
+ ## License
453
314
 
315
+ [Apache 2.0](LICENSE)
@@ -0,0 +1,85 @@
1
+ # Duckstring
2
+ *There is no DAG.*
3
+
4
+ Duckstring treats data transformations as software packages. Upstream dependencies are declared per Pond (unit operation), defining the DAG without the need for its direct management.
5
+
6
+ Ponds are upgraded and deployed to Duckstring's pull-based Catchment (orchestrator) atomically - like upgrading a package - with the earlier version continuing to execute until there are no consumers dependent on it. Upstream defines constraints on what it can consume, downstream defines when it's needed, and the Catchment optimally executes the sequence of Ponds supplying it with the best currency and frequency as possible.
7
+
8
+ You should not need to manage the DAG. You should not need global governance. You should know yourself and your suppliers and trust that you'll get what you need when you need it.
9
+
10
+ ## Core Concepts
11
+
12
+ The main elements:
13
+
14
+ - **Catchment**: Control environment (FastAPI + UI + CLI)
15
+ - **Pond**: Versioned container with declared upstream dependencies
16
+ - **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
17
+
18
+ Ponds are typed or referred to in context:
19
+
20
+ - **Source**: A parent Pond
21
+ - **Sink**: A child Pond
22
+ - **Inlet**: A Pond with external dependencies and no Sources
23
+ - **Outlet**: A Pond with no Sinks (e.g. outputs final data products)
24
+
25
+ To see the orchestration model in action without installing anything, try the [Duckstring Playground](https://playground.duckstring.com).
26
+
27
+ ## Quickstart
28
+
29
+ ```bash
30
+ pip install duckstring
31
+
32
+ # Start a local Catchment (the runtime + web UI) — leave it running
33
+ duckstring catchment init --name dev
34
+ ```
35
+
36
+ Then, in another terminal:
37
+
38
+ ```bash
39
+ # Create the demo pipeline (transactions, products → sales → reports) and deploy it
40
+ mkdir demo && cd demo
41
+ duckstring pond demo
42
+ duckstring pond deploy --all -y
43
+
44
+ # Run it end to end, once
45
+ duckstring trigger pulse reports
46
+
47
+ # Look around
48
+ duckstring status # live state of every Pond
49
+ duckstring query reports monthly_summary # peek at an output table
50
+ ```
51
+
52
+ The Catchment also serves a live web UI at `http://127.0.0.1:7474` — the Pond graph, freshness, run history, and the full trigger/control surface.
53
+
54
+ ## Execution
55
+
56
+ Ponds execute on demand signals sent to an Outlet, in two flavours — **push** runs the lineage forward to a target freshness; **pull** propagates demand upstream so every Pond re-runs as its Sources update, naturally throttled to the bottleneck. Each comes as a one-shot or a standing trigger:
57
+
58
+ | | Once | Continuously |
59
+ |---|---|---|
60
+ | **Push** | Pulse | Tide |
61
+ | **Pull** | Tap | Wave |
62
+
63
+ A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
64
+
65
+ ## Scope & maturity
66
+
67
+ Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
68
+
69
+ ## Going further
70
+
71
+ Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:
72
+
73
+ - [Quickstart](https://docs.duckstring.com/getting-started/quickstart) — the path above, with explanations
74
+ - [Theory](https://docs.duckstring.com/theory) — the freshness-based orchestration model in depth
75
+ - [Versioning](https://docs.duckstring.com/concepts/versioning) — SemVer on Ponds, concurrent major versions, atomic upgrades
76
+ - [Local testing](https://docs.duckstring.com/guides/local-testing) — Puddles: test a Pond before deploying it
77
+ - [Fault tolerance](https://docs.duckstring.com/guides/fault-tolerance) — retry budgets, failure states, recovery
78
+ - [Running a Catchment](https://docs.duckstring.com/guides/running-a-catchment) — hosting, authentication, platform deployment
79
+ - [CLI](https://docs.duckstring.com/reference/cli) / [HTTP API](https://docs.duckstring.com/reference/http-api) — full references
80
+
81
+ There are future plans for a hosted Catchment service at [duckstring.com](https://duckstring.com). If you're interested, please [get in touch](mailto:dev@duckstring.com).
82
+
83
+ ## License
84
+
85
+ [Apache 2.0](LICENSE)
@@ -0,0 +1,65 @@
1
+ [project]
2
+ name = "duckstring"
3
+ version = "0.2.0"
4
+ description = "Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically."
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ license = { file = "LICENSE" }
8
+ authors = [
9
+ { name = "Duckstring", email = "dev@duckstring.com" }
10
+ ]
11
+
12
+ dependencies = [
13
+ "duckdb>=0.10",
14
+ "rich>=13.7",
15
+ "typer>=0.12",
16
+ "fastapi>=0.111",
17
+ "python-multipart>=0.0.9",
18
+ "uvicorn[standard]>=0.29",
19
+ "httpx>=0.25",
20
+ "tomli>=2.0; python_version < '3.11'",
21
+ "tomli-w>=1.0",
22
+ ]
23
+
24
+ [project.optional-dependencies]
25
+ dev = [
26
+ "pytest>=8.0",
27
+ "pytest-timeout>=0.5",
28
+ "ruff>=0.3",
29
+ "mypy>=1.8",
30
+ ]
31
+
32
+ [tool.setuptools]
33
+ package-dir = { "" = "src" }
34
+
35
+ [tool.setuptools.package-data]
36
+ "duckstring.catchment" = ["static/**/*", "schema/*.sql"]
37
+ "duckstring" = ["demo/**/*", "demo/**/.gitignore"]
38
+
39
+ [tool.setuptools.packages.find]
40
+ where = ["src"]
41
+
42
+ [project.urls]
43
+ Homepage = "https://duckstring.com"
44
+ Documentation = "https://docs.duckstring.com"
45
+ Repository = "https://github.com/duckstring-dev/duckstring"
46
+ Playground = "https://playground.duckstring.com"
47
+
48
+ [project.scripts]
49
+ duckstring = "duckstring.cli:main"
50
+ ds = "duckstring.cli:main"
51
+
52
+ [tool.pytest.ini_options]
53
+ testpaths = ["tests"]
54
+ # Tight enough to catch an accidental real sleep (sim tests finish in ms), loose enough to absorb
55
+ # cold-runner I/O on CI (SQLite migrate / uvicorn startup can exceed 1s on shared runners).
56
+ timeout = 5
57
+
58
+ [tool.ruff]
59
+ line-length = 128
60
+ target-version = "py310"
61
+ extend-exclude = ["experiment"]
62
+
63
+ [tool.ruff.lint]
64
+ select = ["E", "F", "I", "B"]
65
+ ignore = ["B008"] # typer.Option/Argument in defaults is intentional
@@ -0,0 +1,28 @@
1
+ from importlib.metadata import PackageNotFoundError
2
+ from importlib.metadata import version as _pkg_version
3
+
4
+ from .core import (
5
+ Catchment,
6
+ Pond,
7
+ Puddle,
8
+ Ripple,
9
+ Trickle,
10
+ puddle,
11
+ ripple,
12
+ )
13
+
14
+ try:
15
+ __version__ = _pkg_version("duckstring")
16
+ except PackageNotFoundError: # running from a source tree without an installed dist
17
+ __version__ = "0.0.0"
18
+
19
+ __all__ = [
20
+ "Catchment",
21
+ "Pond",
22
+ "Puddle",
23
+ "Ripple",
24
+ "Trickle",
25
+ "puddle",
26
+ "ripple",
27
+ "__version__",
28
+ ]
@@ -0,0 +1,110 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import os
5
+ import secrets
6
+ from contextlib import asynccontextmanager
7
+ from datetime import datetime, timezone
8
+ from pathlib import Path
9
+
10
+ from fastapi import FastAPI
11
+ from fastapi.responses import JSONResponse
12
+ from fastapi.staticfiles import StaticFiles
13
+
14
+ from .db import connect, migrate
15
+ from .driver import Driver
16
+ from .launcher import NoopLauncher, SubprocessLauncher
17
+ from .routes import router
18
+
19
+ _STATIC_DIR = Path(__file__).parent / "static"
20
+
21
+
22
+ async def _scheduler(driver: Driver) -> None:
23
+ """Drive clock processes (Tide deadlines, window boundaries, Wave-on-idle) at next_wake."""
24
+ while True:
25
+ nw = driver.next_wake()
26
+ now = datetime.now(timezone.utc)
27
+ delay = (nw - now).total_seconds() if nw else 1.0
28
+ await asyncio.sleep(max(0.05, min(delay, 5.0)))
29
+ try:
30
+ driver.scheduler_tick()
31
+ except Exception as exc: # keep the loop alive
32
+ print(f"[catchment] scheduler error: {exc}", flush=True)
33
+
34
+
35
+ @asynccontextmanager
36
+ async def _lifespan(app: FastAPI):
37
+ base_url = app.state.base_url
38
+ if os.environ.get("DUCKSTRING_DISABLE_DUCKS"):
39
+ launcher = NoopLauncher()
40
+ else:
41
+ # Ducks dial back over the same authenticated surface — they present the API key as their token.
42
+ # base_url None = unknown (the platform picked the bind address): the launcher defers spawns
43
+ # until the dial-back middleware learns the address from the first request.
44
+ launcher = SubprocessLauncher(app.state.root, base_url, token=app.state.api_key or "")
45
+ driver = Driver(app.state.db, app.state.root, base_url, launcher)
46
+ app.state.driver = driver
47
+ app.state.launcher = launcher
48
+
49
+ # Restore: resume any Pond Runs that were in flight when the Catchment last stopped.
50
+ driver.resume_incomplete()
51
+
52
+ scheduler = asyncio.create_task(_scheduler(driver))
53
+ try:
54
+ yield
55
+ finally:
56
+ scheduler.cancel()
57
+ try:
58
+ await scheduler
59
+ except asyncio.CancelledError:
60
+ pass
61
+ launcher.shutdown_all()
62
+
63
+
64
+ def create_app(root: Path, api_key: str | None = None, base_url: str | None = None) -> FastAPI:
65
+ root.mkdir(parents=True, exist_ok=True)
66
+ con = connect(root / "duck.db")
67
+ migrate(con)
68
+
69
+ app = FastAPI(title="Duckstring Catchment", lifespan=_lifespan)
70
+ app.state.root = root
71
+ app.state.db = con
72
+ # API key: explicit argument, or the environment (useful for containers/remote serving). When
73
+ # set, every /api request (except /api/health) must present it — Bearer header or X-Duck-Token.
74
+ app.state.api_key = api_key or os.environ.get("DUCKSTRING_API_KEY") or None
75
+ # The address Ducks dial back to: explicit argument (the CLI passes its bind address), or the
76
+ # environment, or None — unknown, because the host platform picks the bind address (e.g. Posit
77
+ # Connect). When None it is learned from the first request's ASGI scope below.
78
+ app.state.base_url = base_url or os.environ.get("DUCKSTRING_CATCHMENT_URL") or None
79
+
80
+ @app.middleware("http")
81
+ async def _learn_dialback_address(request, call_next):
82
+ launcher = getattr(app.state, "launcher", None)
83
+ if launcher is not None and getattr(launcher, "base_url", "") is None:
84
+ server = request.scope.get("server") # the server's bound (host, port) per the ASGI spec
85
+ if server and server[1]: # a unix socket has port None — nothing TCP to dial
86
+ host = "127.0.0.1" if server[0] in ("0.0.0.0", "::") else server[0]
87
+ url = f"http://{host}:{server[1]}"
88
+ app.state.base_url = url
89
+ app.state.driver.base_url = url
90
+ launcher.set_base_url(url) # spawns any Ducks that were waiting on the address
91
+ return await call_next(request)
92
+
93
+ @app.middleware("http")
94
+ async def _require_api_key(request, call_next):
95
+ key = app.state.api_key
96
+ path = request.url.path
97
+ if key and path.startswith("/api") and path != "/api/health":
98
+ auth = request.headers.get("authorization", "")
99
+ supplied = auth[7:] if auth.lower().startswith("bearer ") else ""
100
+ supplied = supplied or request.headers.get("x-duck-token", "")
101
+ if not secrets.compare_digest(supplied, key):
102
+ return JSONResponse({"detail": "Invalid or missing API key"}, status_code=401)
103
+ return await call_next(request)
104
+
105
+ app.include_router(router, prefix="/api")
106
+
107
+ if _STATIC_DIR.exists():
108
+ app.mount("/", StaticFiles(directory=_STATIC_DIR, html=True), name="frontend")
109
+
110
+ return app
@@ -0,0 +1,30 @@
1
+ """ASGI entry for platform-hosted Catchments (Posit Connect, gunicorn/uvicorn, containers).
2
+
3
+ The platform owns the server lifecycle; this module just exposes the app. A deployable bundle is
4
+ two files — ``requirements.txt`` (``duckstring``) and an ``app.py`` containing::
5
+
6
+ from duckstring.catchment.asgi import app
7
+
8
+ Configuration is environment-only:
9
+
10
+ - ``DUCKSTRING_ROOT`` — the Catchment root. Defaults to ``./.duckstring`` (relative to the working
11
+ directory, i.e. the deployed content directory). On platforms that replace the content directory
12
+ on redeploy (Posit Connect does), the default survives process restarts but **not redeploys of
13
+ the Catchment app itself** — point this at a persistent path for durable state.
14
+ - ``DUCKSTRING_API_KEY`` — optional built-in API key. Leave unset when the platform already gates
15
+ requests (the recommended hosted model).
16
+ - ``DUCKSTRING_CATCHMENT_URL`` — optional Duck dial-back address. Normally unset: the platform
17
+ picks the bind address, and the Catchment learns it from the first request it serves.
18
+
19
+ Run only **one** process of this app (e.g. Posit Connect's "Max processes" = 1): the Catchment is
20
+ a single brain — one scheduler, one SQLite, one set of Ducks.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import os
26
+ from pathlib import Path
27
+
28
+ from .app import create_app
29
+
30
+ app = create_app(Path(os.environ.get("DUCKSTRING_ROOT", ".duckstring")).expanduser().resolve())