duckstring 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {duckstring-0.1.0 → duckstring-0.2.0}/PKG-INFO +61 -199
- duckstring-0.2.0/README.md +85 -0
- duckstring-0.2.0/pyproject.toml +65 -0
- duckstring-0.2.0/src/duckstring/__init__.py +28 -0
- duckstring-0.2.0/src/duckstring/catchment/app.py +110 -0
- duckstring-0.2.0/src/duckstring/catchment/asgi.py +30 -0
- duckstring-0.2.0/src/duckstring/catchment/dag.py +84 -0
- duckstring-0.2.0/src/duckstring/catchment/db.py +39 -0
- duckstring-0.2.0/src/duckstring/catchment/driver.py +836 -0
- duckstring-0.2.0/src/duckstring/catchment/launcher.py +94 -0
- duckstring-0.2.0/src/duckstring/catchment/registry.py +27 -0
- duckstring-0.2.0/src/duckstring/catchment/routes/__init__.py +16 -0
- duckstring-0.2.0/src/duckstring/catchment/routes/catchment.py +121 -0
- duckstring-0.2.0/src/duckstring/catchment/routes/data.py +143 -0
- duckstring-0.2.0/src/duckstring/catchment/routes/deploy.py +212 -0
- duckstring-0.2.0/src/duckstring/catchment/routes/duck.py +31 -0
- duckstring-0.2.0/src/duckstring/catchment/routes/orchestrate.py +209 -0
- duckstring-0.2.0/src/duckstring/catchment/schema/001_init.sql +158 -0
- duckstring-0.2.0/src/duckstring/catchment/static/404.html +1 -0
- duckstring-0.2.0/src/duckstring/catchment/static/__next.__PAGE__.txt +7 -0
- duckstring-0.2.0/src/duckstring/catchment/static/__next._full.txt +16 -0
- duckstring-0.2.0/src/duckstring/catchment/static/__next._head.txt +6 -0
- duckstring-0.2.0/src/duckstring/catchment/static/__next._index.txt +5 -0
- duckstring-0.2.0/src/duckstring/catchment/static/__next._tree.txt +3 -0
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/10u3y4bw1ayzs.js → duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/07lhk_q6pmm3r.js +1 -1
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0dbhjjzl8qfwv.js +1 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0jvmviuftg5e2.css +1 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0kczw6usu-y-f.js +5 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0pgf-_4l3d1gl.css +1 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/0qh0m742pxib~.js +2 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/11kjahy2ntf0n.js +31 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/chunks/turbopack-03~mbvk_uplk_.js +1 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_next/static/uFzbxKIUibAOY83mw7FJ7/_buildManifest.js +16 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._full.txt +16 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._head.txt +6 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._index.txt +5 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/_not-found/__next._not-found.__PAGE__.txt +2 -2
- duckstring-0.2.0/src/duckstring/catchment/static/_not-found/__next._not-found.txt +5 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/_not-found/__next._tree.txt +2 -2
- duckstring-0.2.0/src/duckstring/catchment/static/_not-found.html +1 -0
- duckstring-0.2.0/src/duckstring/catchment/static/_not-found.txt +16 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/favicon.svg +1 -1
- duckstring-0.2.0/src/duckstring/catchment/static/index.html +1 -0
- duckstring-0.2.0/src/duckstring/catchment/static/index.txt +16 -0
- duckstring-0.2.0/src/duckstring/catchment/static/logo-mark.svg +15 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/logo.svg +1 -1
- duckstring-0.2.0/src/duckstring/cli/__init__.py +73 -0
- duckstring-0.2.0/src/duckstring/cli/_http.py +79 -0
- duckstring-0.2.0/src/duckstring/cli/catchment.py +368 -0
- duckstring-0.2.0/src/duckstring/cli/config.py +122 -0
- duckstring-0.2.0/src/duckstring/cli/control.py +126 -0
- duckstring-0.2.0/src/duckstring/cli/data.py +133 -0
- duckstring-0.2.0/src/duckstring/cli/deploy.py +150 -0
- duckstring-0.2.0/src/duckstring/cli/pond.py +212 -0
- duckstring-0.2.0/src/duckstring/cli/puddle.py +141 -0
- duckstring-0.2.0/src/duckstring/cli/status.py +290 -0
- duckstring-0.2.0/src/duckstring/cli/trigger.py +110 -0
- duckstring-0.2.0/src/duckstring/cli/window.py +178 -0
- duckstring-0.2.0/src/duckstring/core.py +368 -0
- duckstring-0.2.0/src/duckstring/demo/products/.gitignore +8 -0
- duckstring-0.2.0/src/duckstring/demo/products/README.md +9 -0
- duckstring-0.2.0/src/duckstring/demo/products/pond.toml +4 -0
- duckstring-0.2.0/src/duckstring/demo/products/src/pond.py +66 -0
- duckstring-0.2.0/src/duckstring/demo/reports/.gitignore +8 -0
- duckstring-0.2.0/src/duckstring/demo/reports/README.md +11 -0
- duckstring-0.2.0/src/duckstring/demo/reports/pond.toml +7 -0
- duckstring-0.2.0/src/duckstring/demo/reports/src/pond.py +24 -0
- duckstring-0.2.0/src/duckstring/demo/sales/.gitignore +8 -0
- duckstring-0.2.0/src/duckstring/demo/sales/README.md +11 -0
- duckstring-0.2.0/src/duckstring/demo/sales/pond.toml +7 -0
- duckstring-0.2.0/src/duckstring/demo/sales/src/pond.py +67 -0
- duckstring-0.2.0/src/duckstring/demo/sales/src/puddles.py +47 -0
- duckstring-0.2.0/src/duckstring/demo/transactions/.gitignore +8 -0
- duckstring-0.2.0/src/duckstring/demo/transactions/README.md +9 -0
- duckstring-0.2.0/src/duckstring/demo/transactions/pond.toml +4 -0
- duckstring-0.2.0/src/duckstring/demo/transactions/src/pond.py +46 -0
- duckstring-0.2.0/src/duckstring/duck/__init__.py +8 -0
- duckstring-0.2.0/src/duckstring/duck/__main__.py +169 -0
- duckstring-0.2.0/src/duckstring/duck/client.py +46 -0
- duckstring-0.2.0/src/duckstring/duck/core.py +144 -0
- duckstring-0.2.0/src/duckstring/duck/executor.py +142 -0
- duckstring-0.2.0/src/duckstring/engine/__init__.py +95 -0
- duckstring-0.2.0/src/duckstring/engine/catchment.py +573 -0
- duckstring-0.2.0/src/duckstring/engine/core.py +219 -0
- duckstring-0.2.0/src/duckstring/engine/pond.py +122 -0
- duckstring-0.2.0/src/duckstring/engine/worker.py +177 -0
- duckstring-0.2.0/src/duckstring/keys.py +22 -0
- duckstring-0.2.0/src/duckstring/local/__init__.py +12 -0
- duckstring-0.2.0/src/duckstring/local/hydrate.py +114 -0
- duckstring-0.2.0/src/duckstring/local/project.py +55 -0
- duckstring-0.2.0/src/duckstring/local/runner.py +147 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/PKG-INFO +61 -199
- duckstring-0.2.0/src/duckstring.egg-info/SOURCES.txt +125 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/requires.txt +7 -0
- duckstring-0.2.0/tests/test_auth.py +258 -0
- duckstring-0.2.0/tests/test_catchment.py +289 -0
- duckstring-0.2.0/tests/test_catchment_app.py +674 -0
- duckstring-0.2.0/tests/test_config.py +70 -0
- duckstring-0.2.0/tests/test_core.py +94 -0
- duckstring-0.2.0/tests/test_data.py +148 -0
- duckstring-0.2.0/tests/test_deploy.py +146 -0
- duckstring-0.2.0/tests/test_download.py +77 -0
- duckstring-0.2.0/tests/test_duck.py +175 -0
- duckstring-0.2.0/tests/test_engine.py +591 -0
- duckstring-0.2.0/tests/test_engine_split.py +228 -0
- duckstring-0.2.0/tests/test_execute.py +149 -0
- duckstring-0.2.0/tests/test_multimajor.py +188 -0
- duckstring-0.2.0/tests/test_platform.py +155 -0
- duckstring-0.2.0/tests/test_pond.py +156 -0
- duckstring-0.2.0/tests/test_puddle.py +392 -0
- duckstring-0.2.0/tests/test_restart.py +98 -0
- duckstring-0.2.0/tests/test_runtime.py +233 -0
- duckstring-0.2.0/tests/test_static.py +23 -0
- duckstring-0.2.0/tests/test_status.py +198 -0
- duckstring-0.2.0/tests/test_status_api.py +233 -0
- duckstring-0.2.0/tests/test_window.py +193 -0
- duckstring-0.1.0/README.md +0 -232
- duckstring-0.1.0/pyproject.toml +0 -46
- duckstring-0.1.0/src/duckstring/__init__.py +0 -13
- duckstring-0.1.0/src/duckstring/catchment/app.py +0 -9
- duckstring-0.1.0/src/duckstring/catchment/routes/__init__.py +0 -0
- duckstring-0.1.0/src/duckstring/catchment/routes/catchment.py +0 -0
- duckstring-0.1.0/src/duckstring/catchment/static/404.html +0 -1
- duckstring-0.1.0/src/duckstring/catchment/static/__next.__PAGE__.txt +0 -6
- duckstring-0.1.0/src/duckstring/catchment/static/__next._full.txt +0 -15
- duckstring-0.1.0/src/duckstring/catchment/static/__next._head.txt +0 -6
- duckstring-0.1.0/src/duckstring/catchment/static/__next._index.txt +0 -5
- duckstring-0.1.0/src/duckstring/catchment/static/__next._tree.txt +0 -2
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/07mne3zcixcz2.js +0 -3
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0d-5~.f~0wtco.js +0 -5
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0d3tkj33oycf2.js +0 -1
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0fsi8lw5utejl.js +0 -1
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/0j18t3ydystkv.css +0 -1
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/11qk_vpw_zwkf.js +0 -31
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/chunks/turbopack-0e47rfx~.~3-8.js +0 -1
- duckstring-0.1.0/src/duckstring/catchment/static/_next/static/v34k52t11dNGCtVyPJ3io/_buildManifest.js +0 -11
- duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._full.txt +0 -16
- duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._head.txt +0 -6
- duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._index.txt +0 -5
- duckstring-0.1.0/src/duckstring/catchment/static/_not-found/__next._not-found.txt +0 -5
- duckstring-0.1.0/src/duckstring/catchment/static/_not-found.html +0 -1
- duckstring-0.1.0/src/duckstring/catchment/static/_not-found.txt +0 -16
- duckstring-0.1.0/src/duckstring/catchment/static/dev/cache/turbopack/ee6e79b1/CURRENT +0 -0
- duckstring-0.1.0/src/duckstring/catchment/static/dev/logs/next-development.log +0 -0
- duckstring-0.1.0/src/duckstring/catchment/static/index.html +0 -1
- duckstring-0.1.0/src/duckstring/catchment/static/index.txt +0 -15
- duckstring-0.1.0/src/duckstring/cli/__init__.py +0 -16
- duckstring-0.1.0/src/duckstring/cli/catchment.py +0 -33
- duckstring-0.1.0/src/duckstring/core.py +0 -17
- duckstring-0.1.0/src/duckstring.egg-info/SOURCES.txt +0 -51
- {duckstring-0.1.0 → duckstring-0.2.0}/LICENSE +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/setup.cfg +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/__main__.py +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/__init__.py +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/catchment/static/_next/static/chunks/03~yq9q893hmn.js +0 -0
- {duckstring-0.1.0/src/duckstring/catchment/static/_next/static/v34k52t11dNGCtVyPJ3io → duckstring-0.2.0/src/duckstring/catchment/static/_next/static/uFzbxKIUibAOY83mw7FJ7}/_clientMiddlewareManifest.js +0 -0
- {duckstring-0.1.0/src/duckstring/catchment/static/_next/static/v34k52t11dNGCtVyPJ3io → duckstring-0.2.0/src/duckstring/catchment/static/_next/static/uFzbxKIUibAOY83mw7FJ7}/_ssgManifest.js +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring/utils.py +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/dependency_links.txt +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/entry_points.txt +0 -0
- {duckstring-0.1.0 → duckstring-0.2.0}/src/duckstring.egg-info/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: duckstring
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically.
|
|
5
5
|
Author-email: Duckstring <dev@duckstring.com>
|
|
6
6
|
License: Apache License
|
|
7
7
|
Version 2.0, January 2004
|
|
@@ -205,6 +205,10 @@ License: Apache License
|
|
|
205
205
|
See the License for the specific language governing permissions and
|
|
206
206
|
limitations under the License.
|
|
207
207
|
|
|
208
|
+
Project-URL: Homepage, https://duckstring.com
|
|
209
|
+
Project-URL: Documentation, https://docs.duckstring.com
|
|
210
|
+
Project-URL: Repository, https://github.com/duckstring-dev/duckstring
|
|
211
|
+
Project-URL: Playground, https://playground.duckstring.com
|
|
208
212
|
Requires-Python: >=3.10
|
|
209
213
|
Description-Content-Type: text/markdown
|
|
210
214
|
License-File: LICENSE
|
|
@@ -212,242 +216,100 @@ Requires-Dist: duckdb>=0.10
|
|
|
212
216
|
Requires-Dist: rich>=13.7
|
|
213
217
|
Requires-Dist: typer>=0.12
|
|
214
218
|
Requires-Dist: fastapi>=0.111
|
|
219
|
+
Requires-Dist: python-multipart>=0.0.9
|
|
215
220
|
Requires-Dist: uvicorn[standard]>=0.29
|
|
221
|
+
Requires-Dist: httpx>=0.25
|
|
222
|
+
Requires-Dist: tomli>=2.0; python_version < "3.11"
|
|
223
|
+
Requires-Dist: tomli-w>=1.0
|
|
216
224
|
Provides-Extra: dev
|
|
217
225
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
226
|
+
Requires-Dist: pytest-timeout>=0.5; extra == "dev"
|
|
218
227
|
Requires-Dist: ruff>=0.3; extra == "dev"
|
|
219
228
|
Requires-Dist: mypy>=1.8; extra == "dev"
|
|
220
229
|
Dynamic: license-file
|
|
221
230
|
|
|
222
231
|
# Duckstring
|
|
232
|
+
*There is no DAG.*
|
|
223
233
|
|
|
224
|
-
Duckstring
|
|
234
|
+
Duckstring treats data transformations as software packages. Upstream dependencies are declared per Pond (unit operation), defining the DAG without the need for its direct management.
|
|
225
235
|
|
|
226
|
-
|
|
236
|
+
Ponds are upgraded and deployed to Duckstring's pull-based Catchment (orchestrator) atomically - like upgrading a package - with the earlier version continuing to execute until there are no consumers dependent on it. Upstream defines constraints on what it can consume, downstream defines when it's needed, and the Catchment optimally executes the sequence of Ponds supplying it with the best currency and frequency as possible.
|
|
227
237
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
The default engine is DuckDB, though this is configurable. Duckstring is however an independent project and is not affiliated with, endorsed by, or maintained by the DuckDB project.
|
|
231
|
-
|
|
232
|
-
-- **Note**: As the project is in development, most of the notes below should be read as *indended functionality*, and most features are not yet implemented.
|
|
238
|
+
You should not need to manage the DAG. You should not need global governance. You should know yourself and your suppliers and trust that you'll get what you need when you need it.
|
|
233
239
|
|
|
234
240
|
## Core Concepts
|
|
235
241
|
|
|
236
|
-
|
|
237
|
-
- **Pond**: Versioned transformation unit with declared upstream dependencies - the main element of version control
|
|
238
|
-
- **Inlet**: Pond with external dependencies and no upstream Ponds
|
|
239
|
-
- **Outlet**: Pond with no downstream Ponds (e.g. outputs final data products)
|
|
240
|
-
- **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
|
|
241
|
-
|
|
242
|
-
## Installation
|
|
243
|
-
|
|
244
|
-
```bash
|
|
245
|
-
pip install duckstring
|
|
246
|
-
```
|
|
247
|
-
|
|
248
|
-
## Quickstart
|
|
249
|
-
|
|
250
|
-
### 1) Connect to a Catchment
|
|
251
|
-
|
|
252
|
-
A Catchment is the execution environment, receiving Ponds and managing runs. It runs either as a local daemon or as a remote server, allowing you to start locally and seamlessly upgrade to a hosted/cloud server if you need to later.
|
|
253
|
-
|
|
254
|
-
#### Start a Catchment Server
|
|
255
|
-
|
|
256
|
-
To run a Catchment locally, run:
|
|
257
|
-
|
|
258
|
-
```bash
|
|
259
|
-
duckstring catchment start --name dev --port 5000 --root ~/.duckstring/dev
|
|
260
|
-
```
|
|
261
|
-
|
|
262
|
-
This will start a server with name 'dev' at port 5000 (the default, if none specified) and store Catchment details at `~/.duckstring/dev` (default is `~/.duckstring/{name}`). If any of these options are omitted you will be prompted on start.
|
|
263
|
-
|
|
264
|
-
#### Connect to a Remote Server
|
|
265
|
-
|
|
266
|
-
Alternatively, you can connect to a server running a Catchment:
|
|
267
|
-
|
|
268
|
-
```bash
|
|
269
|
-
duckstring catchment connect --name dev --path https://path.to.catchment
|
|
270
|
-
```
|
|
271
|
-
|
|
272
|
-
This will prompt for any necessary auth, and will add the Catchment under the specified name.
|
|
273
|
-
|
|
274
|
-
#### Connect to *duckstring.com*
|
|
275
|
-
|
|
276
|
-
There are future plans for a dedicated Catchment service at https://duckstring.com. If you're interested, please [contact me](mailto:dev@duckstring.com).
|
|
277
|
-
|
|
278
|
-
### 2) Define Pond(s)
|
|
279
|
-
|
|
280
|
-
#### Demo Ponds
|
|
281
|
-
|
|
282
|
-
If you want to see an example sequence of Ponds in action immediately, create three project directories and run one of these commands in each:
|
|
283
|
-
|
|
284
|
-
```bash
|
|
285
|
-
duckstring pond demo inlet
|
|
286
|
-
duckstring pond demo pond
|
|
287
|
-
duckstring pond demo outlet
|
|
288
|
-
```
|
|
289
|
-
|
|
290
|
-
It's recommended to do this before attempting to make your own so that you can get a feel for the structure.
|
|
291
|
-
|
|
292
|
-
#### Custom Pond
|
|
293
|
-
|
|
294
|
-
Create a project directory and run:
|
|
295
|
-
|
|
296
|
-
```bash
|
|
297
|
-
duckstring pond init example_pond
|
|
298
|
-
```
|
|
299
|
-
|
|
300
|
-
This will create a duckstring pond structure:
|
|
301
|
-
|
|
302
|
-
```text
|
|
303
|
-
root/
|
|
304
|
-
|-- src/
|
|
305
|
-
| |-- pond.py
|
|
306
|
-
|-- pond.toml
|
|
307
|
-
|-- __main__.py
|
|
308
|
-
|-- .gitignore
|
|
309
|
-
|-- README.md
|
|
310
|
-
```
|
|
311
|
-
|
|
312
|
-
Here `pond.py` contains the code for a single Ripple operation (currently blank), and `pond.toml` specifies the Pond name "example_pond" and version (defaulting to "0.1.0").
|
|
313
|
-
|
|
314
|
-
### 3) Deploy to Catchment
|
|
315
|
-
|
|
316
|
-
#### From Local
|
|
317
|
-
|
|
318
|
-
From a Pond's project root run:
|
|
319
|
-
|
|
320
|
-
```bash
|
|
321
|
-
duckstring deploy dev
|
|
322
|
-
```
|
|
323
|
-
|
|
324
|
-
This will read the pond name, version and type (Inlet, Pond, Outlet) from `pond.toml` and deploy the project contents to the Catchment specified by name (here `dev`).
|
|
325
|
-
|
|
326
|
-
Alternatively, you can import the Pond using the Catchment UI.
|
|
327
|
-
|
|
328
|
-
#### From Git
|
|
242
|
+
The main elements:
|
|
329
243
|
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
duckstring deploy dev --git {branch|commit|tag}
|
|
334
|
-
```
|
|
335
|
-
|
|
336
|
-
This will use the current branch/commit/tag to define the Pond. Upon each execution the Catchment will clone the repository and run it.
|
|
337
|
-
|
|
338
|
-
This can also be specified using the Catchment UI.
|
|
339
|
-
|
|
340
|
-
### 3) Execute
|
|
341
|
-
|
|
342
|
-
Ponds are executed by sending a Demand signal from an Outlet. This propagates backwards through the DAG until it reaches each upstream Inlet, causing them to execute, with children beginning upon completion of all of their parents.
|
|
343
|
-
|
|
344
|
-
These examples will use the Pond `outlet`, version `1.0.0`, as the execution reference. All examples may also be alternatively executed using the Catchment UI.
|
|
345
|
-
|
|
346
|
-
#### Pulse
|
|
347
|
-
|
|
348
|
-
To initiate a single run:
|
|
349
|
-
|
|
350
|
-
```bash
|
|
351
|
-
duckstring pulse dev outlet
|
|
352
|
-
```
|
|
353
|
-
|
|
354
|
-
The `pulse` mode emits a Demand signal from `outlet`, and when it begins execution, sends a Stop signal. This causes it to execute exactly once.
|
|
355
|
-
|
|
356
|
-
This will automatically run against the maximum version available for that Pond. To use a specific version:
|
|
357
|
-
|
|
358
|
-
```bash
|
|
359
|
-
duckstring pulse dev outlet --version 1
|
|
360
|
-
```
|
|
361
|
-
|
|
362
|
-
#### Wave
|
|
363
|
-
|
|
364
|
-
To continuously run:
|
|
365
|
-
|
|
366
|
-
```bash
|
|
367
|
-
duckstring wave dev outlet
|
|
368
|
-
```
|
|
369
|
-
|
|
370
|
-
The `wave` mode emits a Demand signal from `outlet`, and when it begins execution, sends *another* Demand signal. This causes it to execute continuously, as frequently as the DAG allows (i.e. at a period equal to the execution time of the slowest Ripple in any Pond).
|
|
371
|
-
|
|
372
|
-
#### Tide
|
|
373
|
-
|
|
374
|
-
To run at a scheduled frequency:
|
|
375
|
-
|
|
376
|
-
```bash
|
|
377
|
-
duckstring tide dev outlet 15 2 * * * --local
|
|
378
|
-
```
|
|
244
|
+
- **Catchment**: Control environment (FastAPI + UI + CLI)
|
|
245
|
+
- **Pond**: Versioned container with declared upstream dependencies
|
|
246
|
+
- **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
|
|
379
247
|
|
|
380
|
-
|
|
248
|
+
Ponds are typed or referred to in context:
|
|
381
249
|
|
|
382
|
-
|
|
250
|
+
- **Source**: A parent Pond
|
|
251
|
+
- **Sink**: A child Pond
|
|
252
|
+
- **Inlet**: A Pond with external dependencies and no Sources
|
|
253
|
+
- **Outlet**: A Pond with no Sinks (e.g. outputs final data products)
|
|
383
254
|
|
|
384
|
-
To
|
|
255
|
+
To see the orchestration model in action without installing anything, try the [Duckstring Playground](https://playground.duckstring.com).
|
|
385
256
|
|
|
386
|
-
|
|
387
|
-
duckstring status dev
|
|
388
|
-
```
|
|
389
|
-
|
|
390
|
-
This will print to CLI a summary for each Pond that is either currently executing or has Demand.
|
|
391
|
-
|
|
392
|
-
To include all Ponds:
|
|
257
|
+
## Quickstart
|
|
393
258
|
|
|
394
259
|
```bash
|
|
395
|
-
|
|
396
|
-
```
|
|
397
|
-
|
|
398
|
-
### 5) Retrieve Data
|
|
399
|
-
|
|
400
|
-
#### Get
|
|
401
|
-
|
|
402
|
-
The simplest way to retrieve data is to load by the Ripple name. This returns the entire contents of the directory, and does not require that the data be in a tabular format (e.g. SQL-compatible).
|
|
260
|
+
pip install duckstring
|
|
403
261
|
|
|
404
|
-
|
|
405
|
-
duckstring
|
|
262
|
+
# Start a local Catchment (the runtime + web UI) — leave it running
|
|
263
|
+
duckstring catchment init --name dev
|
|
406
264
|
```
|
|
407
265
|
|
|
408
|
-
|
|
266
|
+
Then, in another terminal:
|
|
409
267
|
|
|
410
268
|
```bash
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
269
|
+
# Create the demo pipeline (transactions, products → sales → reports) and deploy it
|
|
270
|
+
mkdir demo && cd demo
|
|
271
|
+
duckstring pond demo
|
|
272
|
+
duckstring pond deploy --all -y
|
|
415
273
|
|
|
416
|
-
|
|
274
|
+
# Run it end to end, once
|
|
275
|
+
duckstring trigger pulse reports
|
|
417
276
|
|
|
418
|
-
|
|
419
|
-
duckstring
|
|
277
|
+
# Look around
|
|
278
|
+
duckstring status # live state of every Pond
|
|
279
|
+
duckstring query reports monthly_summary # peek at an output table
|
|
420
280
|
```
|
|
421
281
|
|
|
422
|
-
|
|
282
|
+
The Catchment also serves a live web UI at `http://127.0.0.1:7474` — the Pond graph, freshness, run history, and the full trigger/control surface.
|
|
423
283
|
|
|
424
|
-
|
|
425
|
-
duckstring query dev outlet --sql @path/to/query.sql
|
|
426
|
-
```
|
|
284
|
+
## Execution
|
|
427
285
|
|
|
428
|
-
|
|
286
|
+
Ponds execute on demand signals sent to an Outlet, in two flavours — **push** runs the lineage forward to a target freshness; **pull** propagates demand upstream so every Pond re-runs as its Sources update, naturally throttled to the bottleneck. Each comes as a one-shot or a standing trigger:
|
|
429
287
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
288
|
+
| | Once | Continuously |
|
|
289
|
+
|---|---|---|
|
|
290
|
+
| **Push** | Pulse | Tide |
|
|
291
|
+
| **Pull** | Tap | Wave |
|
|
433
292
|
|
|
434
|
-
|
|
293
|
+
A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
|
|
435
294
|
|
|
436
|
-
|
|
295
|
+
## Scope & maturity
|
|
437
296
|
|
|
438
|
-
|
|
439
|
-
`--json`: JSON records
|
|
440
|
-
`--parquet`: Parquet file
|
|
297
|
+
Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
|
|
441
298
|
|
|
442
|
-
|
|
299
|
+
## Going further
|
|
443
300
|
|
|
444
|
-
|
|
301
|
+
Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:
|
|
445
302
|
|
|
446
|
-
|
|
447
|
-
duckstring
|
|
448
|
-
|
|
303
|
+
- [Quickstart](https://docs.duckstring.com/getting-started/quickstart) — the path above, with explanations
|
|
304
|
+
- [Theory](https://docs.duckstring.com/theory) — the freshness-based orchestration model in depth
|
|
305
|
+
- [Versioning](https://docs.duckstring.com/concepts/versioning) — SemVer on Ponds, concurrent major versions, atomic upgrades
|
|
306
|
+
- [Local testing](https://docs.duckstring.com/guides/local-testing) — Puddles: test a Pond before deploying it
|
|
307
|
+
- [Fault tolerance](https://docs.duckstring.com/guides/fault-tolerance) — retry budgets, failure states, recovery
|
|
308
|
+
- [Running a Catchment](https://docs.duckstring.com/guides/running-a-catchment) — hosting, authentication, platform deployment
|
|
309
|
+
- [CLI](https://docs.duckstring.com/reference/cli) / [HTTP API](https://docs.duckstring.com/reference/http-api) — full references
|
|
449
310
|
|
|
450
|
-
|
|
311
|
+
There are future plans for a hosted Catchment service at [duckstring.com](https://duckstring.com). If you're interested, please [get in touch](mailto:dev@duckstring.com).
|
|
451
312
|
|
|
452
|
-
|
|
313
|
+
## License
|
|
453
314
|
|
|
315
|
+
[Apache 2.0](LICENSE)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# Duckstring
|
|
2
|
+
*There is no DAG.*
|
|
3
|
+
|
|
4
|
+
Duckstring treats data transformations as software packages. Upstream dependencies are declared per Pond (unit operation), defining the DAG without the need for its direct management.
|
|
5
|
+
|
|
6
|
+
Ponds are upgraded and deployed to Duckstring's pull-based Catchment (orchestrator) atomically - like upgrading a package - with the earlier version continuing to execute until there are no consumers dependent on it. Upstream defines constraints on what it can consume, downstream defines when it's needed, and the Catchment optimally executes the sequence of Ponds supplying it with the best currency and frequency as possible.
|
|
7
|
+
|
|
8
|
+
You should not need to manage the DAG. You should not need global governance. You should know yourself and your suppliers and trust that you'll get what you need when you need it.
|
|
9
|
+
|
|
10
|
+
## Core Concepts
|
|
11
|
+
|
|
12
|
+
The main elements:
|
|
13
|
+
|
|
14
|
+
- **Catchment**: Control environment (FastAPI + UI + CLI)
|
|
15
|
+
- **Pond**: Versioned container with declared upstream dependencies
|
|
16
|
+
- **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
|
|
17
|
+
|
|
18
|
+
Ponds are typed or referred to in context:
|
|
19
|
+
|
|
20
|
+
- **Source**: A parent Pond
|
|
21
|
+
- **Sink**: A child Pond
|
|
22
|
+
- **Inlet**: A Pond with external dependencies and no Sources
|
|
23
|
+
- **Outlet**: A Pond with no Sinks (e.g. outputs final data products)
|
|
24
|
+
|
|
25
|
+
To see the orchestration model in action without installing anything, try the [Duckstring Playground](https://playground.duckstring.com).
|
|
26
|
+
|
|
27
|
+
## Quickstart
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install duckstring
|
|
31
|
+
|
|
32
|
+
# Start a local Catchment (the runtime + web UI) — leave it running
|
|
33
|
+
duckstring catchment init --name dev
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Then, in another terminal:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# Create the demo pipeline (transactions, products → sales → reports) and deploy it
|
|
40
|
+
mkdir demo && cd demo
|
|
41
|
+
duckstring pond demo
|
|
42
|
+
duckstring pond deploy --all -y
|
|
43
|
+
|
|
44
|
+
# Run it end to end, once
|
|
45
|
+
duckstring trigger pulse reports
|
|
46
|
+
|
|
47
|
+
# Look around
|
|
48
|
+
duckstring status # live state of every Pond
|
|
49
|
+
duckstring query reports monthly_summary # peek at an output table
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The Catchment also serves a live web UI at `http://127.0.0.1:7474` — the Pond graph, freshness, run history, and the full trigger/control surface.
|
|
53
|
+
|
|
54
|
+
## Execution
|
|
55
|
+
|
|
56
|
+
Ponds execute on demand signals sent to an Outlet, in two flavours — **push** runs the lineage forward to a target freshness; **pull** propagates demand upstream so every Pond re-runs as its Sources update, naturally throttled to the bottleneck. Each comes as a one-shot or a standing trigger:
|
|
57
|
+
|
|
58
|
+
| | Once | Continuously |
|
|
59
|
+
|---|---|---|
|
|
60
|
+
| **Push** | Pulse | Tide |
|
|
61
|
+
| **Pull** | Tap | Wave |
|
|
62
|
+
|
|
63
|
+
A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
|
|
64
|
+
|
|
65
|
+
## Scope & maturity
|
|
66
|
+
|
|
67
|
+
Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
|
|
68
|
+
|
|
69
|
+
## Going further
|
|
70
|
+
|
|
71
|
+
Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:
|
|
72
|
+
|
|
73
|
+
- [Quickstart](https://docs.duckstring.com/getting-started/quickstart) — the path above, with explanations
|
|
74
|
+
- [Theory](https://docs.duckstring.com/theory) — the freshness-based orchestration model in depth
|
|
75
|
+
- [Versioning](https://docs.duckstring.com/concepts/versioning) — SemVer on Ponds, concurrent major versions, atomic upgrades
|
|
76
|
+
- [Local testing](https://docs.duckstring.com/guides/local-testing) — Puddles: test a Pond before deploying it
|
|
77
|
+
- [Fault tolerance](https://docs.duckstring.com/guides/fault-tolerance) — retry budgets, failure states, recovery
|
|
78
|
+
- [Running a Catchment](https://docs.duckstring.com/guides/running-a-catchment) — hosting, authentication, platform deployment
|
|
79
|
+
- [CLI](https://docs.duckstring.com/reference/cli) / [HTTP API](https://docs.duckstring.com/reference/http-api) — full references
|
|
80
|
+
|
|
81
|
+
There are future plans for a hosted Catchment service at [duckstring.com](https://duckstring.com). If you're interested, please [get in touch](mailto:dev@duckstring.com).
|
|
82
|
+
|
|
83
|
+
## License
|
|
84
|
+
|
|
85
|
+
[Apache 2.0](LICENSE)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "duckstring"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = { file = "LICENSE" }
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Duckstring", email = "dev@duckstring.com" }
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
dependencies = [
|
|
13
|
+
"duckdb>=0.10",
|
|
14
|
+
"rich>=13.7",
|
|
15
|
+
"typer>=0.12",
|
|
16
|
+
"fastapi>=0.111",
|
|
17
|
+
"python-multipart>=0.0.9",
|
|
18
|
+
"uvicorn[standard]>=0.29",
|
|
19
|
+
"httpx>=0.25",
|
|
20
|
+
"tomli>=2.0; python_version < '3.11'",
|
|
21
|
+
"tomli-w>=1.0",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
dev = [
|
|
26
|
+
"pytest>=8.0",
|
|
27
|
+
"pytest-timeout>=0.5",
|
|
28
|
+
"ruff>=0.3",
|
|
29
|
+
"mypy>=1.8",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[tool.setuptools]
|
|
33
|
+
package-dir = { "" = "src" }
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.package-data]
|
|
36
|
+
"duckstring.catchment" = ["static/**/*", "schema/*.sql"]
|
|
37
|
+
"duckstring" = ["demo/**/*", "demo/**/.gitignore"]
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.packages.find]
|
|
40
|
+
where = ["src"]
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://duckstring.com"
|
|
44
|
+
Documentation = "https://docs.duckstring.com"
|
|
45
|
+
Repository = "https://github.com/duckstring-dev/duckstring"
|
|
46
|
+
Playground = "https://playground.duckstring.com"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
duckstring = "duckstring.cli:main"
|
|
50
|
+
ds = "duckstring.cli:main"
|
|
51
|
+
|
|
52
|
+
[tool.pytest.ini_options]
|
|
53
|
+
testpaths = ["tests"]
|
|
54
|
+
# Tight enough to catch an accidental real sleep (sim tests finish in ms), loose enough to absorb
|
|
55
|
+
# cold-runner I/O on CI (SQLite migrate / uvicorn startup can exceed 1s on shared runners).
|
|
56
|
+
timeout = 5
|
|
57
|
+
|
|
58
|
+
[tool.ruff]
|
|
59
|
+
line-length = 128
|
|
60
|
+
target-version = "py310"
|
|
61
|
+
extend-exclude = ["experiment"]
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint]
|
|
64
|
+
select = ["E", "F", "I", "B"]
|
|
65
|
+
ignore = ["B008"] # typer.Option/Argument in defaults is intentional
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError
|
|
2
|
+
from importlib.metadata import version as _pkg_version
|
|
3
|
+
|
|
4
|
+
from .core import (
|
|
5
|
+
Catchment,
|
|
6
|
+
Pond,
|
|
7
|
+
Puddle,
|
|
8
|
+
Ripple,
|
|
9
|
+
Trickle,
|
|
10
|
+
puddle,
|
|
11
|
+
ripple,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
__version__ = _pkg_version("duckstring")
|
|
16
|
+
except PackageNotFoundError: # running from a source tree without an installed dist
|
|
17
|
+
__version__ = "0.0.0"
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"Catchment",
|
|
21
|
+
"Pond",
|
|
22
|
+
"Puddle",
|
|
23
|
+
"Ripple",
|
|
24
|
+
"Trickle",
|
|
25
|
+
"puddle",
|
|
26
|
+
"ripple",
|
|
27
|
+
"__version__",
|
|
28
|
+
]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import os
|
|
5
|
+
import secrets
|
|
6
|
+
from contextlib import asynccontextmanager
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from fastapi import FastAPI
|
|
11
|
+
from fastapi.responses import JSONResponse
|
|
12
|
+
from fastapi.staticfiles import StaticFiles
|
|
13
|
+
|
|
14
|
+
from .db import connect, migrate
|
|
15
|
+
from .driver import Driver
|
|
16
|
+
from .launcher import NoopLauncher, SubprocessLauncher
|
|
17
|
+
from .routes import router
|
|
18
|
+
|
|
19
|
+
_STATIC_DIR = Path(__file__).parent / "static"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
async def _scheduler(driver: Driver) -> None:
|
|
23
|
+
"""Drive clock processes (Tide deadlines, window boundaries, Wave-on-idle) at next_wake."""
|
|
24
|
+
while True:
|
|
25
|
+
nw = driver.next_wake()
|
|
26
|
+
now = datetime.now(timezone.utc)
|
|
27
|
+
delay = (nw - now).total_seconds() if nw else 1.0
|
|
28
|
+
await asyncio.sleep(max(0.05, min(delay, 5.0)))
|
|
29
|
+
try:
|
|
30
|
+
driver.scheduler_tick()
|
|
31
|
+
except Exception as exc: # keep the loop alive
|
|
32
|
+
print(f"[catchment] scheduler error: {exc}", flush=True)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@asynccontextmanager
|
|
36
|
+
async def _lifespan(app: FastAPI):
|
|
37
|
+
base_url = app.state.base_url
|
|
38
|
+
if os.environ.get("DUCKSTRING_DISABLE_DUCKS"):
|
|
39
|
+
launcher = NoopLauncher()
|
|
40
|
+
else:
|
|
41
|
+
# Ducks dial back over the same authenticated surface — they present the API key as their token.
|
|
42
|
+
# base_url None = unknown (the platform picked the bind address): the launcher defers spawns
|
|
43
|
+
# until the dial-back middleware learns the address from the first request.
|
|
44
|
+
launcher = SubprocessLauncher(app.state.root, base_url, token=app.state.api_key or "")
|
|
45
|
+
driver = Driver(app.state.db, app.state.root, base_url, launcher)
|
|
46
|
+
app.state.driver = driver
|
|
47
|
+
app.state.launcher = launcher
|
|
48
|
+
|
|
49
|
+
# Restore: resume any Pond Runs that were in flight when the Catchment last stopped.
|
|
50
|
+
driver.resume_incomplete()
|
|
51
|
+
|
|
52
|
+
scheduler = asyncio.create_task(_scheduler(driver))
|
|
53
|
+
try:
|
|
54
|
+
yield
|
|
55
|
+
finally:
|
|
56
|
+
scheduler.cancel()
|
|
57
|
+
try:
|
|
58
|
+
await scheduler
|
|
59
|
+
except asyncio.CancelledError:
|
|
60
|
+
pass
|
|
61
|
+
launcher.shutdown_all()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def create_app(root: Path, api_key: str | None = None, base_url: str | None = None) -> FastAPI:
|
|
65
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
con = connect(root / "duck.db")
|
|
67
|
+
migrate(con)
|
|
68
|
+
|
|
69
|
+
app = FastAPI(title="Duckstring Catchment", lifespan=_lifespan)
|
|
70
|
+
app.state.root = root
|
|
71
|
+
app.state.db = con
|
|
72
|
+
# API key: explicit argument, or the environment (useful for containers/remote serving). When
|
|
73
|
+
# set, every /api request (except /api/health) must present it — Bearer header or X-Duck-Token.
|
|
74
|
+
app.state.api_key = api_key or os.environ.get("DUCKSTRING_API_KEY") or None
|
|
75
|
+
# The address Ducks dial back to: explicit argument (the CLI passes its bind address), or the
|
|
76
|
+
# environment, or None — unknown, because the host platform picks the bind address (e.g. Posit
|
|
77
|
+
# Connect). When None it is learned from the first request's ASGI scope below.
|
|
78
|
+
app.state.base_url = base_url or os.environ.get("DUCKSTRING_CATCHMENT_URL") or None
|
|
79
|
+
|
|
80
|
+
@app.middleware("http")
|
|
81
|
+
async def _learn_dialback_address(request, call_next):
|
|
82
|
+
launcher = getattr(app.state, "launcher", None)
|
|
83
|
+
if launcher is not None and getattr(launcher, "base_url", "") is None:
|
|
84
|
+
server = request.scope.get("server") # the server's bound (host, port) per the ASGI spec
|
|
85
|
+
if server and server[1]: # a unix socket has port None — nothing TCP to dial
|
|
86
|
+
host = "127.0.0.1" if server[0] in ("0.0.0.0", "::") else server[0]
|
|
87
|
+
url = f"http://{host}:{server[1]}"
|
|
88
|
+
app.state.base_url = url
|
|
89
|
+
app.state.driver.base_url = url
|
|
90
|
+
launcher.set_base_url(url) # spawns any Ducks that were waiting on the address
|
|
91
|
+
return await call_next(request)
|
|
92
|
+
|
|
93
|
+
@app.middleware("http")
|
|
94
|
+
async def _require_api_key(request, call_next):
|
|
95
|
+
key = app.state.api_key
|
|
96
|
+
path = request.url.path
|
|
97
|
+
if key and path.startswith("/api") and path != "/api/health":
|
|
98
|
+
auth = request.headers.get("authorization", "")
|
|
99
|
+
supplied = auth[7:] if auth.lower().startswith("bearer ") else ""
|
|
100
|
+
supplied = supplied or request.headers.get("x-duck-token", "")
|
|
101
|
+
if not secrets.compare_digest(supplied, key):
|
|
102
|
+
return JSONResponse({"detail": "Invalid or missing API key"}, status_code=401)
|
|
103
|
+
return await call_next(request)
|
|
104
|
+
|
|
105
|
+
app.include_router(router, prefix="/api")
|
|
106
|
+
|
|
107
|
+
if _STATIC_DIR.exists():
|
|
108
|
+
app.mount("/", StaticFiles(directory=_STATIC_DIR, html=True), name="frontend")
|
|
109
|
+
|
|
110
|
+
return app
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""ASGI entry for platform-hosted Catchments (Posit Connect, gunicorn/uvicorn, containers).
|
|
2
|
+
|
|
3
|
+
The platform owns the server lifecycle; this module just exposes the app. A deployable bundle is
|
|
4
|
+
two files — ``requirements.txt`` (``duckstring``) and an ``app.py`` containing::
|
|
5
|
+
|
|
6
|
+
from duckstring.catchment.asgi import app
|
|
7
|
+
|
|
8
|
+
Configuration is environment-only:
|
|
9
|
+
|
|
10
|
+
- ``DUCKSTRING_ROOT`` — the Catchment root. Defaults to ``./.duckstring`` (relative to the working
|
|
11
|
+
directory, i.e. the deployed content directory). On platforms that replace the content directory
|
|
12
|
+
on redeploy (Posit Connect does), the default survives process restarts but **not redeploys of
|
|
13
|
+
the Catchment app itself** — point this at a persistent path for durable state.
|
|
14
|
+
- ``DUCKSTRING_API_KEY`` — optional built-in API key. Leave unset when the platform already gates
|
|
15
|
+
requests (the recommended hosted model).
|
|
16
|
+
- ``DUCKSTRING_CATCHMENT_URL`` — optional Duck dial-back address. Normally unset: the platform
|
|
17
|
+
picks the bind address, and the Catchment learns it from the first request it serves.
|
|
18
|
+
|
|
19
|
+
Run only **one** process of this app (e.g. Posit Connect's "Max processes" = 1): the Catchment is
|
|
20
|
+
a single brain — one scheduler, one SQLite, one set of Ducks.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import os
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
from .app import create_app
|
|
29
|
+
|
|
30
|
+
app = create_app(Path(os.environ.get("DUCKSTRING_ROOT", ".duckstring")).expanduser().resolve())
|