daguflow 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- daguflow-0.1.0/LICENSE +21 -0
- daguflow-0.1.0/PKG-INFO +223 -0
- daguflow-0.1.0/README.md +194 -0
- daguflow-0.1.0/pyproject.toml +55 -0
- daguflow-0.1.0/pyproject.toml.orig +49 -0
- daguflow-0.1.0/src/daguflow/__init__.py +35 -0
- daguflow-0.1.0/src/daguflow/_context.py +46 -0
- daguflow-0.1.0/src/daguflow/_types.py +12 -0
- daguflow-0.1.0/src/daguflow/actions/__init__.py +37 -0
- daguflow-0.1.0/src/daguflow/actions/_generated.py +532 -0
- daguflow-0.1.0/src/daguflow/cli.py +246 -0
- daguflow-0.1.0/src/daguflow/client.py +273 -0
- daguflow-0.1.0/src/daguflow/compile.py +69 -0
- daguflow-0.1.0/src/daguflow/dag.py +354 -0
- daguflow-0.1.0/src/daguflow/groups.py +201 -0
- daguflow-0.1.0/src/daguflow/params.py +209 -0
- daguflow-0.1.0/src/daguflow/py.typed +0 -0
- daguflow-0.1.0/src/daguflow/refs.py +185 -0
- daguflow-0.1.0/src/daguflow/schedule.py +59 -0
- daguflow-0.1.0/src/daguflow/schema/__init__.py +82 -0
- daguflow-0.1.0/src/daguflow/step.py +362 -0
- daguflow-0.1.0/src/daguflow/targets.py +79 -0
- daguflow-0.1.0/src/daguflow/task.py +314 -0
- daguflow-0.1.0/src/daguflow/validate.py +99 -0
daguflow-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Stephan Fitzpatrick
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
daguflow-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: daguflow
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Author dagu workflows in Python, Airflow-style: decorators for DAGs and tasks, context managers for sub-DAGs
|
|
5
|
+
Keywords: dagu,workflow,dag,airflow,orchestration,scheduler
|
|
6
|
+
Author: Stephan Fitzpatrick
|
|
7
|
+
Author-email: Stephan Fitzpatrick <stephan@knowsuchagency.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Dist: httpx>=0.27
|
|
20
|
+
Requires-Dist: jsonschema>=4.21
|
|
21
|
+
Requires-Dist: pyyaml>=6
|
|
22
|
+
Requires-Dist: pytest>=8 ; extra == 'test'
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Project-URL: Homepage, https://github.com/execution-associates/daguflow
|
|
25
|
+
Project-URL: Repository, https://github.com/execution-associates/daguflow
|
|
26
|
+
Project-URL: Issues, https://github.com/execution-associates/daguflow/issues
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# daguflow
|
|
31
|
+
|
|
32
|
+
Write [dagu](https://github.com/dagucloud/dagu) workflows in Python, the way Airflow does it: `@dag` and
|
|
33
|
+
`@task` decorators, `>>` for dependencies, and context managers for sub-DAGs and loops. `daguflow build` turns
|
|
34
|
+
them into the YAML dagu runs. Each file is checked against the schema of the dagu version daguflow pins
|
|
35
|
+
(**v2.24.0**), so a typo fails at build time, not when dagu loads the file.
|
|
36
|
+
|
|
37
|
+
```console
|
|
38
|
+
$ pip install daguflow # or: uv add daguflow
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from typing import Annotated
|
|
43
|
+
from daguflow import ITEM, Cron, Range, SSHTarget, SubDAG, dag, env, task
|
|
44
|
+
|
|
45
|
+
pi = SSHTarget(user=env.PI_USER, host="192.168.1.50", key="/etc/dagu/ssh/id_ed25519",
|
|
46
|
+
strict_host_key=True, known_host_file="/etc/dagu/ssh/known_hosts")
|
|
47
|
+
|
|
48
|
+
@task(runs_on=pi)
|
|
49
|
+
def list_lights():
|
|
50
|
+
return "lights.py list --json"
|
|
51
|
+
|
|
52
|
+
@task(runs_on=pi)
|
|
53
|
+
def set_level(light, level):
|
|
54
|
+
return f"lights.py brightness --light {light:q} --value {level}"
|
|
55
|
+
|
|
56
|
+
# Runs on the Pi at run time. unsafe_args: its input is our own lights.py JSON.
|
|
57
|
+
@task.python(runs_on=pi, python="/usr/bin/python3", unsafe_args=True)
|
|
58
|
+
def count(listing: dict) -> int:
|
|
59
|
+
return len(listing["names"])
|
|
60
|
+
|
|
61
|
+
@dag(schedule=Cron("30 22 * * *", tz="America/Los_Angeles"), labels=["lights"])
|
|
62
|
+
def dim_each(level: Annotated[int, Range(0, 100)] = 20):
|
|
63
|
+
"""Dim every light, four at a time."""
|
|
64
|
+
found = list_lights()
|
|
65
|
+
with SubDAG("one", items=found["names"], max_concurrent=4,
|
|
66
|
+
params={"light": ITEM, "level": level}) as one:
|
|
67
|
+
set_level(one.params.light, one.params.level)
|
|
68
|
+
one >> count(found)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
```console
|
|
72
|
+
$ daguflow build workflows -o dags/ # one YAML file per DAG
|
|
73
|
+
$ daguflow build workflows -o dags/ --check # exit 1 with a diff if dags/ is stale
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## How it maps to dagu
|
|
77
|
+
|
|
78
|
+
| Python | dagu YAML |
|
|
79
|
+
|---|---|
|
|
80
|
+
| `@dag` function / `with DAG("name"):` | one file, `<name>.yaml` (the filename is the name) |
|
|
81
|
+
| function parameters | `params:`: plain defaults give `- value: 50`; `Literal`, `Annotated[.., Range/Pattern/Length/Doc]`, `Param(...)` or no default give rich definitions dagu enforces at start |
|
|
82
|
+
| first docstring line / the rest | `description:` / the file's header comment |
|
|
83
|
+
| `@task` call, `shell(...)`, `step(...)`, `target.run(...)`, `actions.x.y(...).step()` | a step |
|
|
84
|
+
| `a >> b`, `[a, b] >> c`, `chain(a, b, c)` | `depends:` |
|
|
85
|
+
| reading another step's output | `depends:` added for you (dagu won't infer it) |
|
|
86
|
+
| `with SubDAG("x"):` | a `---` local DAG in the same file and a `dag.run` step; `items=` adds `parallel:` |
|
|
87
|
+
| `with ForEach(items):` | an inline `foreach:` step |
|
|
88
|
+
| `call_dag("other")` | `action: dag.run` against another file |
|
|
89
|
+
| `with on_failure():` (and `on_success/exit/abort/init/wait`) | `handler_on:` |
|
|
90
|
+
| `Cron("0 9 * * *", tz=...)` | `CRON_TZ=... 0 9 * * *` |
|
|
91
|
+
|
|
92
|
+
Every other dagu key is passed by its own name, e.g. `Step(..., repeat_policy={...})` or
|
|
93
|
+
`DAG(..., max_active_steps=4)`, and checked against the schema. A few Python conveniences translate:
|
|
94
|
+
- `timeout=`: seconds or a `timedelta`, becomes `timeout_sec`.
|
|
95
|
+
- `retries=` with `retry_interval=` (default 5s): dagu requires an interval for any `retry_policy`.
|
|
96
|
+
- `env=`: a dict, becomes dagu's ordered list.
|
|
97
|
+
- `tags=`: becomes `labels`.
|
|
98
|
+
|
|
99
|
+
## References and quoting
|
|
100
|
+
|
|
101
|
+
Refs are build-time stand-ins for dagu's `${...}`. Format them in f-strings:
|
|
102
|
+
|
|
103
|
+
| Python | renders |
|
|
104
|
+
|---|---|
|
|
105
|
+
| `f"{effect}"` | `${effect}` |
|
|
106
|
+
| `f"{effect:q}"` | `"${effect}"` |
|
|
107
|
+
| `f"{value:bare}"` | `$value` |
|
|
108
|
+
| `step.output["field"]` | `${STEP.field}` |
|
|
109
|
+
| `env.HOME` | `${env.HOME}` |
|
|
110
|
+
| `ctx.run.id` | `${context.run.id}` |
|
|
111
|
+
| `ITEM` | `${ITEM}` |
|
|
112
|
+
| `var("NAME")` | `${NAME}` |
|
|
113
|
+
|
|
114
|
+
**Use `{x:q}` for free text in SSH commands.** For an SSH target, dagu replaces a double-quoted `"${x}"` with the
|
|
115
|
+
value re-quoted as one shell word, so `Rock 'n' Roll (Live) & More` arrives as one argument. Unquoted, the
|
|
116
|
+
remote shell splits it. Single-quoted, dagu leaves it unexpanded. Measured on dagu 2.16.2 and 2.24.0
|
|
117
|
+
(`tests/test_ssh_integration.py`).
|
|
118
|
+
|
|
119
|
+
**SSH `with:` fields expand only scoped references.** Write `user=env.PI_USER`, never `"${PI_USER}"`.
|
|
120
|
+
|
|
121
|
+
## `@task` and `@task.python`
|
|
122
|
+
|
|
123
|
+
- **`@task` runs at build time.** It returns a command string, an `Action`, or a `Step`, and calling it inside
|
|
124
|
+
a DAG adds the step. Decorator kwargs are step options (`output=`, `retries=`, `timeout=`, `continue_on=`,
|
|
125
|
+
...). `runs_on=` takes an `SSHTarget` or `container(image)`. `task.override(name=...)` changes them for one
|
|
126
|
+
call. Calling a task twice names the second step `name__1`, as in Airflow.
|
|
127
|
+
- **`@task.python` runs at run time.** The function's own source ships to wherever the step runs: the dagu
|
|
128
|
+
worker (`run:`), an SSH host, or a container.
|
|
129
|
+
- Arguments go in as JSON. The return value comes back as JSON on stdout and becomes the step's output.
|
|
130
|
+
- The function's own `print`s go to stderr, so they can't corrupt that output.
|
|
131
|
+
- Passing another step (or its output) as an argument makes it upstream.
|
|
132
|
+
- `deps=["httpx"]` runs it through `uv run --with`.
|
|
133
|
+
- **The function must be self-contained**: imports inside the body, no closures, no module globals. Build
|
|
134
|
+
fails and names any outside names it uses.
|
|
135
|
+
- **Python must exist where the step runs.** dagu's official image has none.
|
|
136
|
+
- **How run-time values arrive:**
|
|
137
|
+
- Locally, through step `env:`, so values are never spliced into code.
|
|
138
|
+
- Over SSH, each on its own file descriptor as a quoted heredoc, so quotes, `$` and JSON arrive byte for
|
|
139
|
+
byte. A value containing the line `DAGUFLOW_EOF` ends its heredoc early, and the rest runs as shell on
|
|
140
|
+
the host. **So build accepts only params dagu constrains**: an `enum`, a numeric type, or a `pattern`
|
|
141
|
+
that admits no newline. Step outputs, env and unconstrained params are refused; `unsafe_args=True`
|
|
142
|
+
overrides, for values whose every source you trust.
|
|
143
|
+
|
|
144
|
+
## Actions
|
|
145
|
+
|
|
146
|
+
`daguflow.actions` has a typed, keyword-only builder for every builtin action at the pinned dagu version,
|
|
147
|
+
generated from its schema (`tools/gen_actions.py`):
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
from daguflow import actions
|
|
151
|
+
actions.http.request(method="GET", url="https://example.com", timeout=10).step("fetch", output="PAGE")
|
|
152
|
+
actions.jq.filter(filter=".items[]", data=page.output).step("items")
|
|
153
|
+
actions.custom("dagucloud/duckdb@v1", query="select 1").step("duck")
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
- Required keys are enforced, and keys dagu would reject raise `TypeError`.
|
|
157
|
+
- Python keywords take a trailing underscore: `from_=`, `import_=`.
|
|
158
|
+
- `SSHTarget(...)` (also `actions.ssh.Target`) writes SSH settings once. Use `.run(cmd)` for a step, as
|
|
159
|
+
`@task(runs_on=...)`, or as a DAG's `ssh=` default.
|
|
160
|
+
- **A DAG-level `ssh=` moves every step without an action onto that host,** plain `run:` steps included.
|
|
161
|
+
|
|
162
|
+
## CLI
|
|
163
|
+
|
|
164
|
+
- `daguflow build TARGET... -o DIR` writes one YAML per DAG. TARGET is a module, a package directory or a
|
|
165
|
+
`.py` file.
|
|
166
|
+
- `--check` writes nothing and exits 1 with a diff when DIR is stale.
|
|
167
|
+
- `--prune` deletes files daguflow generated that no DAG produces any more. Hand-written files are left
|
|
168
|
+
alone.
|
|
169
|
+
- `daguflow validate FILE...` checks the schema plus dagu's document rules.
|
|
170
|
+
- `--dagu-bin dagu` also runs `dagu validate`.
|
|
171
|
+
- `--server URL` also asks a running dagu's `/dags/validate`.
|
|
172
|
+
- `daguflow run DAG --server URL -p k=v --wait` starts a run and prints its outputs. `--inline MODULE` sends
|
|
173
|
+
the DAG's YAML without a file on the server.
|
|
174
|
+
- `daguflow schema show` names the pinned schema and its cache; `daguflow schema fetch [--out PATH]`
|
|
175
|
+
downloads it ahead of an offline build.
|
|
176
|
+
- `daguflow schema update --version vX.Y.Z`, in a daguflow checkout, repins another dagu release. Then run
|
|
177
|
+
`mise run gen` and the tests.
|
|
178
|
+
|
|
179
|
+
## The dagu schema
|
|
180
|
+
|
|
181
|
+
daguflow validates against the schema of one dagu release, pinned by commit and sha256, but does not ship
|
|
182
|
+
it: dagu is GPL-3.0-or-later and daguflow is MIT. The first build downloads it from dagu's repository,
|
|
183
|
+
checks the hash and caches it in `$XDG_CACHE_HOME/daguflow` (`~/.cache/daguflow`). Offline, run
|
|
184
|
+
`daguflow schema fetch` while you still have a network, or set `DAGUFLOW_SCHEMA` to a copy; either way the
|
|
185
|
+
hash is checked.
|
|
186
|
+
|
|
187
|
+
## Client
|
|
188
|
+
|
|
189
|
+
```python
|
|
190
|
+
from daguflow.client import Client
|
|
191
|
+
with Client("http://127.0.0.1:8080") as dagu:
|
|
192
|
+
run_id = dagu.start("set_light_effect", params={"effect": "Good Vibes"})
|
|
193
|
+
run = dagu.wait("set_light_effect", run_id)
|
|
194
|
+
print(run.status, dagu.outputs("set_light_effect", run_id, decode=True))
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Covered calls:
|
|
198
|
+
- `validate`, `run_inline`, `start`, `enqueue`, `wait`, `status`, `outputs`, `step_log`, `run_log`, `stop`,
|
|
199
|
+
`retry`.
|
|
200
|
+
- `list_dags`, `list_runs`, `sub_runs`, `suspend`, `create_dag` / `update_spec` / `delete_dag`.
|
|
201
|
+
- `request()` for anything else.
|
|
202
|
+
|
|
203
|
+
API quirks the client absorbs:
|
|
204
|
+
- Params go as a JSON object string.
|
|
205
|
+
- The run-detail wrapper key differs by endpoint (`dagRunDetails` vs `dagRun`).
|
|
206
|
+
- Outputs come back as strings under camelCased keys (`output: RESULT` → `result`).
|
|
207
|
+
|
|
208
|
+
## Development
|
|
209
|
+
|
|
210
|
+
`mise run test` runs everything in a throwaway isb sandbox:
|
|
211
|
+
- the unit tests;
|
|
212
|
+
- golden YAML for `examples/`;
|
|
213
|
+
- `dagu validate` on every golden file, using the pinned release binary;
|
|
214
|
+
- the examples running on a real dagu server, and SSH steps against a throwaway sshd.
|
|
215
|
+
|
|
216
|
+
Without isb: `uv run --extra test pytest`. The integration tier runs when `DAGUFLOW_DAGU_BIN` points at
|
|
217
|
+
the pinned dagu binary (`tools/fetch-dagu.sh` downloads it).
|
|
218
|
+
|
|
219
|
+
`mise run gen` regenerates the action builders. A test fails when they drift from the schema.
|
|
220
|
+
|
|
221
|
+
## License
|
|
222
|
+
|
|
223
|
+
MIT. dagu's schema, which daguflow downloads at build time, is GPL-3.0-or-later and stays dagu's.
|
daguflow-0.1.0/README.md
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# daguflow
|
|
2
|
+
|
|
3
|
+
Write [dagu](https://github.com/dagucloud/dagu) workflows in Python, the way Airflow does it: `@dag` and
|
|
4
|
+
`@task` decorators, `>>` for dependencies, and context managers for sub-DAGs and loops. `daguflow build` turns
|
|
5
|
+
them into the YAML dagu runs. Each file is checked against the schema of the dagu version daguflow pins
|
|
6
|
+
(**v2.24.0**), so a typo fails at build time, not when dagu loads the file.
|
|
7
|
+
|
|
8
|
+
```console
|
|
9
|
+
$ pip install daguflow # or: uv add daguflow
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
```python
|
|
13
|
+
from typing import Annotated
|
|
14
|
+
from daguflow import ITEM, Cron, Range, SSHTarget, SubDAG, dag, env, task
|
|
15
|
+
|
|
16
|
+
pi = SSHTarget(user=env.PI_USER, host="192.168.1.50", key="/etc/dagu/ssh/id_ed25519",
|
|
17
|
+
strict_host_key=True, known_host_file="/etc/dagu/ssh/known_hosts")
|
|
18
|
+
|
|
19
|
+
@task(runs_on=pi)
|
|
20
|
+
def list_lights():
|
|
21
|
+
return "lights.py list --json"
|
|
22
|
+
|
|
23
|
+
@task(runs_on=pi)
|
|
24
|
+
def set_level(light, level):
|
|
25
|
+
return f"lights.py brightness --light {light:q} --value {level}"
|
|
26
|
+
|
|
27
|
+
# Runs on the Pi at run time. unsafe_args: its input is our own lights.py JSON.
|
|
28
|
+
@task.python(runs_on=pi, python="/usr/bin/python3", unsafe_args=True)
|
|
29
|
+
def count(listing: dict) -> int:
|
|
30
|
+
return len(listing["names"])
|
|
31
|
+
|
|
32
|
+
@dag(schedule=Cron("30 22 * * *", tz="America/Los_Angeles"), labels=["lights"])
|
|
33
|
+
def dim_each(level: Annotated[int, Range(0, 100)] = 20):
|
|
34
|
+
"""Dim every light, four at a time."""
|
|
35
|
+
found = list_lights()
|
|
36
|
+
with SubDAG("one", items=found["names"], max_concurrent=4,
|
|
37
|
+
params={"light": ITEM, "level": level}) as one:
|
|
38
|
+
set_level(one.params.light, one.params.level)
|
|
39
|
+
one >> count(found)
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
```console
|
|
43
|
+
$ daguflow build workflows -o dags/ # one YAML file per DAG
|
|
44
|
+
$ daguflow build workflows -o dags/ --check # exit 1 with a diff if dags/ is stale
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## How it maps to dagu
|
|
48
|
+
|
|
49
|
+
| Python | dagu YAML |
|
|
50
|
+
|---|---|
|
|
51
|
+
| `@dag` function / `with DAG("name"):` | one file, `<name>.yaml` (the filename is the name) |
|
|
52
|
+
| function parameters | `params:`: plain defaults give `- value: 50`; `Literal`, `Annotated[.., Range/Pattern/Length/Doc]`, `Param(...)` or no default give rich definitions dagu enforces at start |
|
|
53
|
+
| first docstring line / the rest | `description:` / the file's header comment |
|
|
54
|
+
| `@task` call, `shell(...)`, `step(...)`, `target.run(...)`, `actions.x.y(...).step()` | a step |
|
|
55
|
+
| `a >> b`, `[a, b] >> c`, `chain(a, b, c)` | `depends:` |
|
|
56
|
+
| reading another step's output | `depends:` added for you (dagu won't infer it) |
|
|
57
|
+
| `with SubDAG("x"):` | a `---` local DAG in the same file and a `dag.run` step; `items=` adds `parallel:` |
|
|
58
|
+
| `with ForEach(items):` | an inline `foreach:` step |
|
|
59
|
+
| `call_dag("other")` | `action: dag.run` against another file |
|
|
60
|
+
| `with on_failure():` (and `on_success/exit/abort/init/wait`) | `handler_on:` |
|
|
61
|
+
| `Cron("0 9 * * *", tz=...)` | `CRON_TZ=... 0 9 * * *` |
|
|
62
|
+
|
|
63
|
+
Every other dagu key is passed by its own name, e.g. `Step(..., repeat_policy={...})` or
|
|
64
|
+
`DAG(..., max_active_steps=4)`, and checked against the schema. A few Python conveniences translate:
|
|
65
|
+
- `timeout=`: seconds or a `timedelta`, becomes `timeout_sec`.
|
|
66
|
+
- `retries=` with `retry_interval=` (default 5s): dagu requires an interval for any `retry_policy`.
|
|
67
|
+
- `env=`: a dict, becomes dagu's ordered list.
|
|
68
|
+
- `tags=`: becomes `labels`.
|
|
69
|
+
|
|
70
|
+
## References and quoting
|
|
71
|
+
|
|
72
|
+
Refs are build-time stand-ins for dagu's `${...}`. Format them in f-strings:
|
|
73
|
+
|
|
74
|
+
| Python | renders |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `f"{effect}"` | `${effect}` |
|
|
77
|
+
| `f"{effect:q}"` | `"${effect}"` |
|
|
78
|
+
| `f"{value:bare}"` | `$value` |
|
|
79
|
+
| `step.output["field"]` | `${STEP.field}` |
|
|
80
|
+
| `env.HOME` | `${env.HOME}` |
|
|
81
|
+
| `ctx.run.id` | `${context.run.id}` |
|
|
82
|
+
| `ITEM` | `${ITEM}` |
|
|
83
|
+
| `var("NAME")` | `${NAME}` |
|
|
84
|
+
|
|
85
|
+
**Use `{x:q}` for free text in SSH commands.** For an SSH target, dagu replaces a double-quoted `"${x}"` with the
|
|
86
|
+
value re-quoted as one shell word, so `Rock 'n' Roll (Live) & More` arrives as one argument. Unquoted, the
|
|
87
|
+
remote shell splits it. Single-quoted, dagu leaves it unexpanded. Measured on dagu 2.16.2 and 2.24.0
|
|
88
|
+
(`tests/test_ssh_integration.py`).
|
|
89
|
+
|
|
90
|
+
**SSH `with:` fields expand only scoped references.** Write `user=env.PI_USER`, never `"${PI_USER}"`.
|
|
91
|
+
|
|
92
|
+
## `@task` and `@task.python`
|
|
93
|
+
|
|
94
|
+
- **`@task` runs at build time.** It returns a command string, an `Action`, or a `Step`, and calling it inside
|
|
95
|
+
a DAG adds the step. Decorator kwargs are step options (`output=`, `retries=`, `timeout=`, `continue_on=`,
|
|
96
|
+
...). `runs_on=` takes an `SSHTarget` or `container(image)`. `task.override(name=...)` changes them for one
|
|
97
|
+
call. Calling a task twice names the second step `name__1`, as in Airflow.
|
|
98
|
+
- **`@task.python` runs at run time.** The function's own source ships to wherever the step runs: the dagu
|
|
99
|
+
worker (`run:`), an SSH host, or a container.
|
|
100
|
+
- Arguments go in as JSON. The return value comes back as JSON on stdout and becomes the step's output.
|
|
101
|
+
- The function's own `print`s go to stderr, so they can't corrupt that output.
|
|
102
|
+
- Passing another step (or its output) as an argument makes it upstream.
|
|
103
|
+
- `deps=["httpx"]` runs it through `uv run --with`.
|
|
104
|
+
- **The function must be self-contained**: imports inside the body, no closures, no module globals. Build
|
|
105
|
+
fails and names any outside names it uses.
|
|
106
|
+
- **Python must exist where the step runs.** dagu's official image has none.
|
|
107
|
+
- **How run-time values arrive:**
|
|
108
|
+
- Locally, through step `env:`, so values are never spliced into code.
|
|
109
|
+
- Over SSH, each on its own file descriptor as a quoted heredoc, so quotes, `$` and JSON arrive byte for
|
|
110
|
+
byte. A value containing the line `DAGUFLOW_EOF` ends its heredoc early, and the rest runs as shell on
|
|
111
|
+
the host. **So build accepts only params dagu constrains**: an `enum`, a numeric type, or a `pattern`
|
|
112
|
+
that admits no newline. Step outputs, env and unconstrained params are refused; `unsafe_args=True`
|
|
113
|
+
overrides, for values whose every source you trust.
|
|
114
|
+
|
|
115
|
+
## Actions
|
|
116
|
+
|
|
117
|
+
`daguflow.actions` has a typed, keyword-only builder for every builtin action at the pinned dagu version,
|
|
118
|
+
generated from its schema (`tools/gen_actions.py`):
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from daguflow import actions
|
|
122
|
+
actions.http.request(method="GET", url="https://example.com", timeout=10).step("fetch", output="PAGE")
|
|
123
|
+
actions.jq.filter(filter=".items[]", data=page.output).step("items")
|
|
124
|
+
actions.custom("dagucloud/duckdb@v1", query="select 1").step("duck")
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
- Required keys are enforced, and keys dagu would reject raise `TypeError`.
|
|
128
|
+
- Python keywords take a trailing underscore: `from_=`, `import_=`.
|
|
129
|
+
- `SSHTarget(...)` (also `actions.ssh.Target`) writes SSH settings once. Use `.run(cmd)` for a step, as
|
|
130
|
+
`@task(runs_on=...)`, or as a DAG's `ssh=` default.
|
|
131
|
+
- **A DAG-level `ssh=` moves every step without an action onto that host,** plain `run:` steps included.
|
|
132
|
+
|
|
133
|
+
## CLI
|
|
134
|
+
|
|
135
|
+
- `daguflow build TARGET... -o DIR` writes one YAML per DAG. TARGET is a module, a package directory or a
|
|
136
|
+
`.py` file.
|
|
137
|
+
- `--check` writes nothing and exits 1 with a diff when DIR is stale.
|
|
138
|
+
- `--prune` deletes files daguflow generated that no DAG produces any more. Hand-written files are left
|
|
139
|
+
alone.
|
|
140
|
+
- `daguflow validate FILE...` checks the schema plus dagu's document rules.
|
|
141
|
+
- `--dagu-bin dagu` also runs `dagu validate`.
|
|
142
|
+
- `--server URL` also asks a running dagu's `/dags/validate`.
|
|
143
|
+
- `daguflow run DAG --server URL -p k=v --wait` starts a run and prints its outputs. `--inline MODULE` sends
|
|
144
|
+
the DAG's YAML without a file on the server.
|
|
145
|
+
- `daguflow schema show` names the pinned schema and its cache; `daguflow schema fetch [--out PATH]`
|
|
146
|
+
downloads it ahead of an offline build.
|
|
147
|
+
- `daguflow schema update --version vX.Y.Z`, in a daguflow checkout, repins another dagu release. Then run
|
|
148
|
+
`mise run gen` and the tests.
|
|
149
|
+
|
|
150
|
+
## The dagu schema
|
|
151
|
+
|
|
152
|
+
daguflow validates against the schema of one dagu release, pinned by commit and sha256, but does not ship
|
|
153
|
+
it: dagu is GPL-3.0-or-later and daguflow is MIT. The first build downloads it from dagu's repository,
|
|
154
|
+
checks the hash and caches it in `$XDG_CACHE_HOME/daguflow` (`~/.cache/daguflow`). Offline, run
|
|
155
|
+
`daguflow schema fetch` while you still have a network, or set `DAGUFLOW_SCHEMA` to a copy; either way the
|
|
156
|
+
hash is checked.
|
|
157
|
+
|
|
158
|
+
## Client
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
from daguflow.client import Client
|
|
162
|
+
with Client("http://127.0.0.1:8080") as dagu:
|
|
163
|
+
run_id = dagu.start("set_light_effect", params={"effect": "Good Vibes"})
|
|
164
|
+
run = dagu.wait("set_light_effect", run_id)
|
|
165
|
+
print(run.status, dagu.outputs("set_light_effect", run_id, decode=True))
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Covered calls:
|
|
169
|
+
- `validate`, `run_inline`, `start`, `enqueue`, `wait`, `status`, `outputs`, `step_log`, `run_log`, `stop`,
|
|
170
|
+
`retry`.
|
|
171
|
+
- `list_dags`, `list_runs`, `sub_runs`, `suspend`, `create_dag` / `update_spec` / `delete_dag`.
|
|
172
|
+
- `request()` for anything else.
|
|
173
|
+
|
|
174
|
+
API quirks the client absorbs:
|
|
175
|
+
- Params go as a JSON object string.
|
|
176
|
+
- The run-detail wrapper key differs by endpoint (`dagRunDetails` vs `dagRun`).
|
|
177
|
+
- Outputs come back as strings under camelCased keys (`output: RESULT` → `result`).
|
|
178
|
+
|
|
179
|
+
## Development
|
|
180
|
+
|
|
181
|
+
`mise run test` runs everything in a throwaway isb sandbox:
|
|
182
|
+
- the unit tests;
|
|
183
|
+
- golden YAML for `examples/`;
|
|
184
|
+
- `dagu validate` on every golden file, using the pinned release binary;
|
|
185
|
+
- the examples running on a real dagu server, and SSH steps against a throwaway sshd.
|
|
186
|
+
|
|
187
|
+
Without isb: `uv run --extra test pytest`. The integration tier runs when `DAGUFLOW_DAGU_BIN` points at
|
|
188
|
+
the pinned dagu binary (`tools/fetch-dagu.sh` downloads it).
|
|
189
|
+
|
|
190
|
+
`mise run gen` regenerates the action builders. A test fails when they drift from the schema.
|
|
191
|
+
|
|
192
|
+
## License
|
|
193
|
+
|
|
194
|
+
MIT. dagu's schema, which daguflow downloads at build time, is GPL-3.0-or-later and stays dagu's.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "daguflow"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Author dagu workflows in Python, Airflow-style: decorators for DAGs and tasks, context managers for sub-DAGs"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
keywords = [
|
|
9
|
+
"dagu",
|
|
10
|
+
"workflow",
|
|
11
|
+
"dag",
|
|
12
|
+
"airflow",
|
|
13
|
+
"orchestration",
|
|
14
|
+
"scheduler",
|
|
15
|
+
]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: System :: Distributed Computing",
|
|
25
|
+
"Typing :: Typed",
|
|
26
|
+
]
|
|
27
|
+
requires-python = ">=3.11"
|
|
28
|
+
dependencies = [
|
|
29
|
+
"httpx>=0.27",
|
|
30
|
+
"jsonschema>=4.21",
|
|
31
|
+
"pyyaml>=6",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[[project.authors]]
|
|
35
|
+
name = "Stephan Fitzpatrick"
|
|
36
|
+
email = "stephan@knowsuchagency.com"
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
test = ["pytest>=8"]
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/execution-associates/daguflow"
|
|
43
|
+
Repository = "https://github.com/execution-associates/daguflow"
|
|
44
|
+
Issues = "https://github.com/execution-associates/daguflow/issues"
|
|
45
|
+
|
|
46
|
+
[project.scripts]
|
|
47
|
+
daguflow = "daguflow.cli:main"
|
|
48
|
+
|
|
49
|
+
[build-system]
|
|
50
|
+
requires = ["uv_build>=0.9.5,<0.14.0"]
|
|
51
|
+
build-backend = "uv_build"
|
|
52
|
+
|
|
53
|
+
[tool.pytest.ini_options]
|
|
54
|
+
testpaths = ["tests"]
|
|
55
|
+
markers = ["dagu: needs the pinned dagu binary (DAGUFLOW_DAGU_BIN); skipped without it"]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "daguflow"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Author dagu workflows in Python, Airflow-style: decorators for DAGs and tasks, context managers for sub-DAGs"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
keywords = ["dagu", "workflow", "dag", "airflow", "orchestration", "scheduler"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 3 - Alpha",
|
|
11
|
+
"Intended Audience :: Developers",
|
|
12
|
+
"Operating System :: OS Independent",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Topic :: System :: Distributed Computing",
|
|
18
|
+
"Typing :: Typed",
|
|
19
|
+
]
|
|
20
|
+
authors = [
|
|
21
|
+
{ name = "Stephan Fitzpatrick", email = "stephan@knowsuchagency.com" }
|
|
22
|
+
]
|
|
23
|
+
requires-python = ">=3.11"
|
|
24
|
+
dependencies = [
|
|
25
|
+
"httpx>=0.27",
|
|
26
|
+
"jsonschema>=4.21",
|
|
27
|
+
"pyyaml>=6",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
test = ["pytest>=8"]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/execution-associates/daguflow"
|
|
35
|
+
Repository = "https://github.com/execution-associates/daguflow"
|
|
36
|
+
Issues = "https://github.com/execution-associates/daguflow/issues"
|
|
37
|
+
|
|
38
|
+
[project.scripts]
|
|
39
|
+
daguflow = "daguflow.cli:main"
|
|
40
|
+
|
|
41
|
+
[build-system]
|
|
42
|
+
requires = ["uv_build>=0.9.5,<0.14.0"]
|
|
43
|
+
build-backend = "uv_build"
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
testpaths = ["tests"]
|
|
47
|
+
markers = [
|
|
48
|
+
"dagu: needs the pinned dagu binary (DAGUFLOW_DAGU_BIN); skipped without it",
|
|
49
|
+
]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""daguflow: author dagu workflows in Python, Airflow-style.
|
|
2
|
+
|
|
3
|
+
from daguflow import dag, task, SubDAG, ForEach, Cron, env
|
|
4
|
+
|
|
5
|
+
@dag(schedule=Cron("0 9 * * *", tz="America/Los_Angeles"), labels=["reports"])
|
|
6
|
+
def morning_report(day: str = "today"):
|
|
7
|
+
'''Build and send the morning report.'''
|
|
8
|
+
data = fetch(day)
|
|
9
|
+
send(data)
|
|
10
|
+
|
|
11
|
+
`daguflow build <module> -o dags/` writes one YAML file per DAG.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from . import actions
|
|
17
|
+
from .dag import DAG, dag, on_abort, on_exit, on_failure, on_init, on_success, on_wait
|
|
18
|
+
from .groups import ExternalDAG, ForEach, SubDAG, call_dag
|
|
19
|
+
from .params import Doc, Length, Param, Pattern, Range
|
|
20
|
+
from .refs import ITEM, ParamRef, Ref, consts, ctx, env, raw, var
|
|
21
|
+
from .schedule import At, Cron, StartStop
|
|
22
|
+
from .schema import DAGU_VERSION
|
|
23
|
+
from .step import Action, Step, chain, shell, step
|
|
24
|
+
from .targets import ContainerTarget, SSHTarget, container
|
|
25
|
+
from .task import task
|
|
26
|
+
|
|
27
|
+
__version__ = "0.1.0"
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"DAG", "DAGU_VERSION", "ITEM", "Action", "At", "ContainerTarget", "Cron", "Doc", "ExternalDAG",
|
|
31
|
+
"ForEach", "Length", "Param", "ParamRef", "Pattern", "Range", "Ref", "SSHTarget", "StartStop",
|
|
32
|
+
"Step", "SubDAG", "actions", "call_dag", "chain", "consts", "container", "ctx", "dag", "env",
|
|
33
|
+
"on_abort", "on_exit", "on_failure", "on_init", "on_success", "on_wait", "raw", "shell", "step",
|
|
34
|
+
"task", "var",
|
|
35
|
+
]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""The stack of open DAG / sub-DAG / foreach / handler blocks.
|
|
2
|
+
|
|
3
|
+
A Step created while a block is open joins the innermost one, the way an
|
|
4
|
+
Airflow operator joins the `with DAG(...)` it is created in.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import threading
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from .step import Container
|
|
14
|
+
|
|
15
|
+
_local = threading.local()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _stack() -> list[Container]:
|
|
19
|
+
if not hasattr(_local, "stack"):
|
|
20
|
+
_local.stack = []
|
|
21
|
+
return _local.stack
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def push(container: Container) -> None:
|
|
25
|
+
_stack().append(container)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def pop(container: Container) -> None:
|
|
29
|
+
stack = _stack()
|
|
30
|
+
if not stack or stack[-1] is not container:
|
|
31
|
+
raise RuntimeError("daguflow blocks closed out of order")
|
|
32
|
+
stack.pop()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def current() -> Container | None:
|
|
36
|
+
stack = _stack()
|
|
37
|
+
return stack[-1] if stack else None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def current_dag():
|
|
41
|
+
from .dag import DAG
|
|
42
|
+
|
|
43
|
+
for container in reversed(_stack()):
|
|
44
|
+
if isinstance(container, DAG):
|
|
45
|
+
return container
|
|
46
|
+
return None
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Typing helpers shared by the generated builders."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TypeVar, Union
|
|
6
|
+
|
|
7
|
+
from .refs import Ref
|
|
8
|
+
|
|
9
|
+
T = TypeVar("T")
|
|
10
|
+
|
|
11
|
+
# A value or a Ref that dagu fills in at run time.
|
|
12
|
+
Val = Union[T, Ref]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Typed builders for dagu actions.
|
|
2
|
+
|
|
3
|
+
from daguflow import actions
|
|
4
|
+
actions.http.request(method="GET", url="https://example.com").step("fetch")
|
|
5
|
+
|
|
6
|
+
Every builtin action at the vendored dagu version has a builder (generated
|
|
7
|
+
from the schema; see tools/gen_actions.py). Each returns an Action, which
|
|
8
|
+
becomes a step through `.step(...)`, by being returned from a @task, or
|
|
9
|
+
through `Step(action=...)`. `custom()` covers custom actions declared in
|
|
10
|
+
`actions:` and remote ones (`owner/repo@v1`, `name@v1`, `source:...@v1`);
|
|
11
|
+
`redis()` covers the `redis.<op>` family.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from ..step import Action
|
|
19
|
+
from ..targets import SSHTarget
|
|
20
|
+
from ._generated import * # noqa: F403
|
|
21
|
+
from ._generated import __all__ as _generated_all
|
|
22
|
+
from ._generated import ssh
|
|
23
|
+
|
|
24
|
+
ssh.Target = SSHTarget # type: ignore[attr-defined]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def custom(name: str, **with_: Any) -> Action:
|
|
28
|
+
"""A custom or remote action: `custom("dagucloud/duckdb@v1", query=...)`."""
|
|
29
|
+
return Action(name, with_)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def redis(op: str, **with_: Any) -> Action:
|
|
33
|
+
"""`redis.<op>`: `redis("get", key="k")`."""
|
|
34
|
+
return Action(f"redis.{op}", with_)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
__all__ = [*_generated_all, "custom", "redis", "Action", "SSHTarget"]
|