pydagu 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pydagu might be problematic. Click here for more details.
- pydagu-0.2.2/.github/workflows/publish.yml +35 -0
- pydagu-0.2.2/.github/workflows/test.yml +71 -0
- pydagu-0.2.2/.gitignore +13 -0
- pydagu-0.2.2/.importlinter +10 -0
- pydagu-0.2.2/.pre-commit-config.yaml +37 -0
- pydagu-0.2.2/.vscode/settings.json +8 -0
- pydagu-0.2.2/CHANGELOG.md +34 -0
- pydagu-0.2.2/LICENSE +21 -0
- pydagu-0.2.2/MANIFEST.in +8 -0
- pydagu-0.2.2/PKG-INFO +195 -0
- pydagu-0.2.2/README.md +161 -0
- pydagu-0.2.2/RELEASE.md +112 -0
- pydagu-0.2.2/curl-format.txt +8 -0
- pydagu-0.2.2/docs/reference.md +372 -0
- pydagu-0.2.2/pydagu/__init__.py +7 -0
- pydagu-0.2.2/pydagu/builder.py +508 -0
- pydagu-0.2.2/pydagu/http.py +106 -0
- pydagu-0.2.2/pydagu/models/__init__.py +59 -0
- pydagu-0.2.2/pydagu/models/base.py +12 -0
- pydagu-0.2.2/pydagu/models/dag.py +221 -0
- pydagu-0.2.2/pydagu/models/executor.py +183 -0
- pydagu-0.2.2/pydagu/models/handlers.py +30 -0
- pydagu-0.2.2/pydagu/models/infrastructure.py +71 -0
- pydagu-0.2.2/pydagu/models/notifications.py +26 -0
- pydagu-0.2.2/pydagu/models/request.py +14 -0
- pydagu-0.2.2/pydagu/models/response.py +82 -0
- pydagu-0.2.2/pydagu/models/step.py +144 -0
- pydagu-0.2.2/pydagu/models/types.py +16 -0
- pydagu-0.2.2/pydagu/py.typed +0 -0
- pydagu-0.2.2/pyproject.toml +66 -0
- pydagu-0.2.2/tests/__init__.py +1 -0
- pydagu-0.2.2/tests/conftest.py +43 -0
- pydagu-0.2.2/tests/dagu-example.yaml +90 -0
- pydagu-0.2.2/tests/test_builder.py +473 -0
- pydagu-0.2.2/tests/test_http.py +718 -0
- pydagu-0.2.2/tests/test_models.py +354 -0
- pydagu-0.2.2/tests/test_roundtrip.py +10 -0
- pydagu-0.2.2/tests/test_schemathesis.py +377 -0
- pydagu-0.2.2/tests/test_validate.py +393 -0
- pydagu-0.2.2/uv.lock +1499 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build-and-publish:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
permissions:
|
|
11
|
+
id-token: write # Required for trusted publishing
|
|
12
|
+
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- name: Set up Python
|
|
17
|
+
uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.13"
|
|
20
|
+
|
|
21
|
+
- name: Install build tools
|
|
22
|
+
run: |
|
|
23
|
+
python -m pip install --upgrade pip
|
|
24
|
+
pip install build twine
|
|
25
|
+
|
|
26
|
+
- name: Build package
|
|
27
|
+
run: python -m build
|
|
28
|
+
|
|
29
|
+
- name: Check package
|
|
30
|
+
run: twine check dist/*
|
|
31
|
+
|
|
32
|
+
- name: Publish to PyPI
|
|
33
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
34
|
+
with:
|
|
35
|
+
verbose: true
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ main, master ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ main, master ]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.13"]
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
20
|
+
uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
|
|
24
|
+
- name: Install Dagu
|
|
25
|
+
run: |
|
|
26
|
+
# Install Dagu using official installer script
|
|
27
|
+
curl -L https://raw.githubusercontent.com/dagu-org/dagu/main/scripts/installer.sh | bash
|
|
28
|
+
|
|
29
|
+
# Add to PATH for this session
|
|
30
|
+
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
|
31
|
+
|
|
32
|
+
# Verify installation
|
|
33
|
+
$HOME/.local/bin/dagu version
|
|
34
|
+
|
|
35
|
+
- name: Start Dagu server
|
|
36
|
+
run: |
|
|
37
|
+
# Start Dagu server in the background
|
|
38
|
+
$HOME/.local/bin/dagu server --host 0.0.0.0 --port 8080 &
|
|
39
|
+
|
|
40
|
+
# Wait for server to be ready
|
|
41
|
+
echo "Waiting for Dagu server to start..."
|
|
42
|
+
for i in {1..30}; do
|
|
43
|
+
if curl -s http://localhost:8080/api/v2/dags > /dev/null 2>&1; then
|
|
44
|
+
echo "Dagu server is ready!"
|
|
45
|
+
break
|
|
46
|
+
fi
|
|
47
|
+
echo "Attempt $i: Server not ready yet..."
|
|
48
|
+
sleep 2
|
|
49
|
+
done
|
|
50
|
+
|
|
51
|
+
# Verify server is running
|
|
52
|
+
curl -f http://localhost:8080/api/v2/dags || (echo "Failed to connect to Dagu server" && exit 1)
|
|
53
|
+
|
|
54
|
+
- name: Install dependencies
|
|
55
|
+
run: |
|
|
56
|
+
python -m pip install --upgrade pip
|
|
57
|
+
pip install -e ".[dev]"
|
|
58
|
+
|
|
59
|
+
# Verify pytest is available
|
|
60
|
+
pytest --version
|
|
61
|
+
|
|
62
|
+
- name: Run tests
|
|
63
|
+
run: |
|
|
64
|
+
pytest tests/ -v --cov=pydagu --cov-report=xml --cov-report=term
|
|
65
|
+
|
|
66
|
+
- name: Upload coverage to Codecov
|
|
67
|
+
uses: codecov/codecov-action@v4
|
|
68
|
+
if: matrix.python-version == '3.13'
|
|
69
|
+
with:
|
|
70
|
+
file: ./coverage.xml
|
|
71
|
+
fail_ci_if_error: false
|
pydagu-0.2.2/.gitignore
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# See https://pre-commit.com for more information
|
|
2
|
+
# See https://pre-commit.com/hooks.html for more hooks
|
|
3
|
+
repos:
|
|
4
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
5
|
+
rev: v6.0.0
|
|
6
|
+
hooks:
|
|
7
|
+
- id: trailing-whitespace
|
|
8
|
+
- id: end-of-file-fixer
|
|
9
|
+
- id: check-yaml
|
|
10
|
+
- id: check-added-large-files
|
|
11
|
+
|
|
12
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
13
|
+
rev: v0.14.3
|
|
14
|
+
hooks:
|
|
15
|
+
- id: ruff-check
|
|
16
|
+
args: [--fix]
|
|
17
|
+
- id: ruff-format
|
|
18
|
+
|
|
19
|
+
- repo: https://github.com/PyCQA/bandit
|
|
20
|
+
rev: 1.8.6
|
|
21
|
+
hooks:
|
|
22
|
+
- id: bandit
|
|
23
|
+
args: [-ll, pydagu]
|
|
24
|
+
|
|
25
|
+
- repo: local
|
|
26
|
+
hooks:
|
|
27
|
+
- id: lint_imports
|
|
28
|
+
name: "Lint imports"
|
|
29
|
+
entry: ".venv/bin/lint-imports" # Adapt with custom arguments, if need be.
|
|
30
|
+
language: system
|
|
31
|
+
pass_filenames: false
|
|
32
|
+
- id: mypy-strict
|
|
33
|
+
name: "MyPy strict type checking"
|
|
34
|
+
entry: .venv/bin/mypy --strict pydagu/
|
|
35
|
+
language: system
|
|
36
|
+
pass_filenames: false
|
|
37
|
+
types: [python]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] - 2025-01-08
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- Initial release of pydagu
|
|
14
|
+
- `DagBuilder` and `StepBuilder` for fluent DAG construction
|
|
15
|
+
- `DaguHttpClient` for interacting with Dagu HTTP API
|
|
16
|
+
- Pydantic models for type-safe DAG definitions
|
|
17
|
+
- HTTP executor support with retry policies
|
|
18
|
+
- Webhook integration patterns and examples
|
|
19
|
+
- Comprehensive test suite (6 HTTP tests, 95%+ coverage)
|
|
20
|
+
- Support for Python 3.11, 3.12, and 3.13
|
|
21
|
+
|
|
22
|
+
### Features
|
|
23
|
+
- Type-safe DAG creation with validation
|
|
24
|
+
- HTTP executor with headers, body, timeout, and retry configuration
|
|
25
|
+
- Automatic JSON body serialization
|
|
26
|
+
- Step dependencies and output capture
|
|
27
|
+
- Parameter and environment variable support
|
|
28
|
+
- Flask/Django/FastAPI webhook integration examples
|
|
29
|
+
|
|
30
|
+
### Documentation
|
|
31
|
+
- Complete README with quick start guide
|
|
32
|
+
- Webhook integration examples with WebhookManager
|
|
33
|
+
- Production-ready callback patterns
|
|
34
|
+
- GitHub Actions CI/CD workflows
|
pydagu-0.2.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Patrick [Your Last Name]
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pydagu-0.2.2/MANIFEST.in
ADDED
pydagu-0.2.2/PKG-INFO
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pydagu
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Python client library for Dagu - A cron alternative with a Web UI
|
|
5
|
+
Project-URL: Homepage, https://github.com/yourusername/pydagu
|
|
6
|
+
Project-URL: Documentation, https://github.com/yourusername/pydagu#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/yourusername/pydagu
|
|
8
|
+
Project-URL: Issues, https://github.com/yourusername/pydagu/issues
|
|
9
|
+
Author-email: Patrick Dobbs <patrick.dobbs@gmail.com>
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: cron,dag,dagu,scheduler,workflow
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Topic :: System :: Monitoring
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Requires-Dist: httpx>=0.28.1
|
|
23
|
+
Requires-Dist: pydantic>=2.12.3
|
|
24
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: hypothesis-jsonschema>=0.23.1; extra == 'dev'
|
|
27
|
+
Requires-Dist: import-linter>=2.5.2; extra == 'dev'
|
|
28
|
+
Requires-Dist: pre-commit>=4.3.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: pytest-cov>=7.0.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest>=8.4.2; extra == 'dev'
|
|
31
|
+
Requires-Dist: ruff>=0.14.3; extra == 'dev'
|
|
32
|
+
Requires-Dist: schemathesis>=4.3.18; extra == 'dev'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# pydagu
|
|
36
|
+
|
|
37
|
+
[](https://badge.fury.io/py/pydagu)
|
|
38
|
+
[](https://github.com/yourusername/pydagu/actions/workflows/test.yml)
|
|
39
|
+
[](https://pypi.org/project/pydagu/)
|
|
40
|
+
[](https://opensource.org/licenses/MIT)
|
|
41
|
+
|
|
42
|
+
A Python client library for [Dagu](https://github.com/dagu-org/dagu) - providing type-safe DAG creation and HTTP API interaction with Pydantic validation.
|
|
43
|
+
|
|
44
|
+
## Features
|
|
45
|
+
|
|
46
|
+
- 🎯 **Type-safe**: Built with Pydantic models for full type safety and validation
|
|
47
|
+
- 🏗️ **Builder Pattern**: Fluent API for constructing DAGs and steps
|
|
48
|
+
- 🔌 **HTTP Client**: Complete client for Dagu's HTTP API
|
|
49
|
+
- 🔄 **Webhook Support**: Built-in patterns for webhook integration
|
|
50
|
+
- ✅ **Well-tested**: Comprehensive test suite with 95%+ coverage
|
|
51
|
+
- 📝 **Examples**: Production-ready integration examples
|
|
52
|
+
|
|
53
|
+
## Installation
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install pydagu
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Prerequisites
|
|
60
|
+
|
|
61
|
+
You need a running [Dagu server](https://github.com/dagu-org/dagu) to run the tests. Install Dagu:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
# macOS
|
|
65
|
+
brew install dagu-org/brew/dagu
|
|
66
|
+
|
|
67
|
+
# Linux
|
|
68
|
+
curl -L https://github.com/dagu-org/dagu/releases/latest/download/dagu_linux_amd64.tar.gz | tar xz
|
|
69
|
+
sudo mv dagu /usr/local/bin/
|
|
70
|
+
|
|
71
|
+
# Start the server
|
|
72
|
+
dagu server
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Quick Start
|
|
76
|
+
|
|
77
|
+
### Create and Run a Simple DAG
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from pydagu.builder import DagBuilder, StepBuilder
|
|
81
|
+
from pydagu.http import DaguHttpClient
|
|
82
|
+
from pydagu.models.request import StartDagRun
|
|
83
|
+
|
|
84
|
+
# Initialize client
|
|
85
|
+
client = DaguHttpClient(
|
|
86
|
+
dag_name="my-first-dag",
|
|
87
|
+
url_root="http://localhost:8080/api/v2/"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# Build a DAG
|
|
91
|
+
dag = (
|
|
92
|
+
DagBuilder("my-first-dag")
|
|
93
|
+
.description("My first DAG with pydagu")
|
|
94
|
+
.add_step_models(
|
|
95
|
+
StepBuilder("hello-world")
|
|
96
|
+
.command("echo 'Hello from pydagu!'")
|
|
97
|
+
.build()
|
|
98
|
+
)
|
|
99
|
+
.build()
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# Post the DAG to Dagu
|
|
103
|
+
client.post_dag(dag)
|
|
104
|
+
|
|
105
|
+
# Start a run
|
|
106
|
+
dag_run_id = client.start_dag_run(StartDagRun(dagName=dag.name))
|
|
107
|
+
|
|
108
|
+
# Check status
|
|
109
|
+
status = client.get_dag_run_status(dag_run_id.dagRunId)
|
|
110
|
+
print(f"Status: {status.statusLabel}")
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### HTTP Executor with Retry
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from pydagu.builder import StepBuilder
|
|
117
|
+
|
|
118
|
+
step = (
|
|
119
|
+
StepBuilder("api-call")
|
|
120
|
+
.command("POST https://api.example.com/webhook")
|
|
121
|
+
.http_executor(
|
|
122
|
+
headers={
|
|
123
|
+
"Content-Type": "application/json",
|
|
124
|
+
"Authorization": "Bearer ${API_TOKEN}"
|
|
125
|
+
},
|
|
126
|
+
body={"event": "user.created", "user_id": "123"},
|
|
127
|
+
timeout=30
|
|
128
|
+
)
|
|
129
|
+
.retry(limit=3, interval=5)
|
|
130
|
+
.build()
|
|
131
|
+
)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Chained Steps with Dependencies
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from pydagu.builder import DagBuilder, StepBuilder
|
|
138
|
+
|
|
139
|
+
dag = (
|
|
140
|
+
DagBuilder("data-pipeline")
|
|
141
|
+
.add_step_models(
|
|
142
|
+
StepBuilder("extract")
|
|
143
|
+
.command("python extract_data.py")
|
|
144
|
+
.output("EXTRACTED_FILE")
|
|
145
|
+
.build(),
|
|
146
|
+
|
|
147
|
+
StepBuilder("transform")
|
|
148
|
+
.command("python transform_data.py ${EXTRACTED_FILE}")
|
|
149
|
+
.depends_on("extract")
|
|
150
|
+
.output("TRANSFORMED_FILE")
|
|
151
|
+
.build(),
|
|
152
|
+
|
|
153
|
+
StepBuilder("load")
|
|
154
|
+
.command("python load_data.py ${TRANSFORMED_FILE}")
|
|
155
|
+
.depends_on("transform")
|
|
156
|
+
.build()
|
|
157
|
+
)
|
|
158
|
+
.build()
|
|
159
|
+
)
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
## Development
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
# Clone the repository
|
|
166
|
+
git clone https://github.com/yourusername/pydagu.git
|
|
167
|
+
cd pydagu
|
|
168
|
+
|
|
169
|
+
# Install with dev dependencies
|
|
170
|
+
pip install -e ".[dev]"
|
|
171
|
+
|
|
172
|
+
# Run tests (requires Dagu server running)
|
|
173
|
+
pytest tests/ -v
|
|
174
|
+
|
|
175
|
+
# Run with coverage
|
|
176
|
+
pytest tests/ --cov=pydagu --cov-report=html
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## Documentation
|
|
180
|
+
|
|
181
|
+
- [PyDagu Reference](https://github.com/patrickcd/pydagu/blob/main/docs/reference.md)
|
|
182
|
+
- [Dagu Documentation](https://docs.dagu.cloud/)
|
|
183
|
+
|
|
184
|
+
## Contributing
|
|
185
|
+
|
|
186
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
187
|
+
|
|
188
|
+
## License
|
|
189
|
+
|
|
190
|
+
MIT License - see [LICENSE](LICENSE) file for details.
|
|
191
|
+
|
|
192
|
+
## Related Projects
|
|
193
|
+
|
|
194
|
+
- [Dagu](https://github.com/dagu-org/dagu) - The underlying DAG execution engine
|
|
195
|
+
- [Airflow](https://airflow.apache.org/) - Alternative workflow orchestration platform
|
pydagu-0.2.2/README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# pydagu
|
|
2
|
+
|
|
3
|
+
[](https://badge.fury.io/py/pydagu)
|
|
4
|
+
[](https://github.com/yourusername/pydagu/actions/workflows/test.yml)
|
|
5
|
+
[](https://pypi.org/project/pydagu/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
8
|
+
A Python client library for [Dagu](https://github.com/dagu-org/dagu) - providing type-safe DAG creation and HTTP API interaction with Pydantic validation.
|
|
9
|
+
|
|
10
|
+
## Features
|
|
11
|
+
|
|
12
|
+
- 🎯 **Type-safe**: Built with Pydantic models for full type safety and validation
|
|
13
|
+
- 🏗️ **Builder Pattern**: Fluent API for constructing DAGs and steps
|
|
14
|
+
- 🔌 **HTTP Client**: Complete client for Dagu's HTTP API
|
|
15
|
+
- 🔄 **Webhook Support**: Built-in patterns for webhook integration
|
|
16
|
+
- ✅ **Well-tested**: Comprehensive test suite with 95%+ coverage
|
|
17
|
+
- 📝 **Examples**: Production-ready integration examples
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install pydagu
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Prerequisites
|
|
26
|
+
|
|
27
|
+
You need a running [Dagu server](https://github.com/dagu-org/dagu) to run the tests. Install Dagu:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
# macOS
|
|
31
|
+
brew install dagu-org/brew/dagu
|
|
32
|
+
|
|
33
|
+
# Linux
|
|
34
|
+
curl -L https://github.com/dagu-org/dagu/releases/latest/download/dagu_linux_amd64.tar.gz | tar xz
|
|
35
|
+
sudo mv dagu /usr/local/bin/
|
|
36
|
+
|
|
37
|
+
# Start the server
|
|
38
|
+
dagu server
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Quick Start
|
|
42
|
+
|
|
43
|
+
### Create and Run a Simple DAG
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from pydagu.builder import DagBuilder, StepBuilder
|
|
47
|
+
from pydagu.http import DaguHttpClient
|
|
48
|
+
from pydagu.models.request import StartDagRun
|
|
49
|
+
|
|
50
|
+
# Initialize client
|
|
51
|
+
client = DaguHttpClient(
|
|
52
|
+
dag_name="my-first-dag",
|
|
53
|
+
url_root="http://localhost:8080/api/v2/"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# Build a DAG
|
|
57
|
+
dag = (
|
|
58
|
+
DagBuilder("my-first-dag")
|
|
59
|
+
.description("My first DAG with pydagu")
|
|
60
|
+
.add_step_models(
|
|
61
|
+
StepBuilder("hello-world")
|
|
62
|
+
.command("echo 'Hello from pydagu!'")
|
|
63
|
+
.build()
|
|
64
|
+
)
|
|
65
|
+
.build()
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
# Post the DAG to Dagu
|
|
69
|
+
client.post_dag(dag)
|
|
70
|
+
|
|
71
|
+
# Start a run
|
|
72
|
+
dag_run_id = client.start_dag_run(StartDagRun(dagName=dag.name))
|
|
73
|
+
|
|
74
|
+
# Check status
|
|
75
|
+
status = client.get_dag_run_status(dag_run_id.dagRunId)
|
|
76
|
+
print(f"Status: {status.statusLabel}")
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### HTTP Executor with Retry
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from pydagu.builder import StepBuilder
|
|
83
|
+
|
|
84
|
+
step = (
|
|
85
|
+
StepBuilder("api-call")
|
|
86
|
+
.command("POST https://api.example.com/webhook")
|
|
87
|
+
.http_executor(
|
|
88
|
+
headers={
|
|
89
|
+
"Content-Type": "application/json",
|
|
90
|
+
"Authorization": "Bearer ${API_TOKEN}"
|
|
91
|
+
},
|
|
92
|
+
body={"event": "user.created", "user_id": "123"},
|
|
93
|
+
timeout=30
|
|
94
|
+
)
|
|
95
|
+
.retry(limit=3, interval=5)
|
|
96
|
+
.build()
|
|
97
|
+
)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### Chained Steps with Dependencies
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from pydagu.builder import DagBuilder, StepBuilder
|
|
104
|
+
|
|
105
|
+
dag = (
|
|
106
|
+
DagBuilder("data-pipeline")
|
|
107
|
+
.add_step_models(
|
|
108
|
+
StepBuilder("extract")
|
|
109
|
+
.command("python extract_data.py")
|
|
110
|
+
.output("EXTRACTED_FILE")
|
|
111
|
+
.build(),
|
|
112
|
+
|
|
113
|
+
StepBuilder("transform")
|
|
114
|
+
.command("python transform_data.py ${EXTRACTED_FILE}")
|
|
115
|
+
.depends_on("extract")
|
|
116
|
+
.output("TRANSFORMED_FILE")
|
|
117
|
+
.build(),
|
|
118
|
+
|
|
119
|
+
StepBuilder("load")
|
|
120
|
+
.command("python load_data.py ${TRANSFORMED_FILE}")
|
|
121
|
+
.depends_on("transform")
|
|
122
|
+
.build()
|
|
123
|
+
)
|
|
124
|
+
.build()
|
|
125
|
+
)
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## Development
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
# Clone the repository
|
|
132
|
+
git clone https://github.com/yourusername/pydagu.git
|
|
133
|
+
cd pydagu
|
|
134
|
+
|
|
135
|
+
# Install with dev dependencies
|
|
136
|
+
pip install -e ".[dev]"
|
|
137
|
+
|
|
138
|
+
# Run tests (requires Dagu server running)
|
|
139
|
+
pytest tests/ -v
|
|
140
|
+
|
|
141
|
+
# Run with coverage
|
|
142
|
+
pytest tests/ --cov=pydagu --cov-report=html
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Documentation
|
|
146
|
+
|
|
147
|
+
- [PyDagu Reference](https://github.com/patrickcd/pydagu/blob/main/docs/reference.md)
|
|
148
|
+
- [Dagu Documentation](https://docs.dagu.cloud/)
|
|
149
|
+
|
|
150
|
+
## Contributing
|
|
151
|
+
|
|
152
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
153
|
+
|
|
154
|
+
## License
|
|
155
|
+
|
|
156
|
+
MIT License - see [LICENSE](LICENSE) file for details.
|
|
157
|
+
|
|
158
|
+
## Related Projects
|
|
159
|
+
|
|
160
|
+
- [Dagu](https://github.com/dagu-org/dagu) - The underlying DAG execution engine
|
|
161
|
+
- [Airflow](https://airflow.apache.org/) - Alternative workflow orchestration platform
|