dotflow 0.14.1.dev1__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/PKG-INFO +260 -212
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/README.md +239 -211
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/__init__.py +1 -1
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/flow.py +19 -0
- dotflow-0.15.0/dotflow/abc/log.py +122 -0
- dotflow-0.15.0/dotflow/abc/metrics.py +32 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/notify.py +2 -2
- dotflow-0.15.0/dotflow/abc/server.py +30 -0
- dotflow-0.15.0/dotflow/abc/tracer.py +24 -0
- dotflow-0.15.0/dotflow/cli/commands/__init__.py +18 -0
- dotflow-0.15.0/dotflow/cli/commands/cloud.py +202 -0
- dotflow-0.15.0/dotflow/cli/commands/deploy.py +171 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/commands/init.py +1 -3
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/commands/log.py +2 -1
- dotflow-0.15.0/dotflow/cli/commands/storage.py +55 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/setup.py +68 -0
- dotflow-0.15.0/dotflow/cloud/__init__.py +1 -0
- dotflow-0.15.0/dotflow/cloud/alibaba/__init__.py +11 -0
- dotflow-0.15.0/dotflow/cloud/alibaba/constants.py +22 -0
- dotflow-0.15.0/dotflow/cloud/alibaba/deployers/__init__.py +13 -0
- dotflow-0.15.0/dotflow/cloud/alibaba/deployers/fc.py +134 -0
- dotflow-0.15.0/dotflow/cloud/alibaba/deployers/fc_scheduled.py +78 -0
- dotflow-0.15.0/dotflow/cloud/alibaba/services/__init__.py +1 -0
- dotflow-0.15.0/dotflow/cloud/alibaba/services/cr.py +99 -0
- dotflow-0.15.0/dotflow/cloud/aws/__init__.py +19 -0
- dotflow-0.15.0/dotflow/cloud/aws/constants.py +23 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/__init__.py +19 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/base_lambda.py +93 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/ecs.py +116 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/ecs_scheduled.py +154 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/lambda_.py +55 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/lambda_api.py +70 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/lambda_s3.py +62 -0
- dotflow-0.15.0/dotflow/cloud/aws/deployers/lambda_sqs.py +58 -0
- dotflow-0.15.0/dotflow/cloud/aws/schedule.py +79 -0
- dotflow-0.15.0/dotflow/cloud/aws/services/__init__.py +0 -0
- dotflow-0.15.0/dotflow/cloud/aws/services/cloudwatch.py +25 -0
- dotflow-0.15.0/dotflow/cloud/aws/services/ecr.py +75 -0
- dotflow-0.15.0/dotflow/cloud/aws/services/iam.py +82 -0
- dotflow-0.15.0/dotflow/cloud/aws/services/s3.py +47 -0
- dotflow-0.15.0/dotflow/cloud/core.py +95 -0
- dotflow-0.15.0/dotflow/cloud/gcp/__init__.py +5 -0
- dotflow-0.15.0/dotflow/cloud/gcp/constants.py +20 -0
- dotflow-0.15.0/dotflow/cloud/gcp/deployers/__init__.py +5 -0
- dotflow-0.15.0/dotflow/cloud/gcp/deployers/cloudrun.py +113 -0
- dotflow-0.15.0/dotflow/cloud/gcp/services/__init__.py +1 -0
- dotflow-0.15.0/dotflow/cloud/gcp/services/apis.py +45 -0
- dotflow-0.15.0/dotflow/cloud/gcp/services/artifact_registry.py +138 -0
- dotflow-0.15.0/dotflow/cloud/gcp/services/gcs.py +45 -0
- dotflow-0.15.0/dotflow/cloud/github/__init__.py +5 -0
- dotflow-0.15.0/dotflow/cloud/github/constants.py +8 -0
- dotflow-0.15.0/dotflow/cloud/github/deployers/__init__.py +5 -0
- dotflow-0.15.0/dotflow/cloud/github/deployers/actions.py +111 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/action.py +40 -79
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/config.py +19 -5
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/context.py +17 -4
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/dotflow.py +5 -2
- dotflow-0.15.0/dotflow/core/engine.py +280 -0
- dotflow-0.15.0/dotflow/core/module.py +30 -0
- dotflow-0.15.0/dotflow/core/serializers/task.py +135 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/serializers/transport.py +2 -2
- dotflow-0.15.0/dotflow/core/serializers/workflow.py +14 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/task.py +23 -6
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/types/workflow.py +1 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/workflow.py +102 -68
- dotflow-0.15.0/dotflow/logging.py +14 -0
- dotflow-0.15.0/dotflow/providers/__init__.py +79 -0
- dotflow-0.15.0/dotflow/providers/log_default.py +62 -0
- dotflow-0.15.0/dotflow/providers/log_opentelemetry.py +91 -0
- dotflow-0.15.0/dotflow/providers/log_sentry.py +96 -0
- dotflow-0.15.0/dotflow/providers/metrics_default.py +25 -0
- dotflow-0.15.0/dotflow/providers/metrics_opentelemetry.py +95 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/providers/notify_default.py +1 -1
- dotflow-0.15.0/dotflow/providers/notify_discord.py +112 -0
- dotflow-0.15.0/dotflow/providers/notify_telegram.py +109 -0
- dotflow-0.15.0/dotflow/providers/server_default.py +23 -0
- dotflow-0.15.0/dotflow/providers/storage_default.py +22 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/providers/storage_file.py +11 -8
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/providers/storage_gcs.py +4 -32
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/providers/storage_s3.py +4 -34
- dotflow-0.15.0/dotflow/providers/tracer_default.py +19 -0
- dotflow-0.15.0/dotflow/providers/tracer_opentelemetry.py +126 -0
- dotflow-0.15.0/dotflow/providers/tracer_sentry.py +126 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/settings.py +11 -2
- dotflow-0.15.0/dotflow/utils/error_handler.py +13 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/utils/tools.py +0 -10
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/pyproject.toml +11 -2
- dotflow-0.14.1.dev1/dotflow/abc/api.py +0 -24
- dotflow-0.14.1.dev1/dotflow/abc/log.py +0 -16
- dotflow-0.14.1.dev1/dotflow/cli/commands/__init__.py +0 -8
- dotflow-0.14.1.dev1/dotflow/core/execution.py +0 -174
- dotflow-0.14.1.dev1/dotflow/core/module.py +0 -36
- dotflow-0.14.1.dev1/dotflow/core/serializers/task.py +0 -103
- dotflow-0.14.1.dev1/dotflow/core/serializers/workflow.py +0 -23
- dotflow-0.14.1.dev1/dotflow/logging.py +0 -27
- dotflow-0.14.1.dev1/dotflow/providers/__init__.py +0 -39
- dotflow-0.14.1.dev1/dotflow/providers/api_default.py +0 -158
- dotflow-0.14.1.dev1/dotflow/providers/log_default.py +0 -25
- dotflow-0.14.1.dev1/dotflow/providers/notify_telegram.py +0 -59
- dotflow-0.14.1.dev1/dotflow/providers/storage_default.py +0 -20
- dotflow-0.14.1.dev1/dotflow/storage.py +0 -22
- dotflow-0.14.1.dev1/dotflow/utils/error_handler.py +0 -22
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/LICENSE +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/file.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/http.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/scheduler.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/storage.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/abc/tcp.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/command.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/commands/schedule.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/commands/start.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/validators/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/cli/validators/start.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/decorators/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/decorators/time.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/exception.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/serializers/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/types/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/types/execution.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/types/overlap.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/types/status.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/core/types/storage.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/main.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/providers/scheduler_cron.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/providers/scheduler_default.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/types.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/utils/__init__.py +0 -0
- {dotflow-0.14.1.dev1 → dotflow-0.15.0}/dotflow/utils/basic_functions.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dotflow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: 🎲 Dotflow turns an idea into flow! Lightweight Python library for execution pipelines with retry, parallel, cron and async support.
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -42,15 +42,35 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
42
42
|
Classifier: Programming Language :: Python :: 3.12
|
|
43
43
|
Classifier: Programming Language :: Python :: 3.13
|
|
44
44
|
Provides-Extra: aws
|
|
45
|
+
Provides-Extra: deploy-alibaba
|
|
46
|
+
Provides-Extra: deploy-aws
|
|
47
|
+
Provides-Extra: deploy-gcp
|
|
48
|
+
Provides-Extra: deploy-github
|
|
45
49
|
Provides-Extra: gcp
|
|
50
|
+
Provides-Extra: otel
|
|
46
51
|
Provides-Extra: scheduler
|
|
52
|
+
Provides-Extra: sentry
|
|
53
|
+
Requires-Dist: GitPython ; extra == "deploy-github"
|
|
54
|
+
Requires-Dist: PyGithub ; extra == "deploy-github"
|
|
55
|
+
Requires-Dist: alibabacloud-fc-open20210406 ; extra == "deploy-alibaba"
|
|
56
|
+
Requires-Dist: alibabacloud-tea-openapi ; extra == "deploy-alibaba"
|
|
47
57
|
Requires-Dist: boto3 ; extra == "aws"
|
|
58
|
+
Requires-Dist: boto3 ; extra == "deploy-aws"
|
|
48
59
|
Requires-Dist: cookiecutter (>=2.0)
|
|
49
60
|
Requires-Dist: croniter ; extra == "scheduler"
|
|
61
|
+
Requires-Dist: google-auth ; extra == "deploy-gcp"
|
|
62
|
+
Requires-Dist: google-cloud-artifact-registry ; extra == "deploy-gcp"
|
|
63
|
+
Requires-Dist: google-cloud-build ; extra == "deploy-gcp"
|
|
64
|
+
Requires-Dist: google-cloud-run ; extra == "deploy-gcp"
|
|
65
|
+
Requires-Dist: google-cloud-service-usage ; extra == "deploy-gcp"
|
|
66
|
+
Requires-Dist: google-cloud-storage ; extra == "deploy-gcp"
|
|
50
67
|
Requires-Dist: google-cloud-storage ; extra == "gcp"
|
|
68
|
+
Requires-Dist: opentelemetry-api ; extra == "otel"
|
|
69
|
+
Requires-Dist: opentelemetry-sdk ; extra == "otel"
|
|
51
70
|
Requires-Dist: pydantic
|
|
52
71
|
Requires-Dist: requests
|
|
53
72
|
Requires-Dist: rich
|
|
73
|
+
Requires-Dist: sentry-sdk ; extra == "sentry"
|
|
54
74
|
Requires-Dist: typing-extensions
|
|
55
75
|
Project-URL: Changelog, https://dotflow-io.github.io/dotflow/nav/development/release-notes/
|
|
56
76
|
Project-URL: Documentation, https://dotflow-io.github.io/dotflow/
|
|
@@ -60,86 +80,40 @@ Project-URL: Repository, https://github.com/dotflow-io/dotflow
|
|
|
60
80
|
Description-Content-Type: text/markdown
|
|
61
81
|
|
|
62
82
|
<div align="center">
|
|
63
|
-
<a aria-label="Serverless.com" href="https://dotflow.io">Website</a>
|
|
64
|
-
•
|
|
65
|
-
<a aria-label="Dotflow Documentation" href="https://dotflow-io.github.io/dotflow/">Documentation</a>
|
|
66
|
-
•
|
|
67
|
-
<a aria-label="Pypi" href="https://pypi.org/project/dotflow/">Pypi</a>
|
|
68
|
-
</div>
|
|
69
83
|
|
|
70
|
-
|
|
84
|
+

|
|
71
85
|
|
|
72
|
-
|
|
86
|
+
**import dotflow. @action. deploy. Done.**
|
|
73
87
|
|
|
74
|
-
](https://pypi.org/project/dotflow/)
|
|
89
|
+
[](https://pypi.org/project/dotflow/)
|
|
90
|
+
[](https://pypi.org/project/dotflow/)
|
|
91
|
+
[](https://github.com/dotflow-io/dotflow)
|
|
75
92
|
|
|
76
|
-
|
|
77
|
-

|
|
78
|
-

|
|
79
|
-

|
|
80
|
-

|
|
93
|
+
[Website](https://www.cloud.dotflow.io) · [Documentation](https://dotflow-io.github.io/dotflow/) · [PyPI](https://pypi.org/project/dotflow/)
|
|
81
94
|
|
|
82
95
|
</div>
|
|
83
96
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
Dotflow is a lightweight Python library for building execution pipelines. Define tasks with decorators, chain them together, and run workflows in sequential, parallel, or background mode — with built-in retry, timeout, storage, notifications, and more.
|
|
87
|
-
|
|
88
|
-
> **[Read the full documentation](https://dotflow-io.github.io/dotflow/)**
|
|
89
|
-
|
|
90
|
-
## Table of Contents
|
|
97
|
+
---
|
|
91
98
|
|
|
92
|
-
|
|
93
|
-
<summary>Click to expand</summary>
|
|
94
|
-
|
|
95
|
-
- [Getting Help](#getting-help)
|
|
96
|
-
- [Installation](#installation)
|
|
97
|
-
- [Quick Start](#quick-start)
|
|
98
|
-
- [Features](#features)
|
|
99
|
-
- [Execution Modes](#execution-modes)
|
|
100
|
-
- [Retry, Timeout & Backoff](#retry-timeout--backoff)
|
|
101
|
-
- [Context System](#context-system)
|
|
102
|
-
- [Checkpoint & Resume](#checkpoint--resume)
|
|
103
|
-
- [Storage Providers](#storage-providers)
|
|
104
|
-
- [Notifications](#notifications)
|
|
105
|
-
- [Class-Based Steps](#class-based-steps)
|
|
106
|
-
- [Task Groups](#task-groups)
|
|
107
|
-
- [Callbacks](#callbacks)
|
|
108
|
-
- [Error Handling](#error-handling)
|
|
109
|
-
- [Async Support](#async-support)
|
|
110
|
-
- [Scheduler / Cron](#scheduler--cron)
|
|
111
|
-
- [CLI](#cli)
|
|
112
|
-
- [Dependency Injection via Config](#dependency-injection-via-config)
|
|
113
|
-
- [More Examples](#more-examples)
|
|
114
|
-
- [Commit Style](#commit-style)
|
|
115
|
-
- [License](#license)
|
|
99
|
+
# Dotflow
|
|
116
100
|
|
|
117
|
-
|
|
101
|
+
Dotflow is a lightweight Python library for execution pipelines. Define tasks with decorators, chain them together, and deploy to any cloud — with built-in retry, parallel execution, storage, observability, and cloud deployment.
|
|
118
102
|
|
|
119
|
-
##
|
|
103
|
+
## Why Dotflow?
|
|
120
104
|
|
|
121
|
-
|
|
105
|
+
- **Simple** — `@action` decorator + `workflow.start()`. That's it.
|
|
106
|
+
- **Resilient** — Retry, backoff, timeout, checkpoints, and error handling out of the box.
|
|
107
|
+
- **Observable** — OpenTelemetry traces, metrics, and logs. Sentry error tracking.
|
|
108
|
+
- **Deployable** — `dotflow deploy --platform lambda` ships your pipeline to AWS in one command.
|
|
109
|
+
- **Portable** — Same code runs on Lambda, ECS, Cloud Run, Alibaba FC, Kubernetes, Docker, or GitHub Actions.
|
|
122
110
|
|
|
123
|
-
|
|
124
|
-
- [Documentation](https://github.com/dotflow-io/dotflow/issues/new/choose)
|
|
125
|
-
- [Feature Request](https://github.com/dotflow-io/dotflow/issues/new/choose)
|
|
126
|
-
- [Security Issue](https://github.com/dotflow-io/dotflow/issues/new/choose)
|
|
127
|
-
- [General Question](https://github.com/dotflow-io/dotflow/issues/new/choose)
|
|
128
|
-
|
|
129
|
-
## Installation
|
|
111
|
+
## Install
|
|
130
112
|
|
|
131
113
|
```bash
|
|
132
114
|
pip install dotflow
|
|
133
115
|
```
|
|
134
116
|
|
|
135
|
-
**Optional extras:**
|
|
136
|
-
|
|
137
|
-
```bash
|
|
138
|
-
pip install dotflow[aws] # AWS S3 storage
|
|
139
|
-
pip install dotflow[gcp] # Google Cloud Storage
|
|
140
|
-
pip install dotflow[scheduler] # Cron-based scheduler
|
|
141
|
-
```
|
|
142
|
-
|
|
143
117
|
## Quick Start
|
|
144
118
|
|
|
145
119
|
```python
|
|
@@ -166,9 +140,99 @@ workflow.task.add(step=load)
|
|
|
166
140
|
workflow.start()
|
|
167
141
|
```
|
|
168
142
|
|
|
143
|
+
## Deploy anywhere
|
|
144
|
+
|
|
145
|
+
Write your pipeline once. Deploy to any cloud with a single command.
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
dotflow init
|
|
149
|
+
dotflow deploy --platform lambda --project my_pipeline
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
### Supported platforms
|
|
153
|
+
|
|
154
|
+
| Platform | Deploy method |
|
|
155
|
+
|----------|---------------|
|
|
156
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-docker.svg" width="16" /> **Docker** | `docker compose up` |
|
|
157
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-lambda.svg" width="16" /> **AWS Lambda** | `dotflow deploy` |
|
|
158
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-lambda.svg" width="16" /> <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-eventbridge.svg" width="16" /> **AWS Lambda + EventBridge** | `dotflow deploy --schedule` |
|
|
159
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-lambda.svg" width="16" /> <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-s3.svg" width="16" /> **AWS Lambda + S3 Trigger** | `dotflow deploy` |
|
|
160
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-lambda.svg" width="16" /> <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-sqs.svg" width="16" /> **AWS Lambda + SQS Trigger** | `dotflow deploy` |
|
|
161
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-lambda.svg" width="16" /> <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-api-gateway.svg" width="16" /> **AWS Lambda + API Gateway** | `dotflow deploy` |
|
|
162
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-ecs.svg" width="16" /> **AWS ECS Fargate** | `dotflow deploy` |
|
|
163
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-ecs.svg" width="16" /> <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-aws-eventbridge.svg" width="16" /> **AWS ECS + EventBridge** | `dotflow deploy --schedule` |
|
|
164
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-google-cloud.svg" width="16" /> **Google Cloud Run** | `dotflow deploy` |
|
|
165
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-google-cloud.svg" width="16" /> <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-google-cloud-scheduler.svg" width="16" /> **Cloud Run + Scheduler** | `dotflow deploy --schedule` |
|
|
166
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-kubernetes.svg" width="16" /> **Kubernetes** | `kubectl apply` |
|
|
167
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-alibaba-cloud.svg" width="16" /> **Alibaba Cloud FC** | `dotflow deploy` |
|
|
168
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-alibaba-cloud.svg" width="16" /> **Alibaba Cloud FC + Timer** | `dotflow deploy --schedule` |
|
|
169
|
+
| <img src="https://raw.githubusercontent.com/dotflow-io/dotflow/develop/docs/assets/icon-github-actions.svg" width="16" /> **GitHub Actions** | `dotflow deploy` |
|
|
170
|
+
|
|
171
|
+
> [See all 34+ platforms →](https://dotflow-io.github.io/dotflow/nav/cloud/)
|
|
172
|
+
|
|
173
|
+
## Optional extras
|
|
174
|
+
|
|
175
|
+
```bash
|
|
176
|
+
pip install dotflow[aws] # S3 storage
|
|
177
|
+
pip install dotflow[gcp] # Google Cloud Storage
|
|
178
|
+
pip install dotflow[scheduler] # Cron scheduler
|
|
179
|
+
pip install dotflow[otel] # OpenTelemetry
|
|
180
|
+
pip install dotflow[sentry] # Sentry error tracking
|
|
181
|
+
pip install dotflow[deploy-aws] # AWS deploy (Lambda, ECS)
|
|
182
|
+
pip install dotflow[deploy-gcp] # GCP deploy (Cloud Run)
|
|
183
|
+
pip install dotflow[deploy-alibaba] # Alibaba Cloud deploy (FC)
|
|
184
|
+
pip install dotflow[deploy-github] # GitHub Actions deploy
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
> **[Read the full documentation →](https://dotflow-io.github.io/dotflow/)**
|
|
188
|
+
|
|
189
|
+
## Documentation
|
|
190
|
+
|
|
191
|
+
| Section | Description |
|
|
192
|
+
|---------|-------------|
|
|
193
|
+
| [Concepts](https://dotflow-io.github.io/dotflow/nav/concepts/concept-workflow-and-tasks/) | Workflows, tasks, context, providers, process modes |
|
|
194
|
+
| [How-to Guides](https://dotflow-io.github.io/dotflow/nav/how-to/) | Step-by-step tutorials for workflows, tasks, and CLI |
|
|
195
|
+
| [Cloud Deployment](https://dotflow-io.github.io/dotflow/nav/cloud/) | Deploy to AWS, GCP, Alibaba, Kubernetes, Docker, GitHub Actions |
|
|
196
|
+
| [Integrations](https://dotflow-io.github.io/dotflow/nav/integrations/) | OpenTelemetry, Sentry, Telegram, Discord, S3, GCS, Server |
|
|
197
|
+
| [Examples](https://dotflow-io.github.io/dotflow/nav/examples/) | Real-world pipelines: ETL, health checks, async, scheduler |
|
|
198
|
+
| [Reference](https://dotflow-io.github.io/dotflow/nav/reference/dotflow/) | API reference for all classes and providers |
|
|
199
|
+
| [Custom Providers](https://dotflow-io.github.io/dotflow/nav/development/custom-providers/) | Build your own storage, notify, log, tracer, or metrics provider |
|
|
200
|
+
|
|
169
201
|
## Features
|
|
170
202
|
|
|
171
|
-
|
|
203
|
+
<details>
|
|
204
|
+
<summary><strong>Observability</strong></summary>
|
|
205
|
+
|
|
206
|
+
> [OpenTelemetry docs](https://dotflow-io.github.io/dotflow/nav/tutorial/log-opentelemetry/) | [Sentry docs](https://dotflow-io.github.io/dotflow/nav/tutorial/log-sentry/) | [Tracer docs](https://dotflow-io.github.io/dotflow/nav/tutorial/tracer-opentelemetry/) | [Metrics docs](https://dotflow-io.github.io/dotflow/nav/tutorial/metrics-opentelemetry/)
|
|
207
|
+
|
|
208
|
+
Built-in support for OpenTelemetry and Sentry:
|
|
209
|
+
|
|
210
|
+
```python
|
|
211
|
+
from dotflow import Config
|
|
212
|
+
from dotflow.providers import LogOpenTelemetry, TracerOpenTelemetry, MetricsOpenTelemetry
|
|
213
|
+
|
|
214
|
+
# OpenTelemetry: traces, metrics, and structured logs
|
|
215
|
+
config = Config(
|
|
216
|
+
log=LogOpenTelemetry(service_name="my-pipeline"),
|
|
217
|
+
tracer=TracerOpenTelemetry(service_name="my-pipeline"),
|
|
218
|
+
metrics=MetricsOpenTelemetry(service_name="my-pipeline"),
|
|
219
|
+
)
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
from dotflow.providers import LogSentry, TracerSentry
|
|
224
|
+
|
|
225
|
+
# Sentry: error tracking + performance monitoring
|
|
226
|
+
config = Config(
|
|
227
|
+
log=LogSentry(dsn="https://xxx@sentry.io/123"),
|
|
228
|
+
tracer=TracerSentry(),
|
|
229
|
+
)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
</details>
|
|
233
|
+
|
|
234
|
+
<details>
|
|
235
|
+
<summary><strong>Execution Modes</strong></summary>
|
|
172
236
|
|
|
173
237
|
> [Process Mode docs](https://dotflow-io.github.io/dotflow/nav/concepts/process-mode-sequential/)
|
|
174
238
|
|
|
@@ -239,7 +303,10 @@ flowchart TD
|
|
|
239
303
|
|
|
240
304
|
---
|
|
241
305
|
|
|
242
|
-
|
|
306
|
+
</details>
|
|
307
|
+
|
|
308
|
+
<details>
|
|
309
|
+
<summary><strong>Retry, Timeout & Backoff</strong></summary>
|
|
243
310
|
|
|
244
311
|
> [Retry docs](https://dotflow-io.github.io/dotflow/nav/tutorial/task-retry/) | [Backoff docs](https://dotflow-io.github.io/dotflow/nav/tutorial/task-backoff/) | [Timeout docs](https://dotflow-io.github.io/dotflow/nav/tutorial/task-timeout/)
|
|
245
312
|
|
|
@@ -262,7 +329,10 @@ def unreliable_api_call():
|
|
|
262
329
|
|
|
263
330
|
---
|
|
264
331
|
|
|
265
|
-
|
|
332
|
+
</details>
|
|
333
|
+
|
|
334
|
+
<details>
|
|
335
|
+
<summary><strong>Context System</strong></summary>
|
|
266
336
|
|
|
267
337
|
> [Context docs](https://dotflow-io.github.io/dotflow/nav/tutorial/initial-context/) | [Previous Context](https://dotflow-io.github.io/dotflow/nav/tutorial/previous-context/) | [Many Contexts](https://dotflow-io.github.io/dotflow/nav/tutorial/many-contexts/)
|
|
268
338
|
|
|
@@ -293,7 +363,10 @@ Each `Context` object contains:
|
|
|
293
363
|
|
|
294
364
|
---
|
|
295
365
|
|
|
296
|
-
|
|
366
|
+
</details>
|
|
367
|
+
|
|
368
|
+
<details>
|
|
369
|
+
<summary><strong>Checkpoint & Resume</strong></summary>
|
|
297
370
|
|
|
298
371
|
> [Checkpoint docs](https://dotflow-io.github.io/dotflow/nav/tutorial/checkpoint/)
|
|
299
372
|
|
|
@@ -319,7 +392,10 @@ workflow.start(resume=True)
|
|
|
319
392
|
|
|
320
393
|
---
|
|
321
394
|
|
|
322
|
-
|
|
395
|
+
</details>
|
|
396
|
+
|
|
397
|
+
<details>
|
|
398
|
+
<summary><strong>Storage Providers</strong></summary>
|
|
323
399
|
|
|
324
400
|
> [Storage docs](https://dotflow-io.github.io/dotflow/nav/tutorial/storage-default/)
|
|
325
401
|
|
|
@@ -373,32 +449,39 @@ workflow = DotFlow(config=config)
|
|
|
373
449
|
|
|
374
450
|
---
|
|
375
451
|
|
|
376
|
-
|
|
452
|
+
</details>
|
|
453
|
+
|
|
454
|
+
<details>
|
|
455
|
+
<summary><strong>Notifications</strong></summary>
|
|
377
456
|
|
|
378
|
-
> [Telegram docs](https://dotflow-io.github.io/dotflow/nav/tutorial/notify-telegram/)
|
|
457
|
+
> [Telegram docs](https://dotflow-io.github.io/dotflow/nav/tutorial/notify-telegram/) | [Discord docs](https://dotflow-io.github.io/dotflow/nav/tutorial/notify-discord/)
|
|
379
458
|
|
|
380
|
-
Get notified about task status changes via Telegram.
|
|
459
|
+
Get notified about task status changes via Telegram or Discord.
|
|
381
460
|
|
|
382
461
|
```python
|
|
383
|
-
from dotflow import
|
|
462
|
+
from dotflow import Config
|
|
384
463
|
from dotflow.providers import NotifyTelegram
|
|
385
|
-
from dotflow.core.types.status import TypeStatus
|
|
386
464
|
|
|
387
|
-
|
|
465
|
+
config = Config(notify=NotifyTelegram(
|
|
388
466
|
token="YOUR_BOT_TOKEN",
|
|
389
467
|
chat_id=123456789,
|
|
390
|
-
|
|
391
|
-
)
|
|
392
|
-
|
|
393
|
-
config = Config(notify=notify)
|
|
394
|
-
workflow = DotFlow(config=config)
|
|
468
|
+
))
|
|
395
469
|
```
|
|
396
470
|
|
|
397
|
-
|
|
471
|
+
```python
|
|
472
|
+
from dotflow.providers import NotifyDiscord
|
|
473
|
+
|
|
474
|
+
config = Config(notify=NotifyDiscord(
|
|
475
|
+
webhook_url="https://discord.com/api/webhooks/...",
|
|
476
|
+
))
|
|
477
|
+
```
|
|
398
478
|
|
|
399
479
|
---
|
|
400
480
|
|
|
401
|
-
|
|
481
|
+
</details>
|
|
482
|
+
|
|
483
|
+
<details>
|
|
484
|
+
<summary><strong>Class-Based Steps</strong></summary>
|
|
402
485
|
|
|
403
486
|
Return a class instance from a task, and Dotflow will automatically discover and execute all `@action`-decorated methods in source order.
|
|
404
487
|
|
|
@@ -430,7 +513,10 @@ workflow.start()
|
|
|
430
513
|
|
|
431
514
|
---
|
|
432
515
|
|
|
433
|
-
|
|
516
|
+
</details>
|
|
517
|
+
|
|
518
|
+
<details>
|
|
519
|
+
<summary><strong>Task Groups</strong></summary>
|
|
434
520
|
|
|
435
521
|
> [Groups docs](https://dotflow-io.github.io/dotflow/nav/tutorial/groups/)
|
|
436
522
|
|
|
@@ -447,7 +533,10 @@ workflow.start() # groups run in parallel, tasks within each group run sequenti
|
|
|
447
533
|
|
|
448
534
|
---
|
|
449
535
|
|
|
450
|
-
|
|
536
|
+
</details>
|
|
537
|
+
|
|
538
|
+
<details>
|
|
539
|
+
<summary><strong>Callbacks</strong></summary>
|
|
451
540
|
|
|
452
541
|
> [Task Callback docs](https://dotflow-io.github.io/dotflow/nav/tutorial/task-callback/) | [Workflow Callback docs](https://dotflow-io.github.io/dotflow/nav/tutorial/workflow-callback/)
|
|
453
542
|
|
|
@@ -474,7 +563,10 @@ workflow.start(on_success=on_success, on_failure=on_failure)
|
|
|
474
563
|
|
|
475
564
|
---
|
|
476
565
|
|
|
477
|
-
|
|
566
|
+
</details>
|
|
567
|
+
|
|
568
|
+
<details>
|
|
569
|
+
<summary><strong>Error Handling</strong></summary>
|
|
478
570
|
|
|
479
571
|
> [Error Handling docs](https://dotflow-io.github.io/dotflow/nav/tutorial/error-handling/) | [Keep Going docs](https://dotflow-io.github.io/dotflow/nav/tutorial/keep-going/)
|
|
480
572
|
|
|
@@ -504,7 +596,10 @@ for task in workflow.result_task():
|
|
|
504
596
|
|
|
505
597
|
---
|
|
506
598
|
|
|
507
|
-
|
|
599
|
+
</details>
|
|
600
|
+
|
|
601
|
+
<details>
|
|
602
|
+
<summary><strong>Async Support</strong></summary>
|
|
508
603
|
|
|
509
604
|
> [Async docs](https://dotflow-io.github.io/dotflow/nav/tutorial/async-actions/)
|
|
510
605
|
|
|
@@ -527,7 +622,10 @@ workflow.start()
|
|
|
527
622
|
|
|
528
623
|
---
|
|
529
624
|
|
|
530
|
-
|
|
625
|
+
</details>
|
|
626
|
+
|
|
627
|
+
<details>
|
|
628
|
+
<summary><strong>Scheduler / Cron</strong></summary>
|
|
531
629
|
|
|
532
630
|
> [Cron scheduler docs](https://dotflow-io.github.io/dotflow/nav/tutorial/scheduler-cron/) | [Default scheduler](https://dotflow-io.github.io/dotflow/nav/tutorial/scheduler-default/) | [Cron overlap (concepts)](https://dotflow-io.github.io/dotflow/nav/concepts/concept-cron-overlap/)
|
|
533
631
|
|
|
@@ -576,7 +674,10 @@ The scheduler handles graceful shutdown via `SIGINT`/`SIGTERM` signals automatic
|
|
|
576
674
|
|
|
577
675
|
---
|
|
578
676
|
|
|
579
|
-
|
|
677
|
+
</details>
|
|
678
|
+
|
|
679
|
+
<details>
|
|
680
|
+
<summary><strong>CLI</strong></summary>
|
|
580
681
|
|
|
581
682
|
> [CLI docs](https://dotflow-io.github.io/dotflow/nav/how-to/cli/simple-start/)
|
|
582
683
|
|
|
@@ -618,26 +719,76 @@ Available CLI commands:
|
|
|
618
719
|
|
|
619
720
|
| Command | Description |
|
|
620
721
|
|---------|-------------|
|
|
621
|
-
| `dotflow init` |
|
|
722
|
+
| `dotflow init` | Scaffold a new project with cloud support |
|
|
622
723
|
| `dotflow start` | Run a workflow |
|
|
623
724
|
| `dotflow schedule` | Run a workflow on a cron schedule |
|
|
624
|
-
| `dotflow
|
|
725
|
+
| `dotflow logs` | View execution logs |
|
|
726
|
+
| `dotflow cloud list` | Show available cloud platforms |
|
|
727
|
+
| `dotflow cloud generate --platform <name>` | Generate deployment files |
|
|
728
|
+
| `dotflow deploy --platform <name> --project <name>` | Deploy to cloud |
|
|
729
|
+
|
|
730
|
+
---
|
|
731
|
+
|
|
732
|
+
</details>
|
|
733
|
+
|
|
734
|
+
<details>
|
|
735
|
+
<summary><strong>Server Provider</strong></summary>
|
|
736
|
+
|
|
737
|
+
> [Server docs](https://dotflow-io.github.io/dotflow/nav/tutorial/server-default/)
|
|
738
|
+
|
|
739
|
+
Send workflow and task execution data to a remote API (e.g. dotflow-api) in real time.
|
|
740
|
+
|
|
741
|
+
```python
|
|
742
|
+
from dotflow import DotFlow, Config, action
|
|
743
|
+
from dotflow.providers import ServerDefault
|
|
744
|
+
|
|
745
|
+
@action
|
|
746
|
+
def my_task():
|
|
747
|
+
return {"result": "ok"}
|
|
748
|
+
|
|
749
|
+
config = Config(
|
|
750
|
+
server=ServerDefault(
|
|
751
|
+
base_url="http://localhost:8000/api/v1",
|
|
752
|
+
user_token="your-api-token",
|
|
753
|
+
)
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
workflow = DotFlow(config=config)
|
|
757
|
+
workflow.task.add(step=my_task)
|
|
758
|
+
workflow.start()
|
|
759
|
+
```
|
|
760
|
+
|
|
761
|
+
| Parameter | Type | Default | Description |
|
|
762
|
+
|-----------|------|---------|-------------|
|
|
763
|
+
| `base_url` | `str` | `""` | API base URL |
|
|
764
|
+
| `user_token` | `str` | `""` | API token (`X-User-Token` header) |
|
|
765
|
+
| `timeout` | `float` | `5.0` | HTTP request timeout in seconds |
|
|
766
|
+
|
|
767
|
+
The server provider automatically:
|
|
768
|
+
- Creates the workflow on `DotFlow()` init
|
|
769
|
+
- Creates each task on `task.add()`
|
|
770
|
+
- Updates task status on each transition (In progress, Completed, Failed, Retry)
|
|
771
|
+
- Updates workflow status on completion (In progress → Completed)
|
|
625
772
|
|
|
626
773
|
---
|
|
627
774
|
|
|
628
|
-
|
|
775
|
+
</details>
|
|
776
|
+
|
|
777
|
+
<details>
|
|
778
|
+
<summary><strong>Dependency Injection via Config</strong></summary>
|
|
629
779
|
|
|
630
|
-
The `Config` class lets you swap providers for storage, notifications, logging, and
|
|
780
|
+
The `Config` class lets you swap providers for storage, notifications, logging, scheduling, and server:
|
|
631
781
|
|
|
632
782
|
```python
|
|
633
783
|
from dotflow import DotFlow, Config
|
|
634
|
-
from dotflow.providers import StorageFile, NotifyTelegram, LogDefault, SchedulerCron
|
|
784
|
+
from dotflow.providers import StorageFile, NotifyTelegram, LogDefault, SchedulerCron, ServerDefault
|
|
635
785
|
|
|
636
786
|
config = Config(
|
|
637
787
|
storage=StorageFile(path=".output"),
|
|
638
788
|
notify=NotifyTelegram(token="...", chat_id=123),
|
|
639
789
|
log=LogDefault(),
|
|
640
790
|
scheduler=SchedulerCron(cron="0 * * * *"),
|
|
791
|
+
server=ServerDefault(base_url="...", user_token="..."),
|
|
641
792
|
)
|
|
642
793
|
|
|
643
794
|
workflow = DotFlow(config=config)
|
|
@@ -648,13 +799,19 @@ Extend Dotflow by implementing the abstract base classes:
|
|
|
648
799
|
| ABC | Methods | Purpose |
|
|
649
800
|
|-----|---------|---------|
|
|
650
801
|
| `Storage` | `post`, `get`, `key` | Custom storage backends |
|
|
651
|
-
| `Notify` | `
|
|
652
|
-
| `Log` | `info`, `error` | Custom logging |
|
|
802
|
+
| `Notify` | `hook_status_task` | Custom notification channels |
|
|
803
|
+
| `Log` | `info`, `error`, `warning`, `debug` | Custom logging |
|
|
653
804
|
| `Scheduler` | `start`, `stop` | Custom scheduling strategies |
|
|
805
|
+
| `Tracer` | `start_workflow`, `end_workflow`, `start_task`, `end_task` | Distributed tracing |
|
|
806
|
+
| `Metrics` | `workflow_started`, `workflow_completed`, `workflow_failed`, `task_completed`, `task_failed`, `task_retried` | Counters and histograms |
|
|
807
|
+
| `Server` | `create_workflow`, `update_workflow`, `create_task`, `update_task` | Remote API communication |
|
|
654
808
|
|
|
655
809
|
---
|
|
656
810
|
|
|
657
|
-
|
|
811
|
+
</details>
|
|
812
|
+
|
|
813
|
+
<details>
|
|
814
|
+
<summary><strong>Results & Inspection</strong></summary>
|
|
658
815
|
|
|
659
816
|
After execution, inspect results directly from the workflow object:
|
|
660
817
|
|
|
@@ -685,7 +842,10 @@ workflow.task.schema() # Pydantic schema of the workflow
|
|
|
685
842
|
|
|
686
843
|
---
|
|
687
844
|
|
|
688
|
-
|
|
845
|
+
</details>
|
|
846
|
+
|
|
847
|
+
<details>
|
|
848
|
+
<summary><strong>Dynamic Module Import</strong></summary>
|
|
689
849
|
|
|
690
850
|
Reference tasks and callbacks by their module path string instead of importing them directly:
|
|
691
851
|
|
|
@@ -696,124 +856,12 @@ workflow.task.add(step="my_package.tasks.save_results", callback="my_package.cal
|
|
|
696
856
|
|
|
697
857
|
---
|
|
698
858
|
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
All examples are available in the [`docs_src/`](https://github.com/dotflow-io/dotflow/tree/develop/docs_src) directory.
|
|
702
|
-
|
|
703
|
-
#### Basic
|
|
704
|
-
|
|
705
|
-
| Example | Description | Command |
|
|
706
|
-
|---------|-------------|---------|
|
|
707
|
-
| [first_steps](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/first_steps/first_steps.py) | Minimal workflow with callback | `python docs_src/first_steps/first_steps.py` |
|
|
708
|
-
| [simple_function_workflow](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/basic/simple_function_workflow.py) | Simple function-based workflow | `python docs_src/basic/simple_function_workflow.py` |
|
|
709
|
-
| [simple_class_workflow](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/basic/simple_class_workflow.py) | Class-based step with retry | `python docs_src/basic/simple_class_workflow.py` |
|
|
710
|
-
| [simple_function_workflow_with_error](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/basic/simple_function_workflow_with_error.py) | Error inspection after failure | `python docs_src/basic/simple_function_workflow_with_error.py` |
|
|
711
|
-
|
|
712
|
-
#### Async
|
|
713
|
-
|
|
714
|
-
| Example | Description | Command |
|
|
715
|
-
|---------|-------------|---------|
|
|
716
|
-
| [async_action](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/async/async_action.py) | Async task functions | `python docs_src/async/async_action.py` |
|
|
717
|
-
|
|
718
|
-
#### Context
|
|
719
|
-
|
|
720
|
-
| Example | Description | Command |
|
|
721
|
-
|---------|-------------|---------|
|
|
722
|
-
| [context](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/context/context.py) | Creating and inspecting a Context | `python docs_src/context/context.py` |
|
|
723
|
-
| [initial_context](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/initial_context/initial_context.py) | Passing initial context per task | `python docs_src/initial_context/initial_context.py` |
|
|
724
|
-
| [previous_context](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/previous_context/previous_context.py) | Chaining context between tasks | `python docs_src/previous_context/previous_context.py` |
|
|
725
|
-
| [many_contexts](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/context/many_contexts.py) | Using both initial and previous context | `python docs_src/context/many_contexts.py` |
|
|
726
|
-
|
|
727
|
-
#### Process Modes
|
|
728
|
-
|
|
729
|
-
| Example | Description | Command |
|
|
730
|
-
|---------|-------------|---------|
|
|
731
|
-
| [sequential](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/process_mode/sequential.py) | Sequential execution | `python docs_src/process_mode/sequential.py` |
|
|
732
|
-
| [background](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/process_mode/background.py) | Background (non-blocking) execution | `python docs_src/process_mode/background.py` |
|
|
733
|
-
| [parallel](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/process_mode/parallel.py) | Parallel execution | `python docs_src/process_mode/parallel.py` |
|
|
734
|
-
| [parallel_group](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/process_mode/parallel_group.py) | Parallel groups execution | `python docs_src/process_mode/parallel_group.py` |
|
|
735
|
-
| [sequential_group_mode](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/workflow/sequential_group_mode.py) | Sequential with named groups | `python docs_src/workflow/sequential_group_mode.py` |
|
|
736
|
-
|
|
737
|
-
#### Resilience (Retry, Backoff, Timeout)
|
|
738
|
-
|
|
739
|
-
| Example | Description | Command |
|
|
740
|
-
|---------|-------------|---------|
|
|
741
|
-
| [retry](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/retry/retry.py) | Retry on function and class steps | `python docs_src/retry/retry.py` |
|
|
742
|
-
| [retry_delay](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/retry/retry_delay.py) | Retry with delay between attempts | `python docs_src/retry/retry_delay.py` |
|
|
743
|
-
| [backoff](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/backoff/backoff.py) | Exponential backoff on retries | `python docs_src/backoff/backoff.py` |
|
|
744
|
-
| [timeout](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/timeout/timeout.py) | Timeout per task execution | `python docs_src/timeout/timeout.py` |
|
|
745
|
-
|
|
746
|
-
#### Callbacks
|
|
747
|
-
|
|
748
|
-
| Example | Description | Command |
|
|
749
|
-
|---------|-------------|---------|
|
|
750
|
-
| [task_callback](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/callback/task_callback.py) | Per-task callback on completion | `python docs_src/callback/task_callback.py` |
|
|
751
|
-
| [workflow_callback_success](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/callback/workflow_callback_success.py) | Workflow-level success callback | `python docs_src/callback/workflow_callback_success.py` |
|
|
752
|
-
| [workflow_callback_failure](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/callback/workflow_callback_failure.py) | Workflow-level failure callback | `python docs_src/callback/workflow_callback_failure.py` |
|
|
753
|
-
|
|
754
|
-
#### Error Handling
|
|
755
|
-
|
|
756
|
-
| Example | Description | Command |
|
|
757
|
-
|---------|-------------|---------|
|
|
758
|
-
| [errors](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/errors/errors.py) | Inspecting task errors and retry count | `python docs_src/errors/errors.py` |
|
|
759
|
-
| [keep_going_true](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/workflow/keep_going_true.py) | Continue workflow after task failure | `python docs_src/workflow/keep_going_true.py` |
|
|
760
|
-
|
|
761
|
-
#### Groups
|
|
762
|
-
|
|
763
|
-
| Example | Description | Command |
|
|
764
|
-
|---------|-------------|---------|
|
|
765
|
-
| [step_with_groups](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/group/step_with_groups.py) | Tasks in named parallel groups | `python docs_src/group/step_with_groups.py` |
|
|
766
|
-
|
|
767
|
-
#### Storage
|
|
768
|
-
|
|
769
|
-
| Example | Description | Command |
|
|
770
|
-
|---------|-------------|---------|
|
|
771
|
-
| [storage_file](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/storage/storage_file.py) | File-based JSON storage | `python docs_src/storage/storage_file.py` |
|
|
772
|
-
| [storage_s3](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/storage/storage_s3.py) | AWS S3 storage | `python docs_src/storage/storage_s3.py` |
|
|
773
|
-
| [storage_gcs](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/storage/storage_gcs.py) | Google Cloud Storage | `python docs_src/storage/storage_gcs.py` |
|
|
774
|
-
|
|
775
|
-
#### Checkpoint & Resume
|
|
776
|
-
|
|
777
|
-
| Example | Description | Command |
|
|
778
|
-
|---------|-------------|---------|
|
|
779
|
-
| [checkpoint](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/checkpoint/checkpoint.py) | Resume workflow from last checkpoint | `python docs_src/checkpoint/checkpoint.py` |
|
|
780
|
-
|
|
781
|
-
#### Notifications
|
|
782
|
-
|
|
783
|
-
| Example | Description | Command |
|
|
784
|
-
|---------|-------------|---------|
|
|
785
|
-
| [notify_telegram](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/notify/notify_telegram.py) | Telegram notifications on failure | `python docs_src/notify/notify_telegram.py` |
|
|
786
|
-
|
|
787
|
-
#### Config & Providers
|
|
859
|
+
</details>
|
|
788
860
|
|
|
789
|
-
| Example | Description | Command |
|
|
790
|
-
|---------|-------------|---------|
|
|
791
|
-
| [config](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/config/config.py) | Full Config with storage, notify, log | `python docs_src/config/config.py` |
|
|
792
|
-
| [storage_provider](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/config/storage_provider.py) | Swapping storage providers | `python docs_src/config/storage_provider.py` |
|
|
793
|
-
| [notify_provider](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/config/notify_provider.py) | Swapping notification providers | `python docs_src/config/notify_provider.py` |
|
|
794
|
-
| [log_provider](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/config/log_provider.py) | Custom log provider | `python docs_src/config/log_provider.py` |
|
|
795
861
|
|
|
796
|
-
|
|
862
|
+
## More Examples
|
|
797
863
|
|
|
798
|
-
|
|
799
|
-
|---------|-------------|---------|
|
|
800
|
-
| [step_function_result_task](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/output/step_function_result_task.py) | Inspect task results (function) | `python docs_src/output/step_function_result_task.py` |
|
|
801
|
-
| [step_function_result_context](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/output/step_function_result_context.py) | Inspect context results (function) | `python docs_src/output/step_function_result_context.py` |
|
|
802
|
-
| [step_function_result_storage](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/output/step_function_result_storage.py) | Inspect storage results (function) | `python docs_src/output/step_function_result_storage.py` |
|
|
803
|
-
| [step_class_result_task](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/output/step_class_result_task.py) | Inspect task results (class) | `python docs_src/output/step_class_result_task.py` |
|
|
804
|
-
| [step_class_result_context](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/output/step_class_result_context.py) | Inspect context results (class) | `python docs_src/output/step_class_result_context.py` |
|
|
805
|
-
| [step_class_result_storage](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/output/step_class_result_storage.py) | Inspect storage results (class) | `python docs_src/output/step_class_result_storage.py` |
|
|
806
|
-
|
|
807
|
-
#### CLI
|
|
808
|
-
|
|
809
|
-
| Example | Description | Command |
|
|
810
|
-
|---------|-------------|---------|
|
|
811
|
-
| [simple_cli](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/basic/simple_cli.py) | Basic CLI execution | `dotflow start --step docs_src.basic.simple_cli.simple_step` |
|
|
812
|
-
| [cli_with_callback](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/cli/cli_with_callback.py) | CLI with callback function | `dotflow start --step docs_src.cli.cli_with_callback.simple_step --callback docs_src.cli.cli_with_callback.callback` |
|
|
813
|
-
| [cli_with_initial_context](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/cli/cli_with_initial_context.py) | CLI with initial context | `dotflow start --step docs_src.cli.cli_with_initial_context.simple_step --initial-context abc` |
|
|
814
|
-
| [cli_with_mode](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/cli/cli_with_mode.py) | CLI with execution mode | `dotflow start --step docs_src.cli.cli_with_mode.simple_step --mode sequential` |
|
|
815
|
-
| [cli_with_output_context](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/cli/cli_with_output_context.py) | CLI with file storage output | `dotflow start --step docs_src.cli.cli_with_output_context.simple_step --storage file` |
|
|
816
|
-
| [cli_with_path](https://github.com/dotflow-io/dotflow/blob/develop/docs_src/cli/cli_with_path.py) | CLI with custom storage path | `dotflow start --step docs_src.cli.cli_with_path.simple_step --path .storage --storage file` |
|
|
864
|
+
All examples are available in the [`docs_src/`](https://github.com/dotflow-io/dotflow/tree/develop/docs_src) directory.
|
|
817
865
|
|
|
818
866
|
## Commit Style
|
|
819
867
|
|