processes 3.0.1__tar.gz → 3.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {processes-3.0.1 → processes-3.1.0}/.github/workflows/docs.yml +2 -2
- {processes-3.0.1 → processes-3.1.0}/.github/workflows/lint.yml +2 -2
- {processes-3.0.1 → processes-3.1.0}/.github/workflows/mypy.yml +2 -2
- {processes-3.0.1 → processes-3.1.0}/.github/workflows/publish.yml +4 -4
- {processes-3.0.1 → processes-3.1.0}/.github/workflows/tags.yml +2 -2
- {processes-3.0.1 → processes-3.1.0}/.github/workflows/tests.yml +2 -2
- {processes-3.0.1 → processes-3.1.0}/CHANGELOG.md +12 -0
- {processes-3.0.1 → processes-3.1.0}/PKG-INFO +3 -2
- processes-3.1.0/RELEASE_NOTES.md +36 -0
- {processes-3.0.1 → processes-3.1.0}/docs/examples/advanced.md +38 -1
- {processes-3.0.1 → processes-3.1.0}/docs/index.md +35 -19
- {processes-3.0.1 → processes-3.1.0}/mkdocs.yml +10 -0
- {processes-3.0.1 → processes-3.1.0}/pyproject.toml +3 -3
- {processes-3.0.1 → processes-3.1.0}/src/processes/_email_internals.py +12 -17
- processes-3.1.0/src/processes/_error_data.py +38 -0
- processes-3.1.0/src/processes/_log_formatting.py +44 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/email_config.py +2 -10
- {processes-3.0.1 → processes-3.1.0}/src/processes/task.py +11 -24
- processes-3.1.0/tests/manual_tests/__init__.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/manual_tests/manual_themed_tracebacks.py +1 -3
- {processes-3.0.1 → processes-3.1.0}/tests/test_complex_dag_failures.py +2 -5
- {processes-3.0.1 → processes-3.1.0}/tests/test_email_themes.py +39 -38
- {processes-3.0.1 → processes-3.1.0}/tests/test_normal_run_no_errors.py +21 -9
- {processes-3.0.1 → processes-3.1.0}/tests/test_timeout_retry.py +7 -6
- {processes-3.0.1 → processes-3.1.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/.github/ISSUE_TEMPLATE/custom.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/.github/workflows/lint-pr.yml +0 -0
- {processes-3.0.1 → processes-3.1.0}/.gitignore +0 -0
- {processes-3.0.1 → processes-3.1.0}/CONTRIBUTING.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/LICENSE +0 -0
- {processes-3.0.1 → processes-3.1.0}/README.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/assets/banner.svg +0 -0
- {processes-3.0.1 → processes-3.1.0}/assets/logo.png +0 -0
- {processes-3.0.1 → processes-3.1.0}/assets/logo.svg +0 -0
- {processes-3.0.1 → processes-3.1.0}/assets/social_banner.png +0 -0
- {processes-3.0.1 → processes-3.1.0}/assets/social_banner.svg +0 -0
- {processes-3.0.1 → processes-3.1.0}/docs/assets/favicon.svg +0 -0
- {processes-3.0.1 → processes-3.1.0}/docs/examples/basic.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/docs/examples/intro.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/docs/reference.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/examples/01_basic_tasks_and_dependencies/README.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/examples/01_basic_tasks_and_dependencies/example1.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/examples/02_task_dependencies_result_passing/README.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/examples/02_task_dependencies_result_passing/example2.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/examples/README.md +0 -0
- {processes-3.0.1 → processes-3.1.0}/pytest.ini +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/__init__.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/_tb_utils.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/exceptions.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/process.py +0 -0
- /processes-3.0.1/tests/__init__.py → /processes-3.1.0/src/processes/py.typed +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/languages/de.json +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/languages/en.json +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/languages/es.json +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/languages/fr.json +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/languages/it.json +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/languages/pt.json +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/palettes/catppuccin.css +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/palettes/neobones.css +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/palettes/neutral.css +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/palettes/slate.css +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/styles/classic.html +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/styles/compact.html +0 -0
- {processes-3.0.1 → processes-3.1.0}/src/processes/themes/styles/modern.html +0 -0
- {processes-3.0.1/tests/manual_tests → processes-3.1.0/tests}/__init__.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/base_test.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/manual_tests/manual_pipeline_inspect.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/test_args_kwargs.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/test_dependencies.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/test_logfiles.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/test_parallel_race_conditions.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/tests/test_unique_name.py +0 -0
- {processes-3.0.1 → processes-3.1.0}/uv.lock +0 -0
|
@@ -20,8 +20,8 @@ jobs:
|
|
|
20
20
|
deploy:
|
|
21
21
|
runs-on: ubuntu-latest
|
|
22
22
|
steps:
|
|
23
|
-
- uses: actions/checkout@
|
|
24
|
-
- uses: astral-sh/setup-uv@
|
|
23
|
+
- uses: actions/checkout@v6
|
|
24
|
+
- uses: astral-sh/setup-uv@v6
|
|
25
25
|
- name: Install dependencies
|
|
26
26
|
run: uv sync --all-groups
|
|
27
27
|
- name: Build and Deploy
|
|
@@ -14,10 +14,10 @@ jobs:
|
|
|
14
14
|
lint:
|
|
15
15
|
runs-on: ubuntu-latest
|
|
16
16
|
steps:
|
|
17
|
-
- uses: actions/checkout@
|
|
17
|
+
- uses: actions/checkout@v6
|
|
18
18
|
|
|
19
19
|
- name: Install uv
|
|
20
|
-
uses: astral-sh/setup-uv@
|
|
20
|
+
uses: astral-sh/setup-uv@v6
|
|
21
21
|
with:
|
|
22
22
|
enable-cache: true # Speeds up future runs
|
|
23
23
|
|
|
@@ -14,10 +14,10 @@ jobs:
|
|
|
14
14
|
lint:
|
|
15
15
|
runs-on: ubuntu-latest
|
|
16
16
|
steps:
|
|
17
|
-
- uses: actions/checkout@
|
|
17
|
+
- uses: actions/checkout@v6
|
|
18
18
|
|
|
19
19
|
- name: Install uv
|
|
20
|
-
uses: astral-sh/setup-uv@
|
|
20
|
+
uses: astral-sh/setup-uv@v6
|
|
21
21
|
with:
|
|
22
22
|
enable-cache: true # Speeds up future runs
|
|
23
23
|
|
|
@@ -8,9 +8,9 @@ jobs:
|
|
|
8
8
|
quality-gate:
|
|
9
9
|
runs-on: ubuntu-latest
|
|
10
10
|
steps:
|
|
11
|
-
- uses: actions/checkout@
|
|
11
|
+
- uses: actions/checkout@v6
|
|
12
12
|
- name: Install uv
|
|
13
|
-
uses: astral-sh/setup-uv@
|
|
13
|
+
uses: astral-sh/setup-uv@v6
|
|
14
14
|
|
|
15
15
|
- name: Install dependencies
|
|
16
16
|
run: uv sync --all-extras --dev
|
|
@@ -33,9 +33,9 @@ jobs:
|
|
|
33
33
|
contents: read
|
|
34
34
|
|
|
35
35
|
steps:
|
|
36
|
-
- uses: actions/checkout@
|
|
36
|
+
- uses: actions/checkout@v6
|
|
37
37
|
- name: Install uv
|
|
38
|
-
uses: astral-sh/setup-uv@
|
|
38
|
+
uses: astral-sh/setup-uv@v6
|
|
39
39
|
|
|
40
40
|
- name: Build package
|
|
41
41
|
run: uv build
|
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
## v3.1.0 (2026-06-13)
|
|
2
|
+
|
|
3
|
+
### Feat
|
|
4
|
+
|
|
5
|
+
- **logging**: include failure context in task logfiles
|
|
6
|
+
|
|
7
|
+
### Refactor
|
|
8
|
+
|
|
9
|
+
- extract typed _ErrorData from task_context via shared base formatter
|
|
10
|
+
- inline single-use _task_context_lines helper
|
|
11
|
+
- move task logfile formatting out of _email_internals
|
|
12
|
+
|
|
1
13
|
## v3.0.1 (2026-06-13)
|
|
2
14
|
|
|
3
15
|
### Refactor
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: processes
|
|
3
|
-
Version: 3.0
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 3.1.0
|
|
4
|
+
Summary: Orchestrate graphs of callables in Python with automatic dependency resolution, parallel execution, retries, timeouts, and HTML email alerts on failure — zero dependencies
|
|
5
5
|
Project-URL: Homepage, https://github.com/oliverm91/processes
|
|
6
6
|
Project-URL: Documentation, https://oliverm91.github.io/processes/
|
|
7
7
|
Project-URL: Repository, https://github.com/oliverm91/processes
|
|
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Typing :: Typed
|
|
22
23
|
Requires-Python: >=3.10
|
|
23
24
|
Description-Content-Type: text/markdown
|
|
24
25
|
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# v3.0.1 — Email API Overhaul, Task Retries & Timeouts
|
|
2
|
+
|
|
3
|
+
## ⚠️ Breaking Changes
|
|
4
|
+
|
|
5
|
+
- **Removed `HTMLSMTPHandler` and `html_mail_handler`**. Email alerting is now configured with two plain dataclasses passed to `Task`:
|
|
6
|
+
- `SMTPConfig` — SMTP transport settings (host, credentials, sender/recipients, TLS).
|
|
7
|
+
- `HTMLEmailStyle` — presentation settings (style, color palette, language, traced-vars frame filter).
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
Task(..., smtp_config=SMTPConfig(...), email_style=HTMLEmailStyle(style="modern", palette="catppuccin"))
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## ✨ New Features
|
|
14
|
+
|
|
15
|
+
- **Per-task timeouts**: `Task(..., timeout=30)` raises `TimeoutError` if an attempt exceeds the limit.
|
|
16
|
+
- **Automatic retries**: `Task(..., retries=3, retry_on=(ConnectionError, TimeoutError))` retries failed attempts on the specified exception types (defaults to `ConnectionError`/`TimeoutError`).
|
|
17
|
+
- **Structured exceptions**: `DependencyNotFoundError`, `TaskNotFoundError`, and `CircularDependencyError` now carry structured attributes (`task_name`, `missing_dep`) instead of plain messages.
|
|
18
|
+
- **Traced variables in error emails**: failure emails include a "Traced Variables" section showing local variables at the relevant stack frame, with a configurable `traced_vars_frame_filter` to target a specific module.
|
|
19
|
+
|
|
20
|
+
## 🔧 Refactoring
|
|
21
|
+
|
|
22
|
+
- Centralized failure-context extraction: `Task._resolve_args` and `Task._build_failure_context` cleanly separate dependency-result injection from failure reporting.
|
|
23
|
+
- Split traceback/frame-walking utilities (`_tb_utils.py`) from email-rendering internals (`_email_internals.py`) — each module now has a single responsibility.
|
|
24
|
+
- Each `Task` now gets a unique per-instance logger, preventing handler leakage between tasks with the same name.
|
|
25
|
+
- Minor cleanups in `Process`: bare `raise` for re-raised exceptions, extracted `_is_done()` helper for the parallel runner loop.
|
|
26
|
+
|
|
27
|
+
## 🐛 Bug Fixes
|
|
28
|
+
|
|
29
|
+
- Fixed a logging bug where `logger.exception(...)` was called outside an `except` block, resulting in empty exception details (`None`) in failure emails. Errors now log with full `exc_info` and a structured `task_context` payload.
|
|
30
|
+
- Fixed `Task.run()` not reporting the actual exception when retries were exhausted.
|
|
31
|
+
|
|
32
|
+
## 🧪 Testing & CI
|
|
33
|
+
|
|
34
|
+
- Migrated the test suite to a shared `BaseTest` base class with consistent per-test log directories and cleanup.
|
|
35
|
+
- Added dedicated tests for timeout and retry behavior.
|
|
36
|
+
- Bumped GitHub Actions (`actions/checkout` → v6, `astral-sh/setup-uv` → v6) to silence deprecation warnings.
|
|
@@ -217,4 +217,41 @@ style = HTMLEmailStyle(
|
|
|
217
217
|
)
|
|
218
218
|
|
|
219
219
|
t = Task("task_name", "logfile", func_to_run, smtp_config=smtp, email_style=style)
|
|
220
|
-
```
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
## ⏱️ Retries & Timeouts
|
|
223
|
+
|
|
224
|
+
Two `Task` constructor arguments make individual steps more resilient to
|
|
225
|
+
transient failures, without any extra wiring:
|
|
226
|
+
|
|
227
|
+
- **`timeout`** — seconds allowed for a single attempt. If `func` doesn't
|
|
228
|
+
return in time, a `TimeoutError` is raised for that attempt. `None`
|
|
229
|
+
(the default) means no limit. When a timeout fires, the underlying
|
|
230
|
+
thread is detached rather than killed — a Python threading limitation.
|
|
231
|
+
- **`retries`** — additional attempts after the first failure. `0` or
|
|
232
|
+
`None` (the default) means a single attempt with no retry.
|
|
233
|
+
- **`retry_on`** — tuple of exception types that trigger a retry. Only
|
|
234
|
+
evaluated when `retries >= 1`. When `None`, defaults at call time to
|
|
235
|
+
`(ConnectionError, TimeoutError)`.
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
from processes import Task
|
|
239
|
+
|
|
240
|
+
def call_flaky_api() -> dict:
|
|
241
|
+
"""May raise ConnectionError on a bad network blip."""
|
|
242
|
+
...
|
|
243
|
+
|
|
244
|
+
t_fetch = Task(
|
|
245
|
+
"fetch_remote_data",
|
|
246
|
+
"logs/fetch.log",
|
|
247
|
+
call_flaky_api,
|
|
248
|
+
timeout=5, # give up an attempt after 5s
|
|
249
|
+
retries=3, # up to 3 extra attempts (4 total)
|
|
250
|
+
retry_on=(ConnectionError, TimeoutError), # retry on these exceptions only
|
|
251
|
+
)
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
If every attempt fails, the task is marked failed with the **last**
|
|
255
|
+
exception raised — `retries` only controls how many times `func` is
|
|
256
|
+
retried, not whether the failure is eventually reported. Combine with
|
|
257
|
+
`smtp_config` to be paged only once all attempts are exhausted.
|
|
@@ -71,16 +71,20 @@ Define your tasks and their dependencies. **Processes** will handle the executio
|
|
|
71
71
|
```python
|
|
72
72
|
from datetime import date
|
|
73
73
|
|
|
74
|
-
from processes import Process, Task, TaskDependency,
|
|
74
|
+
from processes import Process, Task, TaskDependency, SMTPConfig, HTMLEmailStyle
|
|
75
75
|
|
|
76
76
|
# 1. Setup Email Alerts (Optional)
|
|
77
|
-
|
|
78
|
-
('smtp_server', 587),
|
|
77
|
+
smtp_config = SMTPConfig(
|
|
78
|
+
mailhost=('smtp_server', 587),
|
|
79
|
+
fromaddr='sender@example.com',
|
|
80
|
+
toaddrs=['admin@example.com', 'user@example.com'],
|
|
79
81
|
credentials=('user', 'pass'),
|
|
80
82
|
secure=(), # () = STARTTLS; omit for no encryption
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
)
|
|
84
|
+
email_style = HTMLEmailStyle(
|
|
85
|
+
style='modern', # classic | modern | compact
|
|
86
|
+
palette='neutral', # neutral | catppuccin | neobones | slate
|
|
87
|
+
language='en', # en | es | pt | fr | de | it
|
|
84
88
|
)
|
|
85
89
|
|
|
86
90
|
# 2. If necessary, create wrappers for your Tasks.
|
|
@@ -96,7 +100,7 @@ def sum_data_from_csv_and_x(x, a=1, b=2):
|
|
|
96
100
|
# 3. Create the Task Graph (order is irrelevant, that is handled by Process)
|
|
97
101
|
tasks = [
|
|
98
102
|
Task("t-1", "etl.log", get_previous_working_day),
|
|
99
|
-
Task("intependent", "indep.log", indep_task,
|
|
103
|
+
Task("intependent", "indep.log", indep_task, smtp_config=smtp_config, email_style=email_style), # This task will send email on failure
|
|
100
104
|
Task("sum_csv", "etl.log", search_and_sum_csv,
|
|
101
105
|
dependencies= [
|
|
102
106
|
TaskDependency("t-1",
|
|
@@ -123,36 +127,48 @@ with Process(tasks) as process: # Context Manager ensures correct disposal of lo
|
|
|
123
127
|
|
|
124
128
|
## 📧 Customizing the HTML email
|
|
125
129
|
|
|
126
|
-
When a task with an `
|
|
130
|
+
When a task with an `smtp_config` raises, the alert is a **styled HTML
|
|
127
131
|
email** built from a bundled template. The body includes the exception,
|
|
128
132
|
the traceback (with the user-frame highlighted), the task context, the
|
|
129
133
|
list of downstream tasks that were skipped because of the failure, and
|
|
130
134
|
the local variables at the failing frame (see [Traced Variables](#traced-variables) below).
|
|
131
135
|
|
|
132
|
-
|
|
136
|
+
Email delivery and presentation are configured with two separate dataclasses:
|
|
137
|
+
|
|
138
|
+
- **`SMTPConfig`** — transport settings (host, credentials, sender/recipients, TLS).
|
|
139
|
+
- **`HTMLEmailStyle`** — presentation settings, all optional:
|
|
133
140
|
|
|
134
|
-
|
|
|
141
|
+
| Field | Values | Default |
|
|
135
142
|
|---|---|---|
|
|
136
|
-
| `
|
|
137
|
-
| `
|
|
138
|
-
| `
|
|
139
|
-
| `
|
|
143
|
+
| `style` | `classic`, `modern`, `compact` | `modern` |
|
|
144
|
+
| `palette` | `neutral`, `catppuccin`, `neobones`, `slate` | `neutral` |
|
|
145
|
+
| `language` | `en`, `es`, `pt`, `fr`, `de`, `it` | `en` |
|
|
146
|
+
| `traced_vars_frame_filter` | any path substring, or `None` | `None` (outermost user frame) |
|
|
140
147
|
|
|
141
148
|
```python
|
|
142
|
-
from processes import
|
|
149
|
+
from processes import SMTPConfig, HTMLEmailStyle, Task
|
|
143
150
|
|
|
144
|
-
smtp =
|
|
151
|
+
smtp = SMTPConfig(
|
|
145
152
|
mailhost=("smtp.example.com", 587),
|
|
146
153
|
fromaddr="alerts@example.com",
|
|
147
154
|
toaddrs=["oncall@example.com"],
|
|
148
155
|
credentials=("user", "pass"),
|
|
149
156
|
secure=(), # () = STARTTLS; omit for no encryption
|
|
150
|
-
email_style="compact", # classic | modern | compact
|
|
151
|
-
color_palette="catppuccin", # neutral | catppuccin | neobones | slate
|
|
152
|
-
email_language="es", # en | es | pt | fr | de | it
|
|
153
157
|
)
|
|
158
|
+
|
|
159
|
+
style = HTMLEmailStyle(
|
|
160
|
+
style="compact", # classic | modern | compact
|
|
161
|
+
palette="catppuccin", # neutral | catppuccin | neobones | slate
|
|
162
|
+
language="es", # en | es | pt | fr | de | it
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
t = Task("task_name", "logfile", func_to_run, smtp_config=smtp, email_style=style)
|
|
154
166
|
```
|
|
155
167
|
|
|
168
|
+
If `smtp_config` is set and `email_style` is omitted, `HTMLEmailStyle()` defaults
|
|
169
|
+
(modern, neutral, English) are used. If `smtp_config` is `None`, `email_style` is ignored
|
|
170
|
+
and no email handler is attached.
|
|
171
|
+
|
|
156
172
|
All assets ship inside the wheel — the styles are Jinja-style HTML
|
|
157
173
|
templates at `src/processes/themes/styles/` and the palettes are CSS
|
|
158
174
|
fragments at `src/processes/themes/palettes/`. No template engine or
|
|
@@ -46,6 +46,16 @@ plugins:
|
|
|
46
46
|
handlers:
|
|
47
47
|
python:
|
|
48
48
|
paths: [src]
|
|
49
|
+
options:
|
|
50
|
+
docstring_style: numpy
|
|
51
|
+
show_source: true
|
|
52
|
+
show_root_heading: true
|
|
53
|
+
show_root_full_path: false
|
|
54
|
+
heading_level: 2
|
|
55
|
+
members_order: source
|
|
56
|
+
separate_signature: true
|
|
57
|
+
show_signature_annotations: true
|
|
58
|
+
merge_init_into_class: true
|
|
49
59
|
- social:
|
|
50
60
|
cards_layout: default/variant
|
|
51
61
|
cards_layout_options:
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "processes"
|
|
7
7
|
dynamic = ["version"]
|
|
8
|
-
description = "
|
|
8
|
+
description = "Orchestrate graphs of callables in Python with automatic dependency resolution, parallel execution, retries, timeouts, and HTML email alerts on failure — zero dependencies"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
license = "MIT"
|
|
@@ -22,6 +22,7 @@ classifiers = [
|
|
|
22
22
|
"Programming Language :: Python :: 3.12",
|
|
23
23
|
"Programming Language :: Python :: 3.13",
|
|
24
24
|
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Typing :: Typed",
|
|
25
26
|
]
|
|
26
27
|
|
|
27
28
|
dependencies = []
|
|
@@ -45,10 +46,9 @@ packages = ["src/processes"]
|
|
|
45
46
|
|
|
46
47
|
[tool.commitizen]
|
|
47
48
|
name = "cz_conventional_commits"
|
|
48
|
-
version_provider = "scm"
|
|
49
|
+
version_provider = "scm"
|
|
49
50
|
tag_format = "v$version"
|
|
50
51
|
update_map_description = true
|
|
51
|
-
version_files = ["pyproject.toml:version"]
|
|
52
52
|
update_changelog_on_bump = true
|
|
53
53
|
|
|
54
54
|
[tool.mypy]
|
|
@@ -10,6 +10,7 @@ from email.mime.text import MIMEText
|
|
|
10
10
|
from email.utils import formatdate
|
|
11
11
|
from typing import cast
|
|
12
12
|
|
|
13
|
+
from ._error_data import _ErrorContextFormatter
|
|
13
14
|
from .email_config import HTMLEmailStyle, SMTPConfig
|
|
14
15
|
|
|
15
16
|
_THEMES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "themes")
|
|
@@ -27,7 +28,7 @@ def _load_language_strings(language: str) -> dict[str, str]:
|
|
|
27
28
|
return cast(dict[str, str], json.load(fh))
|
|
28
29
|
|
|
29
30
|
|
|
30
|
-
class _HTMLEmailFormatter(
|
|
31
|
+
class _HTMLEmailFormatter(_ErrorContextFormatter):
|
|
31
32
|
"""Pure renderer: reads all error context from ``record.task_context`` and
|
|
32
33
|
fills the HTML template. No exception-info parsing or frame walking here.
|
|
33
34
|
"""
|
|
@@ -83,37 +84,31 @@ class _HTMLEmailFormatter(logging.Formatter):
|
|
|
83
84
|
|
|
84
85
|
def format(self, record: logging.LogRecord) -> str:
|
|
85
86
|
"""Render a log record as a complete HTML email body."""
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
exception = ctx.get("exception", record.getMessage())
|
|
89
|
-
tb_str = ctx.get("traceback_str", "")
|
|
90
|
-
traced_vars = ctx.get("traced_vars", "")
|
|
91
|
-
traced_vars_location = ctx.get("traced_vars_location", "")
|
|
87
|
+
error = self._error_data(record)
|
|
92
88
|
|
|
93
89
|
tb_before, tb_highlight, tb_after = self._split_traceback_at_target(
|
|
94
|
-
|
|
90
|
+
error.traceback_str, error.traced_vars_location
|
|
95
91
|
)
|
|
96
92
|
|
|
97
|
-
downstream = ctx.get("downstream_impact", []) or []
|
|
98
93
|
downstream_items = "".join(
|
|
99
|
-
f"<li>{html.escape(str(name), quote=True)}</li>" for name in
|
|
94
|
+
f"<li>{html.escape(str(name), quote=True)}</li>" for name in error.downstream_impact
|
|
100
95
|
)
|
|
101
96
|
|
|
102
97
|
substitutions = dict(_load_language_strings(self._email_language))
|
|
103
98
|
substitutions["lang_traced_vars_blurb"] = substitutions.get(
|
|
104
99
|
"lang_traced_vars_blurb", ""
|
|
105
|
-
).replace("{location}", traced_vars_location)
|
|
100
|
+
).replace("{location}", error.traced_vars_location)
|
|
106
101
|
substitutions.update(
|
|
107
102
|
{
|
|
108
|
-
"task_name": html.escape(
|
|
109
|
-
"function": html.escape(
|
|
110
|
-
"args": html.escape(repr(
|
|
111
|
-
"kwargs": html.escape(repr(
|
|
112
|
-
"exception": html.escape(exception, quote=True),
|
|
103
|
+
"task_name": html.escape(error.task_name, quote=True),
|
|
104
|
+
"function": html.escape(error.function, quote=True),
|
|
105
|
+
"args": html.escape(repr(error.args), quote=True),
|
|
106
|
+
"kwargs": html.escape(repr(error.kwargs), quote=True),
|
|
107
|
+
"exception": html.escape(error.exception, quote=True),
|
|
113
108
|
"traceback_before": html.escape(tb_before, quote=True),
|
|
114
109
|
"traceback_highlight": html.escape(tb_highlight, quote=True),
|
|
115
110
|
"traceback_after": html.escape(tb_after, quote=True),
|
|
116
|
-
"traced_vars": traced_vars,
|
|
111
|
+
"traced_vars": error.traced_vars,
|
|
117
112
|
"downstream_items": downstream_items,
|
|
118
113
|
}
|
|
119
114
|
)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class _ErrorData:
|
|
10
|
+
"""Typed view of a task failure, extracted from ``record.task_context``."""
|
|
11
|
+
|
|
12
|
+
task_name: str = "?"
|
|
13
|
+
function: str = "?"
|
|
14
|
+
args: tuple[Any, ...] = ()
|
|
15
|
+
kwargs: dict[str, Any] = field(default_factory=dict)
|
|
16
|
+
downstream_impact: list[str] = field(default_factory=list)
|
|
17
|
+
exception: str = ""
|
|
18
|
+
traceback_str: str = ""
|
|
19
|
+
traced_vars: str = ""
|
|
20
|
+
traced_vars_location: str = ""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class _ErrorContextFormatter(logging.Formatter):
|
|
24
|
+
"""Base formatter providing typed access to a record's failure context."""
|
|
25
|
+
|
|
26
|
+
def _error_data(self, record: logging.LogRecord) -> _ErrorData:
|
|
27
|
+
ctx = getattr(record, "task_context", None) or {}
|
|
28
|
+
return _ErrorData(
|
|
29
|
+
task_name=str(ctx.get("task_name", "?")),
|
|
30
|
+
function=str(ctx.get("function", "?")),
|
|
31
|
+
args=ctx.get("args", ()),
|
|
32
|
+
kwargs=ctx.get("kwargs", {}),
|
|
33
|
+
downstream_impact=ctx.get("downstream_impact", []) or [],
|
|
34
|
+
exception=ctx.get("exception", record.getMessage()),
|
|
35
|
+
traceback_str=ctx.get("traceback_str", ""),
|
|
36
|
+
traced_vars=ctx.get("traced_vars", ""),
|
|
37
|
+
traced_vars_location=ctx.get("traced_vars_location", ""),
|
|
38
|
+
)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from ._error_data import _ErrorContextFormatter
|
|
6
|
+
|
|
7
|
+
_LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class _TaskLogFormatter(_ErrorContextFormatter):
|
|
11
|
+
"""Plain-text formatter for task logfiles.
|
|
12
|
+
|
|
13
|
+
On failure, appends the same failure context shown in the HTML email
|
|
14
|
+
(function, args, kwargs, downstream impact, traced-vars location,
|
|
15
|
+
traceback) as readable text.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
super().__init__(_LOG_FORMAT)
|
|
20
|
+
|
|
21
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
22
|
+
if not getattr(record, "task_context", None):
|
|
23
|
+
return super().format(record)
|
|
24
|
+
|
|
25
|
+
exc_info, record.exc_info = record.exc_info, None
|
|
26
|
+
try:
|
|
27
|
+
base = super().format(record)
|
|
28
|
+
finally:
|
|
29
|
+
record.exc_info = exc_info
|
|
30
|
+
|
|
31
|
+
error = self._error_data(record)
|
|
32
|
+
lines = [
|
|
33
|
+
base,
|
|
34
|
+
"",
|
|
35
|
+
f"Function: {error.function}",
|
|
36
|
+
f"Args: {error.args!r}",
|
|
37
|
+
f"Kwargs: {error.kwargs!r}",
|
|
38
|
+
f"Downstream impact: {', '.join(error.downstream_impact) or '-'}",
|
|
39
|
+
f"Traced vars location: {error.traced_vars_location or '-'}",
|
|
40
|
+
]
|
|
41
|
+
if error.traceback_str:
|
|
42
|
+
lines.append("")
|
|
43
|
+
lines.append(error.traceback_str.rstrip("\n"))
|
|
44
|
+
return "\n".join(lines)
|
|
@@ -37,13 +37,7 @@ class SMTPConfig:
|
|
|
37
37
|
fromaddr: str
|
|
38
38
|
toaddrs: list[str]
|
|
39
39
|
credentials: tuple[str, str] | None = None
|
|
40
|
-
secure: (
|
|
41
|
-
tuple[()]
|
|
42
|
-
| tuple[str]
|
|
43
|
-
| tuple[str, str]
|
|
44
|
-
| tuple[str, str, ssl.SSLContext]
|
|
45
|
-
| None
|
|
46
|
-
) = None
|
|
40
|
+
secure: tuple[()] | tuple[str] | tuple[str, str] | tuple[str, str, ssl.SSLContext] | None = None
|
|
47
41
|
timeout: int = 5
|
|
48
42
|
|
|
49
43
|
|
|
@@ -79,9 +73,7 @@ class HTMLEmailStyle:
|
|
|
79
73
|
|
|
80
74
|
def __post_init__(self) -> None:
|
|
81
75
|
if self.style not in _VALID_STYLES:
|
|
82
|
-
raise ValueError(
|
|
83
|
-
f"style must be one of {sorted(_VALID_STYLES)}, got {self.style!r}"
|
|
84
|
-
)
|
|
76
|
+
raise ValueError(f"style must be one of {sorted(_VALID_STYLES)}, got {self.style!r}")
|
|
85
77
|
if self.palette not in _VALID_PALETTES:
|
|
86
78
|
raise ValueError(
|
|
87
79
|
f"palette must be one of {sorted(_VALID_PALETTES)}, got {self.palette!r}"
|
|
@@ -10,6 +10,7 @@ if TYPE_CHECKING:
|
|
|
10
10
|
import logging
|
|
11
11
|
|
|
12
12
|
from ._email_internals import _build_task_email_handler
|
|
13
|
+
from ._log_formatting import _TaskLogFormatter
|
|
13
14
|
from ._tb_utils import _build_traced_vars_html, _build_traced_vars_location, _format_traceback
|
|
14
15
|
from .email_config import HTMLEmailStyle, SMTPConfig
|
|
15
16
|
from .exceptions import CircularDependencyError
|
|
@@ -243,17 +244,14 @@ class Task:
|
|
|
243
244
|
raise ValueError(f"Duplicate dependency name: {dependency.task_name}")
|
|
244
245
|
depedencies_names.append(dependency.task_name)
|
|
245
246
|
if dependency.task_name == self.name:
|
|
246
|
-
raise CircularDependencyError(
|
|
247
|
-
f"Task '{self.name}' lists itself as a dependency."
|
|
248
|
-
)
|
|
247
|
+
raise CircularDependencyError(f"Task '{self.name}' lists itself as a dependency.")
|
|
249
248
|
|
|
250
249
|
logger = logging.getLogger(f"processes.{self.name}.{id(self)}")
|
|
251
250
|
logger.setLevel(logging.DEBUG)
|
|
252
251
|
|
|
253
252
|
file_handler = logging.FileHandler(self.log_path)
|
|
254
253
|
file_handler.setLevel(logging.INFO)
|
|
255
|
-
|
|
256
|
-
file_handler.setFormatter(formatter)
|
|
254
|
+
file_handler.setFormatter(_TaskLogFormatter())
|
|
257
255
|
logger.addHandler(file_handler)
|
|
258
256
|
|
|
259
257
|
if smtp_config is not None:
|
|
@@ -288,13 +286,9 @@ class Task:
|
|
|
288
286
|
raise TypeError(f"kwargs must be dict. Got {type(self.kwargs)}")
|
|
289
287
|
|
|
290
288
|
if smtp_config is not None and not isinstance(smtp_config, SMTPConfig):
|
|
291
|
-
raise TypeError(
|
|
292
|
-
f"smtp_config must be of type SMTPConfig. Got {type(smtp_config)}"
|
|
293
|
-
)
|
|
289
|
+
raise TypeError(f"smtp_config must be of type SMTPConfig. Got {type(smtp_config)}")
|
|
294
290
|
if email_style is not None and not isinstance(email_style, HTMLEmailStyle):
|
|
295
|
-
raise TypeError(
|
|
296
|
-
f"email_style must be of type HTMLEmailStyle. Got {type(email_style)}"
|
|
297
|
-
)
|
|
291
|
+
raise TypeError(f"email_style must be of type HTMLEmailStyle. Got {type(email_style)}")
|
|
298
292
|
|
|
299
293
|
if not isinstance(self.dependencies, list):
|
|
300
294
|
raise TypeError(f"dependencies must be list. Got {type(self.dependencies)}")
|
|
@@ -308,20 +302,14 @@ class Task:
|
|
|
308
302
|
if self.timeout is not None and (
|
|
309
303
|
not isinstance(self.timeout, (int, float)) or self.timeout <= 0
|
|
310
304
|
):
|
|
311
|
-
raise TypeError(
|
|
312
|
-
f"timeout must be a positive number or None, got {self.timeout!r}"
|
|
313
|
-
)
|
|
305
|
+
raise TypeError(f"timeout must be a positive number or None, got {self.timeout!r}")
|
|
314
306
|
if not isinstance(self.retries, int) or self.retries < 0:
|
|
315
|
-
raise TypeError(
|
|
316
|
-
f"retries must be a non-negative int, got {self.retries!r}"
|
|
317
|
-
)
|
|
307
|
+
raise TypeError(f"retries must be a non-negative int, got {self.retries!r}")
|
|
318
308
|
if self.retry_on is not None and (
|
|
319
309
|
not isinstance(self.retry_on, tuple)
|
|
320
310
|
or not all(isinstance(e, type) and issubclass(e, Exception) for e in self.retry_on)
|
|
321
311
|
):
|
|
322
|
-
raise TypeError(
|
|
323
|
-
"retry_on must be None or a tuple of Exception subclasses"
|
|
324
|
-
)
|
|
312
|
+
raise TypeError("retry_on must be None or a tuple of Exception subclasses")
|
|
325
313
|
|
|
326
314
|
def get_dependencies_names(self) -> set[str]:
|
|
327
315
|
"""
|
|
@@ -352,9 +340,7 @@ class Task:
|
|
|
352
340
|
finally:
|
|
353
341
|
executor.shutdown(wait=False)
|
|
354
342
|
|
|
355
|
-
def _resolve_args(
|
|
356
|
-
self, executing_process: Process | None
|
|
357
|
-
) -> tuple[list[Any], dict[str, Any]]:
|
|
343
|
+
def _resolve_args(self, executing_process: Process | None) -> tuple[list[Any], dict[str, Any]]:
|
|
358
344
|
"""Inject upstream dependency results into args/kwargs."""
|
|
359
345
|
final_args = list(self.args)
|
|
360
346
|
final_kwargs = self.kwargs.copy()
|
|
@@ -373,7 +359,8 @@ class Task:
|
|
|
373
359
|
"""Build the structured failure payload passed to every log handler."""
|
|
374
360
|
downstream_names: list[str] = (
|
|
375
361
|
[d.name for d in executing_process.get_dependant_tasks(self.name)]
|
|
376
|
-
if executing_process is not None
|
|
362
|
+
if executing_process is not None
|
|
363
|
+
else []
|
|
377
364
|
)
|
|
378
365
|
exc_tb = exc.__traceback__
|
|
379
366
|
return {
|
|
File without changes
|
|
@@ -156,9 +156,7 @@ def _log_path(logs_dir: str, name: str) -> str:
|
|
|
156
156
|
return os.path.join(logs_dir, f"{name}.log")
|
|
157
157
|
|
|
158
158
|
|
|
159
|
-
def build_tasks(
|
|
160
|
-
logs_dir: str, smtp: SMTPConfig, style: HTMLEmailStyle
|
|
161
|
-
) -> list[Task]:
|
|
159
|
+
def build_tasks(logs_dir: str, smtp: SMTPConfig, style: HTMLEmailStyle) -> list[Task]:
|
|
162
160
|
dep = TaskDependency
|
|
163
161
|
return [
|
|
164
162
|
Task(
|
|
@@ -152,8 +152,7 @@ class TestComplexDagFailures(BaseTest):
|
|
|
152
152
|
f"got {result.errored_tasks}"
|
|
153
153
|
)
|
|
154
154
|
assert result.skipped_tasks == skipped_task_names, (
|
|
155
|
-
f"skipped_tasks should be the three cascade-skipped tasks, "
|
|
156
|
-
f"got {result.skipped_tasks}"
|
|
155
|
+
f"skipped_tasks should be the three cascade-skipped tasks, got {result.skipped_tasks}"
|
|
157
156
|
)
|
|
158
157
|
assert result.errored_tasks.isdisjoint(result.skipped_tasks), (
|
|
159
158
|
"errored_tasks and skipped_tasks must be disjoint"
|
|
@@ -208,9 +207,7 @@ class TestComplexDagFailures(BaseTest):
|
|
|
208
207
|
assert ctx["args"] == ()
|
|
209
208
|
assert ctx["kwargs"] == {}
|
|
210
209
|
assert isinstance(ctx["downstream_impact"], list)
|
|
211
|
-
expected_downstream = {
|
|
212
|
-
n for n in skipped_task_names if name in _ancestors_of(n, tasks)
|
|
213
|
-
}
|
|
210
|
+
expected_downstream = {n for n in skipped_task_names if name in _ancestors_of(n, tasks)}
|
|
214
211
|
assert set(ctx["downstream_impact"]) == expected_downstream
|
|
215
212
|
|
|
216
213
|
serialized = repr(ctx)
|
|
@@ -166,9 +166,7 @@ class TestEmailRendering(BaseTest):
|
|
|
166
166
|
formatter = _HTMLEmailFormatter(
|
|
167
167
|
HTMLEmailStyle(style=style, palette=palette, language=language)
|
|
168
168
|
)
|
|
169
|
-
output = formatter.format(
|
|
170
|
-
_make_record(f"task_{language}_{style}_{palette}")
|
|
171
|
-
)
|
|
169
|
+
output = formatter.format(_make_record(f"task_{language}_{style}_{palette}"))
|
|
172
170
|
|
|
173
171
|
for key in _LANG_CONTENT_KEYS:
|
|
174
172
|
placeholder = "{{" + key + "}}"
|
|
@@ -214,14 +212,16 @@ class TestEmailRendering(BaseTest):
|
|
|
214
212
|
except Exception as exc:
|
|
215
213
|
record = _make_record("filter_user")
|
|
216
214
|
frame_filter = "test_email_themes"
|
|
217
|
-
record.task_context.update(
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
215
|
+
record.task_context.update(
|
|
216
|
+
{
|
|
217
|
+
"exception": str(exc),
|
|
218
|
+
"traceback_str": _format_traceback(exc),
|
|
219
|
+
"traced_vars": _build_traced_vars_html(exc.__traceback__, frame_filter),
|
|
220
|
+
"traced_vars_location": _build_traced_vars_location(
|
|
221
|
+
exc.__traceback__, frame_filter
|
|
222
|
+
),
|
|
223
|
+
}
|
|
224
|
+
)
|
|
225
225
|
|
|
226
226
|
formatter = _HTMLEmailFormatter(HTMLEmailStyle())
|
|
227
227
|
body = formatter.format(record)
|
|
@@ -254,14 +254,16 @@ class TestEmailRendering(BaseTest):
|
|
|
254
254
|
except Exception as exc:
|
|
255
255
|
record = _make_record("filter_json")
|
|
256
256
|
frame_filter = "json"
|
|
257
|
-
record.task_context.update(
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
257
|
+
record.task_context.update(
|
|
258
|
+
{
|
|
259
|
+
"exception": str(exc),
|
|
260
|
+
"traceback_str": _format_traceback(exc),
|
|
261
|
+
"traced_vars": _build_traced_vars_html(exc.__traceback__, frame_filter),
|
|
262
|
+
"traced_vars_location": _build_traced_vars_location(
|
|
263
|
+
exc.__traceback__, frame_filter
|
|
264
|
+
),
|
|
265
|
+
}
|
|
266
|
+
)
|
|
265
267
|
|
|
266
268
|
formatter = _HTMLEmailFormatter(HTMLEmailStyle())
|
|
267
269
|
body = formatter.format(record)
|
|
@@ -287,12 +289,14 @@ class TestEmailRendering(BaseTest):
|
|
|
287
289
|
_inner()
|
|
288
290
|
except Exception as exc:
|
|
289
291
|
record = _make_record("traced_task")
|
|
290
|
-
record.task_context.update(
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
292
|
+
record.task_context.update(
|
|
293
|
+
{
|
|
294
|
+
"exception": str(exc),
|
|
295
|
+
"traceback_str": _format_traceback(exc),
|
|
296
|
+
"traced_vars": _build_traced_vars_html(exc.__traceback__, None),
|
|
297
|
+
"traced_vars_location": _build_traced_vars_location(exc.__traceback__, None),
|
|
298
|
+
}
|
|
299
|
+
)
|
|
296
300
|
|
|
297
301
|
formatter = _HTMLEmailFormatter(
|
|
298
302
|
HTMLEmailStyle(style="modern", palette="neutral", language="en")
|
|
@@ -393,9 +397,7 @@ class TestTaskEmailWiring(BaseTest):
|
|
|
393
397
|
assert "Fallo en el pipeline: wired" in body, (
|
|
394
398
|
"Rendered email body is missing the Spanish 'lang_failure_header'"
|
|
395
399
|
)
|
|
396
|
-
assert "Función" in body,
|
|
397
|
-
"Rendered email body is missing the Spanish 'lang_function_label'"
|
|
398
|
-
)
|
|
400
|
+
assert "Función" in body, "Rendered email body is missing the Spanish 'lang_function_label'"
|
|
399
401
|
assert "planned end-to-end failure" in body
|
|
400
402
|
|
|
401
403
|
def test_task_subject_carries_language_prefix(self) -> None:
|
|
@@ -477,18 +479,18 @@ class TestTaskEmailWiring(BaseTest):
|
|
|
477
479
|
def boom() -> None:
|
|
478
480
|
raise RuntimeError("boom")
|
|
479
481
|
|
|
480
|
-
task_a = Task(
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
482
|
+
task_a = Task(
|
|
483
|
+
name="iso_task_a", log_path=self._log("iso_task_a.log"), func=boom, smtp_config=smtp_cfg
|
|
484
|
+
)
|
|
485
|
+
task_b = Task(
|
|
486
|
+
name="iso_task_b", log_path=self._log("iso_task_b.log"), func=boom, smtp_config=smtp_cfg
|
|
487
|
+
)
|
|
484
488
|
try:
|
|
485
489
|
email_handlers_a = [
|
|
486
|
-
h for h in task_a.logger.handlers
|
|
487
|
-
if isinstance(h, logging.handlers.SMTPHandler)
|
|
490
|
+
h for h in task_a.logger.handlers if isinstance(h, logging.handlers.SMTPHandler)
|
|
488
491
|
]
|
|
489
492
|
email_handlers_b = [
|
|
490
|
-
h for h in task_b.logger.handlers
|
|
491
|
-
if isinstance(h, logging.handlers.SMTPHandler)
|
|
493
|
+
h for h in task_b.logger.handlers if isinstance(h, logging.handlers.SMTPHandler)
|
|
492
494
|
]
|
|
493
495
|
|
|
494
496
|
assert len(email_handlers_a) == 1, "Task A should have exactly one email handler"
|
|
@@ -519,8 +521,7 @@ class TestTaskEmailWiring(BaseTest):
|
|
|
519
521
|
)
|
|
520
522
|
try:
|
|
521
523
|
email_handlers = [
|
|
522
|
-
h for h in task.logger.handlers
|
|
523
|
-
if isinstance(h, logging.handlers.SMTPHandler)
|
|
524
|
+
h for h in task.logger.handlers if isinstance(h, logging.handlers.SMTPHandler)
|
|
524
525
|
]
|
|
525
526
|
assert len(email_handlers) == 0, (
|
|
526
527
|
f"No email handler should be attached when smtp_config is None, "
|
|
@@ -77,15 +77,27 @@ class TestNormalRun(BaseTest):
|
|
|
77
77
|
Task("task_1", log, task_1),
|
|
78
78
|
Task("task_2", log, task_2),
|
|
79
79
|
Task("task_3", log, task_3, args=(1,)),
|
|
80
|
-
Task(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
80
|
+
Task(
|
|
81
|
+
"task_4",
|
|
82
|
+
log,
|
|
83
|
+
task_4,
|
|
84
|
+
dependencies=[TaskDependency("task_2", use_result_as_additional_args=True)],
|
|
85
|
+
),
|
|
86
|
+
Task(
|
|
87
|
+
"task_5",
|
|
88
|
+
log,
|
|
89
|
+
task_5,
|
|
90
|
+
dependencies=[TaskDependency("task_1", use_result_as_additional_args=True)],
|
|
91
|
+
),
|
|
92
|
+
Task(
|
|
93
|
+
"task_6",
|
|
94
|
+
log,
|
|
95
|
+
task_6,
|
|
96
|
+
dependencies=[
|
|
97
|
+
TaskDependency("task_2", use_result_as_additional_args=True),
|
|
98
|
+
TaskDependency("task_5", use_result_as_additional_args=True),
|
|
99
|
+
],
|
|
100
|
+
),
|
|
89
101
|
]
|
|
90
102
|
|
|
91
103
|
def test_run_dependent_tasks_sequential(self) -> None:
|
|
@@ -58,8 +58,9 @@ class TestTimeout(BaseTest):
|
|
|
58
58
|
|
|
59
59
|
assert "t_seq" in pr.failed_tasks
|
|
60
60
|
assert "t_seq" in pr.errored_tasks
|
|
61
|
-
assert isinstance(
|
|
62
|
-
|
|
61
|
+
assert isinstance(
|
|
62
|
+
pr.passed_tasks_results.get("t_seq", None) or pr.failed_tasks, set
|
|
63
|
+
) # task did not pass
|
|
63
64
|
|
|
64
65
|
def test_timeout_in_parallel_process(self) -> None:
|
|
65
66
|
"""Timeout propagates through Process.run(parallel=True)."""
|
|
@@ -275,13 +276,13 @@ class TestRetry(BaseTest):
|
|
|
275
276
|
|
|
276
277
|
def test_retry_on_string_raises(self) -> None:
|
|
277
278
|
with pytest.raises(TypeError, match="retry_on must be None or a tuple"):
|
|
278
|
-
Task("t", self._log("t.log"), lambda: None, retries=1,
|
|
279
|
-
retry_on="ConnectionError") # type: ignore[arg-type]
|
|
279
|
+
Task("t", self._log("t.log"), lambda: None, retries=1, retry_on="ConnectionError") # type: ignore[arg-type]
|
|
280
280
|
|
|
281
281
|
def test_retry_on_non_exception_raises(self) -> None:
|
|
282
282
|
with pytest.raises(TypeError, match="retry_on must be None or a tuple"):
|
|
283
|
-
Task(
|
|
284
|
-
|
|
283
|
+
Task(
|
|
284
|
+
"t", self._log("t.log"), lambda: None, retries=1, retry_on=(int,)
|
|
285
|
+
) # int is not an Exception subclass
|
|
285
286
|
|
|
286
287
|
|
|
287
288
|
class TestTimeoutWithRetry(BaseTest):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{processes-3.0.1 → processes-3.1.0}/examples/02_task_dependencies_result_passing/example2.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|