processes 3.0.1__tar.gz → 3.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {processes-3.0.1 → processes-3.1.1}/.github/workflows/docs.yml +2 -2
- {processes-3.0.1 → processes-3.1.1}/.github/workflows/lint.yml +2 -2
- {processes-3.0.1 → processes-3.1.1}/.github/workflows/mypy.yml +2 -2
- {processes-3.0.1 → processes-3.1.1}/.github/workflows/publish.yml +4 -4
- {processes-3.0.1 → processes-3.1.1}/.github/workflows/tags.yml +2 -2
- {processes-3.0.1 → processes-3.1.1}/.github/workflows/tests.yml +2 -2
- {processes-3.0.1 → processes-3.1.1}/CHANGELOG.md +18 -0
- {processes-3.0.1 → processes-3.1.1}/PKG-INFO +17 -2
- {processes-3.0.1 → processes-3.1.1}/README.md +14 -0
- {processes-3.0.1 → processes-3.1.1}/docs/examples/advanced.md +38 -1
- {processes-3.0.1 → processes-3.1.1}/docs/index.md +42 -19
- {processes-3.0.1 → processes-3.1.1}/mkdocs.yml +10 -0
- {processes-3.0.1 → processes-3.1.1}/pyproject.toml +3 -3
- {processes-3.0.1 → processes-3.1.1}/src/processes/_email_internals.py +66 -22
- processes-3.1.1/src/processes/_error_data.py +74 -0
- processes-3.1.1/src/processes/_logfile_formatting.py +57 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/_tb_utils.py +82 -4
- {processes-3.0.1 → processes-3.1.1}/src/processes/email_config.py +15 -13
- {processes-3.0.1 → processes-3.1.1}/src/processes/process.py +10 -1
- {processes-3.0.1 → processes-3.1.1}/src/processes/task.py +18 -24
- processes-3.1.1/tests/manual_tests/__init__.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/manual_tests/manual_themed_tracebacks.py +1 -3
- {processes-3.0.1 → processes-3.1.1}/tests/test_complex_dag_failures.py +2 -5
- {processes-3.0.1 → processes-3.1.1}/tests/test_email_themes.py +39 -38
- {processes-3.0.1 → processes-3.1.1}/tests/test_normal_run_no_errors.py +21 -9
- {processes-3.0.1 → processes-3.1.1}/tests/test_timeout_retry.py +7 -6
- {processes-3.0.1 → processes-3.1.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/.github/ISSUE_TEMPLATE/custom.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/.github/workflows/lint-pr.yml +0 -0
- {processes-3.0.1 → processes-3.1.1}/.gitignore +0 -0
- {processes-3.0.1 → processes-3.1.1}/CONTRIBUTING.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/LICENSE +0 -0
- {processes-3.0.1 → processes-3.1.1}/assets/banner.svg +0 -0
- {processes-3.0.1 → processes-3.1.1}/assets/logo.png +0 -0
- {processes-3.0.1 → processes-3.1.1}/assets/logo.svg +0 -0
- {processes-3.0.1 → processes-3.1.1}/assets/social_banner.png +0 -0
- {processes-3.0.1 → processes-3.1.1}/assets/social_banner.svg +0 -0
- {processes-3.0.1 → processes-3.1.1}/docs/assets/favicon.svg +0 -0
- {processes-3.0.1 → processes-3.1.1}/docs/examples/basic.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/docs/examples/intro.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/docs/reference.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/examples/01_basic_tasks_and_dependencies/README.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/examples/01_basic_tasks_and_dependencies/example1.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/examples/02_task_dependencies_result_passing/README.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/examples/02_task_dependencies_result_passing/example2.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/examples/README.md +0 -0
- {processes-3.0.1 → processes-3.1.1}/pytest.ini +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/__init__.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/exceptions.py +0 -0
- /processes-3.0.1/tests/__init__.py → /processes-3.1.1/src/processes/py.typed +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/languages/de.json +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/languages/en.json +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/languages/es.json +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/languages/fr.json +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/languages/it.json +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/languages/pt.json +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/palettes/catppuccin.css +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/palettes/neobones.css +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/palettes/neutral.css +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/palettes/slate.css +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/styles/classic.html +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/styles/compact.html +0 -0
- {processes-3.0.1 → processes-3.1.1}/src/processes/themes/styles/modern.html +0 -0
- {processes-3.0.1/tests/manual_tests → processes-3.1.1/tests}/__init__.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/base_test.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/manual_tests/manual_pipeline_inspect.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/test_args_kwargs.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/test_dependencies.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/test_logfiles.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/test_parallel_race_conditions.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/tests/test_unique_name.py +0 -0
- {processes-3.0.1 → processes-3.1.1}/uv.lock +0 -0
|
@@ -20,8 +20,8 @@ jobs:
|
|
|
20
20
|
deploy:
|
|
21
21
|
runs-on: ubuntu-latest
|
|
22
22
|
steps:
|
|
23
|
-
- uses: actions/checkout@
|
|
24
|
-
- uses: astral-sh/setup-uv@
|
|
23
|
+
- uses: actions/checkout@v6
|
|
24
|
+
- uses: astral-sh/setup-uv@v6
|
|
25
25
|
- name: Install dependencies
|
|
26
26
|
run: uv sync --all-groups
|
|
27
27
|
- name: Build and Deploy
|
|
@@ -14,10 +14,10 @@ jobs:
|
|
|
14
14
|
lint:
|
|
15
15
|
runs-on: ubuntu-latest
|
|
16
16
|
steps:
|
|
17
|
-
- uses: actions/checkout@
|
|
17
|
+
- uses: actions/checkout@v6
|
|
18
18
|
|
|
19
19
|
- name: Install uv
|
|
20
|
-
uses: astral-sh/setup-uv@
|
|
20
|
+
uses: astral-sh/setup-uv@v6
|
|
21
21
|
with:
|
|
22
22
|
enable-cache: true # Speeds up future runs
|
|
23
23
|
|
|
@@ -14,10 +14,10 @@ jobs:
|
|
|
14
14
|
lint:
|
|
15
15
|
runs-on: ubuntu-latest
|
|
16
16
|
steps:
|
|
17
|
-
- uses: actions/checkout@
|
|
17
|
+
- uses: actions/checkout@v6
|
|
18
18
|
|
|
19
19
|
- name: Install uv
|
|
20
|
-
uses: astral-sh/setup-uv@
|
|
20
|
+
uses: astral-sh/setup-uv@v6
|
|
21
21
|
with:
|
|
22
22
|
enable-cache: true # Speeds up future runs
|
|
23
23
|
|
|
@@ -8,9 +8,9 @@ jobs:
|
|
|
8
8
|
quality-gate:
|
|
9
9
|
runs-on: ubuntu-latest
|
|
10
10
|
steps:
|
|
11
|
-
- uses: actions/checkout@
|
|
11
|
+
- uses: actions/checkout@v6
|
|
12
12
|
- name: Install uv
|
|
13
|
-
uses: astral-sh/setup-uv@
|
|
13
|
+
uses: astral-sh/setup-uv@v6
|
|
14
14
|
|
|
15
15
|
- name: Install dependencies
|
|
16
16
|
run: uv sync --all-extras --dev
|
|
@@ -33,9 +33,9 @@ jobs:
|
|
|
33
33
|
contents: read
|
|
34
34
|
|
|
35
35
|
steps:
|
|
36
|
-
- uses: actions/checkout@
|
|
36
|
+
- uses: actions/checkout@v6
|
|
37
37
|
- name: Install uv
|
|
38
|
-
uses: astral-sh/setup-uv@
|
|
38
|
+
uses: astral-sh/setup-uv@v6
|
|
39
39
|
|
|
40
40
|
- name: Build package
|
|
41
41
|
run: uv build
|
|
@@ -1,3 +1,21 @@
|
|
|
1
|
+
## v3.1.1 (2026-06-13)
|
|
2
|
+
|
|
3
|
+
### Refactor
|
|
4
|
+
|
|
5
|
+
- rename _log_formatting to _logfile_formatting and _TaskLogFormatter to _TaskLogfileFormatter
|
|
6
|
+
|
|
7
|
+
## v3.1.0 (2026-06-13)
|
|
8
|
+
|
|
9
|
+
### Feat
|
|
10
|
+
|
|
11
|
+
- **logging**: include failure context in task logfiles
|
|
12
|
+
|
|
13
|
+
### Refactor
|
|
14
|
+
|
|
15
|
+
- extract typed _ErrorData from task_context via shared base formatter
|
|
16
|
+
- inline single-use _task_context_lines helper
|
|
17
|
+
- move task logfile formatting out of _email_internals
|
|
18
|
+
|
|
1
19
|
## v3.0.1 (2026-06-13)
|
|
2
20
|
|
|
3
21
|
### Refactor
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: processes
|
|
3
|
-
Version: 3.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 3.1.1
|
|
4
|
+
Summary: Orchestrate graphs of callables in Python with automatic dependency resolution, parallel execution, retries, timeouts, and HTML email alerts on failure — zero dependencies
|
|
5
5
|
Project-URL: Homepage, https://github.com/oliverm91/processes
|
|
6
6
|
Project-URL: Documentation, https://oliverm91.github.io/processes/
|
|
7
7
|
Project-URL: Repository, https://github.com/oliverm91/processes
|
|
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Typing :: Typed
|
|
22
23
|
Requires-Python: >=3.10
|
|
23
24
|
Description-Content-Type: text/markdown
|
|
24
25
|
|
|
@@ -335,6 +336,20 @@ HTMLEmailStyle(
|
|
|
335
336
|
|
|
336
337
|
All fields are optional — omit `HTMLEmailStyle` entirely to use the defaults.
|
|
337
338
|
|
|
339
|
+
#### Traced Variables
|
|
340
|
+
|
|
341
|
+
On failure, the email body includes the local variables of the **outermost
|
|
342
|
+
user frame in the traceback** — i.e. the last frame that is not inside
|
|
343
|
+
`site-packages` or your virtualenv. A `file:line` reference next to the
|
|
344
|
+
section shows exactly where those values were captured.
|
|
345
|
+
|
|
346
|
+
`traced_vars_frame_filter` lets you point this at a different frame: set it
|
|
347
|
+
to a path substring (e.g. one of your own package or module names) to
|
|
348
|
+
capture locals from the outermost frame whose filename contains that
|
|
349
|
+
substring instead. This is useful for deep-debugging code that runs through
|
|
350
|
+
several layers of internal libraries or wrappers, where the default
|
|
351
|
+
outermost-user-frame would land too high up the call stack.
|
|
352
|
+
|
|
338
353
|
</details>
|
|
339
354
|
|
|
340
355
|
<details>
|
|
@@ -311,6 +311,20 @@ HTMLEmailStyle(
|
|
|
311
311
|
|
|
312
312
|
All fields are optional — omit `HTMLEmailStyle` entirely to use the defaults.
|
|
313
313
|
|
|
314
|
+
#### Traced Variables
|
|
315
|
+
|
|
316
|
+
On failure, the email body includes the local variables of the **outermost
|
|
317
|
+
user frame in the traceback** — i.e. the last frame that is not inside
|
|
318
|
+
`site-packages` or your virtualenv. A `file:line` reference next to the
|
|
319
|
+
section shows exactly where those values were captured.
|
|
320
|
+
|
|
321
|
+
`traced_vars_frame_filter` lets you point this at a different frame: set it
|
|
322
|
+
to a path substring (e.g. one of your own package or module names) to
|
|
323
|
+
capture locals from the outermost frame whose filename contains that
|
|
324
|
+
substring instead. This is useful for deep-debugging code that runs through
|
|
325
|
+
several layers of internal libraries or wrappers, where the default
|
|
326
|
+
outermost-user-frame would land too high up the call stack.
|
|
327
|
+
|
|
314
328
|
</details>
|
|
315
329
|
|
|
316
330
|
<details>
|
|
@@ -217,4 +217,41 @@ style = HTMLEmailStyle(
|
|
|
217
217
|
)
|
|
218
218
|
|
|
219
219
|
t = Task("task_name", "logfile", func_to_run, smtp_config=smtp, email_style=style)
|
|
220
|
-
```
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
## ⏱️ Retries & Timeouts
|
|
223
|
+
|
|
224
|
+
Two `Task` constructor arguments make individual steps more resilient to
|
|
225
|
+
transient failures, without any extra wiring:
|
|
226
|
+
|
|
227
|
+
- **`timeout`** — seconds allowed for a single attempt. If `func` doesn't
|
|
228
|
+
return in time, a `TimeoutError` is raised for that attempt. `None`
|
|
229
|
+
(the default) means no limit. When a timeout fires, the underlying
|
|
230
|
+
thread is detached rather than killed — a Python threading limitation.
|
|
231
|
+
- **`retries`** — additional attempts after the first failure. `0` or
|
|
232
|
+
`None` (the default) means a single attempt with no retry.
|
|
233
|
+
- **`retry_on`** — tuple of exception types that trigger a retry. Only
|
|
234
|
+
evaluated when `retries >= 1`. When `None`, defaults at call time to
|
|
235
|
+
`(ConnectionError, TimeoutError)`.
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
from processes import Task
|
|
239
|
+
|
|
240
|
+
def call_flaky_api() -> dict:
|
|
241
|
+
"""May raise ConnectionError on a bad network blip."""
|
|
242
|
+
...
|
|
243
|
+
|
|
244
|
+
t_fetch = Task(
|
|
245
|
+
"fetch_remote_data",
|
|
246
|
+
"logs/fetch.log",
|
|
247
|
+
call_flaky_api,
|
|
248
|
+
timeout=5, # give up an attempt after 5s
|
|
249
|
+
retries=3, # up to 3 extra attempts (4 total)
|
|
250
|
+
retry_on=(ConnectionError, TimeoutError), # retry on these exceptions only
|
|
251
|
+
)
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
If every attempt fails, the task is marked failed with the **last**
|
|
255
|
+
exception raised — `retries` only controls how many times `func` is
|
|
256
|
+
retried, not whether the failure is eventually reported. Combine with
|
|
257
|
+
`smtp_config` to be paged only once all attempts are exhausted.
|
|
@@ -71,16 +71,20 @@ Define your tasks and their dependencies. **Processes** will handle the executio
|
|
|
71
71
|
```python
|
|
72
72
|
from datetime import date
|
|
73
73
|
|
|
74
|
-
from processes import Process, Task, TaskDependency,
|
|
74
|
+
from processes import Process, Task, TaskDependency, SMTPConfig, HTMLEmailStyle
|
|
75
75
|
|
|
76
76
|
# 1. Setup Email Alerts (Optional)
|
|
77
|
-
|
|
78
|
-
('smtp_server', 587),
|
|
77
|
+
smtp_config = SMTPConfig(
|
|
78
|
+
mailhost=('smtp_server', 587),
|
|
79
|
+
fromaddr='sender@example.com',
|
|
80
|
+
toaddrs=['admin@example.com', 'user@example.com'],
|
|
79
81
|
credentials=('user', 'pass'),
|
|
80
82
|
secure=(), # () = STARTTLS; omit for no encryption
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
)
|
|
84
|
+
email_style = HTMLEmailStyle(
|
|
85
|
+
style='modern', # classic | modern | compact
|
|
86
|
+
palette='neutral', # neutral | catppuccin | neobones | slate
|
|
87
|
+
language='en', # en | es | pt | fr | de | it
|
|
84
88
|
)
|
|
85
89
|
|
|
86
90
|
# 2. If necessary, create wrappers for your Tasks.
|
|
@@ -96,7 +100,7 @@ def sum_data_from_csv_and_x(x, a=1, b=2):
|
|
|
96
100
|
# 3. Create the Task Graph (order is irrelevant, that is handled by Process)
|
|
97
101
|
tasks = [
|
|
98
102
|
Task("t-1", "etl.log", get_previous_working_day),
|
|
99
|
-
Task("intependent", "indep.log", indep_task,
|
|
103
|
+
Task("intependent", "indep.log", indep_task, smtp_config=smtp_config, email_style=email_style), # This task will send email on failure
|
|
100
104
|
Task("sum_csv", "etl.log", search_and_sum_csv,
|
|
101
105
|
dependencies= [
|
|
102
106
|
TaskDependency("t-1",
|
|
@@ -123,36 +127,48 @@ with Process(tasks) as process: # Context Manager ensures correct disposal of lo
|
|
|
123
127
|
|
|
124
128
|
## 📧 Customizing the HTML email
|
|
125
129
|
|
|
126
|
-
When a task with an `
|
|
130
|
+
When a task with an `smtp_config` raises, the alert is a **styled HTML
|
|
127
131
|
email** built from a bundled template. The body includes the exception,
|
|
128
132
|
the traceback (with the user-frame highlighted), the task context, the
|
|
129
133
|
list of downstream tasks that were skipped because of the failure, and
|
|
130
134
|
the local variables at the failing frame (see [Traced Variables](#traced-variables) below).
|
|
131
135
|
|
|
132
|
-
|
|
136
|
+
Email delivery and presentation are configured with two separate dataclasses:
|
|
137
|
+
|
|
138
|
+
- **`SMTPConfig`** — transport settings (host, credentials, sender/recipients, TLS).
|
|
139
|
+
- **`HTMLEmailStyle`** — presentation settings, all optional:
|
|
133
140
|
|
|
134
|
-
|
|
|
141
|
+
| Field | Values | Default |
|
|
135
142
|
|---|---|---|
|
|
136
|
-
| `
|
|
137
|
-
| `
|
|
138
|
-
| `
|
|
139
|
-
| `
|
|
143
|
+
| `style` | `classic`, `modern`, `compact` | `modern` |
|
|
144
|
+
| `palette` | `neutral`, `catppuccin`, `neobones`, `slate` | `neutral` |
|
|
145
|
+
| `language` | `en`, `es`, `pt`, `fr`, `de`, `it` | `en` |
|
|
146
|
+
| `traced_vars_frame_filter` | any path substring, or `None` | `None` (outermost user frame) |
|
|
140
147
|
|
|
141
148
|
```python
|
|
142
|
-
from processes import
|
|
149
|
+
from processes import SMTPConfig, HTMLEmailStyle, Task
|
|
143
150
|
|
|
144
|
-
smtp =
|
|
151
|
+
smtp = SMTPConfig(
|
|
145
152
|
mailhost=("smtp.example.com", 587),
|
|
146
153
|
fromaddr="alerts@example.com",
|
|
147
154
|
toaddrs=["oncall@example.com"],
|
|
148
155
|
credentials=("user", "pass"),
|
|
149
156
|
secure=(), # () = STARTTLS; omit for no encryption
|
|
150
|
-
email_style="compact", # classic | modern | compact
|
|
151
|
-
color_palette="catppuccin", # neutral | catppuccin | neobones | slate
|
|
152
|
-
email_language="es", # en | es | pt | fr | de | it
|
|
153
157
|
)
|
|
158
|
+
|
|
159
|
+
style = HTMLEmailStyle(
|
|
160
|
+
style="compact", # classic | modern | compact
|
|
161
|
+
palette="catppuccin", # neutral | catppuccin | neobones | slate
|
|
162
|
+
language="es", # en | es | pt | fr | de | it
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
t = Task("task_name", "logfile", func_to_run, smtp_config=smtp, email_style=style)
|
|
154
166
|
```
|
|
155
167
|
|
|
168
|
+
If `smtp_config` is set and `email_style` is omitted, `HTMLEmailStyle()` defaults
|
|
169
|
+
(modern, neutral, English) are used. If `smtp_config` is `None`, `email_style` is ignored
|
|
170
|
+
and no email handler is attached.
|
|
171
|
+
|
|
156
172
|
All assets ship inside the wheel — the styles are Jinja-style HTML
|
|
157
173
|
templates at `src/processes/themes/styles/` and the palettes are CSS
|
|
158
174
|
fragments at `src/processes/themes/palettes/`. No template engine or
|
|
@@ -168,6 +184,13 @@ in the source the listed values were captured, which is usually the
|
|
|
168
184
|
fastest way to figure out *why* a complex task broke deep inside a
|
|
169
185
|
wrapper.
|
|
170
186
|
|
|
187
|
+
This default can be overridden with `HTMLEmailStyle.traced_vars_frame_filter`.
|
|
188
|
+
Set it to a path substring (e.g. the name of one of your own packages or
|
|
189
|
+
modules) to capture locals from the outermost frame whose filename contains
|
|
190
|
+
that substring instead — useful for deep-debugging code that runs through
|
|
191
|
+
several layers of internal libraries or wrappers, where the default
|
|
192
|
+
outermost-user-frame would land too high up the call stack.
|
|
193
|
+
|
|
171
194
|
---
|
|
172
195
|
|
|
173
196
|
## 🛡️ Fault Tolerance & Logs
|
|
@@ -46,6 +46,16 @@ plugins:
|
|
|
46
46
|
handlers:
|
|
47
47
|
python:
|
|
48
48
|
paths: [src]
|
|
49
|
+
options:
|
|
50
|
+
docstring_style: numpy
|
|
51
|
+
show_source: true
|
|
52
|
+
show_root_heading: true
|
|
53
|
+
show_root_full_path: false
|
|
54
|
+
heading_level: 2
|
|
55
|
+
members_order: source
|
|
56
|
+
separate_signature: true
|
|
57
|
+
show_signature_annotations: true
|
|
58
|
+
merge_init_into_class: true
|
|
49
59
|
- social:
|
|
50
60
|
cards_layout: default/variant
|
|
51
61
|
cards_layout_options:
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "processes"
|
|
7
7
|
dynamic = ["version"]
|
|
8
|
-
description = "
|
|
8
|
+
description = "Orchestrate graphs of callables in Python with automatic dependency resolution, parallel execution, retries, timeouts, and HTML email alerts on failure — zero dependencies"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
11
11
|
license = "MIT"
|
|
@@ -22,6 +22,7 @@ classifiers = [
|
|
|
22
22
|
"Programming Language :: Python :: 3.12",
|
|
23
23
|
"Programming Language :: Python :: 3.13",
|
|
24
24
|
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Typing :: Typed",
|
|
25
26
|
]
|
|
26
27
|
|
|
27
28
|
dependencies = []
|
|
@@ -45,10 +46,9 @@ packages = ["src/processes"]
|
|
|
45
46
|
|
|
46
47
|
[tool.commitizen]
|
|
47
48
|
name = "cz_conventional_commits"
|
|
48
|
-
version_provider = "scm"
|
|
49
|
+
version_provider = "scm"
|
|
49
50
|
tag_format = "v$version"
|
|
50
51
|
update_map_description = true
|
|
51
|
-
version_files = ["pyproject.toml:version"]
|
|
52
52
|
update_changelog_on_bump = true
|
|
53
53
|
|
|
54
54
|
[tool.mypy]
|
|
@@ -10,6 +10,7 @@ from email.mime.text import MIMEText
|
|
|
10
10
|
from email.utils import formatdate
|
|
11
11
|
from typing import cast
|
|
12
12
|
|
|
13
|
+
from ._error_data import _ErrorContextFormatter
|
|
13
14
|
from .email_config import HTMLEmailStyle, SMTPConfig
|
|
14
15
|
|
|
15
16
|
_THEMES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "themes")
|
|
@@ -21,13 +22,24 @@ _PALETTE_MARKER = "{{__palette_css__}}"
|
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
def _load_language_strings(language: str) -> dict[str, str]:
|
|
24
|
-
"""Load translatable strings for the given ISO 639-1 language code.
|
|
25
|
+
"""Load translatable strings for the given ISO 639-1 language code.
|
|
26
|
+
|
|
27
|
+
Parameters
|
|
28
|
+
----------
|
|
29
|
+
language : str
|
|
30
|
+
ISO 639-1 language code, e.g. ``"en"``.
|
|
31
|
+
|
|
32
|
+
Returns
|
|
33
|
+
-------
|
|
34
|
+
dict[str, str]
|
|
35
|
+
Mapping of translation keys to localized strings.
|
|
36
|
+
"""
|
|
25
37
|
path = os.path.join(_LANGUAGES_DIR, f"{language}.json")
|
|
26
38
|
with open(path, encoding="utf-8") as fh:
|
|
27
39
|
return cast(dict[str, str], json.load(fh))
|
|
28
40
|
|
|
29
41
|
|
|
30
|
-
class _HTMLEmailFormatter(
|
|
42
|
+
class _HTMLEmailFormatter(_ErrorContextFormatter):
|
|
31
43
|
"""Pure renderer: reads all error context from ``record.task_context`` and
|
|
32
44
|
fills the HTML template. No exception-info parsing or frame walking here.
|
|
33
45
|
"""
|
|
@@ -56,9 +68,20 @@ class _HTMLEmailFormatter(logging.Formatter):
|
|
|
56
68
|
def _split_traceback_at_target(self, tb_str: str, location: str) -> tuple[str, str, str]:
|
|
57
69
|
"""Split *tb_str* around the frame line matching *location*.
|
|
58
70
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
71
|
+
Parameters
|
|
72
|
+
----------
|
|
73
|
+
tb_str : str
|
|
74
|
+
Full formatted traceback to split.
|
|
75
|
+
location : str
|
|
76
|
+
``"filename:lineno"`` of the frame to highlight, as produced by
|
|
77
|
+
``_build_traced_vars_location``.
|
|
78
|
+
|
|
79
|
+
Returns
|
|
80
|
+
-------
|
|
81
|
+
tuple[str, str, str]
|
|
82
|
+
``(before, highlight, after)`` where ``highlight`` is the
|
|
83
|
+
``File "<filename>", line <lineno>, in <func>`` line for that
|
|
84
|
+
frame. Returns ``("", "", tb_str)`` if no match is found.
|
|
62
85
|
"""
|
|
63
86
|
if not tb_str or not location:
|
|
64
87
|
return ("", "", tb_str)
|
|
@@ -82,38 +105,43 @@ class _HTMLEmailFormatter(logging.Formatter):
|
|
|
82
105
|
return rendered
|
|
83
106
|
|
|
84
107
|
def format(self, record: logging.LogRecord) -> str:
|
|
85
|
-
"""Render a log record as a complete HTML email body.
|
|
86
|
-
ctx = getattr(record, "task_context", None) or {}
|
|
108
|
+
"""Render a log record as a complete HTML email body.
|
|
87
109
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
110
|
+
Parameters
|
|
111
|
+
----------
|
|
112
|
+
record : logging.LogRecord
|
|
113
|
+
The record being formatted.
|
|
114
|
+
|
|
115
|
+
Returns
|
|
116
|
+
-------
|
|
117
|
+
str
|
|
118
|
+
The fully rendered HTML email body.
|
|
119
|
+
"""
|
|
120
|
+
error = self._error_data(record)
|
|
92
121
|
|
|
93
122
|
tb_before, tb_highlight, tb_after = self._split_traceback_at_target(
|
|
94
|
-
|
|
123
|
+
error.traceback_str, error.traced_vars_location
|
|
95
124
|
)
|
|
96
125
|
|
|
97
|
-
downstream = ctx.get("downstream_impact", []) or []
|
|
98
126
|
downstream_items = "".join(
|
|
99
|
-
f"<li>{html.escape(str(name), quote=True)}</li>" for name in
|
|
127
|
+
f"<li>{html.escape(str(name), quote=True)}</li>" for name in error.downstream_impact
|
|
100
128
|
)
|
|
101
129
|
|
|
102
130
|
substitutions = dict(_load_language_strings(self._email_language))
|
|
103
131
|
substitutions["lang_traced_vars_blurb"] = substitutions.get(
|
|
104
132
|
"lang_traced_vars_blurb", ""
|
|
105
|
-
).replace("{location}", traced_vars_location)
|
|
133
|
+
).replace("{location}", error.traced_vars_location)
|
|
106
134
|
substitutions.update(
|
|
107
135
|
{
|
|
108
|
-
"task_name": html.escape(
|
|
109
|
-
"function": html.escape(
|
|
110
|
-
"args": html.escape(repr(
|
|
111
|
-
"kwargs": html.escape(repr(
|
|
112
|
-
"exception": html.escape(exception, quote=True),
|
|
136
|
+
"task_name": html.escape(error.task_name, quote=True),
|
|
137
|
+
"function": html.escape(error.function, quote=True),
|
|
138
|
+
"args": html.escape(repr(error.args), quote=True),
|
|
139
|
+
"kwargs": html.escape(repr(error.kwargs), quote=True),
|
|
140
|
+
"exception": html.escape(error.exception, quote=True),
|
|
113
141
|
"traceback_before": html.escape(tb_before, quote=True),
|
|
114
142
|
"traceback_highlight": html.escape(tb_highlight, quote=True),
|
|
115
143
|
"traceback_after": html.escape(tb_after, quote=True),
|
|
116
|
-
"traced_vars": traced_vars,
|
|
144
|
+
"traced_vars": error.traced_vars,
|
|
117
145
|
"downstream_items": downstream_items,
|
|
118
146
|
}
|
|
119
147
|
)
|
|
@@ -164,7 +192,23 @@ def _build_task_email_handler(
|
|
|
164
192
|
style: HTMLEmailStyle,
|
|
165
193
|
task_name: str,
|
|
166
194
|
) -> _HTMLEmailHandler:
|
|
167
|
-
"""Create a fully configured email handler bound to one task.
|
|
195
|
+
"""Create a fully configured email handler bound to one task.
|
|
196
|
+
|
|
197
|
+
Parameters
|
|
198
|
+
----------
|
|
199
|
+
smtp_config : SMTPConfig
|
|
200
|
+
SMTP transport configuration for the handler.
|
|
201
|
+
style : HTMLEmailStyle
|
|
202
|
+
HTML presentation settings used by the handler's formatter.
|
|
203
|
+
task_name : str
|
|
204
|
+
Name of the task the handler is bound to, used in the email subject.
|
|
205
|
+
|
|
206
|
+
Returns
|
|
207
|
+
-------
|
|
208
|
+
_HTMLEmailHandler
|
|
209
|
+
A handler at ``logging.ERROR`` level, with its formatter and
|
|
210
|
+
localized subject configured.
|
|
211
|
+
"""
|
|
168
212
|
handler = _HTMLEmailHandler(smtp_config)
|
|
169
213
|
handler.setFormatter(_HTMLEmailFormatter(style))
|
|
170
214
|
handler.setLevel(logging.ERROR)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class _ErrorData:
|
|
10
|
+
"""Typed view of a task failure, extracted from ``record.task_context``.
|
|
11
|
+
|
|
12
|
+
Attributes
|
|
13
|
+
----------
|
|
14
|
+
task_name : str
|
|
15
|
+
Name of the task that failed. Defaults to ``"?"``.
|
|
16
|
+
function : str
|
|
17
|
+
Name of the function that was executing. Defaults to ``"?"``.
|
|
18
|
+
args : tuple[Any, ...]
|
|
19
|
+
Positional arguments the function was called with. Defaults to ``()``.
|
|
20
|
+
kwargs : dict[str, Any]
|
|
21
|
+
Keyword arguments the function was called with. Defaults to ``{}``.
|
|
22
|
+
downstream_impact : list[str]
|
|
23
|
+
Names of tasks skipped as a result of this failure. Defaults to ``[]``.
|
|
24
|
+
exception : str
|
|
25
|
+
String representation of the raised exception. Defaults to ``""``.
|
|
26
|
+
traceback_str : str
|
|
27
|
+
Full formatted traceback. Defaults to ``""``.
|
|
28
|
+
traced_vars : str
|
|
29
|
+
Rendered local variables of the traced frame. Defaults to ``""``.
|
|
30
|
+
traced_vars_location : str
|
|
31
|
+
``"filename:lineno"`` of the traced frame. Defaults to ``""``.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
task_name: str = "?"
|
|
35
|
+
function: str = "?"
|
|
36
|
+
args: tuple[Any, ...] = ()
|
|
37
|
+
kwargs: dict[str, Any] = field(default_factory=dict)
|
|
38
|
+
downstream_impact: list[str] = field(default_factory=list)
|
|
39
|
+
exception: str = ""
|
|
40
|
+
traceback_str: str = ""
|
|
41
|
+
traced_vars: str = ""
|
|
42
|
+
traced_vars_location: str = ""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class _ErrorContextFormatter(logging.Formatter):
|
|
46
|
+
"""Base formatter providing typed access to a record's failure context."""
|
|
47
|
+
|
|
48
|
+
def _error_data(self, record: logging.LogRecord) -> _ErrorData:
|
|
49
|
+
"""Extract the failure context from a log record.
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
record : logging.LogRecord
|
|
54
|
+
The record being formatted. May or may not carry a
|
|
55
|
+
``task_context`` attribute.
|
|
56
|
+
|
|
57
|
+
Returns
|
|
58
|
+
-------
|
|
59
|
+
_ErrorData
|
|
60
|
+
Typed view of ``record.task_context``, with defaults filled in
|
|
61
|
+
for any missing fields.
|
|
62
|
+
"""
|
|
63
|
+
ctx = getattr(record, "task_context", None) or {}
|
|
64
|
+
return _ErrorData(
|
|
65
|
+
task_name=str(ctx.get("task_name", "?")),
|
|
66
|
+
function=str(ctx.get("function", "?")),
|
|
67
|
+
args=ctx.get("args", ()),
|
|
68
|
+
kwargs=ctx.get("kwargs", {}),
|
|
69
|
+
downstream_impact=ctx.get("downstream_impact", []) or [],
|
|
70
|
+
exception=ctx.get("exception", record.getMessage()),
|
|
71
|
+
traceback_str=ctx.get("traceback_str", ""),
|
|
72
|
+
traced_vars=ctx.get("traced_vars", ""),
|
|
73
|
+
traced_vars_location=ctx.get("traced_vars_location", ""),
|
|
74
|
+
)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from ._error_data import _ErrorContextFormatter
|
|
6
|
+
|
|
7
|
+
_LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class _TaskLogfileFormatter(_ErrorContextFormatter):
|
|
11
|
+
"""Plain-text formatter for task logfiles.
|
|
12
|
+
|
|
13
|
+
On failure, appends the same failure context shown in the HTML email
|
|
14
|
+
(function, args, kwargs, downstream impact, traced-vars location,
|
|
15
|
+
traceback) as readable text.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def __init__(self) -> None:
|
|
19
|
+
super().__init__(_LOG_FORMAT)
|
|
20
|
+
|
|
21
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
22
|
+
"""Render a log record as plain text, appending failure context if present.
|
|
23
|
+
|
|
24
|
+
Parameters
|
|
25
|
+
----------
|
|
26
|
+
record : logging.LogRecord
|
|
27
|
+
The record being formatted.
|
|
28
|
+
|
|
29
|
+
Returns
|
|
30
|
+
-------
|
|
31
|
+
str
|
|
32
|
+
The formatted log line, with the failure context appended if
|
|
33
|
+
``record.task_context`` is set.
|
|
34
|
+
"""
|
|
35
|
+
if not getattr(record, "task_context", None):
|
|
36
|
+
return super().format(record)
|
|
37
|
+
|
|
38
|
+
exc_info, record.exc_info = record.exc_info, None
|
|
39
|
+
try:
|
|
40
|
+
base = super().format(record)
|
|
41
|
+
finally:
|
|
42
|
+
record.exc_info = exc_info
|
|
43
|
+
|
|
44
|
+
error = self._error_data(record)
|
|
45
|
+
lines = [
|
|
46
|
+
base,
|
|
47
|
+
"",
|
|
48
|
+
f"Function: {error.function}",
|
|
49
|
+
f"Args: {error.args!r}",
|
|
50
|
+
f"Kwargs: {error.kwargs!r}",
|
|
51
|
+
f"Downstream impact: {', '.join(error.downstream_impact) or '-'}",
|
|
52
|
+
f"Traced vars location: {error.traced_vars_location or '-'}",
|
|
53
|
+
]
|
|
54
|
+
if error.traceback_str:
|
|
55
|
+
lines.append("")
|
|
56
|
+
lines.append(error.traceback_str.rstrip("\n"))
|
|
57
|
+
return "\n".join(lines)
|