wpipe 2.1.4__tar.gz → 2.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {wpipe-2.1.4 → wpipe-2.2.0}/PKG-INFO +50 -5
- {wpipe-2.1.4 → wpipe-2.2.0}/README.md +49 -4
- {wpipe-2.1.4 → wpipe-2.2.0}/pyproject.toml +1 -1
- wpipe-2.2.0/test/test_background.py +252 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/components/logic_blocks.py +37 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/pipe.py +68 -1
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/pipe_async.py +118 -2
- {wpipe-2.1.4 → wpipe-2.2.0}/.gitignore +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/LICENSE +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/01_simple_function/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/01_simple_function/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/02_class_steps/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/02_class_steps/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/03_mixed_steps/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/03_mixed_steps/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/04_default_values/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/04_default_values/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/05_args_kwargs/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/05_args_kwargs/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/06_dict_processing/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/06_dict_processing/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/07_multiple_runs/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/07_multiple_runs/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/08_data_aggregation/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/08_data_aggregation/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/09_empty_data/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/09_empty_data/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/10_lambda_steps/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/10_lambda_steps/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/11_decorator_steps/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/11_decorator_steps/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/12_context_manager/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/12_context_manager/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/13_async_pipeline/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/13_async_pipeline/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/14_pipeline_chaining/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/14_pipeline_chaining/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/15_pipeline_clone/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/15_pipeline_clone/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/16_LogGestor/example.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/01_basic_pipeline/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/15_export/01_json/export_json.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/examples/15_export/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_api_pipeline.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_async.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_basic_pipeline.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_checkpoint.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_core.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_dashboard_and_monitor.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_export.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_misc.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_nested_pipelines.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_pipe.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_pipe_async.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_pipeline_config.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_sqlite.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_tracking.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_util.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/test/test_yaml_config.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/api_client/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/api_client/api_client.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/checkpoint/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/checkpoint/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/checkpoint/checkpoint.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/composition/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/composition/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/composition/pipeline_step.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/__main__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/main.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/static/dashboard.js +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/static/styles.css +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/base.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/alerts.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/analytics.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/data.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/events.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/graph.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/pipelines.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/states.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/dashboard/templates/tabs/timeline.html +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/decorators/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/decorators/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/decorators/step.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/exception/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/exception/api_error.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/export/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/export/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/export/exporter.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/log/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/log/log.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/parallel/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/parallel/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/parallel/executor.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/components/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/components/constants.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/components/logic_blocks_async.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/components/metrics.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/components/progress.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/components/reporting.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/pipe/pipe_async_minimal.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/ram/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/ram/ram.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/resource_monitor/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/resource_monitor/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/resource_monitor/monitor.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/sqlite/Sqlite.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/sqlite/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/sqlite/tables_dto/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/sqlite/tables_dto/log_gestor_model.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/sqlite/tables_dto/records.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/sqlite/tables_dto/tracker_models.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/timeout/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/timeout/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/timeout/timeout.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/tracking/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/tracking/alerts.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/tracking/analysis.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/tracking/queries.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/tracking/tracker.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/type_hinting/README.md +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/type_hinting/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/type_hinting/validators.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/util/__init__.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/util/transform.py +0 -0
- {wpipe-2.1.4 → wpipe-2.2.0}/wpipe/util/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: wpipe
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2.0
|
|
4
4
|
Summary: Pipeline library with API integration for task orchestration and execution tracking
|
|
5
5
|
Project-URL: Homepage, https://github.com/wisrovi/wpipe
|
|
6
6
|
Project-URL: Documentation, https://wpipe.readthedocs.io/en/latest/
|
|
@@ -81,11 +81,11 @@ Requires-Dist: sphinx-sitemap>=2.5.0; extra == 'docs'
|
|
|
81
81
|
Requires-Dist: sphinx>=7.2.0; extra == 'docs'
|
|
82
82
|
Description-Content-Type: text/markdown
|
|
83
83
|
|
|
84
|
-
# 🚀 WPipe v2.
|
|
84
|
+
# 🚀 WPipe v2.2.0
|
|
85
85
|
|
|
86
86
|
**El motor de orquestación de pipelines más rápido, resiliente y puro para Python.**
|
|
87
87
|
|
|
88
|
-
WPipe es una librería profesional diseñada para automatizar flujos de trabajo complejos, garantizando que tus datos viajen seguros, tus procesos sean ultra-rápidos y tus fallos sean fáciles de diagnosticar. Incluye ahora un **Tour de Aprendizaje con
|
|
88
|
+
WPipe es una librería profesional diseñada para automatizar flujos de trabajo complejos, garantizando que tus datos viajen seguros, tus procesos sean ultra-rápidos y tus fallos sean fáciles de diagnosticar. Incluye ahora un **Tour de Aprendizaje con 140 Niveles** para dominar la librería desde lo más básico hasta lo más avanzado.
|
|
89
89
|
|
|
90
90
|
[](https://badge.fury.io/py/wpipe)
|
|
91
91
|
[](https://pypi.org/project/wpipe/)
|
|
@@ -109,7 +109,7 @@ Diferénciate de los scripts lineales. WPipe te ofrece superpoderes:
|
|
|
109
109
|
|
|
110
110
|
---
|
|
111
111
|
|
|
112
|
-
## 📊 Features (
|
|
112
|
+
## 📊 Features (25 Features)
|
|
113
113
|
|
|
114
114
|
| Feature | Descripción |
|
|
115
115
|
|---------|-------------|
|
|
@@ -138,6 +138,7 @@ Diferénciate de los scripts lineales. WPipe te ofrece superpoderes:
|
|
|
138
138
|
| 🔄 Async Pipeline | Soporte completo para pipelines asíncronos |
|
|
139
139
|
| 🏗️ DAG Scheduling | Programación basada en grafos acíclicos dirigida |
|
|
140
140
|
| 🌐 Dashboard Web | Dashboard visual en tiempo real |
|
|
141
|
+
| ⚡ **Background Tasks** | Ejecutar tareas sin bloquear el pipeline (fire & forget) |
|
|
141
142
|
|
|
142
143
|
---
|
|
143
144
|
|
|
@@ -157,6 +158,7 @@ WPipe se basa en **4 pilares** que puedes combinar libremente:
|
|
|
157
158
|
|
|
158
159
|
```python
|
|
159
160
|
from wpipe import Pipeline, step, Condition, For, Parallel
|
|
161
|
+
from wpipe.pipe.components.logic_blocks import Background
|
|
160
162
|
```
|
|
161
163
|
|
|
162
164
|
| Pilar | Uso |
|
|
@@ -166,6 +168,7 @@ from wpipe import Pipeline, step, Condition, For, Parallel
|
|
|
166
168
|
| **`Condition`** | Ramificación condicional basada en expresiones |
|
|
167
169
|
| **`For`** | Bucles con validación de parada |
|
|
168
170
|
| **`Parallel`** | Ejecución paralela de múltiples pasos |
|
|
171
|
+
| **`Background`** | Tareas en background sin bloquear el pipeline |
|
|
169
172
|
|
|
170
173
|
### 2. Tu Primer Pipeline
|
|
171
174
|
|
|
@@ -261,7 +264,36 @@ result = pipeline.run({})
|
|
|
261
264
|
# Las 3 tareas se ejecutan simultáneamente
|
|
262
265
|
```
|
|
263
266
|
|
|
264
|
-
### 6.
|
|
267
|
+
### 6. Background Tasks (Fire & Forget)
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
from wpipe import Pipeline, step
|
|
271
|
+
from wpipe.pipe.components.logic_blocks import Background
|
|
272
|
+
|
|
273
|
+
@step(name="tarea_principal")
|
|
274
|
+
def tarea_principal(data):
|
|
275
|
+
print("Ejecutando tarea principal...")
|
|
276
|
+
return {"status": "completado"}
|
|
277
|
+
|
|
278
|
+
@step(name="tarea_lenta")
|
|
279
|
+
def tarea_lenta(data):
|
|
280
|
+
import time
|
|
281
|
+
print("Enviando telemetría...")
|
|
282
|
+
time.sleep(2) # Simula operación lenta
|
|
283
|
+
print("¡Telemetría enviada!")
|
|
284
|
+
|
|
285
|
+
pipeline = Pipeline(pipeline_name="con_background")
|
|
286
|
+
pipeline.set_steps([
|
|
287
|
+
tarea_principal,
|
|
288
|
+
Background(tarea_lenta), # No bloquea el pipeline
|
|
289
|
+
])
|
|
290
|
+
|
|
291
|
+
result = pipeline.run({})
|
|
292
|
+
# El pipeline NO espera 2 segundos, continúa inmediatamente
|
|
293
|
+
# La tarea lenta se ejecuta en background (daemon thread)
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
### 7. Checkpoints (Resiliencia)
|
|
265
297
|
|
|
266
298
|
```python
|
|
267
299
|
from wpipe import Pipeline, step, CheckpointManager
|
|
@@ -511,6 +543,19 @@ exporter.export_pipeline_logs(format="json", output_path="reporte.json")
|
|
|
511
543
|
|
|
512
544
|
---
|
|
513
545
|
|
|
546
|
+
## 📢 Marketing & Community
|
|
547
|
+
|
|
548
|
+
Hemos expandido nuestra presencia con **+40 nuevos activos de marketing** diseñados para educar y ayudar a los desarrolladores a elegir la mejor arquitectura de orquestación.
|
|
549
|
+
|
|
550
|
+
- **Dev.to**: Guías técnicas profundas sobre Green-IT y ahorro de RAM.
|
|
551
|
+
- **DZone**: Análisis arquitectónicos sobre resiliencia industrial y estados persistentes.
|
|
552
|
+
- **Reddit**: Historias reales y debates técnicos en comunidades de Python y DevOps.
|
|
553
|
+
- **Indie Hackers**: Estrategias para escalar startups con infraestructura mínima y bajo coste.
|
|
554
|
+
|
|
555
|
+
Puedes encontrar todos estos materiales en la carpeta `posters/`, numerados del 79 al 121, cubriendo comparativas con Airflow, n8n, Zapier y más.
|
|
556
|
+
|
|
557
|
+
---
|
|
558
|
+
|
|
514
559
|
## DASHBOARD
|
|
515
560
|
|
|
516
561
|
### Tutorial
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
# 🚀 WPipe v2.
|
|
1
|
+
# 🚀 WPipe v2.2.0
|
|
2
2
|
|
|
3
3
|
**El motor de orquestación de pipelines más rápido, resiliente y puro para Python.**
|
|
4
4
|
|
|
5
|
-
WPipe es una librería profesional diseñada para automatizar flujos de trabajo complejos, garantizando que tus datos viajen seguros, tus procesos sean ultra-rápidos y tus fallos sean fáciles de diagnosticar. Incluye ahora un **Tour de Aprendizaje con
|
|
5
|
+
WPipe es una librería profesional diseñada para automatizar flujos de trabajo complejos, garantizando que tus datos viajen seguros, tus procesos sean ultra-rápidos y tus fallos sean fáciles de diagnosticar. Incluye ahora un **Tour de Aprendizaje con 140 Niveles** para dominar la librería desde lo más básico hasta lo más avanzado.
|
|
6
6
|
|
|
7
7
|
[](https://badge.fury.io/py/wpipe)
|
|
8
8
|
[](https://pypi.org/project/wpipe/)
|
|
@@ -26,7 +26,7 @@ Diferénciate de los scripts lineales. WPipe te ofrece superpoderes:
|
|
|
26
26
|
|
|
27
27
|
---
|
|
28
28
|
|
|
29
|
-
## 📊 Features (
|
|
29
|
+
## 📊 Features (25 Features)
|
|
30
30
|
|
|
31
31
|
| Feature | Descripción |
|
|
32
32
|
|---------|-------------|
|
|
@@ -55,6 +55,7 @@ Diferénciate de los scripts lineales. WPipe te ofrece superpoderes:
|
|
|
55
55
|
| 🔄 Async Pipeline | Soporte completo para pipelines asíncronos |
|
|
56
56
|
| 🏗️ DAG Scheduling | Programación basada en grafos acíclicos dirigida |
|
|
57
57
|
| 🌐 Dashboard Web | Dashboard visual en tiempo real |
|
|
58
|
+
| ⚡ **Background Tasks** | Ejecutar tareas sin bloquear el pipeline (fire & forget) |
|
|
58
59
|
|
|
59
60
|
---
|
|
60
61
|
|
|
@@ -74,6 +75,7 @@ WPipe se basa en **4 pilares** que puedes combinar libremente:
|
|
|
74
75
|
|
|
75
76
|
```python
|
|
76
77
|
from wpipe import Pipeline, step, Condition, For, Parallel
|
|
78
|
+
from wpipe.pipe.components.logic_blocks import Background
|
|
77
79
|
```
|
|
78
80
|
|
|
79
81
|
| Pilar | Uso |
|
|
@@ -83,6 +85,7 @@ from wpipe import Pipeline, step, Condition, For, Parallel
|
|
|
83
85
|
| **`Condition`** | Ramificación condicional basada en expresiones |
|
|
84
86
|
| **`For`** | Bucles con validación de parada |
|
|
85
87
|
| **`Parallel`** | Ejecución paralela de múltiples pasos |
|
|
88
|
+
| **`Background`** | Tareas en background sin bloquear el pipeline |
|
|
86
89
|
|
|
87
90
|
### 2. Tu Primer Pipeline
|
|
88
91
|
|
|
@@ -178,7 +181,36 @@ result = pipeline.run({})
|
|
|
178
181
|
# Las 3 tareas se ejecutan simultáneamente
|
|
179
182
|
```
|
|
180
183
|
|
|
181
|
-
### 6.
|
|
184
|
+
### 6. Background Tasks (Fire & Forget)
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
from wpipe import Pipeline, step
|
|
188
|
+
from wpipe.pipe.components.logic_blocks import Background
|
|
189
|
+
|
|
190
|
+
@step(name="tarea_principal")
|
|
191
|
+
def tarea_principal(data):
|
|
192
|
+
print("Ejecutando tarea principal...")
|
|
193
|
+
return {"status": "completado"}
|
|
194
|
+
|
|
195
|
+
@step(name="tarea_lenta")
|
|
196
|
+
def tarea_lenta(data):
|
|
197
|
+
import time
|
|
198
|
+
print("Enviando telemetría...")
|
|
199
|
+
time.sleep(2) # Simula operación lenta
|
|
200
|
+
print("¡Telemetría enviada!")
|
|
201
|
+
|
|
202
|
+
pipeline = Pipeline(pipeline_name="con_background")
|
|
203
|
+
pipeline.set_steps([
|
|
204
|
+
tarea_principal,
|
|
205
|
+
Background(tarea_lenta), # No bloquea el pipeline
|
|
206
|
+
])
|
|
207
|
+
|
|
208
|
+
result = pipeline.run({})
|
|
209
|
+
# El pipeline NO espera 2 segundos, continúa inmediatamente
|
|
210
|
+
# La tarea lenta se ejecuta en background (daemon thread)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### 7. Checkpoints (Resiliencia)
|
|
182
214
|
|
|
183
215
|
```python
|
|
184
216
|
from wpipe import Pipeline, step, CheckpointManager
|
|
@@ -428,6 +460,19 @@ exporter.export_pipeline_logs(format="json", output_path="reporte.json")
|
|
|
428
460
|
|
|
429
461
|
---
|
|
430
462
|
|
|
463
|
+
## 📢 Marketing & Community
|
|
464
|
+
|
|
465
|
+
Hemos expandido nuestra presencia con **+40 nuevos activos de marketing** diseñados para educar y ayudar a los desarrolladores a elegir la mejor arquitectura de orquestación.
|
|
466
|
+
|
|
467
|
+
- **Dev.to**: Guías técnicas profundas sobre Green-IT y ahorro de RAM.
|
|
468
|
+
- **DZone**: Análisis arquitectónicos sobre resiliencia industrial y estados persistentes.
|
|
469
|
+
- **Reddit**: Historias reales y debates técnicos en comunidades de Python y DevOps.
|
|
470
|
+
- **Indie Hackers**: Estrategias para escalar startups con infraestructura mínima y bajo coste.
|
|
471
|
+
|
|
472
|
+
Puedes encontrar todos estos materiales en la carpeta `posters/`, numerados del 79 al 121, cubriendo comparativas con Airflow, n8n, Zapier y más.
|
|
473
|
+
|
|
474
|
+
---
|
|
475
|
+
|
|
431
476
|
## DASHBOARD
|
|
432
477
|
|
|
433
478
|
### Tutorial
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tests for Background step functionality.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
import time
|
|
7
|
+
import threading
|
|
8
|
+
import asyncio
|
|
9
|
+
import sqlite3
|
|
10
|
+
import threading
|
|
11
|
+
from typing import Dict, Any
|
|
12
|
+
|
|
13
|
+
from wpipe import Pipeline
|
|
14
|
+
from wpipe.pipe.components.logic_blocks import Background
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# --- RE-PARCHE DE EMERGENCIA PARA ARREGLAR BUG DE WPIPE ---
|
|
18
|
+
from wsqlite import WSQLite
|
|
19
|
+
_db_connections = {}
|
|
20
|
+
_db_lock = threading.Lock()
|
|
21
|
+
|
|
22
|
+
def better_get_connection(self):
|
|
23
|
+
db_path = getattr(self, "db_path", getattr(self, "db_name", "register.db"))
|
|
24
|
+
with _db_lock:
|
|
25
|
+
if db_path not in _db_connections:
|
|
26
|
+
conn = sqlite3.connect(db_path, check_same_thread=False)
|
|
27
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
28
|
+
_db_connections[db_path] = conn
|
|
29
|
+
return _db_connections[db_path]
|
|
30
|
+
|
|
31
|
+
WSQLite._get_connection = better_get_connection
|
|
32
|
+
# --- END RE-PARCHE ---
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TestBackgroundStep:
|
|
36
|
+
"""Tests for Background step execution."""
|
|
37
|
+
|
|
38
|
+
def test_background_executes_without_blocking(self) -> None:
|
|
39
|
+
"""Background step should execute without blocking the pipeline."""
|
|
40
|
+
execution_order = []
|
|
41
|
+
|
|
42
|
+
def slow_task(data):
|
|
43
|
+
time.sleep(0.5)
|
|
44
|
+
execution_order.append("slow")
|
|
45
|
+
return {}
|
|
46
|
+
|
|
47
|
+
def fast_task(data):
|
|
48
|
+
execution_order.append("fast")
|
|
49
|
+
return {}
|
|
50
|
+
|
|
51
|
+
pipeline = Pipeline(verbose=False)
|
|
52
|
+
pipeline.set_steps([
|
|
53
|
+
(fast_task, "Fast", "v1.0"),
|
|
54
|
+
Background(slow_task),
|
|
55
|
+
])
|
|
56
|
+
|
|
57
|
+
start = time.time()
|
|
58
|
+
result = pipeline.run({})
|
|
59
|
+
elapsed = time.time() - start
|
|
60
|
+
|
|
61
|
+
assert "fast" in execution_order
|
|
62
|
+
assert elapsed < 0.5, "Pipeline should not wait for background task"
|
|
63
|
+
|
|
64
|
+
def test_background_ignores_return(self) -> None:
|
|
65
|
+
"""Background step return should be ignored by pipeline."""
|
|
66
|
+
def background_step(data):
|
|
67
|
+
return {"bg_value": "should_be_ignored"}
|
|
68
|
+
|
|
69
|
+
def check_step(data):
|
|
70
|
+
data["check_executed"] = True
|
|
71
|
+
return data
|
|
72
|
+
|
|
73
|
+
pipeline = Pipeline(verbose=False)
|
|
74
|
+
pipeline.set_steps([
|
|
75
|
+
Background(background_step),
|
|
76
|
+
check_step,
|
|
77
|
+
])
|
|
78
|
+
|
|
79
|
+
result = pipeline.run({})
|
|
80
|
+
|
|
81
|
+
assert result.get("check_executed") is True
|
|
82
|
+
assert "bg_value" not in result, "Background return should be ignored"
|
|
83
|
+
|
|
84
|
+
def test_background_failure_without_capture_error(self) -> None:
|
|
85
|
+
"""Background step failure with capture_error=False should not stop pipeline."""
|
|
86
|
+
def failing_background(data):
|
|
87
|
+
raise ValueError("Background task failed")
|
|
88
|
+
|
|
89
|
+
def continue_step(data):
|
|
90
|
+
data["continued"] = True
|
|
91
|
+
return data
|
|
92
|
+
|
|
93
|
+
pipeline = Pipeline(verbose=False)
|
|
94
|
+
pipeline.set_steps([
|
|
95
|
+
Background(failing_background, capture_error=False),
|
|
96
|
+
continue_step,
|
|
97
|
+
])
|
|
98
|
+
|
|
99
|
+
result = pipeline.run({})
|
|
100
|
+
|
|
101
|
+
assert result.get("continued") is True, "Pipeline should continue despite background failure"
|
|
102
|
+
|
|
103
|
+
def test_background_failure_with_capture_error(self) -> None:
|
|
104
|
+
"""Background step failure with capture_error=True should trigger error handler."""
|
|
105
|
+
import time
|
|
106
|
+
error_captured = []
|
|
107
|
+
|
|
108
|
+
def failing_background(data):
|
|
109
|
+
raise ValueError("Background task failed")
|
|
110
|
+
|
|
111
|
+
def error_handler(data, error_info):
|
|
112
|
+
error_captured.append(error_info.get("error") or error_info.get("error_message"))
|
|
113
|
+
|
|
114
|
+
def continue_step(data):
|
|
115
|
+
data["continued"] = True
|
|
116
|
+
return data
|
|
117
|
+
|
|
118
|
+
pipeline = Pipeline(verbose=False)
|
|
119
|
+
pipeline.add_error_capture([error_handler])
|
|
120
|
+
pipeline.set_steps([
|
|
121
|
+
Background(failing_background, capture_error=True),
|
|
122
|
+
continue_step,
|
|
123
|
+
])
|
|
124
|
+
|
|
125
|
+
result = pipeline.run({})
|
|
126
|
+
time.sleep(0.1)
|
|
127
|
+
|
|
128
|
+
assert result.get("continued") is True
|
|
129
|
+
assert len(error_captured) > 0
|
|
130
|
+
assert "Background task failed" in str(error_captured[0]) or "502" in str(error_captured[0])
|
|
131
|
+
|
|
132
|
+
def test_background_with_tuple_step(self) -> None:
|
|
133
|
+
"""Background should accept tuple steps (func, name, version)."""
|
|
134
|
+
import time
|
|
135
|
+
executed = []
|
|
136
|
+
|
|
137
|
+
def bg_task(data):
|
|
138
|
+
executed.append(True)
|
|
139
|
+
|
|
140
|
+
def check_task(data):
|
|
141
|
+
return data
|
|
142
|
+
|
|
143
|
+
pipeline = Pipeline(verbose=False)
|
|
144
|
+
pipeline.set_steps([
|
|
145
|
+
Background((bg_task, "My Background", "v1.0")),
|
|
146
|
+
check_task,
|
|
147
|
+
])
|
|
148
|
+
|
|
149
|
+
result = pipeline.run({})
|
|
150
|
+
time.sleep(0.1)
|
|
151
|
+
|
|
152
|
+
assert len(executed) == 1
|
|
153
|
+
|
|
154
|
+
def test_multiple_background_steps(self) -> None:
|
|
155
|
+
"""Multiple background steps should all execute."""
|
|
156
|
+
import time
|
|
157
|
+
counter = {"value": 0}
|
|
158
|
+
|
|
159
|
+
def increment(data):
|
|
160
|
+
counter["value"] += 1
|
|
161
|
+
|
|
162
|
+
def check(data):
|
|
163
|
+
data["counter"] = counter["value"]
|
|
164
|
+
return data
|
|
165
|
+
|
|
166
|
+
pipeline = Pipeline(verbose=False)
|
|
167
|
+
pipeline.set_steps([
|
|
168
|
+
Background(increment),
|
|
169
|
+
Background(increment),
|
|
170
|
+
Background(increment),
|
|
171
|
+
check,
|
|
172
|
+
])
|
|
173
|
+
|
|
174
|
+
pipeline.run({})
|
|
175
|
+
time.sleep(0.3)
|
|
176
|
+
|
|
177
|
+
assert counter["value"] == 3
|
|
178
|
+
|
|
179
|
+
def test_background_with_data_copy(self) -> None:
|
|
180
|
+
"""Background should receive a copy of data, not the original."""
|
|
181
|
+
original_data = {}
|
|
182
|
+
|
|
183
|
+
def background_step(data):
|
|
184
|
+
data["modified_in_bg"] = True
|
|
185
|
+
|
|
186
|
+
def check_step(data):
|
|
187
|
+
return data
|
|
188
|
+
|
|
189
|
+
pipeline = Pipeline(verbose=False)
|
|
190
|
+
pipeline.set_steps([
|
|
191
|
+
Background(background_step),
|
|
192
|
+
check_step,
|
|
193
|
+
])
|
|
194
|
+
|
|
195
|
+
result = pipeline.run({"initial": True})
|
|
196
|
+
|
|
197
|
+
assert "modified_in_bg" not in result
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
class TestBackgroundAsync:
|
|
201
|
+
"""Tests for Background step in async pipeline."""
|
|
202
|
+
|
|
203
|
+
def test_async_background_executes_without_blocking(self) -> None:
|
|
204
|
+
"""Background step should execute without blocking async pipeline."""
|
|
205
|
+
from wpipe import PipelineAsync
|
|
206
|
+
|
|
207
|
+
execution_order = []
|
|
208
|
+
|
|
209
|
+
async def slow_async_task(data):
|
|
210
|
+
await asyncio.sleep(0.3)
|
|
211
|
+
execution_order.append("slow")
|
|
212
|
+
return {}
|
|
213
|
+
|
|
214
|
+
async def fast_task(data):
|
|
215
|
+
execution_order.append("fast")
|
|
216
|
+
return {}
|
|
217
|
+
|
|
218
|
+
pipeline = PipelineAsync(verbose=False)
|
|
219
|
+
pipeline.set_steps([
|
|
220
|
+
(fast_task, "Fast", "v1.0"),
|
|
221
|
+
Background(slow_async_task),
|
|
222
|
+
])
|
|
223
|
+
|
|
224
|
+
start = time.time()
|
|
225
|
+
result = asyncio.run(pipeline.run({}))
|
|
226
|
+
elapsed = time.time() - start
|
|
227
|
+
|
|
228
|
+
assert "fast" in execution_order
|
|
229
|
+
assert elapsed < 0.4, f"Async pipeline should not wait for background task (took {elapsed}s)"
|
|
230
|
+
|
|
231
|
+
def test_async_background_failure_with_capture(self) -> None:
|
|
232
|
+
"""Async background with capture_error=True should trigger error handler."""
|
|
233
|
+
from wpipe import PipelineAsync
|
|
234
|
+
|
|
235
|
+
error_captured = []
|
|
236
|
+
|
|
237
|
+
async def failing_background(data):
|
|
238
|
+
raise ValueError("Async background failed")
|
|
239
|
+
|
|
240
|
+
def error_handler(data, error_info):
|
|
241
|
+
error_captured.append(error_info.get("error") or error_info.get("error_message"))
|
|
242
|
+
|
|
243
|
+
pipeline = PipelineAsync(verbose=False)
|
|
244
|
+
pipeline.add_error_capture([error_handler])
|
|
245
|
+
pipeline.set_steps([
|
|
246
|
+
Background(failing_background, capture_error=True),
|
|
247
|
+
])
|
|
248
|
+
|
|
249
|
+
result = asyncio.run(pipeline.run({}))
|
|
250
|
+
|
|
251
|
+
assert len(error_captured) > 0
|
|
252
|
+
assert "Async background failed" in str(error_captured[0]) or "502" in str(error_captured[0])
|
|
@@ -193,6 +193,43 @@ class For:
|
|
|
193
193
|
return False
|
|
194
194
|
|
|
195
195
|
|
|
196
|
+
class Background:
|
|
197
|
+
"""
|
|
198
|
+
A background task that executes without blocking the pipeline.
|
|
199
|
+
|
|
200
|
+
The step runs in a separate thread and the pipeline continues immediately
|
|
201
|
+
without waiting for completion. The return value is ignored.
|
|
202
|
+
|
|
203
|
+
Attributes:
|
|
204
|
+
step: The step to execute in background (callable or tuple).
|
|
205
|
+
capture_error: If True and step fails, run error capture handlers.
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
def __init__(self, step: Any, capture_error: bool = False) -> None:
|
|
209
|
+
"""
|
|
210
|
+
Initialize a Background block.
|
|
211
|
+
|
|
212
|
+
Args:
|
|
213
|
+
step: The step to execute in background (function, class, or tuple).
|
|
214
|
+
capture_error: Whether to run error handlers if the step fails.
|
|
215
|
+
"""
|
|
216
|
+
self.step = step
|
|
217
|
+
self.capture_error: bool = capture_error
|
|
218
|
+
|
|
219
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
220
|
+
"""
|
|
221
|
+
Convert the block to a dictionary for serialization.
|
|
222
|
+
|
|
223
|
+
Returns:
|
|
224
|
+
Dict[str, Any]: Serialized representation of the background block.
|
|
225
|
+
"""
|
|
226
|
+
return {
|
|
227
|
+
"type": "background",
|
|
228
|
+
"capture_error": self.capture_error,
|
|
229
|
+
"step": _serialize_step(self.step),
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
|
|
196
233
|
class Parallel:
|
|
197
234
|
"""
|
|
198
235
|
Represents a parallel execution block in the pipeline.
|
|
@@ -20,7 +20,7 @@ from wpipe.exception.api_error import logger
|
|
|
20
20
|
from wpipe.tracking import PipelineTracker
|
|
21
21
|
from wpipe.util.utils import clean_for_json
|
|
22
22
|
|
|
23
|
-
from .components.logic_blocks import Condition, For, Parallel
|
|
23
|
+
from .components.logic_blocks import Background, Condition, For, Parallel
|
|
24
24
|
from .components.metrics import SystemMetricsCollector
|
|
25
25
|
from .components.progress import ProgressManager
|
|
26
26
|
|
|
@@ -520,6 +520,23 @@ class Pipeline(APIClient):
|
|
|
520
520
|
max_workers=item.max_workers,
|
|
521
521
|
use_processes=item.use_processes,
|
|
522
522
|
))
|
|
523
|
+
elif isinstance(item, Background):
|
|
524
|
+
normalized_step = normalize_step(item.step)
|
|
525
|
+
if isinstance(normalized_step, tuple):
|
|
526
|
+
bg_step = list(normalized_step)
|
|
527
|
+
if len(bg_step) >= 4 and isinstance(bg_step[3], dict):
|
|
528
|
+
bg_step[3] = {**bg_step[3], "_is_background": True, "_background_capture_error": item.capture_error}
|
|
529
|
+
elif len(bg_step) == 3:
|
|
530
|
+
bg_step.append({"_is_background": True, "_background_capture_error": item.capture_error})
|
|
531
|
+
else:
|
|
532
|
+
bg_step = [normalized_step[0], normalized_step[1] if len(normalized_step) > 1 else "background",
|
|
533
|
+
normalized_step[2] if len(normalized_step) > 2 else "v1.0",
|
|
534
|
+
{"_is_background": True, "_background_capture_error": item.capture_error}]
|
|
535
|
+
else:
|
|
536
|
+
name = getattr(normalized_step, "NAME", getattr(normalized_step, "__name__", "background"))
|
|
537
|
+
version = getattr(normalized_step, "VERSION", "v1.0")
|
|
538
|
+
bg_step = [normalized_step, name, version, {"_is_background": True, "_background_capture_error": item.capture_error}]
|
|
539
|
+
new_list.append(tuple(bg_step))
|
|
523
540
|
elif isinstance(item, Pipeline):
|
|
524
541
|
new_list.append((item, item.pipeline_name or "SubPipeline", "v1.0", {}))
|
|
525
542
|
elif callable(item):
|
|
@@ -772,6 +789,15 @@ class Pipeline(APIClient):
|
|
|
772
789
|
if isinstance(item, Parallel):
|
|
773
790
|
return self._execute_parallel(item, data, parent_step_id, parallel_group, **kwargs)
|
|
774
791
|
|
|
792
|
+
is_background = False
|
|
793
|
+
capture_error = False
|
|
794
|
+
if isinstance(item, tuple) and len(item) >= 4 and isinstance(item[3], dict):
|
|
795
|
+
is_background = item[3].get("_is_background", False)
|
|
796
|
+
capture_error = item[3].get("_background_capture_error", False)
|
|
797
|
+
|
|
798
|
+
if is_background:
|
|
799
|
+
return self._execute_background_step(item, data, capture_error, parent_step_id, parallel_group, **kwargs)
|
|
800
|
+
|
|
775
801
|
return self._execute_task_step(item, data, parent_step_id, parallel_group, **kwargs)
|
|
776
802
|
|
|
777
803
|
def _execute_parallel(
|
|
@@ -859,6 +885,47 @@ class Pipeline(APIClient):
|
|
|
859
885
|
self._end_step_tracking(tracked_id, data if "error" not in data else None, data.get("error"))
|
|
860
886
|
return data
|
|
861
887
|
|
|
888
|
+
def _execute_background_step(
|
|
889
|
+
self,
|
|
890
|
+
item: Any,
|
|
891
|
+
data: Dict[str, Any],
|
|
892
|
+
capture_error: bool,
|
|
893
|
+
parent_step_id: Optional[int],
|
|
894
|
+
parallel_group: Optional[str],
|
|
895
|
+
**kwargs: Any
|
|
896
|
+
) -> Dict[str, Any]:
|
|
897
|
+
"""Execute a background step without blocking the pipeline."""
|
|
898
|
+
import threading as bg_thread
|
|
899
|
+
|
|
900
|
+
func, name, version, step_id, step_meta = None, "background", "v1.0", None, {}
|
|
901
|
+
if isinstance(item, tuple):
|
|
902
|
+
if len(item) >= 2:
|
|
903
|
+
func, name, version = item[0], item[1], item[2] if len(item) > 2 else "v1.0"
|
|
904
|
+
if len(item) >= 4 and isinstance(item[3], dict):
|
|
905
|
+
step_meta = item[3]
|
|
906
|
+
elif callable(item):
|
|
907
|
+
func = item
|
|
908
|
+
name = getattr(item, "NAME", getattr(item, "__name__", "background"))
|
|
909
|
+
|
|
910
|
+
task_data = data.copy()
|
|
911
|
+
task_data.pop("progress_rich", None)
|
|
912
|
+
|
|
913
|
+
def run_background():
|
|
914
|
+
try:
|
|
915
|
+
if func:
|
|
916
|
+
self._task_invoke(func, name, task_data, parent_step_id=parent_step_id, parallel_group=parallel_group, **kwargs)
|
|
917
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
918
|
+
if capture_error:
|
|
919
|
+
error_info = {"step_name": name, "error": str(e), "error_message": str(e)}
|
|
920
|
+
self._execute_error_capture(task_data, error_info)
|
|
921
|
+
if self.verbose:
|
|
922
|
+
print(f"[BACKGROUND ERROR] {name}: {e}")
|
|
923
|
+
|
|
924
|
+
# Use daemon thread so process can exit without waiting
|
|
925
|
+
bg_thread.Thread(target=run_background, daemon=True, name=f"wpipe_bg_{name}").start()
|
|
926
|
+
|
|
927
|
+
return data
|
|
928
|
+
|
|
862
929
|
def _execute_task_step(
|
|
863
930
|
self,
|
|
864
931
|
item: Any,
|
|
@@ -16,7 +16,8 @@ from wpipe.api_client.api_client import APIClient
|
|
|
16
16
|
from wpipe.exception import Codes, TaskError
|
|
17
17
|
from wpipe.tracking import PipelineTracker
|
|
18
18
|
|
|
19
|
-
from .pipe import Condition, Parallel, SystemMetricsCollector
|
|
19
|
+
from .pipe import Background, Condition, Parallel, SystemMetricsCollector
|
|
20
|
+
from .components.logic_blocks import For
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
def _is_async_callable(func: Any) -> bool:
|
|
@@ -418,6 +419,15 @@ class PipelineAsync(APIClient):
|
|
|
418
419
|
if isinstance(item, Parallel):
|
|
419
420
|
return await self._execute_parallel(item, data, parent_step_id, parallel_group, **kwargs)
|
|
420
421
|
|
|
422
|
+
is_background = False
|
|
423
|
+
capture_error = False
|
|
424
|
+
if isinstance(item, tuple) and len(item) >= 4 and isinstance(item[3], dict):
|
|
425
|
+
is_background = item[3].get("_is_background", False)
|
|
426
|
+
capture_error = item[3].get("_background_capture_error", False)
|
|
427
|
+
|
|
428
|
+
if is_background:
|
|
429
|
+
return await self._execute_background_step(item, data, capture_error, parent_step_id, parallel_group, **kwargs)
|
|
430
|
+
|
|
421
431
|
return await self._execute_task(item, data, parent_step_id, parallel_group, **kwargs)
|
|
422
432
|
|
|
423
433
|
async def _execute_parallel(
|
|
@@ -470,6 +480,54 @@ class PipelineAsync(APIClient):
|
|
|
470
480
|
self._end_step_tracking(tracked_parallel_id, data if not error_msg else None, error_msg)
|
|
471
481
|
return data
|
|
472
482
|
|
|
483
|
+
async def _execute_error_capture(self, data: Dict[str, Any], error_info: Dict[str, Any]) -> None:
|
|
484
|
+
"""Execute error capture handlers."""
|
|
485
|
+
if not self._error_capture_tasks:
|
|
486
|
+
return
|
|
487
|
+
|
|
488
|
+
if self.verbose:
|
|
489
|
+
print(f"\n[ERROR CAPTURE] Processing error in state '{error_info.get('step_name', 'unknown')}'...")
|
|
490
|
+
|
|
491
|
+
for handler in self._error_capture_tasks:
|
|
492
|
+
try:
|
|
493
|
+
if _is_async_callable(handler):
|
|
494
|
+
await handler(data, error_info)
|
|
495
|
+
else:
|
|
496
|
+
handler(data, error_info)
|
|
497
|
+
except Exception: # pylint: disable=broad-exception-caught
|
|
498
|
+
pass
|
|
499
|
+
|
|
500
|
+
async def _execute_background_step(
|
|
501
|
+
self,
|
|
502
|
+
item: Any,
|
|
503
|
+
data: Dict[str, Any],
|
|
504
|
+
capture_error: bool,
|
|
505
|
+
parent_step_id: Optional[int],
|
|
506
|
+
parallel_group: Optional[str],
|
|
507
|
+
**kwargs: Any
|
|
508
|
+
) -> Dict[str, Any]:
|
|
509
|
+
"""Execute a background step without blocking the pipeline."""
|
|
510
|
+
func, name, version = None, "background", "v1.0"
|
|
511
|
+
if isinstance(item, tuple) and len(item) >= 2:
|
|
512
|
+
func, name, version = item[0], item[1], item[2] if len(item) > 2 else "v1.0"
|
|
513
|
+
|
|
514
|
+
task_data = data.copy()
|
|
515
|
+
task_data.pop("progress_rich", None)
|
|
516
|
+
|
|
517
|
+
async def run_background():
|
|
518
|
+
try:
|
|
519
|
+
if func:
|
|
520
|
+
await self._execute_task(func, task_data, parent_step_id, parallel_group, **kwargs)
|
|
521
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
522
|
+
if capture_error:
|
|
523
|
+
error_info = {"step_name": name, "error": str(e)}
|
|
524
|
+
await self._execute_error_capture(task_data, error_info)
|
|
525
|
+
if self.verbose:
|
|
526
|
+
print(f"[BACKGROUND ERROR] {name}: {e}")
|
|
527
|
+
|
|
528
|
+
asyncio.create_task(run_background())
|
|
529
|
+
return data
|
|
530
|
+
|
|
473
531
|
async def _execute_task(
|
|
474
532
|
self,
|
|
475
533
|
item: Any,
|
|
@@ -597,4 +655,62 @@ class PipelineAsync(APIClient):
|
|
|
597
655
|
Args:
|
|
598
656
|
steps: List of steps to be executed.
|
|
599
657
|
"""
|
|
600
|
-
|
|
658
|
+
new_list = []
|
|
659
|
+
|
|
660
|
+
def normalize_step(step: Any) -> Any:
|
|
661
|
+
"""Normalize a step tuple to 4 elements (func, name, version, metadata)."""
|
|
662
|
+
if isinstance(step, tuple):
|
|
663
|
+
if len(step) == 3 and callable(step[0]):
|
|
664
|
+
if isinstance(step[2], dict):
|
|
665
|
+
return (step[0], step[1], "v1.0", step[2])
|
|
666
|
+
return (step[0], step[1], step[2], {})
|
|
667
|
+
if len(step) == 2 and callable(step[0]):
|
|
668
|
+
return (step[0], step[1], "v1.0", {})
|
|
669
|
+
if len(step) == 4 and callable(step[0]):
|
|
670
|
+
return step
|
|
671
|
+
return step
|
|
672
|
+
|
|
673
|
+
for item in steps:
|
|
674
|
+
if isinstance(item, Condition):
|
|
675
|
+
normalized_true = [normalize_step(s) for s in item.branch_true]
|
|
676
|
+
normalized_false = [normalize_step(s) for s in item.branch_false]
|
|
677
|
+
new_list.append(Condition(
|
|
678
|
+
expression=item.expression,
|
|
679
|
+
branch_true=normalized_true,
|
|
680
|
+
branch_false=normalized_false,
|
|
681
|
+
))
|
|
682
|
+
elif isinstance(item, For):
|
|
683
|
+
normalized_steps = [normalize_step(s) for s in item.steps]
|
|
684
|
+
new_list.append(For(
|
|
685
|
+
validation_expression=item.validation_expression,
|
|
686
|
+
iterations=item.iterations,
|
|
687
|
+
steps=normalized_steps,
|
|
688
|
+
))
|
|
689
|
+
elif isinstance(item, Parallel):
|
|
690
|
+
normalized_steps = [normalize_step(s) for s in item.steps]
|
|
691
|
+
new_list.append(Parallel(
|
|
692
|
+
steps=normalized_steps,
|
|
693
|
+
max_workers=item.max_workers,
|
|
694
|
+
use_processes=item.use_processes,
|
|
695
|
+
))
|
|
696
|
+
elif isinstance(item, Background):
|
|
697
|
+
normalized_step = normalize_step(item.step)
|
|
698
|
+
if isinstance(normalized_step, tuple):
|
|
699
|
+
bg_step = list(normalized_step)
|
|
700
|
+
else:
|
|
701
|
+
name = getattr(normalized_step, "NAME", getattr(normalized_step, "__name__", "background"))
|
|
702
|
+
version = getattr(normalized_step, "VERSION", "v1.0")
|
|
703
|
+
bg_step = [normalized_step, name, version, {}]
|
|
704
|
+
bg_step.append({"_is_background": True, "_background_capture_error": item.capture_error})
|
|
705
|
+
new_list.append(tuple(bg_step))
|
|
706
|
+
elif callable(item):
|
|
707
|
+
name = getattr(item, "NAME", getattr(item, "__name__", "unknown"))
|
|
708
|
+
version = getattr(item, "VERSION", "v1.0")
|
|
709
|
+
meta = getattr(item, "_wpipe_metadata", {})
|
|
710
|
+
new_list.append((item, name, version, meta))
|
|
711
|
+
elif isinstance(item, tuple) and len(item) >= 2 and callable(item[0]):
|
|
712
|
+
new_list.append(normalize_step(item))
|
|
713
|
+
else:
|
|
714
|
+
new_list.append(item)
|
|
715
|
+
|
|
716
|
+
self.tasks_list = new_list
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|