horsies 0.1.0a3__py3-none-any.whl → 0.1.0a5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- horsies/core/app.py +67 -47
- horsies/core/banner.py +27 -27
- horsies/core/brokers/postgres.py +315 -288
- horsies/core/cli.py +7 -2
- horsies/core/errors.py +3 -0
- horsies/core/models/app.py +87 -64
- horsies/core/models/recovery.py +30 -21
- horsies/core/models/schedule.py +30 -19
- horsies/core/models/tasks.py +1 -0
- horsies/core/models/workflow.py +489 -202
- horsies/core/models/workflow_pg.py +3 -1
- horsies/core/scheduler/service.py +5 -1
- horsies/core/scheduler/state.py +39 -27
- horsies/core/task_decorator.py +138 -0
- horsies/core/types/status.py +14 -12
- horsies/core/utils/imports.py +10 -10
- horsies/core/worker/worker.py +197 -139
- horsies/core/workflows/engine.py +487 -352
- horsies/core/workflows/recovery.py +148 -119
- {horsies-0.1.0a3.dist-info → horsies-0.1.0a5.dist-info}/METADATA +1 -1
- horsies-0.1.0a5.dist-info/RECORD +42 -0
- horsies-0.1.0a3.dist-info/RECORD +0 -42
- {horsies-0.1.0a3.dist-info → horsies-0.1.0a5.dist-info}/WHEEL +0 -0
- {horsies-0.1.0a3.dist-info → horsies-0.1.0a5.dist-info}/entry_points.txt +0 -0
- {horsies-0.1.0a3.dist-info → horsies-0.1.0a5.dist-info}/top_level.txt +0 -0
horsies/core/workflows/engine.py
CHANGED
|
@@ -51,6 +51,48 @@ def _as_str_list(value: object) -> list[str]:
|
|
|
51
51
|
return str_items
|
|
52
52
|
|
|
53
53
|
|
|
54
|
+
# -- SQL constants for start_workflow_async --
|
|
55
|
+
|
|
56
|
+
CHECK_WORKFLOW_EXISTS_SQL = text(
|
|
57
|
+
"""SELECT id FROM horsies_workflows WHERE id = :wf_id"""
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
INSERT_WORKFLOW_SQL = text("""
|
|
61
|
+
INSERT INTO horsies_workflows (id, name, status, on_error, output_task_index,
|
|
62
|
+
success_policy, workflow_def_module, workflow_def_qualname,
|
|
63
|
+
depth, root_workflow_id,
|
|
64
|
+
created_at, started_at, updated_at)
|
|
65
|
+
VALUES (:id, :name, 'RUNNING', :on_error, :output_idx,
|
|
66
|
+
:success_policy, :wf_module, :wf_qualname,
|
|
67
|
+
0, :id,
|
|
68
|
+
NOW(), NOW(), NOW())
|
|
69
|
+
""")
|
|
70
|
+
|
|
71
|
+
INSERT_WORKFLOW_TASK_SUBWORKFLOW_SQL = text("""
|
|
72
|
+
INSERT INTO horsies_workflow_tasks
|
|
73
|
+
(id, workflow_id, task_index, node_id, task_name, task_args, task_kwargs,
|
|
74
|
+
queue_name, priority, dependencies, args_from, workflow_ctx_from,
|
|
75
|
+
allow_failed_deps, join_type, min_success, task_options, status,
|
|
76
|
+
is_subworkflow, sub_workflow_name, sub_workflow_retry_mode,
|
|
77
|
+
sub_workflow_module, sub_workflow_qualname, created_at)
|
|
78
|
+
VALUES (:id, :wf_id, :idx, :node_id, :name, :args, :kwargs, :queue, :priority,
|
|
79
|
+
:deps, :args_from, :ctx_from, :allow_failed, :join_type, :min_success,
|
|
80
|
+
:task_options, :status, TRUE, :sub_wf_name, :sub_wf_retry_mode,
|
|
81
|
+
:sub_wf_module, :sub_wf_qualname, NOW())
|
|
82
|
+
""")
|
|
83
|
+
|
|
84
|
+
INSERT_WORKFLOW_TASK_SQL = text("""
|
|
85
|
+
INSERT INTO horsies_workflow_tasks
|
|
86
|
+
(id, workflow_id, task_index, node_id, task_name, task_args, task_kwargs,
|
|
87
|
+
queue_name, priority, dependencies, args_from, workflow_ctx_from,
|
|
88
|
+
allow_failed_deps, join_type, min_success, task_options, status,
|
|
89
|
+
is_subworkflow, created_at)
|
|
90
|
+
VALUES (:id, :wf_id, :idx, :node_id, :name, :args, :kwargs, :queue, :priority,
|
|
91
|
+
:deps, :args_from, :ctx_from, :allow_failed, :join_type, :min_success,
|
|
92
|
+
:task_options, :status, FALSE, NOW())
|
|
93
|
+
""")
|
|
94
|
+
|
|
95
|
+
|
|
54
96
|
async def start_workflow_async(
|
|
55
97
|
spec: 'WorkflowSpec',
|
|
56
98
|
broker: 'PostgresBroker',
|
|
@@ -106,7 +148,7 @@ async def start_workflow_async(
|
|
|
106
148
|
# Check if workflow already exists (idempotent start)
|
|
107
149
|
if workflow_id:
|
|
108
150
|
existing = await session.execute(
|
|
109
|
-
|
|
151
|
+
CHECK_WORKFLOW_EXISTS_SQL,
|
|
110
152
|
{'wf_id': wf_id},
|
|
111
153
|
)
|
|
112
154
|
if existing.fetchone():
|
|
@@ -137,16 +179,7 @@ async def start_workflow_async(
|
|
|
137
179
|
]
|
|
138
180
|
|
|
139
181
|
await session.execute(
|
|
140
|
-
|
|
141
|
-
INSERT INTO horsies_workflows (id, name, status, on_error, output_task_index,
|
|
142
|
-
success_policy, workflow_def_module, workflow_def_qualname,
|
|
143
|
-
depth, root_workflow_id,
|
|
144
|
-
created_at, started_at, updated_at)
|
|
145
|
-
VALUES (:id, :name, 'RUNNING', :on_error, :output_idx,
|
|
146
|
-
:success_policy, :wf_module, :wf_qualname,
|
|
147
|
-
0, :id,
|
|
148
|
-
NOW(), NOW(), NOW())
|
|
149
|
-
"""),
|
|
182
|
+
INSERT_WORKFLOW_SQL,
|
|
150
183
|
{
|
|
151
184
|
'id': wf_id,
|
|
152
185
|
'name': spec.name,
|
|
@@ -177,18 +210,7 @@ async def start_workflow_async(
|
|
|
177
210
|
if isinstance(node, SubWorkflowNode):
|
|
178
211
|
# SubWorkflowNode: no fn, queue, priority, good_until
|
|
179
212
|
await session.execute(
|
|
180
|
-
|
|
181
|
-
INSERT INTO horsies_workflow_tasks
|
|
182
|
-
(id, workflow_id, task_index, node_id, task_name, task_args, task_kwargs,
|
|
183
|
-
queue_name, priority, dependencies, args_from, workflow_ctx_from,
|
|
184
|
-
allow_failed_deps, join_type, min_success, task_options, status,
|
|
185
|
-
is_subworkflow, sub_workflow_name, sub_workflow_retry_mode,
|
|
186
|
-
sub_workflow_module, sub_workflow_qualname, created_at)
|
|
187
|
-
VALUES (:id, :wf_id, :idx, :node_id, :name, :args, :kwargs, :queue, :priority,
|
|
188
|
-
:deps, :args_from, :ctx_from, :allow_failed, :join_type, :min_success,
|
|
189
|
-
:task_options, :status, TRUE, :sub_wf_name, :sub_wf_retry_mode,
|
|
190
|
-
:sub_wf_module, :sub_wf_qualname, NOW())
|
|
191
|
-
"""),
|
|
213
|
+
INSERT_WORKFLOW_TASK_SUBWORKFLOW_SQL,
|
|
192
214
|
{
|
|
193
215
|
'id': wt_id,
|
|
194
216
|
'wf_id': wf_id,
|
|
@@ -235,16 +257,7 @@ async def start_workflow_async(
|
|
|
235
257
|
task_options_json = dumps_json(base_options)
|
|
236
258
|
|
|
237
259
|
await session.execute(
|
|
238
|
-
|
|
239
|
-
INSERT INTO horsies_workflow_tasks
|
|
240
|
-
(id, workflow_id, task_index, node_id, task_name, task_args, task_kwargs,
|
|
241
|
-
queue_name, priority, dependencies, args_from, workflow_ctx_from,
|
|
242
|
-
allow_failed_deps, join_type, min_success, task_options, status,
|
|
243
|
-
is_subworkflow, created_at)
|
|
244
|
-
VALUES (:id, :wf_id, :idx, :node_id, :name, :args, :kwargs, :queue, :priority,
|
|
245
|
-
:deps, :args_from, :ctx_from, :allow_failed, :join_type, :min_success,
|
|
246
|
-
:task_options, :status, FALSE, NOW())
|
|
247
|
-
"""),
|
|
260
|
+
INSERT_WORKFLOW_TASK_SQL,
|
|
248
261
|
{
|
|
249
262
|
'id': wt_id,
|
|
250
263
|
'wf_id': wf_id,
|
|
@@ -317,6 +330,18 @@ def start_workflow(
|
|
|
317
330
|
runner.stop()
|
|
318
331
|
|
|
319
332
|
|
|
333
|
+
# -- SQL constants for pause_workflow --
|
|
334
|
+
|
|
335
|
+
PAUSE_WORKFLOW_SQL = text("""
|
|
336
|
+
UPDATE horsies_workflows
|
|
337
|
+
SET status = 'PAUSED', updated_at = NOW()
|
|
338
|
+
WHERE id = :wf_id AND status = 'RUNNING'
|
|
339
|
+
RETURNING id
|
|
340
|
+
""")
|
|
341
|
+
|
|
342
|
+
NOTIFY_WORKFLOW_DONE_SQL = text("""SELECT pg_notify('workflow_done', :wf_id)""")
|
|
343
|
+
|
|
344
|
+
|
|
320
345
|
async def pause_workflow(
|
|
321
346
|
broker: 'PostgresBroker',
|
|
322
347
|
workflow_id: str,
|
|
@@ -343,12 +368,7 @@ async def pause_workflow(
|
|
|
343
368
|
"""
|
|
344
369
|
async with broker.session_factory() as session:
|
|
345
370
|
result = await session.execute(
|
|
346
|
-
|
|
347
|
-
UPDATE horsies_workflows
|
|
348
|
-
SET status = 'PAUSED', updated_at = NOW()
|
|
349
|
-
WHERE id = :wf_id AND status = 'RUNNING'
|
|
350
|
-
RETURNING id
|
|
351
|
-
"""),
|
|
371
|
+
PAUSE_WORKFLOW_SQL,
|
|
352
372
|
{'wf_id': workflow_id},
|
|
353
373
|
)
|
|
354
374
|
row = result.fetchone()
|
|
@@ -360,7 +380,7 @@ async def pause_workflow(
|
|
|
360
380
|
|
|
361
381
|
# Notify clients of pause (so get() returns immediately with WORKFLOW_PAUSED)
|
|
362
382
|
await session.execute(
|
|
363
|
-
|
|
383
|
+
NOTIFY_WORKFLOW_DONE_SQL,
|
|
364
384
|
{'wf_id': workflow_id},
|
|
365
385
|
)
|
|
366
386
|
|
|
@@ -368,6 +388,20 @@ async def pause_workflow(
|
|
|
368
388
|
return True
|
|
369
389
|
|
|
370
390
|
|
|
391
|
+
# -- SQL constants for _cascade_pause_to_children --
|
|
392
|
+
|
|
393
|
+
GET_RUNNING_CHILD_WORKFLOWS_SQL = text("""
|
|
394
|
+
SELECT id FROM horsies_workflows
|
|
395
|
+
WHERE parent_workflow_id = :wf_id AND status = 'RUNNING'
|
|
396
|
+
""")
|
|
397
|
+
|
|
398
|
+
PAUSE_CHILD_WORKFLOW_SQL = text("""
|
|
399
|
+
UPDATE horsies_workflows
|
|
400
|
+
SET status = 'PAUSED', updated_at = NOW()
|
|
401
|
+
WHERE id = :wf_id AND status = 'RUNNING'
|
|
402
|
+
""")
|
|
403
|
+
|
|
404
|
+
|
|
371
405
|
async def _cascade_pause_to_children(
|
|
372
406
|
session: AsyncSession,
|
|
373
407
|
workflow_id: str,
|
|
@@ -383,10 +417,7 @@ async def _cascade_pause_to_children(
|
|
|
383
417
|
|
|
384
418
|
# Find running child workflows
|
|
385
419
|
children = await session.execute(
|
|
386
|
-
|
|
387
|
-
SELECT id FROM horsies_workflows
|
|
388
|
-
WHERE parent_workflow_id = :wf_id AND status = 'RUNNING'
|
|
389
|
-
"""),
|
|
420
|
+
GET_RUNNING_CHILD_WORKFLOWS_SQL,
|
|
390
421
|
{'wf_id': current_id},
|
|
391
422
|
)
|
|
392
423
|
|
|
@@ -395,17 +426,13 @@ async def _cascade_pause_to_children(
|
|
|
395
426
|
|
|
396
427
|
# Pause child
|
|
397
428
|
await session.execute(
|
|
398
|
-
|
|
399
|
-
UPDATE horsies_workflows
|
|
400
|
-
SET status = 'PAUSED', updated_at = NOW()
|
|
401
|
-
WHERE id = :wf_id AND status = 'RUNNING'
|
|
402
|
-
"""),
|
|
429
|
+
PAUSE_CHILD_WORKFLOW_SQL,
|
|
403
430
|
{'wf_id': child_id},
|
|
404
431
|
)
|
|
405
432
|
|
|
406
433
|
# Notify of child pause
|
|
407
434
|
await session.execute(
|
|
408
|
-
|
|
435
|
+
NOTIFY_WORKFLOW_DONE_SQL,
|
|
409
436
|
{'wf_id': child_id},
|
|
410
437
|
)
|
|
411
438
|
|
|
@@ -438,6 +465,26 @@ def pause_workflow_sync(
|
|
|
438
465
|
runner.stop()
|
|
439
466
|
|
|
440
467
|
|
|
468
|
+
# -- SQL constants for resume_workflow --
|
|
469
|
+
|
|
470
|
+
RESUME_WORKFLOW_SQL = text("""
|
|
471
|
+
UPDATE horsies_workflows
|
|
472
|
+
SET status = 'RUNNING', updated_at = NOW()
|
|
473
|
+
WHERE id = :wf_id AND status = 'PAUSED'
|
|
474
|
+
RETURNING id, depth, root_workflow_id
|
|
475
|
+
""")
|
|
476
|
+
|
|
477
|
+
GET_PENDING_WORKFLOW_TASKS_SQL = text("""
|
|
478
|
+
SELECT task_index FROM horsies_workflow_tasks
|
|
479
|
+
WHERE workflow_id = :wf_id AND status = 'PENDING'
|
|
480
|
+
""")
|
|
481
|
+
|
|
482
|
+
GET_READY_WORKFLOW_TASKS_SQL = text("""
|
|
483
|
+
SELECT task_index, dependencies, is_subworkflow FROM horsies_workflow_tasks
|
|
484
|
+
WHERE workflow_id = :wf_id AND status = 'READY'
|
|
485
|
+
""")
|
|
486
|
+
|
|
487
|
+
|
|
441
488
|
async def resume_workflow(
|
|
442
489
|
broker: 'PostgresBroker',
|
|
443
490
|
workflow_id: str,
|
|
@@ -460,12 +507,7 @@ async def resume_workflow(
|
|
|
460
507
|
async with broker.session_factory() as session:
|
|
461
508
|
# 1. Transition PAUSED → RUNNING (only if currently PAUSED)
|
|
462
509
|
result = await session.execute(
|
|
463
|
-
|
|
464
|
-
UPDATE horsies_workflows
|
|
465
|
-
SET status = 'RUNNING', updated_at = NOW()
|
|
466
|
-
WHERE id = :wf_id AND status = 'PAUSED'
|
|
467
|
-
RETURNING id, depth, root_workflow_id
|
|
468
|
-
"""),
|
|
510
|
+
RESUME_WORKFLOW_SQL,
|
|
469
511
|
{'wf_id': workflow_id},
|
|
470
512
|
)
|
|
471
513
|
row = result.fetchone()
|
|
@@ -478,10 +520,7 @@ async def resume_workflow(
|
|
|
478
520
|
|
|
479
521
|
# 2. Find all PENDING tasks and try to make them READY
|
|
480
522
|
pending_result = await session.execute(
|
|
481
|
-
|
|
482
|
-
SELECT task_index FROM horsies_workflow_tasks
|
|
483
|
-
WHERE workflow_id = :wf_id AND status = 'PENDING'
|
|
484
|
-
"""),
|
|
523
|
+
GET_PENDING_WORKFLOW_TASKS_SQL,
|
|
485
524
|
{'wf_id': workflow_id},
|
|
486
525
|
)
|
|
487
526
|
pending_indices = [r[0] for r in pending_result.fetchall()]
|
|
@@ -495,10 +534,7 @@ async def resume_workflow(
|
|
|
495
534
|
# (These may be tasks that were READY at pause time, or tasks that
|
|
496
535
|
# couldn't be enqueued during step 2 due to failed deps check)
|
|
497
536
|
ready_result = await session.execute(
|
|
498
|
-
|
|
499
|
-
SELECT task_index, dependencies, is_subworkflow FROM horsies_workflow_tasks
|
|
500
|
-
WHERE workflow_id = :wf_id AND status = 'READY'
|
|
501
|
-
"""),
|
|
537
|
+
GET_READY_WORKFLOW_TASKS_SQL,
|
|
502
538
|
{'wf_id': workflow_id},
|
|
503
539
|
)
|
|
504
540
|
ready_tasks = ready_result.fetchall()
|
|
@@ -534,6 +570,20 @@ async def resume_workflow(
|
|
|
534
570
|
return True
|
|
535
571
|
|
|
536
572
|
|
|
573
|
+
# -- SQL constants for _cascade_resume_to_children --
|
|
574
|
+
|
|
575
|
+
GET_PAUSED_CHILD_WORKFLOWS_SQL = text("""
|
|
576
|
+
SELECT id, depth, root_workflow_id FROM horsies_workflows
|
|
577
|
+
WHERE parent_workflow_id = :wf_id AND status = 'PAUSED'
|
|
578
|
+
""")
|
|
579
|
+
|
|
580
|
+
RESUME_CHILD_WORKFLOW_SQL = text("""
|
|
581
|
+
UPDATE horsies_workflows
|
|
582
|
+
SET status = 'RUNNING', updated_at = NOW()
|
|
583
|
+
WHERE id = :wf_id AND status = 'PAUSED'
|
|
584
|
+
""")
|
|
585
|
+
|
|
586
|
+
|
|
537
587
|
async def _cascade_resume_to_children(
|
|
538
588
|
session: AsyncSession,
|
|
539
589
|
broker: 'PostgresBroker',
|
|
@@ -550,10 +600,7 @@ async def _cascade_resume_to_children(
|
|
|
550
600
|
|
|
551
601
|
# Find paused child workflows
|
|
552
602
|
children = await session.execute(
|
|
553
|
-
|
|
554
|
-
SELECT id, depth, root_workflow_id FROM horsies_workflows
|
|
555
|
-
WHERE parent_workflow_id = :wf_id AND status = 'PAUSED'
|
|
556
|
-
"""),
|
|
603
|
+
GET_PAUSED_CHILD_WORKFLOWS_SQL,
|
|
557
604
|
{'wf_id': current_id},
|
|
558
605
|
)
|
|
559
606
|
|
|
@@ -564,20 +611,13 @@ async def _cascade_resume_to_children(
|
|
|
564
611
|
|
|
565
612
|
# Resume child
|
|
566
613
|
await session.execute(
|
|
567
|
-
|
|
568
|
-
UPDATE horsies_workflows
|
|
569
|
-
SET status = 'RUNNING', updated_at = NOW()
|
|
570
|
-
WHERE id = :wf_id AND status = 'PAUSED'
|
|
571
|
-
"""),
|
|
614
|
+
RESUME_CHILD_WORKFLOW_SQL,
|
|
572
615
|
{'wf_id': child_id},
|
|
573
616
|
)
|
|
574
617
|
|
|
575
618
|
# Re-evaluate and enqueue child's PENDING/READY tasks
|
|
576
619
|
child_pending = await session.execute(
|
|
577
|
-
|
|
578
|
-
SELECT task_index FROM horsies_workflow_tasks
|
|
579
|
-
WHERE workflow_id = :wf_id AND status = 'PENDING'
|
|
580
|
-
"""),
|
|
620
|
+
GET_PENDING_WORKFLOW_TASKS_SQL,
|
|
581
621
|
{'wf_id': child_id},
|
|
582
622
|
)
|
|
583
623
|
for pending_row in child_pending.fetchall():
|
|
@@ -586,10 +626,7 @@ async def _cascade_resume_to_children(
|
|
|
586
626
|
)
|
|
587
627
|
|
|
588
628
|
child_ready = await session.execute(
|
|
589
|
-
|
|
590
|
-
SELECT task_index, dependencies, is_subworkflow FROM horsies_workflow_tasks
|
|
591
|
-
WHERE workflow_id = :wf_id AND status = 'READY'
|
|
592
|
-
"""),
|
|
629
|
+
GET_READY_WORKFLOW_TASKS_SQL,
|
|
593
630
|
{'wf_id': child_id},
|
|
594
631
|
)
|
|
595
632
|
for ready_row in child_ready.fetchall():
|
|
@@ -643,6 +680,34 @@ def resume_workflow_sync(
|
|
|
643
680
|
runner.stop()
|
|
644
681
|
|
|
645
682
|
|
|
683
|
+
# -- SQL constants for _enqueue_workflow_task --
|
|
684
|
+
|
|
685
|
+
ENQUEUE_WORKFLOW_TASK_SQL = text("""
|
|
686
|
+
UPDATE horsies_workflow_tasks wt
|
|
687
|
+
SET status = 'ENQUEUED', started_at = NOW()
|
|
688
|
+
FROM horsies_workflows w
|
|
689
|
+
WHERE wt.workflow_id = :wf_id
|
|
690
|
+
AND wt.task_index = :idx
|
|
691
|
+
AND wt.status = 'READY'
|
|
692
|
+
AND w.id = wt.workflow_id
|
|
693
|
+
AND w.status = 'RUNNING'
|
|
694
|
+
RETURNING wt.id, wt.task_name, wt.task_args, wt.task_kwargs, wt.queue_name, wt.priority,
|
|
695
|
+
wt.args_from, wt.workflow_ctx_from, wt.task_options
|
|
696
|
+
""")
|
|
697
|
+
|
|
698
|
+
INSERT_TASK_FOR_WORKFLOW_SQL = text("""
|
|
699
|
+
INSERT INTO horsies_tasks (id, task_name, queue_name, priority, args, kwargs, status,
|
|
700
|
+
sent_at, created_at, updated_at, claimed, retry_count, max_retries,
|
|
701
|
+
task_options, good_until)
|
|
702
|
+
VALUES (:id, :name, :queue, :priority, :args, :kwargs, 'PENDING',
|
|
703
|
+
NOW(), NOW(), NOW(), FALSE, 0, :max_retries, :task_options, :good_until)
|
|
704
|
+
""")
|
|
705
|
+
|
|
706
|
+
LINK_WORKFLOW_TASK_SQL = text("""
|
|
707
|
+
UPDATE horsies_workflow_tasks SET task_id = :tid WHERE workflow_id = :wf_id AND task_index = :idx
|
|
708
|
+
""")
|
|
709
|
+
|
|
710
|
+
|
|
646
711
|
async def _enqueue_workflow_task(
|
|
647
712
|
session: AsyncSession,
|
|
648
713
|
workflow_id: str,
|
|
@@ -664,18 +729,7 @@ async def _enqueue_workflow_task(
|
|
|
664
729
|
# Atomic: READY → ENQUEUED only if still READY AND workflow is RUNNING
|
|
665
730
|
# PAUSE guard: JOIN with workflows ensures we don't enqueue while paused
|
|
666
731
|
result = await session.execute(
|
|
667
|
-
|
|
668
|
-
UPDATE horsies_workflow_tasks wt
|
|
669
|
-
SET status = 'ENQUEUED', started_at = NOW()
|
|
670
|
-
FROM horsies_workflows w
|
|
671
|
-
WHERE wt.workflow_id = :wf_id
|
|
672
|
-
AND wt.task_index = :idx
|
|
673
|
-
AND wt.status = 'READY'
|
|
674
|
-
AND w.id = wt.workflow_id
|
|
675
|
-
AND w.status = 'RUNNING'
|
|
676
|
-
RETURNING wt.id, wt.task_name, wt.task_args, wt.task_kwargs, wt.queue_name, wt.priority,
|
|
677
|
-
wt.args_from, wt.workflow_ctx_from, wt.task_options
|
|
678
|
-
"""),
|
|
732
|
+
ENQUEUE_WORKFLOW_TASK_SQL,
|
|
679
733
|
{'wf_id': workflow_id, 'idx': task_index},
|
|
680
734
|
)
|
|
681
735
|
|
|
@@ -742,13 +796,7 @@ async def _enqueue_workflow_task(
|
|
|
742
796
|
# Create actual task in tasks table
|
|
743
797
|
task_id = str(uuid.uuid4())
|
|
744
798
|
await session.execute(
|
|
745
|
-
|
|
746
|
-
INSERT INTO horsies_tasks (id, task_name, queue_name, priority, args, kwargs, status,
|
|
747
|
-
sent_at, created_at, updated_at, claimed, retry_count, max_retries,
|
|
748
|
-
task_options, good_until)
|
|
749
|
-
VALUES (:id, :name, :queue, :priority, :args, :kwargs, 'PENDING',
|
|
750
|
-
NOW(), NOW(), NOW(), FALSE, 0, :max_retries, :task_options, :good_until)
|
|
751
|
-
"""),
|
|
799
|
+
INSERT_TASK_FOR_WORKFLOW_SQL,
|
|
752
800
|
{
|
|
753
801
|
'id': task_id,
|
|
754
802
|
'name': row[1], # task_name
|
|
@@ -764,15 +812,57 @@ async def _enqueue_workflow_task(
|
|
|
764
812
|
|
|
765
813
|
# Link workflow_task to actual task
|
|
766
814
|
await session.execute(
|
|
767
|
-
|
|
768
|
-
UPDATE horsies_workflow_tasks SET task_id = :tid WHERE workflow_id = :wf_id AND task_index = :idx
|
|
769
|
-
"""),
|
|
815
|
+
LINK_WORKFLOW_TASK_SQL,
|
|
770
816
|
{'tid': task_id, 'wf_id': workflow_id, 'idx': task_index},
|
|
771
817
|
)
|
|
772
818
|
|
|
773
819
|
return task_id
|
|
774
820
|
|
|
775
821
|
|
|
822
|
+
# -- SQL constants for _enqueue_subworkflow_task --
|
|
823
|
+
|
|
824
|
+
ENQUEUE_SUBWORKFLOW_TASK_SQL = text("""
|
|
825
|
+
UPDATE horsies_workflow_tasks wt
|
|
826
|
+
SET status = 'ENQUEUED', started_at = NOW()
|
|
827
|
+
FROM horsies_workflows w
|
|
828
|
+
WHERE wt.workflow_id = :wf_id
|
|
829
|
+
AND wt.task_index = :idx
|
|
830
|
+
AND wt.status = 'READY'
|
|
831
|
+
AND wt.is_subworkflow = TRUE
|
|
832
|
+
AND w.id = wt.workflow_id
|
|
833
|
+
AND w.status = 'RUNNING'
|
|
834
|
+
RETURNING wt.id, wt.sub_workflow_name, wt.task_args, wt.task_kwargs,
|
|
835
|
+
wt.args_from, wt.node_id, wt.sub_workflow_module,
|
|
836
|
+
wt.sub_workflow_qualname, wt.sub_workflow_retry_mode
|
|
837
|
+
""")
|
|
838
|
+
|
|
839
|
+
GET_WORKFLOW_NAME_SQL = text("""SELECT name FROM horsies_workflows WHERE id = :wf_id""")
|
|
840
|
+
|
|
841
|
+
MARK_WORKFLOW_TASK_FAILED_SQL = text("""
|
|
842
|
+
UPDATE horsies_workflow_tasks
|
|
843
|
+
SET status = 'FAILED', result = :result, completed_at = NOW()
|
|
844
|
+
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
845
|
+
""")
|
|
846
|
+
|
|
847
|
+
INSERT_CHILD_WORKFLOW_SQL = text("""
|
|
848
|
+
INSERT INTO horsies_workflows
|
|
849
|
+
(id, name, status, on_error, output_task_index, success_policy,
|
|
850
|
+
workflow_def_module, workflow_def_qualname,
|
|
851
|
+
parent_workflow_id, parent_task_index, depth, root_workflow_id,
|
|
852
|
+
created_at, started_at, updated_at)
|
|
853
|
+
VALUES (:id, :name, 'RUNNING', :on_error, :output_idx, :success_policy,
|
|
854
|
+
:wf_module, :wf_qualname,
|
|
855
|
+
:parent_wf_id, :parent_idx, :depth, :root_wf_id,
|
|
856
|
+
NOW(), NOW(), NOW())
|
|
857
|
+
""")
|
|
858
|
+
|
|
859
|
+
LINK_SUB_WORKFLOW_SQL = text("""
|
|
860
|
+
UPDATE horsies_workflow_tasks
|
|
861
|
+
SET sub_workflow_id = :child_id, status = 'RUNNING'
|
|
862
|
+
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
863
|
+
""")
|
|
864
|
+
|
|
865
|
+
|
|
776
866
|
async def _enqueue_subworkflow_task(
|
|
777
867
|
session: AsyncSession,
|
|
778
868
|
broker: 'PostgresBroker',
|
|
@@ -799,20 +889,7 @@ async def _enqueue_subworkflow_task(
|
|
|
799
889
|
"""
|
|
800
890
|
# 1. Atomically mark parent node as ENQUEUED (with workflow RUNNING guard)
|
|
801
891
|
result = await session.execute(
|
|
802
|
-
|
|
803
|
-
UPDATE horsies_workflow_tasks wt
|
|
804
|
-
SET status = 'ENQUEUED', started_at = NOW()
|
|
805
|
-
FROM horsies_workflows w
|
|
806
|
-
WHERE wt.workflow_id = :wf_id
|
|
807
|
-
AND wt.task_index = :idx
|
|
808
|
-
AND wt.status = 'READY'
|
|
809
|
-
AND wt.is_subworkflow = TRUE
|
|
810
|
-
AND w.id = wt.workflow_id
|
|
811
|
-
AND w.status = 'RUNNING'
|
|
812
|
-
RETURNING wt.id, wt.sub_workflow_name, wt.task_args, wt.task_kwargs,
|
|
813
|
-
wt.args_from, wt.node_id, wt.sub_workflow_module,
|
|
814
|
-
wt.sub_workflow_qualname, wt.sub_workflow_retry_mode
|
|
815
|
-
"""),
|
|
892
|
+
ENQUEUE_SUBWORKFLOW_TASK_SQL,
|
|
816
893
|
{'wf_id': workflow_id, 'idx': task_index},
|
|
817
894
|
)
|
|
818
895
|
|
|
@@ -840,7 +917,7 @@ async def _enqueue_subworkflow_task(
|
|
|
840
917
|
# Fallback import path (sub_workflow_module/qualname stored in DB) handles
|
|
841
918
|
# the common case where registry is empty.
|
|
842
919
|
workflow_name_result = await session.execute(
|
|
843
|
-
|
|
920
|
+
GET_WORKFLOW_NAME_SQL,
|
|
844
921
|
{'wf_id': workflow_id},
|
|
845
922
|
)
|
|
846
923
|
workflow_name_row = workflow_name_result.fetchone()
|
|
@@ -875,11 +952,7 @@ async def _enqueue_subworkflow_task(
|
|
|
875
952
|
data={'module': sub_workflow_module, 'qualname': sub_workflow_qualname},
|
|
876
953
|
)
|
|
877
954
|
await session.execute(
|
|
878
|
-
|
|
879
|
-
UPDATE horsies_workflow_tasks
|
|
880
|
-
SET status = 'FAILED', result = :result, completed_at = NOW()
|
|
881
|
-
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
882
|
-
"""),
|
|
955
|
+
MARK_WORKFLOW_TASK_FAILED_SQL,
|
|
883
956
|
{
|
|
884
957
|
'wf_id': workflow_id,
|
|
885
958
|
'idx': task_index,
|
|
@@ -964,17 +1037,7 @@ async def _enqueue_subworkflow_task(
|
|
|
964
1037
|
child_output_index = child_spec.output.index if child_spec.output else None
|
|
965
1038
|
|
|
966
1039
|
await session.execute(
|
|
967
|
-
|
|
968
|
-
INSERT INTO horsies_workflows
|
|
969
|
-
(id, name, status, on_error, output_task_index, success_policy,
|
|
970
|
-
workflow_def_module, workflow_def_qualname,
|
|
971
|
-
parent_workflow_id, parent_task_index, depth, root_workflow_id,
|
|
972
|
-
created_at, started_at, updated_at)
|
|
973
|
-
VALUES (:id, :name, 'RUNNING', :on_error, :output_idx, :success_policy,
|
|
974
|
-
:wf_module, :wf_qualname,
|
|
975
|
-
:parent_wf_id, :parent_idx, :depth, :root_wf_id,
|
|
976
|
-
NOW(), NOW(), NOW())
|
|
977
|
-
"""),
|
|
1040
|
+
INSERT_CHILD_WORKFLOW_SQL,
|
|
978
1041
|
{
|
|
979
1042
|
'id': child_id,
|
|
980
1043
|
'name': child_spec.name,
|
|
@@ -1012,18 +1075,7 @@ async def _enqueue_subworkflow_task(
|
|
|
1012
1075
|
if child_is_subworkflow:
|
|
1013
1076
|
child_sub = child_node
|
|
1014
1077
|
await session.execute(
|
|
1015
|
-
|
|
1016
|
-
INSERT INTO horsies_workflow_tasks
|
|
1017
|
-
(id, workflow_id, task_index, node_id, task_name, task_args, task_kwargs,
|
|
1018
|
-
queue_name, priority, dependencies, args_from, workflow_ctx_from,
|
|
1019
|
-
allow_failed_deps, join_type, min_success, task_options, status,
|
|
1020
|
-
is_subworkflow, sub_workflow_name, sub_workflow_retry_mode,
|
|
1021
|
-
sub_workflow_module, sub_workflow_qualname, created_at)
|
|
1022
|
-
VALUES (:id, :wf_id, :idx, :node_id, :name, :args, :kwargs, :queue, :priority,
|
|
1023
|
-
:deps, :args_from, :ctx_from, :allow_failed, :join_type, :min_success,
|
|
1024
|
-
:task_options, :status, TRUE, :sub_wf_name, :sub_wf_retry_mode,
|
|
1025
|
-
:sub_wf_module, :sub_wf_qualname, NOW())
|
|
1026
|
-
"""),
|
|
1078
|
+
INSERT_WORKFLOW_TASK_SUBWORKFLOW_SQL,
|
|
1027
1079
|
{
|
|
1028
1080
|
'id': child_wt_id,
|
|
1029
1081
|
'wf_id': child_id,
|
|
@@ -1065,16 +1117,7 @@ async def _enqueue_subworkflow_task(
|
|
|
1065
1117
|
child_task_options_json = dumps_json(child_base_options)
|
|
1066
1118
|
|
|
1067
1119
|
await session.execute(
|
|
1068
|
-
|
|
1069
|
-
INSERT INTO horsies_workflow_tasks
|
|
1070
|
-
(id, workflow_id, task_index, node_id, task_name, task_args, task_kwargs,
|
|
1071
|
-
queue_name, priority, dependencies, args_from, workflow_ctx_from,
|
|
1072
|
-
allow_failed_deps, join_type, min_success, task_options, status,
|
|
1073
|
-
is_subworkflow, created_at)
|
|
1074
|
-
VALUES (:id, :wf_id, :idx, :node_id, :name, :args, :kwargs, :queue, :priority,
|
|
1075
|
-
:deps, :args_from, :ctx_from, :allow_failed, :join_type, :min_success,
|
|
1076
|
-
:task_options, :status, FALSE, NOW())
|
|
1077
|
-
"""),
|
|
1120
|
+
INSERT_WORKFLOW_TASK_SQL,
|
|
1078
1121
|
{
|
|
1079
1122
|
'id': child_wt_id,
|
|
1080
1123
|
'wf_id': child_id,
|
|
@@ -1104,11 +1147,7 @@ async def _enqueue_subworkflow_task(
|
|
|
1104
1147
|
|
|
1105
1148
|
# 7. Update parent's workflow_task with child_workflow_id and mark RUNNING
|
|
1106
1149
|
await session.execute(
|
|
1107
|
-
|
|
1108
|
-
UPDATE horsies_workflow_tasks
|
|
1109
|
-
SET sub_workflow_id = :child_id, status = 'RUNNING'
|
|
1110
|
-
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
1111
|
-
"""),
|
|
1150
|
+
LINK_SUB_WORKFLOW_SQL,
|
|
1112
1151
|
{'child_id': child_id, 'wf_id': workflow_id, 'idx': task_index},
|
|
1113
1152
|
)
|
|
1114
1153
|
|
|
@@ -1133,6 +1172,18 @@ async def _enqueue_subworkflow_task(
|
|
|
1133
1172
|
return child_id
|
|
1134
1173
|
|
|
1135
1174
|
|
|
1175
|
+
# -- SQL constants for _build_workflow_context_data --
|
|
1176
|
+
|
|
1177
|
+
GET_SUBWORKFLOW_SUMMARIES_SQL = text("""
|
|
1178
|
+
SELECT node_id, sub_workflow_summary
|
|
1179
|
+
FROM horsies_workflow_tasks
|
|
1180
|
+
WHERE workflow_id = :wf_id
|
|
1181
|
+
AND node_id = ANY(:node_ids)
|
|
1182
|
+
AND is_subworkflow = TRUE
|
|
1183
|
+
AND sub_workflow_summary IS NOT NULL
|
|
1184
|
+
""")
|
|
1185
|
+
|
|
1186
|
+
|
|
1136
1187
|
async def _build_workflow_context_data(
|
|
1137
1188
|
session: AsyncSession,
|
|
1138
1189
|
workflow_id: str,
|
|
@@ -1162,14 +1213,7 @@ async def _build_workflow_context_data(
|
|
|
1162
1213
|
summaries_by_id: dict[str, str] = {}
|
|
1163
1214
|
if ctx_from_ids:
|
|
1164
1215
|
summary_result = await session.execute(
|
|
1165
|
-
|
|
1166
|
-
SELECT node_id, sub_workflow_summary
|
|
1167
|
-
FROM horsies_workflow_tasks
|
|
1168
|
-
WHERE workflow_id = :wf_id
|
|
1169
|
-
AND node_id = ANY(:node_ids)
|
|
1170
|
-
AND is_subworkflow = TRUE
|
|
1171
|
-
AND sub_workflow_summary IS NOT NULL
|
|
1172
|
-
"""),
|
|
1216
|
+
GET_SUBWORKFLOW_SUMMARIES_SQL,
|
|
1173
1217
|
{'wf_id': workflow_id, 'node_ids': ctx_from_ids},
|
|
1174
1218
|
)
|
|
1175
1219
|
for row in summary_result.fetchall():
|
|
@@ -1187,6 +1231,25 @@ async def _build_workflow_context_data(
|
|
|
1187
1231
|
}
|
|
1188
1232
|
|
|
1189
1233
|
|
|
1234
|
+
# -- SQL constants for on_workflow_task_complete --
|
|
1235
|
+
|
|
1236
|
+
GET_WORKFLOW_TASK_BY_TASK_ID_SQL = text("""
|
|
1237
|
+
SELECT workflow_id, task_index
|
|
1238
|
+
FROM horsies_workflow_tasks
|
|
1239
|
+
WHERE task_id = :tid
|
|
1240
|
+
""")
|
|
1241
|
+
|
|
1242
|
+
UPDATE_WORKFLOW_TASK_RESULT_SQL = text("""
|
|
1243
|
+
UPDATE horsies_workflow_tasks
|
|
1244
|
+
SET status = :status, result = :result, completed_at = NOW()
|
|
1245
|
+
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
1246
|
+
""")
|
|
1247
|
+
|
|
1248
|
+
GET_WORKFLOW_STATUS_SQL = text(
|
|
1249
|
+
"""SELECT status FROM horsies_workflows WHERE id = :wf_id"""
|
|
1250
|
+
)
|
|
1251
|
+
|
|
1252
|
+
|
|
1190
1253
|
async def on_workflow_task_complete(
|
|
1191
1254
|
session: AsyncSession,
|
|
1192
1255
|
task_id: str,
|
|
@@ -1199,11 +1262,7 @@ async def on_workflow_task_complete(
|
|
|
1199
1262
|
"""
|
|
1200
1263
|
# 1. Find workflow_task by task_id
|
|
1201
1264
|
wt_result = await session.execute(
|
|
1202
|
-
|
|
1203
|
-
SELECT workflow_id, task_index
|
|
1204
|
-
FROM horsies_workflow_tasks
|
|
1205
|
-
WHERE task_id = :tid
|
|
1206
|
-
"""),
|
|
1265
|
+
GET_WORKFLOW_TASK_BY_TASK_ID_SQL,
|
|
1207
1266
|
{'tid': task_id},
|
|
1208
1267
|
)
|
|
1209
1268
|
|
|
@@ -1217,11 +1276,7 @@ async def on_workflow_task_complete(
|
|
|
1217
1276
|
# 2. Update workflow_task status and store result
|
|
1218
1277
|
new_status = 'COMPLETED' if result.is_ok() else 'FAILED'
|
|
1219
1278
|
await session.execute(
|
|
1220
|
-
|
|
1221
|
-
UPDATE horsies_workflow_tasks
|
|
1222
|
-
SET status = :status, result = :result, completed_at = NOW()
|
|
1223
|
-
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
1224
|
-
"""),
|
|
1279
|
+
UPDATE_WORKFLOW_TASK_RESULT_SQL,
|
|
1225
1280
|
{
|
|
1226
1281
|
'status': new_status,
|
|
1227
1282
|
'result': dumps_json(result),
|
|
@@ -1241,7 +1296,7 @@ async def on_workflow_task_complete(
|
|
|
1241
1296
|
|
|
1242
1297
|
# 4. Check if workflow is PAUSED (may have been paused by another task)
|
|
1243
1298
|
status_check = await session.execute(
|
|
1244
|
-
|
|
1299
|
+
GET_WORKFLOW_STATUS_SQL,
|
|
1245
1300
|
{'wf_id': workflow_id},
|
|
1246
1301
|
)
|
|
1247
1302
|
status_row = status_check.fetchone()
|
|
@@ -1255,6 +1310,20 @@ async def on_workflow_task_complete(
|
|
|
1255
1310
|
await _check_workflow_completion(session, workflow_id, broker)
|
|
1256
1311
|
|
|
1257
1312
|
|
|
1313
|
+
# -- SQL constants for _process_dependents --
|
|
1314
|
+
|
|
1315
|
+
GET_DEPENDENT_TASKS_SQL = text("""
|
|
1316
|
+
SELECT task_index FROM horsies_workflow_tasks
|
|
1317
|
+
WHERE workflow_id = :wf_id
|
|
1318
|
+
AND :completed_idx = ANY(dependencies)
|
|
1319
|
+
AND status = 'PENDING'
|
|
1320
|
+
""")
|
|
1321
|
+
|
|
1322
|
+
GET_WORKFLOW_DEPTH_SQL = text(
|
|
1323
|
+
"""SELECT depth, root_workflow_id FROM horsies_workflows WHERE id = :wf_id"""
|
|
1324
|
+
)
|
|
1325
|
+
|
|
1326
|
+
|
|
1258
1327
|
async def _process_dependents(
|
|
1259
1328
|
session: AsyncSession,
|
|
1260
1329
|
workflow_id: str,
|
|
@@ -1272,18 +1341,13 @@ async def _process_dependents(
|
|
|
1272
1341
|
"""
|
|
1273
1342
|
# Find tasks that have completed_task_index in their dependencies
|
|
1274
1343
|
dependents = await session.execute(
|
|
1275
|
-
|
|
1276
|
-
SELECT task_index FROM horsies_workflow_tasks
|
|
1277
|
-
WHERE workflow_id = :wf_id
|
|
1278
|
-
AND :completed_idx = ANY(dependencies)
|
|
1279
|
-
AND status = 'PENDING'
|
|
1280
|
-
"""),
|
|
1344
|
+
GET_DEPENDENT_TASKS_SQL,
|
|
1281
1345
|
{'wf_id': workflow_id, 'completed_idx': completed_task_index},
|
|
1282
1346
|
)
|
|
1283
1347
|
|
|
1284
1348
|
# Get workflow depth and root for subworkflow support
|
|
1285
1349
|
wf_info = await session.execute(
|
|
1286
|
-
|
|
1350
|
+
GET_WORKFLOW_DEPTH_SQL,
|
|
1287
1351
|
{'wf_id': workflow_id},
|
|
1288
1352
|
)
|
|
1289
1353
|
wf_row = wf_info.fetchone()
|
|
@@ -1296,6 +1360,53 @@ async def _process_dependents(
|
|
|
1296
1360
|
)
|
|
1297
1361
|
|
|
1298
1362
|
|
|
1363
|
+
# -- SQL constants for _try_make_ready_and_enqueue --
|
|
1364
|
+
|
|
1365
|
+
GET_TASK_CONFIG_SQL = text("""
|
|
1366
|
+
SELECT wt.status, wt.dependencies, wt.allow_failed_deps,
|
|
1367
|
+
wt.join_type, wt.min_success, wt.workflow_ctx_from,
|
|
1368
|
+
wt.is_subworkflow,
|
|
1369
|
+
w.status as wf_status
|
|
1370
|
+
FROM horsies_workflow_tasks wt
|
|
1371
|
+
JOIN horsies_workflows w ON w.id = wt.workflow_id
|
|
1372
|
+
WHERE wt.workflow_id = :wf_id AND wt.task_index = :idx
|
|
1373
|
+
""")
|
|
1374
|
+
|
|
1375
|
+
GET_DEP_STATUS_COUNTS_SQL = text("""
|
|
1376
|
+
SELECT status, COUNT(*) as cnt
|
|
1377
|
+
FROM horsies_workflow_tasks
|
|
1378
|
+
WHERE workflow_id = :wf_id AND task_index = ANY(:deps)
|
|
1379
|
+
GROUP BY status
|
|
1380
|
+
""")
|
|
1381
|
+
|
|
1382
|
+
SKIP_WORKFLOW_TASK_SQL = text("""
|
|
1383
|
+
UPDATE horsies_workflow_tasks
|
|
1384
|
+
SET status = 'SKIPPED'
|
|
1385
|
+
WHERE workflow_id = :wf_id AND task_index = :idx AND status = 'PENDING'
|
|
1386
|
+
""")
|
|
1387
|
+
|
|
1388
|
+
COUNT_CTX_TERMINAL_DEPS_SQL = text("""
|
|
1389
|
+
SELECT COUNT(*) as cnt
|
|
1390
|
+
FROM horsies_workflow_tasks
|
|
1391
|
+
WHERE workflow_id = :wf_id
|
|
1392
|
+
AND node_id = ANY(:node_ids)
|
|
1393
|
+
AND status = ANY(:wf_task_terminal_states)
|
|
1394
|
+
""")
|
|
1395
|
+
|
|
1396
|
+
MARK_TASK_READY_SQL = text("""
|
|
1397
|
+
UPDATE horsies_workflow_tasks
|
|
1398
|
+
SET status = 'READY'
|
|
1399
|
+
WHERE workflow_id = :wf_id AND task_index = :idx AND status = 'PENDING'
|
|
1400
|
+
RETURNING task_index
|
|
1401
|
+
""")
|
|
1402
|
+
|
|
1403
|
+
SKIP_READY_WORKFLOW_TASK_SQL = text("""
|
|
1404
|
+
UPDATE horsies_workflow_tasks
|
|
1405
|
+
SET status = 'SKIPPED'
|
|
1406
|
+
WHERE workflow_id = :wf_id AND task_index = :idx AND status = 'READY'
|
|
1407
|
+
""")
|
|
1408
|
+
|
|
1409
|
+
|
|
1299
1410
|
async def _try_make_ready_and_enqueue(
|
|
1300
1411
|
session: AsyncSession,
|
|
1301
1412
|
broker: 'PostgresBroker | None',
|
|
@@ -1327,15 +1438,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1327
1438
|
"""
|
|
1328
1439
|
# 1. Fetch task configuration
|
|
1329
1440
|
config_result = await session.execute(
|
|
1330
|
-
|
|
1331
|
-
SELECT wt.status, wt.dependencies, wt.allow_failed_deps,
|
|
1332
|
-
wt.join_type, wt.min_success, wt.workflow_ctx_from,
|
|
1333
|
-
wt.is_subworkflow,
|
|
1334
|
-
w.status as wf_status
|
|
1335
|
-
FROM horsies_workflow_tasks wt
|
|
1336
|
-
JOIN horsies_workflows w ON w.id = wt.workflow_id
|
|
1337
|
-
WHERE wt.workflow_id = :wf_id AND wt.task_index = :idx
|
|
1338
|
-
"""),
|
|
1441
|
+
GET_TASK_CONFIG_SQL,
|
|
1339
1442
|
{'wf_id': workflow_id, 'idx': task_index},
|
|
1340
1443
|
)
|
|
1341
1444
|
config_row = config_result.fetchone()
|
|
@@ -1365,12 +1468,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1365
1468
|
|
|
1366
1469
|
# 2. Get dependency status counts
|
|
1367
1470
|
dep_status_result = await session.execute(
|
|
1368
|
-
|
|
1369
|
-
SELECT status, COUNT(*) as cnt
|
|
1370
|
-
FROM horsies_workflow_tasks
|
|
1371
|
-
WHERE workflow_id = :wf_id AND task_index = ANY(:deps)
|
|
1372
|
-
GROUP BY status
|
|
1373
|
-
"""),
|
|
1471
|
+
GET_DEP_STATUS_COUNTS_SQL,
|
|
1374
1472
|
{'wf_id': workflow_id, 'deps': dependencies},
|
|
1375
1473
|
)
|
|
1376
1474
|
status_counts: dict[str, int] = {
|
|
@@ -1415,11 +1513,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1415
1513
|
if should_skip:
|
|
1416
1514
|
# Mark task as SKIPPED (impossible to meet join condition)
|
|
1417
1515
|
await session.execute(
|
|
1418
|
-
|
|
1419
|
-
UPDATE horsies_workflow_tasks
|
|
1420
|
-
SET status = 'SKIPPED'
|
|
1421
|
-
WHERE workflow_id = :wf_id AND task_index = :idx AND status = 'PENDING'
|
|
1422
|
-
"""),
|
|
1516
|
+
SKIP_WORKFLOW_TASK_SQL,
|
|
1423
1517
|
{'wf_id': workflow_id, 'idx': task_index},
|
|
1424
1518
|
)
|
|
1425
1519
|
# Propagate SKIPPED to dependents
|
|
@@ -1433,13 +1527,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1433
1527
|
ctx_from_ids = _as_str_list(raw_ctx_from)
|
|
1434
1528
|
if ctx_from_ids:
|
|
1435
1529
|
ctx_terminal_result = await session.execute(
|
|
1436
|
-
|
|
1437
|
-
SELECT COUNT(*) as cnt
|
|
1438
|
-
FROM horsies_workflow_tasks
|
|
1439
|
-
WHERE workflow_id = :wf_id
|
|
1440
|
-
AND node_id = ANY(:node_ids)
|
|
1441
|
-
AND status = ANY(:wf_task_terminal_states)
|
|
1442
|
-
"""),
|
|
1530
|
+
COUNT_CTX_TERMINAL_DEPS_SQL,
|
|
1443
1531
|
{
|
|
1444
1532
|
'wf_id': workflow_id,
|
|
1445
1533
|
'node_ids': ctx_from_ids,
|
|
@@ -1452,12 +1540,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1452
1540
|
|
|
1453
1541
|
# 4. Mark task as READY
|
|
1454
1542
|
ready_result = await session.execute(
|
|
1455
|
-
|
|
1456
|
-
UPDATE horsies_workflow_tasks
|
|
1457
|
-
SET status = 'READY'
|
|
1458
|
-
WHERE workflow_id = :wf_id AND task_index = :idx AND status = 'PENDING'
|
|
1459
|
-
RETURNING task_index
|
|
1460
|
-
"""),
|
|
1543
|
+
MARK_TASK_READY_SQL,
|
|
1461
1544
|
{'wf_id': workflow_id, 'idx': task_index},
|
|
1462
1545
|
)
|
|
1463
1546
|
if ready_result.fetchone() is None:
|
|
@@ -1469,11 +1552,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1469
1552
|
failed_or_skipped = failed + skipped
|
|
1470
1553
|
if failed_or_skipped > 0 and not allow_failed_deps:
|
|
1471
1554
|
await session.execute(
|
|
1472
|
-
|
|
1473
|
-
UPDATE horsies_workflow_tasks
|
|
1474
|
-
SET status = 'SKIPPED'
|
|
1475
|
-
WHERE workflow_id = :wf_id AND task_index = :idx AND status = 'READY'
|
|
1476
|
-
"""),
|
|
1555
|
+
SKIP_READY_WORKFLOW_TASK_SQL,
|
|
1477
1556
|
{'wf_id': workflow_id, 'idx': task_index},
|
|
1478
1557
|
)
|
|
1479
1558
|
await _process_dependents(session, workflow_id, task_index, broker)
|
|
@@ -1482,7 +1561,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1482
1561
|
# 6. Evaluate conditions (run_when/skip_when) if set
|
|
1483
1562
|
# Requires workflow name to look up TaskNode from registry
|
|
1484
1563
|
workflow_name_result = await session.execute(
|
|
1485
|
-
|
|
1564
|
+
GET_WORKFLOW_NAME_SQL,
|
|
1486
1565
|
{'wf_id': workflow_id},
|
|
1487
1566
|
)
|
|
1488
1567
|
workflow_name_row = workflow_name_result.fetchone()
|
|
@@ -1494,11 +1573,7 @@ async def _try_make_ready_and_enqueue(
|
|
|
1494
1573
|
)
|
|
1495
1574
|
if should_skip_condition:
|
|
1496
1575
|
await session.execute(
|
|
1497
|
-
|
|
1498
|
-
UPDATE horsies_workflow_tasks
|
|
1499
|
-
SET status = 'SKIPPED'
|
|
1500
|
-
WHERE workflow_id = :wf_id AND task_index = :idx AND status = 'READY'
|
|
1501
|
-
"""),
|
|
1576
|
+
SKIP_READY_WORKFLOW_TASK_SQL,
|
|
1502
1577
|
{'wf_id': workflow_id, 'idx': task_index},
|
|
1503
1578
|
)
|
|
1504
1579
|
await _process_dependents(session, workflow_id, task_index, broker)
|
|
@@ -1518,6 +1593,15 @@ async def _try_make_ready_and_enqueue(
|
|
|
1518
1593
|
await _enqueue_workflow_task(session, workflow_id, task_index, dep_results)
|
|
1519
1594
|
|
|
1520
1595
|
|
|
1596
|
+
# -- SQL constants for _evaluate_conditions --
|
|
1597
|
+
|
|
1598
|
+
GET_WORKFLOW_DEF_PATH_SQL = text("""
|
|
1599
|
+
SELECT workflow_def_module, workflow_def_qualname
|
|
1600
|
+
FROM horsies_workflows
|
|
1601
|
+
WHERE id = :wf_id
|
|
1602
|
+
""")
|
|
1603
|
+
|
|
1604
|
+
|
|
1521
1605
|
async def _evaluate_conditions(
|
|
1522
1606
|
session: AsyncSession,
|
|
1523
1607
|
workflow_id: str,
|
|
@@ -1543,11 +1627,7 @@ async def _evaluate_conditions(
|
|
|
1543
1627
|
if node is None:
|
|
1544
1628
|
# Try to load workflow definition by import path (Option B.1)
|
|
1545
1629
|
def_result = await session.execute(
|
|
1546
|
-
|
|
1547
|
-
SELECT workflow_def_module, workflow_def_qualname
|
|
1548
|
-
FROM horsies_workflows
|
|
1549
|
-
WHERE id = :wf_id
|
|
1550
|
-
"""),
|
|
1630
|
+
GET_WORKFLOW_DEF_PATH_SQL,
|
|
1551
1631
|
{'wf_id': workflow_id},
|
|
1552
1632
|
)
|
|
1553
1633
|
def_row = def_result.fetchone()
|
|
@@ -1592,14 +1672,7 @@ async def _evaluate_conditions(
|
|
|
1592
1672
|
summaries_by_id: dict[str, SubWorkflowSummary[Any]] = {}
|
|
1593
1673
|
if ctx_from_ids:
|
|
1594
1674
|
summary_result = await session.execute(
|
|
1595
|
-
|
|
1596
|
-
SELECT node_id, sub_workflow_summary
|
|
1597
|
-
FROM horsies_workflow_tasks
|
|
1598
|
-
WHERE workflow_id = :wf_id
|
|
1599
|
-
AND node_id = ANY(:node_ids)
|
|
1600
|
-
AND is_subworkflow = TRUE
|
|
1601
|
-
AND sub_workflow_summary IS NOT NULL
|
|
1602
|
-
"""),
|
|
1675
|
+
GET_SUBWORKFLOW_SUMMARIES_SQL,
|
|
1603
1676
|
{'wf_id': workflow_id, 'node_ids': ctx_from_ids},
|
|
1604
1677
|
)
|
|
1605
1678
|
for row in summary_result.fetchall():
|
|
@@ -1648,6 +1721,17 @@ class DependencyResults:
|
|
|
1648
1721
|
self.by_id: dict[str, 'TaskResult[Any, TaskError]'] = {}
|
|
1649
1722
|
|
|
1650
1723
|
|
|
1724
|
+
# -- SQL constants for _get_dependency_results --
|
|
1725
|
+
|
|
1726
|
+
GET_DEPENDENCY_RESULTS_SQL = text("""
|
|
1727
|
+
SELECT task_index, status, result
|
|
1728
|
+
FROM horsies_workflow_tasks
|
|
1729
|
+
WHERE workflow_id = :wf_id
|
|
1730
|
+
AND task_index = ANY(:indices)
|
|
1731
|
+
AND status = ANY(:wf_task_terminal_states)
|
|
1732
|
+
""")
|
|
1733
|
+
|
|
1734
|
+
|
|
1651
1735
|
async def _get_dependency_results(
|
|
1652
1736
|
session: AsyncSession,
|
|
1653
1737
|
workflow_id: str,
|
|
@@ -1665,13 +1749,7 @@ async def _get_dependency_results(
|
|
|
1665
1749
|
return {}
|
|
1666
1750
|
|
|
1667
1751
|
result = await session.execute(
|
|
1668
|
-
|
|
1669
|
-
SELECT task_index, status, result
|
|
1670
|
-
FROM horsies_workflow_tasks
|
|
1671
|
-
WHERE workflow_id = :wf_id
|
|
1672
|
-
AND task_index = ANY(:indices)
|
|
1673
|
-
AND status = ANY(:wf_task_terminal_states)
|
|
1674
|
-
"""),
|
|
1752
|
+
GET_DEPENDENCY_RESULTS_SQL,
|
|
1675
1753
|
{
|
|
1676
1754
|
'wf_id': workflow_id,
|
|
1677
1755
|
'indices': dependency_indices,
|
|
@@ -1700,6 +1778,17 @@ async def _get_dependency_results(
|
|
|
1700
1778
|
return results
|
|
1701
1779
|
|
|
1702
1780
|
|
|
1781
|
+
# -- SQL constants for _get_dependency_results_with_names --
|
|
1782
|
+
|
|
1783
|
+
GET_DEPENDENCY_RESULTS_WITH_NAMES_SQL = text("""
|
|
1784
|
+
SELECT task_index, task_name, node_id, status, result
|
|
1785
|
+
FROM horsies_workflow_tasks
|
|
1786
|
+
WHERE workflow_id = :wf_id
|
|
1787
|
+
AND node_id = ANY(:node_ids)
|
|
1788
|
+
AND status = ANY(:wf_task_terminal_states)
|
|
1789
|
+
""")
|
|
1790
|
+
|
|
1791
|
+
|
|
1703
1792
|
async def _get_dependency_results_with_names(
|
|
1704
1793
|
session: AsyncSession,
|
|
1705
1794
|
workflow_id: str,
|
|
@@ -1720,13 +1809,7 @@ async def _get_dependency_results_with_names(
|
|
|
1720
1809
|
return dep_results
|
|
1721
1810
|
|
|
1722
1811
|
result = await session.execute(
|
|
1723
|
-
|
|
1724
|
-
SELECT task_index, task_name, node_id, status, result
|
|
1725
|
-
FROM horsies_workflow_tasks
|
|
1726
|
-
WHERE workflow_id = :wf_id
|
|
1727
|
-
AND node_id = ANY(:node_ids)
|
|
1728
|
-
AND status = ANY(:wf_task_terminal_states)
|
|
1729
|
-
"""),
|
|
1812
|
+
GET_DEPENDENCY_RESULTS_WITH_NAMES_SQL,
|
|
1730
1813
|
{
|
|
1731
1814
|
'wf_id': workflow_id,
|
|
1732
1815
|
'node_ids': dependency_node_ids,
|
|
@@ -1765,6 +1848,45 @@ async def _get_dependency_results_with_names(
|
|
|
1765
1848
|
return dep_results
|
|
1766
1849
|
|
|
1767
1850
|
|
|
1851
|
+
# -- SQL constants for _check_workflow_completion --
|
|
1852
|
+
|
|
1853
|
+
GET_WORKFLOW_COMPLETION_STATUS_SQL = text("""
|
|
1854
|
+
SELECT
|
|
1855
|
+
w.status,
|
|
1856
|
+
w.completed_at,
|
|
1857
|
+
w.error,
|
|
1858
|
+
w.success_policy,
|
|
1859
|
+
w.name,
|
|
1860
|
+
COUNT(*) FILTER (WHERE NOT (wt.status = ANY(:wf_task_terminal_states))) as incomplete,
|
|
1861
|
+
COUNT(*) FILTER (WHERE wt.status = 'FAILED') as failed,
|
|
1862
|
+
COUNT(*) FILTER (WHERE wt.status = 'COMPLETED') as completed,
|
|
1863
|
+
COUNT(*) as total
|
|
1864
|
+
FROM horsies_workflows w
|
|
1865
|
+
LEFT JOIN horsies_workflow_tasks wt ON wt.workflow_id = w.id
|
|
1866
|
+
WHERE w.id = :wf_id
|
|
1867
|
+
GROUP BY w.id, w.status, w.completed_at, w.error, w.success_policy, w.name
|
|
1868
|
+
""")
|
|
1869
|
+
|
|
1870
|
+
MARK_WORKFLOW_COMPLETED_SQL = text("""
|
|
1871
|
+
UPDATE horsies_workflows
|
|
1872
|
+
SET status = 'COMPLETED', result = :result, completed_at = NOW(), updated_at = NOW()
|
|
1873
|
+
WHERE id = :wf_id AND completed_at IS NULL
|
|
1874
|
+
""")
|
|
1875
|
+
|
|
1876
|
+
MARK_WORKFLOW_FAILED_SQL = text("""
|
|
1877
|
+
UPDATE horsies_workflows
|
|
1878
|
+
SET status = 'FAILED', result = :result,
|
|
1879
|
+
error = COALESCE(:error, error),
|
|
1880
|
+
completed_at = NOW(), updated_at = NOW()
|
|
1881
|
+
WHERE id = :wf_id AND completed_at IS NULL
|
|
1882
|
+
""")
|
|
1883
|
+
|
|
1884
|
+
GET_PARENT_WORKFLOW_INFO_SQL = text("""
|
|
1885
|
+
SELECT parent_workflow_id, parent_task_index
|
|
1886
|
+
FROM horsies_workflows WHERE id = :wf_id
|
|
1887
|
+
""")
|
|
1888
|
+
|
|
1889
|
+
|
|
1768
1890
|
async def _check_workflow_completion(
|
|
1769
1891
|
session: AsyncSession,
|
|
1770
1892
|
workflow_id: str,
|
|
@@ -1778,22 +1900,7 @@ async def _check_workflow_completion(
|
|
|
1778
1900
|
"""
|
|
1779
1901
|
# Get current workflow status, error, success_policy, and task counts
|
|
1780
1902
|
result = await session.execute(
|
|
1781
|
-
|
|
1782
|
-
SELECT
|
|
1783
|
-
w.status,
|
|
1784
|
-
w.completed_at,
|
|
1785
|
-
w.error,
|
|
1786
|
-
w.success_policy,
|
|
1787
|
-
w.name,
|
|
1788
|
-
COUNT(*) FILTER (WHERE NOT (wt.status = ANY(:wf_task_terminal_states))) as incomplete,
|
|
1789
|
-
COUNT(*) FILTER (WHERE wt.status = 'FAILED') as failed,
|
|
1790
|
-
COUNT(*) FILTER (WHERE wt.status = 'COMPLETED') as completed,
|
|
1791
|
-
COUNT(*) as total
|
|
1792
|
-
FROM horsies_workflows w
|
|
1793
|
-
LEFT JOIN horsies_workflow_tasks wt ON wt.workflow_id = w.id
|
|
1794
|
-
WHERE w.id = :wf_id
|
|
1795
|
-
GROUP BY w.id, w.status, w.completed_at, w.error, w.success_policy, w.name
|
|
1796
|
-
"""),
|
|
1903
|
+
GET_WORKFLOW_COMPLETION_STATUS_SQL,
|
|
1797
1904
|
{
|
|
1798
1905
|
'wf_id': workflow_id,
|
|
1799
1906
|
'wf_task_terminal_states': _WF_TASK_TERMINAL_VALUES,
|
|
@@ -1834,16 +1941,12 @@ async def _check_workflow_completion(
|
|
|
1834
1941
|
|
|
1835
1942
|
if workflow_succeeded:
|
|
1836
1943
|
await session.execute(
|
|
1837
|
-
|
|
1838
|
-
UPDATE horsies_workflows
|
|
1839
|
-
SET status = 'COMPLETED', result = :result, completed_at = NOW(), updated_at = NOW()
|
|
1840
|
-
WHERE id = :wf_id AND completed_at IS NULL
|
|
1841
|
-
"""),
|
|
1944
|
+
MARK_WORKFLOW_COMPLETED_SQL,
|
|
1842
1945
|
{'wf_id': workflow_id, 'result': final_result},
|
|
1843
1946
|
)
|
|
1844
1947
|
logger.info(
|
|
1845
1948
|
f"Workflow '{workflow_name}' ({workflow_id[:8]}) COMPLETED: "
|
|
1846
|
-
f
|
|
1949
|
+
f'{completed}/{total} tasks succeeded, {failed} failed'
|
|
1847
1950
|
)
|
|
1848
1951
|
else:
|
|
1849
1952
|
# Compute error based on success_policy semantics
|
|
@@ -1856,32 +1959,23 @@ async def _check_workflow_completion(
|
|
|
1856
1959
|
)
|
|
1857
1960
|
|
|
1858
1961
|
await session.execute(
|
|
1859
|
-
|
|
1860
|
-
UPDATE horsies_workflows
|
|
1861
|
-
SET status = 'FAILED', result = :result,
|
|
1862
|
-
error = COALESCE(:error, error),
|
|
1863
|
-
completed_at = NOW(), updated_at = NOW()
|
|
1864
|
-
WHERE id = :wf_id AND completed_at IS NULL
|
|
1865
|
-
"""),
|
|
1962
|
+
MARK_WORKFLOW_FAILED_SQL,
|
|
1866
1963
|
{'wf_id': workflow_id, 'result': final_result, 'error': error_json},
|
|
1867
1964
|
)
|
|
1868
1965
|
logger.info(
|
|
1869
1966
|
f"Workflow '{workflow_name}' ({workflow_id[:8]}) FAILED: "
|
|
1870
|
-
f
|
|
1967
|
+
f'{completed}/{total} tasks succeeded, {failed} failed'
|
|
1871
1968
|
)
|
|
1872
1969
|
|
|
1873
1970
|
# Send NOTIFY for workflow completion
|
|
1874
1971
|
await session.execute(
|
|
1875
|
-
|
|
1972
|
+
NOTIFY_WORKFLOW_DONE_SQL,
|
|
1876
1973
|
{'wf_id': workflow_id},
|
|
1877
1974
|
)
|
|
1878
1975
|
|
|
1879
1976
|
# If this is a child workflow, notify parent
|
|
1880
1977
|
parent_result = await session.execute(
|
|
1881
|
-
|
|
1882
|
-
SELECT parent_workflow_id, parent_task_index
|
|
1883
|
-
FROM horsies_workflows WHERE id = :wf_id
|
|
1884
|
-
"""),
|
|
1978
|
+
GET_PARENT_WORKFLOW_INFO_SQL,
|
|
1885
1979
|
{'wf_id': workflow_id},
|
|
1886
1980
|
)
|
|
1887
1981
|
parent_row = parent_result.fetchone()
|
|
@@ -1890,6 +1984,25 @@ async def _check_workflow_completion(
|
|
|
1890
1984
|
await _on_subworkflow_complete(session, workflow_id, broker)
|
|
1891
1985
|
|
|
1892
1986
|
|
|
1987
|
+
# -- SQL constants for _on_subworkflow_complete --
|
|
1988
|
+
|
|
1989
|
+
GET_CHILD_WORKFLOW_INFO_SQL = text("""
|
|
1990
|
+
SELECT w.status, w.result, w.error, w.parent_workflow_id, w.parent_task_index,
|
|
1991
|
+
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id) as total,
|
|
1992
|
+
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id AND status = 'COMPLETED') as completed,
|
|
1993
|
+
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id AND status = 'FAILED') as failed,
|
|
1994
|
+
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id AND status = 'SKIPPED') as skipped
|
|
1995
|
+
FROM horsies_workflows w
|
|
1996
|
+
WHERE w.id = :child_id
|
|
1997
|
+
""")
|
|
1998
|
+
|
|
1999
|
+
UPDATE_PARENT_NODE_RESULT_SQL = text("""
|
|
2000
|
+
UPDATE horsies_workflow_tasks
|
|
2001
|
+
SET status = :status, result = :result, sub_workflow_summary = :summary, completed_at = NOW()
|
|
2002
|
+
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
2003
|
+
""")
|
|
2004
|
+
|
|
2005
|
+
|
|
1893
2006
|
async def _on_subworkflow_complete(
|
|
1894
2007
|
session: AsyncSession,
|
|
1895
2008
|
child_workflow_id: str,
|
|
@@ -1903,15 +2016,7 @@ async def _on_subworkflow_complete(
|
|
|
1903
2016
|
|
|
1904
2017
|
# 1. Get child workflow info and task counts
|
|
1905
2018
|
child_result = await session.execute(
|
|
1906
|
-
|
|
1907
|
-
SELECT w.status, w.result, w.error, w.parent_workflow_id, w.parent_task_index,
|
|
1908
|
-
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id) as total,
|
|
1909
|
-
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id AND status = 'COMPLETED') as completed,
|
|
1910
|
-
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id AND status = 'FAILED') as failed,
|
|
1911
|
-
(SELECT COUNT(*) FROM horsies_workflow_tasks WHERE workflow_id = w.id AND status = 'SKIPPED') as skipped
|
|
1912
|
-
FROM horsies_workflows w
|
|
1913
|
-
WHERE w.id = :child_id
|
|
1914
|
-
"""),
|
|
2019
|
+
GET_CHILD_WORKFLOW_INFO_SQL,
|
|
1915
2020
|
{'child_id': child_workflow_id},
|
|
1916
2021
|
)
|
|
1917
2022
|
|
|
@@ -1984,11 +2089,7 @@ async def _on_subworkflow_complete(
|
|
|
1984
2089
|
|
|
1985
2090
|
# 4. Update parent node
|
|
1986
2091
|
await session.execute(
|
|
1987
|
-
|
|
1988
|
-
UPDATE horsies_workflow_tasks
|
|
1989
|
-
SET status = :status, result = :result, sub_workflow_summary = :summary, completed_at = NOW()
|
|
1990
|
-
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
1991
|
-
"""),
|
|
2092
|
+
UPDATE_PARENT_NODE_RESULT_SQL,
|
|
1992
2093
|
{
|
|
1993
2094
|
'status': parent_node_status,
|
|
1994
2095
|
'result': parent_node_result,
|
|
@@ -2018,7 +2119,7 @@ async def _on_subworkflow_complete(
|
|
|
2018
2119
|
|
|
2019
2120
|
# 6. Check if parent workflow is PAUSED
|
|
2020
2121
|
parent_status_check = await session.execute(
|
|
2021
|
-
|
|
2122
|
+
GET_WORKFLOW_STATUS_SQL,
|
|
2022
2123
|
{'wf_id': parent_wf_id},
|
|
2023
2124
|
)
|
|
2024
2125
|
parent_status_row = parent_status_check.fetchone()
|
|
@@ -2032,6 +2133,15 @@ async def _on_subworkflow_complete(
|
|
|
2032
2133
|
await _check_workflow_completion(session, parent_wf_id, broker)
|
|
2033
2134
|
|
|
2034
2135
|
|
|
2136
|
+
# -- SQL constants for _evaluate_workflow_success --
|
|
2137
|
+
|
|
2138
|
+
GET_TASK_STATUSES_SQL = text("""
|
|
2139
|
+
SELECT task_index, status
|
|
2140
|
+
FROM horsies_workflow_tasks
|
|
2141
|
+
WHERE workflow_id = :wf_id
|
|
2142
|
+
""")
|
|
2143
|
+
|
|
2144
|
+
|
|
2035
2145
|
async def _evaluate_workflow_success(
|
|
2036
2146
|
session: AsyncSession,
|
|
2037
2147
|
workflow_id: str,
|
|
@@ -2058,11 +2168,7 @@ async def _evaluate_workflow_success(
|
|
|
2058
2168
|
|
|
2059
2169
|
# Build status map by task_index
|
|
2060
2170
|
result = await session.execute(
|
|
2061
|
-
|
|
2062
|
-
SELECT task_index, status
|
|
2063
|
-
FROM horsies_workflow_tasks
|
|
2064
|
-
WHERE workflow_id = :wf_id
|
|
2065
|
-
"""),
|
|
2171
|
+
GET_TASK_STATUSES_SQL,
|
|
2066
2172
|
{'wf_id': workflow_id},
|
|
2067
2173
|
)
|
|
2068
2174
|
|
|
@@ -2090,6 +2196,23 @@ async def _evaluate_workflow_success(
|
|
|
2090
2196
|
return False
|
|
2091
2197
|
|
|
2092
2198
|
|
|
2199
|
+
# -- SQL constants for _get_workflow_failure_error --
|
|
2200
|
+
|
|
2201
|
+
GET_FIRST_FAILED_TASK_RESULT_SQL = text("""
|
|
2202
|
+
SELECT result FROM horsies_workflow_tasks
|
|
2203
|
+
WHERE workflow_id = :wf_id AND status = 'FAILED'
|
|
2204
|
+
ORDER BY task_index ASC LIMIT 1
|
|
2205
|
+
""")
|
|
2206
|
+
|
|
2207
|
+
GET_FIRST_FAILED_REQUIRED_TASK_SQL = text("""
|
|
2208
|
+
SELECT result FROM horsies_workflow_tasks
|
|
2209
|
+
WHERE workflow_id = :wf_id
|
|
2210
|
+
AND status = 'FAILED'
|
|
2211
|
+
AND task_index = ANY(:required)
|
|
2212
|
+
ORDER BY task_index ASC LIMIT 1
|
|
2213
|
+
""")
|
|
2214
|
+
|
|
2215
|
+
|
|
2093
2216
|
async def _get_workflow_failure_error(
|
|
2094
2217
|
session: AsyncSession,
|
|
2095
2218
|
workflow_id: str,
|
|
@@ -2106,11 +2229,7 @@ async def _get_workflow_failure_error(
|
|
|
2106
2229
|
if success_policy_data is None:
|
|
2107
2230
|
# Default: get first failed task's error
|
|
2108
2231
|
result = await session.execute(
|
|
2109
|
-
|
|
2110
|
-
SELECT result FROM horsies_workflow_tasks
|
|
2111
|
-
WHERE workflow_id = :wf_id AND status = 'FAILED'
|
|
2112
|
-
ORDER BY task_index ASC LIMIT 1
|
|
2113
|
-
"""),
|
|
2232
|
+
GET_FIRST_FAILED_TASK_RESULT_SQL,
|
|
2114
2233
|
{'wf_id': workflow_id},
|
|
2115
2234
|
)
|
|
2116
2235
|
row = result.fetchone()
|
|
@@ -2136,13 +2255,7 @@ async def _get_workflow_failure_error(
|
|
|
2136
2255
|
if all_required:
|
|
2137
2256
|
# Get first failed required task
|
|
2138
2257
|
result = await session.execute(
|
|
2139
|
-
|
|
2140
|
-
SELECT result FROM horsies_workflow_tasks
|
|
2141
|
-
WHERE workflow_id = :wf_id
|
|
2142
|
-
AND status = 'FAILED'
|
|
2143
|
-
AND task_index = ANY(:required)
|
|
2144
|
-
ORDER BY task_index ASC LIMIT 1
|
|
2145
|
-
"""),
|
|
2258
|
+
GET_FIRST_FAILED_REQUIRED_TASK_SQL,
|
|
2146
2259
|
{'wf_id': workflow_id, 'required': list(all_required)},
|
|
2147
2260
|
)
|
|
2148
2261
|
row = result.fetchone()
|
|
@@ -2160,6 +2273,29 @@ async def _get_workflow_failure_error(
|
|
|
2160
2273
|
)
|
|
2161
2274
|
|
|
2162
2275
|
|
|
2276
|
+
# -- SQL constants for _get_workflow_final_result --
|
|
2277
|
+
|
|
2278
|
+
GET_WORKFLOW_OUTPUT_INDEX_SQL = text(
|
|
2279
|
+
"""SELECT output_task_index FROM horsies_workflows WHERE id = :wf_id"""
|
|
2280
|
+
)
|
|
2281
|
+
|
|
2282
|
+
GET_OUTPUT_TASK_RESULT_SQL = text("""
|
|
2283
|
+
SELECT result FROM horsies_workflow_tasks
|
|
2284
|
+
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
2285
|
+
""")
|
|
2286
|
+
|
|
2287
|
+
GET_TERMINAL_TASK_RESULTS_SQL = text("""
|
|
2288
|
+
SELECT wt.node_id, wt.task_index, wt.result
|
|
2289
|
+
FROM horsies_workflow_tasks wt
|
|
2290
|
+
WHERE wt.workflow_id = :wf_id
|
|
2291
|
+
AND NOT EXISTS (
|
|
2292
|
+
SELECT 1 FROM horsies_workflow_tasks other
|
|
2293
|
+
WHERE other.workflow_id = wt.workflow_id
|
|
2294
|
+
AND wt.task_index = ANY(other.dependencies)
|
|
2295
|
+
)
|
|
2296
|
+
""")
|
|
2297
|
+
|
|
2298
|
+
|
|
2163
2299
|
async def _get_workflow_final_result(
|
|
2164
2300
|
session: AsyncSession,
|
|
2165
2301
|
workflow_id: str,
|
|
@@ -2172,7 +2308,7 @@ async def _get_workflow_final_result(
|
|
|
2172
2308
|
"""
|
|
2173
2309
|
# Check for explicit output task
|
|
2174
2310
|
wf_result = await session.execute(
|
|
2175
|
-
|
|
2311
|
+
GET_WORKFLOW_OUTPUT_INDEX_SQL,
|
|
2176
2312
|
{'wf_id': workflow_id},
|
|
2177
2313
|
)
|
|
2178
2314
|
wf_row = wf_result.fetchone()
|
|
@@ -2180,10 +2316,7 @@ async def _get_workflow_final_result(
|
|
|
2180
2316
|
if wf_row and wf_row[0] is not None:
|
|
2181
2317
|
# Return explicit output task's result
|
|
2182
2318
|
output_result = await session.execute(
|
|
2183
|
-
|
|
2184
|
-
SELECT result FROM horsies_workflow_tasks
|
|
2185
|
-
WHERE workflow_id = :wf_id AND task_index = :idx
|
|
2186
|
-
"""),
|
|
2319
|
+
GET_OUTPUT_TASK_RESULT_SQL,
|
|
2187
2320
|
{'wf_id': workflow_id, 'idx': wf_row[0]},
|
|
2188
2321
|
)
|
|
2189
2322
|
output_row = output_result.fetchone()
|
|
@@ -2191,16 +2324,7 @@ async def _get_workflow_final_result(
|
|
|
2191
2324
|
|
|
2192
2325
|
# Find terminal tasks (not in any other task's dependencies)
|
|
2193
2326
|
terminal_results = await session.execute(
|
|
2194
|
-
|
|
2195
|
-
SELECT wt.node_id, wt.task_index, wt.result
|
|
2196
|
-
FROM horsies_workflow_tasks wt
|
|
2197
|
-
WHERE wt.workflow_id = :wf_id
|
|
2198
|
-
AND NOT EXISTS (
|
|
2199
|
-
SELECT 1 FROM horsies_workflow_tasks other
|
|
2200
|
-
WHERE other.workflow_id = wt.workflow_id
|
|
2201
|
-
AND wt.task_index = ANY(other.dependencies)
|
|
2202
|
-
)
|
|
2203
|
-
"""),
|
|
2327
|
+
GET_TERMINAL_TASK_RESULTS_SQL,
|
|
2204
2328
|
{'wf_id': workflow_id},
|
|
2205
2329
|
)
|
|
2206
2330
|
|
|
@@ -2225,6 +2349,25 @@ async def _get_workflow_final_result(
|
|
|
2225
2349
|
return dumps_json(wrapped_result)
|
|
2226
2350
|
|
|
2227
2351
|
|
|
2352
|
+
# -- SQL constants for _handle_workflow_task_failure --
|
|
2353
|
+
|
|
2354
|
+
GET_WORKFLOW_ON_ERROR_SQL = text(
|
|
2355
|
+
"""SELECT on_error FROM horsies_workflows WHERE id = :wf_id"""
|
|
2356
|
+
)
|
|
2357
|
+
|
|
2358
|
+
SET_WORKFLOW_ERROR_SQL = text("""
|
|
2359
|
+
UPDATE horsies_workflows
|
|
2360
|
+
SET error = :error, updated_at = NOW()
|
|
2361
|
+
WHERE id = :wf_id AND status = 'RUNNING'
|
|
2362
|
+
""")
|
|
2363
|
+
|
|
2364
|
+
PAUSE_WORKFLOW_ON_ERROR_SQL = text("""
|
|
2365
|
+
UPDATE horsies_workflows
|
|
2366
|
+
SET status = 'PAUSED', error = :error, updated_at = NOW()
|
|
2367
|
+
WHERE id = :wf_id AND status = 'RUNNING'
|
|
2368
|
+
""")
|
|
2369
|
+
|
|
2370
|
+
|
|
2228
2371
|
async def _handle_workflow_task_failure(
|
|
2229
2372
|
session: AsyncSession,
|
|
2230
2373
|
workflow_id: str,
|
|
@@ -2241,7 +2384,7 @@ async def _handle_workflow_task_failure(
|
|
|
2241
2384
|
"""
|
|
2242
2385
|
# Get workflow's on_error policy
|
|
2243
2386
|
wf_result = await session.execute(
|
|
2244
|
-
|
|
2387
|
+
GET_WORKFLOW_ON_ERROR_SQL,
|
|
2245
2388
|
{'wf_id': workflow_id},
|
|
2246
2389
|
)
|
|
2247
2390
|
|
|
@@ -2259,11 +2402,7 @@ async def _handle_workflow_task_failure(
|
|
|
2259
2402
|
# This allows allow_failed_deps tasks to run and produce meaningful final result
|
|
2260
2403
|
# Status will be set to FAILED in _check_workflow_completion when all tasks are terminal
|
|
2261
2404
|
await session.execute(
|
|
2262
|
-
|
|
2263
|
-
UPDATE horsies_workflows
|
|
2264
|
-
SET error = :error, updated_at = NOW()
|
|
2265
|
-
WHERE id = :wf_id AND status = 'RUNNING'
|
|
2266
|
-
"""),
|
|
2405
|
+
SET_WORKFLOW_ERROR_SQL,
|
|
2267
2406
|
{'wf_id': workflow_id, 'error': error_payload},
|
|
2268
2407
|
)
|
|
2269
2408
|
return True # Continue dependency propagation
|
|
@@ -2271,17 +2410,13 @@ async def _handle_workflow_task_failure(
|
|
|
2271
2410
|
elif on_error == 'pause':
|
|
2272
2411
|
# Pause workflow for manual intervention - STOP all processing
|
|
2273
2412
|
await session.execute(
|
|
2274
|
-
|
|
2275
|
-
UPDATE horsies_workflows
|
|
2276
|
-
SET status = 'PAUSED', error = :error, updated_at = NOW()
|
|
2277
|
-
WHERE id = :wf_id AND status = 'RUNNING'
|
|
2278
|
-
"""),
|
|
2413
|
+
PAUSE_WORKFLOW_ON_ERROR_SQL,
|
|
2279
2414
|
{'wf_id': workflow_id, 'error': error_payload},
|
|
2280
2415
|
)
|
|
2281
2416
|
|
|
2282
2417
|
# Notify of pause (so clients can react via get())
|
|
2283
2418
|
await session.execute(
|
|
2284
|
-
|
|
2419
|
+
NOTIFY_WORKFLOW_DONE_SQL,
|
|
2285
2420
|
{'wf_id': workflow_id},
|
|
2286
2421
|
)
|
|
2287
2422
|
|