FlowerPower 0.11.6.19__py3-none-any.whl → 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/cfg/__init__.py +3 -3
- flowerpower/cfg/pipeline/__init__.py +5 -3
- flowerpower/cfg/project/__init__.py +3 -3
- flowerpower/cfg/project/job_queue.py +1 -128
- flowerpower/cli/__init__.py +5 -5
- flowerpower/cli/cfg.py +0 -3
- flowerpower/cli/job_queue.py +401 -133
- flowerpower/cli/pipeline.py +14 -413
- flowerpower/cli/utils.py +0 -1
- flowerpower/flowerpower.py +537 -28
- flowerpower/job_queue/__init__.py +5 -94
- flowerpower/job_queue/base.py +201 -3
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -3
- flowerpower/job_queue/rq/manager.py +388 -77
- flowerpower/pipeline/__init__.py +2 -0
- flowerpower/pipeline/base.py +2 -2
- flowerpower/pipeline/io.py +14 -16
- flowerpower/pipeline/manager.py +21 -642
- flowerpower/pipeline/pipeline.py +571 -0
- flowerpower/pipeline/registry.py +242 -10
- flowerpower/pipeline/visualizer.py +1 -2
- flowerpower/plugins/_io/__init__.py +8 -0
- flowerpower/plugins/mqtt/manager.py +6 -6
- flowerpower/settings/backend.py +0 -2
- flowerpower/settings/job_queue.py +1 -57
- flowerpower/utils/misc.py +0 -256
- flowerpower/utils/monkey.py +1 -83
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/METADATA +308 -152
- flowerpower-0.20.0.dist-info/RECORD +58 -0
- flowerpower/fs/__init__.py +0 -29
- flowerpower/fs/base.py +0 -662
- flowerpower/fs/ext.py +0 -2143
- flowerpower/fs/storage_options.py +0 -1420
- flowerpower/job_queue/apscheduler/__init__.py +0 -11
- flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
- flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
- flowerpower/job_queue/apscheduler/manager.py +0 -1051
- flowerpower/job_queue/apscheduler/setup.py +0 -554
- flowerpower/job_queue/apscheduler/trigger.py +0 -169
- flowerpower/job_queue/apscheduler/utils.py +0 -311
- flowerpower/pipeline/job_queue.py +0 -583
- flowerpower/pipeline/runner.py +0 -603
- flowerpower/plugins/io/base.py +0 -2520
- flowerpower/plugins/io/helpers/datetime.py +0 -298
- flowerpower/plugins/io/helpers/polars.py +0 -875
- flowerpower/plugins/io/helpers/pyarrow.py +0 -570
- flowerpower/plugins/io/helpers/sql.py +0 -202
- flowerpower/plugins/io/loader/__init__.py +0 -28
- flowerpower/plugins/io/loader/csv.py +0 -37
- flowerpower/plugins/io/loader/deltatable.py +0 -190
- flowerpower/plugins/io/loader/duckdb.py +0 -19
- flowerpower/plugins/io/loader/json.py +0 -37
- flowerpower/plugins/io/loader/mqtt.py +0 -159
- flowerpower/plugins/io/loader/mssql.py +0 -26
- flowerpower/plugins/io/loader/mysql.py +0 -26
- flowerpower/plugins/io/loader/oracle.py +0 -26
- flowerpower/plugins/io/loader/parquet.py +0 -35
- flowerpower/plugins/io/loader/postgres.py +0 -26
- flowerpower/plugins/io/loader/pydala.py +0 -19
- flowerpower/plugins/io/loader/sqlite.py +0 -23
- flowerpower/plugins/io/metadata.py +0 -244
- flowerpower/plugins/io/saver/__init__.py +0 -28
- flowerpower/plugins/io/saver/csv.py +0 -36
- flowerpower/plugins/io/saver/deltatable.py +0 -186
- flowerpower/plugins/io/saver/duckdb.py +0 -19
- flowerpower/plugins/io/saver/json.py +0 -36
- flowerpower/plugins/io/saver/mqtt.py +0 -28
- flowerpower/plugins/io/saver/mssql.py +0 -26
- flowerpower/plugins/io/saver/mysql.py +0 -26
- flowerpower/plugins/io/saver/oracle.py +0 -26
- flowerpower/plugins/io/saver/parquet.py +0 -36
- flowerpower/plugins/io/saver/postgres.py +0 -26
- flowerpower/plugins/io/saver/pydala.py +0 -20
- flowerpower/plugins/io/saver/sqlite.py +0 -24
- flowerpower/utils/scheduler.py +0 -311
- flowerpower-0.11.6.19.dist-info/RECORD +0 -102
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/WHEEL +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/licenses/LICENSE +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/top_level.txt +0 -0
flowerpower/cli/job_queue.py
CHANGED
@@ -1,7 +1,11 @@
|
|
1
|
+
import datetime as dt
|
2
|
+
|
3
|
+
import duration_parser
|
1
4
|
import typer
|
2
5
|
from loguru import logger
|
3
6
|
|
4
7
|
from .. import settings
|
8
|
+
from ..flowerpower import FlowerPowerProject
|
5
9
|
from ..job_queue import JobQueueManager # Adjust import as needed
|
6
10
|
from ..utils.logging import setup_logging
|
7
11
|
from .utils import parse_dict_or_list_param
|
@@ -14,9 +18,7 @@ setup_logging(level=settings.LOG_LEVEL)
|
|
14
18
|
|
15
19
|
@app.command()
|
16
20
|
def start_worker(
|
17
|
-
type: str | None = typer.Option(
|
18
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
19
|
-
),
|
21
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
20
22
|
name: str | None = typer.Option(
|
21
23
|
None, help="Name of the scheduler configuration to use"
|
22
24
|
),
|
@@ -47,7 +49,7 @@ def start_worker(
|
|
47
49
|
or can be run in the background.
|
48
50
|
|
49
51
|
Args:
|
50
|
-
type: Type of job queue backend (rq
|
52
|
+
type: Type of job queue backend (rq)
|
51
53
|
name: Name of the scheduler configuration to use
|
52
54
|
base_dir: Base directory for the scheduler configuration
|
53
55
|
background: Run the worker in the background
|
@@ -80,8 +82,8 @@ def start_worker(
|
|
80
82
|
storage_options=parsed_storage_options,
|
81
83
|
log_level=log_level,
|
82
84
|
) as worker:
|
83
|
-
if num_workers:
|
84
|
-
num_workers = worker.cfg.
|
85
|
+
if not num_workers:
|
86
|
+
num_workers = worker.cfg.num_workers
|
85
87
|
|
86
88
|
if num_workers and num_workers > 1:
|
87
89
|
worker.start_worker_pool(num_workers=num_workers, background=background)
|
@@ -89,77 +91,6 @@ def start_worker(
|
|
89
91
|
worker.start_worker(background=background)
|
90
92
|
|
91
93
|
|
92
|
-
@app.command()
|
93
|
-
def start_scheduler(
|
94
|
-
type: str | None = typer.Option(
|
95
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
96
|
-
),
|
97
|
-
name: str | None = typer.Option(
|
98
|
-
None, help="Name of the scheduler configuration to use"
|
99
|
-
),
|
100
|
-
base_dir: str | None = typer.Option(
|
101
|
-
None, help="Base directory for the scheduler configuration"
|
102
|
-
),
|
103
|
-
background: bool = typer.Option(
|
104
|
-
False, "--background", "-b", help="Run the scheduler in the background"
|
105
|
-
),
|
106
|
-
storage_options: str | None = typer.Option(
|
107
|
-
None, help="Storage options as JSON or key=value pairs"
|
108
|
-
),
|
109
|
-
log_level: str = typer.Option(
|
110
|
-
"info", help="Logging level (debug, info, warning, error, critical)"
|
111
|
-
),
|
112
|
-
interval: int = typer.Option(
|
113
|
-
60, "--interval", "-i", help="Interval for checking jobs in seconds (RQ only)"
|
114
|
-
),
|
115
|
-
):
|
116
|
-
"""
|
117
|
-
Start the scheduler process for queued jobs.
|
118
|
-
|
119
|
-
This command starts a scheduler that manages queued jobs and scheduled tasks.
|
120
|
-
Note that this is only needed for RQ workers, as APScheduler workers have
|
121
|
-
their own built-in scheduler.
|
122
|
-
|
123
|
-
Args:
|
124
|
-
type: Type of job queue backend (rq, apscheduler)
|
125
|
-
name: Name of the scheduler configuration to use
|
126
|
-
base_dir: Base directory for the scheduler configuration
|
127
|
-
background: Run the scheduler in the background
|
128
|
-
storage_options: Storage options as JSON or key=value pairs
|
129
|
-
log_level: Logging level (debug, info, warning, error, critical)
|
130
|
-
interval: Interval for checking jobs in seconds (RQ only)
|
131
|
-
|
132
|
-
Examples:
|
133
|
-
# Start a scheduler with default settings
|
134
|
-
$ flowerpower job-queue start-scheduler
|
135
|
-
|
136
|
-
# Start a scheduler for a specific backend type
|
137
|
-
$ flowerpower job-queue start-scheduler --type rq
|
138
|
-
|
139
|
-
# Run a scheduler in the background
|
140
|
-
$ flowerpower job-queue start-scheduler --background
|
141
|
-
|
142
|
-
# Set a specific scheduler check interval (RQ only)
|
143
|
-
$ flowerpower job-queue start-scheduler --interval 30
|
144
|
-
"""
|
145
|
-
parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
|
146
|
-
|
147
|
-
with JobQueueManager(
|
148
|
-
type=type,
|
149
|
-
name=name,
|
150
|
-
base_dir=base_dir,
|
151
|
-
storage_options=parsed_storage_options,
|
152
|
-
log_level=log_level,
|
153
|
-
) as worker:
|
154
|
-
if worker.cfg.backend.type != "rq":
|
155
|
-
logger.info(
|
156
|
-
f"No scheduler needed for {worker.cfg.backend.type} workers. Skipping."
|
157
|
-
)
|
158
|
-
return
|
159
|
-
|
160
|
-
worker.start_scheduler(background=background, interval=interval)
|
161
|
-
|
162
|
-
|
163
94
|
# @app.command()
|
164
95
|
# def cancel_all_jobs(
|
165
96
|
# type: str | None = None,
|
@@ -175,7 +106,7 @@ def start_scheduler(
|
|
175
106
|
# Note: This is different from deleting jobs as it only stops them from running but keeps their history.
|
176
107
|
|
177
108
|
# Args:
|
178
|
-
# type: Type of the job queue (rq
|
109
|
+
# type: Type of the job queue (rq)
|
179
110
|
# queue_name: Name of the queue (RQ only)
|
180
111
|
# name: Name of the scheduler
|
181
112
|
# base_dir: Base directory for the scheduler
|
@@ -207,7 +138,7 @@ def start_scheduler(
|
|
207
138
|
# Note: This is different from deleting schedules as it only stops them from running but keeps their configuration.
|
208
139
|
|
209
140
|
# Args:
|
210
|
-
# type: Type of the job queue (rq
|
141
|
+
# type: Type of the job queue (rq)
|
211
142
|
# name: Name of the scheduler
|
212
143
|
# base_dir: Base directory for the scheduler
|
213
144
|
# storage_options: Storage options as JSON or key=value pairs
|
@@ -231,9 +162,7 @@ def cancel_job(
|
|
231
162
|
None,
|
232
163
|
help="Name of the queue (RQ only). If provided with --all, cancels all jobs in the queue",
|
233
164
|
),
|
234
|
-
type: str | None = typer.Option(
|
235
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
236
|
-
),
|
165
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
237
166
|
name: str | None = typer.Option(
|
238
167
|
None, help="Name of the scheduler configuration to use"
|
239
168
|
),
|
@@ -258,7 +187,7 @@ def cancel_job(
|
|
258
187
|
job_id: ID of the job to cancel (ignored if --all is used)
|
259
188
|
all: Cancel all jobs instead of a specific one
|
260
189
|
queue_name: For RQ only, specifies the queue to cancel jobs from
|
261
|
-
type: Type of job queue backend (rq
|
190
|
+
type: Type of job queue backend (rq)
|
262
191
|
name: Name of the scheduler configuration to use
|
263
192
|
base_dir: Base directory for the scheduler configuration
|
264
193
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -286,11 +215,6 @@ def cancel_job(
|
|
286
215
|
storage_options=parsed_storage_options,
|
287
216
|
log_level=log_level,
|
288
217
|
) as worker:
|
289
|
-
if worker.cfg.backend.type != "rq":
|
290
|
-
logger.info(
|
291
|
-
f"Job cancellation is not supported for {worker.cfg.backend.type} workers. Skipping."
|
292
|
-
)
|
293
|
-
return
|
294
218
|
if all:
|
295
219
|
count = worker.cancel_all_jobs(
|
296
220
|
queue_name=queue_name if worker.cfg.backend.type == "rq" else None
|
@@ -322,7 +246,7 @@ def cancel_schedule(
|
|
322
246
|
Args:
|
323
247
|
schedule_id: ID of the schedule to cancel
|
324
248
|
all: If True, cancel all schedules
|
325
|
-
type: Type of the job queue (rq
|
249
|
+
type: Type of the job queue (rq)
|
326
250
|
name: Name of the scheduler
|
327
251
|
base_dir: Base directory for the scheduler
|
328
252
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -413,7 +337,7 @@ def delete_job(
|
|
413
337
|
job_id: ID of the job to delete
|
414
338
|
all: If True, delete all jobs
|
415
339
|
queue_name: Name of the queue (RQ only). If provided and all is True, delete all jobs in the queue
|
416
|
-
type: Type of the job queue (rq
|
340
|
+
type: Type of the job queue (rq)
|
417
341
|
name: Name of the scheduler
|
418
342
|
base_dir: Base directory for the scheduler
|
419
343
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -452,7 +376,7 @@ def delete_schedule(
|
|
452
376
|
Args:
|
453
377
|
schedule_id: ID of the schedule to delete
|
454
378
|
all: If True, delete all schedules
|
455
|
-
type: Type of the job queue (rq
|
379
|
+
type: Type of the job queue (rq)
|
456
380
|
name: Name of the scheduler
|
457
381
|
base_dir: Base directory for the scheduler
|
458
382
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -609,9 +533,7 @@ def delete_schedule(
|
|
609
533
|
|
610
534
|
@app.command()
|
611
535
|
def show_job_ids(
|
612
|
-
type: str | None = typer.Option(
|
613
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
614
|
-
),
|
536
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
615
537
|
name: str | None = typer.Option(
|
616
538
|
None, help="Name of the scheduler configuration to use"
|
617
539
|
),
|
@@ -632,7 +554,7 @@ def show_job_ids(
|
|
632
554
|
jobs for other operations like getting results, canceling, or deleting jobs.
|
633
555
|
|
634
556
|
Args:
|
635
|
-
type: Type of job queue backend (rq
|
557
|
+
type: Type of job queue backend (rq)
|
636
558
|
name: Name of the scheduler configuration to use
|
637
559
|
base_dir: Base directory for the scheduler configuration
|
638
560
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -673,9 +595,7 @@ def show_job_ids(
|
|
673
595
|
|
674
596
|
@app.command()
|
675
597
|
def show_schedule_ids(
|
676
|
-
type: str | None = typer.Option(
|
677
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
678
|
-
),
|
598
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
679
599
|
name: str | None = typer.Option(
|
680
600
|
None, help="Name of the scheduler configuration to use"
|
681
601
|
),
|
@@ -696,7 +616,7 @@ def show_schedule_ids(
|
|
696
616
|
identify schedules for other operations like pausing, resuming, or deleting schedules.
|
697
617
|
|
698
618
|
Args:
|
699
|
-
type: Type of job queue backend (rq
|
619
|
+
type: Type of job queue backend (rq)
|
700
620
|
name: Name of the scheduler configuration to use
|
701
621
|
base_dir: Base directory for the scheduler configuration
|
702
622
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -706,8 +626,8 @@ def show_schedule_ids(
|
|
706
626
|
# Show schedule IDs using default settings
|
707
627
|
$ flowerpower job-queue show-schedule-ids
|
708
628
|
|
709
|
-
# Show schedule IDs for
|
710
|
-
$ flowerpower job-queue show-schedule-ids --type
|
629
|
+
# Show schedule IDs for RQ
|
630
|
+
$ flowerpower job-queue show-schedule-ids --type rq
|
711
631
|
|
712
632
|
# Show schedule IDs with a custom scheduler configuration
|
713
633
|
$ flowerpower job-queue show-schedule-ids --name my-scheduler
|
@@ -771,9 +691,7 @@ def pause_schedule(
|
|
771
691
|
all: bool = typer.Option(
|
772
692
|
False, "--all", "-a", help="Pause all schedules instead of a specific one"
|
773
693
|
),
|
774
|
-
type: str | None = typer.Option(
|
775
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
776
|
-
),
|
694
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
777
695
|
name: str | None = typer.Option(
|
778
696
|
None, help="Name of the scheduler configuration to use"
|
779
697
|
),
|
@@ -797,7 +715,7 @@ def pause_schedule(
|
|
797
715
|
Args:
|
798
716
|
schedule_id: ID of the schedule to pause (ignored if --all is used)
|
799
717
|
all: Pause all schedules instead of a specific one
|
800
|
-
type: Type of job queue backend (rq
|
718
|
+
type: Type of job queue backend (rq)
|
801
719
|
name: Name of the scheduler configuration to use
|
802
720
|
base_dir: Base directory for the scheduler configuration
|
803
721
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -810,8 +728,7 @@ def pause_schedule(
|
|
810
728
|
# Pause all schedules
|
811
729
|
$ flowerpower job-queue pause-schedule --all dummy-id
|
812
730
|
|
813
|
-
#
|
814
|
-
$ flowerpower job-queue pause-schedule schedule-123456 --type apscheduler
|
731
|
+
# Note: Schedule pausing is not supported for RQ workers
|
815
732
|
"""
|
816
733
|
parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
|
817
734
|
|
@@ -822,11 +739,8 @@ def pause_schedule(
|
|
822
739
|
storage_options=parsed_storage_options,
|
823
740
|
log_level=log_level,
|
824
741
|
) as worker:
|
825
|
-
|
826
|
-
|
827
|
-
f"Schedule pausing is not supported for {worker.cfg.backend.type} workers."
|
828
|
-
)
|
829
|
-
return
|
742
|
+
logger.info("Schedule pausing is not supported for RQ workers.")
|
743
|
+
return
|
830
744
|
if all:
|
831
745
|
count = worker.pause_all_schedules()
|
832
746
|
logger.info(f"Paused {count} schedules")
|
@@ -874,9 +788,7 @@ def resume_schedule(
|
|
874
788
|
all: bool = typer.Option(
|
875
789
|
False, "--all", "-a", help="Resume all schedules instead of a specific one"
|
876
790
|
),
|
877
|
-
type: str | None = typer.Option(
|
878
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
879
|
-
),
|
791
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
880
792
|
name: str | None = typer.Option(
|
881
793
|
None, help="Name of the scheduler configuration to use"
|
882
794
|
),
|
@@ -900,7 +812,7 @@ def resume_schedule(
|
|
900
812
|
Args:
|
901
813
|
schedule_id: ID of the schedule to resume (ignored if --all is used)
|
902
814
|
all: Resume all schedules instead of a specific one
|
903
|
-
type: Type of job queue backend (rq
|
815
|
+
type: Type of job queue backend (rq)
|
904
816
|
name: Name of the scheduler configuration to use
|
905
817
|
base_dir: Base directory for the scheduler configuration
|
906
818
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -913,8 +825,7 @@ def resume_schedule(
|
|
913
825
|
# Resume all schedules
|
914
826
|
$ flowerpower job-queue resume-schedule --all dummy-id
|
915
827
|
|
916
|
-
#
|
917
|
-
$ flowerpower job-queue resume-schedule schedule-123456 --type apscheduler
|
828
|
+
# Note: Schedule resuming is not supported for RQ workers
|
918
829
|
|
919
830
|
# Set a specific logging level
|
920
831
|
$ flowerpower job-queue resume-schedule schedule-123456 --log-level debug
|
@@ -928,11 +839,8 @@ def resume_schedule(
|
|
928
839
|
storage_options=parsed_storage_options,
|
929
840
|
log_level=log_level,
|
930
841
|
) as worker:
|
931
|
-
|
932
|
-
|
933
|
-
f"Schedule resuming is not supported for {worker.cfg.backend.type} workers."
|
934
|
-
)
|
935
|
-
return
|
842
|
+
logger.info("Schedule resuming is not supported for RQ workers.")
|
843
|
+
return
|
936
844
|
if all:
|
937
845
|
count = worker.resume_all_schedules()
|
938
846
|
logger.info(f"Resumed {count} schedules")
|
@@ -946,9 +854,7 @@ def resume_schedule(
|
|
946
854
|
|
947
855
|
@app.command()
|
948
856
|
def show_jobs(
|
949
|
-
type: str | None = typer.Option(
|
950
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
951
|
-
),
|
857
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
952
858
|
queue_name: str | None = typer.Option(
|
953
859
|
None, help="Name of the queue to show jobs from (RQ only)"
|
954
860
|
),
|
@@ -973,7 +879,7 @@ def show_jobs(
|
|
973
879
|
creation time, execution time, and other details in a user-friendly format.
|
974
880
|
|
975
881
|
Args:
|
976
|
-
type: Type of job queue backend (rq
|
882
|
+
type: Type of job queue backend (rq)
|
977
883
|
queue_name: Name of the queue to show jobs from (RQ only)
|
978
884
|
name: Name of the scheduler configuration to use
|
979
885
|
base_dir: Base directory for the scheduler configuration
|
@@ -1008,9 +914,7 @@ def show_jobs(
|
|
1008
914
|
|
1009
915
|
@app.command()
|
1010
916
|
def show_schedules(
|
1011
|
-
type: str | None = typer.Option(
|
1012
|
-
None, help="Type of job queue backend (rq, apscheduler)"
|
1013
|
-
),
|
917
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
1014
918
|
name: str | None = typer.Option(
|
1015
919
|
None, help="Name of the scheduler configuration to use"
|
1016
920
|
),
|
@@ -1032,7 +936,7 @@ def show_schedules(
|
|
1032
936
|
timing configuration, status, and other details in a user-friendly format.
|
1033
937
|
|
1034
938
|
Args:
|
1035
|
-
type: Type of job queue backend (rq
|
939
|
+
type: Type of job queue backend (rq)
|
1036
940
|
name: Name of the scheduler configuration to use
|
1037
941
|
base_dir: Base directory for the scheduler configuration
|
1038
942
|
storage_options: Storage options as JSON or key=value pairs
|
@@ -1043,8 +947,8 @@ def show_schedules(
|
|
1043
947
|
# Show all schedules using default settings
|
1044
948
|
$ flowerpower job-queue show-schedules
|
1045
949
|
|
1046
|
-
# Show schedules for
|
1047
|
-
$ flowerpower job-queue show-schedules --type
|
950
|
+
# Show schedules for RQ
|
951
|
+
$ flowerpower job-queue show-schedules --type rq
|
1048
952
|
|
1049
953
|
# Display schedules in JSON format
|
1050
954
|
$ flowerpower job-queue show-schedules --format json
|
@@ -1059,3 +963,367 @@ def show_schedules(
|
|
1059
963
|
log_level=log_level,
|
1060
964
|
) as worker:
|
1061
965
|
worker.show_schedules(format=format)
|
966
|
+
|
967
|
+
|
968
|
+
@app.command()
|
969
|
+
def enqueue_pipeline(
|
970
|
+
name: str = typer.Argument(..., help="Name of the pipeline to enqueue"),
|
971
|
+
base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
|
972
|
+
inputs: str | None = typer.Option(
|
973
|
+
None, help="Input parameters as JSON, dict string, or key=value pairs"
|
974
|
+
),
|
975
|
+
final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
|
976
|
+
storage_options: str | None = typer.Option(
|
977
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
978
|
+
),
|
979
|
+
log_level: str | None = typer.Option(
|
980
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
981
|
+
),
|
982
|
+
run_in: str | None = typer.Option(
|
983
|
+
None, help="Schedule job to run after a delay (e.g., '5m', '1h', '30s')"
|
984
|
+
),
|
985
|
+
run_at: str | None = typer.Option(
|
986
|
+
None, help="Schedule job to run at a specific datetime (ISO format)"
|
987
|
+
),
|
988
|
+
):
|
989
|
+
"""
|
990
|
+
Enqueue a pipeline for execution via the job queue.
|
991
|
+
|
992
|
+
This command queues a pipeline for asynchronous execution using the configured
|
993
|
+
job queue backend (RQ). The job can be executed immediately, after a delay,
|
994
|
+
or at a specific time.
|
995
|
+
|
996
|
+
Args:
|
997
|
+
name: Name of the pipeline to enqueue
|
998
|
+
base_dir: Base directory containing pipelines and configurations
|
999
|
+
inputs: Input parameters for the pipeline
|
1000
|
+
final_vars: Final variables to request from the pipeline
|
1001
|
+
storage_options: Options for storage backends
|
1002
|
+
log_level: Set the logging level
|
1003
|
+
run_in: Delay before execution (duration format like '5m', '1h', '30s')
|
1004
|
+
run_at: Specific datetime for execution (ISO format)
|
1005
|
+
|
1006
|
+
Examples:
|
1007
|
+
# Enqueue for immediate execution
|
1008
|
+
$ flowerpower job-queue enqueue-pipeline my_pipeline
|
1009
|
+
|
1010
|
+
# Enqueue with custom inputs
|
1011
|
+
$ flowerpower job-queue enqueue-pipeline my_pipeline --inputs '{"data_path": "data/file.csv"}'
|
1012
|
+
|
1013
|
+
# Enqueue with delay
|
1014
|
+
$ flowerpower job-queue enqueue-pipeline my_pipeline --run-in "30m"
|
1015
|
+
|
1016
|
+
# Enqueue for specific time
|
1017
|
+
$ flowerpower job-queue enqueue-pipeline my_pipeline --run-at "2025-01-01T09:00:00"
|
1018
|
+
"""
|
1019
|
+
parsed_inputs = parse_dict_or_list_param(inputs, "dict")
|
1020
|
+
parsed_final_vars = parse_dict_or_list_param(final_vars, "list")
|
1021
|
+
parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
|
1022
|
+
|
1023
|
+
# Use FlowerPowerProject for consistency
|
1024
|
+
project = FlowerPowerProject.load(
|
1025
|
+
base_dir=base_dir,
|
1026
|
+
storage_options=parsed_storage_options or {},
|
1027
|
+
log_level=log_level,
|
1028
|
+
)
|
1029
|
+
|
1030
|
+
if project is None:
|
1031
|
+
logger.error(f"Failed to load FlowerPower project from {base_dir or '.'}")
|
1032
|
+
raise typer.Exit(1)
|
1033
|
+
|
1034
|
+
if project.job_queue_manager is None:
|
1035
|
+
logger.error("No job queue configured. Cannot enqueue pipeline jobs.")
|
1036
|
+
raise typer.Exit(1)
|
1037
|
+
|
1038
|
+
try:
|
1039
|
+
# Parse run_in duration if provided
|
1040
|
+
kwargs = {}
|
1041
|
+
if run_in:
|
1042
|
+
try:
|
1043
|
+
delay_seconds = duration_parser.parse(run_in).total_seconds()
|
1044
|
+
kwargs["run_in"] = delay_seconds
|
1045
|
+
except Exception as e:
|
1046
|
+
logger.error(f"Invalid duration format '{run_in}': {e}")
|
1047
|
+
raise typer.Exit(1)
|
1048
|
+
|
1049
|
+
# Parse run_at datetime if provided
|
1050
|
+
if run_at:
|
1051
|
+
try:
|
1052
|
+
run_at_dt = dt.datetime.fromisoformat(run_at)
|
1053
|
+
kwargs["run_at"] = run_at_dt
|
1054
|
+
except Exception as e:
|
1055
|
+
logger.error(f"Invalid datetime format '{run_at}': {e}")
|
1056
|
+
raise typer.Exit(1)
|
1057
|
+
|
1058
|
+
# Add pipeline execution parameters
|
1059
|
+
if parsed_inputs:
|
1060
|
+
kwargs["inputs"] = parsed_inputs
|
1061
|
+
if parsed_final_vars:
|
1062
|
+
kwargs["final_vars"] = parsed_final_vars
|
1063
|
+
|
1064
|
+
job_id = project.enqueue(name, **kwargs)
|
1065
|
+
|
1066
|
+
if run_in:
|
1067
|
+
logger.info(
|
1068
|
+
f"Pipeline '{name}' enqueued to run in {run_in}. Job ID: {job_id}"
|
1069
|
+
)
|
1070
|
+
elif run_at:
|
1071
|
+
logger.info(
|
1072
|
+
f"Pipeline '{name}' enqueued to run at {run_at}. Job ID: {job_id}"
|
1073
|
+
)
|
1074
|
+
else:
|
1075
|
+
logger.info(
|
1076
|
+
f"Pipeline '{name}' enqueued for immediate execution. Job ID: {job_id}"
|
1077
|
+
)
|
1078
|
+
|
1079
|
+
except Exception as e:
|
1080
|
+
logger.error(f"Failed to enqueue pipeline '{name}': {e}")
|
1081
|
+
raise typer.Exit(1)
|
1082
|
+
|
1083
|
+
|
1084
|
+
@app.command()
|
1085
|
+
def schedule_pipeline(
|
1086
|
+
name: str = typer.Argument(..., help="Name of the pipeline to schedule"),
|
1087
|
+
base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
|
1088
|
+
cron: str | None = typer.Option(
|
1089
|
+
None, help="Cron expression for recurring execution (e.g., '0 9 * * *')"
|
1090
|
+
),
|
1091
|
+
interval: str | None = typer.Option(
|
1092
|
+
None, help="Interval for recurring execution (e.g., '1h', '30m')"
|
1093
|
+
),
|
1094
|
+
inputs: str | None = typer.Option(
|
1095
|
+
None, help="Input parameters as JSON, dict string, or key=value pairs"
|
1096
|
+
),
|
1097
|
+
final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
|
1098
|
+
storage_options: str | None = typer.Option(
|
1099
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
1100
|
+
),
|
1101
|
+
log_level: str | None = typer.Option(
|
1102
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
1103
|
+
),
|
1104
|
+
schedule_id: str | None = typer.Option(
|
1105
|
+
None, help="Unique identifier for the schedule"
|
1106
|
+
),
|
1107
|
+
):
|
1108
|
+
"""
|
1109
|
+
Schedule a pipeline for recurring or future execution.
|
1110
|
+
|
1111
|
+
This command sets up recurring or future execution of a pipeline using cron
|
1112
|
+
expressions or interval-based scheduling via the configured job queue backend.
|
1113
|
+
|
1114
|
+
Args:
|
1115
|
+
name: Name of the pipeline to schedule
|
1116
|
+
base_dir: Base directory containing pipelines and configurations
|
1117
|
+
cron: Cron expression for scheduling (e.g., '0 9 * * *' for 9 AM daily)
|
1118
|
+
interval: Interval for recurring execution (duration format)
|
1119
|
+
inputs: Input parameters for the pipeline
|
1120
|
+
final_vars: Final variables to request from the pipeline
|
1121
|
+
storage_options: Options for storage backends
|
1122
|
+
log_level: Set the logging level
|
1123
|
+
schedule_id: Custom identifier for the schedule
|
1124
|
+
|
1125
|
+
Examples:
|
1126
|
+
# Schedule daily at 9 AM
|
1127
|
+
$ flowerpower job-queue schedule-pipeline my_pipeline --cron "0 9 * * *"
|
1128
|
+
|
1129
|
+
# Schedule every 30 minutes
|
1130
|
+
$ flowerpower job-queue schedule-pipeline my_pipeline --interval "30m"
|
1131
|
+
|
1132
|
+
# Schedule with custom inputs and ID
|
1133
|
+
$ flowerpower job-queue schedule-pipeline my_pipeline --cron "0 0 * * *" \\
|
1134
|
+
--inputs '{"env": "prod"}' --schedule-id "nightly-prod"
|
1135
|
+
"""
|
1136
|
+
if not cron and not interval:
|
1137
|
+
logger.error("Either --cron or --interval must be specified")
|
1138
|
+
raise typer.Exit(1)
|
1139
|
+
|
1140
|
+
if cron and interval:
|
1141
|
+
logger.error("Cannot specify both --cron and --interval")
|
1142
|
+
raise typer.Exit(1)
|
1143
|
+
|
1144
|
+
parsed_inputs = parse_dict_or_list_param(inputs, "dict")
|
1145
|
+
parsed_final_vars = parse_dict_or_list_param(final_vars, "list")
|
1146
|
+
parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
|
1147
|
+
|
1148
|
+
# Use FlowerPowerProject for consistency
|
1149
|
+
project = FlowerPowerProject.load(
|
1150
|
+
base_dir=base_dir,
|
1151
|
+
storage_options=parsed_storage_options or {},
|
1152
|
+
log_level=log_level,
|
1153
|
+
)
|
1154
|
+
|
1155
|
+
if project is None:
|
1156
|
+
logger.error(f"Failed to load FlowerPower project from {base_dir or '.'}")
|
1157
|
+
raise typer.Exit(1)
|
1158
|
+
|
1159
|
+
if project.job_queue_manager is None:
|
1160
|
+
logger.error("No job queue configured. Cannot schedule pipeline jobs.")
|
1161
|
+
raise typer.Exit(1)
|
1162
|
+
|
1163
|
+
try:
|
1164
|
+
# Prepare schedule parameters
|
1165
|
+
kwargs = {}
|
1166
|
+
if cron:
|
1167
|
+
kwargs["cron"] = cron
|
1168
|
+
if interval:
|
1169
|
+
try:
|
1170
|
+
interval_seconds = duration_parser.parse(interval).total_seconds()
|
1171
|
+
kwargs["interval"] = {"seconds": interval_seconds}
|
1172
|
+
except Exception as e:
|
1173
|
+
logger.error(f"Invalid interval format '{interval}': {e}")
|
1174
|
+
raise typer.Exit(1)
|
1175
|
+
|
1176
|
+
if schedule_id:
|
1177
|
+
kwargs["schedule_id"] = schedule_id
|
1178
|
+
if parsed_inputs:
|
1179
|
+
kwargs["inputs"] = parsed_inputs
|
1180
|
+
if parsed_final_vars:
|
1181
|
+
kwargs["final_vars"] = parsed_final_vars
|
1182
|
+
|
1183
|
+
schedule_result = project.schedule(name, **kwargs)
|
1184
|
+
|
1185
|
+
if cron:
|
1186
|
+
logger.info(
|
1187
|
+
f"Pipeline '{name}' scheduled with cron '{cron}'. Schedule ID: {schedule_result}"
|
1188
|
+
)
|
1189
|
+
elif interval:
|
1190
|
+
logger.info(
|
1191
|
+
f"Pipeline '{name}' scheduled every {interval}. Schedule ID: {schedule_result}"
|
1192
|
+
)
|
1193
|
+
|
1194
|
+
except Exception as e:
|
1195
|
+
logger.error(f"Failed to schedule pipeline '{name}': {e}")
|
1196
|
+
raise typer.Exit(1)
|
1197
|
+
|
1198
|
+
|
1199
|
+
@app.command()
|
1200
|
+
def run_job(
|
1201
|
+
job_id: str = typer.Argument(..., help="ID of the job to run"),
|
1202
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
1203
|
+
name: str | None = typer.Option(
|
1204
|
+
None, help="Name of the scheduler configuration to use"
|
1205
|
+
),
|
1206
|
+
base_dir: str | None = typer.Option(
|
1207
|
+
None, help="Base directory for the scheduler configuration"
|
1208
|
+
),
|
1209
|
+
storage_options: str | None = typer.Option(
|
1210
|
+
None, help="Storage options as JSON or key=value pairs"
|
1211
|
+
),
|
1212
|
+
log_level: str = typer.Option(
|
1213
|
+
"info", help="Logging level (debug, info, warning, error, critical)"
|
1214
|
+
),
|
1215
|
+
):
|
1216
|
+
"""
|
1217
|
+
Execute a specific job by its ID.
|
1218
|
+
|
1219
|
+
This command runs a job that has been previously enqueued in the job queue.
|
1220
|
+
The job will be executed immediately regardless of its original schedule.
|
1221
|
+
|
1222
|
+
Args:
|
1223
|
+
job_id: ID of the job to run
|
1224
|
+
type: Type of job queue backend (rq)
|
1225
|
+
name: Name of the scheduler configuration to use
|
1226
|
+
base_dir: Base directory for the scheduler configuration
|
1227
|
+
storage_options: Storage options as JSON or key=value pairs
|
1228
|
+
log_level: Logging level (debug, info, warning, error, critical)
|
1229
|
+
|
1230
|
+
Examples:
|
1231
|
+
# Run a specific job
|
1232
|
+
$ flowerpower job-queue run-job job-123456
|
1233
|
+
|
1234
|
+
# Run a job with a specific backend type
|
1235
|
+
$ flowerpower job-queue run-job job-123456 --type rq
|
1236
|
+
|
1237
|
+
# Run a job with debug logging
|
1238
|
+
$ flowerpower job-queue run-job job-123456 --log-level debug
|
1239
|
+
"""
|
1240
|
+
parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
|
1241
|
+
|
1242
|
+
with JobQueueManager(
|
1243
|
+
type=type,
|
1244
|
+
name=name,
|
1245
|
+
base_dir=base_dir,
|
1246
|
+
storage_options=parsed_storage_options,
|
1247
|
+
log_level=log_level,
|
1248
|
+
) as worker:
|
1249
|
+
try:
|
1250
|
+
worker.run_job(job_id)
|
1251
|
+
logger.info(f"Job '{job_id}' finished running.")
|
1252
|
+
except Exception as e:
|
1253
|
+
logger.error(f"Failed to run job '{job_id}': {e}")
|
1254
|
+
raise typer.Exit(1)
|
1255
|
+
|
1256
|
+
|
1257
|
+
@app.command()
|
1258
|
+
def list_schedules(
|
1259
|
+
type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
|
1260
|
+
name: str | None = typer.Option(
|
1261
|
+
None, help="Name of the scheduler configuration to use"
|
1262
|
+
),
|
1263
|
+
base_dir: str | None = typer.Option(
|
1264
|
+
None, help="Base directory for the scheduler configuration"
|
1265
|
+
),
|
1266
|
+
storage_options: str | None = typer.Option(
|
1267
|
+
None, help="Storage options as JSON or key=value pairs"
|
1268
|
+
),
|
1269
|
+
log_level: str = typer.Option(
|
1270
|
+
"info", help="Logging level (debug, info, warning, error, critical)"
|
1271
|
+
),
|
1272
|
+
format: str = typer.Option("table", help="Output format (table, json, yaml)"),
|
1273
|
+
show_status: bool = typer.Option(
|
1274
|
+
True, help="Show schedule status (active, paused, etc.)"
|
1275
|
+
),
|
1276
|
+
show_next_run: bool = typer.Option(True, help="Show next scheduled execution time"),
|
1277
|
+
):
|
1278
|
+
"""
|
1279
|
+
List all schedules with detailed status information.
|
1280
|
+
|
1281
|
+
This command provides enhanced schedule listing showing trigger configuration,
|
1282
|
+
status, next run time, and execution history. This is an enhanced version of
|
1283
|
+
show-schedules with more detailed information.
|
1284
|
+
|
1285
|
+
Args:
|
1286
|
+
type: Type of job queue backend (rq)
|
1287
|
+
name: Name of the scheduler configuration to use
|
1288
|
+
base_dir: Base directory for the scheduler configuration
|
1289
|
+
storage_options: Storage options as JSON or key=value pairs
|
1290
|
+
log_level: Logging level (debug, info, warning, error, critical)
|
1291
|
+
format: Output format for the schedule information
|
1292
|
+
show_status: Include schedule status information
|
1293
|
+
show_next_run: Include next execution time information
|
1294
|
+
|
1295
|
+
Examples:
|
1296
|
+
# List all schedules with full details
|
1297
|
+
$ flowerpower job-queue list-schedules
|
1298
|
+
|
1299
|
+
# List schedules in JSON format
|
1300
|
+
$ flowerpower job-queue list-schedules --format json
|
1301
|
+
|
1302
|
+
# List schedules without status information
|
1303
|
+
$ flowerpower job-queue list-schedules --no-show-status
|
1304
|
+
|
1305
|
+
# List schedules for a specific backend
|
1306
|
+
$ flowerpower job-queue list-schedules --type rq
|
1307
|
+
"""
|
1308
|
+
parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
|
1309
|
+
|
1310
|
+
with JobQueueManager(
|
1311
|
+
type=type,
|
1312
|
+
name=name,
|
1313
|
+
base_dir=base_dir,
|
1314
|
+
storage_options=parsed_storage_options,
|
1315
|
+
log_level=log_level,
|
1316
|
+
) as worker:
|
1317
|
+
# This will use the enhanced show_schedules method with additional options
|
1318
|
+
try:
|
1319
|
+
worker.show_schedules(
|
1320
|
+
format=format,
|
1321
|
+
show_status=show_status,
|
1322
|
+
show_next_run=show_next_run,
|
1323
|
+
)
|
1324
|
+
except TypeError:
|
1325
|
+
# Fallback if the show_schedules method doesn't support new parameters
|
1326
|
+
logger.warning(
|
1327
|
+
"Using basic schedule listing (enhanced options not supported)"
|
1328
|
+
)
|
1329
|
+
worker.show_schedules(format=format)
|