FlowerPower 0.11.6.20__py3-none-any.whl → 0.20.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. flowerpower/cfg/__init__.py +3 -3
  2. flowerpower/cfg/pipeline/__init__.py +5 -3
  3. flowerpower/cfg/project/__init__.py +3 -3
  4. flowerpower/cfg/project/job_queue.py +1 -128
  5. flowerpower/cli/__init__.py +5 -5
  6. flowerpower/cli/cfg.py +0 -3
  7. flowerpower/cli/job_queue.py +400 -132
  8. flowerpower/cli/pipeline.py +14 -413
  9. flowerpower/cli/utils.py +0 -1
  10. flowerpower/flowerpower.py +537 -28
  11. flowerpower/job_queue/__init__.py +5 -94
  12. flowerpower/job_queue/base.py +201 -3
  13. flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -3
  14. flowerpower/job_queue/rq/manager.py +388 -77
  15. flowerpower/pipeline/__init__.py +2 -0
  16. flowerpower/pipeline/base.py +2 -2
  17. flowerpower/pipeline/io.py +14 -16
  18. flowerpower/pipeline/manager.py +21 -642
  19. flowerpower/pipeline/pipeline.py +571 -0
  20. flowerpower/pipeline/registry.py +242 -10
  21. flowerpower/pipeline/visualizer.py +1 -2
  22. flowerpower/plugins/_io/__init__.py +8 -0
  23. flowerpower/plugins/mqtt/manager.py +6 -6
  24. flowerpower/settings/backend.py +0 -2
  25. flowerpower/settings/job_queue.py +1 -57
  26. flowerpower/utils/misc.py +0 -256
  27. flowerpower/utils/monkey.py +1 -83
  28. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/METADATA +308 -152
  29. flowerpower-0.20.0.dist-info/RECORD +58 -0
  30. flowerpower/fs/__init__.py +0 -29
  31. flowerpower/fs/base.py +0 -662
  32. flowerpower/fs/ext.py +0 -2143
  33. flowerpower/fs/storage_options.py +0 -1420
  34. flowerpower/job_queue/apscheduler/__init__.py +0 -11
  35. flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
  36. flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
  37. flowerpower/job_queue/apscheduler/manager.py +0 -1051
  38. flowerpower/job_queue/apscheduler/setup.py +0 -554
  39. flowerpower/job_queue/apscheduler/trigger.py +0 -169
  40. flowerpower/job_queue/apscheduler/utils.py +0 -311
  41. flowerpower/pipeline/job_queue.py +0 -583
  42. flowerpower/pipeline/runner.py +0 -603
  43. flowerpower/plugins/io/base.py +0 -2520
  44. flowerpower/plugins/io/helpers/datetime.py +0 -298
  45. flowerpower/plugins/io/helpers/polars.py +0 -875
  46. flowerpower/plugins/io/helpers/pyarrow.py +0 -570
  47. flowerpower/plugins/io/helpers/sql.py +0 -202
  48. flowerpower/plugins/io/loader/__init__.py +0 -28
  49. flowerpower/plugins/io/loader/csv.py +0 -37
  50. flowerpower/plugins/io/loader/deltatable.py +0 -190
  51. flowerpower/plugins/io/loader/duckdb.py +0 -19
  52. flowerpower/plugins/io/loader/json.py +0 -37
  53. flowerpower/plugins/io/loader/mqtt.py +0 -159
  54. flowerpower/plugins/io/loader/mssql.py +0 -26
  55. flowerpower/plugins/io/loader/mysql.py +0 -26
  56. flowerpower/plugins/io/loader/oracle.py +0 -26
  57. flowerpower/plugins/io/loader/parquet.py +0 -35
  58. flowerpower/plugins/io/loader/postgres.py +0 -26
  59. flowerpower/plugins/io/loader/pydala.py +0 -19
  60. flowerpower/plugins/io/loader/sqlite.py +0 -23
  61. flowerpower/plugins/io/metadata.py +0 -244
  62. flowerpower/plugins/io/saver/__init__.py +0 -28
  63. flowerpower/plugins/io/saver/csv.py +0 -36
  64. flowerpower/plugins/io/saver/deltatable.py +0 -186
  65. flowerpower/plugins/io/saver/duckdb.py +0 -19
  66. flowerpower/plugins/io/saver/json.py +0 -36
  67. flowerpower/plugins/io/saver/mqtt.py +0 -28
  68. flowerpower/plugins/io/saver/mssql.py +0 -26
  69. flowerpower/plugins/io/saver/mysql.py +0 -26
  70. flowerpower/plugins/io/saver/oracle.py +0 -26
  71. flowerpower/plugins/io/saver/parquet.py +0 -36
  72. flowerpower/plugins/io/saver/postgres.py +0 -26
  73. flowerpower/plugins/io/saver/pydala.py +0 -20
  74. flowerpower/plugins/io/saver/sqlite.py +0 -24
  75. flowerpower/utils/scheduler.py +0 -311
  76. flowerpower-0.11.6.20.dist-info/RECORD +0 -102
  77. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/WHEEL +0 -0
  78. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/entry_points.txt +0 -0
  79. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/licenses/LICENSE +0 -0
  80. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,11 @@
1
+ import datetime as dt
2
+
3
+ import duration_parser
1
4
  import typer
2
5
  from loguru import logger
3
6
 
4
7
  from .. import settings
8
+ from ..flowerpower import FlowerPowerProject
5
9
  from ..job_queue import JobQueueManager # Adjust import as needed
6
10
  from ..utils.logging import setup_logging
7
11
  from .utils import parse_dict_or_list_param
@@ -14,9 +18,7 @@ setup_logging(level=settings.LOG_LEVEL)
14
18
 
15
19
  @app.command()
16
20
  def start_worker(
17
- type: str | None = typer.Option(
18
- None, help="Type of job queue backend (rq, apscheduler)"
19
- ),
21
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
20
22
  name: str | None = typer.Option(
21
23
  None, help="Name of the scheduler configuration to use"
22
24
  ),
@@ -47,7 +49,7 @@ def start_worker(
47
49
  or can be run in the background.
48
50
 
49
51
  Args:
50
- type: Type of job queue backend (rq, apscheduler)
52
+ type: Type of job queue backend (rq)
51
53
  name: Name of the scheduler configuration to use
52
54
  base_dir: Base directory for the scheduler configuration
53
55
  background: Run the worker in the background
@@ -80,7 +82,7 @@ def start_worker(
80
82
  storage_options=parsed_storage_options,
81
83
  log_level=log_level,
82
84
  ) as worker:
83
- if num_workers:
85
+ if not num_workers:
84
86
  num_workers = worker.cfg.num_workers
85
87
 
86
88
  if num_workers and num_workers > 1:
@@ -89,77 +91,6 @@ def start_worker(
89
91
  worker.start_worker(background=background)
90
92
 
91
93
 
92
- @app.command()
93
- def start_scheduler(
94
- type: str | None = typer.Option(
95
- None, help="Type of job queue backend (rq, apscheduler)"
96
- ),
97
- name: str | None = typer.Option(
98
- None, help="Name of the scheduler configuration to use"
99
- ),
100
- base_dir: str | None = typer.Option(
101
- None, help="Base directory for the scheduler configuration"
102
- ),
103
- background: bool = typer.Option(
104
- False, "--background", "-b", help="Run the scheduler in the background"
105
- ),
106
- storage_options: str | None = typer.Option(
107
- None, help="Storage options as JSON or key=value pairs"
108
- ),
109
- log_level: str = typer.Option(
110
- "info", help="Logging level (debug, info, warning, error, critical)"
111
- ),
112
- interval: int = typer.Option(
113
- 60, "--interval", "-i", help="Interval for checking jobs in seconds (RQ only)"
114
- ),
115
- ):
116
- """
117
- Start the scheduler process for queued jobs.
118
-
119
- This command starts a scheduler that manages queued jobs and scheduled tasks.
120
- Note that this is only needed for RQ workers, as APScheduler workers have
121
- their own built-in scheduler.
122
-
123
- Args:
124
- type: Type of job queue backend (rq, apscheduler)
125
- name: Name of the scheduler configuration to use
126
- base_dir: Base directory for the scheduler configuration
127
- background: Run the scheduler in the background
128
- storage_options: Storage options as JSON or key=value pairs
129
- log_level: Logging level (debug, info, warning, error, critical)
130
- interval: Interval for checking jobs in seconds (RQ only)
131
-
132
- Examples:
133
- # Start a scheduler with default settings
134
- $ flowerpower job-queue start-scheduler
135
-
136
- # Start a scheduler for a specific backend type
137
- $ flowerpower job-queue start-scheduler --type rq
138
-
139
- # Run a scheduler in the background
140
- $ flowerpower job-queue start-scheduler --background
141
-
142
- # Set a specific scheduler check interval (RQ only)
143
- $ flowerpower job-queue start-scheduler --interval 30
144
- """
145
- parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
146
-
147
- with JobQueueManager(
148
- type=type,
149
- name=name,
150
- base_dir=base_dir,
151
- storage_options=parsed_storage_options,
152
- log_level=log_level,
153
- ) as worker:
154
- if worker.cfg.backend.type != "rq":
155
- logger.info(
156
- f"No scheduler needed for {worker.cfg.backend.type} workers. Skipping."
157
- )
158
- return
159
-
160
- worker.start_scheduler(background=background, interval=interval)
161
-
162
-
163
94
  # @app.command()
164
95
  # def cancel_all_jobs(
165
96
  # type: str | None = None,
@@ -175,7 +106,7 @@ def start_scheduler(
175
106
  # Note: This is different from deleting jobs as it only stops them from running but keeps their history.
176
107
 
177
108
  # Args:
178
- # type: Type of the job queue (rq, apscheduler)
109
+ # type: Type of the job queue (rq)
179
110
  # queue_name: Name of the queue (RQ only)
180
111
  # name: Name of the scheduler
181
112
  # base_dir: Base directory for the scheduler
@@ -207,7 +138,7 @@ def start_scheduler(
207
138
  # Note: This is different from deleting schedules as it only stops them from running but keeps their configuration.
208
139
 
209
140
  # Args:
210
- # type: Type of the job queue (rq, apscheduler)
141
+ # type: Type of the job queue (rq)
211
142
  # name: Name of the scheduler
212
143
  # base_dir: Base directory for the scheduler
213
144
  # storage_options: Storage options as JSON or key=value pairs
@@ -231,9 +162,7 @@ def cancel_job(
231
162
  None,
232
163
  help="Name of the queue (RQ only). If provided with --all, cancels all jobs in the queue",
233
164
  ),
234
- type: str | None = typer.Option(
235
- None, help="Type of job queue backend (rq, apscheduler)"
236
- ),
165
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
237
166
  name: str | None = typer.Option(
238
167
  None, help="Name of the scheduler configuration to use"
239
168
  ),
@@ -258,7 +187,7 @@ def cancel_job(
258
187
  job_id: ID of the job to cancel (ignored if --all is used)
259
188
  all: Cancel all jobs instead of a specific one
260
189
  queue_name: For RQ only, specifies the queue to cancel jobs from
261
- type: Type of job queue backend (rq, apscheduler)
190
+ type: Type of job queue backend (rq)
262
191
  name: Name of the scheduler configuration to use
263
192
  base_dir: Base directory for the scheduler configuration
264
193
  storage_options: Storage options as JSON or key=value pairs
@@ -286,11 +215,6 @@ def cancel_job(
286
215
  storage_options=parsed_storage_options,
287
216
  log_level=log_level,
288
217
  ) as worker:
289
- if worker.cfg.backend.type != "rq":
290
- logger.info(
291
- f"Job cancellation is not supported for {worker.cfg.backend.type} workers. Skipping."
292
- )
293
- return
294
218
  if all:
295
219
  count = worker.cancel_all_jobs(
296
220
  queue_name=queue_name if worker.cfg.backend.type == "rq" else None
@@ -322,7 +246,7 @@ def cancel_schedule(
322
246
  Args:
323
247
  schedule_id: ID of the schedule to cancel
324
248
  all: If True, cancel all schedules
325
- type: Type of the job queue (rq, apscheduler)
249
+ type: Type of the job queue (rq)
326
250
  name: Name of the scheduler
327
251
  base_dir: Base directory for the scheduler
328
252
  storage_options: Storage options as JSON or key=value pairs
@@ -413,7 +337,7 @@ def delete_job(
413
337
  job_id: ID of the job to delete
414
338
  all: If True, delete all jobs
415
339
  queue_name: Name of the queue (RQ only). If provided and all is True, delete all jobs in the queue
416
- type: Type of the job queue (rq, apscheduler)
340
+ type: Type of the job queue (rq)
417
341
  name: Name of the scheduler
418
342
  base_dir: Base directory for the scheduler
419
343
  storage_options: Storage options as JSON or key=value pairs
@@ -452,7 +376,7 @@ def delete_schedule(
452
376
  Args:
453
377
  schedule_id: ID of the schedule to delete
454
378
  all: If True, delete all schedules
455
- type: Type of the job queue (rq, apscheduler)
379
+ type: Type of the job queue (rq)
456
380
  name: Name of the scheduler
457
381
  base_dir: Base directory for the scheduler
458
382
  storage_options: Storage options as JSON or key=value pairs
@@ -609,9 +533,7 @@ def delete_schedule(
609
533
 
610
534
  @app.command()
611
535
  def show_job_ids(
612
- type: str | None = typer.Option(
613
- None, help="Type of job queue backend (rq, apscheduler)"
614
- ),
536
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
615
537
  name: str | None = typer.Option(
616
538
  None, help="Name of the scheduler configuration to use"
617
539
  ),
@@ -632,7 +554,7 @@ def show_job_ids(
632
554
  jobs for other operations like getting results, canceling, or deleting jobs.
633
555
 
634
556
  Args:
635
- type: Type of job queue backend (rq, apscheduler)
557
+ type: Type of job queue backend (rq)
636
558
  name: Name of the scheduler configuration to use
637
559
  base_dir: Base directory for the scheduler configuration
638
560
  storage_options: Storage options as JSON or key=value pairs
@@ -673,9 +595,7 @@ def show_job_ids(
673
595
 
674
596
  @app.command()
675
597
  def show_schedule_ids(
676
- type: str | None = typer.Option(
677
- None, help="Type of job queue backend (rq, apscheduler)"
678
- ),
598
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
679
599
  name: str | None = typer.Option(
680
600
  None, help="Name of the scheduler configuration to use"
681
601
  ),
@@ -696,7 +616,7 @@ def show_schedule_ids(
696
616
  identify schedules for other operations like pausing, resuming, or deleting schedules.
697
617
 
698
618
  Args:
699
- type: Type of job queue backend (rq, apscheduler)
619
+ type: Type of job queue backend (rq)
700
620
  name: Name of the scheduler configuration to use
701
621
  base_dir: Base directory for the scheduler configuration
702
622
  storage_options: Storage options as JSON or key=value pairs
@@ -706,8 +626,8 @@ def show_schedule_ids(
706
626
  # Show schedule IDs using default settings
707
627
  $ flowerpower job-queue show-schedule-ids
708
628
 
709
- # Show schedule IDs for a specific queue type
710
- $ flowerpower job-queue show-schedule-ids --type apscheduler
629
+ # Show schedule IDs for RQ
630
+ $ flowerpower job-queue show-schedule-ids --type rq
711
631
 
712
632
  # Show schedule IDs with a custom scheduler configuration
713
633
  $ flowerpower job-queue show-schedule-ids --name my-scheduler
@@ -771,9 +691,7 @@ def pause_schedule(
771
691
  all: bool = typer.Option(
772
692
  False, "--all", "-a", help="Pause all schedules instead of a specific one"
773
693
  ),
774
- type: str | None = typer.Option(
775
- None, help="Type of job queue backend (rq, apscheduler)"
776
- ),
694
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
777
695
  name: str | None = typer.Option(
778
696
  None, help="Name of the scheduler configuration to use"
779
697
  ),
@@ -797,7 +715,7 @@ def pause_schedule(
797
715
  Args:
798
716
  schedule_id: ID of the schedule to pause (ignored if --all is used)
799
717
  all: Pause all schedules instead of a specific one
800
- type: Type of job queue backend (rq, apscheduler)
718
+ type: Type of job queue backend (rq)
801
719
  name: Name of the scheduler configuration to use
802
720
  base_dir: Base directory for the scheduler configuration
803
721
  storage_options: Storage options as JSON or key=value pairs
@@ -810,8 +728,7 @@ def pause_schedule(
810
728
  # Pause all schedules
811
729
  $ flowerpower job-queue pause-schedule --all dummy-id
812
730
 
813
- # Specify the backend type explicitly
814
- $ flowerpower job-queue pause-schedule schedule-123456 --type apscheduler
731
+ # Note: Schedule pausing is not supported for RQ workers
815
732
  """
816
733
  parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
817
734
 
@@ -822,11 +739,8 @@ def pause_schedule(
822
739
  storage_options=parsed_storage_options,
823
740
  log_level=log_level,
824
741
  ) as worker:
825
- if worker.cfg.backend.type != "apscheduler":
826
- logger.info(
827
- f"Schedule pausing is not supported for {worker.cfg.backend.type} workers."
828
- )
829
- return
742
+ logger.info("Schedule pausing is not supported for RQ workers.")
743
+ return
830
744
  if all:
831
745
  count = worker.pause_all_schedules()
832
746
  logger.info(f"Paused {count} schedules")
@@ -874,9 +788,7 @@ def resume_schedule(
874
788
  all: bool = typer.Option(
875
789
  False, "--all", "-a", help="Resume all schedules instead of a specific one"
876
790
  ),
877
- type: str | None = typer.Option(
878
- None, help="Type of job queue backend (rq, apscheduler)"
879
- ),
791
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
880
792
  name: str | None = typer.Option(
881
793
  None, help="Name of the scheduler configuration to use"
882
794
  ),
@@ -900,7 +812,7 @@ def resume_schedule(
900
812
  Args:
901
813
  schedule_id: ID of the schedule to resume (ignored if --all is used)
902
814
  all: Resume all schedules instead of a specific one
903
- type: Type of job queue backend (rq, apscheduler)
815
+ type: Type of job queue backend (rq)
904
816
  name: Name of the scheduler configuration to use
905
817
  base_dir: Base directory for the scheduler configuration
906
818
  storage_options: Storage options as JSON or key=value pairs
@@ -913,8 +825,7 @@ def resume_schedule(
913
825
  # Resume all schedules
914
826
  $ flowerpower job-queue resume-schedule --all dummy-id
915
827
 
916
- # Specify the backend type explicitly
917
- $ flowerpower job-queue resume-schedule schedule-123456 --type apscheduler
828
+ # Note: Schedule resuming is not supported for RQ workers
918
829
 
919
830
  # Set a specific logging level
920
831
  $ flowerpower job-queue resume-schedule schedule-123456 --log-level debug
@@ -928,11 +839,8 @@ def resume_schedule(
928
839
  storage_options=parsed_storage_options,
929
840
  log_level=log_level,
930
841
  ) as worker:
931
- if worker.cfg.backend.type != "apscheduler":
932
- logger.info(
933
- f"Schedule resuming is not supported for {worker.cfg.backend.type} workers."
934
- )
935
- return
842
+ logger.info("Schedule resuming is not supported for RQ workers.")
843
+ return
936
844
  if all:
937
845
  count = worker.resume_all_schedules()
938
846
  logger.info(f"Resumed {count} schedules")
@@ -946,9 +854,7 @@ def resume_schedule(
946
854
 
947
855
  @app.command()
948
856
  def show_jobs(
949
- type: str | None = typer.Option(
950
- None, help="Type of job queue backend (rq, apscheduler)"
951
- ),
857
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
952
858
  queue_name: str | None = typer.Option(
953
859
  None, help="Name of the queue to show jobs from (RQ only)"
954
860
  ),
@@ -973,7 +879,7 @@ def show_jobs(
973
879
  creation time, execution time, and other details in a user-friendly format.
974
880
 
975
881
  Args:
976
- type: Type of job queue backend (rq, apscheduler)
882
+ type: Type of job queue backend (rq)
977
883
  queue_name: Name of the queue to show jobs from (RQ only)
978
884
  name: Name of the scheduler configuration to use
979
885
  base_dir: Base directory for the scheduler configuration
@@ -1008,9 +914,7 @@ def show_jobs(
1008
914
 
1009
915
  @app.command()
1010
916
  def show_schedules(
1011
- type: str | None = typer.Option(
1012
- None, help="Type of job queue backend (rq, apscheduler)"
1013
- ),
917
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
1014
918
  name: str | None = typer.Option(
1015
919
  None, help="Name of the scheduler configuration to use"
1016
920
  ),
@@ -1032,7 +936,7 @@ def show_schedules(
1032
936
  timing configuration, status, and other details in a user-friendly format.
1033
937
 
1034
938
  Args:
1035
- type: Type of job queue backend (rq, apscheduler)
939
+ type: Type of job queue backend (rq)
1036
940
  name: Name of the scheduler configuration to use
1037
941
  base_dir: Base directory for the scheduler configuration
1038
942
  storage_options: Storage options as JSON or key=value pairs
@@ -1043,8 +947,8 @@ def show_schedules(
1043
947
  # Show all schedules using default settings
1044
948
  $ flowerpower job-queue show-schedules
1045
949
 
1046
- # Show schedules for a specific queue type
1047
- $ flowerpower job-queue show-schedules --type apscheduler
950
+ # Show schedules for RQ
951
+ $ flowerpower job-queue show-schedules --type rq
1048
952
 
1049
953
  # Display schedules in JSON format
1050
954
  $ flowerpower job-queue show-schedules --format json
@@ -1059,3 +963,367 @@ def show_schedules(
1059
963
  log_level=log_level,
1060
964
  ) as worker:
1061
965
  worker.show_schedules(format=format)
966
+
967
+
968
+ @app.command()
969
+ def enqueue_pipeline(
970
+ name: str = typer.Argument(..., help="Name of the pipeline to enqueue"),
971
+ base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
972
+ inputs: str | None = typer.Option(
973
+ None, help="Input parameters as JSON, dict string, or key=value pairs"
974
+ ),
975
+ final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
976
+ storage_options: str | None = typer.Option(
977
+ None, help="Storage options as JSON, dict string, or key=value pairs"
978
+ ),
979
+ log_level: str | None = typer.Option(
980
+ None, help="Logging level (debug, info, warning, error, critical)"
981
+ ),
982
+ run_in: str | None = typer.Option(
983
+ None, help="Schedule job to run after a delay (e.g., '5m', '1h', '30s')"
984
+ ),
985
+ run_at: str | None = typer.Option(
986
+ None, help="Schedule job to run at a specific datetime (ISO format)"
987
+ ),
988
+ ):
989
+ """
990
+ Enqueue a pipeline for execution via the job queue.
991
+
992
+ This command queues a pipeline for asynchronous execution using the configured
993
+ job queue backend (RQ). The job can be executed immediately, after a delay,
994
+ or at a specific time.
995
+
996
+ Args:
997
+ name: Name of the pipeline to enqueue
998
+ base_dir: Base directory containing pipelines and configurations
999
+ inputs: Input parameters for the pipeline
1000
+ final_vars: Final variables to request from the pipeline
1001
+ storage_options: Options for storage backends
1002
+ log_level: Set the logging level
1003
+ run_in: Delay before execution (duration format like '5m', '1h', '30s')
1004
+ run_at: Specific datetime for execution (ISO format)
1005
+
1006
+ Examples:
1007
+ # Enqueue for immediate execution
1008
+ $ flowerpower job-queue enqueue-pipeline my_pipeline
1009
+
1010
+ # Enqueue with custom inputs
1011
+ $ flowerpower job-queue enqueue-pipeline my_pipeline --inputs '{"data_path": "data/file.csv"}'
1012
+
1013
+ # Enqueue with delay
1014
+ $ flowerpower job-queue enqueue-pipeline my_pipeline --run-in "30m"
1015
+
1016
+ # Enqueue for specific time
1017
+ $ flowerpower job-queue enqueue-pipeline my_pipeline --run-at "2025-01-01T09:00:00"
1018
+ """
1019
+ parsed_inputs = parse_dict_or_list_param(inputs, "dict")
1020
+ parsed_final_vars = parse_dict_or_list_param(final_vars, "list")
1021
+ parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
1022
+
1023
+ # Use FlowerPowerProject for consistency
1024
+ project = FlowerPowerProject.load(
1025
+ base_dir=base_dir,
1026
+ storage_options=parsed_storage_options or {},
1027
+ log_level=log_level,
1028
+ )
1029
+
1030
+ if project is None:
1031
+ logger.error(f"Failed to load FlowerPower project from {base_dir or '.'}")
1032
+ raise typer.Exit(1)
1033
+
1034
+ if project.job_queue_manager is None:
1035
+ logger.error("No job queue configured. Cannot enqueue pipeline jobs.")
1036
+ raise typer.Exit(1)
1037
+
1038
+ try:
1039
+ # Parse run_in duration if provided
1040
+ kwargs = {}
1041
+ if run_in:
1042
+ try:
1043
+ delay_seconds = duration_parser.parse(run_in).total_seconds()
1044
+ kwargs["run_in"] = delay_seconds
1045
+ except Exception as e:
1046
+ logger.error(f"Invalid duration format '{run_in}': {e}")
1047
+ raise typer.Exit(1)
1048
+
1049
+ # Parse run_at datetime if provided
1050
+ if run_at:
1051
+ try:
1052
+ run_at_dt = dt.datetime.fromisoformat(run_at)
1053
+ kwargs["run_at"] = run_at_dt
1054
+ except Exception as e:
1055
+ logger.error(f"Invalid datetime format '{run_at}': {e}")
1056
+ raise typer.Exit(1)
1057
+
1058
+ # Add pipeline execution parameters
1059
+ if parsed_inputs:
1060
+ kwargs["inputs"] = parsed_inputs
1061
+ if parsed_final_vars:
1062
+ kwargs["final_vars"] = parsed_final_vars
1063
+
1064
+ job_id = project.enqueue(name, **kwargs)
1065
+
1066
+ if run_in:
1067
+ logger.info(
1068
+ f"Pipeline '{name}' enqueued to run in {run_in}. Job ID: {job_id}"
1069
+ )
1070
+ elif run_at:
1071
+ logger.info(
1072
+ f"Pipeline '{name}' enqueued to run at {run_at}. Job ID: {job_id}"
1073
+ )
1074
+ else:
1075
+ logger.info(
1076
+ f"Pipeline '{name}' enqueued for immediate execution. Job ID: {job_id}"
1077
+ )
1078
+
1079
+ except Exception as e:
1080
+ logger.error(f"Failed to enqueue pipeline '{name}': {e}")
1081
+ raise typer.Exit(1)
1082
+
1083
+
1084
+ @app.command()
1085
+ def schedule_pipeline(
1086
+ name: str = typer.Argument(..., help="Name of the pipeline to schedule"),
1087
+ base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
1088
+ cron: str | None = typer.Option(
1089
+ None, help="Cron expression for recurring execution (e.g., '0 9 * * *')"
1090
+ ),
1091
+ interval: str | None = typer.Option(
1092
+ None, help="Interval for recurring execution (e.g., '1h', '30m')"
1093
+ ),
1094
+ inputs: str | None = typer.Option(
1095
+ None, help="Input parameters as JSON, dict string, or key=value pairs"
1096
+ ),
1097
+ final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
1098
+ storage_options: str | None = typer.Option(
1099
+ None, help="Storage options as JSON, dict string, or key=value pairs"
1100
+ ),
1101
+ log_level: str | None = typer.Option(
1102
+ None, help="Logging level (debug, info, warning, error, critical)"
1103
+ ),
1104
+ schedule_id: str | None = typer.Option(
1105
+ None, help="Unique identifier for the schedule"
1106
+ ),
1107
+ ):
1108
+ """
1109
+ Schedule a pipeline for recurring or future execution.
1110
+
1111
+ This command sets up recurring or future execution of a pipeline using cron
1112
+ expressions or interval-based scheduling via the configured job queue backend.
1113
+
1114
+ Args:
1115
+ name: Name of the pipeline to schedule
1116
+ base_dir: Base directory containing pipelines and configurations
1117
+ cron: Cron expression for scheduling (e.g., '0 9 * * *' for 9 AM daily)
1118
+ interval: Interval for recurring execution (duration format)
1119
+ inputs: Input parameters for the pipeline
1120
+ final_vars: Final variables to request from the pipeline
1121
+ storage_options: Options for storage backends
1122
+ log_level: Set the logging level
1123
+ schedule_id: Custom identifier for the schedule
1124
+
1125
+ Examples:
1126
+ # Schedule daily at 9 AM
1127
+ $ flowerpower job-queue schedule-pipeline my_pipeline --cron "0 9 * * *"
1128
+
1129
+ # Schedule every 30 minutes
1130
+ $ flowerpower job-queue schedule-pipeline my_pipeline --interval "30m"
1131
+
1132
+ # Schedule with custom inputs and ID
1133
+ $ flowerpower job-queue schedule-pipeline my_pipeline --cron "0 0 * * *" \\
1134
+ --inputs '{"env": "prod"}' --schedule-id "nightly-prod"
1135
+ """
1136
+ if not cron and not interval:
1137
+ logger.error("Either --cron or --interval must be specified")
1138
+ raise typer.Exit(1)
1139
+
1140
+ if cron and interval:
1141
+ logger.error("Cannot specify both --cron and --interval")
1142
+ raise typer.Exit(1)
1143
+
1144
+ parsed_inputs = parse_dict_or_list_param(inputs, "dict")
1145
+ parsed_final_vars = parse_dict_or_list_param(final_vars, "list")
1146
+ parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
1147
+
1148
+ # Use FlowerPowerProject for consistency
1149
+ project = FlowerPowerProject.load(
1150
+ base_dir=base_dir,
1151
+ storage_options=parsed_storage_options or {},
1152
+ log_level=log_level,
1153
+ )
1154
+
1155
+ if project is None:
1156
+ logger.error(f"Failed to load FlowerPower project from {base_dir or '.'}")
1157
+ raise typer.Exit(1)
1158
+
1159
+ if project.job_queue_manager is None:
1160
+ logger.error("No job queue configured. Cannot schedule pipeline jobs.")
1161
+ raise typer.Exit(1)
1162
+
1163
+ try:
1164
+ # Prepare schedule parameters
1165
+ kwargs = {}
1166
+ if cron:
1167
+ kwargs["cron"] = cron
1168
+ if interval:
1169
+ try:
1170
+ interval_seconds = duration_parser.parse(interval).total_seconds()
1171
+ kwargs["interval"] = {"seconds": interval_seconds}
1172
+ except Exception as e:
1173
+ logger.error(f"Invalid interval format '{interval}': {e}")
1174
+ raise typer.Exit(1)
1175
+
1176
+ if schedule_id:
1177
+ kwargs["schedule_id"] = schedule_id
1178
+ if parsed_inputs:
1179
+ kwargs["inputs"] = parsed_inputs
1180
+ if parsed_final_vars:
1181
+ kwargs["final_vars"] = parsed_final_vars
1182
+
1183
+ schedule_result = project.schedule(name, **kwargs)
1184
+
1185
+ if cron:
1186
+ logger.info(
1187
+ f"Pipeline '{name}' scheduled with cron '{cron}'. Schedule ID: {schedule_result}"
1188
+ )
1189
+ elif interval:
1190
+ logger.info(
1191
+ f"Pipeline '{name}' scheduled every {interval}. Schedule ID: {schedule_result}"
1192
+ )
1193
+
1194
+ except Exception as e:
1195
+ logger.error(f"Failed to schedule pipeline '{name}': {e}")
1196
+ raise typer.Exit(1)
1197
+
1198
+
1199
+ @app.command()
1200
+ def run_job(
1201
+ job_id: str = typer.Argument(..., help="ID of the job to run"),
1202
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
1203
+ name: str | None = typer.Option(
1204
+ None, help="Name of the scheduler configuration to use"
1205
+ ),
1206
+ base_dir: str | None = typer.Option(
1207
+ None, help="Base directory for the scheduler configuration"
1208
+ ),
1209
+ storage_options: str | None = typer.Option(
1210
+ None, help="Storage options as JSON or key=value pairs"
1211
+ ),
1212
+ log_level: str = typer.Option(
1213
+ "info", help="Logging level (debug, info, warning, error, critical)"
1214
+ ),
1215
+ ):
1216
+ """
1217
+ Execute a specific job by its ID.
1218
+
1219
+ This command runs a job that has been previously enqueued in the job queue.
1220
+ The job will be executed immediately regardless of its original schedule.
1221
+
1222
+ Args:
1223
+ job_id: ID of the job to run
1224
+ type: Type of job queue backend (rq)
1225
+ name: Name of the scheduler configuration to use
1226
+ base_dir: Base directory for the scheduler configuration
1227
+ storage_options: Storage options as JSON or key=value pairs
1228
+ log_level: Logging level (debug, info, warning, error, critical)
1229
+
1230
+ Examples:
1231
+ # Run a specific job
1232
+ $ flowerpower job-queue run-job job-123456
1233
+
1234
+ # Run a job with a specific backend type
1235
+ $ flowerpower job-queue run-job job-123456 --type rq
1236
+
1237
+ # Run a job with debug logging
1238
+ $ flowerpower job-queue run-job job-123456 --log-level debug
1239
+ """
1240
+ parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
1241
+
1242
+ with JobQueueManager(
1243
+ type=type,
1244
+ name=name,
1245
+ base_dir=base_dir,
1246
+ storage_options=parsed_storage_options,
1247
+ log_level=log_level,
1248
+ ) as worker:
1249
+ try:
1250
+ worker.run_job(job_id)
1251
+ logger.info(f"Job '{job_id}' finished running.")
1252
+ except Exception as e:
1253
+ logger.error(f"Failed to run job '{job_id}': {e}")
1254
+ raise typer.Exit(1)
1255
+
1256
+
1257
+ @app.command()
1258
+ def list_schedules(
1259
+ type: str | None = typer.Option(None, help="Type of job queue backend (rq)"),
1260
+ name: str | None = typer.Option(
1261
+ None, help="Name of the scheduler configuration to use"
1262
+ ),
1263
+ base_dir: str | None = typer.Option(
1264
+ None, help="Base directory for the scheduler configuration"
1265
+ ),
1266
+ storage_options: str | None = typer.Option(
1267
+ None, help="Storage options as JSON or key=value pairs"
1268
+ ),
1269
+ log_level: str = typer.Option(
1270
+ "info", help="Logging level (debug, info, warning, error, critical)"
1271
+ ),
1272
+ format: str = typer.Option("table", help="Output format (table, json, yaml)"),
1273
+ show_status: bool = typer.Option(
1274
+ True, help="Show schedule status (active, paused, etc.)"
1275
+ ),
1276
+ show_next_run: bool = typer.Option(True, help="Show next scheduled execution time"),
1277
+ ):
1278
+ """
1279
+ List all schedules with detailed status information.
1280
+
1281
+ This command provides enhanced schedule listing showing trigger configuration,
1282
+ status, next run time, and execution history. This is an enhanced version of
1283
+ show-schedules with more detailed information.
1284
+
1285
+ Args:
1286
+ type: Type of job queue backend (rq)
1287
+ name: Name of the scheduler configuration to use
1288
+ base_dir: Base directory for the scheduler configuration
1289
+ storage_options: Storage options as JSON or key=value pairs
1290
+ log_level: Logging level (debug, info, warning, error, critical)
1291
+ format: Output format for the schedule information
1292
+ show_status: Include schedule status information
1293
+ show_next_run: Include next execution time information
1294
+
1295
+ Examples:
1296
+ # List all schedules with full details
1297
+ $ flowerpower job-queue list-schedules
1298
+
1299
+ # List schedules in JSON format
1300
+ $ flowerpower job-queue list-schedules --format json
1301
+
1302
+ # List schedules without status information
1303
+ $ flowerpower job-queue list-schedules --no-show-status
1304
+
1305
+ # List schedules for a specific backend
1306
+ $ flowerpower job-queue list-schedules --type rq
1307
+ """
1308
+ parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
1309
+
1310
+ with JobQueueManager(
1311
+ type=type,
1312
+ name=name,
1313
+ base_dir=base_dir,
1314
+ storage_options=parsed_storage_options,
1315
+ log_level=log_level,
1316
+ ) as worker:
1317
+ # This will use the enhanced show_schedules method with additional options
1318
+ try:
1319
+ worker.show_schedules(
1320
+ format=format,
1321
+ show_status=show_status,
1322
+ show_next_run=show_next_run,
1323
+ )
1324
+ except TypeError:
1325
+ # Fallback if the show_schedules method doesn't support new parameters
1326
+ logger.warning(
1327
+ "Using basic schedule listing (enhanced options not supported)"
1328
+ )
1329
+ worker.show_schedules(format=format)