FlowerPower 1.0.0b2__py3-none-any.whl → 1.0.0b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/__init__.py +2 -3
- flowerpower/cfg/__init__.py +10 -8
- flowerpower/cfg/pipeline/__init__.py +11 -7
- flowerpower/cfg/project/__init__.py +11 -8
- flowerpower/cfg/project/job_queue.py +10 -29
- flowerpower/cli/__init__.py +62 -28
- flowerpower/cli/job_queue.py +306 -123
- flowerpower/cli/mqtt.py +22 -16
- flowerpower/cli/pipeline.py +294 -114
- flowerpower/flowerpower.py +14 -8
- flowerpower/fs/__init__.py +7 -3
- flowerpower/fs/ext.py +6 -2
- flowerpower/io/base.py +17 -10
- flowerpower/io/loader/_duckdb.py +1 -0
- flowerpower/io/loader/deltatable.py +6 -2
- flowerpower/io/saver/deltatable.py +1 -2
- flowerpower/job_queue/__init__.py +16 -12
- flowerpower/job_queue/apscheduler/__init__.py +1 -1
- flowerpower/job_queue/apscheduler/manager.py +11 -6
- flowerpower/job_queue/apscheduler/utils.py +6 -4
- flowerpower/job_queue/base.py +1 -0
- flowerpower/job_queue/rq/__init__.py +1 -1
- flowerpower/job_queue/rq/manager.py +12 -3
- flowerpower/pipeline/io.py +11 -9
- flowerpower/pipeline/job_queue.py +5 -5
- flowerpower/pipeline/manager.py +35 -27
- flowerpower/pipeline/registry.py +26 -16
- flowerpower/pipeline/runner.py +3 -4
- flowerpower/plugins/mqtt/__init__.py +7 -7
- flowerpower/plugins/mqtt/cfg.py +3 -2
- flowerpower/plugins/mqtt/manager.py +25 -23
- flowerpower/utils/misc.py +6 -4
- flowerpower/utils/templates.py +1 -4
- {flowerpower-1.0.0b2.dist-info → flowerpower-1.0.0b4.dist-info}/METADATA +1 -1
- {flowerpower-1.0.0b2.dist-info → flowerpower-1.0.0b4.dist-info}/RECORD +38 -38
- {flowerpower-1.0.0b2.dist-info → flowerpower-1.0.0b4.dist-info}/WHEEL +0 -0
- {flowerpower-1.0.0b2.dist-info → flowerpower-1.0.0b4.dist-info}/entry_points.txt +0 -0
- {flowerpower-1.0.0b2.dist-info → flowerpower-1.0.0b4.dist-info}/top_level.txt +0 -0
flowerpower/cli/pipeline.py
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
# Import necessary libraries
|
2
|
+
import datetime as dt
|
3
|
+
|
4
|
+
import duration_parser
|
2
5
|
import typer
|
3
6
|
from loguru import logger
|
4
7
|
from typing_extensions import Annotated
|
5
|
-
|
6
|
-
import duration_parser
|
8
|
+
|
7
9
|
from ..pipeline.manager import HookType, PipelineManager
|
8
10
|
from ..utils.logging import setup_logging
|
9
|
-
from .utils import parse_dict_or_list_param
|
11
|
+
from .utils import parse_dict_or_list_param # , parse_param_dict
|
10
12
|
|
11
13
|
setup_logging()
|
12
14
|
|
@@ -16,18 +18,38 @@ app = typer.Typer(help="Pipeline management commands")
|
|
16
18
|
@app.command()
|
17
19
|
def run(
|
18
20
|
name: str = typer.Argument(..., help="Name of the pipeline to run"),
|
19
|
-
executor: str | None = typer.Option(
|
21
|
+
executor: str | None = typer.Option(
|
22
|
+
None, help="Executor to use for running the pipeline"
|
23
|
+
),
|
20
24
|
base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
|
21
|
-
inputs: str | None = typer.Option(
|
25
|
+
inputs: str | None = typer.Option(
|
26
|
+
None, help="Input parameters as JSON, dict string, or key=value pairs"
|
27
|
+
),
|
22
28
|
final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
|
23
|
-
config: str | None = typer.Option(
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
29
|
+
config: str | None = typer.Option(
|
30
|
+
None, help="Config for the hamilton pipeline executor"
|
31
|
+
),
|
32
|
+
cache: str | None = typer.Option(
|
33
|
+
None, help="Cache configuration as JSON or dict string"
|
34
|
+
),
|
35
|
+
storage_options: str | None = typer.Option(
|
36
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
37
|
+
),
|
38
|
+
log_level: str | None = typer.Option(
|
39
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
40
|
+
),
|
41
|
+
with_adapter: str | None = typer.Option(
|
42
|
+
None, help="Adapter configuration as JSON or dict string"
|
43
|
+
),
|
44
|
+
max_retries: int = typer.Option(
|
45
|
+
0, help="Maximum number of retry attempts on failure"
|
46
|
+
),
|
47
|
+
retry_delay: float = typer.Option(
|
48
|
+
1.0, help="Base delay between retries in seconds"
|
49
|
+
),
|
50
|
+
jitter_factor: float = typer.Option(
|
51
|
+
0.1, help="Random factor applied to delay for jitter (0-1)"
|
52
|
+
),
|
31
53
|
):
|
32
54
|
"""
|
33
55
|
Run a pipeline immediately.
|
@@ -68,10 +90,10 @@ def run(
|
|
68
90
|
|
69
91
|
# Enable adapters for monitoring/tracking
|
70
92
|
$ pipeline run my_pipeline --with-adapter '{"tracker": true, "opentelemetry": true}'
|
71
|
-
|
93
|
+
|
72
94
|
# Set a specific logging level
|
73
95
|
$ pipeline run my_pipeline --log-level debug
|
74
|
-
|
96
|
+
|
75
97
|
# Configure automatic retries on failure
|
76
98
|
$ pipeline run my_pipeline --max-retries 3 --retry-delay 2.0 --jitter-factor 0.2
|
77
99
|
"""
|
@@ -82,18 +104,18 @@ def run(
|
|
82
104
|
parsed_storage_options = parse_dict_or_list_param(storage_options, "dict")
|
83
105
|
parsed_with_adapter = parse_dict_or_list_param(with_adapter, "dict")
|
84
106
|
|
85
|
-
with PipelineManager(
|
107
|
+
with PipelineManager(
|
86
108
|
base_dir=base_dir,
|
87
|
-
storage_options=parsed_storage_options or {},
|
109
|
+
storage_options=parsed_storage_options or {},
|
88
110
|
log_level=log_level,
|
89
111
|
) as manager:
|
90
|
-
_ = manager.run(
|
112
|
+
_ = manager.run(
|
91
113
|
name=name,
|
92
114
|
inputs=parsed_inputs,
|
93
115
|
final_vars=parsed_final_vars,
|
94
116
|
config=parsed_config,
|
95
117
|
cache=parsed_cache,
|
96
|
-
executor_cfg=executor,
|
118
|
+
executor_cfg=executor,
|
97
119
|
with_adapter_cfg=parsed_with_adapter,
|
98
120
|
max_retries=max_retries,
|
99
121
|
retry_delay=retry_delay,
|
@@ -105,23 +127,43 @@ def run(
|
|
105
127
|
@app.command()
|
106
128
|
def run_job(
|
107
129
|
name: str = typer.Argument(..., help="Name or ID of the pipeline job to run"),
|
108
|
-
executor: str | None = typer.Option(
|
130
|
+
executor: str | None = typer.Option(
|
131
|
+
None, help="Executor to use for running the job"
|
132
|
+
),
|
109
133
|
base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
|
110
|
-
inputs: str | None = typer.Option(
|
134
|
+
inputs: str | None = typer.Option(
|
135
|
+
None, help="Input parameters as JSON, dict string, or key=value pairs"
|
136
|
+
),
|
111
137
|
final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
|
112
|
-
config: str | None = typer.Option(
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
138
|
+
config: str | None = typer.Option(
|
139
|
+
None, help="Config for the hamilton pipeline executor"
|
140
|
+
),
|
141
|
+
cache: str | None = typer.Option(
|
142
|
+
None, help="Cache configuration as JSON or dict string"
|
143
|
+
),
|
144
|
+
storage_options: str | None = typer.Option(
|
145
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
146
|
+
),
|
147
|
+
log_level: str | None = typer.Option(
|
148
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
149
|
+
),
|
150
|
+
with_adapter: str | None = typer.Option(
|
151
|
+
None, help="Adapter configuration as JSON or dict string"
|
152
|
+
),
|
153
|
+
max_retries: int = typer.Option(
|
154
|
+
0, help="Maximum number of retry attempts on failure"
|
155
|
+
),
|
156
|
+
retry_delay: float = typer.Option(
|
157
|
+
1.0, help="Base delay between retries in seconds"
|
158
|
+
),
|
159
|
+
jitter_factor: float = typer.Option(
|
160
|
+
0.1, help="Random factor applied to delay for jitter (0-1)"
|
161
|
+
),
|
120
162
|
):
|
121
163
|
"""
|
122
164
|
Run a specific pipeline job.
|
123
165
|
|
124
|
-
This command runs an existing job by its ID. The job should have been previously
|
166
|
+
This command runs an existing job by its ID. The job should have been previously
|
125
167
|
added to the system via the add-job command or through scheduling.
|
126
168
|
|
127
169
|
Args:
|
@@ -154,7 +196,7 @@ def run_job(
|
|
154
196
|
|
155
197
|
# Configure adapters for monitoring
|
156
198
|
$ pipeline run-job job-123456 --with-adapter '{"tracker": true, "opentelemetry": false}'
|
157
|
-
|
199
|
+
|
158
200
|
# Set up automatic retries for resilience
|
159
201
|
$ pipeline run-job job-123456 --max-retries 3 --retry-delay 2.0
|
160
202
|
"""
|
@@ -167,7 +209,7 @@ def run_job(
|
|
167
209
|
|
168
210
|
with PipelineManager(
|
169
211
|
base_dir=base_dir,
|
170
|
-
storage_options=parsed_storage_options or {},
|
212
|
+
storage_options=parsed_storage_options or {},
|
171
213
|
log_level=log_level,
|
172
214
|
) as manager:
|
173
215
|
_ = manager.run_job(
|
@@ -188,25 +230,47 @@ def run_job(
|
|
188
230
|
@app.command()
|
189
231
|
def add_job(
|
190
232
|
name: str = typer.Argument(..., help="Name of the pipeline to add as a job"),
|
191
|
-
executor: str | None = typer.Option(
|
233
|
+
executor: str | None = typer.Option(
|
234
|
+
None, help="Executor to use for running the job"
|
235
|
+
),
|
192
236
|
base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
|
193
|
-
inputs: str | None = typer.Option(
|
237
|
+
inputs: str | None = typer.Option(
|
238
|
+
None, help="Input parameters as JSON, dict string, or key=value pairs"
|
239
|
+
),
|
194
240
|
final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
|
195
|
-
config: str | None = typer.Option(
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
241
|
+
config: str | None = typer.Option(
|
242
|
+
None, help="Config for the hamilton pipeline executor"
|
243
|
+
),
|
244
|
+
cache: str | None = typer.Option(
|
245
|
+
None, help="Cache configuration as JSON or dict string"
|
246
|
+
),
|
247
|
+
storage_options: str | None = typer.Option(
|
248
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
249
|
+
),
|
250
|
+
log_level: str | None = typer.Option(
|
251
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
252
|
+
),
|
253
|
+
with_adapter: str | None = typer.Option(
|
254
|
+
None, help="Adapter configuration as JSON or dict string"
|
255
|
+
),
|
200
256
|
run_at: str | None = typer.Option(None, help="Run at a specific time (ISO format)"),
|
201
|
-
run_in: str | None = typer.Option(
|
202
|
-
|
203
|
-
|
204
|
-
|
257
|
+
run_in: str | None = typer.Option(
|
258
|
+
None, help="Run in a specific interval (e.g., '5m', '1h', '12m34s')"
|
259
|
+
),
|
260
|
+
max_retries: int = typer.Option(
|
261
|
+
3, help="Maximum number of retry attempts on failure"
|
262
|
+
),
|
263
|
+
retry_delay: float = typer.Option(
|
264
|
+
1.0, help="Base delay between retries in seconds"
|
265
|
+
),
|
266
|
+
jitter_factor: float = typer.Option(
|
267
|
+
0.1, help="Random factor applied to delay for jitter (0-1)"
|
268
|
+
),
|
205
269
|
):
|
206
270
|
"""
|
207
271
|
Add a pipeline job to the queue.
|
208
272
|
|
209
|
-
This command adds a job to the queue for later execution. The job is based on
|
273
|
+
This command adds a job to the queue for later execution. The job is based on
|
210
274
|
an existing pipeline with customized inputs and configuration.
|
211
275
|
|
212
276
|
Args:
|
@@ -241,7 +305,7 @@ def add_job(
|
|
241
305
|
|
242
306
|
# Use a specific log level
|
243
307
|
$ pipeline add-job my_pipeline --log-level debug
|
244
|
-
|
308
|
+
|
245
309
|
# Configure automatic retries for resilience
|
246
310
|
$ pipeline add-job my_pipeline --max-retries 5 --retry-delay 2.0 --jitter-factor 0.2
|
247
311
|
"""
|
@@ -279,23 +343,51 @@ def add_job(
|
|
279
343
|
@app.command()
|
280
344
|
def schedule(
|
281
345
|
name: str = typer.Argument(..., help="Name of the pipeline to schedule"),
|
282
|
-
executor: str | None = typer.Option(
|
346
|
+
executor: str | None = typer.Option(
|
347
|
+
None, help="Executor to use for running the job"
|
348
|
+
),
|
283
349
|
base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
|
284
|
-
inputs: str | None = typer.Option(
|
350
|
+
inputs: str | None = typer.Option(
|
351
|
+
None, help="Input parameters as JSON, dict string, or key=value pairs"
|
352
|
+
),
|
285
353
|
final_vars: str | None = typer.Option(None, help="Final variables as JSON or list"),
|
286
|
-
config: str | None = typer.Option(
|
287
|
-
|
354
|
+
config: str | None = typer.Option(
|
355
|
+
None, help="Config for the hamilton pipeline executor"
|
356
|
+
),
|
357
|
+
cache: str | None = typer.Option(
|
358
|
+
None, help="Cache configuration as JSON or dict string"
|
359
|
+
),
|
288
360
|
cron: str | None = typer.Option(None, help="Cron expression for scheduling"),
|
289
|
-
interval: str | None = typer.Option(
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
361
|
+
interval: str | None = typer.Option(
|
362
|
+
None, help="Interval for scheduling (e.g., '5m', '1h')"
|
363
|
+
),
|
364
|
+
date: str | None = typer.Option(
|
365
|
+
None, help="Specific date and time for scheduling (ISO format)"
|
366
|
+
),
|
367
|
+
storage_options: str | None = typer.Option(
|
368
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
369
|
+
),
|
370
|
+
log_level: str | None = typer.Option(
|
371
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
372
|
+
),
|
373
|
+
with_adapter: str | None = typer.Option(
|
374
|
+
None, help="Adapter configuration as JSON or dict string"
|
375
|
+
),
|
376
|
+
overwrite: bool = typer.Option(
|
377
|
+
False, help="Overwrite existing schedule if it exists"
|
378
|
+
),
|
379
|
+
schedule_id: str | None = typer.Option(
|
380
|
+
None, help="Custom ID for the schedule (autogenerated if not provided)"
|
381
|
+
),
|
382
|
+
max_retries: int = typer.Option(
|
383
|
+
3, help="Maximum number of retry attempts on failure"
|
384
|
+
),
|
385
|
+
retry_delay: float = typer.Option(
|
386
|
+
1.0, help="Base delay between retries in seconds"
|
387
|
+
),
|
388
|
+
jitter_factor: float = typer.Option(
|
389
|
+
0.1, help="Random factor applied to delay for jitter (0-1)"
|
390
|
+
),
|
299
391
|
):
|
300
392
|
"""
|
301
393
|
Schedule a pipeline to run at specified times.
|
@@ -341,7 +433,7 @@ def schedule(
|
|
341
433
|
|
342
434
|
# Set a custom schedule ID
|
343
435
|
$ pipeline schedule my_pipeline --crontab "0 12 * * *" --schedule_id "daily-noon-run"
|
344
|
-
|
436
|
+
|
345
437
|
# Configure automatic retries for resilience
|
346
438
|
$ pipeline schedule my_pipeline --max-retries 5 --retry-delay 2.0 --jitter-factor 0.2
|
347
439
|
"""
|
@@ -353,7 +445,7 @@ def schedule(
|
|
353
445
|
parsed_with_adapter = parse_dict_or_list_param(with_adapter, "dict")
|
354
446
|
interval = duration_parser.parse(interval) if interval else None
|
355
447
|
cron = cron if cron else None
|
356
|
-
date = dt.datetime.fromisoformat(date) if date else None
|
448
|
+
date = dt.datetime.fromisoformat(date) if date else None
|
357
449
|
|
358
450
|
with PipelineManager(
|
359
451
|
base_dir=base_dir,
|
@@ -361,7 +453,7 @@ def schedule(
|
|
361
453
|
log_level=log_level,
|
362
454
|
) as manager:
|
363
455
|
# Combine common schedule kwargs
|
364
|
-
|
456
|
+
|
365
457
|
id_ = manager.schedule(
|
366
458
|
name=name,
|
367
459
|
inputs=parsed_inputs,
|
@@ -385,11 +477,21 @@ def schedule(
|
|
385
477
|
|
386
478
|
@app.command()
|
387
479
|
def schedule_all(
|
388
|
-
executor: str | None = typer.Option(
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
480
|
+
executor: str | None = typer.Option(
|
481
|
+
None, help="Override executor specified in pipeline configs"
|
482
|
+
),
|
483
|
+
base_dir: str | None = typer.Option(
|
484
|
+
None, help="Base directory containing pipelines and configurations"
|
485
|
+
),
|
486
|
+
storage_options: str | None = typer.Option(
|
487
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
488
|
+
),
|
489
|
+
log_level: str | None = typer.Option(
|
490
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
491
|
+
),
|
492
|
+
overwrite: bool = typer.Option(
|
493
|
+
False, help="Overwrite existing schedules if they exist"
|
494
|
+
),
|
393
495
|
):
|
394
496
|
"""
|
395
497
|
Schedule all pipelines based on their individual configurations.
|
@@ -425,10 +527,7 @@ def schedule_all(
|
|
425
527
|
storage_options=parsed_storage_options or {},
|
426
528
|
log_level=log_level,
|
427
529
|
) as manager:
|
428
|
-
manager.schedule_all(
|
429
|
-
overwrite=overwrite,
|
430
|
-
executor_cfg=executor
|
431
|
-
)
|
530
|
+
manager.schedule_all(overwrite=overwrite, executor_cfg=executor)
|
432
531
|
logger.info("Scheduled all pipelines based on their configurations.")
|
433
532
|
|
434
533
|
|
@@ -436,14 +535,20 @@ def schedule_all(
|
|
436
535
|
def new(
|
437
536
|
name: str = typer.Argument(..., help="Name of the pipeline to create"),
|
438
537
|
base_dir: str | None = typer.Option(None, help="Base directory for the pipeline"),
|
439
|
-
storage_options: str | None = typer.Option(
|
440
|
-
|
441
|
-
|
538
|
+
storage_options: str | None = typer.Option(
|
539
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
540
|
+
),
|
541
|
+
log_level: str | None = typer.Option(
|
542
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
543
|
+
),
|
544
|
+
overwrite: bool = typer.Option(
|
545
|
+
False, help="Overwrite existing pipeline if it exists"
|
546
|
+
),
|
442
547
|
):
|
443
548
|
"""
|
444
549
|
Create a new pipeline structure.
|
445
550
|
|
446
|
-
This command creates a new pipeline with the necessary directory structure,
|
551
|
+
This command creates a new pipeline with the necessary directory structure,
|
447
552
|
configuration file, and skeleton module file. It prepares all the required
|
448
553
|
components for you to start implementing your pipeline logic.
|
449
554
|
|
@@ -477,11 +582,21 @@ def new(
|
|
477
582
|
@app.command()
|
478
583
|
def delete(
|
479
584
|
name: str = typer.Argument(..., help="Name of the pipeline to delete"),
|
480
|
-
base_dir: str | None = typer.Option(
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
585
|
+
base_dir: str | None = typer.Option(
|
586
|
+
None, help="Base directory containing the pipeline"
|
587
|
+
),
|
588
|
+
cfg: bool = typer.Option(
|
589
|
+
False, "--cfg", "-c", help="Delete only the configuration file"
|
590
|
+
),
|
591
|
+
module: bool = typer.Option(
|
592
|
+
False, "--module", "-m", help="Delete only the pipeline module"
|
593
|
+
),
|
594
|
+
storage_options: str | None = typer.Option(
|
595
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
596
|
+
),
|
597
|
+
log_level: str | None = typer.Option(
|
598
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
599
|
+
),
|
485
600
|
):
|
486
601
|
"""
|
487
602
|
Delete a pipeline's configuration and/or module files.
|
@@ -525,16 +640,28 @@ def delete(
|
|
525
640
|
deleted_parts.append("config")
|
526
641
|
if delete_module:
|
527
642
|
deleted_parts.append("module")
|
528
|
-
logger.info(
|
643
|
+
logger.info(
|
644
|
+
f"Pipeline '{name}' deleted ({', '.join(deleted_parts)})."
|
645
|
+
if deleted_parts
|
646
|
+
else f"Pipeline '{name}' - nothing specified to delete."
|
647
|
+
)
|
529
648
|
|
530
649
|
|
531
650
|
@app.command()
|
532
651
|
def show_dag(
|
533
652
|
name: str = typer.Argument(..., help="Name of the pipeline to visualize"),
|
534
|
-
base_dir: str | None = typer.Option(
|
535
|
-
|
536
|
-
|
537
|
-
|
653
|
+
base_dir: str | None = typer.Option(
|
654
|
+
None, help="Base directory containing the pipeline"
|
655
|
+
),
|
656
|
+
storage_options: str | None = typer.Option(
|
657
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
658
|
+
),
|
659
|
+
log_level: str | None = typer.Option(
|
660
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
661
|
+
),
|
662
|
+
format: str = typer.Option(
|
663
|
+
"png", help="Output format (e.g., png, svg, pdf). If 'raw', returns object."
|
664
|
+
),
|
538
665
|
):
|
539
666
|
"""
|
540
667
|
Show the DAG (Directed Acyclic Graph) of a pipeline.
|
@@ -569,14 +696,20 @@ def show_dag(
|
|
569
696
|
) as manager:
|
570
697
|
# Manager's show_dag likely handles rendering or returning raw object
|
571
698
|
try:
|
572
|
-
graph_or_none = manager.show_dag(
|
699
|
+
graph_or_none = manager.show_dag(
|
700
|
+
name=name, format=format if not is_raw else "png", raw=is_raw
|
701
|
+
)
|
573
702
|
if is_raw and graph_or_none:
|
574
703
|
print("Graphviz object returned (not rendered):")
|
575
704
|
# print(graph_or_none) # Or handle as needed
|
576
705
|
elif not is_raw:
|
577
|
-
|
706
|
+
logger.info(
|
707
|
+
f"DAG for pipeline '{name}' displayed/saved (format: {format})."
|
708
|
+
)
|
578
709
|
except ImportError:
|
579
|
-
|
710
|
+
logger.error(
|
711
|
+
"Graphviz is not installed. Cannot show/save DAG. Install with: pip install graphviz"
|
712
|
+
)
|
580
713
|
except Exception as e:
|
581
714
|
logger.error(f"Failed to generate DAG for pipeline '{name}': {e}")
|
582
715
|
|
@@ -584,11 +717,19 @@ def show_dag(
|
|
584
717
|
@app.command()
|
585
718
|
def save_dag(
|
586
719
|
name: str = typer.Argument(..., help="Name of the pipeline to visualize"),
|
587
|
-
base_dir: str | None = typer.Option(
|
588
|
-
|
589
|
-
|
720
|
+
base_dir: str | None = typer.Option(
|
721
|
+
None, help="Base directory containing the pipeline"
|
722
|
+
),
|
723
|
+
storage_options: str | None = typer.Option(
|
724
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
725
|
+
),
|
726
|
+
log_level: str | None = typer.Option(
|
727
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
728
|
+
),
|
590
729
|
format: str = typer.Option("png", help="Output format (e.g., png, svg, pdf)"),
|
591
|
-
output_path: str | None = typer.Option(
|
730
|
+
output_path: str | None = typer.Option(
|
731
|
+
None, help="Custom path to save the file (default: <name>.<format>)"
|
732
|
+
),
|
592
733
|
):
|
593
734
|
"""
|
594
735
|
Save the DAG (Directed Acyclic Graph) of a pipeline to a file.
|
@@ -621,19 +762,29 @@ def save_dag(
|
|
621
762
|
log_level=log_level,
|
622
763
|
) as manager:
|
623
764
|
try:
|
624
|
-
file_path = manager.save_dag(
|
765
|
+
file_path = manager.save_dag(
|
766
|
+
name=name, format=format, output_path=output_path
|
767
|
+
)
|
625
768
|
logger.info(f"DAG for pipeline '{name}' saved to {file_path}.")
|
626
769
|
except ImportError:
|
627
|
-
|
770
|
+
logger.error(
|
771
|
+
"Graphviz is not installed. Cannot save DAG. Install with: pip install graphviz"
|
772
|
+
)
|
628
773
|
except Exception as e:
|
629
774
|
logger.error(f"Failed to save DAG for pipeline '{name}': {e}")
|
630
775
|
|
631
776
|
|
632
777
|
@app.command()
|
633
778
|
def show_pipelines(
|
634
|
-
base_dir: str | None = typer.Option(
|
635
|
-
|
636
|
-
|
779
|
+
base_dir: str | None = typer.Option(
|
780
|
+
None, help="Base directory containing pipelines"
|
781
|
+
),
|
782
|
+
storage_options: str | None = typer.Option(
|
783
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
784
|
+
),
|
785
|
+
log_level: str | None = typer.Option(
|
786
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
787
|
+
),
|
637
788
|
format: str = typer.Option("table", help="Output format (table, json, yaml)"),
|
638
789
|
):
|
639
790
|
"""
|
@@ -654,7 +805,7 @@ def show_pipelines(
|
|
654
805
|
|
655
806
|
# Output in JSON format
|
656
807
|
$ pipeline show-pipelines --format json
|
657
|
-
|
808
|
+
|
658
809
|
# List pipelines from a specific directory
|
659
810
|
$ pipeline show-pipelines --base-dir /path/to/project
|
660
811
|
"""
|
@@ -669,16 +820,26 @@ def show_pipelines(
|
|
669
820
|
|
670
821
|
@app.command()
|
671
822
|
def show_summary(
|
672
|
-
name: str | None = typer.Option(
|
823
|
+
name: str | None = typer.Option(
|
824
|
+
None, help="Name of specific pipeline to show (all pipelines if not specified)"
|
825
|
+
),
|
673
826
|
cfg: bool = typer.Option(True, help="Include configuration details"),
|
674
827
|
code: bool = typer.Option(True, help="Include code/module details"),
|
675
828
|
project: bool = typer.Option(True, help="Include project context"),
|
676
|
-
base_dir: str | None = typer.Option(
|
677
|
-
|
678
|
-
|
829
|
+
base_dir: str | None = typer.Option(
|
830
|
+
None, help="Base directory containing pipelines"
|
831
|
+
),
|
832
|
+
storage_options: str | None = typer.Option(
|
833
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
834
|
+
),
|
835
|
+
log_level: str | None = typer.Option(
|
836
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
837
|
+
),
|
679
838
|
to_html: bool = typer.Option(False, help="Output summary as HTML"),
|
680
839
|
to_svg: bool = typer.Option(False, help="Output summary as SVG (if applicable)"),
|
681
|
-
output_file: str | None = typer.Option(
|
840
|
+
output_file: str | None = typer.Option(
|
841
|
+
None, help="Save output to specified file instead of printing"
|
842
|
+
),
|
682
843
|
):
|
683
844
|
"""
|
684
845
|
Show summary information for one or all pipelines.
|
@@ -708,7 +869,7 @@ def show_summary(
|
|
708
869
|
|
709
870
|
# Show only configuration information
|
710
871
|
$ pipeline show-summary --name my_pipeline --cfg --no-code --no-project
|
711
|
-
|
872
|
+
|
712
873
|
# Generate HTML report
|
713
874
|
$ pipeline show-summary --to-html --output-file pipeline_report.html
|
714
875
|
"""
|
@@ -727,10 +888,10 @@ def show_summary(
|
|
727
888
|
to_html=to_html,
|
728
889
|
to_svg=to_svg,
|
729
890
|
)
|
730
|
-
|
891
|
+
|
731
892
|
if summary_output:
|
732
893
|
if output_file:
|
733
|
-
with open(output_file,
|
894
|
+
with open(output_file, "w") as f:
|
734
895
|
f.write(summary_output)
|
735
896
|
logger.info(f"Summary saved to {output_file}")
|
736
897
|
else:
|
@@ -741,12 +902,27 @@ def show_summary(
|
|
741
902
|
@app.command()
|
742
903
|
def add_hook(
|
743
904
|
name: str = typer.Argument(..., help="Name of the pipeline to add the hook to"),
|
744
|
-
function_name: str = typer.Option(
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
905
|
+
function_name: str = typer.Option(
|
906
|
+
...,
|
907
|
+
"--function",
|
908
|
+
"-f",
|
909
|
+
help="Name of the hook function defined in the pipeline module",
|
910
|
+
),
|
911
|
+
type: Annotated[
|
912
|
+
HookType, typer.Option(help="Type of hook to add")
|
913
|
+
] = HookType.MQTT_BUILD_CONFIG,
|
914
|
+
to: str | None = typer.Option(
|
915
|
+
None, help="Target node name or tag (required for node hooks)"
|
916
|
+
),
|
917
|
+
base_dir: str | None = typer.Option(
|
918
|
+
None, help="Base directory containing the pipeline"
|
919
|
+
),
|
920
|
+
storage_options: str | None = typer.Option(
|
921
|
+
None, help="Storage options as JSON, dict string, or key=value pairs"
|
922
|
+
),
|
923
|
+
log_level: str | None = typer.Option(
|
924
|
+
None, help="Logging level (debug, info, warning, error, critical)"
|
925
|
+
),
|
750
926
|
):
|
751
927
|
"""
|
752
928
|
Add a hook to a pipeline configuration.
|
@@ -781,7 +957,9 @@ def add_hook(
|
|
781
957
|
|
782
958
|
# Validate 'to' argument for node hooks
|
783
959
|
if type in (HookType.NODE_PRE_EXECUTE, HookType.NODE_POST_EXECUTE) and not to:
|
784
|
-
raise typer.BadParameter(
|
960
|
+
raise typer.BadParameter(
|
961
|
+
"The '--to' option (target node/tag) is required for node hooks."
|
962
|
+
)
|
785
963
|
|
786
964
|
with PipelineManager(
|
787
965
|
base_dir=base_dir,
|
@@ -795,6 +973,8 @@ def add_hook(
|
|
795
973
|
to=to,
|
796
974
|
function_name=function_name,
|
797
975
|
)
|
798
|
-
logger.info(
|
976
|
+
logger.info(
|
977
|
+
f"Hook '{function_name}' added to pipeline '{name}' (type: {type.value})."
|
978
|
+
)
|
799
979
|
except Exception as e:
|
800
980
|
logger.error(f"Failed to add hook to pipeline '{name}': {e}")
|