FlowerPower 0.20.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. flowerpower/__init__.py +2 -6
  2. flowerpower/cfg/__init__.py +4 -11
  3. flowerpower/cfg/base.py +29 -25
  4. flowerpower/cfg/pipeline/__init__.py +3 -3
  5. flowerpower/cfg/pipeline/_schedule.py +32 -0
  6. flowerpower/cfg/pipeline/adapter.py +0 -5
  7. flowerpower/cfg/pipeline/builder.py +377 -0
  8. flowerpower/cfg/pipeline/run.py +89 -0
  9. flowerpower/cfg/project/__init__.py +8 -21
  10. flowerpower/cfg/project/adapter.py +0 -12
  11. flowerpower/cli/__init__.py +2 -28
  12. flowerpower/cli/pipeline.py +10 -4
  13. flowerpower/flowerpower.py +275 -585
  14. flowerpower/pipeline/base.py +19 -10
  15. flowerpower/pipeline/io.py +52 -46
  16. flowerpower/pipeline/manager.py +149 -91
  17. flowerpower/pipeline/pipeline.py +159 -87
  18. flowerpower/pipeline/registry.py +68 -33
  19. flowerpower/pipeline/visualizer.py +4 -4
  20. flowerpower/plugins/{_io → io}/__init__.py +1 -1
  21. flowerpower/settings/__init__.py +0 -2
  22. flowerpower/settings/{backend.py → _backend.py} +0 -19
  23. flowerpower/settings/logging.py +1 -1
  24. flowerpower/utils/logging.py +24 -12
  25. flowerpower/utils/misc.py +17 -0
  26. flowerpower-0.30.0.dist-info/METADATA +451 -0
  27. flowerpower-0.30.0.dist-info/RECORD +42 -0
  28. flowerpower/cfg/pipeline/schedule.py +0 -74
  29. flowerpower/cfg/project/job_queue.py +0 -111
  30. flowerpower/cli/job_queue.py +0 -1329
  31. flowerpower/cli/mqtt.py +0 -174
  32. flowerpower/job_queue/__init__.py +0 -205
  33. flowerpower/job_queue/base.py +0 -611
  34. flowerpower/job_queue/rq/__init__.py +0 -10
  35. flowerpower/job_queue/rq/_trigger.py +0 -37
  36. flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -226
  37. flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -228
  38. flowerpower/job_queue/rq/manager.py +0 -1893
  39. flowerpower/job_queue/rq/setup.py +0 -154
  40. flowerpower/job_queue/rq/utils.py +0 -69
  41. flowerpower/mqtt.py +0 -12
  42. flowerpower/plugins/mqtt/__init__.py +0 -12
  43. flowerpower/plugins/mqtt/cfg.py +0 -17
  44. flowerpower/plugins/mqtt/manager.py +0 -962
  45. flowerpower/settings/job_queue.py +0 -31
  46. flowerpower-0.20.0.dist-info/METADATA +0 -693
  47. flowerpower-0.20.0.dist-info/RECORD +0 -58
  48. {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/WHEEL +0 -0
  49. {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/entry_points.txt +0 -0
  50. {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/licenses/LICENSE +0 -0
  51. {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/top_level.txt +0 -0
@@ -1,23 +1,35 @@
1
+ import os
1
2
  import sys
2
3
 
3
4
  from loguru import logger
4
5
 
5
- from ..settings import LOG_LEVEL # Import the setting
6
+ from ..settings import LOG_LEVEL
6
7
 
7
8
 
8
- def setup_logging(level: str = LOG_LEVEL, disable: bool = False) -> None:
9
+ def setup_logging(level: str | None = None) -> None:
9
10
  """
10
11
  Configures the Loguru logger.
11
12
 
12
- Removes the default handler and adds a new one targeting stderr
13
- with the level specified by the FP_LOG_LEVEL setting.
13
+ Determines the logging level based on the following precedence:
14
+ 1. The 'level' argument passed to the function.
15
+ 2. The 'FP_LOG_LEVEL' environment variable.
16
+ 3. The 'LOG_LEVEL' from ..settings (which defaults to "CRITICAL").
17
+
18
+ If the effective logging level is "CRITICAL", logging for the "flowerpower" module
19
+ is disabled. Otherwise, logging is enabled and configured.
14
20
  """
15
- logger.remove() # Remove the default handler added by Loguru
16
- logger.add(
17
- sys.stderr,
18
- level=level.upper(), # Use the level from the parameter, ensure it's uppercase
19
- format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>", # Example format
20
- )
21
- if disable:
21
+ # Remove all existing handlers to prevent duplicate logs
22
+ logger.remove()
23
+
24
+ # Determine the effective logging level
25
+ effective_level = level or os.getenv("FP_LOG_LEVEL") or LOG_LEVEL
26
+
27
+ if effective_level.upper() == "CRITICAL":
22
28
  logger.disable("flowerpower")
23
- # logger.info(f"Log level set to: {FP_LOG_LEVEL.upper()}")
29
+ else:
30
+ logger.enable("flowerpower")
31
+ logger.add(
32
+ sys.stderr,
33
+ level=effective_level.upper(),
34
+ format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
35
+ )
flowerpower/utils/misc.py CHANGED
@@ -7,6 +7,7 @@ import time
7
7
  from typing import Any
8
8
 
9
9
  import msgspec
10
+ from fsspec_utils import AbstractFileSystem, filesystem
10
11
 
11
12
  if importlib.util.find_spec("joblib"):
12
13
  from joblib import Parallel, delayed
@@ -228,3 +229,19 @@ def update_nested_dict(
228
229
  # Direct update
229
230
  result[key] = value
230
231
  return result
232
+
233
+
234
+ def get_filesystem(fs: AbstractFileSystem | None = None, fs_type: str = "file") -> AbstractFileSystem:
235
+ """
236
+ Helper function to get a filesystem instance.
237
+
238
+ Args:
239
+ fs: An optional filesystem instance to use. If provided, this will be returned directly.
240
+ fs_type: The type of filesystem to create if fs is None. Defaults to "file".
241
+
242
+ Returns:
243
+ An AbstractFileSystem instance.
244
+ """
245
+ if fs is None:
246
+ fs = filesystem(fs_type)
247
+ return fs
@@ -0,0 +1,451 @@
1
+ Metadata-Version: 2.4
2
+ Name: FlowerPower
3
+ Version: 0.30.0
4
+ Summary: A simple workflow framework for building and managing data processing pipelines
5
+ Author-email: "Volker L." <ligno.blades@gmail.com>
6
+ Project-URL: Homepage, https://github.com/legout/flowerpower
7
+ Project-URL: Bug Tracker, https://github.com/legout/flowerpower/issues
8
+ Keywords: hamilton,workflow,pipeline,scheduler,dask,ray
9
+ Requires-Python: >=3.11
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: duration-parser>=1.0.1
13
+ Requires-Dist: fsspec>=2024.10.0
14
+ Requires-Dist: fsspec-utils[full]>=0.1.0
15
+ Requires-Dist: humanize>=4.12.2
16
+ Requires-Dist: msgspec>=0.19.0
17
+ Requires-Dist: munch>=4.0.0
18
+ Requires-Dist: pyyaml>=6.0.1
19
+ Requires-Dist: rich>=13.9.3
20
+ Requires-Dist: s3fs>=2024.10.0
21
+ Requires-Dist: sf-hamilton-sdk>=0.5.2
22
+ Requires-Dist: sf-hamilton[rich,tqdm,visualization]>=1.69.0
23
+ Requires-Dist: typer>=0.12.3
24
+ Provides-Extra: io
25
+ Requires-Dist: flowerpower-io>=0.1.1; extra == "io"
26
+ Provides-Extra: io-legacy
27
+ Requires-Dist: flowerpower-io[legacy]>=0.1.1; extra == "io-legacy"
28
+ Provides-Extra: opentelemetry
29
+ Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
30
+ Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
31
+ Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
32
+ Provides-Extra: ray
33
+ Requires-Dist: ray>=2.34.0; extra == "ray"
34
+ Provides-Extra: ui
35
+ Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
36
+ Provides-Extra: openlineage
37
+ Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
38
+ Dynamic: license-file
39
+
40
+ <div align="center">
41
+ <h1>FlowerPower 🌸 - Build & Orchestrate Data Pipelines</h1>
42
+ <h3>Simple Workflow Framework - Hamilton = FlowerPower</h3>
43
+ <img src="./image.png" alt="FlowerPower Logo" width="400" height="300">
44
+ </div>
45
+
46
+
47
+
48
+ [![PyPI version](https://img.shields.io/pypi/v/flowerpower.svg?style=flat-square)](https://pypi.org/project/flowerpower/) <!-- Placeholder -->
49
+ [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/legout/flowerpower/blob/main/LICENSE)
50
+ [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/legout/flowerpower)
51
+ [![Documentation Status](https://readthedocs.org/projects/flowerpower/badge/?version=latest)](https://legout.github.io/flowerpower/)
52
+
53
+
54
+ **FlowerPower** is a Python framework designed for building, configuring, and executing data processing pipelines with ease and flexibility. It promotes a modular, configuration-driven approach, allowing you to focus on your pipeline logic while FlowerPower handles the orchestration.
55
+
56
+ It leverages the [Hamilton](https://github.com/apache/hamilton) library for defining dataflows in a clean, functional way within your Python pipeline scripts. Pipelines are defined in Python modules and configured using YAML files, making it easy to manage and understand your data workflows.
57
+ FlowerPower provides a unified project interface that makes it easy to work with pipeline execution. It also provides a web UI (Hamilton UI) for monitoring and managing your pipelines.
58
+ FlowerPower is designed to be extensible, allowing you to easily add custom I/O plugins and adapt to different deployment scenarios. This flexibility makes it suitable for a wide range of data processing tasks, from simple ETL jobs to complex data workflows.
59
+
60
+
61
+ ## ✨ Key Features
62
+
63
+ * **Modular Pipeline Design:** Thanks to [Hamilton](https://github.com/apache/hamilton), you can define your data processing logic in Python modules, using functions as nodes in a directed acyclic graph (DAG).
64
+ * **Configuration-Driven:** Define pipeline parameters, execution logic, and scheduling declaratively using simple YAML files.
65
+ * **Extensible I/O Plugins:** Connect to various data sources and destinations (CSV, JSON, Parquet, DeltaTable, DuckDB, PostgreSQL, MySQL, MSSQL, Oracle, MQTT, SQLite, and more).
66
+ * **Unified Project Interface:** Interact with your pipelines via:
67
+ * **FlowerPowerProject API:** A unified interface for pipeline execution, supporting both `RunConfig` objects and flexible `**kwargs` overrides.
68
+ * **Command Line Interface (CLI):** For running, managing, and inspecting pipelines, with enhanced `run` command capabilities.
69
+ * **Web UI:** A graphical interface for monitoring and managing pipelines and schedules. ([Hamilton UI](https://hamilton.dagworks.io/en/latest/hamilton-ui/ui/))
70
+ * **Filesystem Abstraction:** Simplified file handling with support for local and remote filesystems (e.g., S3, GCS).
71
+
72
+ ## 📦 Installation
73
+
74
+ We recommend using [uv](https://github.com/astral-sh/uv) for installing FlowerPower and managing your project environments. `uv` is an extremely fast Python package installer and resolver.
75
+
76
+ ```bash
77
+ # Create and activate a virtual environment (recommended)
78
+ uv venv
79
+ source .venv/bin/activate # Or .\.venv\Scripts\activate on Windows
80
+
81
+ # Install FlowerPower
82
+ uv pip install flowerpower
83
+
84
+ # Optional: Install additional dependencies for specific features
85
+ uv pip install flowerpower[io] # For I/O plugins (CSV, JSON, Parquet, DeltaTable, DuckDB, PostgreSQL, MySQL, MSSQL, Oracle, SQLite)
86
+ uv pip install flowerpower[ui] # For Hamilton UI
87
+ uv pip install flowerpower[all] # Install all optional dependencies
88
+ ```
89
+
90
+ *(Note: Specify required Python versions if known, e.g., Python 3.8+)*
91
+
92
+ ## 🚀 Getting Started
93
+
94
+ Let's build a simple "Hello World" pipeline.
95
+
96
+ ### 1. Initialize Your Project:
97
+
98
+ You can quickly set up the standard FlowerPower project structure using the CLI or Python.
99
+
100
+ **Using the CLI:**
101
+
102
+ Navigate to your desired parent directory and run:
103
+ ```bash
104
+ flowerpower init --name hello-flowerpower-project
105
+ ```
106
+
107
+
108
+ **Using Python:**
109
+
110
+ Alternatively, you can initialize programmatically:
111
+ ```python
112
+ from flowerpower import FlowerPowerProject
113
+
114
+ # Initialize a new project
115
+ project = FlowerPowerProject.init(
116
+ name='hello-flowerpower-project',
117
+ )
118
+ ```
119
+
120
+ This will create a `hello-flowerpower-project` directory with the necessary `conf/` and `pipelines/` subdirectories and default configuration files.
121
+
122
+ ```
123
+ hello-flowerpower-project/
124
+ ├── conf/
125
+ │ ├── project.yml
126
+ │ └── pipelines/
127
+ └── pipelines/
128
+ ```
129
+
130
+ Now, navigate into your new project directory:
131
+
132
+ ```bash
133
+ cd hello-flowerpower-project
134
+ ```
135
+
136
+ **Configure Project (`conf/project.yml`):**
137
+
138
+ Open `conf/project.yml` and define your project name:
139
+
140
+ ```yaml
141
+ name: hello-flowerpower
142
+ # adapter: ... # Optional adapter configurations (e.g., Hamilton Tracker, MLflow), see `conf/project.yml` for details
143
+ ```
144
+
145
+ ### 2. Create Your Pipeline
146
+
147
+ You can create a new pipeline using the CLI or programmatically.
148
+
149
+ **Using the CLI:**
150
+
151
+ ```bash
152
+ flowerpower pipeline new hello_world
153
+ ```
154
+
155
+ **Using Python:**
156
+
157
+ You can create pipelines programmatically using the FlowerPowerProject interface:
158
+
159
+ ```python
160
+ from flowerpower import FlowerPowerProject
161
+
162
+ # Load the project
163
+ project = FlowerPowerProject.load('.')
164
+
165
+ # Create a new pipeline
166
+ project.pipeline_manager.new(name='hello_world')
167
+ ```
168
+
169
+ This will create a new file `hello_world.py` in the `pipelines/` directory and a corresponding configuration file `hello_world.yml` in `conf/pipelines/`.
170
+
171
+ **Implement Pipeline (`pipelines/hello_world.py`):**
172
+
173
+ Open `pipelines/hello_world.py` and write your pipeline logic using Python and Hamilton. FlowerPower makes configuration easily accessible.
174
+
175
+ ```python
176
+ # FlowerPower pipeline hello_world.py
177
+ # Created on 2025-05-03 22:34:09
178
+
179
+ ####################################################################################################
180
+ # Import necessary libraries
181
+ # NOTE: Remove or comment out imports that are not used in the pipeline
182
+
183
+ from hamilton.function_modifiers import parameterize
184
+
185
+ from pathlib import Path
186
+
187
+ from flowerpower.cfg import Config
188
+
189
+ ####################################################################################################
190
+ # Load pipeline parameters. Do not modify this section.
191
+
192
+ PARAMS = Config.load(
193
+ Path(__file__).parents[1], pipeline_name="hello_world"
194
+ ).pipeline.h_params
195
+
196
+
197
+ ####################################################################################################
198
+ # Helper functions.
199
+ # This functions have to start with an underscore (_).
200
+
201
+
202
+ ####################################################################################################
203
+ # Pipeline functions
204
+
205
+ @parameterize(**PARAMS.greeting_message) # Inject 'message' from params
206
+ def greeting_message(message: str) -> str:
207
+ """Provides the greeting part."""
208
+ return f"{message},"
209
+
210
+ @parameterize(**PARAMS.target_name) # Inject 'name' from params
211
+ def target_name(name: str) -> str:
212
+ """Provides the target name."""
213
+ return f"{name}!"
214
+
215
+ def full_greeting(greeting_message: str, target_name: str) -> str:
216
+ """Combines the greeting and target."""
217
+ print(f"Generating greeting: {greeting_message} {target_name}")
218
+ return f"{greeting_message} {target_name}"
219
+
220
+ # You can add more complex Hamilton functions here...
221
+ ```
222
+
223
+ **Configure Pipeline (`conf/pipelines/hello_world.yml`):**
224
+
225
+ Open `conf/pipelines/hello_world.yml` and specify parameters, run configurations, and scheduling for your pipeline.
226
+
227
+ ```yaml
228
+ # adapter: ... # Pipeline-specific adapter overrides
229
+
230
+ params: # Parameters accessible in your Python code
231
+ greeting_message:
232
+ message: "Hello"
233
+ target_name:
234
+ name: "World"
235
+
236
+ run: # How to execute the pipeline
237
+ final_vars: # Specify the desired output(s) from your Hamilton DAG
238
+ - full_greeting
239
+ # inputs: # Optional: Specify input variables to the pipeline
240
+ # message: "Hello"
241
+ # config: ... # Runtime configuration overrides for Hamilton
242
+ # executor: ... # Execution backend (e.g., threadpool, multiprocessing)
243
+
244
+ ```
245
+ ### 3. Run Your Pipeline 🏃‍♀️
246
+
247
+ FlowerPower allows you to execute your pipelines synchronously, with flexible configuration options.
248
+
249
+ #### Synchronous Execution:
250
+
251
+ For quick testing or local runs, you can execute your pipeline synchronously. This is useful for debugging or running pipelines in a local environment.
252
+
253
+ * **Via CLI:**
254
+
255
+ The `flowerpower pipeline run` command now supports `RunConfig` objects (via file path or JSON string) and direct `**kwargs` for overriding.
256
+
257
+ ```bash
258
+ # Basic pipeline execution
259
+ flowerpower pipeline run hello_world
260
+
261
+ # Run with individual parameters (kwargs)
262
+ flowerpower pipeline run hello_world --inputs '{"greeting_message": "Hi", "target_name": "FlowerPower"}' --final-vars '["full_greeting"]' --log-level DEBUG
263
+
264
+ # Run using a RunConfig from a YAML file
265
+ # Assuming you have a run_config.yaml like:
266
+ # inputs:
267
+ # greeting_message: "Hola"
268
+ # target_name: "Amigo"
269
+ # log_level: "INFO"
270
+ flowerpower pipeline run hello_world --run-config ./run_config.yaml
271
+
272
+ # Run using a RunConfig provided as a JSON string
273
+ flowerpower pipeline run hello_world --run-config '{"inputs": {"greeting_message": "Bonjour", "target_name": "Monde"}, "log_level": "INFO"}'
274
+
275
+ # Mixing RunConfig with individual parameters (kwargs overrides RunConfig)
276
+ # This will run with log_level="DEBUG" and inputs={"greeting_message": "Howdy", "target_name": "Partner"}
277
+ flowerpower pipeline run hello_world --run-config '{"inputs": {"greeting_message": "Original", "target_name": "Value"}, "log_level": "INFO"}' --inputs '{"greeting_message": "Howdy", "target_name": "Partner"}' --log-level DEBUG
278
+ ```
279
+
280
+ * **Via Python:**
281
+
282
+ The `run` methods (`FlowerPowerProject.run`, `PipelineManager.run`) now primarily accept a `RunConfig` object, but also allow individual parameters to be passed via `**kwargs` which override `RunConfig` attributes.
283
+
284
+ ```python
285
+ from flowerpower import FlowerPowerProject
286
+ from flowerpower.cfg.pipeline.run import RunConfig
287
+ from flowerpower.cfg.pipeline.builder import RunConfigBuilder
288
+
289
+ # Load the project
290
+ project = FlowerPowerProject.load('.')
291
+
292
+ # Basic execution
293
+ result = project.run('hello_world')
294
+ print(result)
295
+
296
+ # Using individual parameters (kwargs)
297
+ result = project.run(
298
+ 'hello_world',
299
+ inputs={"greeting_message": "Hi", "target_name": "FlowerPower"},
300
+ final_vars=["full_greeting"],
301
+ log_level="DEBUG"
302
+ )
303
+ print(result)
304
+
305
+ # Using RunConfig directly
306
+ config = RunConfig(
307
+ inputs={"greeting_message": "Aloha", "target_name": "World"},
308
+ final_vars=["full_greeting"],
309
+ log_level="INFO"
310
+ )
311
+ result = project.run('hello_world', run_config=config)
312
+ print(result)
313
+
314
+ # Using RunConfigBuilder (recommended)
315
+ config = (
316
+ RunConfigBuilder(pipeline_name='hello_world')
317
+ .with_inputs({"greeting_message": "Greetings", "target_name": "Earth"})
318
+ .with_final_vars(["full_greeting"])
319
+ .with_log_level("DEBUG")
320
+ .with_retries(max_attempts=3, delay=1.0)
321
+ .build()
322
+ )
323
+ result = project.run('hello_world', run_config=config)
324
+ print(result)
325
+
326
+ # Mixing RunConfig with individual parameters (kwargs overrides RunConfig)
327
+ base_config = RunConfigBuilder().with_log_level("INFO").build()
328
+ result = project.run(
329
+ 'hello_world',
330
+ run_config=base_config,
331
+ inputs={"greeting_message": "Howdy", "target_name": "Partner"}, # Overrides inputs in base_config
332
+ log_level="DEBUG" # Overrides log_level in base_config
333
+ )
334
+ print(result)
335
+ ```
336
+
337
+
338
+ ## ⚙️ Configuration Overview
339
+
340
+ FlowerPower uses a layered configuration system:
341
+
342
+ * **`conf/project.yml`:** Defines global settings for your project, including integrated `adapter`s (like Hamilton Tracker, MLflow, etc.).
343
+ * **`conf/pipelines/*.yml`:** Each file defines a specific pipeline. It contains:
344
+ * `params`: Input parameters for your Hamilton functions.
345
+ * `run`: Execution details like target outputs (`final_vars`), Hamilton runtime `config`, and `executor` settings.
346
+ * `adapter`: Pipeline-specific overrides for adapter settings.
347
+
348
+ ## 🛠️ Basic Usage
349
+
350
+ You can interact with FlowerPower pipelines through multiple interfaces:
351
+
352
+ **Python API (Recommended):**
353
+ ```python
354
+ from flowerpower import FlowerPowerProject
355
+ from flowerpower.cfg.pipeline.run import RunConfig
356
+ from flowerpower.cfg.pipeline.builder import RunConfigBuilder
357
+
358
+ # Load the project
359
+ project = FlowerPowerProject.load('.')
360
+
361
+ # Run a pipeline using RunConfig
362
+ config = RunConfig(inputs={"greeting_message": "Hello", "target_name": "API"})
363
+ result = project.run('hello_world', run_config=config)
364
+ print(result)
365
+
366
+ # Run a pipeline using kwargs
367
+ result = project.run('hello_world', inputs={"greeting_message": "Hi", "target_name": "Kwargs"})
368
+ print(result)
369
+ ```
370
+
371
+ **CLI:**
372
+ ```bash
373
+ # Run a pipeline using RunConfig from a file
374
+ # flowerpower pipeline run hello_world --run-config ./path/to/run_config.yaml
375
+
376
+ # Run a pipeline using kwargs
377
+ flowerpower pipeline run hello_world --inputs '{"greeting_message": "CLI", "target_name": "Kwargs"}'
378
+
379
+ # List all available commands
380
+ flowerpower --help
381
+ ```
382
+
383
+ ## 🔧 Direct Module Usage
384
+
385
+ While the unified `FlowerPowerProject` interface is recommended for most use cases, you can also use the pipeline module directly for more granular control or when you only need specific functionality.
386
+
387
+ ### Pipeline-Only Usage
388
+
389
+ If you only need pipeline execution, you can use the `PipelineManager` directly:
390
+
391
+ ```python
392
+ from flowerpower.pipeline import PipelineManager
393
+ from flowerpower.cfg.pipeline.run import RunConfig
394
+ from flowerpower.cfg.pipeline.builder import RunConfigBuilder
395
+
396
+ # Initialize pipeline manager
397
+ pm = PipelineManager(base_dir='.')
398
+
399
+ # Create a new pipeline
400
+ pm.new(name='my_pipeline')
401
+
402
+ # Run a pipeline synchronously using RunConfig
403
+ config = RunConfig(inputs={'param': 'value'}, final_vars=['output_var'])
404
+ result = pm.run(name='my_pipeline', run_config=config)
405
+ print(result)
406
+
407
+ # Run a pipeline synchronously using kwargs
408
+ result = pm.run(name='my_pipeline', inputs={'param': 'new_value'}, final_vars=['output_var'])
409
+ print(result)
410
+
411
+ # List available pipelines
412
+ pipelines = pm.list()
413
+ print(f"Available pipelines: {pipelines}")
414
+
415
+ # Get pipeline information
416
+ info = pm.get('my_pipeline')
417
+ print(f"Pipeline config: {info}")
418
+
419
+ # Delete a pipeline
420
+ pm.delete('old_pipeline')
421
+ ```
422
+
423
+ **When to use Pipeline-only approach:**
424
+ - Simple synchronous workflows
425
+ - Testing and development
426
+ - Lightweight applications with minimal dependencies
427
+
428
+ **Benefits of FlowerPowerProject vs Direct Usage:**
429
+
430
+ | Approach | Benefits | Use Cases |
431
+ |----------|----------|-----------|
432
+ | **FlowerPowerProject** | - Unified interface<br>- Automatic dependency injection<br>- Simplified configuration<br>- Best practices built-in | - Most applications<br>- Rapid development<br>- Full feature integration |
433
+ | **Pipeline-only** | - Lightweight<br>- Simple synchronous execution | - Testing<br>- Simple workflows |
434
+
435
+ ## 🖥️ UI
436
+
437
+ The FlowerPower web UI (Hamilton UI) provides a graphical interface for monitoring and managing your pipelines. It allows you to visualize pipeline runs, schedules, and potentially manage configurations.
438
+
439
+ ```bash
440
+ # Start the web UI
441
+ flowerpower ui
442
+ ```
443
+
444
+ ## 📖 Documentation
445
+
446
+ You can find the full documentation for FlowerPower, including installation instructions, usage examples, and API references, at [https://legout.github.io/flowerpower/](https://legout.github.io/flowerpower/).
447
+
448
+
449
+ ## 📜 License
450
+
451
+ This project is licensed under the MIT License - see the `LICENSE` file for details. (Placeholder - update with actual license)
@@ -0,0 +1,42 @@
1
+ flowerpower/__init__.py,sha256=nHFrD6twJyR-Ti6Yn0lLcyXXcKOi0tFfOD96oAu17Js,431
2
+ flowerpower/flowerpower.py,sha256=-taA0uemnoy1B5ugNxzwQ_Jir8jfmaGNXTk_uewaN88,23755
3
+ flowerpower/cfg/__init__.py,sha256=aWeKlBNgJ4LCtWBzZns5RmQm9GN9SpTbozE-NsS2_9w,8481
4
+ flowerpower/cfg/base.py,sha256=IpvCcUvGTjmjN69D34IFNn692hxTwZzYQJ1utgKANWo,4795
5
+ flowerpower/cfg/pipeline/__init__.py,sha256=HSUlDt9smo-zPqOljEl329CQg8qsd0EJfCH1FOtuMdk,9276
6
+ flowerpower/cfg/pipeline/_schedule.py,sha256=kaDyij3eUj0u6VdmJW4x_AQbSAZ-r7s2Fk7QFkXU5JQ,1029
7
+ flowerpower/cfg/pipeline/adapter.py,sha256=uBKV6BZlsRRqSYNyC1oEWPchsaH7rFPCBobG5BrF3ss,2265
8
+ flowerpower/cfg/pipeline/builder.py,sha256=rZ-cspbV-nwtckvOA49vt7DNJNb-gRLHHWiejmJPsFs,13192
9
+ flowerpower/cfg/pipeline/run.py,sha256=cirZbiHsY8D87idBvbfNv1gEtKpgv3xLfiiFIXJFxrA,6733
10
+ flowerpower/cfg/project/__init__.py,sha256=bMMZ5Zo3YgkgOGt82H1mW8I4WnoCUpa0bSLPpZiB7K4,4560
11
+ flowerpower/cfg/project/adapter.py,sha256=2k2U25NziDEiUacLTjxaSxOVkaQBRt6ECWGRGX0v4J0,1481
12
+ flowerpower/cli/__init__.py,sha256=RkJT3mPlSOi55dNNP8kpm_omF0zEfbmWVP16N-lkijE,4897
13
+ flowerpower/cli/cfg.py,sha256=P7qEcjraitMxbzVWJMqWeitIdpUkW41QkUi7ol0ksW0,1455
14
+ flowerpower/cli/pipeline.py,sha256=I58cWjK7J4dkXdJXIePcPiD7iILnYqoje0ztkR5mNwg,21958
15
+ flowerpower/cli/utils.py,sha256=tsxvKIqUhl4m9IzuaSoc5a3_gb6Fu4LYyV8fVodqIdA,5127
16
+ flowerpower/pipeline/__init__.py,sha256=ltr4LQnM5Boa9w7t-lNVmmcSglIkTPuIoeYnEKWU4Og,122
17
+ flowerpower/pipeline/base.py,sha256=oQSDfEAyY2_kDRlHNnHr6ihZvfGeOjyMJRPKob1i7U8,3560
18
+ flowerpower/pipeline/io.py,sha256=phYJhN4LZ0c6d8_udEQ4C9cGzeV3Ku0hsj0gyE1n2UY,16246
19
+ flowerpower/pipeline/manager.py,sha256=gJMWe_T1WpxO2H9GDrNtf61S4oJuLugMHwX82jqAuEU,47579
20
+ flowerpower/pipeline/pipeline.py,sha256=8Vhc3GGoAJI-5zv8qX4gZ5JwJrTdVoiXHWNpv5OeKGw,26207
21
+ flowerpower/pipeline/registry.py,sha256=iHdEBPiRpZorAUe6sDkqONoP8t6cicdQYGdK8UyxuAQ,28791
22
+ flowerpower/pipeline/visualizer.py,sha256=EVpjv-TUe1zGvdEAWyShJcVXurm02W0jkLbj7z1uAv4,4953
23
+ flowerpower/plugins/io/__init__.py,sha256=ZmSdKoh3TavJagOz0vUItnEqAh3mAM1QpAWj0KufF_k,222
24
+ flowerpower/settings/__init__.py,sha256=XKQa8AI9VrX8ievs-suq3Cm6PBt4cJ78ZHVIjUbXCyA,130
25
+ flowerpower/settings/_backend.py,sha256=Up1RBqAs3jtDUOV-9wEpL68Qmom-dRWMOeHXIh0F3lw,4273
26
+ flowerpower/settings/executor.py,sha256=vNF383g1gMGkq_CXUzateGMNcJZig-vHkVVb0Hi_b74,249
27
+ flowerpower/settings/general.py,sha256=RxY6PGF_L8ApFlLPHulZ2I8_-aHYqOj63fUu9kSQTjI,227
28
+ flowerpower/settings/hamilton.py,sha256=GVzWKz3B-wy07etY1mNUstEa4DFrQ_lM2cjE0qG_6qw,623
29
+ flowerpower/settings/logging.py,sha256=BHahxfuiofByiTU8TYQ2AObPDRz9SYH-MalSEu1DRro,71
30
+ flowerpower/settings/retry.py,sha256=W3AAVSxmTUAeeSbzRGA37RxqtKKyUdi2MkwDzCsXP94,181
31
+ flowerpower/utils/callback.py,sha256=sGYSrEbnl0xfRa1X-mA-om3erpH7pmpsWdyPQ9HpU7E,6736
32
+ flowerpower/utils/logging.py,sha256=WUnpoEmr9H5MGFF68jlshhEJcuMtW49M7VVRstmqIAg,1192
33
+ flowerpower/utils/misc.py,sha256=gY1KVJp4lF4S0THUu1LUrjXaf87TzLO-TcwHxZZh2Zk,8414
34
+ flowerpower/utils/monkey.py,sha256=vJMYANjZI13PNbEQThdX0EFP1_6bGNHgnpF7HwReNyM,58
35
+ flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
36
+ flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
37
+ flowerpower-0.30.0.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
38
+ flowerpower-0.30.0.dist-info/METADATA,sha256=3nPehMWbDcryhbkgbieU-_13I_8S3-1aNL54bMdjdDs,17208
39
+ flowerpower-0.30.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
40
+ flowerpower-0.30.0.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
41
+ flowerpower-0.30.0.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
42
+ flowerpower-0.30.0.dist-info/RECORD,,
@@ -1,74 +0,0 @@
1
- import datetime as dt
2
-
3
- import msgspec
4
- from munch import munchify
5
-
6
- from ..base import BaseConfig
7
-
8
- # class ScheduleCronTriggerConfig(BaseConfig):
9
- # year: str | int | None = None
10
- # month: str | int | None = None
11
- # week: str | int | None = None
12
- # day: str | int | None = None
13
- # day_of_week: str | int | None = None
14
- # hour: str | int | None = None
15
- # minute: str | int | None = None
16
- # second: str | int | None = None
17
- # start_time: dt.datetime | None = None
18
- # end_time: dt.datetime | None = None
19
- # timezone: str | None = None
20
- # crontab: str | None = None
21
-
22
-
23
- # class ScheduleIntervalTriggerConfig(BaseConfig):
24
- # weeks: int | float | None = None
25
- # days: int | float | None = None
26
- # hours: int | float | None = None
27
- # minutes: int | float | None = None
28
- # seconds: int | float | None = None
29
- # microseconds: int | float | None = None
30
- # start_time: dt.datetime | None = None
31
- # end_time: dt.datetime | None = None
32
-
33
-
34
- # class ScheduleCalendarTriggerConfig(BaseConfig):
35
- # years: int | float | None = None
36
- # months: int | float | None = None
37
- # weeks: int | float | None = None
38
- # days: int | float | None = None
39
- # hour: int | float | None = None
40
- # minute: int | float | None = None
41
- # second: int | float | None = None
42
- # start_date: dt.datetime | None = None
43
- # end_date: dt.datetime | None = None
44
- # timezone: str | None = None
45
-
46
-
47
- # class ScheduleDateTriggerConfig(BaseConfig):
48
- # run_time: dt.datetime | None = None
49
-
50
-
51
- class ScheduleConfig(BaseConfig):
52
- cron: str | dict | None = msgspec.field(default=None)
53
- interval: str | int | dict | None = msgspec.field(default=None)
54
- date: str | None = msgspec.field(default=None)
55
-
56
- def __post_init__(self):
57
- if isinstance(self.date, str):
58
- try:
59
- self.date = dt.datetime.fromisoformat(self.date)
60
- except ValueError:
61
- raise ValueError(
62
- f"Invalid date format: {self.date}. Expected ISO format."
63
- )
64
- if isinstance(self.cron, dict):
65
- self.cron = munchify(self.cron)
66
- if isinstance(self.interval, dict):
67
- self.interval = munchify(self.interval)
68
-
69
-
70
- # class ScheduleConfig(BaseConfig):
71
- # run: ScheduleRunConfig = msgspec.field(default_factory=ScheduleRunConfig)
72
- # trigger: ScheduleTriggerConfig = msgspec.field(
73
- # default_factory=ScheduleTriggerConfig
74
- # )