FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. flowerpower/__init__.py +2 -6
  2. flowerpower/cfg/__init__.py +7 -14
  3. flowerpower/cfg/base.py +29 -25
  4. flowerpower/cfg/pipeline/__init__.py +8 -6
  5. flowerpower/cfg/pipeline/_schedule.py +32 -0
  6. flowerpower/cfg/pipeline/adapter.py +0 -5
  7. flowerpower/cfg/pipeline/builder.py +377 -0
  8. flowerpower/cfg/pipeline/run.py +36 -0
  9. flowerpower/cfg/project/__init__.py +11 -24
  10. flowerpower/cfg/project/adapter.py +0 -12
  11. flowerpower/cli/__init__.py +2 -21
  12. flowerpower/cli/cfg.py +0 -3
  13. flowerpower/cli/mqtt.py +0 -6
  14. flowerpower/cli/pipeline.py +22 -415
  15. flowerpower/cli/utils.py +0 -1
  16. flowerpower/flowerpower.py +345 -146
  17. flowerpower/pipeline/__init__.py +2 -0
  18. flowerpower/pipeline/base.py +21 -12
  19. flowerpower/pipeline/io.py +58 -54
  20. flowerpower/pipeline/manager.py +165 -726
  21. flowerpower/pipeline/pipeline.py +643 -0
  22. flowerpower/pipeline/registry.py +285 -18
  23. flowerpower/pipeline/visualizer.py +5 -6
  24. flowerpower/plugins/io/__init__.py +8 -0
  25. flowerpower/plugins/mqtt/__init__.py +7 -11
  26. flowerpower/settings/__init__.py +0 -2
  27. flowerpower/settings/{backend.py → _backend.py} +0 -21
  28. flowerpower/settings/logging.py +1 -1
  29. flowerpower/utils/logging.py +24 -12
  30. flowerpower/utils/misc.py +17 -256
  31. flowerpower/utils/monkey.py +1 -83
  32. flowerpower-0.21.0.dist-info/METADATA +463 -0
  33. flowerpower-0.21.0.dist-info/RECORD +44 -0
  34. flowerpower/cfg/pipeline/schedule.py +0 -74
  35. flowerpower/cfg/project/job_queue.py +0 -238
  36. flowerpower/cli/job_queue.py +0 -1061
  37. flowerpower/fs/__init__.py +0 -29
  38. flowerpower/fs/base.py +0 -662
  39. flowerpower/fs/ext.py +0 -2143
  40. flowerpower/fs/storage_options.py +0 -1420
  41. flowerpower/job_queue/__init__.py +0 -294
  42. flowerpower/job_queue/apscheduler/__init__.py +0 -11
  43. flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
  44. flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
  45. flowerpower/job_queue/apscheduler/manager.py +0 -1051
  46. flowerpower/job_queue/apscheduler/setup.py +0 -554
  47. flowerpower/job_queue/apscheduler/trigger.py +0 -169
  48. flowerpower/job_queue/apscheduler/utils.py +0 -311
  49. flowerpower/job_queue/base.py +0 -413
  50. flowerpower/job_queue/rq/__init__.py +0 -10
  51. flowerpower/job_queue/rq/_trigger.py +0 -37
  52. flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -226
  53. flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -231
  54. flowerpower/job_queue/rq/manager.py +0 -1582
  55. flowerpower/job_queue/rq/setup.py +0 -154
  56. flowerpower/job_queue/rq/utils.py +0 -69
  57. flowerpower/mqtt.py +0 -12
  58. flowerpower/pipeline/job_queue.py +0 -583
  59. flowerpower/pipeline/runner.py +0 -603
  60. flowerpower/plugins/io/base.py +0 -2520
  61. flowerpower/plugins/io/helpers/datetime.py +0 -298
  62. flowerpower/plugins/io/helpers/polars.py +0 -875
  63. flowerpower/plugins/io/helpers/pyarrow.py +0 -570
  64. flowerpower/plugins/io/helpers/sql.py +0 -202
  65. flowerpower/plugins/io/loader/__init__.py +0 -28
  66. flowerpower/plugins/io/loader/csv.py +0 -37
  67. flowerpower/plugins/io/loader/deltatable.py +0 -190
  68. flowerpower/plugins/io/loader/duckdb.py +0 -19
  69. flowerpower/plugins/io/loader/json.py +0 -37
  70. flowerpower/plugins/io/loader/mqtt.py +0 -159
  71. flowerpower/plugins/io/loader/mssql.py +0 -26
  72. flowerpower/plugins/io/loader/mysql.py +0 -26
  73. flowerpower/plugins/io/loader/oracle.py +0 -26
  74. flowerpower/plugins/io/loader/parquet.py +0 -35
  75. flowerpower/plugins/io/loader/postgres.py +0 -26
  76. flowerpower/plugins/io/loader/pydala.py +0 -19
  77. flowerpower/plugins/io/loader/sqlite.py +0 -23
  78. flowerpower/plugins/io/metadata.py +0 -244
  79. flowerpower/plugins/io/saver/__init__.py +0 -28
  80. flowerpower/plugins/io/saver/csv.py +0 -36
  81. flowerpower/plugins/io/saver/deltatable.py +0 -186
  82. flowerpower/plugins/io/saver/duckdb.py +0 -19
  83. flowerpower/plugins/io/saver/json.py +0 -36
  84. flowerpower/plugins/io/saver/mqtt.py +0 -28
  85. flowerpower/plugins/io/saver/mssql.py +0 -26
  86. flowerpower/plugins/io/saver/mysql.py +0 -26
  87. flowerpower/plugins/io/saver/oracle.py +0 -26
  88. flowerpower/plugins/io/saver/parquet.py +0 -36
  89. flowerpower/plugins/io/saver/postgres.py +0 -26
  90. flowerpower/plugins/io/saver/pydala.py +0 -20
  91. flowerpower/plugins/io/saver/sqlite.py +0 -24
  92. flowerpower/plugins/mqtt/cfg.py +0 -17
  93. flowerpower/plugins/mqtt/manager.py +0 -962
  94. flowerpower/settings/job_queue.py +0 -87
  95. flowerpower/utils/scheduler.py +0 -311
  96. flowerpower-0.11.6.20.dist-info/METADATA +0 -537
  97. flowerpower-0.11.6.20.dist-info/RECORD +0 -102
  98. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/WHEEL +0 -0
  99. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/entry_points.txt +0 -0
  100. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/licenses/LICENSE +0 -0
  101. {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,463 @@
1
+ Metadata-Version: 2.4
2
+ Name: FlowerPower
3
+ Version: 0.21.0
4
+ Summary: A simple workflow framework for building and managing data processing pipelines
5
+ Author-email: "Volker L." <ligno.blades@gmail.com>
6
+ Project-URL: Homepage, https://github.com/legout/flowerpower
7
+ Project-URL: Bug Tracker, https://github.com/legout/flowerpower/issues
8
+ Keywords: hamilton,workflow,pipeline,scheduler,dask,ray
9
+ Requires-Python: >=3.11
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: duration-parser>=1.0.1
13
+ Requires-Dist: fsspec>=2024.10.0
14
+ Requires-Dist: fsspec-utils>=0.1.0
15
+ Requires-Dist: humanize>=4.12.2
16
+ Requires-Dist: msgspec>=0.19.0
17
+ Requires-Dist: munch>=4.0.0
18
+ Requires-Dist: pyyaml>=6.0.1
19
+ Requires-Dist: rich>=13.9.3
20
+ Requires-Dist: s3fs>=2024.10.0
21
+ Requires-Dist: sf-hamilton-sdk>=0.5.2
22
+ Requires-Dist: sf-hamilton[rich,tqdm,visualization]>=1.69.0
23
+ Requires-Dist: typer>=0.12.3
24
+ Provides-Extra: io
25
+ Requires-Dist: flowerpower-io>=0.1.1; extra == "io"
26
+ Provides-Extra: io-legacy
27
+ Requires-Dist: flowerpower-io[legacy]>=0.1.1; extra == "io-legacy"
28
+ Provides-Extra: mongodb
29
+ Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
30
+ Provides-Extra: mqtt
31
+ Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
32
+ Requires-Dist: orjson>=3.10.11; extra == "mqtt"
33
+ Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
34
+ Provides-Extra: opentelemetry
35
+ Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
36
+ Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
37
+ Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
38
+ Provides-Extra: ray
39
+ Requires-Dist: ray>=2.34.0; extra == "ray"
40
+ Provides-Extra: tui
41
+ Requires-Dist: textual>=0.85.2; extra == "tui"
42
+ Provides-Extra: ui
43
+ Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
44
+ Provides-Extra: webserver
45
+ Requires-Dist: sanic>=24.6.0; extra == "webserver"
46
+ Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
47
+ Requires-Dist: orjson>=3.10.11; extra == "webserver"
48
+ Provides-Extra: openlineage
49
+ Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
50
+ Dynamic: license-file
51
+
52
+ <div align="center">
53
+ <h1>FlowerPower 🌸 - Build & Orchestrate Data Pipelines</h1>
54
+ <h3>Simple Workflow Framework - Hamilton = FlowerPower</h3>
55
+ <img src="./image.png" alt="FlowerPower Logo" width="400" height="300">
56
+ </div>
57
+
58
+
59
+
60
+ [![PyPI version](https://img.shields.io/pypi/v/flowerpower.svg?style=flat-square)](https://pypi.org/project/flowerpower/) <!-- Placeholder -->
61
+ [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/legout/flowerpower/blob/main/LICENSE)
62
+ [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/legout/flowerpower)
63
+ [![Documentation Status](https://readthedocs.org/projects/flowerpower/badge/?version=latest)](https://legout.github.io/flowerpower/)
64
+
65
+
66
+ **FlowerPower** is a Python framework designed for building, configuring, and executing data processing pipelines with ease and flexibility. It promotes a modular, configuration-driven approach, allowing you to focus on your pipeline logic while FlowerPower handles the orchestration.
67
+
68
+ It leverages the [Hamilton](https://github.com/apache/hamilton) library for defining dataflows in a clean, functional way within your Python pipeline scripts. Pipelines are defined in Python modules and configured using YAML files, making it easy to manage and understand your data workflows.
69
+ FlowerPower provides a unified project interface that makes it easy to work with pipeline execution. It also provides a web UI (Hamilton UI) for monitoring and managing your pipelines.
70
+ FlowerPower is designed to be extensible, allowing you to easily add custom I/O plugins and adapt to different deployment scenarios. This flexibility makes it suitable for a wide range of data processing tasks, from simple ETL jobs to complex data workflows.
71
+
72
+
73
+ ## ✨ Key Features
74
+
75
+ * **Modular Pipeline Design:** Thanks to [Hamilton](https://github.com/apache/hamilton), you can define your data processing logic in Python modules, using functions as nodes in a directed acyclic graph (DAG).
76
+ * **Configuration-Driven:** Define pipeline parameters, execution logic, and scheduling declaratively using simple YAML files.
77
+ * **Extensible I/O Plugins:** Connect to various data sources and destinations (CSV, JSON, Parquet, DeltaTable, DuckDB, PostgreSQL, MySQL, MSSQL, Oracle, MQTT, SQLite, and more).
78
+ * **Unified Project Interface:** Interact with your pipelines via:
79
+ * **FlowerPowerProject API:** A unified interface for pipeline execution, supporting both `RunConfig` objects and flexible `**kwargs` overrides.
80
+ * **Command Line Interface (CLI):** For running, managing, and inspecting pipelines, with enhanced `run` command capabilities.
81
+ * **Web UI:** A graphical interface for monitoring and managing pipelines and schedules. ([Hamilton UI](https://hamilton.dagworks.io/en/latest/hamilton-ui/ui/))
82
+ * **Filesystem Abstraction:** Simplified file handling with support for local and remote filesystems (e.g., S3, GCS).
83
+
84
+ ## 📦 Installation
85
+
86
+ We recommend using [uv](https://github.com/astral-sh/uv) for installing FlowerPower and managing your project environments. `uv` is an extremely fast Python package installer and resolver.
87
+
88
+ ```bash
89
+ # Create and activate a virtual environment (recommended)
90
+ uv venv
91
+ source .venv/bin/activate # Or .\.venv\Scripts\activate on Windows
92
+
93
+ # Install FlowerPower
94
+ uv pip install flowerpower
95
+
96
+ # Optional: Install additional dependencies for specific features
97
+ uv pip install flowerpower[io] # For I/O plugins (CSV, JSON, Parquet, DeltaTable, DuckDB, PostgreSQL, MySQL, MSSQL, Oracle, SQLite)
98
+ uv pip install flowerpower[ui] # For Hamilton UI
99
+ uv pip install flowerpower[all] # Install all optional dependencies
100
+ ```
101
+
102
+ *(Note: Specify required Python versions if known, e.g., Python 3.8+)*
103
+
104
+ ## 🚀 Getting Started
105
+
106
+ Let's build a simple "Hello World" pipeline.
107
+
108
+ ### 1. Initialize Your Project:
109
+
110
+ You can quickly set up the standard FlowerPower project structure using the CLI or Python.
111
+
112
+ **Using the CLI:**
113
+
114
+ Navigate to your desired parent directory and run:
115
+ ```bash
116
+ flowerpower init --name hello-flowerpower-project
117
+ ```
118
+
119
+
120
+ **Using Python:**
121
+
122
+ Alternatively, you can initialize programmatically:
123
+ ```python
124
+ from flowerpower import FlowerPowerProject
125
+
126
+ # Initialize a new project
127
+ project = FlowerPowerProject.init(
128
+ name='hello-flowerpower-project',
129
+ )
130
+ ```
131
+
132
+ This will create a `hello-flowerpower-project` directory with the necessary `conf/` and `pipelines/` subdirectories and default configuration files.
133
+
134
+ ```
135
+ hello-flowerpower-project/
136
+ ├── conf/
137
+ │ ├── project.yml
138
+ │ └── pipelines/
139
+ └── pipelines/
140
+ ```
141
+
142
+ Now, navigate into your new project directory:
143
+
144
+ ```bash
145
+ cd hello-flowerpower-project
146
+ ```
147
+
148
+ **Configure Project (`conf/project.yml`):**
149
+
150
+ Open `conf/project.yml` and define your project name:
151
+
152
+ ```yaml
153
+ name: hello-flowerpower
154
+ # adapter: ... # Optional adapter configurations (e.g., Hamilton Tracker, MLflow), see `conf/project.yml` for details
155
+ ```
156
+
157
+ ### 2. Create Your Pipeline
158
+
159
+ You can create a new pipeline using the CLI or programmatically.
160
+
161
+ **Using the CLI:**
162
+
163
+ ```bash
164
+ flowerpower pipeline new hello_world
165
+ ```
166
+
167
+ **Using Python:**
168
+
169
+ You can create pipelines programmatically using the FlowerPowerProject interface:
170
+
171
+ ```python
172
+ from flowerpower import FlowerPowerProject
173
+
174
+ # Load the project
175
+ project = FlowerPowerProject.load('.')
176
+
177
+ # Create a new pipeline
178
+ project.pipeline_manager.new(name='hello_world')
179
+ ```
180
+
181
+ This will create a new file `hello_world.py` in the `pipelines/` directory and a corresponding configuration file `hello_world.yml` in `conf/pipelines/`.
182
+
183
+ **Implement Pipeline (`pipelines/hello_world.py`):**
184
+
185
+ Open `pipelines/hello_world.py` and write your pipeline logic using Python and Hamilton. FlowerPower makes configuration easily accessible.
186
+
187
+ ```python
188
+ # FlowerPower pipeline hello_world.py
189
+ # Created on 2025-05-03 22:34:09
190
+
191
+ ####################################################################################################
192
+ # Import necessary libraries
193
+ # NOTE: Remove or comment out imports that are not used in the pipeline
194
+
195
+ from hamilton.function_modifiers import parameterize
196
+
197
+ from pathlib import Path
198
+
199
+ from flowerpower.cfg import Config
200
+
201
+ ####################################################################################################
202
+ # Load pipeline parameters. Do not modify this section.
203
+
204
+ PARAMS = Config.load(
205
+ Path(__file__).parents[1], pipeline_name="hello_world"
206
+ ).pipeline.h_params
207
+
208
+
209
+ ####################################################################################################
210
+ # Helper functions.
211
+ # This functions have to start with an underscore (_).
212
+
213
+
214
+ ####################################################################################################
215
+ # Pipeline functions
216
+
217
+ @parameterize(**PARAMS.greeting_message) # Inject 'message' from params
218
+ def greeting_message(message: str) -> str:
219
+ """Provides the greeting part."""
220
+ return f"{message},"
221
+
222
+ @parameterize(**PARAMS.target_name) # Inject 'name' from params
223
+ def target_name(name: str) -> str:
224
+ """Provides the target name."""
225
+ return f"{name}!"
226
+
227
+ def full_greeting(greeting_message: str, target_name: str) -> str:
228
+ """Combines the greeting and target."""
229
+ print(f"Generating greeting: {greeting_message} {target_name}")
230
+ return f"{greeting_message} {target_name}"
231
+
232
+ # You can add more complex Hamilton functions here...
233
+ ```
234
+
235
+ **Configure Pipeline (`conf/pipelines/hello_world.yml`):**
236
+
237
+ Open `conf/pipelines/hello_world.yml` and specify parameters, run configurations, and scheduling for your pipeline.
238
+
239
+ ```yaml
240
+ # adapter: ... # Pipeline-specific adapter overrides
241
+
242
+ params: # Parameters accessible in your Python code
243
+ greeting_message:
244
+ message: "Hello"
245
+ target_name:
246
+ name: "World"
247
+
248
+ run: # How to execute the pipeline
249
+ final_vars: # Specify the desired output(s) from your Hamilton DAG
250
+ - full_greeting
251
+ # inputs: # Optional: Specify input variables to the pipeline
252
+ # message: "Hello"
253
+ # config: ... # Runtime configuration overrides for Hamilton
254
+ # executor: ... # Execution backend (e.g., threadpool, multiprocessing)
255
+
256
+ ```
257
+ ### 3. Run Your Pipeline 🏃‍♀️
258
+
259
+ FlowerPower allows you to execute your pipelines synchronously, with flexible configuration options.
260
+
261
+ #### Synchronous Execution:
262
+
263
+ For quick testing or local runs, you can execute your pipeline synchronously. This is useful for debugging or running pipelines in a local environment.
264
+
265
+ * **Via CLI:**
266
+
267
+ The `flowerpower pipeline run` command now supports `RunConfig` objects (via file path or JSON string) and direct `**kwargs` for overriding.
268
+
269
+ ```bash
270
+ # Basic pipeline execution
271
+ flowerpower pipeline run hello_world
272
+
273
+ # Run with individual parameters (kwargs)
274
+ flowerpower pipeline run hello_world --inputs '{"greeting_message": "Hi", "target_name": "FlowerPower"}' --final-vars '["full_greeting"]' --log-level DEBUG
275
+
276
+ # Run using a RunConfig from a YAML file
277
+ # Assuming you have a run_config.yaml like:
278
+ # inputs:
279
+ # greeting_message: "Hola"
280
+ # target_name: "Amigo"
281
+ # log_level: "INFO"
282
+ flowerpower pipeline run hello_world --run-config ./run_config.yaml
283
+
284
+ # Run using a RunConfig provided as a JSON string
285
+ flowerpower pipeline run hello_world --run-config '{"inputs": {"greeting_message": "Bonjour", "target_name": "Monde"}, "log_level": "INFO"}'
286
+
287
+ # Mixing RunConfig with individual parameters (kwargs overrides RunConfig)
288
+ # This will run with log_level="DEBUG" and inputs={"greeting_message": "Howdy", "target_name": "Partner"}
289
+ flowerpower pipeline run hello_world --run-config '{"inputs": {"greeting_message": "Original", "target_name": "Value"}, "log_level": "INFO"}' --inputs '{"greeting_message": "Howdy", "target_name": "Partner"}' --log-level DEBUG
290
+ ```
291
+
292
+ * **Via Python:**
293
+
294
+ The `run` methods (`FlowerPowerProject.run`, `PipelineManager.run`) now primarily accept a `RunConfig` object, but also allow individual parameters to be passed via `**kwargs` which override `RunConfig` attributes.
295
+
296
+ ```python
297
+ from flowerpower import FlowerPowerProject
298
+ from flowerpower.cfg.pipeline.run import RunConfig
299
+ from flowerpower.cfg.pipeline.builder import RunConfigBuilder
300
+
301
+ # Load the project
302
+ project = FlowerPowerProject.load('.')
303
+
304
+ # Basic execution
305
+ result = project.run('hello_world')
306
+ print(result)
307
+
308
+ # Using individual parameters (kwargs)
309
+ result = project.run(
310
+ 'hello_world',
311
+ inputs={"greeting_message": "Hi", "target_name": "FlowerPower"},
312
+ final_vars=["full_greeting"],
313
+ log_level="DEBUG"
314
+ )
315
+ print(result)
316
+
317
+ # Using RunConfig directly
318
+ config = RunConfig(
319
+ inputs={"greeting_message": "Aloha", "target_name": "World"},
320
+ final_vars=["full_greeting"],
321
+ log_level="INFO"
322
+ )
323
+ result = project.run('hello_world', run_config=config)
324
+ print(result)
325
+
326
+ # Using RunConfigBuilder (recommended)
327
+ config = (
328
+ RunConfigBuilder(pipeline_name='hello_world')
329
+ .with_inputs({"greeting_message": "Greetings", "target_name": "Earth"})
330
+ .with_final_vars(["full_greeting"])
331
+ .with_log_level("DEBUG")
332
+ .with_retries(max_attempts=3, delay=1.0)
333
+ .build()
334
+ )
335
+ result = project.run('hello_world', run_config=config)
336
+ print(result)
337
+
338
+ # Mixing RunConfig with individual parameters (kwargs overrides RunConfig)
339
+ base_config = RunConfigBuilder().with_log_level("INFO").build()
340
+ result = project.run(
341
+ 'hello_world',
342
+ run_config=base_config,
343
+ inputs={"greeting_message": "Howdy", "target_name": "Partner"}, # Overrides inputs in base_config
344
+ log_level="DEBUG" # Overrides log_level in base_config
345
+ )
346
+ print(result)
347
+ ```
348
+
349
+
350
+ ## ⚙️ Configuration Overview
351
+
352
+ FlowerPower uses a layered configuration system:
353
+
354
+ * **`conf/project.yml`:** Defines global settings for your project, including integrated `adapter`s (like Hamilton Tracker, MLflow, etc.).
355
+ * **`conf/pipelines/*.yml`:** Each file defines a specific pipeline. It contains:
356
+ * `params`: Input parameters for your Hamilton functions.
357
+ * `run`: Execution details like target outputs (`final_vars`), Hamilton runtime `config`, and `executor` settings.
358
+ * `adapter`: Pipeline-specific overrides for adapter settings.
359
+
360
+ ## 🛠️ Basic Usage
361
+
362
+ You can interact with FlowerPower pipelines through multiple interfaces:
363
+
364
+ **Python API (Recommended):**
365
+ ```python
366
+ from flowerpower import FlowerPowerProject
367
+ from flowerpower.cfg.pipeline.run import RunConfig
368
+ from flowerpower.cfg.pipeline.builder import RunConfigBuilder
369
+
370
+ # Load the project
371
+ project = FlowerPowerProject.load('.')
372
+
373
+ # Run a pipeline using RunConfig
374
+ config = RunConfig(inputs={"greeting_message": "Hello", "target_name": "API"})
375
+ result = project.run('hello_world', run_config=config)
376
+ print(result)
377
+
378
+ # Run a pipeline using kwargs
379
+ result = project.run('hello_world', inputs={"greeting_message": "Hi", "target_name": "Kwargs"})
380
+ print(result)
381
+ ```
382
+
383
+ **CLI:**
384
+ ```bash
385
+ # Run a pipeline using RunConfig from a file
386
+ # flowerpower pipeline run hello_world --run-config ./path/to/run_config.yaml
387
+
388
+ # Run a pipeline using kwargs
389
+ flowerpower pipeline run hello_world --inputs '{"greeting_message": "CLI", "target_name": "Kwargs"}'
390
+
391
+ # List all available commands
392
+ flowerpower --help
393
+ ```
394
+
395
+ ## 🔧 Direct Module Usage
396
+
397
+ While the unified `FlowerPowerProject` interface is recommended for most use cases, you can also use the pipeline module directly for more granular control or when you only need specific functionality.
398
+
399
+ ### Pipeline-Only Usage
400
+
401
+ If you only need pipeline execution, you can use the `PipelineManager` directly:
402
+
403
+ ```python
404
+ from flowerpower.pipeline import PipelineManager
405
+ from flowerpower.cfg.pipeline.run import RunConfig
406
+ from flowerpower.cfg.pipeline.builder import RunConfigBuilder
407
+
408
+ # Initialize pipeline manager
409
+ pm = PipelineManager(base_dir='.')
410
+
411
+ # Create a new pipeline
412
+ pm.new(name='my_pipeline')
413
+
414
+ # Run a pipeline synchronously using RunConfig
415
+ config = RunConfig(inputs={'param': 'value'}, final_vars=['output_var'])
416
+ result = pm.run(name='my_pipeline', run_config=config)
417
+ print(result)
418
+
419
+ # Run a pipeline synchronously using kwargs
420
+ result = pm.run(name='my_pipeline', inputs={'param': 'new_value'}, final_vars=['output_var'])
421
+ print(result)
422
+
423
+ # List available pipelines
424
+ pipelines = pm.list()
425
+ print(f"Available pipelines: {pipelines}")
426
+
427
+ # Get pipeline information
428
+ info = pm.get('my_pipeline')
429
+ print(f"Pipeline config: {info}")
430
+
431
+ # Delete a pipeline
432
+ pm.delete('old_pipeline')
433
+ ```
434
+
435
+ **When to use Pipeline-only approach:**
436
+ - Simple synchronous workflows
437
+ - Testing and development
438
+ - Lightweight applications with minimal dependencies
439
+
440
+ **Benefits of FlowerPowerProject vs Direct Usage:**
441
+
442
+ | Approach | Benefits | Use Cases |
443
+ |----------|----------|-----------|
444
+ | **FlowerPowerProject** | - Unified interface<br>- Automatic dependency injection<br>- Simplified configuration<br>- Best practices built-in | - Most applications<br>- Rapid development<br>- Full feature integration |
445
+ | **Pipeline-only** | - Lightweight<br>- Simple synchronous execution | - Testing<br>- Simple workflows |
446
+
447
+ ## 🖥️ UI
448
+
449
+ The FlowerPower web UI (Hamilton UI) provides a graphical interface for monitoring and managing your pipelines. It allows you to visualize pipeline runs, schedules, and potentially manage configurations.
450
+
451
+ ```bash
452
+ # Start the web UI
453
+ flowerpower ui
454
+ ```
455
+
456
+ ## 📖 Documentation
457
+
458
+ You can find the full documentation for FlowerPower, including installation instructions, usage examples, and API references, at [https://legout.github.io/flowerpower/](https://legout.github.io/flowerpower/).
459
+
460
+
461
+ ## 📜 License
462
+
463
+ This project is licensed under the MIT License - see the `LICENSE` file for details. (Placeholder - update with actual license)
@@ -0,0 +1,44 @@
1
+ flowerpower/__init__.py,sha256=nHFrD6twJyR-Ti6Yn0lLcyXXcKOi0tFfOD96oAu17Js,431
2
+ flowerpower/flowerpower.py,sha256=-taA0uemnoy1B5ugNxzwQ_Jir8jfmaGNXTk_uewaN88,23755
3
+ flowerpower/cfg/__init__.py,sha256=aWeKlBNgJ4LCtWBzZns5RmQm9GN9SpTbozE-NsS2_9w,8481
4
+ flowerpower/cfg/base.py,sha256=IpvCcUvGTjmjN69D34IFNn692hxTwZzYQJ1utgKANWo,4795
5
+ flowerpower/cfg/pipeline/__init__.py,sha256=HSUlDt9smo-zPqOljEl329CQg8qsd0EJfCH1FOtuMdk,9276
6
+ flowerpower/cfg/pipeline/_schedule.py,sha256=kaDyij3eUj0u6VdmJW4x_AQbSAZ-r7s2Fk7QFkXU5JQ,1029
7
+ flowerpower/cfg/pipeline/adapter.py,sha256=uBKV6BZlsRRqSYNyC1oEWPchsaH7rFPCBobG5BrF3ss,2265
8
+ flowerpower/cfg/pipeline/builder.py,sha256=rZ-cspbV-nwtckvOA49vt7DNJNb-gRLHHWiejmJPsFs,13192
9
+ flowerpower/cfg/pipeline/run.py,sha256=xdHkB3Dx2ejkRUeK9bGHrIcO_JkFH3Hok_AFSO0WWT8,4212
10
+ flowerpower/cfg/project/__init__.py,sha256=bMMZ5Zo3YgkgOGt82H1mW8I4WnoCUpa0bSLPpZiB7K4,4560
11
+ flowerpower/cfg/project/adapter.py,sha256=2k2U25NziDEiUacLTjxaSxOVkaQBRt6ECWGRGX0v4J0,1481
12
+ flowerpower/cli/__init__.py,sha256=ZojMSbKvwtsOu0gryA5_GOEtUy3l6ZxYNvvKyhWsTxk,5079
13
+ flowerpower/cli/cfg.py,sha256=P7qEcjraitMxbzVWJMqWeitIdpUkW41QkUi7ol0ksW0,1455
14
+ flowerpower/cli/mqtt.py,sha256=NuQtvKA3DrmLdGRsR7CFhbVhKZ4qObzwRlvENSkTc0k,6009
15
+ flowerpower/cli/pipeline.py,sha256=I58cWjK7J4dkXdJXIePcPiD7iILnYqoje0ztkR5mNwg,21958
16
+ flowerpower/cli/utils.py,sha256=tsxvKIqUhl4m9IzuaSoc5a3_gb6Fu4LYyV8fVodqIdA,5127
17
+ flowerpower/pipeline/__init__.py,sha256=ltr4LQnM5Boa9w7t-lNVmmcSglIkTPuIoeYnEKWU4Og,122
18
+ flowerpower/pipeline/base.py,sha256=oQSDfEAyY2_kDRlHNnHr6ihZvfGeOjyMJRPKob1i7U8,3560
19
+ flowerpower/pipeline/io.py,sha256=phYJhN4LZ0c6d8_udEQ4C9cGzeV3Ku0hsj0gyE1n2UY,16246
20
+ flowerpower/pipeline/manager.py,sha256=ILnFUI8YeyCJsu7wS8g8Os6cNIhxoK6EihfkN5azWaI,47698
21
+ flowerpower/pipeline/pipeline.py,sha256=8Vhc3GGoAJI-5zv8qX4gZ5JwJrTdVoiXHWNpv5OeKGw,26207
22
+ flowerpower/pipeline/registry.py,sha256=iHdEBPiRpZorAUe6sDkqONoP8t6cicdQYGdK8UyxuAQ,28791
23
+ flowerpower/pipeline/visualizer.py,sha256=EVpjv-TUe1zGvdEAWyShJcVXurm02W0jkLbj7z1uAv4,4953
24
+ flowerpower/plugins/io/__init__.py,sha256=ZmSdKoh3TavJagOz0vUItnEqAh3mAM1QpAWj0KufF_k,222
25
+ flowerpower/plugins/mqtt/__init__.py,sha256=4eTTvYY4SjC7O2lMaK-Hw5LQGvZE8TVl6HsXuVVD9zQ,228
26
+ flowerpower/settings/__init__.py,sha256=XKQa8AI9VrX8ievs-suq3Cm6PBt4cJ78ZHVIjUbXCyA,130
27
+ flowerpower/settings/_backend.py,sha256=Up1RBqAs3jtDUOV-9wEpL68Qmom-dRWMOeHXIh0F3lw,4273
28
+ flowerpower/settings/executor.py,sha256=vNF383g1gMGkq_CXUzateGMNcJZig-vHkVVb0Hi_b74,249
29
+ flowerpower/settings/general.py,sha256=RxY6PGF_L8ApFlLPHulZ2I8_-aHYqOj63fUu9kSQTjI,227
30
+ flowerpower/settings/hamilton.py,sha256=GVzWKz3B-wy07etY1mNUstEa4DFrQ_lM2cjE0qG_6qw,623
31
+ flowerpower/settings/logging.py,sha256=BHahxfuiofByiTU8TYQ2AObPDRz9SYH-MalSEu1DRro,71
32
+ flowerpower/settings/retry.py,sha256=W3AAVSxmTUAeeSbzRGA37RxqtKKyUdi2MkwDzCsXP94,181
33
+ flowerpower/utils/callback.py,sha256=sGYSrEbnl0xfRa1X-mA-om3erpH7pmpsWdyPQ9HpU7E,6736
34
+ flowerpower/utils/logging.py,sha256=WUnpoEmr9H5MGFF68jlshhEJcuMtW49M7VVRstmqIAg,1192
35
+ flowerpower/utils/misc.py,sha256=gY1KVJp4lF4S0THUu1LUrjXaf87TzLO-TcwHxZZh2Zk,8414
36
+ flowerpower/utils/monkey.py,sha256=vJMYANjZI13PNbEQThdX0EFP1_6bGNHgnpF7HwReNyM,58
37
+ flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
38
+ flowerpower/utils/templates.py,sha256=ouyEeSDqa9PjW8c32fGpcINlpC0WToawRFZkMPtwsLE,1591
39
+ flowerpower-0.21.0.dist-info/licenses/LICENSE,sha256=9AkLexxrmr0aBgSHiqxpJk9wgazpP1CTJyiDyr56J9k,1063
40
+ flowerpower-0.21.0.dist-info/METADATA,sha256=fkVSaKr-aU6eiaRAvydDHWfXf1J_a02Ca8Zw_zCjgHA,17691
41
+ flowerpower-0.21.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
42
+ flowerpower-0.21.0.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
43
+ flowerpower-0.21.0.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
44
+ flowerpower-0.21.0.dist-info/RECORD,,
@@ -1,74 +0,0 @@
1
- import datetime as dt
2
-
3
- import msgspec
4
- from munch import munchify
5
-
6
- from ..base import BaseConfig
7
-
8
- # class ScheduleCronTriggerConfig(BaseConfig):
9
- # year: str | int | None = None
10
- # month: str | int | None = None
11
- # week: str | int | None = None
12
- # day: str | int | None = None
13
- # day_of_week: str | int | None = None
14
- # hour: str | int | None = None
15
- # minute: str | int | None = None
16
- # second: str | int | None = None
17
- # start_time: dt.datetime | None = None
18
- # end_time: dt.datetime | None = None
19
- # timezone: str | None = None
20
- # crontab: str | None = None
21
-
22
-
23
- # class ScheduleIntervalTriggerConfig(BaseConfig):
24
- # weeks: int | float | None = None
25
- # days: int | float | None = None
26
- # hours: int | float | None = None
27
- # minutes: int | float | None = None
28
- # seconds: int | float | None = None
29
- # microseconds: int | float | None = None
30
- # start_time: dt.datetime | None = None
31
- # end_time: dt.datetime | None = None
32
-
33
-
34
- # class ScheduleCalendarTriggerConfig(BaseConfig):
35
- # years: int | float | None = None
36
- # months: int | float | None = None
37
- # weeks: int | float | None = None
38
- # days: int | float | None = None
39
- # hour: int | float | None = None
40
- # minute: int | float | None = None
41
- # second: int | float | None = None
42
- # start_date: dt.datetime | None = None
43
- # end_date: dt.datetime | None = None
44
- # timezone: str | None = None
45
-
46
-
47
- # class ScheduleDateTriggerConfig(BaseConfig):
48
- # run_time: dt.datetime | None = None
49
-
50
-
51
- class ScheduleConfig(BaseConfig):
52
- cron: str | dict | None = msgspec.field(default=None)
53
- interval: str | int | dict | None = msgspec.field(default=None)
54
- date: str | None = msgspec.field(default=None)
55
-
56
- def __post_init__(self):
57
- if isinstance(self.date, str):
58
- try:
59
- self.date = dt.datetime.fromisoformat(self.date)
60
- except ValueError:
61
- raise ValueError(
62
- f"Invalid date format: {self.date}. Expected ISO format."
63
- )
64
- if isinstance(self.cron, dict):
65
- self.cron = munchify(self.cron)
66
- if isinstance(self.interval, dict):
67
- self.interval = munchify(self.interval)
68
-
69
-
70
- # class ScheduleConfig(BaseConfig):
71
- # run: ScheduleRunConfig = msgspec.field(default_factory=ScheduleRunConfig)
72
- # trigger: ScheduleTriggerConfig = msgspec.field(
73
- # default_factory=ScheduleTriggerConfig
74
- # )