FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/__init__.py +17 -2
- flowerpower/cfg/__init__.py +201 -149
- flowerpower/cfg/base.py +122 -24
- flowerpower/cfg/pipeline/__init__.py +254 -0
- flowerpower/cfg/pipeline/adapter.py +66 -0
- flowerpower/cfg/pipeline/run.py +40 -11
- flowerpower/cfg/pipeline/schedule.py +69 -79
- flowerpower/cfg/project/__init__.py +149 -0
- flowerpower/cfg/project/adapter.py +57 -0
- flowerpower/cfg/project/job_queue.py +165 -0
- flowerpower/cli/__init__.py +92 -37
- flowerpower/cli/job_queue.py +878 -0
- flowerpower/cli/mqtt.py +32 -1
- flowerpower/cli/pipeline.py +559 -406
- flowerpower/cli/utils.py +29 -18
- flowerpower/flowerpower.py +12 -8
- flowerpower/fs/__init__.py +20 -2
- flowerpower/fs/base.py +350 -26
- flowerpower/fs/ext.py +797 -216
- flowerpower/fs/storage_options.py +1097 -55
- flowerpower/io/base.py +13 -18
- flowerpower/io/loader/__init__.py +28 -0
- flowerpower/io/loader/deltatable.py +7 -10
- flowerpower/io/metadata.py +1 -0
- flowerpower/io/saver/__init__.py +28 -0
- flowerpower/io/saver/deltatable.py +4 -3
- flowerpower/job_queue/__init__.py +252 -0
- flowerpower/job_queue/apscheduler/__init__.py +11 -0
- flowerpower/job_queue/apscheduler/_setup/datastore.py +110 -0
- flowerpower/job_queue/apscheduler/_setup/eventbroker.py +93 -0
- flowerpower/job_queue/apscheduler/manager.py +1063 -0
- flowerpower/job_queue/apscheduler/setup.py +524 -0
- flowerpower/job_queue/apscheduler/trigger.py +169 -0
- flowerpower/job_queue/apscheduler/utils.py +309 -0
- flowerpower/job_queue/base.py +382 -0
- flowerpower/job_queue/rq/__init__.py +10 -0
- flowerpower/job_queue/rq/_trigger.py +37 -0
- flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +226 -0
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +231 -0
- flowerpower/job_queue/rq/manager.py +1449 -0
- flowerpower/job_queue/rq/setup.py +150 -0
- flowerpower/job_queue/rq/utils.py +69 -0
- flowerpower/pipeline/__init__.py +5 -0
- flowerpower/pipeline/base.py +118 -0
- flowerpower/pipeline/io.py +407 -0
- flowerpower/pipeline/job_queue.py +505 -0
- flowerpower/pipeline/manager.py +1586 -0
- flowerpower/pipeline/registry.py +560 -0
- flowerpower/pipeline/runner.py +560 -0
- flowerpower/pipeline/visualizer.py +142 -0
- flowerpower/plugins/mqtt/__init__.py +12 -0
- flowerpower/plugins/mqtt/cfg.py +16 -0
- flowerpower/plugins/mqtt/manager.py +789 -0
- flowerpower/settings.py +110 -0
- flowerpower/utils/logging.py +21 -0
- flowerpower/utils/misc.py +57 -9
- flowerpower/utils/sql.py +122 -24
- flowerpower/utils/templates.py +2 -142
- flowerpower-1.0.0b2.dist-info/METADATA +324 -0
- flowerpower-1.0.0b2.dist-info/RECORD +94 -0
- flowerpower/_web/__init__.py +0 -61
- flowerpower/_web/routes/config.py +0 -103
- flowerpower/_web/routes/pipelines.py +0 -173
- flowerpower/_web/routes/scheduler.py +0 -136
- flowerpower/cfg/pipeline/tracker.py +0 -14
- flowerpower/cfg/project/open_telemetry.py +0 -8
- flowerpower/cfg/project/tracker.py +0 -11
- flowerpower/cfg/project/worker.py +0 -19
- flowerpower/cli/scheduler.py +0 -309
- flowerpower/cli/web.py +0 -44
- flowerpower/event_handler.py +0 -23
- flowerpower/mqtt.py +0 -609
- flowerpower/pipeline.py +0 -2499
- flowerpower/scheduler.py +0 -680
- flowerpower/tui.py +0 -79
- flowerpower/utils/datastore.py +0 -186
- flowerpower/utils/eventbroker.py +0 -127
- flowerpower/utils/executor.py +0 -58
- flowerpower/utils/trigger.py +0 -140
- flowerpower-0.9.13.1.dist-info/METADATA +0 -586
- flowerpower-0.9.13.1.dist-info/RECORD +0 -76
- /flowerpower/{cfg/pipeline/params.py → cli/worker.py} +0 -0
- {flowerpower-0.9.13.1.dist-info → flowerpower-1.0.0b2.dist-info}/WHEEL +0 -0
- {flowerpower-0.9.13.1.dist-info → flowerpower-1.0.0b2.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.9.13.1.dist-info → flowerpower-1.0.0b2.dist-info}/top_level.txt +0 -0
@@ -1,586 +0,0 @@
|
|
1
|
-
Metadata-Version: 2.4
|
2
|
-
Name: FlowerPower
|
3
|
-
Version: 0.9.13.1
|
4
|
-
Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
|
5
|
-
Author-email: "Volker L." <ligno.blades@gmail.com>
|
6
|
-
Project-URL: Homepage, https://github.com/legout/flowerpower
|
7
|
-
Project-URL: Bug Tracker, https://github.com/legout/flowerpower/issues
|
8
|
-
Keywords: hamilton,workflow,pipeline,scheduler,apscheduler,dask,ray
|
9
|
-
Requires-Python: >=3.11
|
10
|
-
Description-Content-Type: text/markdown
|
11
|
-
Requires-Dist: aiobotocore<2.18.0
|
12
|
-
Requires-Dist: aiosqlite>=0.21.0
|
13
|
-
Requires-Dist: dill>=0.3.8
|
14
|
-
Requires-Dist: fsspec>=2024.10.0
|
15
|
-
Requires-Dist: munch>=4.0.0
|
16
|
-
Requires-Dist: orjson>=3.10.15
|
17
|
-
Requires-Dist: pendulum>=3.0.0
|
18
|
-
Requires-Dist: pyarrow<19.0.0
|
19
|
-
Requires-Dist: pydantic>=2.10.2
|
20
|
-
Requires-Dist: python-dotenv>=1.0.1
|
21
|
-
Requires-Dist: pyyaml>=6.0.1
|
22
|
-
Requires-Dist: rich>=13.9.3
|
23
|
-
Requires-Dist: s3fs>=2024.10.0
|
24
|
-
Requires-Dist: sf-hamilton-sdk>=0.5.2
|
25
|
-
Requires-Dist: sf-hamilton[rich,tqdm,visualization]>=1.69.0
|
26
|
-
Requires-Dist: tqdm>=4.67.1
|
27
|
-
Requires-Dist: typer>=0.12.3
|
28
|
-
Provides-Extra: io
|
29
|
-
Requires-Dist: adbc-driver-manager>=1.4.0; extra == "io"
|
30
|
-
Requires-Dist: datafusion>=43.1.0; extra == "io"
|
31
|
-
Requires-Dist: deltalake>=0.24.0; extra == "io"
|
32
|
-
Requires-Dist: duckdb>=1.1.3; extra == "io"
|
33
|
-
Requires-Dist: orjson>=3.10.12; extra == "io"
|
34
|
-
Requires-Dist: pandas>=2.2.3; extra == "io"
|
35
|
-
Requires-Dist: polars>=1.15.0; extra == "io"
|
36
|
-
Requires-Dist: pyarrow>=18.1.0; extra == "io"
|
37
|
-
Requires-Dist: pydala2>=0.9.4.5; extra == "io"
|
38
|
-
Requires-Dist: redis>=5.2.1; extra == "io"
|
39
|
-
Requires-Dist: sherlock>=0.4.1; extra == "io"
|
40
|
-
Provides-Extra: io-legacy
|
41
|
-
Requires-Dist: adbc-driver-manager>=1.4.0; extra == "io-legacy"
|
42
|
-
Requires-Dist: datafusion>=43.1.0; extra == "io-legacy"
|
43
|
-
Requires-Dist: deltalake>=0.24.0; extra == "io-legacy"
|
44
|
-
Requires-Dist: duckdb>=1.1.3; extra == "io-legacy"
|
45
|
-
Requires-Dist: orjson>=3.10.12; extra == "io-legacy"
|
46
|
-
Requires-Dist: pandas>=2.2.3; extra == "io-legacy"
|
47
|
-
Requires-Dist: polars-lts-cpu>=1.15.0; extra == "io-legacy"
|
48
|
-
Requires-Dist: pyarrow>=18.1.0; extra == "io-legacy"
|
49
|
-
Requires-Dist: pydala2>=0.9.4.5; extra == "io-legacy"
|
50
|
-
Requires-Dist: redis>=5.2.1; extra == "io-legacy"
|
51
|
-
Requires-Dist: sherlock>=0.4.1; extra == "io-legacy"
|
52
|
-
Provides-Extra: mongodb
|
53
|
-
Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
|
54
|
-
Provides-Extra: mqtt
|
55
|
-
Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
|
56
|
-
Requires-Dist: orjson>=3.10.11; extra == "mqtt"
|
57
|
-
Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
|
58
|
-
Provides-Extra: opentelemetry
|
59
|
-
Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
|
60
|
-
Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
|
61
|
-
Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
|
62
|
-
Provides-Extra: ray
|
63
|
-
Requires-Dist: ray>=2.34.0; extra == "ray"
|
64
|
-
Provides-Extra: redis
|
65
|
-
Requires-Dist: redis>=5.0.4; extra == "redis"
|
66
|
-
Provides-Extra: scheduler
|
67
|
-
Requires-Dist: aiosqlite>=0.21.0; extra == "scheduler"
|
68
|
-
Requires-Dist: apscheduler>=4.0.0a5; extra == "scheduler"
|
69
|
-
Requires-Dist: asyncpg>=0.29.0; extra == "scheduler"
|
70
|
-
Requires-Dist: greenlet>=3.0.3; extra == "scheduler"
|
71
|
-
Requires-Dist: sqlalchemy>=2.0.30; extra == "scheduler"
|
72
|
-
Provides-Extra: tui
|
73
|
-
Requires-Dist: textual>=0.85.2; extra == "tui"
|
74
|
-
Provides-Extra: ui
|
75
|
-
Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
|
76
|
-
Provides-Extra: webserver
|
77
|
-
Requires-Dist: sanic>=24.6.0; extra == "webserver"
|
78
|
-
Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
|
79
|
-
Requires-Dist: orjson>=3.10.11; extra == "webserver"
|
80
|
-
Provides-Extra: web
|
81
|
-
Requires-Dist: datastar-py>=0.4.2; extra == "web"
|
82
|
-
Requires-Dist: fastapi>=0.115.10; extra == "web"
|
83
|
-
Requires-Dist: htmy>=0.5.0; extra == "web"
|
84
|
-
Requires-Dist: htpy>=25.2.0; extra == "web"
|
85
|
-
Requires-Dist: python-fasthtml>=0.12.4; extra == "web"
|
86
|
-
Requires-Dist: python-multipart>=0.0.20; extra == "web"
|
87
|
-
Requires-Dist: sanic>=24.12.0; extra == "web"
|
88
|
-
Requires-Dist: starlette>=0.45.3; extra == "web"
|
89
|
-
|
90
|
-
<div align="center">
|
91
|
-
<h1>FlowerPower</h1>
|
92
|
-
<h3>Simple Workflow Framework - Hamilton + APScheduler = FlowerPower</h3>
|
93
|
-
<img src="./image.png" alt="FlowerPower Logo" width="600" height="400">
|
94
|
-
</div>
|
95
|
-
|
96
|
-
---
|
97
|
-
|
98
|
-
## 📚 Table of Contents
|
99
|
-
1. [Overview](#overview)
|
100
|
-
2. [Installation](#installation)
|
101
|
-
3. [Getting Started](#getting-started)
|
102
|
-
- [Initialize Project](#initialize-project)
|
103
|
-
- [Add Pipeline](#add-pipeline)
|
104
|
-
- [Setup Pipeline](#setup-pipeline)
|
105
|
-
- [Run Pipeline](#run-pipeline)
|
106
|
-
- [Schedule Pipeline](#schedule-pipeline)
|
107
|
-
- [Start Worker](#start-worker)
|
108
|
-
- [Track Pipeline](#track-pipeline)
|
109
|
-
4. [Development](#development)
|
110
|
-
- [Dev Services](#dev-services)
|
111
|
-
|
112
|
-
---
|
113
|
-
|
114
|
-
## 🔍 Overview
|
115
|
-
|
116
|
-
FlowerPower is a simple workflow framework based on two fantastic Python libraries:
|
117
|
-
|
118
|
-
- **[Hamilton](https://github.com/DAGWorks-Inc/hamilton)**: Creates DAGs from your pipeline functions
|
119
|
-
- **[APScheduler](https://github.com/agronholm/apscheduler)**: Handles pipeline scheduling
|
120
|
-
|
121
|
-
### Key Features
|
122
|
-
|
123
|
-
- 🔄 **Pipeline Workflows**: Create and execute complex DAG-based workflows
|
124
|
-
- ⏰ **Scheduling**: Run pipelines at specific times or intervals
|
125
|
-
- ⚙️ **Parameterization**: Easily configure pipeline parameters
|
126
|
-
- 📊 **Tracking**: Monitor executions with Hamilton UI
|
127
|
-
- 🛠️ **Flexible Configuration**: Simple YAML-based setup
|
128
|
-
- 📡 **Distributed Execution**: Support for distributed environments
|
129
|
-
|
130
|
-
[More details in Hamilton docs](https://hamilton.dagworks.io/en/latest/)
|
131
|
-
|
132
|
-
---
|
133
|
-
|
134
|
-
## 📦 Installation
|
135
|
-
|
136
|
-
```bash
|
137
|
-
# Basic installation
|
138
|
-
pip install flowerpower
|
139
|
-
|
140
|
-
# With scheduling support
|
141
|
-
pip install "flowerpower[scheduler]"
|
142
|
-
|
143
|
-
# Additional components
|
144
|
-
pip install "flowerpower[mqtt]" # MQTT broker
|
145
|
-
pip install "flowerpower[redis]" # Redis broker
|
146
|
-
pip install "flowerpower[mongodb]" # MongoDB store
|
147
|
-
pip install "flowerpower[ray]" # Ray computing
|
148
|
-
pip install "flowerpower[dask]" # Dask computing
|
149
|
-
pip install "flowerpower[ui]" # Hamilton UI
|
150
|
-
pip install "flowerpower[websever]" # Web server
|
151
|
-
```
|
152
|
-
|
153
|
-
---
|
154
|
-
|
155
|
-
## 🚀 Getting Started
|
156
|
-
|
157
|
-
### Initialize Project
|
158
|
-
|
159
|
-
**Option 1: Command Line**
|
160
|
-
```bash
|
161
|
-
flowerpower init new-project
|
162
|
-
cd new-project
|
163
|
-
```
|
164
|
-
|
165
|
-
**Option 2: Python**
|
166
|
-
```python
|
167
|
-
from flowerpower import init
|
168
|
-
init("new-project")
|
169
|
-
```
|
170
|
-
|
171
|
-
This creates basic config files:
|
172
|
-
- `conf/project.yml`
|
173
|
-
|
174
|
-
|
175
|
-
### 📦 Optional: Project Management with UV (Recommended)
|
176
|
-
|
177
|
-
It is recommended to use the project manager `uv` to manage your project dependencies.
|
178
|
-
|
179
|
-
**Installation**
|
180
|
-
```bash
|
181
|
-
pip install uv
|
182
|
-
```
|
183
|
-
> For more installation options, visit: https://docs.astral.sh/uv/getting-started/installation/
|
184
|
-
|
185
|
-
**Project Initialization**
|
186
|
-
```bash
|
187
|
-
uv init --app --no-readme --vcs git
|
188
|
-
```
|
189
|
-
---
|
190
|
-
|
191
|
-
### Pipeline Management
|
192
|
-
|
193
|
-
#### Creating a New Pipeline
|
194
|
-
|
195
|
-
**Option 1: Command Line**
|
196
|
-
```bash
|
197
|
-
flowerpower new my_flow
|
198
|
-
```
|
199
|
-
|
200
|
-
**Option 2: Python**
|
201
|
-
```python
|
202
|
-
# Using PipelineManager
|
203
|
-
from flowerpower.pipeline import PipelineManager
|
204
|
-
pm = PipelineManager()
|
205
|
-
pm.new("my_flow")
|
206
|
-
|
207
|
-
# Or using the new function directly
|
208
|
-
from flowerpower.pipeline import new
|
209
|
-
new("my_flow")
|
210
|
-
```
|
211
|
-
|
212
|
-
This creates the new pipeline and configuration file:
|
213
|
-
- `pipelines/my_flow.py`
|
214
|
-
- `conf/pipelines/my_flow.yml`
|
215
|
-
|
216
|
-
#### Setting Up a Pipeline
|
217
|
-
|
218
|
-
1. **Add Pipeline Functions**
|
219
|
-
Build your pipeline by adding the functions (nodes) to `pipelines/my_flow.py` that build the DAG, following the Hamilton paradigm.
|
220
|
-
|
221
|
-
2. **Parameterize Functions**
|
222
|
-
|
223
|
-
You can parameterize functions in two ways:
|
224
|
-
|
225
|
-
**Method 1: Default Values**
|
226
|
-
```python
|
227
|
-
def add_int_col(
|
228
|
-
df: pd.DataFrame,
|
229
|
-
col_name: str = "foo",
|
230
|
-
values: str = "bar"
|
231
|
-
) -> pd.DataFrame:
|
232
|
-
return df.assign(**{col_name: values})
|
233
|
-
```
|
234
|
-
|
235
|
-
**Method 2: Configuration File**
|
236
|
-
|
237
|
-
In `conf/pipelines/my_flow.yml`:
|
238
|
-
```yaml
|
239
|
-
...
|
240
|
-
func:
|
241
|
-
add_int_col:
|
242
|
-
col_name: foo
|
243
|
-
values: bar
|
244
|
-
...
|
245
|
-
```
|
246
|
-
|
247
|
-
Add the `@parameterize` decorator to the function in your pipeline file:
|
248
|
-
```python
|
249
|
-
@parameterize(**PARAMS.add_int_col)
|
250
|
-
def add_int_col(
|
251
|
-
df: pd.DataFrame,
|
252
|
-
col_name: str,
|
253
|
-
values: int
|
254
|
-
) -> pd.DataFrame:
|
255
|
-
return df.assign(**{col_name: values})
|
256
|
-
```
|
257
|
-
|
258
|
-
---
|
259
|
-
|
260
|
-
### Running Pipelines
|
261
|
-
|
262
|
-
#### Configuration
|
263
|
-
|
264
|
-
You can configure the pipeline parameters `inputs`, and `final_vars`, and other parameters in the pipeline
|
265
|
-
configuration file `conf/pipelines/my_flow.yml` or directly in the pipeline execution function.
|
266
|
-
|
267
|
-
#### Using the Pipeline Configuration
|
268
|
-
```yaml
|
269
|
-
...
|
270
|
-
run:
|
271
|
-
inputs:
|
272
|
-
data_path: path/to/data.csv
|
273
|
-
fs_protocol: local
|
274
|
-
final_vars: [add_int_col, final_df]
|
275
|
-
# optional parameters
|
276
|
-
with_tracker: false
|
277
|
-
executor: threadpool # or processpool, ray, dask
|
278
|
-
...
|
279
|
-
```
|
280
|
-
|
281
|
-
#### Execution Methods
|
282
|
-
There are three ways to execute a pipeline:
|
283
|
-
|
284
|
-
1. **Direct Execution**
|
285
|
-
- Runs in current process
|
286
|
-
- No data store required
|
287
|
-
|
288
|
-
2. **Job Execution**
|
289
|
-
- Runs as APScheduler job
|
290
|
-
- Returns job results
|
291
|
-
- Requires data store and event broker
|
292
|
-
|
293
|
-
3. **Async Job Addition**
|
294
|
-
- Adds to APScheduler
|
295
|
-
- Returns job ID
|
296
|
-
- Results retrievable from data store
|
297
|
-
|
298
|
-
|
299
|
-
#### Command Line Usage
|
300
|
-
```bash
|
301
|
-
# Note: add --inputs and --final-vars and other optional parameters if not specified in the config file
|
302
|
-
# Direct execution
|
303
|
-
flowerpower run my_flow
|
304
|
-
# Job execution
|
305
|
-
flowerpower run-job my_flow
|
306
|
-
|
307
|
-
# Add as scheduled job
|
308
|
-
flowerpower add-job my_flow
|
309
|
-
```
|
310
|
-
|
311
|
-
You can also use the `--inputs` and `--final-vars` flags to override the configuration file parameters or if they are not specified in the configuration file.
|
312
|
-
|
313
|
-
```bash
|
314
|
-
flowerpower run my_flow \
|
315
|
-
--inputs data_path=path/to/data.csv,fs_protocol=local \
|
316
|
-
--final-vars final_df \
|
317
|
-
--executor threadpool
|
318
|
-
--without-tracker
|
319
|
-
```
|
320
|
-
|
321
|
-
#### Python Usage
|
322
|
-
```python
|
323
|
-
from flowerpower.pipeline import Pipeline, run, run_job, add_job
|
324
|
-
|
325
|
-
# Using Pipeline class
|
326
|
-
p = Pipeline("my_flow")
|
327
|
-
# Note: add inputs, final_vars, and other optional arguments if not specified in the config file
|
328
|
-
result = p.run()
|
329
|
-
result = p.run_job()
|
330
|
-
job_id = p.add_job()
|
331
|
-
|
332
|
-
# Using functions
|
333
|
-
result = run("my_flow")
|
334
|
-
result = run_job("my_flow")
|
335
|
-
job_id = add_job("my_flow")
|
336
|
-
```
|
337
|
-
|
338
|
-
You can also use the `inputs` and `final-vars` arguments to override the configuration file parameters or if they are not specified in the configuration file.
|
339
|
-
|
340
|
-
```python
|
341
|
-
result = run(
|
342
|
-
"my_flow",
|
343
|
-
inputs={
|
344
|
-
"data_path": "path/to/data.csv",
|
345
|
-
"fs_protocol": "local"
|
346
|
-
},
|
347
|
-
final_vars=["final_df"],
|
348
|
-
executor="threadpool",
|
349
|
-
with_tracker=False
|
350
|
-
)
|
351
|
-
```
|
352
|
-
|
353
|
-
---
|
354
|
-
## ⏰ Scheduling Pipelines
|
355
|
-
|
356
|
-
### Setting Up Schedules
|
357
|
-
|
358
|
-
#### Command Line Options
|
359
|
-
|
360
|
-
```bash
|
361
|
-
# Run every 30 seconds
|
362
|
-
flowerpower schedule my_flow \
|
363
|
-
--type interval \
|
364
|
-
--interval-params seconds=30
|
365
|
-
|
366
|
-
# Run at specific date/time
|
367
|
-
flowerpower schedule my_flow \
|
368
|
-
--type date \
|
369
|
-
--date-params year=2022,month=1,day=1,hour=0,minute=0,second=0
|
370
|
-
|
371
|
-
# Run with cron parameters
|
372
|
-
flowerpower schedule my_flow \
|
373
|
-
--type cron \
|
374
|
-
--cron-params second=0,minute=0,hour=0,day=1,month=1,day_of_week=0
|
375
|
-
|
376
|
-
# Run with crontab expression
|
377
|
-
flowerpower schedule my_flow \
|
378
|
-
--type cron \
|
379
|
-
--crontab "0 0 1 1 0"
|
380
|
-
```
|
381
|
-
|
382
|
-
#### Python Usage
|
383
|
-
```python
|
384
|
-
from flowerpower.scheduler import schedule, Pipeline
|
385
|
-
|
386
|
-
# Using Pipeline class
|
387
|
-
p = Pipeline("my_flow")
|
388
|
-
p.schedule("interval", seconds=30)
|
389
|
-
|
390
|
-
# Using schedule function
|
391
|
-
schedule("my_flow", "interval", seconds=30)
|
392
|
-
```
|
393
|
-
|
394
|
-
---
|
395
|
-
|
396
|
-
## 👷 Worker Management
|
397
|
-
|
398
|
-
### Starting a Worker
|
399
|
-
|
400
|
-
**Command Line**
|
401
|
-
```bash
|
402
|
-
flowerpower start-worker
|
403
|
-
```
|
404
|
-
|
405
|
-
**Python**
|
406
|
-
```python
|
407
|
-
# Using the SchedulerManager class
|
408
|
-
from flowerpower.scheduler import SchedulerManager
|
409
|
-
sm = SchedulerManager()
|
410
|
-
sm.start_worker()
|
411
|
-
|
412
|
-
# Using the start_worker function
|
413
|
-
from flowerpower.scheduler import start_worker
|
414
|
-
start_worker()
|
415
|
-
```
|
416
|
-
|
417
|
-
### Worker Configuration
|
418
|
-
|
419
|
-
Configure your worker in `conf/project.yml`:
|
420
|
-
|
421
|
-
```yaml
|
422
|
-
# PostgreSQL Configuration
|
423
|
-
data_store:
|
424
|
-
type: postgres
|
425
|
-
uri: postgresql+asyncpq://user:password@localhost:5432/flowerpower
|
426
|
-
|
427
|
-
# Redis Event Broker
|
428
|
-
event_broker:
|
429
|
-
type: redis
|
430
|
-
uri: redis://localhost:6379
|
431
|
-
# Alternative configuration:
|
432
|
-
# host: localhost
|
433
|
-
# port: 6379
|
434
|
-
```
|
435
|
-
|
436
|
-
#### Alternative Data Store Options
|
437
|
-
|
438
|
-
**SQLite**
|
439
|
-
```yaml
|
440
|
-
data_store:
|
441
|
-
type: sqlite
|
442
|
-
uri: sqlite+aiosqlite:///flowerpower.db
|
443
|
-
```
|
444
|
-
|
445
|
-
**MySQL**
|
446
|
-
```yaml
|
447
|
-
data_store:
|
448
|
-
type: mysql
|
449
|
-
uri: mysql+aiomysql://user:password@localhost:3306/flowerpower
|
450
|
-
```
|
451
|
-
|
452
|
-
**MongoDB**
|
453
|
-
```yaml
|
454
|
-
data_store:
|
455
|
-
type: mongodb
|
456
|
-
uri: mongodb://localhost:27017/flowerpower
|
457
|
-
```
|
458
|
-
|
459
|
-
**In-Memory**
|
460
|
-
```yaml
|
461
|
-
data_store:
|
462
|
-
type: memory
|
463
|
-
```
|
464
|
-
|
465
|
-
#### Alternative Event Broker Options
|
466
|
-
|
467
|
-
**MQTT**
|
468
|
-
```yaml
|
469
|
-
event_broker:
|
470
|
-
type: mqtt
|
471
|
-
host: localhost
|
472
|
-
port: 1883
|
473
|
-
username: user # optional if required
|
474
|
-
password: supersecret # optional if required
|
475
|
-
```
|
476
|
-
**Redis**
|
477
|
-
```yaml
|
478
|
-
event_broker:
|
479
|
-
type: redis
|
480
|
-
uri: redis://localhost:6379
|
481
|
-
# Alternative configuration:
|
482
|
-
# host: localhost
|
483
|
-
# port: 6379
|
484
|
-
```
|
485
|
-
|
486
|
-
**In-Memory**
|
487
|
-
```yaml
|
488
|
-
event_broker:
|
489
|
-
type: memory
|
490
|
-
```
|
491
|
-
|
492
|
-
---
|
493
|
-
|
494
|
-
## 📊 Pipeline Tracking
|
495
|
-
|
496
|
-
### Hamilton UI Setup
|
497
|
-
|
498
|
-
#### Local Installation
|
499
|
-
```bash
|
500
|
-
# Install UI package
|
501
|
-
pip install "flowerpower[ui]"
|
502
|
-
|
503
|
-
# Start UI server
|
504
|
-
flowerpower hamilton-ui
|
505
|
-
```
|
506
|
-
> Access the UI at: http://localhost:8241
|
507
|
-
|
508
|
-
#### Docker Installation
|
509
|
-
```bash
|
510
|
-
# Clone Hamilton repository
|
511
|
-
git clone https://github.com/dagworks-inc/hamilton
|
512
|
-
cd hamilton/ui
|
513
|
-
|
514
|
-
# Start UI server
|
515
|
-
./run.sh
|
516
|
-
```
|
517
|
-
> Access the UI at: http://localhost:8242
|
518
|
-
|
519
|
-
### Tracker Configuration
|
520
|
-
|
521
|
-
Configure tracking in `conf/project.yml`:
|
522
|
-
|
523
|
-
```yaml
|
524
|
-
username: my_email@example.com
|
525
|
-
api_url: http://localhost:8241
|
526
|
-
ui_url: http://localhost:8242
|
527
|
-
api_key: optional_key
|
528
|
-
```
|
529
|
-
|
530
|
-
And specify the `tracker` parameter in the pipeline configuration `conf/pipelines/my_flow.yml:
|
531
|
-
|
532
|
-
```yaml
|
533
|
-
...
|
534
|
-
tracker:
|
535
|
-
project_id: 1
|
536
|
-
tags:
|
537
|
-
environment: dev
|
538
|
-
version: 1.0
|
539
|
-
dag_name: my_flow_123
|
540
|
-
...
|
541
|
-
```
|
542
|
-
|
543
|
-
---
|
544
|
-
|
545
|
-
## 🛠️ Development Services
|
546
|
-
|
547
|
-
### Local Development Setup
|
548
|
-
|
549
|
-
Download the docker-compose configuration:
|
550
|
-
```bash
|
551
|
-
curl -O https://raw.githubusercontent.com/legout/flowerpower/main/docker/docker-compose.yml
|
552
|
-
```
|
553
|
-
|
554
|
-
### Starting Services
|
555
|
-
|
556
|
-
```bash
|
557
|
-
# MQTT Broker (EMQX)
|
558
|
-
docker-compose up mqtt -d
|
559
|
-
|
560
|
-
# Redis
|
561
|
-
docker-compose up redis -d
|
562
|
-
|
563
|
-
# MongoDB
|
564
|
-
docker-compose up mongodb -d
|
565
|
-
|
566
|
-
# PostgreSQL
|
567
|
-
docker-compose up postgres -d
|
568
|
-
```
|
569
|
-
|
570
|
-
---
|
571
|
-
|
572
|
-
## 📝 License
|
573
|
-
|
574
|
-
[MIT License](LICENSE)
|
575
|
-
|
576
|
-
---
|
577
|
-
|
578
|
-
## 🤝 Contributing
|
579
|
-
|
580
|
-
Contributions are welcome! Please feel free to submit a Pull Request.
|
581
|
-
|
582
|
-
---
|
583
|
-
|
584
|
-
## 📫 Support
|
585
|
-
|
586
|
-
For support, please open an issue in the GitHub repository.
|
@@ -1,76 +0,0 @@
|
|
1
|
-
flowerpower/__init__.py,sha256=eD08iJU0y6w8BMCvJou9HMbvH8I62ed9lUs30mh2wTA,134
|
2
|
-
flowerpower/event_handler.py,sha256=JH1o_GUTlYemrwTgOD0291_SUynLeeReSAOk_oLWLYg,534
|
3
|
-
flowerpower/flowerpower.py,sha256=JPy7h6jYexrenBGBekkAWAKJhsxTG76jW5KgLGRgAPs,3272
|
4
|
-
flowerpower/mqtt.py,sha256=Ud2d4vnUijq-oVcGrK-zshtjqypgTnUtMjDZWOovlkQ,21602
|
5
|
-
flowerpower/pipeline.py,sha256=eaznQR5rUyejtRqvY3Gw_BCFWe1MXFuyg3LNzbuf-uk,92417
|
6
|
-
flowerpower/scheduler.py,sha256=ylYnrxec7i9nuCK0QvJlukz6GcnIxNPoiHi9jmelrPQ,20982
|
7
|
-
flowerpower/tui.py,sha256=2ff_yOfOQnkFctIKKHV1RDW4M5AsX5FUAT0L6ygkvVU,2294
|
8
|
-
flowerpower/_web/__init__.py,sha256=pTgLgNInGZzLEO-TyO-jHI1VrSoUKFVFroo9XEDYUUI,1981
|
9
|
-
flowerpower/_web/routes/config.py,sha256=E9mPL9O1CdPqBpyl-zwYPhmAOZzOZgHeiq6WaDZ9oHo,3199
|
10
|
-
flowerpower/_web/routes/pipelines.py,sha256=wVVyVsnxOPL5hM9XhgCHEmTDhuFTt_I1-mWVlQhD-pw,5871
|
11
|
-
flowerpower/_web/routes/scheduler.py,sha256=6raCvsiUByj1l775qXDecpoUcmeJRvF26xZySr6-Aj4,4898
|
12
|
-
flowerpower/cfg/__init__.py,sha256=YAn0DF2koQ47gACBTTo4PeDxXah20ndAVosutzHsZYk,7457
|
13
|
-
flowerpower/cfg/base.py,sha256=JUhxWzC3XesnhEpBwHbyqOG6_dra-pEbsJsBbGX59dM,1291
|
14
|
-
flowerpower/cfg/pipeline/params.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
15
|
-
flowerpower/cfg/pipeline/run.py,sha256=Y3jU9dYYh3R5Us-xkbj2ol3oizlGvxTc_H8N88bhYdU,565
|
16
|
-
flowerpower/cfg/pipeline/schedule.py,sha256=JAREhzvkYqPH6Op-FY8UuqTmGVqrl3yt8aByXscCkgk,2753
|
17
|
-
flowerpower/cfg/pipeline/tracker.py,sha256=cew9F4UIEgkR9u6p12xRBW5aGN22hDRExKbnMEbA0EM,370
|
18
|
-
flowerpower/cfg/project/open_telemetry.py,sha256=3tJQ92qgZHsS2tYQ3ZlHtpVREz5DA4NzhBVfjzyvxTU,185
|
19
|
-
flowerpower/cfg/project/tracker.py,sha256=KcvSaJtLLOibQtgpIuUm-pe0ZgdV9HBfLNzSZlKKYio,303
|
20
|
-
flowerpower/cfg/project/worker.py,sha256=5M2bUousYXLpLJGVnUyFrDB-vJp7b8VBhZhzISbszac,665
|
21
|
-
flowerpower/cli/__init__.py,sha256=MMXmU7xKao4_UXJNIH8OzcueEWaqLjrsoXyhStcxOn8,2532
|
22
|
-
flowerpower/cli/cfg.py,sha256=B9yV6g9bTGRXOR1PewQTxf_XfcYbcsE3GeLAxlbE2qc,1530
|
23
|
-
flowerpower/cli/mqtt.py,sha256=HirYiqMXTbxHUipAMTMXvshSXNBh2WWyacxVVMM7c7Y,4390
|
24
|
-
flowerpower/cli/pipeline.py,sha256=wheTMsSy4Ebj6ri6Nvgb6P3MfVYEMC2H1iWWczJU7NI,21133
|
25
|
-
flowerpower/cli/scheduler.py,sha256=1i-Qm5GF5Ra4hmXlKLXPveELo7frObXbhDzQsA9Iulc,8681
|
26
|
-
flowerpower/cli/utils.py,sha256=psVQy_9zjmHYIN4bTsebD65ERvGRLtPO5w5YulpJ5To,5025
|
27
|
-
flowerpower/cli/web.py,sha256=ed8ECYxmQz2ANLwzBELTshuWv9RDpcdbPer6GtV95UU,956
|
28
|
-
flowerpower/fs/__init__.py,sha256=0vmjffXYEE5PnQtSOO07EOpMZgEG2QDmFlDn7G-ZTUg,216
|
29
|
-
flowerpower/fs/base.py,sha256=vI6XFkzI_AeJ6_JtBnaMKjZE0-vjqHYvpDo1w4yCxl8,10523
|
30
|
-
flowerpower/fs/ext.py,sha256=ipiwQX4fpdr_o_ywkim8d2CKAYnDIsRv8dH75Xzhph8,45186
|
31
|
-
flowerpower/fs/storage_options.py,sha256=B-rCbwPookMqvzL2H4YPR3aOnpgaHusI2Zd-6d_z97Q,10667
|
32
|
-
flowerpower/io/base.py,sha256=lDTwer9KgKcdOlr2hvswEXdBPgLgNC3iUEM6QsbG56U,75863
|
33
|
-
flowerpower/io/metadata.py,sha256=_Q1gf8F8RMaSFEvMOozLtR0PmCO-4rJCH5Da9WLbGF8,7269
|
34
|
-
flowerpower/io/loader/_duckdb.py,sha256=WB9CKI7H4eumcV3VKqiEXiiLSDkHmjjyzrUytQWf6Qc,10311
|
35
|
-
flowerpower/io/loader/csv.py,sha256=2geJIMljENZGCvjUPg1CCBgnb4LN8TzUGXcPi9HWotQ,890
|
36
|
-
flowerpower/io/loader/deltatable.py,sha256=6eNjO5tfGHoHpJg9hHFMxeWT4VClSrldgr1aWJ0ceRU,6564
|
37
|
-
flowerpower/io/loader/duckdb.py,sha256=lRhPyRaOhDEFund3wm7Fd7MpatluoDakhwipltfANrY,486
|
38
|
-
flowerpower/io/loader/json.py,sha256=wdhZhb4xO538c5SqDzNHB7U_Ao0E6MlLIHfSpGT-o2Y,802
|
39
|
-
flowerpower/io/loader/mqtt.py,sha256=2eAnavk9KcapQcZqoKOi81FvUFGbhkeJWzB6y01-3Pc,5147
|
40
|
-
flowerpower/io/loader/mssql.py,sha256=0nwa71KvdRLEUxDFnlFH7hIqWDRmZoVhfqt4ua8Xcqs,825
|
41
|
-
flowerpower/io/loader/mysql.py,sha256=6zuOvBntZca6bQ25ReUTeLp8J9SnwuYcoQL5WRu3B_0,825
|
42
|
-
flowerpower/io/loader/oracle.py,sha256=6YlIsim_XlDbBZ6Uje8FuxovxXXn6oPZ4Vmsp37KLrw,841
|
43
|
-
flowerpower/io/loader/parquet.py,sha256=8bXmZuzVCA2eFsnrp-vupdyLihvzSpb2LYAIXuX0aXo,850
|
44
|
-
flowerpower/io/loader/postgres.py,sha256=rb9HhbWG1mPxVU7ZhtpR-S3BMQ0DgE9Mj27_Me6Yeqc,853
|
45
|
-
flowerpower/io/loader/pydala.py,sha256=17plL-oVi0hgrYGAfCW8vxM9d4zTjSKLOF9LJSVIbSo,438
|
46
|
-
flowerpower/io/loader/sqlite.py,sha256=5SFeBfhgibjYemUhaPrMfg1XkFCAaIqu-B5gIcQjTN8,627
|
47
|
-
flowerpower/io/saver/_duckdb.py,sha256=GMQohIkpNVUcwT7tUv4MqtaPhaIPCwGl1kkdtp9DocE,9274
|
48
|
-
flowerpower/io/saver/csv.py,sha256=q6ndV12bobRWNJBibLaDbpvBiOsTcARmYMn8B1uegZk,799
|
49
|
-
flowerpower/io/saver/deltatable.py,sha256=xPzLpdXz26x94_aiWy64awVB5Pv62iiGVsGt42lVc68,6947
|
50
|
-
flowerpower/io/saver/duckdb.py,sha256=XNHgn8Qc2m7xovKZWOYEjNTf6CTQ4me5UToo1-kpzi0,433
|
51
|
-
flowerpower/io/saver/json.py,sha256=Qx17Yghia7RmXYoee7MXQjc6uKdyHxqI-5JhCdkBKuA,789
|
52
|
-
flowerpower/io/saver/mqtt.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
53
|
-
flowerpower/io/saver/mssql.py,sha256=4oDblnkm99gOHMXkEeMTd91hqPNHjy7eVUYxZWMGN0A,758
|
54
|
-
flowerpower/io/saver/mysql.py,sha256=l88MfCoMFdPtn5XlR4IlnLXpeYlQz4ljUJoljoJpw6I,758
|
55
|
-
flowerpower/io/saver/oracle.py,sha256=8mw45vKDL7HeFcIhL95yvg2ODAwUBLPZIfnb5sWpjIk,774
|
56
|
-
flowerpower/io/saver/parquet.py,sha256=IkYODtvcuwECjNlMdEogd0CkC3KEn00tgxQfvbzYXac,815
|
57
|
-
flowerpower/io/saver/postgres.py,sha256=lDSGnN3Y6xDuqn5NjnSxAWk84QmOhwrYCTQhXMlABLo,784
|
58
|
-
flowerpower/io/saver/pydala.py,sha256=0hcyFNWOVbdNlC_F0KbeVeTbgOZX2HbTvY1i6UU3kUw,478
|
59
|
-
flowerpower/io/saver/sqlite.py,sha256=q7v3fWtiaGslfKbgHpdaV8p7WtXc2MMyx3yMxmW7G5s,625
|
60
|
-
flowerpower/utils/datastore.py,sha256=LiAJlDLgHdKGzRdUwxvucjjTj8tF9gJR_dzj2qmLTEE,5468
|
61
|
-
flowerpower/utils/eventbroker.py,sha256=lx19N5qqlPZX6JGG70MmM0Mu3V8jTCb30D0-CUynfRY,3819
|
62
|
-
flowerpower/utils/executor.py,sha256=drX8Phew5K7d_Y3tkt30DaQcXHlq8um6dYLSYJrIIL0,2057
|
63
|
-
flowerpower/utils/misc.py,sha256=a7Wnm0k3dOOSbX9zmZU1TiCF6a75x1xSraxcm2sIqf8,14404
|
64
|
-
flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2903
|
65
|
-
flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
|
66
|
-
flowerpower/utils/polars.py,sha256=-b4SrVVtsS4g03rD1vdsd6vu7zQogua1p57fYGhe6vo,17739
|
67
|
-
flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
|
68
|
-
flowerpower/utils/sql.py,sha256=lGcZEicPRIgwEzvBJmGau8PWzsVXUj5p0ozcVhCDagE,8709
|
69
|
-
flowerpower/utils/templates.py,sha256=JlcAcXmak6XmfRdynb7dJg2mUvjyPjE9NsaOXpDMjQQ,4762
|
70
|
-
flowerpower/utils/trigger.py,sha256=NU1hC9Azu2xH8pQ-nVt4DrqVDIs9bqGM3nXHvdwKCEM,3315
|
71
|
-
flowerpower/web/app.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
72
|
-
flowerpower-0.9.13.1.dist-info/METADATA,sha256=8Npkq8UI1n_DL45-4Ozn4ODmjBqeARDYqXnpUUpk5ow,13943
|
73
|
-
flowerpower-0.9.13.1.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
|
74
|
-
flowerpower-0.9.13.1.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
|
75
|
-
flowerpower-0.9.13.1.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
|
76
|
-
flowerpower-0.9.13.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|