FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/__init__.py +17 -2
- flowerpower/cfg/__init__.py +201 -149
- flowerpower/cfg/base.py +122 -24
- flowerpower/cfg/pipeline/__init__.py +254 -0
- flowerpower/cfg/pipeline/adapter.py +66 -0
- flowerpower/cfg/pipeline/run.py +40 -11
- flowerpower/cfg/pipeline/schedule.py +69 -79
- flowerpower/cfg/project/__init__.py +149 -0
- flowerpower/cfg/project/adapter.py +57 -0
- flowerpower/cfg/project/job_queue.py +165 -0
- flowerpower/cli/__init__.py +92 -37
- flowerpower/cli/job_queue.py +878 -0
- flowerpower/cli/mqtt.py +32 -1
- flowerpower/cli/pipeline.py +559 -406
- flowerpower/cli/utils.py +29 -18
- flowerpower/flowerpower.py +12 -8
- flowerpower/fs/__init__.py +20 -2
- flowerpower/fs/base.py +350 -26
- flowerpower/fs/ext.py +797 -216
- flowerpower/fs/storage_options.py +1097 -55
- flowerpower/io/base.py +13 -18
- flowerpower/io/loader/__init__.py +28 -0
- flowerpower/io/loader/deltatable.py +7 -10
- flowerpower/io/metadata.py +1 -0
- flowerpower/io/saver/__init__.py +28 -0
- flowerpower/io/saver/deltatable.py +4 -3
- flowerpower/job_queue/__init__.py +252 -0
- flowerpower/job_queue/apscheduler/__init__.py +11 -0
- flowerpower/job_queue/apscheduler/_setup/datastore.py +110 -0
- flowerpower/job_queue/apscheduler/_setup/eventbroker.py +93 -0
- flowerpower/job_queue/apscheduler/manager.py +1063 -0
- flowerpower/job_queue/apscheduler/setup.py +524 -0
- flowerpower/job_queue/apscheduler/trigger.py +169 -0
- flowerpower/job_queue/apscheduler/utils.py +309 -0
- flowerpower/job_queue/base.py +382 -0
- flowerpower/job_queue/rq/__init__.py +10 -0
- flowerpower/job_queue/rq/_trigger.py +37 -0
- flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +226 -0
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +231 -0
- flowerpower/job_queue/rq/manager.py +1449 -0
- flowerpower/job_queue/rq/setup.py +150 -0
- flowerpower/job_queue/rq/utils.py +69 -0
- flowerpower/pipeline/__init__.py +5 -0
- flowerpower/pipeline/base.py +118 -0
- flowerpower/pipeline/io.py +407 -0
- flowerpower/pipeline/job_queue.py +505 -0
- flowerpower/pipeline/manager.py +1586 -0
- flowerpower/pipeline/registry.py +560 -0
- flowerpower/pipeline/runner.py +560 -0
- flowerpower/pipeline/visualizer.py +142 -0
- flowerpower/plugins/mqtt/__init__.py +12 -0
- flowerpower/plugins/mqtt/cfg.py +16 -0
- flowerpower/plugins/mqtt/manager.py +789 -0
- flowerpower/settings.py +110 -0
- flowerpower/utils/logging.py +21 -0
- flowerpower/utils/misc.py +57 -9
- flowerpower/utils/sql.py +122 -24
- flowerpower/utils/templates.py +2 -142
- flowerpower-1.0.0b2.dist-info/METADATA +324 -0
- flowerpower-1.0.0b2.dist-info/RECORD +94 -0
- flowerpower/_web/__init__.py +0 -61
- flowerpower/_web/routes/config.py +0 -103
- flowerpower/_web/routes/pipelines.py +0 -173
- flowerpower/_web/routes/scheduler.py +0 -136
- flowerpower/cfg/pipeline/tracker.py +0 -14
- flowerpower/cfg/project/open_telemetry.py +0 -8
- flowerpower/cfg/project/tracker.py +0 -11
- flowerpower/cfg/project/worker.py +0 -19
- flowerpower/cli/scheduler.py +0 -309
- flowerpower/cli/web.py +0 -44
- flowerpower/event_handler.py +0 -23
- flowerpower/mqtt.py +0 -609
- flowerpower/pipeline.py +0 -2499
- flowerpower/scheduler.py +0 -680
- flowerpower/tui.py +0 -79
- flowerpower/utils/datastore.py +0 -186
- flowerpower/utils/eventbroker.py +0 -127
- flowerpower/utils/executor.py +0 -58
- flowerpower/utils/trigger.py +0 -140
- flowerpower-0.9.13.1.dist-info/METADATA +0 -586
- flowerpower-0.9.13.1.dist-info/RECORD +0 -76
- /flowerpower/{cfg/pipeline/params.py → cli/worker.py} +0 -0
- {flowerpower-0.9.13.1.dist-info → flowerpower-1.0.0b2.dist-info}/WHEEL +0 -0
- {flowerpower-0.9.13.1.dist-info → flowerpower-1.0.0b2.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.9.13.1.dist-info → flowerpower-1.0.0b2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,324 @@
|
|
1
|
+
Metadata-Version: 2.4
|
2
|
+
Name: FlowerPower
|
3
|
+
Version: 1.0.0b2
|
4
|
+
Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
|
5
|
+
Author-email: "Volker L." <ligno.blades@gmail.com>
|
6
|
+
Project-URL: Homepage, https://github.com/legout/flowerpower
|
7
|
+
Project-URL: Bug Tracker, https://github.com/legout/flowerpower/issues
|
8
|
+
Keywords: hamilton,workflow,pipeline,scheduler,apscheduler,dask,ray
|
9
|
+
Requires-Python: >=3.11
|
10
|
+
Description-Content-Type: text/markdown
|
11
|
+
Requires-Dist: aiobotocore<2.18.0
|
12
|
+
Requires-Dist: aiosqlite>=0.21.0
|
13
|
+
Requires-Dist: dill>=0.3.8
|
14
|
+
Requires-Dist: duration-parser>=1.0.1
|
15
|
+
Requires-Dist: fsspec>=2024.10.0
|
16
|
+
Requires-Dist: humanize>=4.12.2
|
17
|
+
Requires-Dist: msgspec>=0.19.0
|
18
|
+
Requires-Dist: munch>=4.0.0
|
19
|
+
Requires-Dist: orjson>=3.10.15
|
20
|
+
Requires-Dist: pyarrow<19.0.0
|
21
|
+
Requires-Dist: pydantic>=2.10.2
|
22
|
+
Requires-Dist: python-dotenv>=1.0.1
|
23
|
+
Requires-Dist: pyyaml>=6.0.1
|
24
|
+
Requires-Dist: rich>=13.9.3
|
25
|
+
Requires-Dist: s3fs>=2024.10.0
|
26
|
+
Requires-Dist: sf-hamilton-sdk>=0.5.2
|
27
|
+
Requires-Dist: sf-hamilton[rich,tqdm,visualization]>=1.69.0
|
28
|
+
Requires-Dist: typer>=0.12.3
|
29
|
+
Provides-Extra: apscheduler
|
30
|
+
Requires-Dist: aiosqlite>=0.21.0; extra == "apscheduler"
|
31
|
+
Requires-Dist: apscheduler==4.0.0a5; extra == "apscheduler"
|
32
|
+
Requires-Dist: asyncpg>=0.29.0; extra == "apscheduler"
|
33
|
+
Requires-Dist: greenlet>=3.0.3; extra == "apscheduler"
|
34
|
+
Requires-Dist: sqlalchemy>=2.0.30; extra == "apscheduler"
|
35
|
+
Requires-Dist: cron-descriptor>=1.4.5; extra == "apscheduler"
|
36
|
+
Provides-Extra: io
|
37
|
+
Requires-Dist: adbc-driver-manager>=1.4.0; extra == "io"
|
38
|
+
Requires-Dist: datafusion>=43.1.0; extra == "io"
|
39
|
+
Requires-Dist: deltalake>=0.24.0; extra == "io"
|
40
|
+
Requires-Dist: duckdb>=1.1.3; extra == "io"
|
41
|
+
Requires-Dist: orjson>=3.10.12; extra == "io"
|
42
|
+
Requires-Dist: pandas>=2.2.3; extra == "io"
|
43
|
+
Requires-Dist: polars>=1.15.0; extra == "io"
|
44
|
+
Requires-Dist: pyarrow>=18.1.0; extra == "io"
|
45
|
+
Requires-Dist: pydala2>=0.9.4.5; extra == "io"
|
46
|
+
Requires-Dist: redis>=5.2.1; extra == "io"
|
47
|
+
Requires-Dist: sherlock>=0.4.1; extra == "io"
|
48
|
+
Provides-Extra: io-legacy
|
49
|
+
Requires-Dist: adbc-driver-manager>=1.4.0; extra == "io-legacy"
|
50
|
+
Requires-Dist: datafusion>=43.1.0; extra == "io-legacy"
|
51
|
+
Requires-Dist: deltalake>=0.24.0; extra == "io-legacy"
|
52
|
+
Requires-Dist: duckdb>=1.1.3; extra == "io-legacy"
|
53
|
+
Requires-Dist: orjson>=3.10.12; extra == "io-legacy"
|
54
|
+
Requires-Dist: pandas>=2.2.3; extra == "io-legacy"
|
55
|
+
Requires-Dist: polars-lts-cpu>=1.15.0; extra == "io-legacy"
|
56
|
+
Requires-Dist: pyarrow>=18.1.0; extra == "io-legacy"
|
57
|
+
Requires-Dist: pydala2>=0.9.4.5; extra == "io-legacy"
|
58
|
+
Requires-Dist: redis>=5.2.1; extra == "io-legacy"
|
59
|
+
Requires-Dist: sherlock>=0.4.1; extra == "io-legacy"
|
60
|
+
Provides-Extra: mongodb
|
61
|
+
Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
|
62
|
+
Provides-Extra: mqtt
|
63
|
+
Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
|
64
|
+
Requires-Dist: orjson>=3.10.11; extra == "mqtt"
|
65
|
+
Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
|
66
|
+
Provides-Extra: opentelemetry
|
67
|
+
Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
|
68
|
+
Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
|
69
|
+
Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
|
70
|
+
Provides-Extra: ray
|
71
|
+
Requires-Dist: ray>=2.34.0; extra == "ray"
|
72
|
+
Provides-Extra: redis
|
73
|
+
Requires-Dist: redis>=5.0.4; extra == "redis"
|
74
|
+
Provides-Extra: rq
|
75
|
+
Requires-Dist: rq>=2.3.1; extra == "rq"
|
76
|
+
Requires-Dist: rq-scheduler>=0.14.0; extra == "rq"
|
77
|
+
Requires-Dist: cron-descriptor>=1.4.5; extra == "rq"
|
78
|
+
Provides-Extra: tui
|
79
|
+
Requires-Dist: textual>=0.85.2; extra == "tui"
|
80
|
+
Provides-Extra: ui
|
81
|
+
Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
|
82
|
+
Provides-Extra: webserver
|
83
|
+
Requires-Dist: sanic>=24.6.0; extra == "webserver"
|
84
|
+
Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
|
85
|
+
Requires-Dist: orjson>=3.10.11; extra == "webserver"
|
86
|
+
Provides-Extra: openlineage
|
87
|
+
Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
|
88
|
+
|
89
|
+
<div align="center">
|
90
|
+
<h1>FlowerPower 🌸</h1>
|
91
|
+
<h3>Simple Workflow Framework - Hamilton + APScheduler = FlowerPower</h3>
|
92
|
+
<img src="./image.png" alt="FlowerPower Logo" width="400" height="300">
|
93
|
+
</div>
|
94
|
+
|
95
|
+
A powerful and flexible data pipeline framework that simplifies data processing workflows, job scheduling, and event-driven architectures. FlowerPower combines modern data processing capabilities with robust job queue management and MQTT integration.
|
96
|
+
|
97
|
+
## ✨ Features
|
98
|
+
|
99
|
+
### Core Features
|
100
|
+
- 📊 **Pipeline Management**: Build and run data processing pipelines with support for multiple data formats and computation engines
|
101
|
+
- 🔄 **Job Queue Integration**: Support for multiple job queue backends (RQ, APScheduler)
|
102
|
+
- 📡 **MQTT Integration**: Built-in support for MQTT-based event processing
|
103
|
+
- 🎯 **Resilient Execution**: Automatic retries with configurable backoff and jitter
|
104
|
+
- 📊 **Data Format Support**: Work with CSV, JSON, Parquet files and more
|
105
|
+
- 🗄️ **Database Connectivity**: Connect to PostgreSQL, MySQL, SQLite, DuckDB, Oracle, and MSSQL
|
106
|
+
|
107
|
+
### Additional Features
|
108
|
+
- 🛠️ **CLI Tools**: Comprehensive command-line interface for all operations
|
109
|
+
- 📈 **Pipeline Visualization**: DAG visualization for pipeline understanding
|
110
|
+
- 🔍 **Monitoring**: Integration with OpenTelemetry for observability
|
111
|
+
- 🐳 **Docker Support**: Ready-to-use Docker configurations
|
112
|
+
|
113
|
+
## 🚀 Quick Start
|
114
|
+
|
115
|
+
### Installation
|
116
|
+
|
117
|
+
```bash
|
118
|
+
# Using pip
|
119
|
+
pip install flowerpower
|
120
|
+
|
121
|
+
# For development installation
|
122
|
+
git clone https://github.com/yourusername/flowerpower.git
|
123
|
+
cd flowerpower
|
124
|
+
pip install -e ".[dev]"
|
125
|
+
```
|
126
|
+
|
127
|
+
### Create Your First Pipeline
|
128
|
+
|
129
|
+
1. Initialize a new project:
|
130
|
+
```bash
|
131
|
+
flowerpower init --name my-first-project
|
132
|
+
```
|
133
|
+
|
134
|
+
2. Create a simple pipeline in `pipelines/hello_world.py`:
|
135
|
+
```python
|
136
|
+
import pandas as pd
|
137
|
+
|
138
|
+
def load_data() -> pd.DataFrame:
|
139
|
+
"""Load sample data"""
|
140
|
+
return pd.DataFrame({
|
141
|
+
'name': ['Alice', 'Bob', 'Charlie'],
|
142
|
+
'age': [25, 30, 35]
|
143
|
+
})
|
144
|
+
|
145
|
+
def process_data(df: pd.DataFrame) -> pd.DataFrame:
|
146
|
+
"""Add a greeting column"""
|
147
|
+
df['greeting'] = 'Hello, ' + df['name']
|
148
|
+
return df
|
149
|
+
|
150
|
+
def save_output(df: pd.DataFrame) -> None:
|
151
|
+
"""Save the processed data"""
|
152
|
+
print(df)
|
153
|
+
```
|
154
|
+
|
155
|
+
3. Run the pipeline:
|
156
|
+
```bash
|
157
|
+
flowerpower pipeline run hello_world
|
158
|
+
```
|
159
|
+
|
160
|
+
## 💡 Key Concepts
|
161
|
+
|
162
|
+
### Pipeline Management
|
163
|
+
|
164
|
+
Pipelines are the core building blocks of FlowerPower. They can be:
|
165
|
+
- Run directly
|
166
|
+
- Scheduled
|
167
|
+
- Triggered by MQTT messages
|
168
|
+
- Executed as background jobs
|
169
|
+
|
170
|
+
```bash
|
171
|
+
# Run a pipeline
|
172
|
+
flowerpower pipeline run my_pipeline --inputs '{"source": "data.csv"}'
|
173
|
+
|
174
|
+
# Schedule a pipeline
|
175
|
+
flowerpower pipeline schedule my_pipeline --cron "0 * * * *"
|
176
|
+
|
177
|
+
# Show pipeline structure
|
178
|
+
flowerpower pipeline show-dag my_pipeline
|
179
|
+
```
|
180
|
+
|
181
|
+
### Job Queue Integration
|
182
|
+
|
183
|
+
FlowerPower supports multiple job queue backends:
|
184
|
+
|
185
|
+
```bash
|
186
|
+
# Start a worker with RQ backend
|
187
|
+
flowerpower job-queue start-worker --type rq
|
188
|
+
|
189
|
+
# Start APScheduler worker
|
190
|
+
flowerpower job-queue start-worker --type apscheduler
|
191
|
+
|
192
|
+
# Add a job with retry configuration
|
193
|
+
flowerpower job-queue add-job my_pipeline \
|
194
|
+
--max-retries 3 \
|
195
|
+
--retry-delay 2.0 \
|
196
|
+
--jitter-factor 0.1
|
197
|
+
```
|
198
|
+
|
199
|
+
### MQTT Integration
|
200
|
+
|
201
|
+
Connect your pipelines to MQTT message brokers:
|
202
|
+
|
203
|
+
```bash
|
204
|
+
# Run a pipeline when messages arrive
|
205
|
+
flowerpower mqtt run-pipeline-on-message my_pipeline \
|
206
|
+
--topic "sensors/data" \
|
207
|
+
--max-retries 3 \
|
208
|
+
--retry-delay 1.0
|
209
|
+
|
210
|
+
# Start a custom message listener
|
211
|
+
flowerpower mqtt start-listener \
|
212
|
+
--on-message process_message \
|
213
|
+
--topic "events/#"
|
214
|
+
```
|
215
|
+
|
216
|
+
## 📁 Project Structure
|
217
|
+
|
218
|
+
```
|
219
|
+
my-project/
|
220
|
+
├── conf/
|
221
|
+
│ ├── project.yml # Project configuration
|
222
|
+
│ └── pipelines/ # Pipeline configurations
|
223
|
+
│ └── my_pipeline.yml
|
224
|
+
├── pipelines/ # Pipeline implementations
|
225
|
+
│ └── my_pipeline.py
|
226
|
+
└── data/ # Data files (optional)
|
227
|
+
```
|
228
|
+
|
229
|
+
## 🔌 Data Connectors
|
230
|
+
|
231
|
+
### Supported File Formats
|
232
|
+
- CSV
|
233
|
+
- JSON
|
234
|
+
- Parquet
|
235
|
+
- Pydala Datasets
|
236
|
+
|
237
|
+
### Supported Databases
|
238
|
+
- PostgreSQL
|
239
|
+
- MySQL
|
240
|
+
- SQLite
|
241
|
+
- Oracle
|
242
|
+
- Microsoft SQL Server
|
243
|
+
- DuckDB
|
244
|
+
|
245
|
+
## 🐳 Docker Support
|
246
|
+
|
247
|
+
Run FlowerPower in containers:
|
248
|
+
|
249
|
+
```bash
|
250
|
+
cd docker
|
251
|
+
docker-compose up
|
252
|
+
```
|
253
|
+
|
254
|
+
The Docker setup includes:
|
255
|
+
- Python worker environment
|
256
|
+
- MQTT broker (Mosquitto)
|
257
|
+
- Built-in configuration
|
258
|
+
|
259
|
+
## 🛠️ Configuration
|
260
|
+
|
261
|
+
### Pipeline Configuration
|
262
|
+
```yaml
|
263
|
+
# conf/pipelines/my_pipeline.yml
|
264
|
+
name: my_pipeline
|
265
|
+
description: Example pipeline configuration
|
266
|
+
inputs:
|
267
|
+
source_data:
|
268
|
+
type: csv
|
269
|
+
path: data/input.csv
|
270
|
+
outputs:
|
271
|
+
processed_data:
|
272
|
+
type: parquet
|
273
|
+
path: data/output.parquet
|
274
|
+
```
|
275
|
+
|
276
|
+
### Job Queue Configuration
|
277
|
+
```yaml
|
278
|
+
# conf/project.yml
|
279
|
+
job_queue:
|
280
|
+
type: rq # or apscheduler
|
281
|
+
redis_url: redis://localhost:6379
|
282
|
+
max_retries: 3
|
283
|
+
retry_delay: 1.0
|
284
|
+
```
|
285
|
+
|
286
|
+
## 📚 API Documentation
|
287
|
+
|
288
|
+
Visit our [API Documentation](docs/api.md) for detailed information about:
|
289
|
+
- Pipeline API
|
290
|
+
- Job Queue API
|
291
|
+
- MQTT Integration
|
292
|
+
- Data Connectors
|
293
|
+
- Configuration Options
|
294
|
+
|
295
|
+
## 🧪 Running Tests
|
296
|
+
|
297
|
+
```bash
|
298
|
+
# Run all tests
|
299
|
+
pytest tests/
|
300
|
+
|
301
|
+
# Run specific test category
|
302
|
+
pytest tests/test_pipeline/
|
303
|
+
```
|
304
|
+
|
305
|
+
## 🤝 Contributing
|
306
|
+
|
307
|
+
We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) for details.
|
308
|
+
|
309
|
+
1. Fork the repository
|
310
|
+
2. Create your feature branch
|
311
|
+
3. Commit your changes
|
312
|
+
4. Push to the branch
|
313
|
+
5. Create a Pull Request
|
314
|
+
|
315
|
+
## 📄 License
|
316
|
+
|
317
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
318
|
+
|
319
|
+
## 🙏 Acknowledgments
|
320
|
+
|
321
|
+
- Built with [Hamilton](https://github.com/DAGWorks-Inc/hamilton) for pipeline execution
|
322
|
+
- Uses [RQ](https://python-rq.org/) and [APScheduler](https://apscheduler.readthedocs.io/) for job queues
|
323
|
+
- MQTT support via [Paho MQTT](https://www.eclipse.org/paho/)
|
324
|
+
- Database connectivity through SQLAlchemy and native connectors
|
@@ -0,0 +1,94 @@
|
|
1
|
+
flowerpower/__init__.py,sha256=gAnAmZMeikphdIY3C55kJqwe2UFtZsvegs7Pk3VoL4M,456
|
2
|
+
flowerpower/flowerpower.py,sha256=7wmmLoWjz8B39amj0H23iRvr2DPuMtJyEnA0rLBbmXg,3542
|
3
|
+
flowerpower/settings.py,sha256=8mzXgHTMjnZOKn8QP035wLRxhMF1r6Fp5NavB62Xjno,3496
|
4
|
+
flowerpower/cfg/__init__.py,sha256=L-MI8JvYr9YTxuLvYlQCt2-uC9xZ_GFad3eUmm8XE1I,8709
|
5
|
+
flowerpower/cfg/base.py,sha256=W1_trbWUs9d56T06QHR5YlSLMbf7GnvtL6lQNAujJcY,4709
|
6
|
+
flowerpower/cfg/pipeline/__init__.py,sha256=9Ylc-t6uWWff8EYbrWsieulENzPZ92n_XSBIVvvNdN8,9015
|
7
|
+
flowerpower/cfg/pipeline/adapter.py,sha256=PuXEAl6YbluoRj5jdoABD8xHPstgHsKJUIDs2HJBgRk,2517
|
8
|
+
flowerpower/cfg/pipeline/run.py,sha256=tI5EhK7V-GNJgCQ9q-SnZLg_Ewxb312osLKQi8_lD7k,2049
|
9
|
+
flowerpower/cfg/pipeline/schedule.py,sha256=viw1Gd8pKq73Yw2-Tm9074YHP_yB5RPl9pYaqI8VDqI,2450
|
10
|
+
flowerpower/cfg/project/__init__.py,sha256=wVf5v3-hlsP3ZIJXvYXwnu-4VfvDr_bsNqH1dOLUIe8,5216
|
11
|
+
flowerpower/cfg/project/adapter.py,sha256=zIBwelSoQJ277_SA506tYEVSf0kEPJODaggocYm5rK4,2044
|
12
|
+
flowerpower/cfg/project/job_queue.py,sha256=Gc8B4xK894Dnby1B1Rf0czHjVzbGVtPmRVwT24fItnE,5911
|
13
|
+
flowerpower/cli/__init__.py,sha256=ifnlGUYIZDAu2fZ8TOUVQCnoYvMSiGCiPDJCrR6tR5M,5553
|
14
|
+
flowerpower/cli/cfg.py,sha256=B9yV6g9bTGRXOR1PewQTxf_XfcYbcsE3GeLAxlbE2qc,1530
|
15
|
+
flowerpower/cli/job_queue.py,sha256=8Zk8C3pP26lIXK_EDGtDVEPMAUMHrD26uNdhfG95n8w,35701
|
16
|
+
flowerpower/cli/mqtt.py,sha256=vu5A6IFX_nkR-1ShA6uLvumL5ymZIAYuqrhhNHRT1X4,6250
|
17
|
+
flowerpower/cli/pipeline.py,sha256=29LtaiBL4xIWh_z5FwP02TWVm9RNP13xcLn5w2mh0e0,36573
|
18
|
+
flowerpower/cli/utils.py,sha256=nDSSj_1nlYlMmj252kRZeohhFqHv9yvdgDEduQCyWOc,5152
|
19
|
+
flowerpower/cli/worker.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
|
+
flowerpower/fs/__init__.py,sha256=R0vqECRSUYi4TjLkLVpzMDRqivVIi25pYVpNU6nZwCo,862
|
21
|
+
flowerpower/fs/base.py,sha256=TZQv3jmuP6JT014JcamLz8ZGkl6c9TGL84tANvyyKOI,22084
|
22
|
+
flowerpower/fs/ext.py,sha256=iBmisaKRQBDbODeKRP8ZkzLX3SHtF15LzrBTGRGJ7zE,63885
|
23
|
+
flowerpower/fs/storage_options.py,sha256=O8lDfXyQ-Y4NuYemcmW9JE9t1IqWAqPdd0L7gVMUoAQ,45103
|
24
|
+
flowerpower/io/base.py,sha256=a81MS_lwTPcm8MbDHJoraeNUCHriGoO0V7UhomR5-jg,75955
|
25
|
+
flowerpower/io/metadata.py,sha256=ha1M4nFAwcmyTAbnxQeouIsAd7ROqRCeS1YsglqGhgw,7270
|
26
|
+
flowerpower/io/loader/__init__.py,sha256=MKH42nvVokaWas0wFgX1yrpU5iLpvHjRqqF-KzwLHCg,780
|
27
|
+
flowerpower/io/loader/_duckdb.py,sha256=WB9CKI7H4eumcV3VKqiEXiiLSDkHmjjyzrUytQWf6Qc,10311
|
28
|
+
flowerpower/io/loader/csv.py,sha256=2geJIMljENZGCvjUPg1CCBgnb4LN8TzUGXcPi9HWotQ,890
|
29
|
+
flowerpower/io/loader/deltatable.py,sha256=MhLJzahy3rBCz7ikIUSk8QhcBiUik0qQrt_hMRkpF2E,6573
|
30
|
+
flowerpower/io/loader/duckdb.py,sha256=lRhPyRaOhDEFund3wm7Fd7MpatluoDakhwipltfANrY,486
|
31
|
+
flowerpower/io/loader/json.py,sha256=wdhZhb4xO538c5SqDzNHB7U_Ao0E6MlLIHfSpGT-o2Y,802
|
32
|
+
flowerpower/io/loader/mqtt.py,sha256=2eAnavk9KcapQcZqoKOi81FvUFGbhkeJWzB6y01-3Pc,5147
|
33
|
+
flowerpower/io/loader/mssql.py,sha256=0nwa71KvdRLEUxDFnlFH7hIqWDRmZoVhfqt4ua8Xcqs,825
|
34
|
+
flowerpower/io/loader/mysql.py,sha256=6zuOvBntZca6bQ25ReUTeLp8J9SnwuYcoQL5WRu3B_0,825
|
35
|
+
flowerpower/io/loader/oracle.py,sha256=6YlIsim_XlDbBZ6Uje8FuxovxXXn6oPZ4Vmsp37KLrw,841
|
36
|
+
flowerpower/io/loader/parquet.py,sha256=8bXmZuzVCA2eFsnrp-vupdyLihvzSpb2LYAIXuX0aXo,850
|
37
|
+
flowerpower/io/loader/postgres.py,sha256=rb9HhbWG1mPxVU7ZhtpR-S3BMQ0DgE9Mj27_Me6Yeqc,853
|
38
|
+
flowerpower/io/loader/pydala.py,sha256=17plL-oVi0hgrYGAfCW8vxM9d4zTjSKLOF9LJSVIbSo,438
|
39
|
+
flowerpower/io/loader/sqlite.py,sha256=5SFeBfhgibjYemUhaPrMfg1XkFCAaIqu-B5gIcQjTN8,627
|
40
|
+
flowerpower/io/saver/__init__.py,sha256=RWcUdrcmi2JR4mb77jPZd-lSG5rQBIqS7Pl4vbPfUes,780
|
41
|
+
flowerpower/io/saver/_duckdb.py,sha256=GMQohIkpNVUcwT7tUv4MqtaPhaIPCwGl1kkdtp9DocE,9274
|
42
|
+
flowerpower/io/saver/csv.py,sha256=q6ndV12bobRWNJBibLaDbpvBiOsTcARmYMn8B1uegZk,799
|
43
|
+
flowerpower/io/saver/deltatable.py,sha256=tc4O5CZapCR8eEiM7WRU4Bi8Jv6XlTxy2YGCAmuNw78,6978
|
44
|
+
flowerpower/io/saver/duckdb.py,sha256=XNHgn8Qc2m7xovKZWOYEjNTf6CTQ4me5UToo1-kpzi0,433
|
45
|
+
flowerpower/io/saver/json.py,sha256=Qx17Yghia7RmXYoee7MXQjc6uKdyHxqI-5JhCdkBKuA,789
|
46
|
+
flowerpower/io/saver/mqtt.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
47
|
+
flowerpower/io/saver/mssql.py,sha256=4oDblnkm99gOHMXkEeMTd91hqPNHjy7eVUYxZWMGN0A,758
|
48
|
+
flowerpower/io/saver/mysql.py,sha256=l88MfCoMFdPtn5XlR4IlnLXpeYlQz4ljUJoljoJpw6I,758
|
49
|
+
flowerpower/io/saver/oracle.py,sha256=8mw45vKDL7HeFcIhL95yvg2ODAwUBLPZIfnb5sWpjIk,774
|
50
|
+
flowerpower/io/saver/parquet.py,sha256=IkYODtvcuwECjNlMdEogd0CkC3KEn00tgxQfvbzYXac,815
|
51
|
+
flowerpower/io/saver/postgres.py,sha256=lDSGnN3Y6xDuqn5NjnSxAWk84QmOhwrYCTQhXMlABLo,784
|
52
|
+
flowerpower/io/saver/pydala.py,sha256=0hcyFNWOVbdNlC_F0KbeVeTbgOZX2HbTvY1i6UU3kUw,478
|
53
|
+
flowerpower/io/saver/sqlite.py,sha256=q7v3fWtiaGslfKbgHpdaV8p7WtXc2MMyx3yMxmW7G5s,625
|
54
|
+
flowerpower/job_queue/__init__.py,sha256=vrTKNHrNnqUQaAf17JGNGtKRfigvH0lAm74kjCGkSEM,8888
|
55
|
+
flowerpower/job_queue/base.py,sha256=3QMs5E4tyLetUSBwBWtbDcxfSlG6PKyyy-hC5yz6cPM,12498
|
56
|
+
flowerpower/job_queue/apscheduler/__init__.py,sha256=CersYA0cZGPr_q-C1zVRsJJE8R7DXjK2HQpY8nsZ45s,235
|
57
|
+
flowerpower/job_queue/apscheduler/manager.py,sha256=P1EfC9WeZ9NuuX-NN00HvrNnKFtq7ESpshIivsMMWRk,36541
|
58
|
+
flowerpower/job_queue/apscheduler/setup.py,sha256=cjaJHOvn3BE-8XfqkPNvuVyvHB8yqalvcK2WY9xQIe8,18350
|
59
|
+
flowerpower/job_queue/apscheduler/trigger.py,sha256=kStQO_ipAn2xBbX3fSg4X3ORNOulalRyOhKBb0lkSAg,5111
|
60
|
+
flowerpower/job_queue/apscheduler/utils.py,sha256=ji-u98h3pF_ADCVSBIZIiYlxI3Zz3juYyMYA7WGVTWg,9312
|
61
|
+
flowerpower/job_queue/apscheduler/_setup/datastore.py,sha256=oV8pw7x0dZBDxhR54d-Y2O9hVanxhBuZbUNpxcizxak,3490
|
62
|
+
flowerpower/job_queue/apscheduler/_setup/eventbroker.py,sha256=qh8v9vAc-3v4e66wdXyrVZyIxFyqmOpzwKkolE8mVko,2872
|
63
|
+
flowerpower/job_queue/rq/__init__.py,sha256=m_vFSwmq7Cis-2mYhkDHxFaucSw7ODUfxWd3-hWZ50Y,193
|
64
|
+
flowerpower/job_queue/rq/_trigger.py,sha256=w1WKs_8nd9d4e_Y25yuG0vV2N4A58rWZz8RUORJepH0,1134
|
65
|
+
flowerpower/job_queue/rq/manager.py,sha256=7vduemVSaBdwb9_tVNOCbGn58fO5mFIPMLF28iR3lFI,51328
|
66
|
+
flowerpower/job_queue/rq/setup.py,sha256=fJPb7wztkefipulv4NQQjW3fEKFH3EMDaBLNVS8dkV8,5126
|
67
|
+
flowerpower/job_queue/rq/utils.py,sha256=QjyNhSM1_gMZkV2cO8eR99XeEji4AMwpxE1TANaRgTg,2009
|
68
|
+
flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py,sha256=X8rKfSbuITpWiwCyNqN-WVeodBkafFuj-zvbZb7z6lw,7511
|
69
|
+
flowerpower/job_queue/rq/concurrent_workers/thread_worker.py,sha256=M_jjci-pypEsbHe-gCQS6jmJBrA-Tb7W19sEWi6htFU,7801
|
70
|
+
flowerpower/pipeline/__init__.py,sha256=xbEn_RN0vVNqLZMSFOCdV41ggUkYrghFVJYd_EC0C44,75
|
71
|
+
flowerpower/pipeline/base.py,sha256=1srub-3gy2BSbdqyva8V6p5VljUDtutq-fXcQWKzFUs,3140
|
72
|
+
flowerpower/pipeline/io.py,sha256=rjj-V9wf0OJKxvDXgruKOZuCLkNqc56J9VcMegy5ols,15948
|
73
|
+
flowerpower/pipeline/job_queue.py,sha256=4LhOx2boUh7ET8-a6GfiJpVYHTKjmeRJtQrJ6GIi-Nc,22045
|
74
|
+
flowerpower/pipeline/manager.py,sha256=cWLX7CW01jv7Y4E0ofcRXmYtAf_qLjQZh1qaQyYeR0s,63907
|
75
|
+
flowerpower/pipeline/registry.py,sha256=fqTR2hKyJwxyUTgzLHeBP8aJdZENWd9Jz22KD052lTE,18892
|
76
|
+
flowerpower/pipeline/runner.py,sha256=tZ9WByCx7MtAWwovoEbEHscDvQffTmmfhtUN1cz463M,23603
|
77
|
+
flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpuNLoTI,5013
|
78
|
+
flowerpower/plugins/mqtt/__init__.py,sha256=kTlWgXrSScSxLrSQdqQESi35OEDbCbO3LLkTIcI90jQ,252
|
79
|
+
flowerpower/plugins/mqtt/cfg.py,sha256=_0eEubHU3n6Ybd0LOX6vr21-l3nhoIHwu6g9OXpwdJE,483
|
80
|
+
flowerpower/plugins/mqtt/manager.py,sha256=klft9kKmj3P4Pq7EutlT38rViQ4mVXPO1vDJDSzJZrU,29525
|
81
|
+
flowerpower/utils/logging.py,sha256=cdvlA9uJTbzPFRNC8bB5yEHKtn2G6mrGQyuZleFPiNM,787
|
82
|
+
flowerpower/utils/misc.py,sha256=ILznXwiBKynLKhpALMkx8l9DNFHvWYNa8ezTj8lHmqw,15962
|
83
|
+
flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2903
|
84
|
+
flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
|
85
|
+
flowerpower/utils/polars.py,sha256=-b4SrVVtsS4g03rD1vdsd6vu7zQogua1p57fYGhe6vo,17739
|
86
|
+
flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
|
87
|
+
flowerpower/utils/sql.py,sha256=wn10GJhfVg02uKppP0BY9g2DvdZbvynHJ4MPuKRUqbc,13085
|
88
|
+
flowerpower/utils/templates.py,sha256=5UsZUvxDaKMuoUURq4yoN8ijFytSvyrGrr-XDoLMDp8,1593
|
89
|
+
flowerpower/web/app.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
90
|
+
flowerpower-1.0.0b2.dist-info/METADATA,sha256=fIGwZCIYh5rgyvzfeAAAr375LyjzWAZJ01wN-EqSjI0,9530
|
91
|
+
flowerpower-1.0.0b2.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
|
92
|
+
flowerpower-1.0.0b2.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
|
93
|
+
flowerpower-1.0.0b2.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
|
94
|
+
flowerpower-1.0.0b2.dist-info/RECORD,,
|
flowerpower/_web/__init__.py
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
import importlib.util
|
2
|
-
import os
|
3
|
-
from pathlib import Path
|
4
|
-
|
5
|
-
from fastapi import FastAPI, Request
|
6
|
-
from fastapi.responses import HTMLResponse
|
7
|
-
from fastapi.staticfiles import StaticFiles
|
8
|
-
from fastapi.templating import Jinja2Templates
|
9
|
-
|
10
|
-
from ..pipeline import PipelineManager
|
11
|
-
from ..cfg import Config
|
12
|
-
|
13
|
-
# Initialize FastAPI app
|
14
|
-
app = FastAPI(
|
15
|
-
title="FlowerPower UI",
|
16
|
-
description="Web UI for the FlowerPower framework",
|
17
|
-
version="1.0.0",
|
18
|
-
)
|
19
|
-
|
20
|
-
# Set base directory
|
21
|
-
base_dir = os.environ.get("FLOWERPOWER_BASE_DIR", str(Path.cwd()))
|
22
|
-
|
23
|
-
# Set up template and static file directories
|
24
|
-
package_root = Path(__file__).parent
|
25
|
-
templates = Jinja2Templates(directory=str(package_root / "templates"))
|
26
|
-
app.mount("/static", StaticFiles(directory=str(package_root / "static")), name="static")
|
27
|
-
|
28
|
-
# Import and include routers
|
29
|
-
from .routes import pipelines, scheduler, config
|
30
|
-
|
31
|
-
app.include_router(pipelines.router)
|
32
|
-
app.include_router(scheduler.router)
|
33
|
-
app.include_router(config.router)
|
34
|
-
|
35
|
-
# Check for scheduler availability
|
36
|
-
has_scheduler = importlib.util.find_spec("apscheduler") is not None
|
37
|
-
|
38
|
-
@app.get("/", response_class=HTMLResponse)
|
39
|
-
async def home(request: Request):
|
40
|
-
"""Home page with dashboard overview"""
|
41
|
-
# Create pipeline manager to get pipelines
|
42
|
-
pipeline_manager = PipelineManager(base_dir=base_dir)
|
43
|
-
pipelines_list = pipeline_manager.list_pipelines() or [] # Ensure we have a list, even if empty
|
44
|
-
|
45
|
-
# Get schedules if scheduler is available
|
46
|
-
schedules = []
|
47
|
-
if has_scheduler:
|
48
|
-
from ..scheduler import SchedulerManager
|
49
|
-
with SchedulerManager(fs=pipeline_manager._fs, role="scheduler") as sm:
|
50
|
-
schedules = sm.get_schedules(as_dict=True) or [] # Ensure we have a list
|
51
|
-
|
52
|
-
return templates.TemplateResponse(
|
53
|
-
"index.html",
|
54
|
-
{
|
55
|
-
"request": request,
|
56
|
-
"pipelines": pipelines_list,
|
57
|
-
"schedules": schedules,
|
58
|
-
"has_scheduler": has_scheduler,
|
59
|
-
"base_dir": base_dir,
|
60
|
-
}
|
61
|
-
)
|
@@ -1,103 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
import yaml
|
3
|
-
from pathlib import Path
|
4
|
-
|
5
|
-
from fastapi import APIRouter, Request, Form, HTTPException
|
6
|
-
from fastapi.responses import HTMLResponse
|
7
|
-
|
8
|
-
from ...cfg import Config
|
9
|
-
from ...pipeline import PipelineManager
|
10
|
-
from .. import templates
|
11
|
-
|
12
|
-
router = APIRouter(prefix="/config", tags=["config"])
|
13
|
-
|
14
|
-
base_dir = os.environ.get("FLOWERPOWER_BASE_DIR", str(Path.cwd()))
|
15
|
-
|
16
|
-
@router.get("/project", response_class=HTMLResponse)
|
17
|
-
async def get_project_config(request: Request):
|
18
|
-
"""Get project configuration"""
|
19
|
-
config = Config.load(base_dir=base_dir)
|
20
|
-
|
21
|
-
# Get raw YAML content
|
22
|
-
config_path = Path(base_dir) / "conf" / "project.yml"
|
23
|
-
config_content = ""
|
24
|
-
if config_path.exists():
|
25
|
-
with open(config_path, "r") as f:
|
26
|
-
config_content = f.read()
|
27
|
-
|
28
|
-
return templates.TemplateResponse(
|
29
|
-
"config/project.html",
|
30
|
-
{
|
31
|
-
"request": request,
|
32
|
-
"config": config.project,
|
33
|
-
"config_content": config_content,
|
34
|
-
"base_dir": base_dir
|
35
|
-
}
|
36
|
-
)
|
37
|
-
|
38
|
-
@router.post("/project/update")
|
39
|
-
async def update_project_config(
|
40
|
-
request: Request,
|
41
|
-
config_content: str = Form(...)
|
42
|
-
):
|
43
|
-
"""Update project configuration"""
|
44
|
-
try:
|
45
|
-
# Validate YAML
|
46
|
-
config_data = yaml.safe_load(config_content)
|
47
|
-
|
48
|
-
# Write to file
|
49
|
-
config_path = Path(base_dir) / "conf" / "project.yml"
|
50
|
-
with open(config_path, "w") as f:
|
51
|
-
f.write(config_content)
|
52
|
-
|
53
|
-
return {"success": True, "message": "Project configuration updated successfully"}
|
54
|
-
except Exception as e:
|
55
|
-
raise HTTPException(status_code=500, detail=str(e))
|
56
|
-
|
57
|
-
@router.get("/pipeline/{name}", response_class=HTMLResponse)
|
58
|
-
async def get_pipeline_config(request: Request, name: str):
|
59
|
-
"""Get pipeline configuration"""
|
60
|
-
pipeline_manager = PipelineManager(base_dir=base_dir)
|
61
|
-
pipeline_manager.load_config(name=name)
|
62
|
-
|
63
|
-
# Get raw YAML content
|
64
|
-
config_path = Path(base_dir) / "conf" / "pipelines" / f"{name}.yml"
|
65
|
-
config_content = ""
|
66
|
-
if config_path.exists():
|
67
|
-
with open(config_path, "r") as f:
|
68
|
-
config_content = f.read()
|
69
|
-
|
70
|
-
return templates.TemplateResponse(
|
71
|
-
"config/pipeline.html",
|
72
|
-
{
|
73
|
-
"request": request,
|
74
|
-
"pipeline": name,
|
75
|
-
"config": pipeline_manager.cfg.pipeline,
|
76
|
-
"config_content": config_content,
|
77
|
-
"base_dir": base_dir
|
78
|
-
}
|
79
|
-
)
|
80
|
-
|
81
|
-
@router.post("/pipeline/{name}/update")
|
82
|
-
async def update_pipeline_config(
|
83
|
-
request: Request,
|
84
|
-
name: str,
|
85
|
-
config_content: str = Form(...)
|
86
|
-
):
|
87
|
-
"""Update pipeline configuration"""
|
88
|
-
try:
|
89
|
-
# Validate YAML
|
90
|
-
config_data = yaml.safe_load(config_content)
|
91
|
-
|
92
|
-
# Write to file
|
93
|
-
config_path = Path(base_dir) / "conf" / "pipelines" / f"{name}.yml"
|
94
|
-
|
95
|
-
# Ensure directory exists
|
96
|
-
config_path.parent.mkdir(parents=True, exist_ok=True)
|
97
|
-
|
98
|
-
with open(config_path, "w") as f:
|
99
|
-
f.write(config_content)
|
100
|
-
|
101
|
-
return {"success": True, "message": f"Pipeline {name} configuration updated successfully"}
|
102
|
-
except Exception as e:
|
103
|
-
raise HTTPException(status_code=500, detail=str(e))
|