FlowerPower 0.9.12.4__py3-none-any.whl → 1.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/__init__.py +17 -2
- flowerpower/cfg/__init__.py +201 -149
- flowerpower/cfg/base.py +122 -24
- flowerpower/cfg/pipeline/__init__.py +254 -0
- flowerpower/cfg/pipeline/adapter.py +66 -0
- flowerpower/cfg/pipeline/run.py +40 -11
- flowerpower/cfg/pipeline/schedule.py +69 -79
- flowerpower/cfg/project/__init__.py +149 -0
- flowerpower/cfg/project/adapter.py +57 -0
- flowerpower/cfg/project/job_queue.py +165 -0
- flowerpower/cli/__init__.py +92 -35
- flowerpower/cli/job_queue.py +878 -0
- flowerpower/cli/mqtt.py +49 -4
- flowerpower/cli/pipeline.py +576 -381
- flowerpower/cli/utils.py +55 -0
- flowerpower/flowerpower.py +12 -7
- flowerpower/fs/__init__.py +20 -2
- flowerpower/fs/base.py +350 -26
- flowerpower/fs/ext.py +797 -216
- flowerpower/fs/storage_options.py +1097 -55
- flowerpower/io/base.py +13 -18
- flowerpower/io/loader/__init__.py +28 -0
- flowerpower/io/loader/deltatable.py +7 -10
- flowerpower/io/metadata.py +1 -0
- flowerpower/io/saver/__init__.py +28 -0
- flowerpower/io/saver/deltatable.py +4 -3
- flowerpower/job_queue/__init__.py +252 -0
- flowerpower/job_queue/apscheduler/__init__.py +11 -0
- flowerpower/job_queue/apscheduler/_setup/datastore.py +110 -0
- flowerpower/job_queue/apscheduler/_setup/eventbroker.py +93 -0
- flowerpower/job_queue/apscheduler/manager.py +1063 -0
- flowerpower/job_queue/apscheduler/setup.py +524 -0
- flowerpower/job_queue/apscheduler/trigger.py +169 -0
- flowerpower/job_queue/apscheduler/utils.py +309 -0
- flowerpower/job_queue/base.py +382 -0
- flowerpower/job_queue/rq/__init__.py +10 -0
- flowerpower/job_queue/rq/_trigger.py +37 -0
- flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +226 -0
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +231 -0
- flowerpower/job_queue/rq/manager.py +1449 -0
- flowerpower/job_queue/rq/setup.py +150 -0
- flowerpower/job_queue/rq/utils.py +69 -0
- flowerpower/pipeline/__init__.py +5 -0
- flowerpower/pipeline/base.py +118 -0
- flowerpower/pipeline/io.py +407 -0
- flowerpower/pipeline/job_queue.py +505 -0
- flowerpower/pipeline/manager.py +1586 -0
- flowerpower/pipeline/registry.py +560 -0
- flowerpower/pipeline/runner.py +560 -0
- flowerpower/pipeline/visualizer.py +142 -0
- flowerpower/plugins/mqtt/__init__.py +12 -0
- flowerpower/plugins/mqtt/cfg.py +16 -0
- flowerpower/plugins/mqtt/manager.py +789 -0
- flowerpower/settings.py +110 -0
- flowerpower/utils/logging.py +21 -0
- flowerpower/utils/misc.py +57 -9
- flowerpower/utils/sql.py +122 -24
- flowerpower/utils/templates.py +18 -142
- flowerpower/web/app.py +0 -0
- flowerpower-1.0.0b1.dist-info/METADATA +324 -0
- flowerpower-1.0.0b1.dist-info/RECORD +94 -0
- {flowerpower-0.9.12.4.dist-info → flowerpower-1.0.0b1.dist-info}/WHEEL +1 -1
- flowerpower/cfg/pipeline/tracker.py +0 -14
- flowerpower/cfg/project/open_telemetry.py +0 -8
- flowerpower/cfg/project/tracker.py +0 -11
- flowerpower/cfg/project/worker.py +0 -19
- flowerpower/cli/scheduler.py +0 -309
- flowerpower/event_handler.py +0 -23
- flowerpower/mqtt.py +0 -525
- flowerpower/pipeline.py +0 -2419
- flowerpower/scheduler.py +0 -680
- flowerpower/tui.py +0 -79
- flowerpower/utils/datastore.py +0 -186
- flowerpower/utils/eventbroker.py +0 -127
- flowerpower/utils/executor.py +0 -58
- flowerpower/utils/trigger.py +0 -140
- flowerpower-0.9.12.4.dist-info/METADATA +0 -575
- flowerpower-0.9.12.4.dist-info/RECORD +0 -70
- /flowerpower/{cfg/pipeline/params.py → cli/worker.py} +0 -0
- {flowerpower-0.9.12.4.dist-info → flowerpower-1.0.0b1.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.9.12.4.dist-info → flowerpower-1.0.0b1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,324 @@
|
|
1
|
+
Metadata-Version: 2.4
|
2
|
+
Name: FlowerPower
|
3
|
+
Version: 1.0.0b1
|
4
|
+
Summary: A simple workflow framework. Hamilton + APScheduler = FlowerPower
|
5
|
+
Author-email: "Volker L." <ligno.blades@gmail.com>
|
6
|
+
Project-URL: Homepage, https://github.com/legout/flowerpower
|
7
|
+
Project-URL: Bug Tracker, https://github.com/legout/flowerpower/issues
|
8
|
+
Keywords: hamilton,workflow,pipeline,scheduler,apscheduler,dask,ray
|
9
|
+
Requires-Python: >=3.11
|
10
|
+
Description-Content-Type: text/markdown
|
11
|
+
Requires-Dist: aiobotocore<2.18.0
|
12
|
+
Requires-Dist: aiosqlite>=0.21.0
|
13
|
+
Requires-Dist: dill>=0.3.8
|
14
|
+
Requires-Dist: duration-parser>=1.0.1
|
15
|
+
Requires-Dist: fsspec>=2024.10.0
|
16
|
+
Requires-Dist: humanize>=4.12.2
|
17
|
+
Requires-Dist: msgspec>=0.19.0
|
18
|
+
Requires-Dist: munch>=4.0.0
|
19
|
+
Requires-Dist: orjson>=3.10.15
|
20
|
+
Requires-Dist: pyarrow<19.0.0
|
21
|
+
Requires-Dist: pydantic>=2.10.2
|
22
|
+
Requires-Dist: python-dotenv>=1.0.1
|
23
|
+
Requires-Dist: pyyaml>=6.0.1
|
24
|
+
Requires-Dist: rich>=13.9.3
|
25
|
+
Requires-Dist: s3fs>=2024.10.0
|
26
|
+
Requires-Dist: sf-hamilton-sdk>=0.5.2
|
27
|
+
Requires-Dist: sf-hamilton[rich,tqdm,visualization]>=1.69.0
|
28
|
+
Requires-Dist: typer>=0.12.3
|
29
|
+
Provides-Extra: apscheduler
|
30
|
+
Requires-Dist: aiosqlite>=0.21.0; extra == "apscheduler"
|
31
|
+
Requires-Dist: apscheduler==4.0.0a5; extra == "apscheduler"
|
32
|
+
Requires-Dist: asyncpg>=0.29.0; extra == "apscheduler"
|
33
|
+
Requires-Dist: greenlet>=3.0.3; extra == "apscheduler"
|
34
|
+
Requires-Dist: sqlalchemy>=2.0.30; extra == "apscheduler"
|
35
|
+
Requires-Dist: cron-descriptor>=1.4.5; extra == "apscheduler"
|
36
|
+
Provides-Extra: io
|
37
|
+
Requires-Dist: adbc-driver-manager>=1.4.0; extra == "io"
|
38
|
+
Requires-Dist: datafusion>=43.1.0; extra == "io"
|
39
|
+
Requires-Dist: deltalake>=0.24.0; extra == "io"
|
40
|
+
Requires-Dist: duckdb>=1.1.3; extra == "io"
|
41
|
+
Requires-Dist: orjson>=3.10.12; extra == "io"
|
42
|
+
Requires-Dist: pandas>=2.2.3; extra == "io"
|
43
|
+
Requires-Dist: polars>=1.15.0; extra == "io"
|
44
|
+
Requires-Dist: pyarrow>=18.1.0; extra == "io"
|
45
|
+
Requires-Dist: pydala2>=0.9.4.5; extra == "io"
|
46
|
+
Requires-Dist: redis>=5.2.1; extra == "io"
|
47
|
+
Requires-Dist: sherlock>=0.4.1; extra == "io"
|
48
|
+
Provides-Extra: io-legacy
|
49
|
+
Requires-Dist: adbc-driver-manager>=1.4.0; extra == "io-legacy"
|
50
|
+
Requires-Dist: datafusion>=43.1.0; extra == "io-legacy"
|
51
|
+
Requires-Dist: deltalake>=0.24.0; extra == "io-legacy"
|
52
|
+
Requires-Dist: duckdb>=1.1.3; extra == "io-legacy"
|
53
|
+
Requires-Dist: orjson>=3.10.12; extra == "io-legacy"
|
54
|
+
Requires-Dist: pandas>=2.2.3; extra == "io-legacy"
|
55
|
+
Requires-Dist: polars-lts-cpu>=1.15.0; extra == "io-legacy"
|
56
|
+
Requires-Dist: pyarrow>=18.1.0; extra == "io-legacy"
|
57
|
+
Requires-Dist: pydala2>=0.9.4.5; extra == "io-legacy"
|
58
|
+
Requires-Dist: redis>=5.2.1; extra == "io-legacy"
|
59
|
+
Requires-Dist: sherlock>=0.4.1; extra == "io-legacy"
|
60
|
+
Provides-Extra: mongodb
|
61
|
+
Requires-Dist: pymongo>=4.7.2; extra == "mongodb"
|
62
|
+
Provides-Extra: mqtt
|
63
|
+
Requires-Dist: paho-mqtt>=2.1.0; extra == "mqtt"
|
64
|
+
Requires-Dist: orjson>=3.10.11; extra == "mqtt"
|
65
|
+
Requires-Dist: mmh3>=5.1.0; extra == "mqtt"
|
66
|
+
Provides-Extra: opentelemetry
|
67
|
+
Requires-Dist: opentelemetry-api>=1.5.0; extra == "opentelemetry"
|
68
|
+
Requires-Dist: opentelemetry-sdk>=1.5.0; extra == "opentelemetry"
|
69
|
+
Requires-Dist: opentelemetry-exporter-jaeger>=1.21.0; extra == "opentelemetry"
|
70
|
+
Provides-Extra: ray
|
71
|
+
Requires-Dist: ray>=2.34.0; extra == "ray"
|
72
|
+
Provides-Extra: redis
|
73
|
+
Requires-Dist: redis>=5.0.4; extra == "redis"
|
74
|
+
Provides-Extra: rq
|
75
|
+
Requires-Dist: rq>=2.3.1; extra == "rq"
|
76
|
+
Requires-Dist: rq-scheduler>=0.14.0; extra == "rq"
|
77
|
+
Requires-Dist: cron-descriptor>=1.4.5; extra == "rq"
|
78
|
+
Provides-Extra: tui
|
79
|
+
Requires-Dist: textual>=0.85.2; extra == "tui"
|
80
|
+
Provides-Extra: ui
|
81
|
+
Requires-Dist: sf-hamilton-ui>=0.0.11; extra == "ui"
|
82
|
+
Provides-Extra: webserver
|
83
|
+
Requires-Dist: sanic>=24.6.0; extra == "webserver"
|
84
|
+
Requires-Dist: sanic-ext>=23.12.0; extra == "webserver"
|
85
|
+
Requires-Dist: orjson>=3.10.11; extra == "webserver"
|
86
|
+
Provides-Extra: openlineage
|
87
|
+
Requires-Dist: openlineage-python>=1.32.0; extra == "openlineage"
|
88
|
+
|
89
|
+
<div align="center">
|
90
|
+
<h1>FlowerPower 🌸</h1>
|
91
|
+
<h3>Simple Workflow Framework - Hamilton + APScheduler = FlowerPower</h3>
|
92
|
+
<img src="./image.png" alt="FlowerPower Logo" width="400" height="300">
|
93
|
+
</div>
|
94
|
+
|
95
|
+
A powerful and flexible data pipeline framework that simplifies data processing workflows, job scheduling, and event-driven architectures. FlowerPower combines modern data processing capabilities with robust job queue management and MQTT integration.
|
96
|
+
|
97
|
+
## ✨ Features
|
98
|
+
|
99
|
+
### Core Features
|
100
|
+
- 📊 **Pipeline Management**: Build and run data processing pipelines with support for multiple data formats and computation engines
|
101
|
+
- 🔄 **Job Queue Integration**: Support for multiple job queue backends (RQ, APScheduler)
|
102
|
+
- 📡 **MQTT Integration**: Built-in support for MQTT-based event processing
|
103
|
+
- 🎯 **Resilient Execution**: Automatic retries with configurable backoff and jitter
|
104
|
+
- 📊 **Data Format Support**: Work with CSV, JSON, Parquet files and more
|
105
|
+
- 🗄️ **Database Connectivity**: Connect to PostgreSQL, MySQL, SQLite, DuckDB, Oracle, and MSSQL
|
106
|
+
|
107
|
+
### Additional Features
|
108
|
+
- 🛠️ **CLI Tools**: Comprehensive command-line interface for all operations
|
109
|
+
- 📈 **Pipeline Visualization**: DAG visualization for pipeline understanding
|
110
|
+
- 🔍 **Monitoring**: Integration with OpenTelemetry for observability
|
111
|
+
- 🐳 **Docker Support**: Ready-to-use Docker configurations
|
112
|
+
|
113
|
+
## 🚀 Quick Start
|
114
|
+
|
115
|
+
### Installation
|
116
|
+
|
117
|
+
```bash
|
118
|
+
# Using pip
|
119
|
+
pip install flowerpower
|
120
|
+
|
121
|
+
# For development installation
|
122
|
+
git clone https://github.com/yourusername/flowerpower.git
|
123
|
+
cd flowerpower
|
124
|
+
pip install -e ".[dev]"
|
125
|
+
```
|
126
|
+
|
127
|
+
### Create Your First Pipeline
|
128
|
+
|
129
|
+
1. Initialize a new project:
|
130
|
+
```bash
|
131
|
+
flowerpower init --name my-first-project
|
132
|
+
```
|
133
|
+
|
134
|
+
2. Create a simple pipeline in `pipelines/hello_world.py`:
|
135
|
+
```python
|
136
|
+
import pandas as pd
|
137
|
+
|
138
|
+
def load_data() -> pd.DataFrame:
|
139
|
+
"""Load sample data"""
|
140
|
+
return pd.DataFrame({
|
141
|
+
'name': ['Alice', 'Bob', 'Charlie'],
|
142
|
+
'age': [25, 30, 35]
|
143
|
+
})
|
144
|
+
|
145
|
+
def process_data(df: pd.DataFrame) -> pd.DataFrame:
|
146
|
+
"""Add a greeting column"""
|
147
|
+
df['greeting'] = 'Hello, ' + df['name']
|
148
|
+
return df
|
149
|
+
|
150
|
+
def save_output(df: pd.DataFrame) -> None:
|
151
|
+
"""Save the processed data"""
|
152
|
+
print(df)
|
153
|
+
```
|
154
|
+
|
155
|
+
3. Run the pipeline:
|
156
|
+
```bash
|
157
|
+
flowerpower pipeline run hello_world
|
158
|
+
```
|
159
|
+
|
160
|
+
## 💡 Key Concepts
|
161
|
+
|
162
|
+
### Pipeline Management
|
163
|
+
|
164
|
+
Pipelines are the core building blocks of FlowerPower. They can be:
|
165
|
+
- Run directly
|
166
|
+
- Scheduled
|
167
|
+
- Triggered by MQTT messages
|
168
|
+
- Executed as background jobs
|
169
|
+
|
170
|
+
```bash
|
171
|
+
# Run a pipeline
|
172
|
+
flowerpower pipeline run my_pipeline --inputs '{"source": "data.csv"}'
|
173
|
+
|
174
|
+
# Schedule a pipeline
|
175
|
+
flowerpower pipeline schedule my_pipeline --cron "0 * * * *"
|
176
|
+
|
177
|
+
# Show pipeline structure
|
178
|
+
flowerpower pipeline show-dag my_pipeline
|
179
|
+
```
|
180
|
+
|
181
|
+
### Job Queue Integration
|
182
|
+
|
183
|
+
FlowerPower supports multiple job queue backends:
|
184
|
+
|
185
|
+
```bash
|
186
|
+
# Start a worker with RQ backend
|
187
|
+
flowerpower job-queue start-worker --type rq
|
188
|
+
|
189
|
+
# Start APScheduler worker
|
190
|
+
flowerpower job-queue start-worker --type apscheduler
|
191
|
+
|
192
|
+
# Add a job with retry configuration
|
193
|
+
flowerpower job-queue add-job my_pipeline \
|
194
|
+
--max-retries 3 \
|
195
|
+
--retry-delay 2.0 \
|
196
|
+
--jitter-factor 0.1
|
197
|
+
```
|
198
|
+
|
199
|
+
### MQTT Integration
|
200
|
+
|
201
|
+
Connect your pipelines to MQTT message brokers:
|
202
|
+
|
203
|
+
```bash
|
204
|
+
# Run a pipeline when messages arrive
|
205
|
+
flowerpower mqtt run-pipeline-on-message my_pipeline \
|
206
|
+
--topic "sensors/data" \
|
207
|
+
--max-retries 3 \
|
208
|
+
--retry-delay 1.0
|
209
|
+
|
210
|
+
# Start a custom message listener
|
211
|
+
flowerpower mqtt start-listener \
|
212
|
+
--on-message process_message \
|
213
|
+
--topic "events/#"
|
214
|
+
```
|
215
|
+
|
216
|
+
## 📁 Project Structure
|
217
|
+
|
218
|
+
```
|
219
|
+
my-project/
|
220
|
+
├── conf/
|
221
|
+
│ ├── project.yml # Project configuration
|
222
|
+
│ └── pipelines/ # Pipeline configurations
|
223
|
+
│ └── my_pipeline.yml
|
224
|
+
├── pipelines/ # Pipeline implementations
|
225
|
+
│ └── my_pipeline.py
|
226
|
+
└── data/ # Data files (optional)
|
227
|
+
```
|
228
|
+
|
229
|
+
## 🔌 Data Connectors
|
230
|
+
|
231
|
+
### Supported File Formats
|
232
|
+
- CSV
|
233
|
+
- JSON
|
234
|
+
- Parquet
|
235
|
+
- Pydala Datasets
|
236
|
+
|
237
|
+
### Supported Databases
|
238
|
+
- PostgreSQL
|
239
|
+
- MySQL
|
240
|
+
- SQLite
|
241
|
+
- Oracle
|
242
|
+
- Microsoft SQL Server
|
243
|
+
- DuckDB
|
244
|
+
|
245
|
+
## 🐳 Docker Support
|
246
|
+
|
247
|
+
Run FlowerPower in containers:
|
248
|
+
|
249
|
+
```bash
|
250
|
+
cd docker
|
251
|
+
docker-compose up
|
252
|
+
```
|
253
|
+
|
254
|
+
The Docker setup includes:
|
255
|
+
- Python worker environment
|
256
|
+
- MQTT broker (Mosquitto)
|
257
|
+
- Built-in configuration
|
258
|
+
|
259
|
+
## 🛠️ Configuration
|
260
|
+
|
261
|
+
### Pipeline Configuration
|
262
|
+
```yaml
|
263
|
+
# conf/pipelines/my_pipeline.yml
|
264
|
+
name: my_pipeline
|
265
|
+
description: Example pipeline configuration
|
266
|
+
inputs:
|
267
|
+
source_data:
|
268
|
+
type: csv
|
269
|
+
path: data/input.csv
|
270
|
+
outputs:
|
271
|
+
processed_data:
|
272
|
+
type: parquet
|
273
|
+
path: data/output.parquet
|
274
|
+
```
|
275
|
+
|
276
|
+
### Job Queue Configuration
|
277
|
+
```yaml
|
278
|
+
# conf/project.yml
|
279
|
+
job_queue:
|
280
|
+
type: rq # or apscheduler
|
281
|
+
redis_url: redis://localhost:6379
|
282
|
+
max_retries: 3
|
283
|
+
retry_delay: 1.0
|
284
|
+
```
|
285
|
+
|
286
|
+
## 📚 API Documentation
|
287
|
+
|
288
|
+
Visit our [API Documentation](docs/api.md) for detailed information about:
|
289
|
+
- Pipeline API
|
290
|
+
- Job Queue API
|
291
|
+
- MQTT Integration
|
292
|
+
- Data Connectors
|
293
|
+
- Configuration Options
|
294
|
+
|
295
|
+
## 🧪 Running Tests
|
296
|
+
|
297
|
+
```bash
|
298
|
+
# Run all tests
|
299
|
+
pytest tests/
|
300
|
+
|
301
|
+
# Run specific test category
|
302
|
+
pytest tests/test_pipeline/
|
303
|
+
```
|
304
|
+
|
305
|
+
## 🤝 Contributing
|
306
|
+
|
307
|
+
We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) for details.
|
308
|
+
|
309
|
+
1. Fork the repository
|
310
|
+
2. Create your feature branch
|
311
|
+
3. Commit your changes
|
312
|
+
4. Push to the branch
|
313
|
+
5. Create a Pull Request
|
314
|
+
|
315
|
+
## 📄 License
|
316
|
+
|
317
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
318
|
+
|
319
|
+
## 🙏 Acknowledgments
|
320
|
+
|
321
|
+
- Built with [Hamilton](https://github.com/DAGWorks-Inc/hamilton) for pipeline execution
|
322
|
+
- Uses [RQ](https://python-rq.org/) and [APScheduler](https://apscheduler.readthedocs.io/) for job queues
|
323
|
+
- MQTT support via [Paho MQTT](https://www.eclipse.org/paho/)
|
324
|
+
- Database connectivity through SQLAlchemy and native connectors
|
@@ -0,0 +1,94 @@
|
|
1
|
+
flowerpower/__init__.py,sha256=gAnAmZMeikphdIY3C55kJqwe2UFtZsvegs7Pk3VoL4M,456
|
2
|
+
flowerpower/flowerpower.py,sha256=7wmmLoWjz8B39amj0H23iRvr2DPuMtJyEnA0rLBbmXg,3542
|
3
|
+
flowerpower/settings.py,sha256=8mzXgHTMjnZOKn8QP035wLRxhMF1r6Fp5NavB62Xjno,3496
|
4
|
+
flowerpower/cfg/__init__.py,sha256=L-MI8JvYr9YTxuLvYlQCt2-uC9xZ_GFad3eUmm8XE1I,8709
|
5
|
+
flowerpower/cfg/base.py,sha256=W1_trbWUs9d56T06QHR5YlSLMbf7GnvtL6lQNAujJcY,4709
|
6
|
+
flowerpower/cfg/pipeline/__init__.py,sha256=9Ylc-t6uWWff8EYbrWsieulENzPZ92n_XSBIVvvNdN8,9015
|
7
|
+
flowerpower/cfg/pipeline/adapter.py,sha256=PuXEAl6YbluoRj5jdoABD8xHPstgHsKJUIDs2HJBgRk,2517
|
8
|
+
flowerpower/cfg/pipeline/run.py,sha256=tI5EhK7V-GNJgCQ9q-SnZLg_Ewxb312osLKQi8_lD7k,2049
|
9
|
+
flowerpower/cfg/pipeline/schedule.py,sha256=viw1Gd8pKq73Yw2-Tm9074YHP_yB5RPl9pYaqI8VDqI,2450
|
10
|
+
flowerpower/cfg/project/__init__.py,sha256=wVf5v3-hlsP3ZIJXvYXwnu-4VfvDr_bsNqH1dOLUIe8,5216
|
11
|
+
flowerpower/cfg/project/adapter.py,sha256=zIBwelSoQJ277_SA506tYEVSf0kEPJODaggocYm5rK4,2044
|
12
|
+
flowerpower/cfg/project/job_queue.py,sha256=Gc8B4xK894Dnby1B1Rf0czHjVzbGVtPmRVwT24fItnE,5911
|
13
|
+
flowerpower/cli/__init__.py,sha256=ifnlGUYIZDAu2fZ8TOUVQCnoYvMSiGCiPDJCrR6tR5M,5553
|
14
|
+
flowerpower/cli/cfg.py,sha256=B9yV6g9bTGRXOR1PewQTxf_XfcYbcsE3GeLAxlbE2qc,1530
|
15
|
+
flowerpower/cli/job_queue.py,sha256=8Zk8C3pP26lIXK_EDGtDVEPMAUMHrD26uNdhfG95n8w,35701
|
16
|
+
flowerpower/cli/mqtt.py,sha256=vu5A6IFX_nkR-1ShA6uLvumL5ymZIAYuqrhhNHRT1X4,6250
|
17
|
+
flowerpower/cli/pipeline.py,sha256=29LtaiBL4xIWh_z5FwP02TWVm9RNP13xcLn5w2mh0e0,36573
|
18
|
+
flowerpower/cli/utils.py,sha256=nDSSj_1nlYlMmj252kRZeohhFqHv9yvdgDEduQCyWOc,5152
|
19
|
+
flowerpower/cli/worker.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
20
|
+
flowerpower/fs/__init__.py,sha256=R0vqECRSUYi4TjLkLVpzMDRqivVIi25pYVpNU6nZwCo,862
|
21
|
+
flowerpower/fs/base.py,sha256=TZQv3jmuP6JT014JcamLz8ZGkl6c9TGL84tANvyyKOI,22084
|
22
|
+
flowerpower/fs/ext.py,sha256=iBmisaKRQBDbODeKRP8ZkzLX3SHtF15LzrBTGRGJ7zE,63885
|
23
|
+
flowerpower/fs/storage_options.py,sha256=O8lDfXyQ-Y4NuYemcmW9JE9t1IqWAqPdd0L7gVMUoAQ,45103
|
24
|
+
flowerpower/io/base.py,sha256=a81MS_lwTPcm8MbDHJoraeNUCHriGoO0V7UhomR5-jg,75955
|
25
|
+
flowerpower/io/metadata.py,sha256=ha1M4nFAwcmyTAbnxQeouIsAd7ROqRCeS1YsglqGhgw,7270
|
26
|
+
flowerpower/io/loader/__init__.py,sha256=MKH42nvVokaWas0wFgX1yrpU5iLpvHjRqqF-KzwLHCg,780
|
27
|
+
flowerpower/io/loader/_duckdb.py,sha256=WB9CKI7H4eumcV3VKqiEXiiLSDkHmjjyzrUytQWf6Qc,10311
|
28
|
+
flowerpower/io/loader/csv.py,sha256=2geJIMljENZGCvjUPg1CCBgnb4LN8TzUGXcPi9HWotQ,890
|
29
|
+
flowerpower/io/loader/deltatable.py,sha256=MhLJzahy3rBCz7ikIUSk8QhcBiUik0qQrt_hMRkpF2E,6573
|
30
|
+
flowerpower/io/loader/duckdb.py,sha256=lRhPyRaOhDEFund3wm7Fd7MpatluoDakhwipltfANrY,486
|
31
|
+
flowerpower/io/loader/json.py,sha256=wdhZhb4xO538c5SqDzNHB7U_Ao0E6MlLIHfSpGT-o2Y,802
|
32
|
+
flowerpower/io/loader/mqtt.py,sha256=2eAnavk9KcapQcZqoKOi81FvUFGbhkeJWzB6y01-3Pc,5147
|
33
|
+
flowerpower/io/loader/mssql.py,sha256=0nwa71KvdRLEUxDFnlFH7hIqWDRmZoVhfqt4ua8Xcqs,825
|
34
|
+
flowerpower/io/loader/mysql.py,sha256=6zuOvBntZca6bQ25ReUTeLp8J9SnwuYcoQL5WRu3B_0,825
|
35
|
+
flowerpower/io/loader/oracle.py,sha256=6YlIsim_XlDbBZ6Uje8FuxovxXXn6oPZ4Vmsp37KLrw,841
|
36
|
+
flowerpower/io/loader/parquet.py,sha256=8bXmZuzVCA2eFsnrp-vupdyLihvzSpb2LYAIXuX0aXo,850
|
37
|
+
flowerpower/io/loader/postgres.py,sha256=rb9HhbWG1mPxVU7ZhtpR-S3BMQ0DgE9Mj27_Me6Yeqc,853
|
38
|
+
flowerpower/io/loader/pydala.py,sha256=17plL-oVi0hgrYGAfCW8vxM9d4zTjSKLOF9LJSVIbSo,438
|
39
|
+
flowerpower/io/loader/sqlite.py,sha256=5SFeBfhgibjYemUhaPrMfg1XkFCAaIqu-B5gIcQjTN8,627
|
40
|
+
flowerpower/io/saver/__init__.py,sha256=RWcUdrcmi2JR4mb77jPZd-lSG5rQBIqS7Pl4vbPfUes,780
|
41
|
+
flowerpower/io/saver/_duckdb.py,sha256=GMQohIkpNVUcwT7tUv4MqtaPhaIPCwGl1kkdtp9DocE,9274
|
42
|
+
flowerpower/io/saver/csv.py,sha256=q6ndV12bobRWNJBibLaDbpvBiOsTcARmYMn8B1uegZk,799
|
43
|
+
flowerpower/io/saver/deltatable.py,sha256=tc4O5CZapCR8eEiM7WRU4Bi8Jv6XlTxy2YGCAmuNw78,6978
|
44
|
+
flowerpower/io/saver/duckdb.py,sha256=XNHgn8Qc2m7xovKZWOYEjNTf6CTQ4me5UToo1-kpzi0,433
|
45
|
+
flowerpower/io/saver/json.py,sha256=Qx17Yghia7RmXYoee7MXQjc6uKdyHxqI-5JhCdkBKuA,789
|
46
|
+
flowerpower/io/saver/mqtt.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
47
|
+
flowerpower/io/saver/mssql.py,sha256=4oDblnkm99gOHMXkEeMTd91hqPNHjy7eVUYxZWMGN0A,758
|
48
|
+
flowerpower/io/saver/mysql.py,sha256=l88MfCoMFdPtn5XlR4IlnLXpeYlQz4ljUJoljoJpw6I,758
|
49
|
+
flowerpower/io/saver/oracle.py,sha256=8mw45vKDL7HeFcIhL95yvg2ODAwUBLPZIfnb5sWpjIk,774
|
50
|
+
flowerpower/io/saver/parquet.py,sha256=IkYODtvcuwECjNlMdEogd0CkC3KEn00tgxQfvbzYXac,815
|
51
|
+
flowerpower/io/saver/postgres.py,sha256=lDSGnN3Y6xDuqn5NjnSxAWk84QmOhwrYCTQhXMlABLo,784
|
52
|
+
flowerpower/io/saver/pydala.py,sha256=0hcyFNWOVbdNlC_F0KbeVeTbgOZX2HbTvY1i6UU3kUw,478
|
53
|
+
flowerpower/io/saver/sqlite.py,sha256=q7v3fWtiaGslfKbgHpdaV8p7WtXc2MMyx3yMxmW7G5s,625
|
54
|
+
flowerpower/job_queue/__init__.py,sha256=vrTKNHrNnqUQaAf17JGNGtKRfigvH0lAm74kjCGkSEM,8888
|
55
|
+
flowerpower/job_queue/base.py,sha256=3QMs5E4tyLetUSBwBWtbDcxfSlG6PKyyy-hC5yz6cPM,12498
|
56
|
+
flowerpower/job_queue/apscheduler/__init__.py,sha256=CersYA0cZGPr_q-C1zVRsJJE8R7DXjK2HQpY8nsZ45s,235
|
57
|
+
flowerpower/job_queue/apscheduler/manager.py,sha256=P1EfC9WeZ9NuuX-NN00HvrNnKFtq7ESpshIivsMMWRk,36541
|
58
|
+
flowerpower/job_queue/apscheduler/setup.py,sha256=cjaJHOvn3BE-8XfqkPNvuVyvHB8yqalvcK2WY9xQIe8,18350
|
59
|
+
flowerpower/job_queue/apscheduler/trigger.py,sha256=kStQO_ipAn2xBbX3fSg4X3ORNOulalRyOhKBb0lkSAg,5111
|
60
|
+
flowerpower/job_queue/apscheduler/utils.py,sha256=ji-u98h3pF_ADCVSBIZIiYlxI3Zz3juYyMYA7WGVTWg,9312
|
61
|
+
flowerpower/job_queue/apscheduler/_setup/datastore.py,sha256=oV8pw7x0dZBDxhR54d-Y2O9hVanxhBuZbUNpxcizxak,3490
|
62
|
+
flowerpower/job_queue/apscheduler/_setup/eventbroker.py,sha256=qh8v9vAc-3v4e66wdXyrVZyIxFyqmOpzwKkolE8mVko,2872
|
63
|
+
flowerpower/job_queue/rq/__init__.py,sha256=m_vFSwmq7Cis-2mYhkDHxFaucSw7ODUfxWd3-hWZ50Y,193
|
64
|
+
flowerpower/job_queue/rq/_trigger.py,sha256=w1WKs_8nd9d4e_Y25yuG0vV2N4A58rWZz8RUORJepH0,1134
|
65
|
+
flowerpower/job_queue/rq/manager.py,sha256=7vduemVSaBdwb9_tVNOCbGn58fO5mFIPMLF28iR3lFI,51328
|
66
|
+
flowerpower/job_queue/rq/setup.py,sha256=fJPb7wztkefipulv4NQQjW3fEKFH3EMDaBLNVS8dkV8,5126
|
67
|
+
flowerpower/job_queue/rq/utils.py,sha256=QjyNhSM1_gMZkV2cO8eR99XeEji4AMwpxE1TANaRgTg,2009
|
68
|
+
flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py,sha256=X8rKfSbuITpWiwCyNqN-WVeodBkafFuj-zvbZb7z6lw,7511
|
69
|
+
flowerpower/job_queue/rq/concurrent_workers/thread_worker.py,sha256=M_jjci-pypEsbHe-gCQS6jmJBrA-Tb7W19sEWi6htFU,7801
|
70
|
+
flowerpower/pipeline/__init__.py,sha256=xbEn_RN0vVNqLZMSFOCdV41ggUkYrghFVJYd_EC0C44,75
|
71
|
+
flowerpower/pipeline/base.py,sha256=1srub-3gy2BSbdqyva8V6p5VljUDtutq-fXcQWKzFUs,3140
|
72
|
+
flowerpower/pipeline/io.py,sha256=rjj-V9wf0OJKxvDXgruKOZuCLkNqc56J9VcMegy5ols,15948
|
73
|
+
flowerpower/pipeline/job_queue.py,sha256=4LhOx2boUh7ET8-a6GfiJpVYHTKjmeRJtQrJ6GIi-Nc,22045
|
74
|
+
flowerpower/pipeline/manager.py,sha256=cWLX7CW01jv7Y4E0ofcRXmYtAf_qLjQZh1qaQyYeR0s,63907
|
75
|
+
flowerpower/pipeline/registry.py,sha256=fqTR2hKyJwxyUTgzLHeBP8aJdZENWd9Jz22KD052lTE,18892
|
76
|
+
flowerpower/pipeline/runner.py,sha256=tZ9WByCx7MtAWwovoEbEHscDvQffTmmfhtUN1cz463M,23603
|
77
|
+
flowerpower/pipeline/visualizer.py,sha256=amjMrl5NetErE198HzZBPWVZBi_t5jj9ydxWpuNLoTI,5013
|
78
|
+
flowerpower/plugins/mqtt/__init__.py,sha256=kTlWgXrSScSxLrSQdqQESi35OEDbCbO3LLkTIcI90jQ,252
|
79
|
+
flowerpower/plugins/mqtt/cfg.py,sha256=_0eEubHU3n6Ybd0LOX6vr21-l3nhoIHwu6g9OXpwdJE,483
|
80
|
+
flowerpower/plugins/mqtt/manager.py,sha256=klft9kKmj3P4Pq7EutlT38rViQ4mVXPO1vDJDSzJZrU,29525
|
81
|
+
flowerpower/utils/logging.py,sha256=cdvlA9uJTbzPFRNC8bB5yEHKtn2G6mrGQyuZleFPiNM,787
|
82
|
+
flowerpower/utils/misc.py,sha256=ILznXwiBKynLKhpALMkx8l9DNFHvWYNa8ezTj8lHmqw,15962
|
83
|
+
flowerpower/utils/monkey.py,sha256=VPl3yimoWhwD9kI05BFsjNvtyQiDyLfY4Q85Bb6Ma0w,2903
|
84
|
+
flowerpower/utils/open_telemetry.py,sha256=fQWJWbIQFtKIxMBjAWeF12NGnqT0isO3A3j-DSOv_vE,949
|
85
|
+
flowerpower/utils/polars.py,sha256=-b4SrVVtsS4g03rD1vdsd6vu7zQogua1p57fYGhe6vo,17739
|
86
|
+
flowerpower/utils/scheduler.py,sha256=2zJ_xmLXpvXUQNF1XS2Gqm3Ogo907ctZ50GtvQB_rhE,9354
|
87
|
+
flowerpower/utils/sql.py,sha256=wn10GJhfVg02uKppP0BY9g2DvdZbvynHJ4MPuKRUqbc,13085
|
88
|
+
flowerpower/utils/templates.py,sha256=5UsZUvxDaKMuoUURq4yoN8ijFytSvyrGrr-XDoLMDp8,1593
|
89
|
+
flowerpower/web/app.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
90
|
+
flowerpower-1.0.0b1.dist-info/METADATA,sha256=_syCgxe-qj2aFZm66TOMfr3KyNgzkhh4UUNuCH1HG3o,9530
|
91
|
+
flowerpower-1.0.0b1.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
|
92
|
+
flowerpower-1.0.0b1.dist-info/entry_points.txt,sha256=61X11i5a2IwC9LBiP20XCDl5zMOigGCjMCx17B7bDbQ,52
|
93
|
+
flowerpower-1.0.0b1.dist-info/top_level.txt,sha256=VraH4WtEUfSxs5L-rXwDQhzQb9eLHTUtgvmFZ2dAYnA,12
|
94
|
+
flowerpower-1.0.0b1.dist-info/RECORD,,
|
@@ -1,14 +0,0 @@
|
|
1
|
-
from munch import Munch, munchify
|
2
|
-
from pydantic import Field
|
3
|
-
|
4
|
-
from ..base import BaseConfig
|
5
|
-
|
6
|
-
|
7
|
-
class PipelineTrackerConfig(BaseConfig):
|
8
|
-
project_id: int | None = None
|
9
|
-
# version: str | None = None
|
10
|
-
dag_name: str | None = None
|
11
|
-
tags: dict | Munch = Field(default_factory=dict)
|
12
|
-
|
13
|
-
def model_post_init(self, __context):
|
14
|
-
self.tags = munchify(self.tags)
|
@@ -1,11 +0,0 @@
|
|
1
|
-
from pydantic import Field
|
2
|
-
|
3
|
-
from ..base import BaseConfig
|
4
|
-
|
5
|
-
|
6
|
-
class ProjectTrackerConfig(BaseConfig):
|
7
|
-
username: str | None = Field(default=None)
|
8
|
-
api_url: str = "http://localhost:8241"
|
9
|
-
ui_url: str = "http://localhost:8242"
|
10
|
-
api_key: str | None = Field(default=None)
|
11
|
-
verify: bool = False
|
@@ -1,19 +0,0 @@
|
|
1
|
-
import datetime as dt
|
2
|
-
|
3
|
-
from munch import Munch, munchify
|
4
|
-
from pydantic import Field
|
5
|
-
|
6
|
-
from ..base import BaseConfig
|
7
|
-
|
8
|
-
|
9
|
-
class ProjectWorkerConfig(BaseConfig):
|
10
|
-
data_store: dict | Munch = Field(default_factory=dict)
|
11
|
-
event_broker: dict | Munch = Field(default_factory=dict)
|
12
|
-
cleanup_interval: int | float | dt.timedelta = Field(default=300) # int in secods
|
13
|
-
max_concurrent_jobs: int = Field(default=10)
|
14
|
-
|
15
|
-
def model_post_init(self, __context):
|
16
|
-
if isinstance(self.data_store, dict):
|
17
|
-
self.data_store = munchify(self.data_store)
|
18
|
-
if isinstance(self.event_broker, dict):
|
19
|
-
self.event_broker = munchify(self.event_broker)
|