FlowerPower 0.11.6.20__py3-none-any.whl → 0.21.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/__init__.py +2 -6
- flowerpower/cfg/__init__.py +7 -14
- flowerpower/cfg/base.py +29 -25
- flowerpower/cfg/pipeline/__init__.py +8 -6
- flowerpower/cfg/pipeline/_schedule.py +32 -0
- flowerpower/cfg/pipeline/adapter.py +0 -5
- flowerpower/cfg/pipeline/builder.py +377 -0
- flowerpower/cfg/pipeline/run.py +36 -0
- flowerpower/cfg/project/__init__.py +11 -24
- flowerpower/cfg/project/adapter.py +0 -12
- flowerpower/cli/__init__.py +2 -21
- flowerpower/cli/cfg.py +0 -3
- flowerpower/cli/mqtt.py +0 -6
- flowerpower/cli/pipeline.py +22 -415
- flowerpower/cli/utils.py +0 -1
- flowerpower/flowerpower.py +345 -146
- flowerpower/pipeline/__init__.py +2 -0
- flowerpower/pipeline/base.py +21 -12
- flowerpower/pipeline/io.py +58 -54
- flowerpower/pipeline/manager.py +165 -726
- flowerpower/pipeline/pipeline.py +643 -0
- flowerpower/pipeline/registry.py +285 -18
- flowerpower/pipeline/visualizer.py +5 -6
- flowerpower/plugins/io/__init__.py +8 -0
- flowerpower/plugins/mqtt/__init__.py +7 -11
- flowerpower/settings/__init__.py +0 -2
- flowerpower/settings/{backend.py → _backend.py} +0 -21
- flowerpower/settings/logging.py +1 -1
- flowerpower/utils/logging.py +24 -12
- flowerpower/utils/misc.py +17 -256
- flowerpower/utils/monkey.py +1 -83
- flowerpower-0.21.0.dist-info/METADATA +463 -0
- flowerpower-0.21.0.dist-info/RECORD +44 -0
- flowerpower/cfg/pipeline/schedule.py +0 -74
- flowerpower/cfg/project/job_queue.py +0 -238
- flowerpower/cli/job_queue.py +0 -1061
- flowerpower/fs/__init__.py +0 -29
- flowerpower/fs/base.py +0 -662
- flowerpower/fs/ext.py +0 -2143
- flowerpower/fs/storage_options.py +0 -1420
- flowerpower/job_queue/__init__.py +0 -294
- flowerpower/job_queue/apscheduler/__init__.py +0 -11
- flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
- flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
- flowerpower/job_queue/apscheduler/manager.py +0 -1051
- flowerpower/job_queue/apscheduler/setup.py +0 -554
- flowerpower/job_queue/apscheduler/trigger.py +0 -169
- flowerpower/job_queue/apscheduler/utils.py +0 -311
- flowerpower/job_queue/base.py +0 -413
- flowerpower/job_queue/rq/__init__.py +0 -10
- flowerpower/job_queue/rq/_trigger.py +0 -37
- flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -226
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -231
- flowerpower/job_queue/rq/manager.py +0 -1582
- flowerpower/job_queue/rq/setup.py +0 -154
- flowerpower/job_queue/rq/utils.py +0 -69
- flowerpower/mqtt.py +0 -12
- flowerpower/pipeline/job_queue.py +0 -583
- flowerpower/pipeline/runner.py +0 -603
- flowerpower/plugins/io/base.py +0 -2520
- flowerpower/plugins/io/helpers/datetime.py +0 -298
- flowerpower/plugins/io/helpers/polars.py +0 -875
- flowerpower/plugins/io/helpers/pyarrow.py +0 -570
- flowerpower/plugins/io/helpers/sql.py +0 -202
- flowerpower/plugins/io/loader/__init__.py +0 -28
- flowerpower/plugins/io/loader/csv.py +0 -37
- flowerpower/plugins/io/loader/deltatable.py +0 -190
- flowerpower/plugins/io/loader/duckdb.py +0 -19
- flowerpower/plugins/io/loader/json.py +0 -37
- flowerpower/plugins/io/loader/mqtt.py +0 -159
- flowerpower/plugins/io/loader/mssql.py +0 -26
- flowerpower/plugins/io/loader/mysql.py +0 -26
- flowerpower/plugins/io/loader/oracle.py +0 -26
- flowerpower/plugins/io/loader/parquet.py +0 -35
- flowerpower/plugins/io/loader/postgres.py +0 -26
- flowerpower/plugins/io/loader/pydala.py +0 -19
- flowerpower/plugins/io/loader/sqlite.py +0 -23
- flowerpower/plugins/io/metadata.py +0 -244
- flowerpower/plugins/io/saver/__init__.py +0 -28
- flowerpower/plugins/io/saver/csv.py +0 -36
- flowerpower/plugins/io/saver/deltatable.py +0 -186
- flowerpower/plugins/io/saver/duckdb.py +0 -19
- flowerpower/plugins/io/saver/json.py +0 -36
- flowerpower/plugins/io/saver/mqtt.py +0 -28
- flowerpower/plugins/io/saver/mssql.py +0 -26
- flowerpower/plugins/io/saver/mysql.py +0 -26
- flowerpower/plugins/io/saver/oracle.py +0 -26
- flowerpower/plugins/io/saver/parquet.py +0 -36
- flowerpower/plugins/io/saver/postgres.py +0 -26
- flowerpower/plugins/io/saver/pydala.py +0 -20
- flowerpower/plugins/io/saver/sqlite.py +0 -24
- flowerpower/plugins/mqtt/cfg.py +0 -17
- flowerpower/plugins/mqtt/manager.py +0 -962
- flowerpower/settings/job_queue.py +0 -87
- flowerpower/utils/scheduler.py +0 -311
- flowerpower-0.11.6.20.dist-info/METADATA +0 -537
- flowerpower-0.11.6.20.dist-info/RECORD +0 -102
- {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/WHEEL +0 -0
- {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/licenses/LICENSE +0 -0
- {flowerpower-0.11.6.20.dist-info → flowerpower-0.21.0.dist-info}/top_level.txt +0 -0
@@ -1,36 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseFileWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class JsonFileWriter(BaseFileWriter, gc=False):
|
8
|
-
"""JSON file writer.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to JSON files.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = JsonFileWriter(df, "data.json")
|
15
|
-
writer.write()
|
16
|
-
```
|
17
|
-
"""
|
18
|
-
|
19
|
-
format: str = field(default="json")
|
20
|
-
|
21
|
-
|
22
|
-
# @attrs.define
|
23
|
-
class JsonDatasetWriter(BaseFileWriter, gc=False):
|
24
|
-
"""JSON dataset writer.
|
25
|
-
|
26
|
-
This class is responsible for writing dataframes to JSON dataset.
|
27
|
-
|
28
|
-
Examples:
|
29
|
-
```python
|
30
|
-
writer = JsonDatasetWriter([df1, df2], "json_data/")
|
31
|
-
writer.write()
|
32
|
-
```
|
33
|
-
|
34
|
-
"""
|
35
|
-
|
36
|
-
format: str = field(default="json")
|
@@ -1,28 +0,0 @@
|
|
1
|
-
import msgspec
|
2
|
-
|
3
|
-
|
4
|
-
class MQTTWriter(msgspec.Struct):
|
5
|
-
"""MQTT writer.
|
6
|
-
|
7
|
-
This class is responsible for writing dataframes to MQTT broker.
|
8
|
-
|
9
|
-
Examples:
|
10
|
-
```python
|
11
|
-
writer = MQTTWriter(broker="localhost", port=1883, topic="data")
|
12
|
-
writer.write(df)
|
13
|
-
```
|
14
|
-
"""
|
15
|
-
|
16
|
-
broker: str
|
17
|
-
port: int = 1883
|
18
|
-
topic: str
|
19
|
-
username: str | None = None
|
20
|
-
password: str | None = None
|
21
|
-
|
22
|
-
def __post_init__(self):
|
23
|
-
pass
|
24
|
-
|
25
|
-
def write(self, data):
|
26
|
-
"""Write data to MQTT broker."""
|
27
|
-
# Implementation would go here
|
28
|
-
pass
|
@@ -1,26 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseDatabaseWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class MSSQLWriter(BaseDatabaseWriter, gc=False):
|
8
|
-
"""MSSQL writer.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to MsSQL database.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = MSSQLWriter(table_name="table", host="localhost",
|
15
|
-
port=5432, username="user", password="password",
|
16
|
-
database="database")
|
17
|
-
writer.write(df)
|
18
|
-
|
19
|
-
# or
|
20
|
-
writer = MSSQLWriter(table_name="table",
|
21
|
-
connection_string="mssql+pyodbc://user:password@localhost:5432/database")
|
22
|
-
writer.write(df)
|
23
|
-
```
|
24
|
-
"""
|
25
|
-
|
26
|
-
type_: str = field(default="mssql")
|
@@ -1,26 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseDatabaseWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class MySQLWriter(BaseDatabaseWriter, gc=False):
|
8
|
-
"""MySQL writer.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to MySQL database.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = MySQLWriter(table_name="table", host="localhost",
|
15
|
-
port=5432, username="user", password="password",
|
16
|
-
database="database")
|
17
|
-
writer.write(df)
|
18
|
-
|
19
|
-
# or
|
20
|
-
writer = MySQLWriter(table_name="table",
|
21
|
-
connection_string="mysql+pymsql://user:password@localhost:5432/database")
|
22
|
-
writer.write(df)
|
23
|
-
```
|
24
|
-
"""
|
25
|
-
|
26
|
-
type_: str = field(default="mysql")
|
@@ -1,26 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseDatabaseWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class OracleDBWriter(BaseDatabaseWriter, gc=False):
|
8
|
-
"""OracleDB writer.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to OracleDB database.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = OracleDBWriter(table_name="table", host="localhost",
|
15
|
-
port=5432, username="user", password="password",
|
16
|
-
database="database")
|
17
|
-
writer.write(df)
|
18
|
-
|
19
|
-
# or
|
20
|
-
writer = OracleDBWriter(table_name="table",
|
21
|
-
connection_string="mysql+pymsql://user:password@localhost:5432/database")
|
22
|
-
writer.write(df)
|
23
|
-
```
|
24
|
-
"""
|
25
|
-
|
26
|
-
type_: str = field(default="oracle")
|
@@ -1,36 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseDatasetWriter, BaseFileWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class ParquetFileWriter(BaseFileWriter, gc=False):
|
8
|
-
"""Parquet file writer.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to Parquet files.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = ParquetFileWriter(df, "data.parquet")
|
15
|
-
writer.write()
|
16
|
-
```
|
17
|
-
"""
|
18
|
-
|
19
|
-
format: str = field(default="parquet")
|
20
|
-
|
21
|
-
|
22
|
-
# @attrs.define
|
23
|
-
class ParquetDatasetWriter(BaseDatasetWriter, gc=False):
|
24
|
-
"""Parquet dataset writer.
|
25
|
-
|
26
|
-
This class is responsible for writing dataframes to Parquet dataset.
|
27
|
-
|
28
|
-
Examples:
|
29
|
-
```python
|
30
|
-
writer = ParquetDatasetWriter(df, "parquet_data/")
|
31
|
-
writer.write()
|
32
|
-
```
|
33
|
-
|
34
|
-
"""
|
35
|
-
|
36
|
-
format: str = field(default="parquet")
|
@@ -1,26 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseDatabaseWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class PostgreSQLWriter(BaseDatabaseWriter, gc=False):
|
8
|
-
"""PostgreSQL writer.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to PostgreSQL database.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = PostgreSQLWriter(table_name="table", host="localhost",
|
15
|
-
port=5432, username="user", password="password",
|
16
|
-
database="database")
|
17
|
-
writer.write(df)
|
18
|
-
|
19
|
-
# or
|
20
|
-
writer = PostgreSQLWriter(table_name="table",
|
21
|
-
connection_string="postgresql://user:password@localhost:5432/database")
|
22
|
-
writer.write(df)
|
23
|
-
```
|
24
|
-
"""
|
25
|
-
|
26
|
-
type_: str = field(default="postgres")
|
@@ -1,20 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseDatasetWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class PydalaDatasetWriter(BaseDatasetWriter, gc=False):
|
8
|
-
"""Writer for Pydala dataset.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to Pydala dataset.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = PydalaDatasetWriter(path="pydala_data/")
|
15
|
-
writer.write(df)
|
16
|
-
```
|
17
|
-
"""
|
18
|
-
|
19
|
-
format: str = field(default="parquet")
|
20
|
-
is_pydala_dataset: bool = field(default=True)
|
@@ -1,24 +0,0 @@
|
|
1
|
-
from msgspec import field
|
2
|
-
|
3
|
-
from ..base import BaseDatabaseWriter
|
4
|
-
|
5
|
-
|
6
|
-
# @attrs.define
|
7
|
-
class SQLiteWriter(BaseDatabaseWriter, gc=False):
|
8
|
-
"""SQLite writer.
|
9
|
-
|
10
|
-
This class is responsible for writing dataframes to SQLite database.
|
11
|
-
|
12
|
-
Examples:
|
13
|
-
```python
|
14
|
-
writer = SQLiteWriter(table_name="table", path="data.db")
|
15
|
-
writer.write(df)
|
16
|
-
|
17
|
-
# or
|
18
|
-
writer = SQLiteWriter(table_name="table",
|
19
|
-
connection_string="sqkite:///data.db")
|
20
|
-
writer.write(df)
|
21
|
-
```
|
22
|
-
"""
|
23
|
-
|
24
|
-
type_: str = field(default="sqlite")
|
flowerpower/plugins/mqtt/cfg.py
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
from ...cfg.base import BaseConfig
|
2
|
-
|
3
|
-
|
4
|
-
class MqttConfig(BaseConfig):
|
5
|
-
username: str | None = None
|
6
|
-
password: str | None = None
|
7
|
-
host: str | None = "localhost"
|
8
|
-
port: int | None = 1883
|
9
|
-
topic: str | None = None
|
10
|
-
first_reconnect_delay: int = 1
|
11
|
-
max_reconnect_count: int = 5
|
12
|
-
reconnect_rate: int = 2
|
13
|
-
max_reconnect_delay: int = 60
|
14
|
-
transport: str = "tcp"
|
15
|
-
clean_session: bool = True
|
16
|
-
client_id: str | None = None
|
17
|
-
client_id_suffix: str | None = None
|