FlowerPower 0.11.6.19__py3-none-any.whl → 0.20.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/cfg/__init__.py +3 -3
- flowerpower/cfg/pipeline/__init__.py +5 -3
- flowerpower/cfg/project/__init__.py +3 -3
- flowerpower/cfg/project/job_queue.py +1 -128
- flowerpower/cli/__init__.py +5 -5
- flowerpower/cli/cfg.py +0 -3
- flowerpower/cli/job_queue.py +401 -133
- flowerpower/cli/pipeline.py +14 -413
- flowerpower/cli/utils.py +0 -1
- flowerpower/flowerpower.py +537 -28
- flowerpower/job_queue/__init__.py +5 -94
- flowerpower/job_queue/base.py +201 -3
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -3
- flowerpower/job_queue/rq/manager.py +388 -77
- flowerpower/pipeline/__init__.py +2 -0
- flowerpower/pipeline/base.py +2 -2
- flowerpower/pipeline/io.py +14 -16
- flowerpower/pipeline/manager.py +21 -642
- flowerpower/pipeline/pipeline.py +571 -0
- flowerpower/pipeline/registry.py +242 -10
- flowerpower/pipeline/visualizer.py +1 -2
- flowerpower/plugins/_io/__init__.py +8 -0
- flowerpower/plugins/mqtt/manager.py +6 -6
- flowerpower/settings/backend.py +0 -2
- flowerpower/settings/job_queue.py +1 -57
- flowerpower/utils/misc.py +0 -256
- flowerpower/utils/monkey.py +1 -83
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/METADATA +308 -152
- flowerpower-0.20.0.dist-info/RECORD +58 -0
- flowerpower/fs/__init__.py +0 -29
- flowerpower/fs/base.py +0 -662
- flowerpower/fs/ext.py +0 -2143
- flowerpower/fs/storage_options.py +0 -1420
- flowerpower/job_queue/apscheduler/__init__.py +0 -11
- flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
- flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
- flowerpower/job_queue/apscheduler/manager.py +0 -1051
- flowerpower/job_queue/apscheduler/setup.py +0 -554
- flowerpower/job_queue/apscheduler/trigger.py +0 -169
- flowerpower/job_queue/apscheduler/utils.py +0 -311
- flowerpower/pipeline/job_queue.py +0 -583
- flowerpower/pipeline/runner.py +0 -603
- flowerpower/plugins/io/base.py +0 -2520
- flowerpower/plugins/io/helpers/datetime.py +0 -298
- flowerpower/plugins/io/helpers/polars.py +0 -875
- flowerpower/plugins/io/helpers/pyarrow.py +0 -570
- flowerpower/plugins/io/helpers/sql.py +0 -202
- flowerpower/plugins/io/loader/__init__.py +0 -28
- flowerpower/plugins/io/loader/csv.py +0 -37
- flowerpower/plugins/io/loader/deltatable.py +0 -190
- flowerpower/plugins/io/loader/duckdb.py +0 -19
- flowerpower/plugins/io/loader/json.py +0 -37
- flowerpower/plugins/io/loader/mqtt.py +0 -159
- flowerpower/plugins/io/loader/mssql.py +0 -26
- flowerpower/plugins/io/loader/mysql.py +0 -26
- flowerpower/plugins/io/loader/oracle.py +0 -26
- flowerpower/plugins/io/loader/parquet.py +0 -35
- flowerpower/plugins/io/loader/postgres.py +0 -26
- flowerpower/plugins/io/loader/pydala.py +0 -19
- flowerpower/plugins/io/loader/sqlite.py +0 -23
- flowerpower/plugins/io/metadata.py +0 -244
- flowerpower/plugins/io/saver/__init__.py +0 -28
- flowerpower/plugins/io/saver/csv.py +0 -36
- flowerpower/plugins/io/saver/deltatable.py +0 -186
- flowerpower/plugins/io/saver/duckdb.py +0 -19
- flowerpower/plugins/io/saver/json.py +0 -36
- flowerpower/plugins/io/saver/mqtt.py +0 -28
- flowerpower/plugins/io/saver/mssql.py +0 -26
- flowerpower/plugins/io/saver/mysql.py +0 -26
- flowerpower/plugins/io/saver/oracle.py +0 -26
- flowerpower/plugins/io/saver/parquet.py +0 -36
- flowerpower/plugins/io/saver/postgres.py +0 -26
- flowerpower/plugins/io/saver/pydala.py +0 -20
- flowerpower/plugins/io/saver/sqlite.py +0 -24
- flowerpower/utils/scheduler.py +0 -311
- flowerpower-0.11.6.19.dist-info/RECORD +0 -102
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/WHEEL +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/licenses/LICENSE +0 -0
- {flowerpower-0.11.6.19.dist-info → flowerpower-0.20.0.dist-info}/top_level.txt +0 -0
@@ -1,110 +0,0 @@
|
|
1
|
-
from apscheduler.datastores.base import BaseDataStore
|
2
|
-
from sqlalchemy import text
|
3
|
-
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
4
|
-
|
5
|
-
from ...base import BackendType, BaseBackend
|
6
|
-
|
7
|
-
|
8
|
-
class APSDataStoreType(BackendType):
|
9
|
-
POSTGRESQL = "postgresql"
|
10
|
-
SQLITE = "sqlite"
|
11
|
-
MYSQL = "mysql"
|
12
|
-
MONGODB = "mongodb"
|
13
|
-
MEMORY = "memory"
|
14
|
-
|
15
|
-
|
16
|
-
class APSDataStore(BaseBackend):
|
17
|
-
"""Data store for APScheduler."""
|
18
|
-
|
19
|
-
def __post_init__(self):
|
20
|
-
super().__post_init__(backend_type=APSDataStoreType)
|
21
|
-
self._validate_inputs()
|
22
|
-
|
23
|
-
@classmethod
|
24
|
-
def from_dict(cls, d: dict[str, any]) -> "APSDataStore":
|
25
|
-
return cls(**d)
|
26
|
-
|
27
|
-
def _validate_inputs(self) -> None:
|
28
|
-
if self.type.value not in [ds.value for ds in APSDataStoreType]:
|
29
|
-
raise ValueError(
|
30
|
-
f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSDataStoreType]}"
|
31
|
-
)
|
32
|
-
|
33
|
-
async def _setup_db(self) -> None:
|
34
|
-
sqla_engine = create_async_engine(self.uri)
|
35
|
-
|
36
|
-
try:
|
37
|
-
await self._create_schema(sqla_engine)
|
38
|
-
except Exception:
|
39
|
-
await self._create_database_and_schema(sqla_engine)
|
40
|
-
|
41
|
-
async def _create_schema(self, engine: AsyncEngine) -> None:
|
42
|
-
if not self.schema_or_queue:
|
43
|
-
return
|
44
|
-
|
45
|
-
async with engine.begin() as conn:
|
46
|
-
await conn.execute(
|
47
|
-
text(f"CREATE SCHEMA IF NOT EXISTS {self.schema_or_queue}")
|
48
|
-
)
|
49
|
-
await conn.commit()
|
50
|
-
|
51
|
-
async def _create_database_and_schema(self, engine: AsyncEngine) -> None:
|
52
|
-
database_name = self.uri.split("/")[-1].split("?")[0]
|
53
|
-
temp_uri = self.uri.replace(f"/{database_name}", "/template1")
|
54
|
-
temp_engine = create_async_engine(temp_uri)
|
55
|
-
|
56
|
-
async with temp_engine.begin() as conn:
|
57
|
-
await conn.execute(text("COMMIT"))
|
58
|
-
try:
|
59
|
-
await conn.execute(text(f"CREATE DATABASE {database_name}"))
|
60
|
-
finally:
|
61
|
-
await conn.execute(text("COMMIT"))
|
62
|
-
|
63
|
-
if self.schema_or_queue:
|
64
|
-
await self._create_schema(engine)
|
65
|
-
|
66
|
-
def setup_db(self) -> None:
|
67
|
-
from anyio.from_thread import start_blocking_portal
|
68
|
-
|
69
|
-
with start_blocking_portal() as portal:
|
70
|
-
portal.call(self._setup_db)
|
71
|
-
|
72
|
-
def _setup_sqlalchemy(self) -> None:
|
73
|
-
from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore
|
74
|
-
|
75
|
-
if not self.type.is_sqlite_type:
|
76
|
-
self.setup_db()
|
77
|
-
self._sqla_engine = create_async_engine(self.uri)
|
78
|
-
self._client = SQLAlchemyDataStore(
|
79
|
-
self._sqla_engine, schema=self.schema_or_queue
|
80
|
-
)
|
81
|
-
|
82
|
-
def _setup_mongodb(self) -> None:
|
83
|
-
from apscheduler.datastores.mongodb import MongoDBDataStore
|
84
|
-
|
85
|
-
self._client = MongoDBDataStore(self.uri, database=self.schema_or_queue)
|
86
|
-
|
87
|
-
def _setup_memory(self) -> None:
|
88
|
-
from apscheduler.datastores.memory import MemoryDataStore
|
89
|
-
|
90
|
-
self._client = MemoryDataStore()
|
91
|
-
|
92
|
-
def setup(self) -> None:
|
93
|
-
if self.type.is_sqla_type:
|
94
|
-
self._setup_sqlalchemy()
|
95
|
-
elif self.type.is_mongodb_type:
|
96
|
-
self._setup_mongodb()
|
97
|
-
else:
|
98
|
-
self._setup_memory()
|
99
|
-
|
100
|
-
@property
|
101
|
-
def client(self) -> BaseDataStore:
|
102
|
-
if self._client is None:
|
103
|
-
self.setup()
|
104
|
-
return self._client
|
105
|
-
|
106
|
-
@property
|
107
|
-
def sqla_engine(self) -> AsyncEngine | None:
|
108
|
-
if self._sqla_engine is None:
|
109
|
-
self.setup()
|
110
|
-
return self._sqla_engine
|
@@ -1,93 +0,0 @@
|
|
1
|
-
from apscheduler.eventbrokers.base import BaseEventBroker
|
2
|
-
from sqlalchemy.ext.asyncio import AsyncEngine
|
3
|
-
|
4
|
-
from ...base import BackendType, BaseBackend
|
5
|
-
|
6
|
-
|
7
|
-
class APSEventBrokerType(BackendType):
|
8
|
-
POSTGRESQL = "postgresql"
|
9
|
-
MEMORY = "memory"
|
10
|
-
REDIS = "redis"
|
11
|
-
MQTT = "mqtt"
|
12
|
-
|
13
|
-
|
14
|
-
class APSEventBroker(BaseBackend):
|
15
|
-
"""Data store for APScheduler."""
|
16
|
-
|
17
|
-
def __post_init__(self):
|
18
|
-
super().__post_init__(backend_type=APSEventBrokerType)
|
19
|
-
|
20
|
-
@classmethod
|
21
|
-
def from_dict(cls, d: dict[str, any]) -> "APSEventBroker":
|
22
|
-
return cls(**d)
|
23
|
-
|
24
|
-
def _validate_inputs(self) -> None:
|
25
|
-
if self.type.value not in [ds.value for ds in APSEventBrokerType]:
|
26
|
-
raise ValueError(
|
27
|
-
f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSEventBrokerType]}"
|
28
|
-
)
|
29
|
-
|
30
|
-
def _setup_asyncpg_event_broker(self):
|
31
|
-
from apscheduler.eventbrokers.asyncpg import AsyncpgEventBroker
|
32
|
-
|
33
|
-
if self._sqla_engine is None:
|
34
|
-
self._event_broker = AsyncpgEventBroker.from_dsn(dsn=self.uri)
|
35
|
-
else:
|
36
|
-
self._event_broker = AsyncpgEventBroker.from_async_sqla_engine(
|
37
|
-
engine=self._sqla_engine
|
38
|
-
)
|
39
|
-
|
40
|
-
def _setup_mqtt_event_broker(self):
|
41
|
-
import urllib.parse
|
42
|
-
|
43
|
-
from apscheduler.eventbrokers.mqtt import MQTTEventBroker
|
44
|
-
|
45
|
-
# Parse the URI
|
46
|
-
parsed = urllib.parse.urlparse(self.uri)
|
47
|
-
|
48
|
-
hostname = parsed.hostname
|
49
|
-
port = parsed.port
|
50
|
-
username = parsed.username
|
51
|
-
password = parsed.password
|
52
|
-
use_ssl = parsed.scheme == "mqtts"
|
53
|
-
|
54
|
-
self._event_broker = MQTTEventBroker(
|
55
|
-
host=hostname, port=port, ssl=use_ssl, topic="flowerpower/scheduler"
|
56
|
-
)
|
57
|
-
if (self.username is not None) and (self.password is not None):
|
58
|
-
self._event_broker._client.username_pw_set(
|
59
|
-
username,
|
60
|
-
password,
|
61
|
-
)
|
62
|
-
|
63
|
-
def _setup_redis_event_broker(self):
|
64
|
-
from apscheduler.eventbrokers.redis import RedisEventBroker
|
65
|
-
|
66
|
-
self._event_broker = RedisEventBroker(self.uri)
|
67
|
-
|
68
|
-
def _setup_local_event_broker(self):
|
69
|
-
from apscheduler.eventbrokers.local import LocalEventBroker
|
70
|
-
|
71
|
-
self._event_broker = LocalEventBroker()
|
72
|
-
|
73
|
-
def setup(self):
|
74
|
-
if self.is_sqla_type:
|
75
|
-
self._setup_asyncpg_event_broker()
|
76
|
-
elif self.is_mqtt_type:
|
77
|
-
self._setup_mqtt_event_broker()
|
78
|
-
elif self.is_redis_type:
|
79
|
-
self._setup_redis_event_broker()
|
80
|
-
else:
|
81
|
-
self._setup_local_event_broker()
|
82
|
-
|
83
|
-
@property
|
84
|
-
def client(self) -> BaseEventBroker:
|
85
|
-
if self._event_broker is None:
|
86
|
-
self.setup()
|
87
|
-
return self._event_broker
|
88
|
-
|
89
|
-
@property
|
90
|
-
def sqla_engine(self) -> AsyncEngine | None:
|
91
|
-
if self._sqla_engine is None:
|
92
|
-
self.setup()
|
93
|
-
return self._sqla_engine
|