FlowerPower 0.11.6.20__py3-none-any.whl → 0.20.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. flowerpower/cfg/__init__.py +3 -3
  2. flowerpower/cfg/pipeline/__init__.py +5 -3
  3. flowerpower/cfg/project/__init__.py +3 -3
  4. flowerpower/cfg/project/job_queue.py +1 -128
  5. flowerpower/cli/__init__.py +5 -5
  6. flowerpower/cli/cfg.py +0 -3
  7. flowerpower/cli/job_queue.py +400 -132
  8. flowerpower/cli/pipeline.py +14 -413
  9. flowerpower/cli/utils.py +0 -1
  10. flowerpower/flowerpower.py +537 -28
  11. flowerpower/job_queue/__init__.py +5 -94
  12. flowerpower/job_queue/base.py +201 -3
  13. flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -3
  14. flowerpower/job_queue/rq/manager.py +388 -77
  15. flowerpower/pipeline/__init__.py +2 -0
  16. flowerpower/pipeline/base.py +2 -2
  17. flowerpower/pipeline/io.py +14 -16
  18. flowerpower/pipeline/manager.py +21 -642
  19. flowerpower/pipeline/pipeline.py +571 -0
  20. flowerpower/pipeline/registry.py +242 -10
  21. flowerpower/pipeline/visualizer.py +1 -2
  22. flowerpower/plugins/_io/__init__.py +8 -0
  23. flowerpower/plugins/mqtt/manager.py +6 -6
  24. flowerpower/settings/backend.py +0 -2
  25. flowerpower/settings/job_queue.py +1 -57
  26. flowerpower/utils/misc.py +0 -256
  27. flowerpower/utils/monkey.py +1 -83
  28. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/METADATA +308 -152
  29. flowerpower-0.20.0.dist-info/RECORD +58 -0
  30. flowerpower/fs/__init__.py +0 -29
  31. flowerpower/fs/base.py +0 -662
  32. flowerpower/fs/ext.py +0 -2143
  33. flowerpower/fs/storage_options.py +0 -1420
  34. flowerpower/job_queue/apscheduler/__init__.py +0 -11
  35. flowerpower/job_queue/apscheduler/_setup/datastore.py +0 -110
  36. flowerpower/job_queue/apscheduler/_setup/eventbroker.py +0 -93
  37. flowerpower/job_queue/apscheduler/manager.py +0 -1051
  38. flowerpower/job_queue/apscheduler/setup.py +0 -554
  39. flowerpower/job_queue/apscheduler/trigger.py +0 -169
  40. flowerpower/job_queue/apscheduler/utils.py +0 -311
  41. flowerpower/pipeline/job_queue.py +0 -583
  42. flowerpower/pipeline/runner.py +0 -603
  43. flowerpower/plugins/io/base.py +0 -2520
  44. flowerpower/plugins/io/helpers/datetime.py +0 -298
  45. flowerpower/plugins/io/helpers/polars.py +0 -875
  46. flowerpower/plugins/io/helpers/pyarrow.py +0 -570
  47. flowerpower/plugins/io/helpers/sql.py +0 -202
  48. flowerpower/plugins/io/loader/__init__.py +0 -28
  49. flowerpower/plugins/io/loader/csv.py +0 -37
  50. flowerpower/plugins/io/loader/deltatable.py +0 -190
  51. flowerpower/plugins/io/loader/duckdb.py +0 -19
  52. flowerpower/plugins/io/loader/json.py +0 -37
  53. flowerpower/plugins/io/loader/mqtt.py +0 -159
  54. flowerpower/plugins/io/loader/mssql.py +0 -26
  55. flowerpower/plugins/io/loader/mysql.py +0 -26
  56. flowerpower/plugins/io/loader/oracle.py +0 -26
  57. flowerpower/plugins/io/loader/parquet.py +0 -35
  58. flowerpower/plugins/io/loader/postgres.py +0 -26
  59. flowerpower/plugins/io/loader/pydala.py +0 -19
  60. flowerpower/plugins/io/loader/sqlite.py +0 -23
  61. flowerpower/plugins/io/metadata.py +0 -244
  62. flowerpower/plugins/io/saver/__init__.py +0 -28
  63. flowerpower/plugins/io/saver/csv.py +0 -36
  64. flowerpower/plugins/io/saver/deltatable.py +0 -186
  65. flowerpower/plugins/io/saver/duckdb.py +0 -19
  66. flowerpower/plugins/io/saver/json.py +0 -36
  67. flowerpower/plugins/io/saver/mqtt.py +0 -28
  68. flowerpower/plugins/io/saver/mssql.py +0 -26
  69. flowerpower/plugins/io/saver/mysql.py +0 -26
  70. flowerpower/plugins/io/saver/oracle.py +0 -26
  71. flowerpower/plugins/io/saver/parquet.py +0 -36
  72. flowerpower/plugins/io/saver/postgres.py +0 -26
  73. flowerpower/plugins/io/saver/pydala.py +0 -20
  74. flowerpower/plugins/io/saver/sqlite.py +0 -24
  75. flowerpower/utils/scheduler.py +0 -311
  76. flowerpower-0.11.6.20.dist-info/RECORD +0 -102
  77. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/WHEEL +0 -0
  78. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/entry_points.txt +0 -0
  79. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/licenses/LICENSE +0 -0
  80. {flowerpower-0.11.6.20.dist-info → flowerpower-0.20.0.dist-info}/top_level.txt +0 -0
@@ -1,11 +0,0 @@
1
- from .manager import APSManager
2
- from .setup import APSBackend, APSDataStore, APSEventBroker
3
- from .trigger import APSTrigger
4
-
5
- __all__ = [
6
- "APSManager",
7
- "APSTrigger",
8
- "APSBackend",
9
- "APSDataStore",
10
- "APSEventBroker",
11
- ]
@@ -1,110 +0,0 @@
1
- from apscheduler.datastores.base import BaseDataStore
2
- from sqlalchemy import text
3
- from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
4
-
5
- from ...base import BackendType, BaseBackend
6
-
7
-
8
- class APSDataStoreType(BackendType):
9
- POSTGRESQL = "postgresql"
10
- SQLITE = "sqlite"
11
- MYSQL = "mysql"
12
- MONGODB = "mongodb"
13
- MEMORY = "memory"
14
-
15
-
16
- class APSDataStore(BaseBackend):
17
- """Data store for APScheduler."""
18
-
19
- def __post_init__(self):
20
- super().__post_init__(backend_type=APSDataStoreType)
21
- self._validate_inputs()
22
-
23
- @classmethod
24
- def from_dict(cls, d: dict[str, any]) -> "APSDataStore":
25
- return cls(**d)
26
-
27
- def _validate_inputs(self) -> None:
28
- if self.type.value not in [ds.value for ds in APSDataStoreType]:
29
- raise ValueError(
30
- f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSDataStoreType]}"
31
- )
32
-
33
- async def _setup_db(self) -> None:
34
- sqla_engine = create_async_engine(self.uri)
35
-
36
- try:
37
- await self._create_schema(sqla_engine)
38
- except Exception:
39
- await self._create_database_and_schema(sqla_engine)
40
-
41
- async def _create_schema(self, engine: AsyncEngine) -> None:
42
- if not self.schema_or_queue:
43
- return
44
-
45
- async with engine.begin() as conn:
46
- await conn.execute(
47
- text(f"CREATE SCHEMA IF NOT EXISTS {self.schema_or_queue}")
48
- )
49
- await conn.commit()
50
-
51
- async def _create_database_and_schema(self, engine: AsyncEngine) -> None:
52
- database_name = self.uri.split("/")[-1].split("?")[0]
53
- temp_uri = self.uri.replace(f"/{database_name}", "/template1")
54
- temp_engine = create_async_engine(temp_uri)
55
-
56
- async with temp_engine.begin() as conn:
57
- await conn.execute(text("COMMIT"))
58
- try:
59
- await conn.execute(text(f"CREATE DATABASE {database_name}"))
60
- finally:
61
- await conn.execute(text("COMMIT"))
62
-
63
- if self.schema_or_queue:
64
- await self._create_schema(engine)
65
-
66
- def setup_db(self) -> None:
67
- from anyio.from_thread import start_blocking_portal
68
-
69
- with start_blocking_portal() as portal:
70
- portal.call(self._setup_db)
71
-
72
- def _setup_sqlalchemy(self) -> None:
73
- from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore
74
-
75
- if not self.type.is_sqlite_type:
76
- self.setup_db()
77
- self._sqla_engine = create_async_engine(self.uri)
78
- self._client = SQLAlchemyDataStore(
79
- self._sqla_engine, schema=self.schema_or_queue
80
- )
81
-
82
- def _setup_mongodb(self) -> None:
83
- from apscheduler.datastores.mongodb import MongoDBDataStore
84
-
85
- self._client = MongoDBDataStore(self.uri, database=self.schema_or_queue)
86
-
87
- def _setup_memory(self) -> None:
88
- from apscheduler.datastores.memory import MemoryDataStore
89
-
90
- self._client = MemoryDataStore()
91
-
92
- def setup(self) -> None:
93
- if self.type.is_sqla_type:
94
- self._setup_sqlalchemy()
95
- elif self.type.is_mongodb_type:
96
- self._setup_mongodb()
97
- else:
98
- self._setup_memory()
99
-
100
- @property
101
- def client(self) -> BaseDataStore:
102
- if self._client is None:
103
- self.setup()
104
- return self._client
105
-
106
- @property
107
- def sqla_engine(self) -> AsyncEngine | None:
108
- if self._sqla_engine is None:
109
- self.setup()
110
- return self._sqla_engine
@@ -1,93 +0,0 @@
1
- from apscheduler.eventbrokers.base import BaseEventBroker
2
- from sqlalchemy.ext.asyncio import AsyncEngine
3
-
4
- from ...base import BackendType, BaseBackend
5
-
6
-
7
- class APSEventBrokerType(BackendType):
8
- POSTGRESQL = "postgresql"
9
- MEMORY = "memory"
10
- REDIS = "redis"
11
- MQTT = "mqtt"
12
-
13
-
14
- class APSEventBroker(BaseBackend):
15
- """Data store for APScheduler."""
16
-
17
- def __post_init__(self):
18
- super().__post_init__(backend_type=APSEventBrokerType)
19
-
20
- @classmethod
21
- def from_dict(cls, d: dict[str, any]) -> "APSEventBroker":
22
- return cls(**d)
23
-
24
- def _validate_inputs(self) -> None:
25
- if self.type.value not in [ds.value for ds in APSEventBrokerType]:
26
- raise ValueError(
27
- f"Invalid data store type: {self.type}. Valid types: {[ds.value for ds in APSEventBrokerType]}"
28
- )
29
-
30
- def _setup_asyncpg_event_broker(self):
31
- from apscheduler.eventbrokers.asyncpg import AsyncpgEventBroker
32
-
33
- if self._sqla_engine is None:
34
- self._event_broker = AsyncpgEventBroker.from_dsn(dsn=self.uri)
35
- else:
36
- self._event_broker = AsyncpgEventBroker.from_async_sqla_engine(
37
- engine=self._sqla_engine
38
- )
39
-
40
- def _setup_mqtt_event_broker(self):
41
- import urllib.parse
42
-
43
- from apscheduler.eventbrokers.mqtt import MQTTEventBroker
44
-
45
- # Parse the URI
46
- parsed = urllib.parse.urlparse(self.uri)
47
-
48
- hostname = parsed.hostname
49
- port = parsed.port
50
- username = parsed.username
51
- password = parsed.password
52
- use_ssl = parsed.scheme == "mqtts"
53
-
54
- self._event_broker = MQTTEventBroker(
55
- host=hostname, port=port, ssl=use_ssl, topic="flowerpower/scheduler"
56
- )
57
- if (self.username is not None) and (self.password is not None):
58
- self._event_broker._client.username_pw_set(
59
- username,
60
- password,
61
- )
62
-
63
- def _setup_redis_event_broker(self):
64
- from apscheduler.eventbrokers.redis import RedisEventBroker
65
-
66
- self._event_broker = RedisEventBroker(self.uri)
67
-
68
- def _setup_local_event_broker(self):
69
- from apscheduler.eventbrokers.local import LocalEventBroker
70
-
71
- self._event_broker = LocalEventBroker()
72
-
73
- def setup(self):
74
- if self.is_sqla_type:
75
- self._setup_asyncpg_event_broker()
76
- elif self.is_mqtt_type:
77
- self._setup_mqtt_event_broker()
78
- elif self.is_redis_type:
79
- self._setup_redis_event_broker()
80
- else:
81
- self._setup_local_event_broker()
82
-
83
- @property
84
- def client(self) -> BaseEventBroker:
85
- if self._event_broker is None:
86
- self.setup()
87
- return self._event_broker
88
-
89
- @property
90
- def sqla_engine(self) -> AsyncEngine | None:
91
- if self._sqla_engine is None:
92
- self.setup()
93
- return self._sqla_engine