FlowerPower 0.20.0__py3-none-any.whl → 0.30.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowerpower/__init__.py +2 -6
- flowerpower/cfg/__init__.py +4 -11
- flowerpower/cfg/base.py +29 -25
- flowerpower/cfg/pipeline/__init__.py +3 -3
- flowerpower/cfg/pipeline/_schedule.py +32 -0
- flowerpower/cfg/pipeline/adapter.py +0 -5
- flowerpower/cfg/pipeline/builder.py +377 -0
- flowerpower/cfg/pipeline/run.py +89 -0
- flowerpower/cfg/project/__init__.py +8 -21
- flowerpower/cfg/project/adapter.py +0 -12
- flowerpower/cli/__init__.py +2 -28
- flowerpower/cli/pipeline.py +10 -4
- flowerpower/flowerpower.py +275 -585
- flowerpower/pipeline/base.py +19 -10
- flowerpower/pipeline/io.py +52 -46
- flowerpower/pipeline/manager.py +149 -91
- flowerpower/pipeline/pipeline.py +159 -87
- flowerpower/pipeline/registry.py +68 -33
- flowerpower/pipeline/visualizer.py +4 -4
- flowerpower/plugins/{_io → io}/__init__.py +1 -1
- flowerpower/settings/__init__.py +0 -2
- flowerpower/settings/{backend.py → _backend.py} +0 -19
- flowerpower/settings/logging.py +1 -1
- flowerpower/utils/logging.py +24 -12
- flowerpower/utils/misc.py +17 -0
- flowerpower-0.30.0.dist-info/METADATA +451 -0
- flowerpower-0.30.0.dist-info/RECORD +42 -0
- flowerpower/cfg/pipeline/schedule.py +0 -74
- flowerpower/cfg/project/job_queue.py +0 -111
- flowerpower/cli/job_queue.py +0 -1329
- flowerpower/cli/mqtt.py +0 -174
- flowerpower/job_queue/__init__.py +0 -205
- flowerpower/job_queue/base.py +0 -611
- flowerpower/job_queue/rq/__init__.py +0 -10
- flowerpower/job_queue/rq/_trigger.py +0 -37
- flowerpower/job_queue/rq/concurrent_workers/gevent_worker.py +0 -226
- flowerpower/job_queue/rq/concurrent_workers/thread_worker.py +0 -228
- flowerpower/job_queue/rq/manager.py +0 -1893
- flowerpower/job_queue/rq/setup.py +0 -154
- flowerpower/job_queue/rq/utils.py +0 -69
- flowerpower/mqtt.py +0 -12
- flowerpower/plugins/mqtt/__init__.py +0 -12
- flowerpower/plugins/mqtt/cfg.py +0 -17
- flowerpower/plugins/mqtt/manager.py +0 -962
- flowerpower/settings/job_queue.py +0 -31
- flowerpower-0.20.0.dist-info/METADATA +0 -693
- flowerpower-0.20.0.dist-info/RECORD +0 -58
- {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/WHEEL +0 -0
- {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/entry_points.txt +0 -0
- {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/licenses/LICENSE +0 -0
- {flowerpower-0.20.0.dist-info → flowerpower-0.30.0.dist-info}/top_level.txt +0 -0
@@ -1,154 +0,0 @@
|
|
1
|
-
from dataclasses import dataclass, field
|
2
|
-
|
3
|
-
import redis
|
4
|
-
|
5
|
-
from ..base import BaseBackend
|
6
|
-
|
7
|
-
# Enums for RQ DataStore and EventBroker types
|
8
|
-
# class RQBackendType(BackendType):
|
9
|
-
# REDIS = "redis"
|
10
|
-
# MEMORY = "memory"
|
11
|
-
|
12
|
-
|
13
|
-
@dataclass # (slots=True)
|
14
|
-
class RQBackend(BaseBackend):
|
15
|
-
"""RQ Backend implementation for Redis Queue (RQ) job storage and queuing.
|
16
|
-
|
17
|
-
This class provides a Redis-based backend for RQ job storage and queue management.
|
18
|
-
It supports both Redis and in-memory storage options for development/testing.
|
19
|
-
|
20
|
-
Args:
|
21
|
-
queues (str | list[str] | None): Names of queues to create. Defaults to ["default"].
|
22
|
-
num_workers (int): Number of worker processes to use. Defaults to 1.
|
23
|
-
|
24
|
-
Attributes:
|
25
|
-
type (str): Backend type, either "redis" or "memory". Inherited from BaseBackend.
|
26
|
-
uri (str): Connection URI. Inherited from BaseBackend.
|
27
|
-
result_namespace (str): Namespace for storing job results in Redis.
|
28
|
-
_client (redis.Redis | dict): Redis client or dict for memory storage.
|
29
|
-
|
30
|
-
Raises:
|
31
|
-
ValueError: If an invalid backend type is specified.
|
32
|
-
|
33
|
-
Example:
|
34
|
-
```python
|
35
|
-
# Create Redis backend with default queue
|
36
|
-
backend = RQBackend(
|
37
|
-
type="redis",
|
38
|
-
uri="redis://localhost:6379/0"
|
39
|
-
)
|
40
|
-
|
41
|
-
# Create Redis backend with multiple queues
|
42
|
-
backend = RQBackend(
|
43
|
-
type="redis",
|
44
|
-
uri="redis://localhost:6379/0",
|
45
|
-
queues=["high", "default", "low"]
|
46
|
-
)
|
47
|
-
|
48
|
-
# Create in-memory backend for testing
|
49
|
-
backend = RQBackend(type="memory", queues=["test"])
|
50
|
-
```
|
51
|
-
"""
|
52
|
-
|
53
|
-
queues: str | list[str] | None = field(default_factory=lambda: ["default"])
|
54
|
-
num_workers: int = field(default=1)
|
55
|
-
|
56
|
-
def __post_init__(self) -> None:
|
57
|
-
"""Initialize and validate the backend configuration.
|
58
|
-
|
59
|
-
This method is called automatically after instance creation. It:
|
60
|
-
1. Sets default type to "redis" if not specified
|
61
|
-
2. Calls parent class initialization
|
62
|
-
3. Validates backend type
|
63
|
-
4. Sets default result namespace
|
64
|
-
|
65
|
-
Raises:
|
66
|
-
ValueError: If an unsupported backend type is specified.
|
67
|
-
Only "redis" and "memory" types are supported.
|
68
|
-
"""
|
69
|
-
if self.type is None:
|
70
|
-
self.type = "redis"
|
71
|
-
super().__post_init__()
|
72
|
-
|
73
|
-
if not self.type.is_memory_type and not self.type.is_redis_type:
|
74
|
-
raise ValueError(
|
75
|
-
f"Invalid backend type: {self.type}. Valid types: {[self.type.REDIS, self.type.MEMORY]}"
|
76
|
-
)
|
77
|
-
|
78
|
-
self.result_namespace = getattr(self, "result_namespace", "flowerpower:results")
|
79
|
-
|
80
|
-
def setup(self) -> None:
|
81
|
-
"""Set up the Redis client or in-memory storage.
|
82
|
-
|
83
|
-
This method initializes the backend storage based on the configured type.
|
84
|
-
For Redis, it creates a Redis client with the specified connection parameters.
|
85
|
-
For in-memory storage, it creates a simple dictionary.
|
86
|
-
|
87
|
-
Raises:
|
88
|
-
ValueError: If an unsupported backend type is specified.
|
89
|
-
redis.RedisError: If Redis connection fails.
|
90
|
-
|
91
|
-
Example:
|
92
|
-
```python
|
93
|
-
backend = RQBackend(
|
94
|
-
type="redis",
|
95
|
-
host="localhost",
|
96
|
-
port=6379,
|
97
|
-
password="secret",
|
98
|
-
database="0",
|
99
|
-
ssl=True
|
100
|
-
)
|
101
|
-
backend.setup()
|
102
|
-
```
|
103
|
-
"""
|
104
|
-
# Use connection info from BaseBackend to create Redis client
|
105
|
-
if self.type.is_redis_type:
|
106
|
-
# Parse db from database or default to 0
|
107
|
-
db = 0
|
108
|
-
if self.database is not None:
|
109
|
-
try:
|
110
|
-
db = int(self.database)
|
111
|
-
except Exception:
|
112
|
-
db = 0
|
113
|
-
self._client = redis.Redis(
|
114
|
-
host=self.host or self.type.default_host,
|
115
|
-
port=self.port or self.type.default_port,
|
116
|
-
db=db,
|
117
|
-
password=self.password,
|
118
|
-
ssl=self.ssl,
|
119
|
-
ssl_cert_reqs=None if not self.verify_ssl else "required",
|
120
|
-
ssl_ca_certs=self.ca_file,
|
121
|
-
ssl_certfile=self.cert_file,
|
122
|
-
ssl_keyfile=self.key_file,
|
123
|
-
)
|
124
|
-
elif self.type.is_memory_type:
|
125
|
-
# Simple in-memory dict for testing
|
126
|
-
self._client = {}
|
127
|
-
else:
|
128
|
-
raise ValueError(f"Unsupported RQBackend type: {self.type}")
|
129
|
-
|
130
|
-
@property
|
131
|
-
def client(self) -> redis.Redis | dict:
|
132
|
-
"""Get the initialized storage client.
|
133
|
-
|
134
|
-
This property provides access to the Redis client or in-memory dictionary,
|
135
|
-
initializing it if needed.
|
136
|
-
|
137
|
-
Returns:
|
138
|
-
redis.Redis | dict: The Redis client for Redis backend,
|
139
|
-
or dictionary for in-memory backend.
|
140
|
-
|
141
|
-
Example:
|
142
|
-
```python
|
143
|
-
backend = RQBackend(type="redis", uri="redis://localhost:6379/0")
|
144
|
-
redis_client = backend.client # Gets Redis client
|
145
|
-
redis_client.set("key", "value")
|
146
|
-
|
147
|
-
backend = RQBackend(type="memory")
|
148
|
-
mem_dict = backend.client # Gets dict for testing
|
149
|
-
mem_dict["key"] = "value"
|
150
|
-
```
|
151
|
-
"""
|
152
|
-
if self._client is None:
|
153
|
-
self.setup()
|
154
|
-
return self._client
|
@@ -1,69 +0,0 @@
|
|
1
|
-
from rich.console import Console
|
2
|
-
from rich.table import Table
|
3
|
-
from rq import Queue
|
4
|
-
from rq_scheduler import Scheduler
|
5
|
-
|
6
|
-
|
7
|
-
def show_schedules(scheduler: Scheduler) -> None:
|
8
|
-
"""
|
9
|
-
Display the schedules in a user-friendly format.
|
10
|
-
|
11
|
-
Args:
|
12
|
-
scheduler (Scheduler): An instance of rq_scheduler.Scheduler.
|
13
|
-
"""
|
14
|
-
console = Console()
|
15
|
-
table = Table(title="Scheduled Jobs")
|
16
|
-
|
17
|
-
table.add_column("ID", style="cyan")
|
18
|
-
table.add_column("Function", style="green")
|
19
|
-
table.add_column("Schedule", style="yellow")
|
20
|
-
table.add_column("Next Run", style="magenta")
|
21
|
-
|
22
|
-
for job in scheduler.get_jobs():
|
23
|
-
# Determine schedule type and format
|
24
|
-
schedule_type = "Unknown"
|
25
|
-
if hasattr(job, "meta"):
|
26
|
-
if job.meta.get("cron"):
|
27
|
-
schedule_type = f"Cron: {job.meta['cron']}"
|
28
|
-
elif job.meta.get("interval"):
|
29
|
-
schedule_type = f"Interval: {job.meta['interval']}s"
|
30
|
-
|
31
|
-
next_run = (
|
32
|
-
job.scheduled_at.strftime("%Y-%m-%d %H:%M:%S")
|
33
|
-
if hasattr(job, "scheduled_at") and job.scheduled_at
|
34
|
-
else "Unknown"
|
35
|
-
)
|
36
|
-
|
37
|
-
table.add_row(job.id, job.func_name, schedule_type, next_run)
|
38
|
-
|
39
|
-
console.print(table)
|
40
|
-
|
41
|
-
|
42
|
-
def show_jobs(queue: Queue) -> None:
|
43
|
-
"""
|
44
|
-
Display the jobs in a user-friendly format.
|
45
|
-
|
46
|
-
Args:
|
47
|
-
queue (Queue): An instance of rq.Queue.
|
48
|
-
"""
|
49
|
-
console = Console()
|
50
|
-
table = Table(title="Jobs")
|
51
|
-
|
52
|
-
table.add_column("ID", style="cyan")
|
53
|
-
table.add_column("Function", style="green")
|
54
|
-
table.add_column("Status", style="yellow")
|
55
|
-
table.add_column("Enqueued At", style="magenta")
|
56
|
-
table.add_column("Result", style="blue")
|
57
|
-
|
58
|
-
for job in queue.get_jobs():
|
59
|
-
table.add_row(
|
60
|
-
job.id,
|
61
|
-
job.func_name,
|
62
|
-
job.get_status(),
|
63
|
-
job.enqueued_at.strftime("%Y-%m-%d %H:%M:%S")
|
64
|
-
if job.enqueued_at
|
65
|
-
else "Unknown",
|
66
|
-
str(job.result) if job.result else "None",
|
67
|
-
)
|
68
|
-
|
69
|
-
console.print(table)
|
flowerpower/mqtt.py
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
import warnings
|
2
|
-
|
3
|
-
warnings.warn(
|
4
|
-
"`flowerpower.mqtt` is deprecated, use `flowerpower.plugins.mqtt` instead",
|
5
|
-
DeprecationWarning,
|
6
|
-
stacklevel=2,
|
7
|
-
)
|
8
|
-
|
9
|
-
from flowerpower.plugins.mqtt import MqttManager # noqa: E402
|
10
|
-
from flowerpower.plugins.mqtt import MqttConfig, MQTTManager
|
11
|
-
|
12
|
-
__all__ = ["MqttConfig", "MqttManager", "MQTTManager"]
|
flowerpower/plugins/mqtt/cfg.py
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
from ...cfg.base import BaseConfig
|
2
|
-
|
3
|
-
|
4
|
-
class MqttConfig(BaseConfig):
|
5
|
-
username: str | None = None
|
6
|
-
password: str | None = None
|
7
|
-
host: str | None = "localhost"
|
8
|
-
port: int | None = 1883
|
9
|
-
topic: str | None = None
|
10
|
-
first_reconnect_delay: int = 1
|
11
|
-
max_reconnect_count: int = 5
|
12
|
-
reconnect_rate: int = 2
|
13
|
-
max_reconnect_delay: int = 60
|
14
|
-
transport: str = "tcp"
|
15
|
-
clean_session: bool = True
|
16
|
-
client_id: str | None = None
|
17
|
-
client_id_suffix: str | None = None
|