pyworkflow-engine 0.1.15__py3-none-any.whl → 0.1.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyworkflow/__init__.py +1 -1
- pyworkflow/celery/app.py +128 -9
- pyworkflow/celery/singleton.py +91 -6
- pyworkflow/cli/commands/worker.py +94 -2
- {pyworkflow_engine-0.1.15.dist-info → pyworkflow_engine-0.1.16.dist-info}/METADATA +1 -1
- {pyworkflow_engine-0.1.15.dist-info → pyworkflow_engine-0.1.16.dist-info}/RECORD +10 -10
- {pyworkflow_engine-0.1.15.dist-info → pyworkflow_engine-0.1.16.dist-info}/WHEEL +0 -0
- {pyworkflow_engine-0.1.15.dist-info → pyworkflow_engine-0.1.16.dist-info}/entry_points.txt +0 -0
- {pyworkflow_engine-0.1.15.dist-info → pyworkflow_engine-0.1.16.dist-info}/licenses/LICENSE +0 -0
- {pyworkflow_engine-0.1.15.dist-info → pyworkflow_engine-0.1.16.dist-info}/top_level.txt +0 -0
pyworkflow/__init__.py
CHANGED
pyworkflow/celery/app.py
CHANGED
|
@@ -13,6 +13,7 @@ garbage collector and Celery's saferepr module. It does not affect functionality
|
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
import os
|
|
16
|
+
from typing import Any
|
|
16
17
|
|
|
17
18
|
from celery import Celery
|
|
18
19
|
from celery.signals import worker_init, worker_process_init, worker_shutdown
|
|
@@ -38,6 +39,74 @@ def _configure_worker_logging() -> None:
|
|
|
38
39
|
_logging_configured = True
|
|
39
40
|
|
|
40
41
|
|
|
42
|
+
def is_sentinel_url(url: str) -> bool:
|
|
43
|
+
"""Check if URL uses sentinel:// or sentinel+ssl:// protocol."""
|
|
44
|
+
return url.startswith("sentinel://") or url.startswith("sentinel+ssl://")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def parse_sentinel_url(url: str) -> tuple[list[tuple[str, int]], int, str | None]:
|
|
48
|
+
"""
|
|
49
|
+
Parse a sentinel:// URL into sentinel hosts, database number, and password.
|
|
50
|
+
|
|
51
|
+
Format: sentinel://[password@]host1:port1,host2:port2/db_number
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
url: Sentinel URL (sentinel:// or sentinel+ssl://)
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
Tuple of ([(host, port), ...], db_number, password or None)
|
|
58
|
+
|
|
59
|
+
Examples:
|
|
60
|
+
>>> parse_sentinel_url("sentinel://host1:26379,host2:26379/0")
|
|
61
|
+
([('host1', 26379), ('host2', 26379)], 0, None)
|
|
62
|
+
|
|
63
|
+
>>> parse_sentinel_url("sentinel://mypassword@host1:26379/0")
|
|
64
|
+
([('host1', 26379)], 0, 'mypassword')
|
|
65
|
+
"""
|
|
66
|
+
# Remove protocol prefix
|
|
67
|
+
if url.startswith("sentinel+ssl://"):
|
|
68
|
+
url_without_protocol = url[len("sentinel+ssl://") :]
|
|
69
|
+
elif url.startswith("sentinel://"):
|
|
70
|
+
url_without_protocol = url[len("sentinel://") :]
|
|
71
|
+
else:
|
|
72
|
+
raise ValueError(f"Invalid sentinel URL: {url}")
|
|
73
|
+
|
|
74
|
+
# Extract password if present (password@hosts)
|
|
75
|
+
password: str | None = None
|
|
76
|
+
if "@" in url_without_protocol:
|
|
77
|
+
password, url_without_protocol = url_without_protocol.split("@", 1)
|
|
78
|
+
|
|
79
|
+
# Extract database number from path
|
|
80
|
+
db_number = 0
|
|
81
|
+
if "/" in url_without_protocol:
|
|
82
|
+
hosts_part, db_part = url_without_protocol.rsplit("/", 1)
|
|
83
|
+
# Handle query params in db part
|
|
84
|
+
if "?" in db_part:
|
|
85
|
+
db_part = db_part.split("?")[0]
|
|
86
|
+
if db_part:
|
|
87
|
+
db_number = int(db_part)
|
|
88
|
+
else:
|
|
89
|
+
hosts_part = url_without_protocol
|
|
90
|
+
# Handle query params
|
|
91
|
+
if "?" in hosts_part:
|
|
92
|
+
hosts_part = hosts_part.split("?")[0]
|
|
93
|
+
|
|
94
|
+
# Parse hosts
|
|
95
|
+
sentinels: list[tuple[str, int]] = []
|
|
96
|
+
for host_port in hosts_part.split(","):
|
|
97
|
+
host_port = host_port.strip()
|
|
98
|
+
if not host_port:
|
|
99
|
+
continue
|
|
100
|
+
if ":" in host_port:
|
|
101
|
+
host, port_str = host_port.rsplit(":", 1)
|
|
102
|
+
sentinels.append((host, int(port_str)))
|
|
103
|
+
else:
|
|
104
|
+
# Default Sentinel port
|
|
105
|
+
sentinels.append((host_port, 26379))
|
|
106
|
+
|
|
107
|
+
return sentinels, db_number, password
|
|
108
|
+
|
|
109
|
+
|
|
41
110
|
def discover_workflows(modules: list[str] | None = None) -> None:
|
|
42
111
|
"""
|
|
43
112
|
Discover and import workflow modules to register workflows with Celery workers.
|
|
@@ -79,6 +148,9 @@ def create_celery_app(
|
|
|
79
148
|
broker_url: str | None = None,
|
|
80
149
|
result_backend: str | None = None,
|
|
81
150
|
app_name: str = "pyworkflow",
|
|
151
|
+
sentinel_master_name: str | None = None,
|
|
152
|
+
broker_transport_options: dict[str, Any] | None = None,
|
|
153
|
+
result_backend_transport_options: dict[str, Any] | None = None,
|
|
82
154
|
) -> Celery:
|
|
83
155
|
"""
|
|
84
156
|
Create and configure a Celery application for PyWorkflow.
|
|
@@ -87,6 +159,9 @@ def create_celery_app(
|
|
|
87
159
|
broker_url: Celery broker URL. Priority: parameter > PYWORKFLOW_CELERY_BROKER env var > redis://localhost:6379/0
|
|
88
160
|
result_backend: Result backend URL. Priority: parameter > PYWORKFLOW_CELERY_RESULT_BACKEND env var > redis://localhost:6379/1
|
|
89
161
|
app_name: Application name
|
|
162
|
+
sentinel_master_name: Redis Sentinel master name. Priority: parameter > PYWORKFLOW_CELERY_SENTINEL_MASTER env var > "mymaster"
|
|
163
|
+
broker_transport_options: Additional transport options for the broker (merged with defaults)
|
|
164
|
+
result_backend_transport_options: Additional transport options for the result backend (merged with defaults)
|
|
90
165
|
|
|
91
166
|
Returns:
|
|
92
167
|
Configured Celery application
|
|
@@ -94,6 +169,7 @@ def create_celery_app(
|
|
|
94
169
|
Environment Variables:
|
|
95
170
|
PYWORKFLOW_CELERY_BROKER: Celery broker URL (used if broker_url param not provided)
|
|
96
171
|
PYWORKFLOW_CELERY_RESULT_BACKEND: Result backend URL (used if result_backend param not provided)
|
|
172
|
+
PYWORKFLOW_CELERY_SENTINEL_MASTER: Sentinel master name (used if sentinel_master_name param not provided)
|
|
97
173
|
|
|
98
174
|
Examples:
|
|
99
175
|
# Default configuration (uses env vars if set, otherwise localhost Redis)
|
|
@@ -110,6 +186,13 @@ def create_celery_app(
|
|
|
110
186
|
broker_url="amqp://guest:guest@rabbitmq:5672//",
|
|
111
187
|
result_backend="redis://localhost:6379/1"
|
|
112
188
|
)
|
|
189
|
+
|
|
190
|
+
# Redis Sentinel for high availability
|
|
191
|
+
app = create_celery_app(
|
|
192
|
+
broker_url="sentinel://sentinel1:26379,sentinel2:26379,sentinel3:26379/0",
|
|
193
|
+
result_backend="sentinel://sentinel1:26379,sentinel2:26379,sentinel3:26379/1",
|
|
194
|
+
sentinel_master_name="mymaster"
|
|
195
|
+
)
|
|
113
196
|
"""
|
|
114
197
|
# Priority: parameter > environment variable > hardcoded default
|
|
115
198
|
broker_url = broker_url or os.getenv("PYWORKFLOW_CELERY_BROKER") or "redis://localhost:6379/0"
|
|
@@ -119,6 +202,45 @@ def create_celery_app(
|
|
|
119
202
|
or "redis://localhost:6379/1"
|
|
120
203
|
)
|
|
121
204
|
|
|
205
|
+
# Detect broker and backend types
|
|
206
|
+
is_sentinel_broker = is_sentinel_url(broker_url)
|
|
207
|
+
is_sentinel_backend = is_sentinel_url(result_backend)
|
|
208
|
+
is_redis_broker = broker_url.startswith("redis://") or broker_url.startswith("rediss://")
|
|
209
|
+
|
|
210
|
+
# Get Sentinel master name from param, env, or default
|
|
211
|
+
master_name = (
|
|
212
|
+
sentinel_master_name or os.getenv("PYWORKFLOW_CELERY_SENTINEL_MASTER") or "mymaster"
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# Build transport options for broker
|
|
216
|
+
if is_sentinel_broker:
|
|
217
|
+
sentinel_broker_opts: dict[str, Any] = {"master_name": master_name}
|
|
218
|
+
# Merge with user options (user takes precedence)
|
|
219
|
+
final_broker_opts: dict[str, Any] = {
|
|
220
|
+
"visibility_timeout": 3600,
|
|
221
|
+
**sentinel_broker_opts,
|
|
222
|
+
**(broker_transport_options or {}),
|
|
223
|
+
}
|
|
224
|
+
else:
|
|
225
|
+
final_broker_opts = {
|
|
226
|
+
"visibility_timeout": 3600,
|
|
227
|
+
**(broker_transport_options or {}),
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
# Build transport options for result backend
|
|
231
|
+
if is_sentinel_backend:
|
|
232
|
+
sentinel_backend_opts: dict[str, Any] = {"master_name": master_name}
|
|
233
|
+
final_backend_opts: dict[str, Any] = {
|
|
234
|
+
"visibility_timeout": 3600,
|
|
235
|
+
**sentinel_backend_opts,
|
|
236
|
+
**(result_backend_transport_options or {}),
|
|
237
|
+
}
|
|
238
|
+
else:
|
|
239
|
+
final_backend_opts = {
|
|
240
|
+
"visibility_timeout": 3600,
|
|
241
|
+
**(result_backend_transport_options or {}),
|
|
242
|
+
}
|
|
243
|
+
|
|
122
244
|
app = Celery(
|
|
123
245
|
app_name,
|
|
124
246
|
broker=broker_url,
|
|
@@ -138,12 +260,8 @@ def create_celery_app(
|
|
|
138
260
|
enable_utc=True,
|
|
139
261
|
# Broker transport options - prevent task redelivery
|
|
140
262
|
# See: https://github.com/celery/celery/issues/5935
|
|
141
|
-
broker_transport_options=
|
|
142
|
-
|
|
143
|
-
},
|
|
144
|
-
result_backend_transport_options={
|
|
145
|
-
"visibility_timeout": 3600,
|
|
146
|
-
},
|
|
263
|
+
broker_transport_options=final_broker_opts,
|
|
264
|
+
result_backend_transport_options=final_backend_opts,
|
|
147
265
|
# Task routing
|
|
148
266
|
task_default_queue="pyworkflow.default",
|
|
149
267
|
task_default_exchange="pyworkflow",
|
|
@@ -194,12 +312,13 @@ def create_celery_app(
|
|
|
194
312
|
worker_task_log_format="[%(asctime)s: %(levelname)s/%(processName)s] [%(task_name)s(%(task_id)s)] %(message)s",
|
|
195
313
|
)
|
|
196
314
|
|
|
197
|
-
# Configure singleton locking for Redis brokers
|
|
315
|
+
# Configure singleton locking for Redis or Sentinel brokers
|
|
198
316
|
# This enables distributed locking to prevent duplicate task execution
|
|
199
|
-
is_redis_broker
|
|
200
|
-
if is_redis_broker:
|
|
317
|
+
if is_redis_broker or is_sentinel_broker:
|
|
201
318
|
app.conf.update(
|
|
202
319
|
singleton_backend_url=broker_url,
|
|
320
|
+
singleton_backend_is_sentinel=is_sentinel_broker,
|
|
321
|
+
singleton_sentinel_master=master_name if is_sentinel_broker else None,
|
|
203
322
|
singleton_key_prefix="pyworkflow:lock:",
|
|
204
323
|
singleton_lock_expiry=3600, # 1 hour TTL (safety net)
|
|
205
324
|
)
|
pyworkflow/celery/singleton.py
CHANGED
|
@@ -12,13 +12,16 @@ Based on:
|
|
|
12
12
|
import inspect
|
|
13
13
|
import json
|
|
14
14
|
from hashlib import md5
|
|
15
|
-
from typing import Any
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
16
|
from uuid import uuid4
|
|
17
17
|
|
|
18
18
|
from celery import Task
|
|
19
19
|
from celery.exceptions import WorkerLostError
|
|
20
20
|
from loguru import logger
|
|
21
21
|
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from redis.sentinel import Sentinel
|
|
24
|
+
|
|
22
25
|
|
|
23
26
|
def generate_lock_key(
|
|
24
27
|
task_name: str,
|
|
@@ -59,14 +62,83 @@ class SingletonConfig:
|
|
|
59
62
|
def raise_on_duplicate(self) -> bool:
|
|
60
63
|
return self.app.conf.get("singleton_raise_on_duplicate", False)
|
|
61
64
|
|
|
65
|
+
@property
|
|
66
|
+
def is_sentinel(self) -> bool:
|
|
67
|
+
"""Check if the backend uses Redis Sentinel."""
|
|
68
|
+
return self.app.conf.get("singleton_backend_is_sentinel", False)
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def sentinel_master(self) -> str | None:
|
|
72
|
+
"""Get the Sentinel master name."""
|
|
73
|
+
return self.app.conf.get("singleton_sentinel_master")
|
|
74
|
+
|
|
62
75
|
|
|
63
76
|
class RedisLockBackend:
|
|
64
|
-
"""Redis backend for distributed locking."""
|
|
77
|
+
"""Redis backend for distributed locking with Sentinel support."""
|
|
65
78
|
|
|
66
|
-
|
|
79
|
+
_sentinel: "Sentinel | None"
|
|
80
|
+
_master_name: str | None
|
|
81
|
+
|
|
82
|
+
def __init__(
|
|
83
|
+
self,
|
|
84
|
+
url: str,
|
|
85
|
+
is_sentinel: bool = False,
|
|
86
|
+
sentinel_master: str | None = None,
|
|
87
|
+
):
|
|
67
88
|
import redis
|
|
68
89
|
|
|
69
|
-
|
|
90
|
+
if is_sentinel:
|
|
91
|
+
from redis.sentinel import Sentinel
|
|
92
|
+
|
|
93
|
+
sentinels = self._parse_sentinel_url(url)
|
|
94
|
+
self._sentinel = Sentinel(
|
|
95
|
+
sentinels,
|
|
96
|
+
socket_timeout=0.5,
|
|
97
|
+
decode_responses=True,
|
|
98
|
+
)
|
|
99
|
+
self._master_name = sentinel_master or "mymaster"
|
|
100
|
+
self.redis = self._sentinel.master_for(
|
|
101
|
+
self._master_name,
|
|
102
|
+
decode_responses=True,
|
|
103
|
+
)
|
|
104
|
+
else:
|
|
105
|
+
self._sentinel = None
|
|
106
|
+
self._master_name = None
|
|
107
|
+
self.redis = redis.from_url(url, decode_responses=True)
|
|
108
|
+
|
|
109
|
+
@staticmethod
|
|
110
|
+
def _parse_sentinel_url(url: str) -> list[tuple[str, int]]:
|
|
111
|
+
"""
|
|
112
|
+
Parse sentinel:// URL to list of (host, port) tuples.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
url: Sentinel URL (sentinel:// or sentinel+ssl://)
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
List of (host, port) tuples for Sentinel servers
|
|
119
|
+
"""
|
|
120
|
+
# Remove protocol
|
|
121
|
+
url = url.replace("sentinel://", "").replace("sentinel+ssl://", "")
|
|
122
|
+
# Remove database suffix and query params
|
|
123
|
+
if "/" in url:
|
|
124
|
+
url = url.split("/")[0]
|
|
125
|
+
if "?" in url:
|
|
126
|
+
url = url.split("?")[0]
|
|
127
|
+
# Handle password prefix (password@hosts)
|
|
128
|
+
if "@" in url:
|
|
129
|
+
url = url.split("@", 1)[1]
|
|
130
|
+
|
|
131
|
+
sentinels: list[tuple[str, int]] = []
|
|
132
|
+
for host_port in url.split(","):
|
|
133
|
+
host_port = host_port.strip()
|
|
134
|
+
if not host_port:
|
|
135
|
+
continue
|
|
136
|
+
if ":" in host_port:
|
|
137
|
+
host, port = host_port.rsplit(":", 1)
|
|
138
|
+
sentinels.append((host, int(port)))
|
|
139
|
+
else:
|
|
140
|
+
sentinels.append((host_port, 26379)) # Default Sentinel port
|
|
141
|
+
return sentinels
|
|
70
142
|
|
|
71
143
|
def lock(self, lock_key: str, task_id: str, expiry: int | None = None) -> bool:
|
|
72
144
|
"""Acquire lock atomically. Returns True if acquired."""
|
|
@@ -144,13 +216,26 @@ class SingletonWorkflowTask(Task):
|
|
|
144
216
|
def singleton_backend(self) -> RedisLockBackend | None:
|
|
145
217
|
if self._singleton_backend is None:
|
|
146
218
|
url = self.singleton_config.backend_url
|
|
219
|
+
is_sentinel = self.singleton_config.is_sentinel
|
|
220
|
+
sentinel_master = self.singleton_config.sentinel_master
|
|
221
|
+
|
|
147
222
|
if not url:
|
|
148
|
-
# Try broker URL if it's Redis
|
|
223
|
+
# Try broker URL if it's Redis or Sentinel
|
|
149
224
|
broker = self.app.conf.broker_url or ""
|
|
150
225
|
if broker.startswith("redis://") or broker.startswith("rediss://"):
|
|
151
226
|
url = broker
|
|
227
|
+
is_sentinel = False
|
|
228
|
+
elif broker.startswith("sentinel://") or broker.startswith("sentinel+ssl://"):
|
|
229
|
+
url = broker
|
|
230
|
+
is_sentinel = True
|
|
231
|
+
sentinel_master = self.app.conf.get("singleton_sentinel_master", "mymaster")
|
|
232
|
+
|
|
152
233
|
if url:
|
|
153
|
-
self._singleton_backend = RedisLockBackend(
|
|
234
|
+
self._singleton_backend = RedisLockBackend(
|
|
235
|
+
url,
|
|
236
|
+
is_sentinel=is_sentinel,
|
|
237
|
+
sentinel_master=sentinel_master,
|
|
238
|
+
)
|
|
154
239
|
return self._singleton_backend
|
|
155
240
|
|
|
156
241
|
@property
|
|
@@ -20,7 +20,13 @@ def worker() -> None:
|
|
|
20
20
|
pass
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
@worker.command(
|
|
23
|
+
@worker.command(
|
|
24
|
+
name="run",
|
|
25
|
+
context_settings={
|
|
26
|
+
"allow_extra_args": True,
|
|
27
|
+
"allow_interspersed_args": False,
|
|
28
|
+
},
|
|
29
|
+
)
|
|
24
30
|
@click.option(
|
|
25
31
|
"--workflow",
|
|
26
32
|
"queue_workflow",
|
|
@@ -70,6 +76,40 @@ def worker() -> None:
|
|
|
70
76
|
default="prefork",
|
|
71
77
|
help="Worker pool type (default: prefork). Use 'solo' for debugging with breakpoints",
|
|
72
78
|
)
|
|
79
|
+
@click.option(
|
|
80
|
+
"--sentinel-master",
|
|
81
|
+
default=None,
|
|
82
|
+
help="Redis Sentinel master name (required for sentinel:// URLs)",
|
|
83
|
+
)
|
|
84
|
+
@click.option(
|
|
85
|
+
"--autoscale",
|
|
86
|
+
default=None,
|
|
87
|
+
help="Enable autoscaling: MIN,MAX (e.g., '2,10')",
|
|
88
|
+
)
|
|
89
|
+
@click.option(
|
|
90
|
+
"--max-tasks-per-child",
|
|
91
|
+
type=int,
|
|
92
|
+
default=None,
|
|
93
|
+
help="Maximum tasks per worker child before replacement",
|
|
94
|
+
)
|
|
95
|
+
@click.option(
|
|
96
|
+
"--prefetch-multiplier",
|
|
97
|
+
type=int,
|
|
98
|
+
default=None,
|
|
99
|
+
help="Task prefetch count per worker process",
|
|
100
|
+
)
|
|
101
|
+
@click.option(
|
|
102
|
+
"--time-limit",
|
|
103
|
+
type=float,
|
|
104
|
+
default=None,
|
|
105
|
+
help="Hard time limit for tasks in seconds",
|
|
106
|
+
)
|
|
107
|
+
@click.option(
|
|
108
|
+
"--soft-time-limit",
|
|
109
|
+
type=float,
|
|
110
|
+
default=None,
|
|
111
|
+
help="Soft time limit for tasks in seconds",
|
|
112
|
+
)
|
|
73
113
|
@click.pass_context
|
|
74
114
|
def run_worker(
|
|
75
115
|
ctx: click.Context,
|
|
@@ -81,6 +121,12 @@ def run_worker(
|
|
|
81
121
|
hostname: str | None,
|
|
82
122
|
beat: bool,
|
|
83
123
|
pool: str | None,
|
|
124
|
+
sentinel_master: str | None,
|
|
125
|
+
autoscale: str | None,
|
|
126
|
+
max_tasks_per_child: int | None,
|
|
127
|
+
prefetch_multiplier: int | None,
|
|
128
|
+
time_limit: float | None,
|
|
129
|
+
soft_time_limit: float | None,
|
|
84
130
|
) -> None:
|
|
85
131
|
"""
|
|
86
132
|
Start a Celery worker for processing workflows.
|
|
@@ -88,6 +134,8 @@ def run_worker(
|
|
|
88
134
|
By default, processes all queues. Use --workflow, --step, or --schedule
|
|
89
135
|
flags to limit to specific queue types.
|
|
90
136
|
|
|
137
|
+
Use -- to pass arbitrary Celery arguments directly to the worker.
|
|
138
|
+
|
|
91
139
|
Examples:
|
|
92
140
|
|
|
93
141
|
# Start a worker processing all queues
|
|
@@ -107,6 +155,15 @@ def run_worker(
|
|
|
107
155
|
|
|
108
156
|
# Start with custom log level
|
|
109
157
|
pyworkflow worker run --loglevel debug
|
|
158
|
+
|
|
159
|
+
# Enable autoscaling (min 2, max 10 workers)
|
|
160
|
+
pyworkflow worker run --step --autoscale 2,10
|
|
161
|
+
|
|
162
|
+
# Set task limits
|
|
163
|
+
pyworkflow worker run --max-tasks-per-child 100 --time-limit 300
|
|
164
|
+
|
|
165
|
+
# Pass arbitrary Celery arguments after --
|
|
166
|
+
pyworkflow worker run -- --max-memory-per-child=200000
|
|
110
167
|
"""
|
|
111
168
|
# Get config from CLI context (TOML config)
|
|
112
169
|
config = ctx.obj.get("config", {})
|
|
@@ -124,6 +181,9 @@ def run_worker(
|
|
|
124
181
|
merged_config["celery"] = yaml_config["celery"]
|
|
125
182
|
config = merged_config
|
|
126
183
|
|
|
184
|
+
# Get extra args passed after --
|
|
185
|
+
extra_args = ctx.args
|
|
186
|
+
|
|
127
187
|
# Determine queues to process
|
|
128
188
|
queues = []
|
|
129
189
|
if queue_workflow:
|
|
@@ -158,11 +218,21 @@ def run_worker(
|
|
|
158
218
|
|
|
159
219
|
loguru_logger.enable("pyworkflow")
|
|
160
220
|
|
|
221
|
+
# Get Sentinel master from CLI option, config file, or environment
|
|
222
|
+
sentinel_master_name = sentinel_master or celery_config.get(
|
|
223
|
+
"sentinel_master",
|
|
224
|
+
os.getenv("PYWORKFLOW_CELERY_SENTINEL_MASTER"),
|
|
225
|
+
)
|
|
226
|
+
|
|
161
227
|
print_info("Starting Celery worker...")
|
|
162
228
|
print_info(f"Broker: {broker_url}")
|
|
229
|
+
if broker_url.startswith("sentinel://") or broker_url.startswith("sentinel+ssl://"):
|
|
230
|
+
print_info(f"Sentinel master: {sentinel_master_name or 'mymaster'}")
|
|
163
231
|
print_info(f"Queues: {', '.join(queues)}")
|
|
164
232
|
print_info(f"Concurrency: {concurrency}")
|
|
165
233
|
print_info(f"Pool: {pool}")
|
|
234
|
+
if extra_args:
|
|
235
|
+
print_info(f"Extra args: {' '.join(extra_args)}")
|
|
166
236
|
|
|
167
237
|
try:
|
|
168
238
|
# Discover workflows using CLI discovery (reads from --module, env var, or YAML config)
|
|
@@ -177,6 +247,7 @@ def run_worker(
|
|
|
177
247
|
app = create_celery_app(
|
|
178
248
|
broker_url=broker_url,
|
|
179
249
|
result_backend=result_backend,
|
|
250
|
+
sentinel_master_name=sentinel_master_name,
|
|
180
251
|
)
|
|
181
252
|
|
|
182
253
|
# Log discovered workflows and steps
|
|
@@ -212,11 +283,12 @@ def run_worker(
|
|
|
212
283
|
worker_args = [
|
|
213
284
|
"worker",
|
|
214
285
|
f"--loglevel={loglevel.upper()}",
|
|
215
|
-
f"--queues={','.join(queues)}",
|
|
216
286
|
f"--concurrency={concurrency}", # Always set (default: 1)
|
|
217
287
|
f"--pool={pool}", # Always set (default: prefork)
|
|
218
288
|
]
|
|
219
289
|
|
|
290
|
+
worker_args.append(f"--queues={','.join(queues)}")
|
|
291
|
+
|
|
220
292
|
if hostname:
|
|
221
293
|
worker_args.append(f"--hostname={hostname}")
|
|
222
294
|
|
|
@@ -224,6 +296,26 @@ def run_worker(
|
|
|
224
296
|
worker_args.append("--beat")
|
|
225
297
|
worker_args.append("--scheduler=pyworkflow.celery.scheduler:PyWorkflowScheduler")
|
|
226
298
|
|
|
299
|
+
# Add new explicit options
|
|
300
|
+
if autoscale:
|
|
301
|
+
worker_args.append(f"--autoscale={autoscale}")
|
|
302
|
+
|
|
303
|
+
if max_tasks_per_child is not None:
|
|
304
|
+
worker_args.append(f"--max-tasks-per-child={max_tasks_per_child}")
|
|
305
|
+
|
|
306
|
+
if prefetch_multiplier is not None:
|
|
307
|
+
worker_args.append(f"--prefetch-multiplier={prefetch_multiplier}")
|
|
308
|
+
|
|
309
|
+
if time_limit is not None:
|
|
310
|
+
worker_args.append(f"--time-limit={time_limit}")
|
|
311
|
+
|
|
312
|
+
if soft_time_limit is not None:
|
|
313
|
+
worker_args.append(f"--soft-time-limit={soft_time_limit}")
|
|
314
|
+
|
|
315
|
+
# Append extra args last (highest priority - they can override anything)
|
|
316
|
+
if extra_args:
|
|
317
|
+
worker_args.extend(extra_args)
|
|
318
|
+
|
|
227
319
|
print_success("Worker starting...")
|
|
228
320
|
print_info("Press Ctrl+C to stop")
|
|
229
321
|
print_info("")
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
pyworkflow/__init__.py,sha256=
|
|
1
|
+
pyworkflow/__init__.py,sha256=s41mjcGKUwCL-2vfD77bvwvoWhPXncFNzylphNfQnG0,6281
|
|
2
2
|
pyworkflow/config.py,sha256=pKwPrpCwBJiDpB-MIjM0U7GW1TFmQFO341pihL5-vTM,14455
|
|
3
3
|
pyworkflow/discovery.py,sha256=snW3l4nvY3Nc067TGlwtn_qdzTU9ybN7YPr8FbvY8iM,8066
|
|
4
4
|
pyworkflow/aws/__init__.py,sha256=Ak_xHcR9LTRX-CwcS0XecYmzrXZw4EM3V9aKBBDEmIk,1741
|
|
@@ -6,10 +6,10 @@ pyworkflow/aws/context.py,sha256=Vjyjip6U1Emg-WA5TlBaxFhcg15rf9mVJiPfT4VywHc,821
|
|
|
6
6
|
pyworkflow/aws/handler.py,sha256=0SnQuIfQVD99QKMCRFPtrsrV_l1LYKFkzPIRx_2UkSI,5849
|
|
7
7
|
pyworkflow/aws/testing.py,sha256=WrRk9wjbycM-UyHFQWNnA83UE9IrYnhfT38WrbxQT2U,8844
|
|
8
8
|
pyworkflow/celery/__init__.py,sha256=FywVyqnT8AYz9cXkr-wel7_-N7dHFsPNASEPMFESf4Q,1179
|
|
9
|
-
pyworkflow/celery/app.py,sha256=
|
|
9
|
+
pyworkflow/celery/app.py,sha256=QXpPXVVuwJv3ToylT0pyz9SgmwjC9hW-9WaIO4wH5OQ,14349
|
|
10
10
|
pyworkflow/celery/loop.py,sha256=mu8cIfMJYgHAoGCN_DdDoNoXK3QHzHpLmrPCyFDQYIY,3016
|
|
11
11
|
pyworkflow/celery/scheduler.py,sha256=Ms4rqRpdpMiLM8l4y3DK-Divunj9afYuUaGGoNQe7P4,11288
|
|
12
|
-
pyworkflow/celery/singleton.py,sha256=
|
|
12
|
+
pyworkflow/celery/singleton.py,sha256=9gdVHzqFjShZ9OJOJlJNABUg9oqnl6ITGROtomcOtsg,16070
|
|
13
13
|
pyworkflow/celery/tasks.py,sha256=BNHZwWTSRc3q8EgAy4tEmXAm6O0vtVLgrG7MrO0ZZXA,86049
|
|
14
14
|
pyworkflow/cli/__init__.py,sha256=tcbe-fcZmyeEKUy_aEo8bsEF40HsNKOwvyMBZIJZPwc,3844
|
|
15
15
|
pyworkflow/cli/__main__.py,sha256=LxLLS4FEEPXa5rWpLTtKuivn6Xp9pGia-QKGoxt9SS0,148
|
|
@@ -20,7 +20,7 @@ pyworkflow/cli/commands/runs.py,sha256=dkAx0WSBLyooD-vUUDPqgrmM3ElFwqO4nycEZGkNq
|
|
|
20
20
|
pyworkflow/cli/commands/scheduler.py,sha256=w2iUoJ1CtEtOg_4TWslTHbzEPVsV-YybqWU9jkf38gs,3706
|
|
21
21
|
pyworkflow/cli/commands/schedules.py,sha256=UCKZLTWsiLwCewCEXmqOVQnptvvuIKsWSTXai61RYbM,23466
|
|
22
22
|
pyworkflow/cli/commands/setup.py,sha256=J-9lvz3m2sZiiLzQtQIfjmX0l8IpJ4L-xp5U4P7UmRY,32256
|
|
23
|
-
pyworkflow/cli/commands/worker.py,sha256=
|
|
23
|
+
pyworkflow/cli/commands/worker.py,sha256=3TfKAQ3MxYyAEGt4TuPTLfuLN-igtrYeu4cTiLsJoq8,14982
|
|
24
24
|
pyworkflow/cli/commands/workflows.py,sha256=zRBFeqCa4Uo_wwEjgk0SBmkqgcaMznS6ghe1N0ub8Zs,42673
|
|
25
25
|
pyworkflow/cli/output/__init__.py,sha256=5VxKL3mXah5rCKmctxcAKVwp42T47qT1oBK5LFVHHEg,48
|
|
26
26
|
pyworkflow/cli/output/formatters.py,sha256=QzsgPR3cjIbH0723wuG_HzUx9xC7XMA6-NkT2y2lwtM,8785
|
|
@@ -86,9 +86,9 @@ pyworkflow/storage/sqlite.py,sha256=qDhFjyFAenwYq6MF_66FFhDaBG7CEr7ni9Uy72X7MvQ,
|
|
|
86
86
|
pyworkflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
87
87
|
pyworkflow/utils/duration.py,sha256=C-itmiSQQlplw7j6XB679hLF9xYGnyCwm7twO88OF8U,3978
|
|
88
88
|
pyworkflow/utils/schedule.py,sha256=dO_MkGFyfwZpb0LDlW6BGyZzlPuQIA6dc6j9nk9lc4Y,10691
|
|
89
|
-
pyworkflow_engine-0.1.
|
|
90
|
-
pyworkflow_engine-0.1.
|
|
91
|
-
pyworkflow_engine-0.1.
|
|
92
|
-
pyworkflow_engine-0.1.
|
|
93
|
-
pyworkflow_engine-0.1.
|
|
94
|
-
pyworkflow_engine-0.1.
|
|
89
|
+
pyworkflow_engine-0.1.16.dist-info/licenses/LICENSE,sha256=Y49RCTZ5ayn_yzBcRxnyIFdcMCyuYm150aty_FIznfY,1080
|
|
90
|
+
pyworkflow_engine-0.1.16.dist-info/METADATA,sha256=a0_gpG8bYfcmgaJ_G28Ko5mRdWWls_Gyh3bs4bcnJ_k,19628
|
|
91
|
+
pyworkflow_engine-0.1.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
92
|
+
pyworkflow_engine-0.1.16.dist-info/entry_points.txt,sha256=3IGAfuylnS39U0YX0pxnjrj54kB4iT_bNYrmsiDB-dE,51
|
|
93
|
+
pyworkflow_engine-0.1.16.dist-info/top_level.txt,sha256=FLTv9pQmLDBXrQdLOhTMIS3njFibliMsQEfumqmdzBE,11
|
|
94
|
+
pyworkflow_engine-0.1.16.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|