parsl 2025.6.9__py3-none-any.whl → 2025.6.23__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsl/executors/base.py +5 -3
- parsl/executors/flux/executor.py +2 -0
- parsl/executors/globus_compute.py +2 -0
- parsl/executors/high_throughput/executor.py +2 -0
- parsl/executors/high_throughput/mpi_executor.py +7 -0
- parsl/executors/radical/executor.py +3 -0
- parsl/executors/taskvine/executor.py +2 -0
- parsl/executors/threads.py +11 -2
- parsl/executors/workqueue/executor.py +2 -0
- parsl/monitoring/errors.py +0 -5
- parsl/monitoring/monitoring.py +19 -63
- parsl/monitoring/radios/filesystem.py +1 -1
- parsl/monitoring/radios/filesystem_router.py +36 -4
- parsl/monitoring/radios/htex.py +1 -1
- parsl/monitoring/radios/udp_router.py +76 -24
- parsl/monitoring/radios/zmq_router.py +9 -10
- parsl/tests/conftest.py +7 -4
- parsl/tests/test_monitoring/{test_fuzz_zmq.py → test_htex_fuzz_zmq.py} +7 -2
- parsl/tests/test_monitoring/test_radio_filesystem.py +50 -0
- parsl/tests/test_monitoring/test_radio_udp.py +53 -0
- parsl/tests/test_shutdown/test_kill_monitoring.py +1 -1
- parsl/version.py +1 -1
- {parsl-2025.6.9.dist-info → parsl-2025.6.23.dist-info}/METADATA +2 -2
- {parsl-2025.6.9.dist-info → parsl-2025.6.23.dist-info}/RECORD +32 -32
- parsl/tests/configs/local_threads_monitoring.py +0 -10
- parsl/tests/manual_tests/test_udp_simple.py +0 -51
- {parsl-2025.6.9.data → parsl-2025.6.23.data}/scripts/exec_parsl_function.py +0 -0
- {parsl-2025.6.9.data → parsl-2025.6.23.data}/scripts/interchange.py +0 -0
- {parsl-2025.6.9.data → parsl-2025.6.23.data}/scripts/parsl_coprocess.py +0 -0
- {parsl-2025.6.9.data → parsl-2025.6.23.data}/scripts/process_worker_pool.py +0 -0
- {parsl-2025.6.9.dist-info → parsl-2025.6.23.dist-info}/LICENSE +0 -0
- {parsl-2025.6.9.dist-info → parsl-2025.6.23.dist-info}/WHEEL +0 -0
- {parsl-2025.6.9.dist-info → parsl-2025.6.23.dist-info}/entry_points.txt +0 -0
- {parsl-2025.6.9.dist-info → parsl-2025.6.23.dist-info}/top_level.txt +0 -0
parsl/executors/base.py
CHANGED
@@ -98,15 +98,17 @@ class ParslExecutor(metaclass=ABCMeta):
|
|
98
98
|
def shutdown(self) -> None:
|
99
99
|
"""Shutdown the executor.
|
100
100
|
|
101
|
-
|
101
|
+
Executors should call super().shutdown() as part of their overridden
|
102
|
+
implementation.
|
102
103
|
"""
|
103
104
|
pass
|
104
105
|
|
105
106
|
def monitor_resources(self) -> bool:
|
106
107
|
"""Should resource monitoring happen for tasks on running on this executor?
|
107
108
|
|
108
|
-
Parsl resource monitoring conflicts with execution styles which
|
109
|
-
|
109
|
+
Parsl resource monitoring conflicts with execution styles which do
|
110
|
+
not directly use a process tree - for example, the ThreadPoolExecutor
|
111
|
+
and the MPIExecutor.
|
110
112
|
|
111
113
|
This function allows resource monitoring to be disabled per executor implementation.
|
112
114
|
"""
|
parsl/executors/flux/executor.py
CHANGED
@@ -134,3 +134,5 @@ class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
|
|
134
134
|
self.executor.shutdown(wait=False, cancel_futures=True)
|
135
135
|
result_watcher = self.executor._get_result_watcher()
|
136
136
|
result_watcher.shutdown(wait=False, cancel_futures=True)
|
137
|
+
|
138
|
+
super().shutdown()
|
@@ -881,6 +881,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
|
|
881
881
|
if self.zmq_monitoring:
|
882
882
|
self.zmq_monitoring.close()
|
883
883
|
|
884
|
+
super().shutdown()
|
885
|
+
|
884
886
|
logger.info("Finished HighThroughputExecutor shutdown attempt")
|
885
887
|
|
886
888
|
def get_usage_information(self):
|
@@ -111,3 +111,10 @@ class MPIExecutor(HighThroughputExecutor):
|
|
111
111
|
|
112
112
|
def validate_resource_spec(self, resource_specification: dict):
|
113
113
|
return validate_resource_spec(resource_specification)
|
114
|
+
|
115
|
+
def monitor_resources(self):
|
116
|
+
"""Resource monitoring does not make sense when using the
|
117
|
+
MPIExecutor, as the process tree launched for each task is spread
|
118
|
+
across multiple OS images/worker nodes.
|
119
|
+
"""
|
120
|
+
return False
|
@@ -601,6 +601,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
|
|
601
601
|
self._finished_task_queue.close()
|
602
602
|
self._finished_task_queue.join_thread()
|
603
603
|
|
604
|
+
super().shutdown()
|
605
|
+
|
604
606
|
logger.debug("TaskVine shutdown completed")
|
605
607
|
|
606
608
|
@wrap_with_logs
|
parsl/executors/threads.py
CHANGED
@@ -73,9 +73,18 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
|
|
73
73
|
"""
|
74
74
|
logger.debug("Shutting down executor, which involves waiting for running tasks to complete")
|
75
75
|
self.executor.shutdown(wait=block)
|
76
|
+
|
77
|
+
super().shutdown()
|
78
|
+
|
76
79
|
logger.debug("Done with executor shutdown")
|
77
80
|
|
78
81
|
def monitor_resources(self):
|
79
|
-
"""Resource monitoring
|
80
|
-
|
82
|
+
"""Resource monitoring does not make sense when using the
|
83
|
+
ThreadPoolExecutor, as there is no per-task process tree: all tasks
|
84
|
+
run inside the same single submitting process.
|
85
|
+
|
86
|
+
In addition, the use of fork-based multiprocessing in the remote
|
87
|
+
wrapper in parsl/monitoring/remote.py was especially prone to deadlock
|
88
|
+
with this executor.
|
89
|
+
"""
|
81
90
|
return False
|
@@ -714,6 +714,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
|
|
714
714
|
self.collector_queue.close()
|
715
715
|
self.collector_queue.join_thread()
|
716
716
|
|
717
|
+
super().shutdown()
|
718
|
+
|
717
719
|
logger.debug("Work Queue shutdown completed")
|
718
720
|
|
719
721
|
@wrap_with_logs
|
parsl/monitoring/errors.py
CHANGED
@@ -1,11 +1,6 @@
|
|
1
1
|
from parsl.errors import ParslError
|
2
2
|
|
3
3
|
|
4
|
-
class MonitoringHubStartError(ParslError):
|
5
|
-
def __str__(self) -> str:
|
6
|
-
return "Hub failed to start"
|
7
|
-
|
8
|
-
|
9
4
|
class MonitoringRouterStartError(ParslError):
|
10
5
|
def __str__(self) -> str:
|
11
6
|
return "Monitoring router failed to start"
|
parsl/monitoring/monitoring.py
CHANGED
@@ -3,16 +3,14 @@ from __future__ import annotations
|
|
3
3
|
import logging
|
4
4
|
import multiprocessing.synchronize as ms
|
5
5
|
import os
|
6
|
-
import queue
|
7
6
|
import warnings
|
8
7
|
from multiprocessing.queues import Queue
|
9
|
-
from typing import
|
8
|
+
from typing import Any, Optional, Union
|
10
9
|
|
11
10
|
import typeguard
|
12
11
|
|
13
|
-
from parsl.monitoring.
|
14
|
-
from parsl.monitoring.radios.
|
15
|
-
from parsl.monitoring.radios.udp_router import udp_router_starter
|
12
|
+
from parsl.monitoring.radios.filesystem_router import start_filesystem_receiver
|
13
|
+
from parsl.monitoring.radios.udp_router import start_udp_receiver
|
16
14
|
from parsl.monitoring.types import TaggedMonitoringMessage
|
17
15
|
from parsl.multiprocessing import (
|
18
16
|
SizedQueue,
|
@@ -119,40 +117,14 @@ class MonitoringHub(RepresentationMixin):
|
|
119
117
|
|
120
118
|
self.monitoring_hub_active = True
|
121
119
|
|
122
|
-
# This annotation is incompatible with typeguard 4.x instrumentation
|
123
|
-
# of local variables: Queue is not subscriptable at runtime, as far
|
124
|
-
# as typeguard is concerned. The more general Queue annotation works,
|
125
|
-
# but does not restrict the contents of the Queue. Using TYPE_CHECKING
|
126
|
-
# here allows the stricter definition to be seen by mypy, and the
|
127
|
-
# simpler definition to be seen by typeguard. Hopefully at some point
|
128
|
-
# in the future, Queue will allow runtime subscripts.
|
129
|
-
|
130
|
-
if TYPE_CHECKING:
|
131
|
-
udp_comm_q: Queue[Union[int, str]]
|
132
|
-
else:
|
133
|
-
udp_comm_q: Queue
|
134
|
-
|
135
|
-
udp_comm_q = SizedQueue(maxsize=10)
|
136
|
-
|
137
120
|
self.resource_msgs: Queue[TaggedMonitoringMessage]
|
138
121
|
self.resource_msgs = SizedQueue()
|
139
122
|
|
140
|
-
self.
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
"resource_msgs": self.resource_msgs,
|
146
|
-
"exit_event": self.router_exit_event,
|
147
|
-
"hub_address": self.hub_address,
|
148
|
-
"udp_port": self.hub_port,
|
149
|
-
"run_dir": dfk_run_dir,
|
150
|
-
"logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
|
151
|
-
},
|
152
|
-
name="Monitoring-UDP-Router-Process",
|
153
|
-
daemon=True,
|
154
|
-
)
|
155
|
-
self.udp_router_proc.start()
|
123
|
+
self.udp_receiver = start_udp_receiver(debug=self.monitoring_debug,
|
124
|
+
logdir=dfk_run_dir,
|
125
|
+
monitoring_messages=self.resource_msgs,
|
126
|
+
port=self.hub_port
|
127
|
+
)
|
156
128
|
|
157
129
|
self.dbm_exit_event: ms.Event
|
158
130
|
self.dbm_exit_event = SpawnEvent()
|
@@ -169,30 +141,15 @@ class MonitoringHub(RepresentationMixin):
|
|
169
141
|
)
|
170
142
|
self.dbm_proc.start()
|
171
143
|
logger.info("Started UDP router process %s and DBM process %s",
|
172
|
-
self.
|
173
|
-
|
174
|
-
self.
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
try:
|
183
|
-
udp_comm_q_result = udp_comm_q.get(block=True, timeout=120)
|
184
|
-
udp_comm_q.close()
|
185
|
-
udp_comm_q.join_thread()
|
186
|
-
except queue.Empty:
|
187
|
-
logger.error("Monitoring UDP router has not reported port in 120s. Aborting")
|
188
|
-
raise MonitoringHubStartError()
|
189
|
-
|
190
|
-
if isinstance(udp_comm_q_result, str):
|
191
|
-
logger.error("MonitoringRouter sent an error message: %s", udp_comm_q_result)
|
192
|
-
raise RuntimeError(f"MonitoringRouter failed to start: {udp_comm_q_result}")
|
193
|
-
|
194
|
-
udp_port = udp_comm_q_result
|
195
|
-
self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, udp_port)
|
144
|
+
self.udp_receiver.process.pid, self.dbm_proc.pid)
|
145
|
+
|
146
|
+
self.filesystem_receiver = start_filesystem_receiver(debug=self.monitoring_debug,
|
147
|
+
logdir=dfk_run_dir,
|
148
|
+
monitoring_messages=self.resource_msgs
|
149
|
+
)
|
150
|
+
logger.info("Started filesystem radio receiver process %s", self.filesystem_receiver.process.pid)
|
151
|
+
|
152
|
+
self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, self.udp_receiver.port)
|
196
153
|
|
197
154
|
logger.info("Monitoring Hub initialized")
|
198
155
|
|
@@ -201,10 +158,9 @@ class MonitoringHub(RepresentationMixin):
|
|
201
158
|
if self.monitoring_hub_active:
|
202
159
|
self.monitoring_hub_active = False
|
203
160
|
logger.info("Setting router termination event")
|
204
|
-
self.router_exit_event.set()
|
205
161
|
|
206
162
|
logger.info("Waiting for UDP router to terminate")
|
207
|
-
|
163
|
+
self.udp_receiver.close()
|
208
164
|
|
209
165
|
logger.debug("Finished waiting for router termination")
|
210
166
|
logger.debug("Waiting for DB termination")
|
@@ -213,7 +169,7 @@ class MonitoringHub(RepresentationMixin):
|
|
213
169
|
logger.debug("Finished waiting for DBM termination")
|
214
170
|
|
215
171
|
logger.info("Terminating filesystem radio receiver process")
|
216
|
-
|
172
|
+
self.filesystem_receiver.close()
|
217
173
|
|
218
174
|
logger.info("Closing monitoring multiprocessing queues")
|
219
175
|
self.resource_msgs.close()
|
@@ -27,7 +27,7 @@ class FilesystemRadioSender(MonitoringRadioSender):
|
|
27
27
|
"""
|
28
28
|
|
29
29
|
def __init__(self, *, monitoring_url: str, timeout: int = 10, run_dir: str):
|
30
|
-
logger.info("filesystem based monitoring
|
30
|
+
logger.info("filesystem based monitoring radio initializing")
|
31
31
|
self.base_path = f"{run_dir}/monitor-fs-radio/"
|
32
32
|
self.tmp_path = f"{self.base_path}/tmp"
|
33
33
|
self.new_path = f"{self.base_path}/new"
|
@@ -4,6 +4,7 @@ import logging
|
|
4
4
|
import os
|
5
5
|
import pickle
|
6
6
|
import time
|
7
|
+
from multiprocessing.context import SpawnProcess
|
7
8
|
from multiprocessing.queues import Queue
|
8
9
|
from multiprocessing.synchronize import Event
|
9
10
|
from typing import cast
|
@@ -11,15 +12,17 @@ from typing import cast
|
|
11
12
|
from parsl.log_utils import set_file_logger
|
12
13
|
from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
|
13
14
|
from parsl.monitoring.types import TaggedMonitoringMessage
|
15
|
+
from parsl.multiprocessing import SpawnEvent, join_terminate_close_proc
|
14
16
|
from parsl.process_loggers import wrap_with_logs
|
15
17
|
from parsl.utils import setproctitle
|
16
18
|
|
19
|
+
logger = logging.getLogger(__name__)
|
20
|
+
|
17
21
|
|
18
22
|
@wrap_with_logs
|
19
|
-
def filesystem_router_starter(q: Queue[TaggedMonitoringMessage], run_dir: str, exit_event: Event) -> None:
|
20
|
-
|
21
|
-
|
22
|
-
level=logging.INFO)
|
23
|
+
def filesystem_router_starter(*, q: Queue[TaggedMonitoringMessage], run_dir: str, exit_event: Event) -> None:
|
24
|
+
set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
|
25
|
+
level=logging.INFO)
|
23
26
|
|
24
27
|
logger.info("Starting filesystem radio receiver")
|
25
28
|
setproctitle("parsl: monitoring filesystem receiver")
|
@@ -52,3 +55,32 @@ def filesystem_router_starter(q: Queue[TaggedMonitoringMessage], run_dir: str, e
|
|
52
55
|
|
53
56
|
time.sleep(1) # whats a good time for this poll?
|
54
57
|
logger.info("Ending filesystem radio receiver")
|
58
|
+
|
59
|
+
|
60
|
+
class FilesystemRadioReceiver():
|
61
|
+
def __init__(self, *, process: SpawnProcess, exit_event: Event) -> None:
|
62
|
+
self.process = process
|
63
|
+
self.exit_event = exit_event
|
64
|
+
|
65
|
+
def close(self) -> None:
|
66
|
+
self.exit_event.set()
|
67
|
+
join_terminate_close_proc(self.process)
|
68
|
+
|
69
|
+
|
70
|
+
def start_filesystem_receiver(*,
|
71
|
+
monitoring_messages: Queue,
|
72
|
+
logdir: str,
|
73
|
+
debug: bool) -> FilesystemRadioReceiver:
|
74
|
+
|
75
|
+
router_exit_event = SpawnEvent()
|
76
|
+
|
77
|
+
filesystem_proc = SpawnProcess(target=filesystem_router_starter,
|
78
|
+
kwargs={"q": monitoring_messages,
|
79
|
+
"run_dir": logdir,
|
80
|
+
"exit_event": router_exit_event},
|
81
|
+
name="Monitoring-Filesystem-Process",
|
82
|
+
daemon=True
|
83
|
+
)
|
84
|
+
filesystem_proc.start()
|
85
|
+
|
86
|
+
return FilesystemRadioReceiver(process=filesystem_proc, exit_event=router_exit_event)
|
parsl/monitoring/radios/htex.py
CHANGED
@@ -18,7 +18,7 @@ class HTEXRadioSender(MonitoringRadioSender):
|
|
18
18
|
timeout : int
|
19
19
|
timeout, default=10s
|
20
20
|
"""
|
21
|
-
logger.info("htex-based monitoring
|
21
|
+
logger.info("htex-based monitoring radio initialising")
|
22
22
|
|
23
23
|
def send(self, message: object) -> None:
|
24
24
|
""" Sends a message to the UDP receiver
|
@@ -4,15 +4,26 @@ import logging
|
|
4
4
|
import multiprocessing.queues as mpq
|
5
5
|
import os
|
6
6
|
import pickle
|
7
|
+
import queue
|
7
8
|
import socket
|
8
9
|
import time
|
10
|
+
from multiprocessing.context import SpawnProcess as SpawnProcessType
|
11
|
+
from multiprocessing.queues import Queue
|
9
12
|
from multiprocessing.synchronize import Event
|
10
|
-
from
|
13
|
+
from multiprocessing.synchronize import Event as EventType
|
14
|
+
from typing import Optional, Union
|
11
15
|
|
12
16
|
import typeguard
|
13
17
|
|
14
18
|
from parsl.log_utils import set_file_logger
|
19
|
+
from parsl.monitoring.errors import MonitoringRouterStartError
|
15
20
|
from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
|
21
|
+
from parsl.multiprocessing import (
|
22
|
+
SizedQueue,
|
23
|
+
SpawnEvent,
|
24
|
+
SpawnProcess,
|
25
|
+
join_terminate_close_proc,
|
26
|
+
)
|
16
27
|
from parsl.process_loggers import wrap_with_logs
|
17
28
|
from parsl.utils import setproctitle
|
18
29
|
|
@@ -23,10 +34,7 @@ class MonitoringRouter:
|
|
23
34
|
|
24
35
|
def __init__(self,
|
25
36
|
*,
|
26
|
-
hub_address: str,
|
27
37
|
udp_port: Optional[int] = None,
|
28
|
-
|
29
|
-
monitoring_hub_address: str = "127.0.0.1",
|
30
38
|
run_dir: str = ".",
|
31
39
|
logging_level: int = logging.INFO,
|
32
40
|
atexit_timeout: int = 3, # in seconds
|
@@ -37,8 +45,6 @@ class MonitoringRouter:
|
|
37
45
|
|
38
46
|
Parameters
|
39
47
|
----------
|
40
|
-
hub_address : str
|
41
|
-
The ip address at which the workers will be able to reach the Hub.
|
42
48
|
udp_port : int
|
43
49
|
The specific port at which workers will be able to reach the Hub via UDP. Default: None
|
44
50
|
run_dir : str
|
@@ -53,12 +59,10 @@ class MonitoringRouter:
|
|
53
59
|
An event that the main Parsl process will set to signal that the monitoring router should shut down.
|
54
60
|
"""
|
55
61
|
os.makedirs(run_dir, exist_ok=True)
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
self.logger.debug("Monitoring router starting")
|
62
|
+
set_file_logger(f"{run_dir}/monitoring_udp_router.log",
|
63
|
+
level=logging_level)
|
64
|
+
logger.debug("Monitoring router starting")
|
60
65
|
|
61
|
-
self.hub_address = hub_address
|
62
66
|
self.atexit_timeout = atexit_timeout
|
63
67
|
|
64
68
|
self.loop_freq = 10.0 # milliseconds
|
@@ -79,39 +83,39 @@ class MonitoringRouter:
|
|
79
83
|
except Exception as e:
|
80
84
|
raise RuntimeError(f"Could not bind to udp_port {udp_port} because: {e}")
|
81
85
|
self.udp_sock.settimeout(self.loop_freq / 1000)
|
82
|
-
|
86
|
+
logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
|
83
87
|
|
84
88
|
self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
|
85
89
|
self.exit_event = exit_event
|
86
90
|
|
87
|
-
@wrap_with_logs
|
91
|
+
@wrap_with_logs
|
88
92
|
def start(self) -> None:
|
89
|
-
|
93
|
+
logger.info("Starting UDP listener")
|
90
94
|
try:
|
91
95
|
while not self.exit_event.is_set():
|
92
96
|
try:
|
93
97
|
data, addr = self.udp_sock.recvfrom(2048)
|
94
98
|
resource_msg = pickle.loads(data)
|
95
|
-
|
99
|
+
logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
|
96
100
|
self.target_radio.send(resource_msg)
|
97
101
|
except socket.timeout:
|
98
102
|
pass
|
99
103
|
|
100
|
-
|
104
|
+
logger.info("UDP listener draining")
|
101
105
|
last_msg_received_time = time.time()
|
102
106
|
while time.time() - last_msg_received_time < self.atexit_timeout:
|
103
107
|
try:
|
104
108
|
data, addr = self.udp_sock.recvfrom(2048)
|
105
109
|
msg = pickle.loads(data)
|
106
|
-
|
110
|
+
logger.debug("Got UDP Message from {}: {}".format(addr, msg))
|
107
111
|
self.target_radio.send(msg)
|
108
112
|
last_msg_received_time = time.time()
|
109
113
|
except socket.timeout:
|
110
114
|
pass
|
111
115
|
|
112
|
-
|
116
|
+
logger.info("UDP listener finishing normally")
|
113
117
|
finally:
|
114
|
-
|
118
|
+
logger.info("UDP listener finished")
|
115
119
|
|
116
120
|
|
117
121
|
@wrap_with_logs
|
@@ -121,15 +125,13 @@ def udp_router_starter(*,
|
|
121
125
|
resource_msgs: mpq.Queue,
|
122
126
|
exit_event: Event,
|
123
127
|
|
124
|
-
hub_address: str,
|
125
128
|
udp_port: Optional[int],
|
126
129
|
|
127
130
|
run_dir: str,
|
128
131
|
logging_level: int) -> None:
|
129
132
|
setproctitle("parsl: monitoring UDP router")
|
130
133
|
try:
|
131
|
-
router = MonitoringRouter(
|
132
|
-
udp_port=udp_port,
|
134
|
+
router = MonitoringRouter(udp_port=udp_port,
|
133
135
|
run_dir=run_dir,
|
134
136
|
logging_level=logging_level,
|
135
137
|
resource_msgs=resource_msgs,
|
@@ -140,8 +142,58 @@ def udp_router_starter(*,
|
|
140
142
|
else:
|
141
143
|
comm_q.put(router.udp_port)
|
142
144
|
|
143
|
-
|
145
|
+
logger.info("Starting MonitoringRouter in router_starter")
|
144
146
|
try:
|
145
147
|
router.start()
|
146
148
|
except Exception:
|
147
|
-
|
149
|
+
logger.exception("UDP router start exception")
|
150
|
+
|
151
|
+
|
152
|
+
class UDPRadioReceiver():
|
153
|
+
def __init__(self, *, process: SpawnProcessType, exit_event: EventType, port: int) -> None:
|
154
|
+
self.process = process
|
155
|
+
self.exit_event = exit_event
|
156
|
+
self.port = port
|
157
|
+
|
158
|
+
def close(self) -> None:
|
159
|
+
self.exit_event.set()
|
160
|
+
join_terminate_close_proc(self.process)
|
161
|
+
|
162
|
+
|
163
|
+
def start_udp_receiver(*,
|
164
|
+
monitoring_messages: Queue,
|
165
|
+
port: Optional[int],
|
166
|
+
logdir: str,
|
167
|
+
debug: bool) -> UDPRadioReceiver:
|
168
|
+
|
169
|
+
udp_comm_q: Queue[Union[int, str]]
|
170
|
+
udp_comm_q = SizedQueue(maxsize=10)
|
171
|
+
|
172
|
+
router_exit_event = SpawnEvent()
|
173
|
+
|
174
|
+
router_proc = SpawnProcess(target=udp_router_starter,
|
175
|
+
kwargs={"comm_q": udp_comm_q,
|
176
|
+
"resource_msgs": monitoring_messages,
|
177
|
+
"exit_event": router_exit_event,
|
178
|
+
"udp_port": port,
|
179
|
+
"run_dir": logdir,
|
180
|
+
"logging_level": logging.DEBUG if debug else logging.INFO,
|
181
|
+
},
|
182
|
+
name="Monitoring-UDP-Router-Process",
|
183
|
+
daemon=True,
|
184
|
+
)
|
185
|
+
router_proc.start()
|
186
|
+
|
187
|
+
try:
|
188
|
+
udp_comm_q_result = udp_comm_q.get(block=True, timeout=120)
|
189
|
+
udp_comm_q.close()
|
190
|
+
udp_comm_q.join_thread()
|
191
|
+
except queue.Empty:
|
192
|
+
logger.error("Monitoring UDP router has not reported port in 120s. Aborting")
|
193
|
+
raise MonitoringRouterStartError()
|
194
|
+
|
195
|
+
if isinstance(udp_comm_q_result, str):
|
196
|
+
logger.error("MonitoringRouter sent an error message: %s", udp_comm_q_result)
|
197
|
+
raise RuntimeError(f"MonitoringRouter failed to start: {udp_comm_q_result}")
|
198
|
+
|
199
|
+
return UDPRadioReceiver(process=router_proc, exit_event=router_exit_event, port=udp_comm_q_result)
|
@@ -61,10 +61,9 @@ class MonitoringRouter:
|
|
61
61
|
An event that the main Parsl process will set to signal that the monitoring router should shut down.
|
62
62
|
"""
|
63
63
|
os.makedirs(run_dir, exist_ok=True)
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
self.logger.debug("Monitoring router starting")
|
64
|
+
set_file_logger(f"{run_dir}/monitoring_zmq_router.log",
|
65
|
+
level=logging_level)
|
66
|
+
logger.debug("Monitoring router starting")
|
68
67
|
|
69
68
|
self.address = address
|
70
69
|
|
@@ -75,7 +74,7 @@ class MonitoringRouter:
|
|
75
74
|
self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
|
76
75
|
self.zmq_receiver_channel.set_hwm(0)
|
77
76
|
self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq) # in milliseconds
|
78
|
-
|
77
|
+
logger.debug("address: {}. port_range {}".format(address, port_range))
|
79
78
|
self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port(tcp_url(address),
|
80
79
|
min_port=port_range[0],
|
81
80
|
max_port=port_range[1])
|
@@ -83,9 +82,9 @@ class MonitoringRouter:
|
|
83
82
|
self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
|
84
83
|
self.exit_event = exit_event
|
85
84
|
|
86
|
-
@wrap_with_logs
|
85
|
+
@wrap_with_logs
|
87
86
|
def start(self) -> None:
|
88
|
-
|
87
|
+
logger.info("Starting ZMQ listener")
|
89
88
|
try:
|
90
89
|
while not self.exit_event.is_set():
|
91
90
|
try:
|
@@ -107,11 +106,11 @@ class MonitoringRouter:
|
|
107
106
|
# channel is broken in such a way that it always raises
|
108
107
|
# an exception? Looping on this would maybe be the wrong
|
109
108
|
# thing to do.
|
110
|
-
|
109
|
+
logger.warning("Failure processing a ZMQ message", exc_info=True)
|
111
110
|
|
112
|
-
|
111
|
+
logger.info("ZMQ listener finishing normally")
|
113
112
|
finally:
|
114
|
-
|
113
|
+
logger.info("ZMQ listener finished")
|
115
114
|
|
116
115
|
|
117
116
|
@wrap_with_logs
|
parsl/tests/conftest.py
CHANGED
@@ -406,13 +406,15 @@ def try_assert():
|
|
406
406
|
timeout_ms: float = 5000,
|
407
407
|
attempts: int = 0,
|
408
408
|
check_period_ms: int = 20,
|
409
|
+
factor: float = 2,
|
409
410
|
):
|
410
411
|
tb = create_traceback(start=1)
|
411
412
|
check_period_s = abs(check_period_ms) / 1000.0
|
412
413
|
if attempts > 0:
|
413
414
|
for _attempt_no in range(attempts):
|
414
|
-
|
415
|
-
check_period_s
|
415
|
+
fraction = random.random()
|
416
|
+
time.sleep(fraction * check_period_s) # jitter
|
417
|
+
check_period_s *= factor ** fraction
|
416
418
|
if test_func():
|
417
419
|
return
|
418
420
|
else:
|
@@ -427,9 +429,10 @@ def try_assert():
|
|
427
429
|
timeout_s = abs(timeout_ms) / 1000.0
|
428
430
|
end = time.monotonic() + timeout_s
|
429
431
|
while time.monotonic() < end:
|
430
|
-
|
432
|
+
fraction = random.random()
|
433
|
+
wait_for = fraction * check_period_s # jitter
|
431
434
|
time.sleep(min(wait_for, end - time.monotonic()))
|
432
|
-
check_period_s *=
|
435
|
+
check_period_s *= factor ** fraction
|
433
436
|
if test_func():
|
434
437
|
return
|
435
438
|
att_fail = (
|
@@ -17,7 +17,12 @@ def this_app():
|
|
17
17
|
|
18
18
|
|
19
19
|
@pytest.mark.local
|
20
|
-
def
|
20
|
+
def test_fuzz():
|
21
|
+
"""This test sends fuzz into the ZMQ radio receiver that HTEX starts
|
22
|
+
for receiving monitoring messages from the interchange, and checks
|
23
|
+
that monitoring still records things ok.
|
24
|
+
"""
|
25
|
+
|
21
26
|
import sqlalchemy
|
22
27
|
from sqlalchemy import text
|
23
28
|
|
@@ -44,7 +49,7 @@ def test_row_counts():
|
|
44
49
|
# the latter is what i'm most suspicious of in my present investigation
|
45
50
|
|
46
51
|
# dig out the interchange port...
|
47
|
-
hub_address = parsl.dfk().
|
52
|
+
hub_address = parsl.dfk().executors["htex_Local"].loopback_address
|
48
53
|
hub_zmq_port = parsl.dfk().executors["htex_Local"].hub_zmq_port
|
49
54
|
|
50
55
|
# this will send a string to a new socket connection
|
@@ -0,0 +1,50 @@
|
|
1
|
+
import pytest
|
2
|
+
|
3
|
+
from parsl.monitoring.message_type import MessageType
|
4
|
+
from parsl.monitoring.radios.filesystem import FilesystemRadioSender
|
5
|
+
from parsl.monitoring.radios.filesystem_router import start_filesystem_receiver
|
6
|
+
from parsl.multiprocessing import SpawnQueue
|
7
|
+
|
8
|
+
|
9
|
+
@pytest.mark.local
|
10
|
+
def test_filesystem(tmpd_cwd):
|
11
|
+
"""Test filesystem radio/receiver pair.
|
12
|
+
This test checks that the pair can be started up locally, that a message
|
13
|
+
is conveyed from radio to receiver, and that the receiver process goes
|
14
|
+
away at shutdown.
|
15
|
+
"""
|
16
|
+
|
17
|
+
resource_msgs = SpawnQueue()
|
18
|
+
|
19
|
+
# start receiver
|
20
|
+
receiver = start_filesystem_receiver(debug=True,
|
21
|
+
logdir=str(tmpd_cwd),
|
22
|
+
monitoring_messages=resource_msgs,
|
23
|
+
)
|
24
|
+
|
25
|
+
# make radio
|
26
|
+
|
27
|
+
radio_sender = FilesystemRadioSender(run_dir=str(tmpd_cwd),
|
28
|
+
monitoring_url="irrelevant:")
|
29
|
+
|
30
|
+
# send message into radio
|
31
|
+
|
32
|
+
message = (MessageType.RESOURCE_INFO, {})
|
33
|
+
|
34
|
+
radio_sender.send(message)
|
35
|
+
|
36
|
+
# verify it comes out of the receiver
|
37
|
+
|
38
|
+
m = resource_msgs.get()
|
39
|
+
|
40
|
+
assert m == message, "The sent message should appear in the queue"
|
41
|
+
|
42
|
+
# shut down router
|
43
|
+
|
44
|
+
receiver.close()
|
45
|
+
|
46
|
+
# we can't inspect the process if it has been closed properly, but
|
47
|
+
# we can verify that it raises the expected ValueError the closed
|
48
|
+
# processes raise on access.
|
49
|
+
with pytest.raises(ValueError):
|
50
|
+
receiver.process.exitcode
|
@@ -0,0 +1,53 @@
|
|
1
|
+
import pytest
|
2
|
+
|
3
|
+
from parsl.monitoring.message_type import MessageType
|
4
|
+
from parsl.monitoring.radios.udp import UDPRadioSender
|
5
|
+
from parsl.monitoring.radios.udp_router import start_udp_receiver
|
6
|
+
from parsl.multiprocessing import SpawnQueue
|
7
|
+
|
8
|
+
|
9
|
+
@pytest.mark.local
|
10
|
+
def test_udp(tmpd_cwd):
|
11
|
+
"""Test UDP radio/receiver pair.
|
12
|
+
This test checks that the pair can be started up locally, that a message
|
13
|
+
is conveyed from radio to receiver, and that the receiver process goes
|
14
|
+
away at shutdown.
|
15
|
+
"""
|
16
|
+
|
17
|
+
resource_msgs = SpawnQueue()
|
18
|
+
|
19
|
+
# start receiver
|
20
|
+
udp_receiver = start_udp_receiver(debug=True,
|
21
|
+
logdir=str(tmpd_cwd),
|
22
|
+
monitoring_messages=resource_msgs,
|
23
|
+
port=None
|
24
|
+
)
|
25
|
+
|
26
|
+
# make radio
|
27
|
+
|
28
|
+
# this comes from monitoring.py:
|
29
|
+
url = "udp://{}:{}".format("localhost", udp_receiver.port)
|
30
|
+
|
31
|
+
radio_sender = UDPRadioSender(url)
|
32
|
+
|
33
|
+
# send message into radio
|
34
|
+
|
35
|
+
message = (MessageType.RESOURCE_INFO, {})
|
36
|
+
|
37
|
+
radio_sender.send(message)
|
38
|
+
|
39
|
+
# verify it comes out of the receiver
|
40
|
+
|
41
|
+
m = resource_msgs.get()
|
42
|
+
|
43
|
+
assert m == message, "The sent message should appear in the queue"
|
44
|
+
|
45
|
+
# shut down router
|
46
|
+
|
47
|
+
udp_receiver.close()
|
48
|
+
|
49
|
+
# we can't inspect the process if it has been closed properly, but
|
50
|
+
# we can verify that it raises the expected ValueError the closed
|
51
|
+
# processes raise on access.
|
52
|
+
with pytest.raises(ValueError):
|
53
|
+
udp_receiver.process.exitcode
|
@@ -30,7 +30,7 @@ def test_no_kills():
|
|
30
30
|
|
31
31
|
@pytest.mark.local
|
32
32
|
@pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM, signal.SIGKILL, signal.SIGQUIT])
|
33
|
-
@pytest.mark.parametrize("process_attr", ["
|
33
|
+
@pytest.mark.parametrize("process_attr", ["dbm_proc"])
|
34
34
|
def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
|
35
35
|
"""This tests that we can kill a monitoring process and still have successful shutdown.
|
36
36
|
SIGINT emulates some racy behaviour when ctrl-C is pressed: that
|
parsl/version.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: parsl
|
3
|
-
Version: 2025.6.
|
3
|
+
Version: 2025.6.23
|
4
4
|
Summary: Simple data dependent workflows in Python
|
5
5
|
Home-page: https://github.com/Parsl/parsl
|
6
|
-
Download-URL: https://github.com/Parsl/parsl/archive/2025.06.
|
6
|
+
Download-URL: https://github.com/Parsl/parsl/archive/2025.06.23.tar.gz
|
7
7
|
Author: The Parsl Team
|
8
8
|
Author-email: parsl@googlegroups.com
|
9
9
|
License: Apache 2.0
|
@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=JNAfgdZvQSsxVyUp229OOUqWwf_ZUhpmw8X9CdF3i6k,3614
|
|
8
8
|
parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
|
9
9
|
parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
10
|
parsl/utils.py,sha256=codTX6_KLhgeTwNkRzc1lo4bgc1M93eJ-lkqOO98fvk,14331
|
11
|
-
parsl/version.py,sha256=
|
11
|
+
parsl/version.py,sha256=Ic4ZXjmRNaNVe4VgUeJZzbs04AL3PcIpDFnIlIUjmIQ,131
|
12
12
|
parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
13
13
|
parsl/app/app.py,sha256=0gbM4AH2OtFOLsv07I5nglpElcwMSOi-FzdZZfrk7So,8532
|
14
14
|
parsl/app/bash.py,sha256=jm2AvePlCT9DZR7H_4ANDWxatp5dN_22FUlT_gWhZ-g,5528
|
@@ -61,37 +61,37 @@ parsl/dataflow/rundirs.py,sha256=JZdzybVGubY35jL2YiKcDo65ZmRl1WyOApc8ajYxztc,108
|
|
61
61
|
parsl/dataflow/states.py,sha256=hV6mfv-y4A6xrujeQglcomnfEs7y3Xm2g6JFwC6dvgQ,2612
|
62
62
|
parsl/dataflow/taskrecord.py,sha256=qIW7T6hn9dYTuNPdUura3HQwwUpUJACwPP5REm5COf4,3042
|
63
63
|
parsl/executors/__init__.py,sha256=PEuXYrnVqwlaz_nt82s9D_YNaVsX7ET29DeIZRUR8hw,577
|
64
|
-
parsl/executors/base.py,sha256=
|
64
|
+
parsl/executors/base.py,sha256=OApNwTcBrzVXweWFPj8rMrP87qStcvF2facsxaevSqA,4568
|
65
65
|
parsl/executors/errors.py,sha256=ZxL3nK5samPos8Xixo_jpRtPIiRJfZ5D397_qaXj2g0,2515
|
66
66
|
parsl/executors/execute_task.py,sha256=PtqHxk778UQaNah1AN-TJV5emZbOcU5TGtWDxFn3_F4,1079
|
67
|
-
parsl/executors/globus_compute.py,sha256=
|
67
|
+
parsl/executors/globus_compute.py,sha256=p59iBrv2BvYUAZ3YZSpOrm_Wpai592ueiJm3zFS7gvY,5304
|
68
68
|
parsl/executors/status_handling.py,sha256=n2DLASEvKZNgFpviAMYDfqcAsxMiU7QxFemw91YSenc,15746
|
69
|
-
parsl/executors/threads.py,sha256=
|
69
|
+
parsl/executors/threads.py,sha256=JkbjJGtZPJNzsVF3DFQHOsZpKmFr7wcGbH1wbwDPSrw,3754
|
70
70
|
parsl/executors/flux/__init__.py,sha256=P9grTTeRPXfqXurFhlSS7XhmE6tTbnCnyQ1f9b-oYHE,136
|
71
71
|
parsl/executors/flux/execute_parsl_task.py,sha256=zHP5M7ILGiwnoalZ8WsfVVdZM7uP4iQo2ThVh4crxpM,1530
|
72
|
-
parsl/executors/flux/executor.py,sha256=
|
72
|
+
parsl/executors/flux/executor.py,sha256=NLdjOli5VjrSdEfyWbfqKN_8APvFkp_qFCouS_9NeLQ,17069
|
73
73
|
parsl/executors/flux/flux_instance_manager.py,sha256=5T3Rp7ZM-mlT0Pf0Gxgs5_YmnaPrSF9ec7zvRfLfYJw,2129
|
74
74
|
parsl/executors/high_throughput/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
75
75
|
parsl/executors/high_throughput/errors.py,sha256=k2XuvvFdUfNs2foHFnxmS-BToRMfdXpYEa4EF3ELKq4,1554
|
76
|
-
parsl/executors/high_throughput/executor.py,sha256=
|
76
|
+
parsl/executors/high_throughput/executor.py,sha256=JTNgVfZfwzgH9RheMWCCDuNd_1i3LN69B_SYzL7rY3Q,39840
|
77
77
|
parsl/executors/high_throughput/interchange.py,sha256=lnPhV-5OR_rp4fa_Wj0tmG3avwUdPgJH4LTTBZQcrf8,28922
|
78
78
|
parsl/executors/high_throughput/manager_record.py,sha256=ZMsqFxvreGLRXAw3N-JnODDa9Qfizw2tMmcBhm4lco4,490
|
79
79
|
parsl/executors/high_throughput/manager_selector.py,sha256=UKcUE6v0tO7PDMTThpKSKxVpOpOUilxDL7UbNgpZCxo,2116
|
80
80
|
parsl/executors/high_throughput/monitoring_info.py,sha256=HC0drp6nlXQpAop5PTUKNjdXMgtZVvrBL0JzZJebPP4,298
|
81
|
-
parsl/executors/high_throughput/mpi_executor.py,sha256=
|
81
|
+
parsl/executors/high_throughput/mpi_executor.py,sha256=AKc5OX7mNH9eYPiBL1g7Xlrj3P-ouniDgE17bhpIems,5052
|
82
82
|
parsl/executors/high_throughput/mpi_prefix_composer.py,sha256=DmpKugANNa1bdYlqQBLHkrFc15fJpefPPhW9hkAlh1s,4308
|
83
83
|
parsl/executors/high_throughput/mpi_resource_management.py,sha256=73bTW2ZbHRfcrPN318cyjiqDN50AM1cOCQqUGJDIlBg,8199
|
84
84
|
parsl/executors/high_throughput/probe.py,sha256=QOEaliO3x5cB6ltMOZMsZQ-ath9AAuFqXcBzRgWOM60,2754
|
85
85
|
parsl/executors/high_throughput/process_worker_pool.py,sha256=Q7FN0MdXIAOouxDarim6etYVHEgbXFiaMhBahC2ZtIQ,41137
|
86
86
|
parsl/executors/high_throughput/zmq_pipes.py,sha256=NUK25IEh0UkxzdqQQyM8tMtuZmjSiTeWu1DzkkAIOhA,8980
|
87
87
|
parsl/executors/radical/__init__.py,sha256=CKbtV2numw5QvgIBq1htMUrt9TqDCIC2zifyf2svTNU,186
|
88
|
-
parsl/executors/radical/executor.py,sha256=
|
88
|
+
parsl/executors/radical/executor.py,sha256=e3XS4mvug1uJ6wrt4UH6hBgfbDbc-mQH3xUW2ZmBsMQ,22888
|
89
89
|
parsl/executors/radical/rpex_resources.py,sha256=Q7-0u3K447LBCe2y7mVcdw6jqWI7SdPXxCKhkr6FoRQ,5139
|
90
90
|
parsl/executors/radical/rpex_worker.py,sha256=z6r82ZujKb6sdKIdHsQ_5EBMDIQieeGcrlt6kGLFo4M,1830
|
91
91
|
parsl/executors/taskvine/__init__.py,sha256=9rwp3M8B0YyEhZMLO0RHaNw7u1nc01WHbXLqnBTanu0,293
|
92
92
|
parsl/executors/taskvine/errors.py,sha256=euIYkSslrNSI85kyi2s0xzOaO9ik4c1fYHstMIeiBJk,652
|
93
93
|
parsl/executors/taskvine/exec_parsl_function.py,sha256=ftGdJU78lKPPkphSHlEi4rj164mhuMHJjghVqfgeXKk,7085
|
94
|
-
parsl/executors/taskvine/executor.py,sha256=
|
94
|
+
parsl/executors/taskvine/executor.py,sha256=h47c90QoaeoWcqTdJKgfqwQBZVW-kkF44O95sKUfJ8g,30978
|
95
95
|
parsl/executors/taskvine/factory.py,sha256=GU5JryEAKJuYKwrSc162BN-lhcKhapvBZHT820pxwic,2772
|
96
96
|
parsl/executors/taskvine/factory_config.py,sha256=ZQC5vyDe8cM0nuv7fbBCV2xnWGAZ87iLlT2UqmFFI1U,3695
|
97
97
|
parsl/executors/taskvine/manager.py,sha256=SUi5mqqMm_rnkBLrZtTQe7RiHqWDn1oOejQscYzfwAU,25797
|
@@ -100,7 +100,7 @@ parsl/executors/taskvine/utils.py,sha256=iSrIogeiauL3UNy_9tiZp1cBSNn6fIJkMYQRVi1
|
|
100
100
|
parsl/executors/workqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
101
101
|
parsl/executors/workqueue/errors.py,sha256=XO2naYhAsHHyiOBH6hpObg3mPNDmvMoFqErsj0-v7jc,541
|
102
102
|
parsl/executors/workqueue/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
|
103
|
-
parsl/executors/workqueue/executor.py,sha256=
|
103
|
+
parsl/executors/workqueue/executor.py,sha256=R4cBX3Ac2OePpKsXo-4Kq68X6wAkyltYkhLysk45QwY,49855
|
104
104
|
parsl/executors/workqueue/parsl_coprocess.py,sha256=cF1UmTgVLoey6QzBcbYgEiEsRidSaFfuO54f1HFw_EM,5737
|
105
105
|
parsl/executors/workqueue/parsl_coprocess_stub.py,sha256=_bJmpPIgL42qM6bVzeEKt1Mn1trSP41rtJguXxPGfHI,735
|
106
106
|
parsl/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -115,23 +115,23 @@ parsl/launchers/errors.py,sha256=8YMV_CHpBNVa4eXkGE4x5DaFQlZkDCRCHmBktYcY6TA,467
|
|
115
115
|
parsl/launchers/launchers.py,sha256=cQsNsHuCOL_nQTjPXf0--YsgsDoMoJ77bO1Wt4ncLjs,15134
|
116
116
|
parsl/monitoring/__init__.py,sha256=0ywNz6i0lM1xo_7_BIxhETDGeVd2C_0wwD7qgeaMR4c,83
|
117
117
|
parsl/monitoring/db_manager.py,sha256=L0c5S9ockq0UIchT2bjmkSAWXS-t0G-Q_neOIBfLbm0,33444
|
118
|
-
parsl/monitoring/errors.py,sha256=
|
118
|
+
parsl/monitoring/errors.py,sha256=pol4Xp9lkpTurc1F5vJ0g48R4R-nUr8GBHubD7SU9Aw,165
|
119
119
|
parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
|
120
|
-
parsl/monitoring/monitoring.py,sha256=
|
120
|
+
parsl/monitoring/monitoring.py,sha256=8g8JVbR9Nq2JV17EBBwMaFFR-kvKDVd_PWtHyO0vUlQ,7564
|
121
121
|
parsl/monitoring/remote.py,sha256=t0qCTUMCzeJ_JOARFpjqlTNrAWdEb20BxhmZh9X7kEM,13728
|
122
122
|
parsl/monitoring/types.py,sha256=oOCrzv-ab-_rv4pb8o58Sdb8G_RGp1aZriRbdf9zBEk,339
|
123
123
|
parsl/monitoring/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
124
124
|
parsl/monitoring/queries/pandas.py,sha256=0Z2r0rjTKCemf0eaDkF1irvVHn5g7KC5SYETvQPRxwU,2232
|
125
125
|
parsl/monitoring/radios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
126
126
|
parsl/monitoring/radios/base.py,sha256=MBW3aAbxQY7TMuNGZEN15dWoy3mAELOqz-GMN1d7vF4,221
|
127
|
-
parsl/monitoring/radios/filesystem.py,sha256=
|
128
|
-
parsl/monitoring/radios/filesystem_router.py,sha256=
|
129
|
-
parsl/monitoring/radios/htex.py,sha256=
|
127
|
+
parsl/monitoring/radios/filesystem.py,sha256=nn9NJ7EteEAixkqS2b0jJ9M7j55xdouXmYiK0qu5AjY,1763
|
128
|
+
parsl/monitoring/radios/filesystem_router.py,sha256=T6Okb5deWeLwaugb1Bkskjfdz1_KpM6pX2EyQLey8p4,3312
|
129
|
+
parsl/monitoring/radios/htex.py,sha256=1v4on4_KmJa_PuEc3a9OrL4NJV2h0bIuUKh_mdoPVKc,1641
|
130
130
|
parsl/monitoring/radios/multiprocessing.py,sha256=fsfaaoMDp6VJv1DSAl-P0R2ofO6jp13byx6NsPItV3Y,655
|
131
131
|
parsl/monitoring/radios/udp.py,sha256=bTpt7JYp-5hyBBLzgiLj1_BlSTn28UVp39OYgVGLXCw,1613
|
132
|
-
parsl/monitoring/radios/udp_router.py,sha256=
|
132
|
+
parsl/monitoring/radios/udp_router.py,sha256=M7UtHx5dYphKU4YKB2KmkA7aZRtqOFcxuaLVvBn8QJA,7590
|
133
133
|
parsl/monitoring/radios/zmq.py,sha256=fhoHp9ylhf-D3eTJb2aSHRsuic8-FJ_oRNGnniGkCAI,592
|
134
|
-
parsl/monitoring/radios/zmq_router.py,sha256=
|
134
|
+
parsl/monitoring/radios/zmq_router.py,sha256=oqGlOI4OkmvGLNKGp5F0iLFLYj3JdUkpnwg40JOvEak,7826
|
135
135
|
parsl/monitoring/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
136
136
|
parsl/monitoring/visualization/app.py,sha256=xMeRlAnzl5lHddAOdSBcqY3D5lmOYw3Z3Z2_YyoVwnw,1425
|
137
137
|
parsl/monitoring/visualization/models.py,sha256=C7CcF6w6PhtrdvDX9VgDH-aSrpLfvYU1fJ4-HDUeFVQ,5138
|
@@ -196,7 +196,7 @@ parsl/serialize/facade.py,sha256=3uOuVp0epfyLn7qDzuWqLfsy971YVGD3sqwqcAiRwh0,668
|
|
196
196
|
parsl/serialize/proxystore.py,sha256=o-ha9QAvVhbN8y9S1itk3W0O75eyHYZw2AvB2xu5_Lg,1624
|
197
197
|
parsl/tests/__init__.py,sha256=VTtJzOzz_x6fWNh8IOnsgFqVbdiJShi2AZH21mcmID4,204
|
198
198
|
parsl/tests/callables_helper.py,sha256=ceP1YYsNtrZgKT6MAIvpgdccEjQ_CpFEOnZBGHKGOx0,30
|
199
|
-
parsl/tests/conftest.py,sha256=
|
199
|
+
parsl/tests/conftest.py,sha256=PqXpj1AxpPQrcKXJBQ83WIF8TIzZ4-YhAjKQPahE1Tw,15618
|
200
200
|
parsl/tests/test_aalst_patterns.py,sha256=lNIxb7nIgh1yX7hR2fr_ck_mxYJxx8ASKK9zHUVqPno,9614
|
201
201
|
parsl/tests/test_callables.py,sha256=97vrIF1_hfDGd81FM1bhR6FemZMWFcALrH6pVHMTCt8,1974
|
202
202
|
parsl/tests/test_curvezmq.py,sha256=yyhlS4vmaZdMitiySoy4l_ih9H1bsPiN-tMdwIh3H20,12431
|
@@ -230,7 +230,6 @@ parsl/tests/configs/local_threads_checkpoint_task_exit.py,sha256=zHKN68T-xhAVQwQ
|
|
230
230
|
parsl/tests/configs/local_threads_ftp_in_task.py,sha256=c9odRbxgj1bM_ttpkWTh2Ch_MV7f5cmn-68BOjLeJ70,444
|
231
231
|
parsl/tests/configs/local_threads_globus.py,sha256=NhY27cD4vcqLh762Ye0BINZnt63EmTyHXg7FQMffOBw,1097
|
232
232
|
parsl/tests/configs/local_threads_http_in_task.py,sha256=csDY-C50tXKO2ntbbPBvppCRlXBcB7UCQOHN_FyfFYc,447
|
233
|
-
parsl/tests/configs/local_threads_monitoring.py,sha256=me-wWCtDwg_CGLQ7j4JRXcc_YTq3FGrffznjht07L08,368
|
234
233
|
parsl/tests/configs/local_threads_no_cache.py,sha256=2LM8rYhl62LIFUMjAs2_VI_R25YW5AI3RfVK_e5bdN8,236
|
235
234
|
parsl/tests/configs/midway.py,sha256=ZLdAUDR5paPA8gheRNLI0q9Vj5HcnCYuIttu-C-TlJs,1335
|
236
235
|
parsl/tests/configs/nscc_singapore.py,sha256=ECENZcBuCjkY6OWZstEMhfMrmjRmjCc7ELdfGEp7ly4,1481
|
@@ -254,7 +253,6 @@ parsl/tests/manual_tests/test_basic.py,sha256=3uCS9BqaZmKTNtNfJSsJIg2exlTAdC0Sdc
|
|
254
253
|
parsl/tests/manual_tests/test_log_filter.py,sha256=jwKclAVuESdlGK_giBuHDkY6ryX6rZB7q01lXT5K0XU,1872
|
255
254
|
parsl/tests/manual_tests/test_memory_limits.py,sha256=fjQhGsZu1PUl2Z4v75mssqve51QmtVrK94q9zJWQefU,2593
|
256
255
|
parsl/tests/manual_tests/test_regression_220.py,sha256=Jo2puWt1W0r1rJfaJFgd2ZPgR3i6uOXzrLRcfYDZWDo,931
|
257
|
-
parsl/tests/manual_tests/test_udp_simple.py,sha256=cleSqvzblTFN6sCG0egQtLYVSQeCo7JMpCMjYKjBZUw,1009
|
258
256
|
parsl/tests/manual_tests/test_worker_count.py,sha256=Cv8nAWMXAREiiGEBUr_8JyI87ffp8JGAyDqVXzcjX_0,2072
|
259
257
|
parsl/tests/scaling_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
260
258
|
parsl/tests/scaling_tests/htex_local.py,sha256=opgR-l18aunWeReuqBvNP7nSAk8CpsYqhAZE9PNPV8M,483
|
@@ -341,10 +339,12 @@ parsl/tests/test_monitoring/test_app_names.py,sha256=A-mOMCVhZDnUyJp32fsTUkHdcyv
|
|
341
339
|
parsl/tests/test_monitoring/test_basic.py,sha256=QJS0SJkf1KV6A4UvikBu-wAIcBJZVoVe-BSVWgyFonI,6180
|
342
340
|
parsl/tests/test_monitoring/test_db_locks.py,sha256=3s3c1xhKo230ZZIJ3f1Ca4U7LcEdXnanOGVXQyNlk2U,2895
|
343
341
|
parsl/tests/test_monitoring/test_exit_helper.py,sha256=ob8Qd1hlkq_mowygfPetTnYN9LfuqeXHRpPilSfDSog,1232
|
344
|
-
parsl/tests/test_monitoring/
|
342
|
+
parsl/tests/test_monitoring/test_htex_fuzz_zmq.py,sha256=MUaVBVjmoKSVg00DlMIRdj9KUizYvHjzW95Y3n_ViqE,3659
|
345
343
|
parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py,sha256=_QV8zjBKVF_qBbBnhT0C3X9AmfS7IKLcOnEw_cU6HeM,2622
|
346
344
|
parsl/tests/test_monitoring/test_incomplete_futures.py,sha256=ZnO1sFSwlWUBHX64C_zwfTVRVC_UFNlU4h0POgx6NEo,2005
|
347
345
|
parsl/tests/test_monitoring/test_memoization_representation.py,sha256=dknv2nO7pNZ1jGxWGsC_AW3rs90gjMIeC5d7pIJ75Xc,2645
|
346
|
+
parsl/tests/test_monitoring/test_radio_filesystem.py,sha256=hA-RXmU29nEG4Dm6eLcyVYoTdGog_wvYDcvNsN5-3lU,1531
|
347
|
+
parsl/tests/test_monitoring/test_radio_udp.py,sha256=dyb9SOY-ceSuZ4R2rAIn-7DqsRljYl4U6zNdACyCACU,1538
|
348
348
|
parsl/tests/test_monitoring/test_radio_zmq.py,sha256=7ARjDbde9kSuP4NCx_z-UtmMf2X5opbkkKM7mZcayuA,783
|
349
349
|
parsl/tests/test_monitoring/test_stdouterr.py,sha256=SEMKBk4v5Saoq3QiraFpzUpVe5vS_132GQPSf1Qu0qM,4573
|
350
350
|
parsl/tests/test_monitoring/test_viz_colouring.py,sha256=83Qdmn3gM0j7IL6kPDcuIsp_nl4zj-liPijyIN632SY,592
|
@@ -425,7 +425,7 @@ parsl/tests/test_serialization/test_pack_resource_spec.py,sha256=-Vtyh8KyezZw8e7
|
|
425
425
|
parsl/tests/test_serialization/test_proxystore_configured.py,sha256=lGWOSEWul16enDWhW-s7CK0d3eMDzm1324Fmj0cZMVU,2293
|
426
426
|
parsl/tests/test_serialization/test_proxystore_impl.py,sha256=uGd45sfPm9rJhzqKV0rI3lqdSOAUddQf-diEpcJAlcY,1228
|
427
427
|
parsl/tests/test_shutdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
428
|
-
parsl/tests/test_shutdown/test_kill_monitoring.py,sha256=
|
428
|
+
parsl/tests/test_shutdown/test_kill_monitoring.py,sha256=yWeYgizqGAq1ezlKWwRrJGcc0U9d2Vuzp0YuXYO1b20,1891
|
429
429
|
parsl/tests/test_staging/__init__.py,sha256=WZl9EHSkfYiSoE3Gbulcq2ifmn7IFGUkasJIobL5T5A,208
|
430
430
|
parsl/tests/test_staging/staging_provider.py,sha256=6FDpImkWOLgysqM68NbCAoXZciZokI8dmBWRAxnggBk,3242
|
431
431
|
parsl/tests/test_staging/test_1316.py,sha256=eS0e2BDM2vmPNF60aDr35wcuGgDPfXjTjRV6kyBZOQc,2652
|
@@ -460,13 +460,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
|
|
460
460
|
parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
|
461
461
|
parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
|
462
462
|
parsl/usage_tracking/usage.py,sha256=hbMo5BYgIWqMcFWqN-HYP1TbwNrTonpv-usfwnCFJKY,9212
|
463
|
-
parsl-2025.6.
|
464
|
-
parsl-2025.6.
|
465
|
-
parsl-2025.6.
|
466
|
-
parsl-2025.6.
|
467
|
-
parsl-2025.6.
|
468
|
-
parsl-2025.6.
|
469
|
-
parsl-2025.6.
|
470
|
-
parsl-2025.6.
|
471
|
-
parsl-2025.6.
|
472
|
-
parsl-2025.6.
|
463
|
+
parsl-2025.6.23.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
|
464
|
+
parsl-2025.6.23.data/scripts/interchange.py,sha256=_FRB1LxkL9vnT3y24NTXHOzotMlDJEXwF5ZZCjGmcww,28909
|
465
|
+
parsl-2025.6.23.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
|
466
|
+
parsl-2025.6.23.data/scripts/process_worker_pool.py,sha256=__gFeFQJpV5moRofj3WKQCnKp6gmzieXjzkmzVuTmX4,41123
|
467
|
+
parsl-2025.6.23.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
|
468
|
+
parsl-2025.6.23.dist-info/METADATA,sha256=MzCr6YRZ0einE2SSqmtGkBDaX8tdfR0I57QP3W3QOXE,4055
|
469
|
+
parsl-2025.6.23.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
470
|
+
parsl-2025.6.23.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
|
471
|
+
parsl-2025.6.23.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
|
472
|
+
parsl-2025.6.23.dist-info/RECORD,,
|
@@ -1,10 +0,0 @@
|
|
1
|
-
from parsl import ThreadPoolExecutor
|
2
|
-
from parsl.config import Config
|
3
|
-
from parsl.monitoring import MonitoringHub
|
4
|
-
|
5
|
-
config = Config(executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
|
6
|
-
monitoring=MonitoringHub(
|
7
|
-
hub_address="localhost",
|
8
|
-
resource_monitoring_interval=3,
|
9
|
-
)
|
10
|
-
)
|
@@ -1,51 +0,0 @@
|
|
1
|
-
import logging
|
2
|
-
|
3
|
-
import parsl
|
4
|
-
from parsl import python_app
|
5
|
-
from parsl.config import Config
|
6
|
-
from parsl.executors import ThreadPoolExecutor
|
7
|
-
from parsl.monitoring.monitoring import MonitoringHub
|
8
|
-
|
9
|
-
|
10
|
-
def local_setup():
|
11
|
-
threads_config = Config(
|
12
|
-
executors=[ThreadPoolExecutor(
|
13
|
-
label='threads',
|
14
|
-
max_threads=4)
|
15
|
-
],
|
16
|
-
monitoring=MonitoringHub(
|
17
|
-
hub_address="127.0.0.1",
|
18
|
-
logging_level=logging.INFO,
|
19
|
-
resource_monitoring_interval=10))
|
20
|
-
|
21
|
-
parsl.load(threads_config)
|
22
|
-
|
23
|
-
|
24
|
-
def local_teardown():
|
25
|
-
parsl.clear()
|
26
|
-
|
27
|
-
|
28
|
-
@python_app
|
29
|
-
def sleeper(dur=25):
|
30
|
-
import time
|
31
|
-
time.sleep(dur)
|
32
|
-
|
33
|
-
|
34
|
-
@python_app
|
35
|
-
def cpu_stress(dur=30):
|
36
|
-
import time
|
37
|
-
s = 0
|
38
|
-
start = time.time()
|
39
|
-
for i in range(10**8):
|
40
|
-
s += i
|
41
|
-
if time.time() - start >= dur:
|
42
|
-
break
|
43
|
-
return s
|
44
|
-
|
45
|
-
|
46
|
-
if __name__ == "__main__":
|
47
|
-
|
48
|
-
tasks = [sleeper() for i in range(8)]
|
49
|
-
# tasks = [cpu_stress() for i in range(10)]
|
50
|
-
|
51
|
-
print([i.result() for i in tasks])
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|