parsl 2025.6.16__py3-none-any.whl → 2025.6.23__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsl/executors/base.py +5 -3
- parsl/executors/flux/executor.py +2 -0
- parsl/executors/globus_compute.py +2 -0
- parsl/executors/high_throughput/executor.py +2 -0
- parsl/executors/high_throughput/mpi_executor.py +7 -0
- parsl/executors/radical/executor.py +3 -0
- parsl/executors/taskvine/executor.py +2 -0
- parsl/executors/threads.py +11 -2
- parsl/executors/workqueue/executor.py +2 -0
- parsl/monitoring/errors.py +0 -5
- parsl/monitoring/monitoring.py +19 -64
- parsl/monitoring/radios/filesystem.py +1 -1
- parsl/monitoring/radios/filesystem_router.py +35 -3
- parsl/monitoring/radios/htex.py +1 -1
- parsl/monitoring/radios/udp_router.py +75 -15
- parsl/monitoring/radios/zmq_router.py +9 -10
- parsl/tests/conftest.py +7 -4
- parsl/tests/test_monitoring/{test_fuzz_zmq.py → test_htex_fuzz_zmq.py} +7 -2
- parsl/tests/test_monitoring/test_radio_filesystem.py +50 -0
- parsl/tests/test_monitoring/test_radio_udp.py +53 -0
- parsl/tests/test_shutdown/test_kill_monitoring.py +1 -1
- parsl/version.py +1 -1
- {parsl-2025.6.16.dist-info → parsl-2025.6.23.dist-info}/METADATA +2 -2
- {parsl-2025.6.16.dist-info → parsl-2025.6.23.dist-info}/RECORD +32 -32
- parsl/tests/configs/local_threads_monitoring.py +0 -10
- parsl/tests/manual_tests/test_udp_simple.py +0 -51
- {parsl-2025.6.16.data → parsl-2025.6.23.data}/scripts/exec_parsl_function.py +0 -0
- {parsl-2025.6.16.data → parsl-2025.6.23.data}/scripts/interchange.py +0 -0
- {parsl-2025.6.16.data → parsl-2025.6.23.data}/scripts/parsl_coprocess.py +0 -0
- {parsl-2025.6.16.data → parsl-2025.6.23.data}/scripts/process_worker_pool.py +0 -0
- {parsl-2025.6.16.dist-info → parsl-2025.6.23.dist-info}/LICENSE +0 -0
- {parsl-2025.6.16.dist-info → parsl-2025.6.23.dist-info}/WHEEL +0 -0
- {parsl-2025.6.16.dist-info → parsl-2025.6.23.dist-info}/entry_points.txt +0 -0
- {parsl-2025.6.16.dist-info → parsl-2025.6.23.dist-info}/top_level.txt +0 -0
parsl/executors/base.py
CHANGED
@@ -98,15 +98,17 @@ class ParslExecutor(metaclass=ABCMeta):
|
|
98
98
|
def shutdown(self) -> None:
|
99
99
|
"""Shutdown the executor.
|
100
100
|
|
101
|
-
|
101
|
+
Executors should call super().shutdown() as part of their overridden
|
102
|
+
implementation.
|
102
103
|
"""
|
103
104
|
pass
|
104
105
|
|
105
106
|
def monitor_resources(self) -> bool:
|
106
107
|
"""Should resource monitoring happen for tasks on running on this executor?
|
107
108
|
|
108
|
-
Parsl resource monitoring conflicts with execution styles which
|
109
|
-
|
109
|
+
Parsl resource monitoring conflicts with execution styles which do
|
110
|
+
not directly use a process tree - for example, the ThreadPoolExecutor
|
111
|
+
and the MPIExecutor.
|
110
112
|
|
111
113
|
This function allows resource monitoring to be disabled per executor implementation.
|
112
114
|
"""
|
parsl/executors/flux/executor.py
CHANGED
@@ -134,3 +134,5 @@ class GlobusComputeExecutor(ParslExecutor, RepresentationMixin):
|
|
134
134
|
self.executor.shutdown(wait=False, cancel_futures=True)
|
135
135
|
result_watcher = self.executor._get_result_watcher()
|
136
136
|
result_watcher.shutdown(wait=False, cancel_futures=True)
|
137
|
+
|
138
|
+
super().shutdown()
|
@@ -881,6 +881,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
|
|
881
881
|
if self.zmq_monitoring:
|
882
882
|
self.zmq_monitoring.close()
|
883
883
|
|
884
|
+
super().shutdown()
|
885
|
+
|
884
886
|
logger.info("Finished HighThroughputExecutor shutdown attempt")
|
885
887
|
|
886
888
|
def get_usage_information(self):
|
@@ -111,3 +111,10 @@ class MPIExecutor(HighThroughputExecutor):
|
|
111
111
|
|
112
112
|
def validate_resource_spec(self, resource_specification: dict):
|
113
113
|
return validate_resource_spec(resource_specification)
|
114
|
+
|
115
|
+
def monitor_resources(self):
|
116
|
+
"""Resource monitoring does not make sense when using the
|
117
|
+
MPIExecutor, as the process tree launched for each task is spread
|
118
|
+
across multiple OS images/worker nodes.
|
119
|
+
"""
|
120
|
+
return False
|
@@ -601,6 +601,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
|
|
601
601
|
self._finished_task_queue.close()
|
602
602
|
self._finished_task_queue.join_thread()
|
603
603
|
|
604
|
+
super().shutdown()
|
605
|
+
|
604
606
|
logger.debug("TaskVine shutdown completed")
|
605
607
|
|
606
608
|
@wrap_with_logs
|
parsl/executors/threads.py
CHANGED
@@ -73,9 +73,18 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
|
|
73
73
|
"""
|
74
74
|
logger.debug("Shutting down executor, which involves waiting for running tasks to complete")
|
75
75
|
self.executor.shutdown(wait=block)
|
76
|
+
|
77
|
+
super().shutdown()
|
78
|
+
|
76
79
|
logger.debug("Done with executor shutdown")
|
77
80
|
|
78
81
|
def monitor_resources(self):
|
79
|
-
"""Resource monitoring
|
80
|
-
|
82
|
+
"""Resource monitoring does not make sense when using the
|
83
|
+
ThreadPoolExecutor, as there is no per-task process tree: all tasks
|
84
|
+
run inside the same single submitting process.
|
85
|
+
|
86
|
+
In addition, the use of fork-based multiprocessing in the remote
|
87
|
+
wrapper in parsl/monitoring/remote.py was especially prone to deadlock
|
88
|
+
with this executor.
|
89
|
+
"""
|
81
90
|
return False
|
@@ -714,6 +714,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
|
|
714
714
|
self.collector_queue.close()
|
715
715
|
self.collector_queue.join_thread()
|
716
716
|
|
717
|
+
super().shutdown()
|
718
|
+
|
717
719
|
logger.debug("Work Queue shutdown completed")
|
718
720
|
|
719
721
|
@wrap_with_logs
|
parsl/monitoring/errors.py
CHANGED
@@ -1,11 +1,6 @@
|
|
1
1
|
from parsl.errors import ParslError
|
2
2
|
|
3
3
|
|
4
|
-
class MonitoringHubStartError(ParslError):
|
5
|
-
def __str__(self) -> str:
|
6
|
-
return "Hub failed to start"
|
7
|
-
|
8
|
-
|
9
4
|
class MonitoringRouterStartError(ParslError):
|
10
5
|
def __str__(self) -> str:
|
11
6
|
return "Monitoring router failed to start"
|
parsl/monitoring/monitoring.py
CHANGED
@@ -3,16 +3,14 @@ from __future__ import annotations
|
|
3
3
|
import logging
|
4
4
|
import multiprocessing.synchronize as ms
|
5
5
|
import os
|
6
|
-
import queue
|
7
6
|
import warnings
|
8
7
|
from multiprocessing.queues import Queue
|
9
|
-
from typing import
|
8
|
+
from typing import Any, Optional, Union
|
10
9
|
|
11
10
|
import typeguard
|
12
11
|
|
13
|
-
from parsl.monitoring.
|
14
|
-
from parsl.monitoring.radios.
|
15
|
-
from parsl.monitoring.radios.udp_router import udp_router_starter
|
12
|
+
from parsl.monitoring.radios.filesystem_router import start_filesystem_receiver
|
13
|
+
from parsl.monitoring.radios.udp_router import start_udp_receiver
|
16
14
|
from parsl.monitoring.types import TaggedMonitoringMessage
|
17
15
|
from parsl.multiprocessing import (
|
18
16
|
SizedQueue,
|
@@ -119,39 +117,14 @@ class MonitoringHub(RepresentationMixin):
|
|
119
117
|
|
120
118
|
self.monitoring_hub_active = True
|
121
119
|
|
122
|
-
# This annotation is incompatible with typeguard 4.x instrumentation
|
123
|
-
# of local variables: Queue is not subscriptable at runtime, as far
|
124
|
-
# as typeguard is concerned. The more general Queue annotation works,
|
125
|
-
# but does not restrict the contents of the Queue. Using TYPE_CHECKING
|
126
|
-
# here allows the stricter definition to be seen by mypy, and the
|
127
|
-
# simpler definition to be seen by typeguard. Hopefully at some point
|
128
|
-
# in the future, Queue will allow runtime subscripts.
|
129
|
-
|
130
|
-
if TYPE_CHECKING:
|
131
|
-
udp_comm_q: Queue[Union[int, str]]
|
132
|
-
else:
|
133
|
-
udp_comm_q: Queue
|
134
|
-
|
135
|
-
udp_comm_q = SizedQueue(maxsize=10)
|
136
|
-
|
137
120
|
self.resource_msgs: Queue[TaggedMonitoringMessage]
|
138
121
|
self.resource_msgs = SizedQueue()
|
139
122
|
|
140
|
-
self.
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
"resource_msgs": self.resource_msgs,
|
146
|
-
"exit_event": self.router_exit_event,
|
147
|
-
"udp_port": self.hub_port,
|
148
|
-
"run_dir": dfk_run_dir,
|
149
|
-
"logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
|
150
|
-
},
|
151
|
-
name="Monitoring-UDP-Router-Process",
|
152
|
-
daemon=True,
|
153
|
-
)
|
154
|
-
self.udp_router_proc.start()
|
123
|
+
self.udp_receiver = start_udp_receiver(debug=self.monitoring_debug,
|
124
|
+
logdir=dfk_run_dir,
|
125
|
+
monitoring_messages=self.resource_msgs,
|
126
|
+
port=self.hub_port
|
127
|
+
)
|
155
128
|
|
156
129
|
self.dbm_exit_event: ms.Event
|
157
130
|
self.dbm_exit_event = SpawnEvent()
|
@@ -168,32 +141,15 @@ class MonitoringHub(RepresentationMixin):
|
|
168
141
|
)
|
169
142
|
self.dbm_proc.start()
|
170
143
|
logger.info("Started UDP router process %s and DBM process %s",
|
171
|
-
self.
|
172
|
-
|
173
|
-
self.
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
self.filesystem_proc.start()
|
181
|
-
logger.info("Started filesystem radio receiver process %s", self.filesystem_proc.pid)
|
182
|
-
|
183
|
-
try:
|
184
|
-
udp_comm_q_result = udp_comm_q.get(block=True, timeout=120)
|
185
|
-
udp_comm_q.close()
|
186
|
-
udp_comm_q.join_thread()
|
187
|
-
except queue.Empty:
|
188
|
-
logger.error("Monitoring UDP router has not reported port in 120s. Aborting")
|
189
|
-
raise MonitoringHubStartError()
|
190
|
-
|
191
|
-
if isinstance(udp_comm_q_result, str):
|
192
|
-
logger.error("MonitoringRouter sent an error message: %s", udp_comm_q_result)
|
193
|
-
raise RuntimeError(f"MonitoringRouter failed to start: {udp_comm_q_result}")
|
194
|
-
|
195
|
-
udp_port = udp_comm_q_result
|
196
|
-
self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, udp_port)
|
144
|
+
self.udp_receiver.process.pid, self.dbm_proc.pid)
|
145
|
+
|
146
|
+
self.filesystem_receiver = start_filesystem_receiver(debug=self.monitoring_debug,
|
147
|
+
logdir=dfk_run_dir,
|
148
|
+
monitoring_messages=self.resource_msgs
|
149
|
+
)
|
150
|
+
logger.info("Started filesystem radio receiver process %s", self.filesystem_receiver.process.pid)
|
151
|
+
|
152
|
+
self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, self.udp_receiver.port)
|
197
153
|
|
198
154
|
logger.info("Monitoring Hub initialized")
|
199
155
|
|
@@ -202,10 +158,9 @@ class MonitoringHub(RepresentationMixin):
|
|
202
158
|
if self.monitoring_hub_active:
|
203
159
|
self.monitoring_hub_active = False
|
204
160
|
logger.info("Setting router termination event")
|
205
|
-
self.router_exit_event.set()
|
206
161
|
|
207
162
|
logger.info("Waiting for UDP router to terminate")
|
208
|
-
|
163
|
+
self.udp_receiver.close()
|
209
164
|
|
210
165
|
logger.debug("Finished waiting for router termination")
|
211
166
|
logger.debug("Waiting for DB termination")
|
@@ -214,7 +169,7 @@ class MonitoringHub(RepresentationMixin):
|
|
214
169
|
logger.debug("Finished waiting for DBM termination")
|
215
170
|
|
216
171
|
logger.info("Terminating filesystem radio receiver process")
|
217
|
-
|
172
|
+
self.filesystem_receiver.close()
|
218
173
|
|
219
174
|
logger.info("Closing monitoring multiprocessing queues")
|
220
175
|
self.resource_msgs.close()
|
@@ -27,7 +27,7 @@ class FilesystemRadioSender(MonitoringRadioSender):
|
|
27
27
|
"""
|
28
28
|
|
29
29
|
def __init__(self, *, monitoring_url: str, timeout: int = 10, run_dir: str):
|
30
|
-
logger.info("filesystem based monitoring
|
30
|
+
logger.info("filesystem based monitoring radio initializing")
|
31
31
|
self.base_path = f"{run_dir}/monitor-fs-radio/"
|
32
32
|
self.tmp_path = f"{self.base_path}/tmp"
|
33
33
|
self.new_path = f"{self.base_path}/new"
|
@@ -4,6 +4,7 @@ import logging
|
|
4
4
|
import os
|
5
5
|
import pickle
|
6
6
|
import time
|
7
|
+
from multiprocessing.context import SpawnProcess
|
7
8
|
from multiprocessing.queues import Queue
|
8
9
|
from multiprocessing.synchronize import Event
|
9
10
|
from typing import cast
|
@@ -11,15 +12,17 @@ from typing import cast
|
|
11
12
|
from parsl.log_utils import set_file_logger
|
12
13
|
from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
|
13
14
|
from parsl.monitoring.types import TaggedMonitoringMessage
|
15
|
+
from parsl.multiprocessing import SpawnEvent, join_terminate_close_proc
|
14
16
|
from parsl.process_loggers import wrap_with_logs
|
15
17
|
from parsl.utils import setproctitle
|
16
18
|
|
19
|
+
logger = logging.getLogger(__name__)
|
20
|
+
|
17
21
|
|
18
22
|
@wrap_with_logs
|
19
23
|
def filesystem_router_starter(*, q: Queue[TaggedMonitoringMessage], run_dir: str, exit_event: Event) -> None:
|
20
|
-
|
21
|
-
|
22
|
-
level=logging.INFO)
|
24
|
+
set_file_logger(f"{run_dir}/monitoring_filesystem_radio.log",
|
25
|
+
level=logging.INFO)
|
23
26
|
|
24
27
|
logger.info("Starting filesystem radio receiver")
|
25
28
|
setproctitle("parsl: monitoring filesystem receiver")
|
@@ -52,3 +55,32 @@ def filesystem_router_starter(*, q: Queue[TaggedMonitoringMessage], run_dir: str
|
|
52
55
|
|
53
56
|
time.sleep(1) # whats a good time for this poll?
|
54
57
|
logger.info("Ending filesystem radio receiver")
|
58
|
+
|
59
|
+
|
60
|
+
class FilesystemRadioReceiver():
|
61
|
+
def __init__(self, *, process: SpawnProcess, exit_event: Event) -> None:
|
62
|
+
self.process = process
|
63
|
+
self.exit_event = exit_event
|
64
|
+
|
65
|
+
def close(self) -> None:
|
66
|
+
self.exit_event.set()
|
67
|
+
join_terminate_close_proc(self.process)
|
68
|
+
|
69
|
+
|
70
|
+
def start_filesystem_receiver(*,
|
71
|
+
monitoring_messages: Queue,
|
72
|
+
logdir: str,
|
73
|
+
debug: bool) -> FilesystemRadioReceiver:
|
74
|
+
|
75
|
+
router_exit_event = SpawnEvent()
|
76
|
+
|
77
|
+
filesystem_proc = SpawnProcess(target=filesystem_router_starter,
|
78
|
+
kwargs={"q": monitoring_messages,
|
79
|
+
"run_dir": logdir,
|
80
|
+
"exit_event": router_exit_event},
|
81
|
+
name="Monitoring-Filesystem-Process",
|
82
|
+
daemon=True
|
83
|
+
)
|
84
|
+
filesystem_proc.start()
|
85
|
+
|
86
|
+
return FilesystemRadioReceiver(process=filesystem_proc, exit_event=router_exit_event)
|
parsl/monitoring/radios/htex.py
CHANGED
@@ -18,7 +18,7 @@ class HTEXRadioSender(MonitoringRadioSender):
|
|
18
18
|
timeout : int
|
19
19
|
timeout, default=10s
|
20
20
|
"""
|
21
|
-
logger.info("htex-based monitoring
|
21
|
+
logger.info("htex-based monitoring radio initialising")
|
22
22
|
|
23
23
|
def send(self, message: object) -> None:
|
24
24
|
""" Sends a message to the UDP receiver
|
@@ -4,15 +4,26 @@ import logging
|
|
4
4
|
import multiprocessing.queues as mpq
|
5
5
|
import os
|
6
6
|
import pickle
|
7
|
+
import queue
|
7
8
|
import socket
|
8
9
|
import time
|
10
|
+
from multiprocessing.context import SpawnProcess as SpawnProcessType
|
11
|
+
from multiprocessing.queues import Queue
|
9
12
|
from multiprocessing.synchronize import Event
|
10
|
-
from
|
13
|
+
from multiprocessing.synchronize import Event as EventType
|
14
|
+
from typing import Optional, Union
|
11
15
|
|
12
16
|
import typeguard
|
13
17
|
|
14
18
|
from parsl.log_utils import set_file_logger
|
19
|
+
from parsl.monitoring.errors import MonitoringRouterStartError
|
15
20
|
from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
|
21
|
+
from parsl.multiprocessing import (
|
22
|
+
SizedQueue,
|
23
|
+
SpawnEvent,
|
24
|
+
SpawnProcess,
|
25
|
+
join_terminate_close_proc,
|
26
|
+
)
|
16
27
|
from parsl.process_loggers import wrap_with_logs
|
17
28
|
from parsl.utils import setproctitle
|
18
29
|
|
@@ -48,10 +59,9 @@ class MonitoringRouter:
|
|
48
59
|
An event that the main Parsl process will set to signal that the monitoring router should shut down.
|
49
60
|
"""
|
50
61
|
os.makedirs(run_dir, exist_ok=True)
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
self.logger.debug("Monitoring router starting")
|
62
|
+
set_file_logger(f"{run_dir}/monitoring_udp_router.log",
|
63
|
+
level=logging_level)
|
64
|
+
logger.debug("Monitoring router starting")
|
55
65
|
|
56
66
|
self.atexit_timeout = atexit_timeout
|
57
67
|
|
@@ -73,39 +83,39 @@ class MonitoringRouter:
|
|
73
83
|
except Exception as e:
|
74
84
|
raise RuntimeError(f"Could not bind to udp_port {udp_port} because: {e}")
|
75
85
|
self.udp_sock.settimeout(self.loop_freq / 1000)
|
76
|
-
|
86
|
+
logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
|
77
87
|
|
78
88
|
self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
|
79
89
|
self.exit_event = exit_event
|
80
90
|
|
81
|
-
@wrap_with_logs
|
91
|
+
@wrap_with_logs
|
82
92
|
def start(self) -> None:
|
83
|
-
|
93
|
+
logger.info("Starting UDP listener")
|
84
94
|
try:
|
85
95
|
while not self.exit_event.is_set():
|
86
96
|
try:
|
87
97
|
data, addr = self.udp_sock.recvfrom(2048)
|
88
98
|
resource_msg = pickle.loads(data)
|
89
|
-
|
99
|
+
logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
|
90
100
|
self.target_radio.send(resource_msg)
|
91
101
|
except socket.timeout:
|
92
102
|
pass
|
93
103
|
|
94
|
-
|
104
|
+
logger.info("UDP listener draining")
|
95
105
|
last_msg_received_time = time.time()
|
96
106
|
while time.time() - last_msg_received_time < self.atexit_timeout:
|
97
107
|
try:
|
98
108
|
data, addr = self.udp_sock.recvfrom(2048)
|
99
109
|
msg = pickle.loads(data)
|
100
|
-
|
110
|
+
logger.debug("Got UDP Message from {}: {}".format(addr, msg))
|
101
111
|
self.target_radio.send(msg)
|
102
112
|
last_msg_received_time = time.time()
|
103
113
|
except socket.timeout:
|
104
114
|
pass
|
105
115
|
|
106
|
-
|
116
|
+
logger.info("UDP listener finishing normally")
|
107
117
|
finally:
|
108
|
-
|
118
|
+
logger.info("UDP listener finished")
|
109
119
|
|
110
120
|
|
111
121
|
@wrap_with_logs
|
@@ -132,8 +142,58 @@ def udp_router_starter(*,
|
|
132
142
|
else:
|
133
143
|
comm_q.put(router.udp_port)
|
134
144
|
|
135
|
-
|
145
|
+
logger.info("Starting MonitoringRouter in router_starter")
|
136
146
|
try:
|
137
147
|
router.start()
|
138
148
|
except Exception:
|
139
|
-
|
149
|
+
logger.exception("UDP router start exception")
|
150
|
+
|
151
|
+
|
152
|
+
class UDPRadioReceiver():
|
153
|
+
def __init__(self, *, process: SpawnProcessType, exit_event: EventType, port: int) -> None:
|
154
|
+
self.process = process
|
155
|
+
self.exit_event = exit_event
|
156
|
+
self.port = port
|
157
|
+
|
158
|
+
def close(self) -> None:
|
159
|
+
self.exit_event.set()
|
160
|
+
join_terminate_close_proc(self.process)
|
161
|
+
|
162
|
+
|
163
|
+
def start_udp_receiver(*,
|
164
|
+
monitoring_messages: Queue,
|
165
|
+
port: Optional[int],
|
166
|
+
logdir: str,
|
167
|
+
debug: bool) -> UDPRadioReceiver:
|
168
|
+
|
169
|
+
udp_comm_q: Queue[Union[int, str]]
|
170
|
+
udp_comm_q = SizedQueue(maxsize=10)
|
171
|
+
|
172
|
+
router_exit_event = SpawnEvent()
|
173
|
+
|
174
|
+
router_proc = SpawnProcess(target=udp_router_starter,
|
175
|
+
kwargs={"comm_q": udp_comm_q,
|
176
|
+
"resource_msgs": monitoring_messages,
|
177
|
+
"exit_event": router_exit_event,
|
178
|
+
"udp_port": port,
|
179
|
+
"run_dir": logdir,
|
180
|
+
"logging_level": logging.DEBUG if debug else logging.INFO,
|
181
|
+
},
|
182
|
+
name="Monitoring-UDP-Router-Process",
|
183
|
+
daemon=True,
|
184
|
+
)
|
185
|
+
router_proc.start()
|
186
|
+
|
187
|
+
try:
|
188
|
+
udp_comm_q_result = udp_comm_q.get(block=True, timeout=120)
|
189
|
+
udp_comm_q.close()
|
190
|
+
udp_comm_q.join_thread()
|
191
|
+
except queue.Empty:
|
192
|
+
logger.error("Monitoring UDP router has not reported port in 120s. Aborting")
|
193
|
+
raise MonitoringRouterStartError()
|
194
|
+
|
195
|
+
if isinstance(udp_comm_q_result, str):
|
196
|
+
logger.error("MonitoringRouter sent an error message: %s", udp_comm_q_result)
|
197
|
+
raise RuntimeError(f"MonitoringRouter failed to start: {udp_comm_q_result}")
|
198
|
+
|
199
|
+
return UDPRadioReceiver(process=router_proc, exit_event=router_exit_event, port=udp_comm_q_result)
|
@@ -61,10 +61,9 @@ class MonitoringRouter:
|
|
61
61
|
An event that the main Parsl process will set to signal that the monitoring router should shut down.
|
62
62
|
"""
|
63
63
|
os.makedirs(run_dir, exist_ok=True)
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
self.logger.debug("Monitoring router starting")
|
64
|
+
set_file_logger(f"{run_dir}/monitoring_zmq_router.log",
|
65
|
+
level=logging_level)
|
66
|
+
logger.debug("Monitoring router starting")
|
68
67
|
|
69
68
|
self.address = address
|
70
69
|
|
@@ -75,7 +74,7 @@ class MonitoringRouter:
|
|
75
74
|
self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
|
76
75
|
self.zmq_receiver_channel.set_hwm(0)
|
77
76
|
self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq) # in milliseconds
|
78
|
-
|
77
|
+
logger.debug("address: {}. port_range {}".format(address, port_range))
|
79
78
|
self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port(tcp_url(address),
|
80
79
|
min_port=port_range[0],
|
81
80
|
max_port=port_range[1])
|
@@ -83,9 +82,9 @@ class MonitoringRouter:
|
|
83
82
|
self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
|
84
83
|
self.exit_event = exit_event
|
85
84
|
|
86
|
-
@wrap_with_logs
|
85
|
+
@wrap_with_logs
|
87
86
|
def start(self) -> None:
|
88
|
-
|
87
|
+
logger.info("Starting ZMQ listener")
|
89
88
|
try:
|
90
89
|
while not self.exit_event.is_set():
|
91
90
|
try:
|
@@ -107,11 +106,11 @@ class MonitoringRouter:
|
|
107
106
|
# channel is broken in such a way that it always raises
|
108
107
|
# an exception? Looping on this would maybe be the wrong
|
109
108
|
# thing to do.
|
110
|
-
|
109
|
+
logger.warning("Failure processing a ZMQ message", exc_info=True)
|
111
110
|
|
112
|
-
|
111
|
+
logger.info("ZMQ listener finishing normally")
|
113
112
|
finally:
|
114
|
-
|
113
|
+
logger.info("ZMQ listener finished")
|
115
114
|
|
116
115
|
|
117
116
|
@wrap_with_logs
|
parsl/tests/conftest.py
CHANGED
@@ -406,13 +406,15 @@ def try_assert():
|
|
406
406
|
timeout_ms: float = 5000,
|
407
407
|
attempts: int = 0,
|
408
408
|
check_period_ms: int = 20,
|
409
|
+
factor: float = 2,
|
409
410
|
):
|
410
411
|
tb = create_traceback(start=1)
|
411
412
|
check_period_s = abs(check_period_ms) / 1000.0
|
412
413
|
if attempts > 0:
|
413
414
|
for _attempt_no in range(attempts):
|
414
|
-
|
415
|
-
check_period_s
|
415
|
+
fraction = random.random()
|
416
|
+
time.sleep(fraction * check_period_s) # jitter
|
417
|
+
check_period_s *= factor ** fraction
|
416
418
|
if test_func():
|
417
419
|
return
|
418
420
|
else:
|
@@ -427,9 +429,10 @@ def try_assert():
|
|
427
429
|
timeout_s = abs(timeout_ms) / 1000.0
|
428
430
|
end = time.monotonic() + timeout_s
|
429
431
|
while time.monotonic() < end:
|
430
|
-
|
432
|
+
fraction = random.random()
|
433
|
+
wait_for = fraction * check_period_s # jitter
|
431
434
|
time.sleep(min(wait_for, end - time.monotonic()))
|
432
|
-
check_period_s *=
|
435
|
+
check_period_s *= factor ** fraction
|
433
436
|
if test_func():
|
434
437
|
return
|
435
438
|
att_fail = (
|
@@ -17,7 +17,12 @@ def this_app():
|
|
17
17
|
|
18
18
|
|
19
19
|
@pytest.mark.local
|
20
|
-
def
|
20
|
+
def test_fuzz():
|
21
|
+
"""This test sends fuzz into the ZMQ radio receiver that HTEX starts
|
22
|
+
for receiving monitoring messages from the interchange, and checks
|
23
|
+
that monitoring still records things ok.
|
24
|
+
"""
|
25
|
+
|
21
26
|
import sqlalchemy
|
22
27
|
from sqlalchemy import text
|
23
28
|
|
@@ -44,7 +49,7 @@ def test_row_counts():
|
|
44
49
|
# the latter is what i'm most suspicious of in my present investigation
|
45
50
|
|
46
51
|
# dig out the interchange port...
|
47
|
-
hub_address = parsl.dfk().
|
52
|
+
hub_address = parsl.dfk().executors["htex_Local"].loopback_address
|
48
53
|
hub_zmq_port = parsl.dfk().executors["htex_Local"].hub_zmq_port
|
49
54
|
|
50
55
|
# this will send a string to a new socket connection
|
@@ -0,0 +1,50 @@
|
|
1
|
+
import pytest
|
2
|
+
|
3
|
+
from parsl.monitoring.message_type import MessageType
|
4
|
+
from parsl.monitoring.radios.filesystem import FilesystemRadioSender
|
5
|
+
from parsl.monitoring.radios.filesystem_router import start_filesystem_receiver
|
6
|
+
from parsl.multiprocessing import SpawnQueue
|
7
|
+
|
8
|
+
|
9
|
+
@pytest.mark.local
|
10
|
+
def test_filesystem(tmpd_cwd):
|
11
|
+
"""Test filesystem radio/receiver pair.
|
12
|
+
This test checks that the pair can be started up locally, that a message
|
13
|
+
is conveyed from radio to receiver, and that the receiver process goes
|
14
|
+
away at shutdown.
|
15
|
+
"""
|
16
|
+
|
17
|
+
resource_msgs = SpawnQueue()
|
18
|
+
|
19
|
+
# start receiver
|
20
|
+
receiver = start_filesystem_receiver(debug=True,
|
21
|
+
logdir=str(tmpd_cwd),
|
22
|
+
monitoring_messages=resource_msgs,
|
23
|
+
)
|
24
|
+
|
25
|
+
# make radio
|
26
|
+
|
27
|
+
radio_sender = FilesystemRadioSender(run_dir=str(tmpd_cwd),
|
28
|
+
monitoring_url="irrelevant:")
|
29
|
+
|
30
|
+
# send message into radio
|
31
|
+
|
32
|
+
message = (MessageType.RESOURCE_INFO, {})
|
33
|
+
|
34
|
+
radio_sender.send(message)
|
35
|
+
|
36
|
+
# verify it comes out of the receiver
|
37
|
+
|
38
|
+
m = resource_msgs.get()
|
39
|
+
|
40
|
+
assert m == message, "The sent message should appear in the queue"
|
41
|
+
|
42
|
+
# shut down router
|
43
|
+
|
44
|
+
receiver.close()
|
45
|
+
|
46
|
+
# we can't inspect the process if it has been closed properly, but
|
47
|
+
# we can verify that it raises the expected ValueError the closed
|
48
|
+
# processes raise on access.
|
49
|
+
with pytest.raises(ValueError):
|
50
|
+
receiver.process.exitcode
|
@@ -0,0 +1,53 @@
|
|
1
|
+
import pytest
|
2
|
+
|
3
|
+
from parsl.monitoring.message_type import MessageType
|
4
|
+
from parsl.monitoring.radios.udp import UDPRadioSender
|
5
|
+
from parsl.monitoring.radios.udp_router import start_udp_receiver
|
6
|
+
from parsl.multiprocessing import SpawnQueue
|
7
|
+
|
8
|
+
|
9
|
+
@pytest.mark.local
|
10
|
+
def test_udp(tmpd_cwd):
|
11
|
+
"""Test UDP radio/receiver pair.
|
12
|
+
This test checks that the pair can be started up locally, that a message
|
13
|
+
is conveyed from radio to receiver, and that the receiver process goes
|
14
|
+
away at shutdown.
|
15
|
+
"""
|
16
|
+
|
17
|
+
resource_msgs = SpawnQueue()
|
18
|
+
|
19
|
+
# start receiver
|
20
|
+
udp_receiver = start_udp_receiver(debug=True,
|
21
|
+
logdir=str(tmpd_cwd),
|
22
|
+
monitoring_messages=resource_msgs,
|
23
|
+
port=None
|
24
|
+
)
|
25
|
+
|
26
|
+
# make radio
|
27
|
+
|
28
|
+
# this comes from monitoring.py:
|
29
|
+
url = "udp://{}:{}".format("localhost", udp_receiver.port)
|
30
|
+
|
31
|
+
radio_sender = UDPRadioSender(url)
|
32
|
+
|
33
|
+
# send message into radio
|
34
|
+
|
35
|
+
message = (MessageType.RESOURCE_INFO, {})
|
36
|
+
|
37
|
+
radio_sender.send(message)
|
38
|
+
|
39
|
+
# verify it comes out of the receiver
|
40
|
+
|
41
|
+
m = resource_msgs.get()
|
42
|
+
|
43
|
+
assert m == message, "The sent message should appear in the queue"
|
44
|
+
|
45
|
+
# shut down router
|
46
|
+
|
47
|
+
udp_receiver.close()
|
48
|
+
|
49
|
+
# we can't inspect the process if it has been closed properly, but
|
50
|
+
# we can verify that it raises the expected ValueError the closed
|
51
|
+
# processes raise on access.
|
52
|
+
with pytest.raises(ValueError):
|
53
|
+
udp_receiver.process.exitcode
|
@@ -30,7 +30,7 @@ def test_no_kills():
|
|
30
30
|
|
31
31
|
@pytest.mark.local
|
32
32
|
@pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM, signal.SIGKILL, signal.SIGQUIT])
|
33
|
-
@pytest.mark.parametrize("process_attr", ["
|
33
|
+
@pytest.mark.parametrize("process_attr", ["dbm_proc"])
|
34
34
|
def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
|
35
35
|
"""This tests that we can kill a monitoring process and still have successful shutdown.
|
36
36
|
SIGINT emulates some racy behaviour when ctrl-C is pressed: that
|
parsl/version.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: parsl
|
3
|
-
Version: 2025.6.
|
3
|
+
Version: 2025.6.23
|
4
4
|
Summary: Simple data dependent workflows in Python
|
5
5
|
Home-page: https://github.com/Parsl/parsl
|
6
|
-
Download-URL: https://github.com/Parsl/parsl/archive/2025.06.
|
6
|
+
Download-URL: https://github.com/Parsl/parsl/archive/2025.06.23.tar.gz
|
7
7
|
Author: The Parsl Team
|
8
8
|
Author-email: parsl@googlegroups.com
|
9
9
|
License: Apache 2.0
|
@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=JNAfgdZvQSsxVyUp229OOUqWwf_ZUhpmw8X9CdF3i6k,3614
|
|
8
8
|
parsl/process_loggers.py,sha256=uQ7Gd0W72Jz7rrcYlOMfLsAEhkRltxXJL2MgdduJjEw,1136
|
9
9
|
parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
10
|
parsl/utils.py,sha256=codTX6_KLhgeTwNkRzc1lo4bgc1M93eJ-lkqOO98fvk,14331
|
11
|
-
parsl/version.py,sha256=
|
11
|
+
parsl/version.py,sha256=Ic4ZXjmRNaNVe4VgUeJZzbs04AL3PcIpDFnIlIUjmIQ,131
|
12
12
|
parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
13
13
|
parsl/app/app.py,sha256=0gbM4AH2OtFOLsv07I5nglpElcwMSOi-FzdZZfrk7So,8532
|
14
14
|
parsl/app/bash.py,sha256=jm2AvePlCT9DZR7H_4ANDWxatp5dN_22FUlT_gWhZ-g,5528
|
@@ -61,37 +61,37 @@ parsl/dataflow/rundirs.py,sha256=JZdzybVGubY35jL2YiKcDo65ZmRl1WyOApc8ajYxztc,108
|
|
61
61
|
parsl/dataflow/states.py,sha256=hV6mfv-y4A6xrujeQglcomnfEs7y3Xm2g6JFwC6dvgQ,2612
|
62
62
|
parsl/dataflow/taskrecord.py,sha256=qIW7T6hn9dYTuNPdUura3HQwwUpUJACwPP5REm5COf4,3042
|
63
63
|
parsl/executors/__init__.py,sha256=PEuXYrnVqwlaz_nt82s9D_YNaVsX7ET29DeIZRUR8hw,577
|
64
|
-
parsl/executors/base.py,sha256=
|
64
|
+
parsl/executors/base.py,sha256=OApNwTcBrzVXweWFPj8rMrP87qStcvF2facsxaevSqA,4568
|
65
65
|
parsl/executors/errors.py,sha256=ZxL3nK5samPos8Xixo_jpRtPIiRJfZ5D397_qaXj2g0,2515
|
66
66
|
parsl/executors/execute_task.py,sha256=PtqHxk778UQaNah1AN-TJV5emZbOcU5TGtWDxFn3_F4,1079
|
67
|
-
parsl/executors/globus_compute.py,sha256=
|
67
|
+
parsl/executors/globus_compute.py,sha256=p59iBrv2BvYUAZ3YZSpOrm_Wpai592ueiJm3zFS7gvY,5304
|
68
68
|
parsl/executors/status_handling.py,sha256=n2DLASEvKZNgFpviAMYDfqcAsxMiU7QxFemw91YSenc,15746
|
69
|
-
parsl/executors/threads.py,sha256=
|
69
|
+
parsl/executors/threads.py,sha256=JkbjJGtZPJNzsVF3DFQHOsZpKmFr7wcGbH1wbwDPSrw,3754
|
70
70
|
parsl/executors/flux/__init__.py,sha256=P9grTTeRPXfqXurFhlSS7XhmE6tTbnCnyQ1f9b-oYHE,136
|
71
71
|
parsl/executors/flux/execute_parsl_task.py,sha256=zHP5M7ILGiwnoalZ8WsfVVdZM7uP4iQo2ThVh4crxpM,1530
|
72
|
-
parsl/executors/flux/executor.py,sha256=
|
72
|
+
parsl/executors/flux/executor.py,sha256=NLdjOli5VjrSdEfyWbfqKN_8APvFkp_qFCouS_9NeLQ,17069
|
73
73
|
parsl/executors/flux/flux_instance_manager.py,sha256=5T3Rp7ZM-mlT0Pf0Gxgs5_YmnaPrSF9ec7zvRfLfYJw,2129
|
74
74
|
parsl/executors/high_throughput/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
75
75
|
parsl/executors/high_throughput/errors.py,sha256=k2XuvvFdUfNs2foHFnxmS-BToRMfdXpYEa4EF3ELKq4,1554
|
76
|
-
parsl/executors/high_throughput/executor.py,sha256=
|
76
|
+
parsl/executors/high_throughput/executor.py,sha256=JTNgVfZfwzgH9RheMWCCDuNd_1i3LN69B_SYzL7rY3Q,39840
|
77
77
|
parsl/executors/high_throughput/interchange.py,sha256=lnPhV-5OR_rp4fa_Wj0tmG3avwUdPgJH4LTTBZQcrf8,28922
|
78
78
|
parsl/executors/high_throughput/manager_record.py,sha256=ZMsqFxvreGLRXAw3N-JnODDa9Qfizw2tMmcBhm4lco4,490
|
79
79
|
parsl/executors/high_throughput/manager_selector.py,sha256=UKcUE6v0tO7PDMTThpKSKxVpOpOUilxDL7UbNgpZCxo,2116
|
80
80
|
parsl/executors/high_throughput/monitoring_info.py,sha256=HC0drp6nlXQpAop5PTUKNjdXMgtZVvrBL0JzZJebPP4,298
|
81
|
-
parsl/executors/high_throughput/mpi_executor.py,sha256=
|
81
|
+
parsl/executors/high_throughput/mpi_executor.py,sha256=AKc5OX7mNH9eYPiBL1g7Xlrj3P-ouniDgE17bhpIems,5052
|
82
82
|
parsl/executors/high_throughput/mpi_prefix_composer.py,sha256=DmpKugANNa1bdYlqQBLHkrFc15fJpefPPhW9hkAlh1s,4308
|
83
83
|
parsl/executors/high_throughput/mpi_resource_management.py,sha256=73bTW2ZbHRfcrPN318cyjiqDN50AM1cOCQqUGJDIlBg,8199
|
84
84
|
parsl/executors/high_throughput/probe.py,sha256=QOEaliO3x5cB6ltMOZMsZQ-ath9AAuFqXcBzRgWOM60,2754
|
85
85
|
parsl/executors/high_throughput/process_worker_pool.py,sha256=Q7FN0MdXIAOouxDarim6etYVHEgbXFiaMhBahC2ZtIQ,41137
|
86
86
|
parsl/executors/high_throughput/zmq_pipes.py,sha256=NUK25IEh0UkxzdqQQyM8tMtuZmjSiTeWu1DzkkAIOhA,8980
|
87
87
|
parsl/executors/radical/__init__.py,sha256=CKbtV2numw5QvgIBq1htMUrt9TqDCIC2zifyf2svTNU,186
|
88
|
-
parsl/executors/radical/executor.py,sha256=
|
88
|
+
parsl/executors/radical/executor.py,sha256=e3XS4mvug1uJ6wrt4UH6hBgfbDbc-mQH3xUW2ZmBsMQ,22888
|
89
89
|
parsl/executors/radical/rpex_resources.py,sha256=Q7-0u3K447LBCe2y7mVcdw6jqWI7SdPXxCKhkr6FoRQ,5139
|
90
90
|
parsl/executors/radical/rpex_worker.py,sha256=z6r82ZujKb6sdKIdHsQ_5EBMDIQieeGcrlt6kGLFo4M,1830
|
91
91
|
parsl/executors/taskvine/__init__.py,sha256=9rwp3M8B0YyEhZMLO0RHaNw7u1nc01WHbXLqnBTanu0,293
|
92
92
|
parsl/executors/taskvine/errors.py,sha256=euIYkSslrNSI85kyi2s0xzOaO9ik4c1fYHstMIeiBJk,652
|
93
93
|
parsl/executors/taskvine/exec_parsl_function.py,sha256=ftGdJU78lKPPkphSHlEi4rj164mhuMHJjghVqfgeXKk,7085
|
94
|
-
parsl/executors/taskvine/executor.py,sha256=
|
94
|
+
parsl/executors/taskvine/executor.py,sha256=h47c90QoaeoWcqTdJKgfqwQBZVW-kkF44O95sKUfJ8g,30978
|
95
95
|
parsl/executors/taskvine/factory.py,sha256=GU5JryEAKJuYKwrSc162BN-lhcKhapvBZHT820pxwic,2772
|
96
96
|
parsl/executors/taskvine/factory_config.py,sha256=ZQC5vyDe8cM0nuv7fbBCV2xnWGAZ87iLlT2UqmFFI1U,3695
|
97
97
|
parsl/executors/taskvine/manager.py,sha256=SUi5mqqMm_rnkBLrZtTQe7RiHqWDn1oOejQscYzfwAU,25797
|
@@ -100,7 +100,7 @@ parsl/executors/taskvine/utils.py,sha256=iSrIogeiauL3UNy_9tiZp1cBSNn6fIJkMYQRVi1
|
|
100
100
|
parsl/executors/workqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
101
101
|
parsl/executors/workqueue/errors.py,sha256=XO2naYhAsHHyiOBH6hpObg3mPNDmvMoFqErsj0-v7jc,541
|
102
102
|
parsl/executors/workqueue/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
|
103
|
-
parsl/executors/workqueue/executor.py,sha256=
|
103
|
+
parsl/executors/workqueue/executor.py,sha256=R4cBX3Ac2OePpKsXo-4Kq68X6wAkyltYkhLysk45QwY,49855
|
104
104
|
parsl/executors/workqueue/parsl_coprocess.py,sha256=cF1UmTgVLoey6QzBcbYgEiEsRidSaFfuO54f1HFw_EM,5737
|
105
105
|
parsl/executors/workqueue/parsl_coprocess_stub.py,sha256=_bJmpPIgL42qM6bVzeEKt1Mn1trSP41rtJguXxPGfHI,735
|
106
106
|
parsl/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -115,23 +115,23 @@ parsl/launchers/errors.py,sha256=8YMV_CHpBNVa4eXkGE4x5DaFQlZkDCRCHmBktYcY6TA,467
|
|
115
115
|
parsl/launchers/launchers.py,sha256=cQsNsHuCOL_nQTjPXf0--YsgsDoMoJ77bO1Wt4ncLjs,15134
|
116
116
|
parsl/monitoring/__init__.py,sha256=0ywNz6i0lM1xo_7_BIxhETDGeVd2C_0wwD7qgeaMR4c,83
|
117
117
|
parsl/monitoring/db_manager.py,sha256=L0c5S9ockq0UIchT2bjmkSAWXS-t0G-Q_neOIBfLbm0,33444
|
118
|
-
parsl/monitoring/errors.py,sha256=
|
118
|
+
parsl/monitoring/errors.py,sha256=pol4Xp9lkpTurc1F5vJ0g48R4R-nUr8GBHubD7SU9Aw,165
|
119
119
|
parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
|
120
|
-
parsl/monitoring/monitoring.py,sha256=
|
120
|
+
parsl/monitoring/monitoring.py,sha256=8g8JVbR9Nq2JV17EBBwMaFFR-kvKDVd_PWtHyO0vUlQ,7564
|
121
121
|
parsl/monitoring/remote.py,sha256=t0qCTUMCzeJ_JOARFpjqlTNrAWdEb20BxhmZh9X7kEM,13728
|
122
122
|
parsl/monitoring/types.py,sha256=oOCrzv-ab-_rv4pb8o58Sdb8G_RGp1aZriRbdf9zBEk,339
|
123
123
|
parsl/monitoring/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
124
124
|
parsl/monitoring/queries/pandas.py,sha256=0Z2r0rjTKCemf0eaDkF1irvVHn5g7KC5SYETvQPRxwU,2232
|
125
125
|
parsl/monitoring/radios/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
126
126
|
parsl/monitoring/radios/base.py,sha256=MBW3aAbxQY7TMuNGZEN15dWoy3mAELOqz-GMN1d7vF4,221
|
127
|
-
parsl/monitoring/radios/filesystem.py,sha256=
|
128
|
-
parsl/monitoring/radios/filesystem_router.py,sha256=
|
129
|
-
parsl/monitoring/radios/htex.py,sha256=
|
127
|
+
parsl/monitoring/radios/filesystem.py,sha256=nn9NJ7EteEAixkqS2b0jJ9M7j55xdouXmYiK0qu5AjY,1763
|
128
|
+
parsl/monitoring/radios/filesystem_router.py,sha256=T6Okb5deWeLwaugb1Bkskjfdz1_KpM6pX2EyQLey8p4,3312
|
129
|
+
parsl/monitoring/radios/htex.py,sha256=1v4on4_KmJa_PuEc3a9OrL4NJV2h0bIuUKh_mdoPVKc,1641
|
130
130
|
parsl/monitoring/radios/multiprocessing.py,sha256=fsfaaoMDp6VJv1DSAl-P0R2ofO6jp13byx6NsPItV3Y,655
|
131
131
|
parsl/monitoring/radios/udp.py,sha256=bTpt7JYp-5hyBBLzgiLj1_BlSTn28UVp39OYgVGLXCw,1613
|
132
|
-
parsl/monitoring/radios/udp_router.py,sha256=
|
132
|
+
parsl/monitoring/radios/udp_router.py,sha256=M7UtHx5dYphKU4YKB2KmkA7aZRtqOFcxuaLVvBn8QJA,7590
|
133
133
|
parsl/monitoring/radios/zmq.py,sha256=fhoHp9ylhf-D3eTJb2aSHRsuic8-FJ_oRNGnniGkCAI,592
|
134
|
-
parsl/monitoring/radios/zmq_router.py,sha256=
|
134
|
+
parsl/monitoring/radios/zmq_router.py,sha256=oqGlOI4OkmvGLNKGp5F0iLFLYj3JdUkpnwg40JOvEak,7826
|
135
135
|
parsl/monitoring/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
136
136
|
parsl/monitoring/visualization/app.py,sha256=xMeRlAnzl5lHddAOdSBcqY3D5lmOYw3Z3Z2_YyoVwnw,1425
|
137
137
|
parsl/monitoring/visualization/models.py,sha256=C7CcF6w6PhtrdvDX9VgDH-aSrpLfvYU1fJ4-HDUeFVQ,5138
|
@@ -196,7 +196,7 @@ parsl/serialize/facade.py,sha256=3uOuVp0epfyLn7qDzuWqLfsy971YVGD3sqwqcAiRwh0,668
|
|
196
196
|
parsl/serialize/proxystore.py,sha256=o-ha9QAvVhbN8y9S1itk3W0O75eyHYZw2AvB2xu5_Lg,1624
|
197
197
|
parsl/tests/__init__.py,sha256=VTtJzOzz_x6fWNh8IOnsgFqVbdiJShi2AZH21mcmID4,204
|
198
198
|
parsl/tests/callables_helper.py,sha256=ceP1YYsNtrZgKT6MAIvpgdccEjQ_CpFEOnZBGHKGOx0,30
|
199
|
-
parsl/tests/conftest.py,sha256=
|
199
|
+
parsl/tests/conftest.py,sha256=PqXpj1AxpPQrcKXJBQ83WIF8TIzZ4-YhAjKQPahE1Tw,15618
|
200
200
|
parsl/tests/test_aalst_patterns.py,sha256=lNIxb7nIgh1yX7hR2fr_ck_mxYJxx8ASKK9zHUVqPno,9614
|
201
201
|
parsl/tests/test_callables.py,sha256=97vrIF1_hfDGd81FM1bhR6FemZMWFcALrH6pVHMTCt8,1974
|
202
202
|
parsl/tests/test_curvezmq.py,sha256=yyhlS4vmaZdMitiySoy4l_ih9H1bsPiN-tMdwIh3H20,12431
|
@@ -230,7 +230,6 @@ parsl/tests/configs/local_threads_checkpoint_task_exit.py,sha256=zHKN68T-xhAVQwQ
|
|
230
230
|
parsl/tests/configs/local_threads_ftp_in_task.py,sha256=c9odRbxgj1bM_ttpkWTh2Ch_MV7f5cmn-68BOjLeJ70,444
|
231
231
|
parsl/tests/configs/local_threads_globus.py,sha256=NhY27cD4vcqLh762Ye0BINZnt63EmTyHXg7FQMffOBw,1097
|
232
232
|
parsl/tests/configs/local_threads_http_in_task.py,sha256=csDY-C50tXKO2ntbbPBvppCRlXBcB7UCQOHN_FyfFYc,447
|
233
|
-
parsl/tests/configs/local_threads_monitoring.py,sha256=me-wWCtDwg_CGLQ7j4JRXcc_YTq3FGrffznjht07L08,368
|
234
233
|
parsl/tests/configs/local_threads_no_cache.py,sha256=2LM8rYhl62LIFUMjAs2_VI_R25YW5AI3RfVK_e5bdN8,236
|
235
234
|
parsl/tests/configs/midway.py,sha256=ZLdAUDR5paPA8gheRNLI0q9Vj5HcnCYuIttu-C-TlJs,1335
|
236
235
|
parsl/tests/configs/nscc_singapore.py,sha256=ECENZcBuCjkY6OWZstEMhfMrmjRmjCc7ELdfGEp7ly4,1481
|
@@ -254,7 +253,6 @@ parsl/tests/manual_tests/test_basic.py,sha256=3uCS9BqaZmKTNtNfJSsJIg2exlTAdC0Sdc
|
|
254
253
|
parsl/tests/manual_tests/test_log_filter.py,sha256=jwKclAVuESdlGK_giBuHDkY6ryX6rZB7q01lXT5K0XU,1872
|
255
254
|
parsl/tests/manual_tests/test_memory_limits.py,sha256=fjQhGsZu1PUl2Z4v75mssqve51QmtVrK94q9zJWQefU,2593
|
256
255
|
parsl/tests/manual_tests/test_regression_220.py,sha256=Jo2puWt1W0r1rJfaJFgd2ZPgR3i6uOXzrLRcfYDZWDo,931
|
257
|
-
parsl/tests/manual_tests/test_udp_simple.py,sha256=cleSqvzblTFN6sCG0egQtLYVSQeCo7JMpCMjYKjBZUw,1009
|
258
256
|
parsl/tests/manual_tests/test_worker_count.py,sha256=Cv8nAWMXAREiiGEBUr_8JyI87ffp8JGAyDqVXzcjX_0,2072
|
259
257
|
parsl/tests/scaling_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
260
258
|
parsl/tests/scaling_tests/htex_local.py,sha256=opgR-l18aunWeReuqBvNP7nSAk8CpsYqhAZE9PNPV8M,483
|
@@ -341,10 +339,12 @@ parsl/tests/test_monitoring/test_app_names.py,sha256=A-mOMCVhZDnUyJp32fsTUkHdcyv
|
|
341
339
|
parsl/tests/test_monitoring/test_basic.py,sha256=QJS0SJkf1KV6A4UvikBu-wAIcBJZVoVe-BSVWgyFonI,6180
|
342
340
|
parsl/tests/test_monitoring/test_db_locks.py,sha256=3s3c1xhKo230ZZIJ3f1Ca4U7LcEdXnanOGVXQyNlk2U,2895
|
343
341
|
parsl/tests/test_monitoring/test_exit_helper.py,sha256=ob8Qd1hlkq_mowygfPetTnYN9LfuqeXHRpPilSfDSog,1232
|
344
|
-
parsl/tests/test_monitoring/
|
342
|
+
parsl/tests/test_monitoring/test_htex_fuzz_zmq.py,sha256=MUaVBVjmoKSVg00DlMIRdj9KUizYvHjzW95Y3n_ViqE,3659
|
345
343
|
parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py,sha256=_QV8zjBKVF_qBbBnhT0C3X9AmfS7IKLcOnEw_cU6HeM,2622
|
346
344
|
parsl/tests/test_monitoring/test_incomplete_futures.py,sha256=ZnO1sFSwlWUBHX64C_zwfTVRVC_UFNlU4h0POgx6NEo,2005
|
347
345
|
parsl/tests/test_monitoring/test_memoization_representation.py,sha256=dknv2nO7pNZ1jGxWGsC_AW3rs90gjMIeC5d7pIJ75Xc,2645
|
346
|
+
parsl/tests/test_monitoring/test_radio_filesystem.py,sha256=hA-RXmU29nEG4Dm6eLcyVYoTdGog_wvYDcvNsN5-3lU,1531
|
347
|
+
parsl/tests/test_monitoring/test_radio_udp.py,sha256=dyb9SOY-ceSuZ4R2rAIn-7DqsRljYl4U6zNdACyCACU,1538
|
348
348
|
parsl/tests/test_monitoring/test_radio_zmq.py,sha256=7ARjDbde9kSuP4NCx_z-UtmMf2X5opbkkKM7mZcayuA,783
|
349
349
|
parsl/tests/test_monitoring/test_stdouterr.py,sha256=SEMKBk4v5Saoq3QiraFpzUpVe5vS_132GQPSf1Qu0qM,4573
|
350
350
|
parsl/tests/test_monitoring/test_viz_colouring.py,sha256=83Qdmn3gM0j7IL6kPDcuIsp_nl4zj-liPijyIN632SY,592
|
@@ -425,7 +425,7 @@ parsl/tests/test_serialization/test_pack_resource_spec.py,sha256=-Vtyh8KyezZw8e7
|
|
425
425
|
parsl/tests/test_serialization/test_proxystore_configured.py,sha256=lGWOSEWul16enDWhW-s7CK0d3eMDzm1324Fmj0cZMVU,2293
|
426
426
|
parsl/tests/test_serialization/test_proxystore_impl.py,sha256=uGd45sfPm9rJhzqKV0rI3lqdSOAUddQf-diEpcJAlcY,1228
|
427
427
|
parsl/tests/test_shutdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
428
|
-
parsl/tests/test_shutdown/test_kill_monitoring.py,sha256=
|
428
|
+
parsl/tests/test_shutdown/test_kill_monitoring.py,sha256=yWeYgizqGAq1ezlKWwRrJGcc0U9d2Vuzp0YuXYO1b20,1891
|
429
429
|
parsl/tests/test_staging/__init__.py,sha256=WZl9EHSkfYiSoE3Gbulcq2ifmn7IFGUkasJIobL5T5A,208
|
430
430
|
parsl/tests/test_staging/staging_provider.py,sha256=6FDpImkWOLgysqM68NbCAoXZciZokI8dmBWRAxnggBk,3242
|
431
431
|
parsl/tests/test_staging/test_1316.py,sha256=eS0e2BDM2vmPNF60aDr35wcuGgDPfXjTjRV6kyBZOQc,2652
|
@@ -460,13 +460,13 @@ parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
|
|
460
460
|
parsl/usage_tracking/api.py,sha256=iaCY58Dc5J4UM7_dJzEEs871P1p1HdxBMtNGyVdzc9g,1821
|
461
461
|
parsl/usage_tracking/levels.py,sha256=xbfzYEsd55KiZJ-mzNgPebvOH4rRHum04hROzEf41tU,291
|
462
462
|
parsl/usage_tracking/usage.py,sha256=hbMo5BYgIWqMcFWqN-HYP1TbwNrTonpv-usfwnCFJKY,9212
|
463
|
-
parsl-2025.6.
|
464
|
-
parsl-2025.6.
|
465
|
-
parsl-2025.6.
|
466
|
-
parsl-2025.6.
|
467
|
-
parsl-2025.6.
|
468
|
-
parsl-2025.6.
|
469
|
-
parsl-2025.6.
|
470
|
-
parsl-2025.6.
|
471
|
-
parsl-2025.6.
|
472
|
-
parsl-2025.6.
|
463
|
+
parsl-2025.6.23.data/scripts/exec_parsl_function.py,sha256=YXKVVIa4zXmOtz-0Ca4E_5nQfN_3S2bh2tB75uZZB4w,7774
|
464
|
+
parsl-2025.6.23.data/scripts/interchange.py,sha256=_FRB1LxkL9vnT3y24NTXHOzotMlDJEXwF5ZZCjGmcww,28909
|
465
|
+
parsl-2025.6.23.data/scripts/parsl_coprocess.py,sha256=zrVjEqQvFOHxsLufPi00xzMONagjVwLZbavPM7bbjK4,5722
|
466
|
+
parsl-2025.6.23.data/scripts/process_worker_pool.py,sha256=__gFeFQJpV5moRofj3WKQCnKp6gmzieXjzkmzVuTmX4,41123
|
467
|
+
parsl-2025.6.23.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
|
468
|
+
parsl-2025.6.23.dist-info/METADATA,sha256=MzCr6YRZ0einE2SSqmtGkBDaX8tdfR0I57QP3W3QOXE,4055
|
469
|
+
parsl-2025.6.23.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
|
470
|
+
parsl-2025.6.23.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
|
471
|
+
parsl-2025.6.23.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
|
472
|
+
parsl-2025.6.23.dist-info/RECORD,,
|
@@ -1,10 +0,0 @@
|
|
1
|
-
from parsl import ThreadPoolExecutor
|
2
|
-
from parsl.config import Config
|
3
|
-
from parsl.monitoring import MonitoringHub
|
4
|
-
|
5
|
-
config = Config(executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
|
6
|
-
monitoring=MonitoringHub(
|
7
|
-
hub_address="localhost",
|
8
|
-
resource_monitoring_interval=3,
|
9
|
-
)
|
10
|
-
)
|
@@ -1,51 +0,0 @@
|
|
1
|
-
import logging
|
2
|
-
|
3
|
-
import parsl
|
4
|
-
from parsl import python_app
|
5
|
-
from parsl.config import Config
|
6
|
-
from parsl.executors import ThreadPoolExecutor
|
7
|
-
from parsl.monitoring.monitoring import MonitoringHub
|
8
|
-
|
9
|
-
|
10
|
-
def local_setup():
|
11
|
-
threads_config = Config(
|
12
|
-
executors=[ThreadPoolExecutor(
|
13
|
-
label='threads',
|
14
|
-
max_threads=4)
|
15
|
-
],
|
16
|
-
monitoring=MonitoringHub(
|
17
|
-
hub_address="127.0.0.1",
|
18
|
-
logging_level=logging.INFO,
|
19
|
-
resource_monitoring_interval=10))
|
20
|
-
|
21
|
-
parsl.load(threads_config)
|
22
|
-
|
23
|
-
|
24
|
-
def local_teardown():
|
25
|
-
parsl.clear()
|
26
|
-
|
27
|
-
|
28
|
-
@python_app
|
29
|
-
def sleeper(dur=25):
|
30
|
-
import time
|
31
|
-
time.sleep(dur)
|
32
|
-
|
33
|
-
|
34
|
-
@python_app
|
35
|
-
def cpu_stress(dur=30):
|
36
|
-
import time
|
37
|
-
s = 0
|
38
|
-
start = time.time()
|
39
|
-
for i in range(10**8):
|
40
|
-
s += i
|
41
|
-
if time.time() - start >= dur:
|
42
|
-
break
|
43
|
-
return s
|
44
|
-
|
45
|
-
|
46
|
-
if __name__ == "__main__":
|
47
|
-
|
48
|
-
tasks = [sleeper() for i in range(8)]
|
49
|
-
# tasks = [cpu_stress() for i in range(10)]
|
50
|
-
|
51
|
-
print([i.result() for i in tasks])
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|