parsl 2025.8.4__py3-none-any.whl → 2025.11.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. parsl/__init__.py +0 -4
  2. parsl/app/bash.py +1 -1
  3. parsl/benchmark/perf.py +73 -17
  4. parsl/concurrent/__init__.py +95 -14
  5. parsl/curvezmq.py +0 -16
  6. parsl/data_provider/globus.py +3 -1
  7. parsl/dataflow/dflow.py +107 -207
  8. parsl/dataflow/memoization.py +144 -31
  9. parsl/dataflow/states.py +5 -5
  10. parsl/executors/base.py +2 -2
  11. parsl/executors/execute_task.py +2 -8
  12. parsl/executors/flux/executor.py +4 -6
  13. parsl/executors/globus_compute.py +0 -4
  14. parsl/executors/high_throughput/executor.py +86 -25
  15. parsl/executors/high_throughput/interchange.py +55 -42
  16. parsl/executors/high_throughput/mpi_executor.py +1 -2
  17. parsl/executors/high_throughput/mpi_resource_management.py +7 -14
  18. parsl/executors/high_throughput/process_worker_pool.py +32 -7
  19. parsl/executors/high_throughput/zmq_pipes.py +36 -67
  20. parsl/executors/radical/executor.py +2 -6
  21. parsl/executors/radical/rpex_worker.py +2 -2
  22. parsl/executors/taskvine/executor.py +5 -1
  23. parsl/executors/threads.py +5 -2
  24. parsl/jobs/states.py +2 -2
  25. parsl/jobs/strategy.py +7 -6
  26. parsl/monitoring/db_manager.py +21 -23
  27. parsl/monitoring/monitoring.py +2 -2
  28. parsl/monitoring/radios/filesystem.py +2 -1
  29. parsl/monitoring/radios/htex.py +2 -1
  30. parsl/monitoring/radios/multiprocessing.py +2 -1
  31. parsl/monitoring/radios/udp.py +2 -1
  32. parsl/monitoring/radios/udp_router.py +2 -2
  33. parsl/monitoring/radios/zmq_router.py +2 -2
  34. parsl/multiprocessing.py +0 -49
  35. parsl/providers/base.py +24 -37
  36. parsl/providers/pbspro/pbspro.py +1 -1
  37. parsl/serialize/__init__.py +6 -9
  38. parsl/serialize/facade.py +0 -32
  39. parsl/tests/configs/local_threads_globus.py +18 -14
  40. parsl/tests/configs/taskvine_ex.py +1 -1
  41. parsl/tests/manual_tests/test_memory_limits.py +1 -1
  42. parsl/tests/sites/test_concurrent.py +51 -3
  43. parsl/tests/test_checkpointing/test_periodic.py +15 -9
  44. parsl/tests/test_checkpointing/test_python_checkpoint_1.py +6 -3
  45. parsl/tests/test_checkpointing/test_regression_233.py +0 -1
  46. parsl/tests/test_curvezmq.py +0 -42
  47. parsl/tests/test_execute_task.py +2 -11
  48. parsl/tests/test_htex/test_command_concurrency_regression_1321.py +54 -0
  49. parsl/tests/test_htex/test_htex.py +36 -1
  50. parsl/tests/test_htex/test_interchange_exit_bad_registration.py +2 -2
  51. parsl/tests/test_htex/test_priority_queue.py +26 -3
  52. parsl/tests/test_htex/test_zmq_binding.py +2 -1
  53. parsl/tests/test_mpi_apps/test_mpi_scheduler.py +18 -43
  54. parsl/tests/test_python_apps/test_basic.py +0 -14
  55. parsl/tests/test_python_apps/test_depfail_propagation.py +11 -1
  56. parsl/tests/test_python_apps/test_exception.py +19 -0
  57. parsl/tests/test_python_apps/test_garbage_collect.py +1 -6
  58. parsl/tests/test_python_apps/test_memoize_2.py +11 -1
  59. parsl/tests/test_python_apps/test_memoize_exception.py +41 -0
  60. parsl/tests/test_regression/test_3874.py +47 -0
  61. parsl/tests/test_scaling/test_regression_3696_oscillation.py +1 -0
  62. parsl/tests/test_staging/test_staging_globus.py +2 -2
  63. parsl/tests/test_utils/test_representation_mixin.py +53 -0
  64. parsl/tests/unit/test_globus_compute_executor.py +11 -2
  65. parsl/utils.py +11 -3
  66. parsl/version.py +1 -1
  67. {parsl-2025.8.4.data → parsl-2025.11.10.data}/scripts/interchange.py +55 -42
  68. {parsl-2025.8.4.data → parsl-2025.11.10.data}/scripts/process_worker_pool.py +32 -7
  69. {parsl-2025.8.4.dist-info → parsl-2025.11.10.dist-info}/METADATA +64 -50
  70. {parsl-2025.8.4.dist-info → parsl-2025.11.10.dist-info}/RECORD +76 -81
  71. {parsl-2025.8.4.dist-info → parsl-2025.11.10.dist-info}/WHEEL +1 -1
  72. parsl/tests/configs/local_threads_checkpoint_periodic.py +0 -11
  73. parsl/tests/configs/local_threads_no_cache.py +0 -11
  74. parsl/tests/site_tests/test_provider.py +0 -88
  75. parsl/tests/site_tests/test_site.py +0 -70
  76. parsl/tests/test_aalst_patterns.py +0 -474
  77. parsl/tests/test_docs/test_workflow2.py +0 -42
  78. parsl/tests/test_error_handling/test_rand_fail.py +0 -171
  79. parsl/tests/test_regression/test_854.py +0 -62
  80. parsl/tests/test_serialization/test_pack_resource_spec.py +0 -23
  81. {parsl-2025.8.4.data → parsl-2025.11.10.data}/scripts/exec_parsl_function.py +0 -0
  82. {parsl-2025.8.4.data → parsl-2025.11.10.data}/scripts/parsl_coprocess.py +0 -0
  83. {parsl-2025.8.4.dist-info → parsl-2025.11.10.dist-info}/entry_points.txt +0 -0
  84. {parsl-2025.8.4.dist-info → parsl-2025.11.10.dist-info/licenses}/LICENSE +0 -0
  85. {parsl-2025.8.4.dist-info → parsl-2025.11.10.dist-info}/top_level.txt +0 -0
@@ -9,6 +9,18 @@ class GoodRepr(RepresentationMixin):
9
9
  self.y = y
10
10
 
11
11
 
12
+ class GoodReprDefaults(RepresentationMixin):
13
+ def __init__(self, x, y="default 2"):
14
+ self.x = x
15
+ self.y = y
16
+
17
+
18
+ class GoodReprKeywordOnly(RepresentationMixin):
19
+ def __init__(self, *, x, y):
20
+ self.x = x
21
+ self.y = y
22
+
23
+
12
24
  class BadRepr(RepresentationMixin):
13
25
  """This class incorrectly subclasses RepresentationMixin.
14
26
  It does not store the parameter x on self.
@@ -31,6 +43,47 @@ def test_repr_good():
31
43
  assert p2 in r
32
44
 
33
45
 
46
+ @pytest.mark.local
47
+ def test_repr_good_defaults_overridden():
48
+ p1 = "parameter 1"
49
+ p2 = "the second parameter"
50
+
51
+ # repr should not raise an exception
52
+ r = repr(GoodReprDefaults(p1, p2))
53
+
54
+ # representation should contain both values supplied
55
+ # at object creation.
56
+ assert p1 in r
57
+ assert p2 in r
58
+
59
+
60
+ @pytest.mark.local
61
+ def test_repr_good_defaults_defaulted():
62
+ p1 = "parameter 1"
63
+
64
+ # repr should not raise an exception
65
+ r = repr(GoodReprDefaults(p1))
66
+
67
+ # representation should contain one value supplied
68
+ # at object creation, and the other defaulted.
69
+ assert p1 in r
70
+ assert "default 2" in r
71
+
72
+
73
+ @pytest.mark.local
74
+ def test_repr_good_keyword_only():
75
+ p1 = "parameter 1"
76
+ p2 = "the second parameter"
77
+
78
+ # repr should not raise an exception
79
+ r = repr(GoodReprKeywordOnly(x=p1, y=p2))
80
+
81
+ # representation should contain both values supplied
82
+ # at object creation.
83
+ assert p1 in r
84
+ assert p2 in r
85
+
86
+
34
87
  @pytest.mark.local
35
88
  def test_repr_bad():
36
89
  p1 = "parameter 1"
@@ -2,18 +2,21 @@ import random
2
2
  from unittest import mock
3
3
 
4
4
  import pytest
5
- from globus_compute_sdk import Executor
6
5
 
7
6
  from parsl.executors import GlobusComputeExecutor
8
7
 
9
8
 
10
9
  @pytest.fixture
11
10
  def mock_ex():
12
- # Not Parsl's job to test GC's Executor
11
+ # Not Parsl's job to test GC's Executor, although it
12
+ # still needs to be importable for these test cases.
13
+ from globus_compute_sdk import Executor
14
+
13
15
  yield mock.Mock(spec=Executor)
14
16
 
15
17
 
16
18
  @pytest.mark.local
19
+ @pytest.mark.globus_compute
17
20
  def test_gc_executor_mock_spec(mock_ex):
18
21
  # a test of tests -- make sure we're using spec= in the mock
19
22
  with pytest.raises(AttributeError):
@@ -21,12 +24,14 @@ def test_gc_executor_mock_spec(mock_ex):
21
24
 
22
25
 
23
26
  @pytest.mark.local
27
+ @pytest.mark.globus_compute
24
28
  def test_gc_executor_label_default(mock_ex):
25
29
  gce = GlobusComputeExecutor(mock_ex)
26
30
  assert gce.label == type(gce).__name__, "Expect reasonable default label"
27
31
 
28
32
 
29
33
  @pytest.mark.local
34
+ @pytest.mark.globus_compute
30
35
  def test_gc_executor_label(mock_ex, randomstring):
31
36
  exp_label = randomstring()
32
37
  gce = GlobusComputeExecutor(mock_ex, label=exp_label)
@@ -34,6 +39,7 @@ def test_gc_executor_label(mock_ex, randomstring):
34
39
 
35
40
 
36
41
  @pytest.mark.local
42
+ @pytest.mark.globus_compute
37
43
  def test_gc_executor_resets_spec_after_submit(mock_ex, randomstring):
38
44
  submit_res = {randomstring(): "some submit res"}
39
45
  res = {"some": randomstring(), "spec": randomstring()}
@@ -57,6 +63,7 @@ def test_gc_executor_resets_spec_after_submit(mock_ex, randomstring):
57
63
 
58
64
 
59
65
  @pytest.mark.local
66
+ @pytest.mark.globus_compute
60
67
  def test_gc_executor_resets_uep_after_submit(mock_ex, randomstring):
61
68
  uep_conf = randomstring()
62
69
  res = {"some": randomstring()}
@@ -79,6 +86,7 @@ def test_gc_executor_resets_uep_after_submit(mock_ex, randomstring):
79
86
 
80
87
 
81
88
  @pytest.mark.local
89
+ @pytest.mark.globus_compute
82
90
  def test_gc_executor_happy_path(mock_ex, randomstring):
83
91
  mock_fn = mock.Mock()
84
92
  args = tuple(randomstring() for _ in range(random.randint(0, 3)))
@@ -95,6 +103,7 @@ def test_gc_executor_happy_path(mock_ex, randomstring):
95
103
 
96
104
 
97
105
  @pytest.mark.local
106
+ @pytest.mark.globus_compute
98
107
  def test_gc_executor_shuts_down_asynchronously(mock_ex):
99
108
  gce = GlobusComputeExecutor(mock_ex)
100
109
  gce.shutdown()
parsl/utils.py CHANGED
@@ -11,7 +11,6 @@ from types import TracebackType
11
11
  from typing import (
12
12
  IO,
13
13
  Any,
14
- AnyStr,
15
14
  Callable,
16
15
  Dict,
17
16
  Generator,
@@ -132,7 +131,13 @@ def get_std_fname_mode(
132
131
  mode = 'a+'
133
132
  elif isinstance(stdfspec, tuple):
134
133
  if len(stdfspec) != 2:
135
- msg = (f"std descriptor {fdname} has incorrect tuple length "
134
+ # this is annotated as unreachable because the type annotation says
135
+ # it cannot be reached. Earlier versions of typeguard did not enforce
136
+ # that type annotation at runtime, though, and the parameters to this
137
+ # function come from the user.
138
+ # When typeguard lower bound is raised to around version 4, this
139
+ # unreachable can be removed.
140
+ msg = (f"std descriptor {fdname} has incorrect tuple length " # type: ignore[unreachable]
136
141
  f"{len(stdfspec)}")
137
142
  raise pe.BadStdStreamFile(msg)
138
143
  fname, mode = stdfspec
@@ -157,7 +162,7 @@ def wait_for_file(path: str, seconds: int = 10) -> Generator[None, None, None]:
157
162
 
158
163
 
159
164
  @contextmanager
160
- def time_limited_open(path: str, mode: str, seconds: int = 1) -> Generator[IO[AnyStr], None, None]:
165
+ def time_limited_open(path: str, mode: str, seconds: int = 1) -> Generator[IO, None, None]:
161
166
  with wait_for_file(path, seconds):
162
167
  logger.debug("wait_for_file yielded")
163
168
  f = open(path, mode)
@@ -250,6 +255,9 @@ class RepresentationMixin:
250
255
  args = [getattr(self, a, default) for a in argspec.args[1:]]
251
256
  kwargs = {key: getattr(self, key, default) for key in defaults}
252
257
 
258
+ kwonlyargs = {key: getattr(self, key, default) for key in argspec.kwonlyargs}
259
+ kwargs.update(kwonlyargs)
260
+
253
261
  def assemble_multiline(args: List[str], kwargs: Dict[str, object]) -> str:
254
262
  def indent(text: str) -> str:
255
263
  lines = text.splitlines()
parsl/version.py CHANGED
@@ -3,4 +3,4 @@
3
3
  Year.Month.Day[alpha/beta/..]
4
4
  Alphas will be numbered like this -> 2024.12.10a0
5
5
  """
6
- VERSION = '2025.08.04'
6
+ VERSION = '2025.11.10'
@@ -23,7 +23,6 @@ from parsl.monitoring.radios.base import MonitoringRadioSender
23
23
  from parsl.monitoring.radios.zmq import ZMQRadioSender
24
24
  from parsl.process_loggers import wrap_with_logs
25
25
  from parsl.serialize import serialize as serialize_object
26
- from parsl.utils import setproctitle
27
26
  from parsl.version import VERSION as PARSL_VERSION
28
27
 
29
28
  PKL_HEARTBEAT_CODE = pickle.dumps((2 ** 32) - 1)
@@ -56,6 +55,7 @@ class Interchange:
56
55
  cert_dir: Optional[str],
57
56
  manager_selector: ManagerSelector,
58
57
  run_id: str,
58
+ _check_python_mismatch: bool,
59
59
  ) -> None:
60
60
  """
61
61
  Parameters
@@ -99,6 +99,11 @@ class Interchange:
99
99
 
100
100
  cert_dir : str | None
101
101
  Path to the certificate directory.
102
+
103
+ _check_python_mismatch : bool
104
+ If True, the interchange and worker managers must run the same version of
105
+ Python. Running different versions can cause inter-process communication
106
+ errors, so proceed with caution.
102
107
  """
103
108
  self.cert_dir = cert_dir
104
109
  self.logdir = logdir
@@ -126,15 +131,13 @@ class Interchange:
126
131
  logger.info("Connected to client")
127
132
 
128
133
  self.run_id = run_id
134
+ self._check_python_mismatch = _check_python_mismatch
129
135
 
130
136
  self.hub_address = hub_address
131
137
  self.hub_zmq_port = hub_zmq_port
132
138
 
133
139
  self.pending_task_queue: SortedList[Any] = SortedList(key=lambda tup: (tup[0], tup[1]))
134
140
 
135
- # count of tasks that have been received from the submit side
136
- self.task_counter = 0
137
-
138
141
  # count of tasks that have been sent out to worker pools
139
142
  self.count = 0
140
143
 
@@ -157,6 +160,7 @@ class Interchange:
157
160
  logger.info(f"Bound to port {worker_port} for incoming worker connections")
158
161
 
159
162
  self._ready_managers: Dict[bytes, ManagerRecord] = {}
163
+ self._logged_manager_count_token: object = None
160
164
  self.connected_block_history: List[str] = []
161
165
 
162
166
  self.heartbeat_threshold = heartbeat_threshold
@@ -213,7 +217,7 @@ class Interchange:
213
217
 
214
218
  reply: Any # the type of reply depends on the command_req received (aka this needs dependent types...)
215
219
 
216
- if self.command_channel in self.socks and self.socks[self.command_channel] == zmq.POLLIN:
220
+ if self.socks.get(self.command_channel) == zmq.POLLIN:
217
221
  logger.debug("entering command_server section")
218
222
 
219
223
  command_req = self.command_channel.recv_pyobj()
@@ -222,35 +226,29 @@ class Interchange:
222
226
  reply = self.connected_block_history
223
227
 
224
228
  elif command_req == "WORKERS":
225
- num_workers = 0
226
- for manager in self._ready_managers.values():
227
- num_workers += manager['worker_count']
228
- reply = num_workers
229
+ reply = sum(m['worker_count'] for m in self._ready_managers.values())
229
230
 
230
231
  elif command_req == "MANAGERS":
231
232
  reply = []
232
- for manager_id in self._ready_managers:
233
- m = self._ready_managers[manager_id]
234
- idle_since = m['idle_since']
235
- if idle_since is not None:
236
- idle_duration = time.time() - idle_since
237
- else:
238
- idle_duration = 0.0
239
- resp = {'manager': manager_id.decode('utf-8'),
240
- 'block_id': m['block_id'],
241
- 'worker_count': m['worker_count'],
242
- 'tasks': len(m['tasks']),
243
- 'idle_duration': idle_duration,
244
- 'active': m['active'],
245
- 'parsl_version': m['parsl_version'],
246
- 'python_version': m['python_version'],
247
- 'draining': m['draining']}
233
+ now = time.time()
234
+ for manager_id, m in self._ready_managers.items():
235
+ idle_duration = now - (m['idle_since'] or now)
236
+ resp = {
237
+ 'manager': manager_id.decode('utf-8'),
238
+ 'block_id': m['block_id'],
239
+ 'worker_count': m['worker_count'],
240
+ 'tasks': len(m['tasks']),
241
+ 'idle_duration': idle_duration,
242
+ 'active': m['active'],
243
+ 'parsl_version': m['parsl_version'],
244
+ 'python_version': m['python_version'],
245
+ 'draining': m['draining']
246
+ }
248
247
  reply.append(resp)
249
248
 
250
249
  elif command_req == "MANAGERS_PACKAGES":
251
250
  reply = {}
252
- for manager_id in self._ready_managers:
253
- m = self._ready_managers[manager_id]
251
+ for manager_id, m in self._ready_managers.items():
254
252
  manager_id_str = manager_id.decode('utf-8')
255
253
  reply[manager_id_str] = m["packages"]
256
254
 
@@ -316,6 +314,7 @@ class Interchange:
316
314
  self.process_manager_socket_message(interesting_managers, monitoring_radio, kill_event)
317
315
  self.expire_bad_managers(interesting_managers, monitoring_radio)
318
316
  self.expire_drained_managers(interesting_managers, monitoring_radio)
317
+ self.log_manager_counts(interesting_managers)
319
318
  self.process_tasks_to_send(interesting_managers, monitoring_radio)
320
319
 
321
320
  self.zmq_context.destroy()
@@ -327,20 +326,20 @@ class Interchange:
327
326
  """Process incoming task message(s).
328
327
  """
329
328
 
330
- if self.task_incoming in self.socks and self.socks[self.task_incoming] == zmq.POLLIN:
329
+ if self.socks.get(self.task_incoming) == zmq.POLLIN:
331
330
  logger.debug("start task_incoming section")
332
331
  msg = self.task_incoming.recv_pyobj()
333
332
 
334
333
  # Process priority, higher number = lower priority
335
- resource_spec = msg.get('resource_spec', {})
334
+ task_id = msg['task_id']
335
+ resource_spec = msg['context'].get('resource_spec', {})
336
336
  priority = resource_spec.get('priority', float('inf'))
337
- queue_entry = (-priority, -self.task_counter, msg)
337
+ queue_entry = (-priority, -task_id, msg)
338
338
 
339
- logger.debug("putting message onto pending_task_queue")
339
+ logger.debug("Putting task %s onto pending_task_queue", task_id)
340
340
 
341
341
  self.pending_task_queue.add(queue_entry)
342
- self.task_counter += 1
343
- logger.debug(f"Fetched {self.task_counter} tasks so far")
342
+ logger.debug("Put task %s onto pending_task_queue", task_id)
344
343
 
345
344
  def process_manager_socket_message(
346
345
  self,
@@ -360,9 +359,10 @@ class Interchange:
360
359
  mtype = meta['type']
361
360
  except Exception as e:
362
361
  logger.warning(
363
- f'Failed to read manager message ([{type(e).__name__}] {e})'
362
+ 'Failed to read manager message; ignoring message'
363
+ f' (Exception: [{type(e).__name__}] {e})'
364
364
  )
365
- logger.debug('Message:\n %r\n', msg_parts, exc_info=e)
365
+ logger.debug('Raw message bytes:\n %r\n', msg_parts, exc_info=e)
366
366
  return
367
367
 
368
368
  logger.debug(
@@ -402,7 +402,9 @@ class Interchange:
402
402
  logger.info(f'Registration info for manager {manager_id!r}: {meta}')
403
403
  self._send_monitoring_info(monitoring_radio, new_rec)
404
404
 
405
- if (mgr_minor_py, mgr_parsl_v) != (ix_minor_py, ix_parsl_v):
405
+ python_mismatch: bool = ix_minor_py != mgr_minor_py
406
+ parsl_mismatch: bool = ix_parsl_v != mgr_parsl_v
407
+ if parsl_mismatch or (self._check_python_mismatch and python_mismatch):
406
408
  kill_event.set()
407
409
  vm_exc = VersionMismatch(
408
410
  f"py.v={ix_minor_py} parsl.v={ix_parsl_v}",
@@ -523,15 +525,24 @@ class Interchange:
523
525
  m['active'] = False
524
526
  self._send_monitoring_info(monitoring_radio, m)
525
527
 
528
+ def log_manager_counts(self, interesting_managers: Set[bytes]) -> None:
529
+ count_interesting = len(interesting_managers)
530
+ count_ready = len(self._ready_managers)
531
+
532
+ new_logged_manager_count_token = (count_interesting, count_ready)
533
+
534
+ if self._logged_manager_count_token != new_logged_manager_count_token:
535
+
536
+ logger.debug(
537
+ "Managers count (interesting/total): %d/%d",
538
+ count_interesting,
539
+ count_ready
540
+ )
541
+ self._logged_manager_count_token = new_logged_manager_count_token
542
+
526
543
  def process_tasks_to_send(self, interesting_managers: Set[bytes], monitoring_radio: Optional[MonitoringRadioSender]) -> None:
527
544
  # Check if there are tasks that could be sent to managers
528
545
 
529
- logger.debug(
530
- "Managers count (interesting/total): %d/%d",
531
- len(interesting_managers),
532
- len(self._ready_managers)
533
- )
534
-
535
546
  if interesting_managers and self.pending_task_queue:
536
547
  shuffled_managers = self.manager_selector.sort_managers(self._ready_managers, interesting_managers)
537
548
 
@@ -624,6 +635,8 @@ def start_file_logger(filename: str, level: int = logging.DEBUG, format_string:
624
635
 
625
636
 
626
637
  if __name__ == "__main__":
638
+ from parsl.utils import setproctitle
639
+
627
640
  setproctitle("parsl: HTEX interchange")
628
641
 
629
642
  config = pickle.load(sys.stdin.buffer)
@@ -1,6 +1,7 @@
1
1
  #!python
2
2
 
3
3
  import argparse
4
+ import importlib
4
5
  import logging
5
6
  import math
6
7
  import multiprocessing
@@ -17,7 +18,7 @@ from importlib.metadata import distributions
17
18
  from multiprocessing.context import SpawnProcess
18
19
  from multiprocessing.managers import DictProxy
19
20
  from multiprocessing.sharedctypes import Synchronized
20
- from typing import Dict, List, Optional, Sequence
21
+ from typing import Callable, Dict, List, Optional, Sequence
21
22
 
22
23
  import psutil
23
24
  import zmq
@@ -348,7 +349,7 @@ class Manager:
348
349
 
349
350
  logger.debug(
350
351
  'ready workers: %d, pending tasks: %d',
351
- self.ready_worker_count.value, # type: ignore[attr-defined]
352
+ self.ready_worker_count.value,
352
353
  pending_task_count,
353
354
  )
354
355
 
@@ -373,10 +374,12 @@ class Manager:
373
374
  if socks.get(ix_sock) == zmq.POLLIN:
374
375
  pkl_msg = ix_sock.recv()
375
376
  tasks = pickle.loads(pkl_msg)
377
+ del pkl_msg
378
+
376
379
  last_interchange_contact = time.time()
377
380
 
378
381
  if tasks == HEARTBEAT_CODE:
379
- logger.debug("Got heartbeat from interchange")
382
+ logger.debug("Got heartbeat response from interchange")
380
383
  elif tasks == DRAINED_CODE:
381
384
  logger.info("Got fully drained message from interchange - setting kill flag")
382
385
  self._stop_event.set()
@@ -454,6 +457,7 @@ class Manager:
454
457
  'exception': serialize(RemoteExceptionWrapper(*sys.exc_info()))}
455
458
  pkl_package = pickle.dumps(result_package)
456
459
  self.pending_result_queue.put(pkl_package)
460
+ del pkl_package
457
461
  except KeyError:
458
462
  logger.info("Worker {} was not busy when it died".format(worker_id))
459
463
 
@@ -603,6 +607,10 @@ def update_resource_spec_env_vars(mpi_launcher: str, resource_spec: Dict, node_i
603
607
 
604
608
 
605
609
  def _init_mpi_env(mpi_launcher: str, resource_spec: Dict):
610
+ for varname in resource_spec:
611
+ envname = "PARSL_" + str(varname).upper()
612
+ os.environ[envname] = str(resource_spec[varname])
613
+
606
614
  node_list = resource_spec.get("MPI_NODELIST")
607
615
  if node_list is None:
608
616
  return
@@ -753,8 +761,8 @@ def worker(
753
761
  worker_enqueued = True
754
762
 
755
763
  try:
756
- # The worker will receive {'task_id':<tid>, 'buffer':<buf>}
757
764
  req = task_queue.get(timeout=task_queue_timeout)
765
+ # req is {'task_id':<tid>, 'buffer':<buf>, 'resource_spec':<dict>}
758
766
  except queue.Empty:
759
767
  continue
760
768
 
@@ -766,17 +774,33 @@ def worker(
766
774
  ready_worker_count.value -= 1
767
775
  worker_enqueued = False
768
776
 
769
- _init_mpi_env(mpi_launcher=mpi_launcher, resource_spec=req["resource_spec"])
777
+ ctxt = req["context"]
778
+ res_spec = ctxt.get("resource_spec", {})
779
+
780
+ _init_mpi_env(mpi_launcher=mpi_launcher, resource_spec=res_spec)
781
+
782
+ exec_func: Callable = execute_task
783
+ exec_args = ()
784
+ exec_kwargs = {}
770
785
 
771
786
  try:
772
- result = execute_task(req['buffer'])
787
+ if task_executor := ctxt.get("task_executor", None):
788
+ mod_name, _, fn_name = task_executor["f"].rpartition(".")
789
+ exec_mod = importlib.import_module(mod_name)
790
+ exec_func = getattr(exec_mod, fn_name)
791
+
792
+ exec_args = task_executor.get("a", ())
793
+ exec_kwargs = task_executor.get("k", {})
794
+
795
+ result = exec_func(req['buffer'], *exec_args, **exec_kwargs)
773
796
  serialized_result = serialize(result, buffer_threshold=1000000)
774
797
  except Exception as e:
775
798
  logger.info('Caught an exception: {}'.format(e))
776
799
  result_package = {'type': 'result', 'task_id': tid, 'exception': serialize(RemoteExceptionWrapper(*sys.exc_info()))}
777
800
  else:
778
801
  result_package = {'type': 'result', 'task_id': tid, 'result': serialized_result}
779
- # logger.debug("Result: {}".format(result))
802
+ del serialized_result
803
+ del req
780
804
 
781
805
  logger.info("Completed executor task {}".format(tid))
782
806
  try:
@@ -788,6 +812,7 @@ def worker(
788
812
  })
789
813
 
790
814
  result_queue.put(pkl_package)
815
+ del pkl_package, result_package
791
816
  tasks_in_progress.pop(worker_id)
792
817
  logger.info("All processing finished for executor task {}".format(tid))
793
818
 
@@ -1,9 +1,9 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: parsl
3
- Version: 2025.8.4
3
+ Version: 2025.11.10
4
4
  Summary: Simple data dependent workflows in Python
5
5
  Home-page: https://github.com/Parsl/parsl
6
- Download-URL: https://github.com/Parsl/parsl/archive/2025.08.04.tar.gz
6
+ Download-URL: https://github.com/Parsl/parsl/archive/2025.11.10.tar.gz
7
7
  Author: The Parsl Team
8
8
  Author-email: parsl@googlegroups.com
9
9
  License: Apache 2.0
@@ -11,16 +11,14 @@ Keywords: Workflows,Scientific computing
11
11
  Classifier: Development Status :: 5 - Production/Stable
12
12
  Classifier: Intended Audience :: Developers
13
13
  Classifier: License :: OSI Approved :: Apache Software License
14
- Classifier: Programming Language :: Python :: 3.9
15
14
  Classifier: Programming Language :: Python :: 3.10
16
15
  Classifier: Programming Language :: Python :: 3.11
17
16
  Classifier: Programming Language :: Python :: 3.12
18
- Requires-Python: >=3.9.0
17
+ Requires-Python: >=3.10.0
19
18
  License-File: LICENSE
20
19
  Requires-Dist: pyzmq>=17.1.2
21
20
  Requires-Dist: typeguard!=3.*,<5,>=2.10
22
21
  Requires-Dist: typing-extensions<5,>=4.6
23
- Requires-Dist: globus-sdk
24
22
  Requires-Dist: dill
25
23
  Requires-Dist: tblib
26
24
  Requires-Dist: requests
@@ -28,12 +26,54 @@ Requires-Dist: sortedcontainers
28
26
  Requires-Dist: psutil>=5.5.1
29
27
  Requires-Dist: setproctitle
30
28
  Requires-Dist: filelock<4,>=3.13
29
+ Provides-Extra: monitoring
30
+ Requires-Dist: sqlalchemy<2.1,>=2; extra == "monitoring"
31
+ Provides-Extra: visualization
32
+ Requires-Dist: pydot>=1.4.2; extra == "visualization"
33
+ Requires-Dist: networkx<3.3,>=3.2; extra == "visualization"
34
+ Requires-Dist: Flask>=1.0.2; extra == "visualization"
35
+ Requires-Dist: flask_sqlalchemy; extra == "visualization"
36
+ Requires-Dist: pandas<3,>=2.2; extra == "visualization"
37
+ Requires-Dist: plotly; extra == "visualization"
38
+ Requires-Dist: python-daemon; extra == "visualization"
39
+ Provides-Extra: aws
40
+ Requires-Dist: boto3; extra == "aws"
41
+ Provides-Extra: kubernetes
42
+ Requires-Dist: kubernetes; extra == "kubernetes"
43
+ Provides-Extra: docs
44
+ Requires-Dist: ipython<=8.6.0; extra == "docs"
45
+ Requires-Dist: nbsphinx; extra == "docs"
46
+ Requires-Dist: sphinx<8,>=7.4; extra == "docs"
47
+ Requires-Dist: sphinx_rtd_theme; extra == "docs"
48
+ Provides-Extra: google-cloud
49
+ Requires-Dist: google-auth; extra == "google-cloud"
50
+ Requires-Dist: google-api-python-client; extra == "google-cloud"
51
+ Provides-Extra: gssapi
52
+ Requires-Dist: python-gssapi; extra == "gssapi"
53
+ Provides-Extra: azure
54
+ Requires-Dist: azure<=4; extra == "azure"
55
+ Requires-Dist: msrestazure; extra == "azure"
56
+ Provides-Extra: workqueue
57
+ Requires-Dist: work_queue; extra == "workqueue"
58
+ Provides-Extra: flux
59
+ Requires-Dist: pyyaml; extra == "flux"
60
+ Requires-Dist: cffi; extra == "flux"
61
+ Requires-Dist: jsonschema; extra == "flux"
62
+ Provides-Extra: proxystore
63
+ Requires-Dist: proxystore; extra == "proxystore"
64
+ Provides-Extra: radical-pilot
65
+ Requires-Dist: radical.pilot==1.90; extra == "radical-pilot"
66
+ Requires-Dist: radical.utils==1.90; extra == "radical-pilot"
67
+ Provides-Extra: globus-compute
68
+ Requires-Dist: globus_compute_sdk>=2.34.0; extra == "globus-compute"
69
+ Provides-Extra: globus-transfer
70
+ Requires-Dist: globus-sdk; extra == "globus-transfer"
31
71
  Provides-Extra: all
32
72
  Requires-Dist: sqlalchemy<2.1,>=2; extra == "all"
33
73
  Requires-Dist: pydot>=1.4.2; extra == "all"
34
74
  Requires-Dist: networkx<3.3,>=3.2; extra == "all"
35
75
  Requires-Dist: Flask>=1.0.2; extra == "all"
36
- Requires-Dist: flask-sqlalchemy; extra == "all"
76
+ Requires-Dist: flask_sqlalchemy; extra == "all"
37
77
  Requires-Dist: pandas<3,>=2.2; extra == "all"
38
78
  Requires-Dist: plotly; extra == "all"
39
79
  Requires-Dist: python-daemon; extra == "all"
@@ -42,59 +82,33 @@ Requires-Dist: kubernetes; extra == "all"
42
82
  Requires-Dist: ipython<=8.6.0; extra == "all"
43
83
  Requires-Dist: nbsphinx; extra == "all"
44
84
  Requires-Dist: sphinx<8,>=7.4; extra == "all"
45
- Requires-Dist: sphinx-rtd-theme; extra == "all"
85
+ Requires-Dist: sphinx_rtd_theme; extra == "all"
46
86
  Requires-Dist: google-auth; extra == "all"
47
87
  Requires-Dist: google-api-python-client; extra == "all"
48
88
  Requires-Dist: python-gssapi; extra == "all"
49
89
  Requires-Dist: azure<=4; extra == "all"
50
90
  Requires-Dist: msrestazure; extra == "all"
51
- Requires-Dist: work-queue; extra == "all"
91
+ Requires-Dist: work_queue; extra == "all"
52
92
  Requires-Dist: pyyaml; extra == "all"
53
93
  Requires-Dist: cffi; extra == "all"
54
94
  Requires-Dist: jsonschema; extra == "all"
55
95
  Requires-Dist: proxystore; extra == "all"
56
96
  Requires-Dist: radical.pilot==1.90; extra == "all"
57
97
  Requires-Dist: radical.utils==1.90; extra == "all"
58
- Requires-Dist: globus-compute-sdk>=2.34.0; extra == "all"
59
- Provides-Extra: aws
60
- Requires-Dist: boto3; extra == "aws"
61
- Provides-Extra: azure
62
- Requires-Dist: azure<=4; extra == "azure"
63
- Requires-Dist: msrestazure; extra == "azure"
64
- Provides-Extra: docs
65
- Requires-Dist: ipython<=8.6.0; extra == "docs"
66
- Requires-Dist: nbsphinx; extra == "docs"
67
- Requires-Dist: sphinx<8,>=7.4; extra == "docs"
68
- Requires-Dist: sphinx-rtd-theme; extra == "docs"
69
- Provides-Extra: flux
70
- Requires-Dist: pyyaml; extra == "flux"
71
- Requires-Dist: cffi; extra == "flux"
72
- Requires-Dist: jsonschema; extra == "flux"
73
- Provides-Extra: globus_compute
74
- Requires-Dist: globus-compute-sdk>=2.34.0; extra == "globus-compute"
75
- Provides-Extra: google_cloud
76
- Requires-Dist: google-auth; extra == "google-cloud"
77
- Requires-Dist: google-api-python-client; extra == "google-cloud"
78
- Provides-Extra: gssapi
79
- Requires-Dist: python-gssapi; extra == "gssapi"
80
- Provides-Extra: kubernetes
81
- Requires-Dist: kubernetes; extra == "kubernetes"
82
- Provides-Extra: monitoring
83
- Requires-Dist: sqlalchemy<2.1,>=2; extra == "monitoring"
84
- Provides-Extra: proxystore
85
- Requires-Dist: proxystore; extra == "proxystore"
86
- Provides-Extra: radical-pilot
87
- Requires-Dist: radical.pilot==1.90; extra == "radical-pilot"
88
- Requires-Dist: radical.utils==1.90; extra == "radical-pilot"
89
- Provides-Extra: visualization
90
- Requires-Dist: pydot>=1.4.2; extra == "visualization"
91
- Requires-Dist: networkx<3.3,>=3.2; extra == "visualization"
92
- Requires-Dist: Flask>=1.0.2; extra == "visualization"
93
- Requires-Dist: flask-sqlalchemy; extra == "visualization"
94
- Requires-Dist: pandas<3,>=2.2; extra == "visualization"
95
- Requires-Dist: plotly; extra == "visualization"
96
- Requires-Dist: python-daemon; extra == "visualization"
97
- Provides-Extra: workqueue
98
- Requires-Dist: work-queue; extra == "workqueue"
98
+ Requires-Dist: globus_compute_sdk>=2.34.0; extra == "all"
99
+ Requires-Dist: globus-sdk; extra == "all"
100
+ Dynamic: author
101
+ Dynamic: author-email
102
+ Dynamic: classifier
103
+ Dynamic: description
104
+ Dynamic: download-url
105
+ Dynamic: home-page
106
+ Dynamic: keywords
107
+ Dynamic: license
108
+ Dynamic: license-file
109
+ Dynamic: provides-extra
110
+ Dynamic: requires-dist
111
+ Dynamic: requires-python
112
+ Dynamic: summary
99
113
 
100
114
  Simple parallel workflows system for Python