parsl 2024.11.18__py3-none-any.whl → 2024.12.2__py3-none-any.whl

parsl/providers/local/local.py

@@ -32,9 +32,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
         Ratio of provisioned task slots to active tasks. A parallelism value of 1 represents aggressive
         scaling where as many resources as possible are used; parallelism close to 0 represents
         the opposite situation in which as few resources as possible (i.e., min_blocks) are used.
-    move_files : Optional[Bool]
-        Should files be moved? By default, Parsl will try to figure this out itself (= None).
-        If True, then will always move. If False, will never move.
     worker_init : str
         Command to be run before starting a worker, such as 'module load Anaconda; source activate env'.
     """
@@ -48,8 +45,7 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
                  max_blocks=1,
                  worker_init='',
                  cmd_timeout=30,
-                 parallelism=1,
-                 move_files=None):
+                 parallelism=1):
         self.channel = channel
         self._label = 'local'
         self.nodes_per_block = nodes_per_block
@@ -61,7 +57,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
         self.parallelism = parallelism
         self.script_dir = None
         self.cmd_timeout = cmd_timeout
-        self.move_files = move_files
 
         # Dictionary that keeps track of jobs, keyed on job_id
         self.resources = {}
@@ -83,7 +78,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
             if job_dict['status'] and job_dict['status'].terminal:
                 # We already checked this and it can't change after that
                 continue
-            # Script path should point to remote path if _should_move_files() is True
             script_path = job_dict['script_path']
 
             alive = self._is_alive(job_dict)
@@ -137,8 +131,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
 
     def _job_file_path(self, script_path: str, suffix: str) -> str:
         path = '{0}{1}'.format(script_path, suffix)
-        if self._should_move_files():
-            path = self.channel.pull_file(path, self.script_dir)
         return path
 
     def _read_job_file(self, script_path: str, suffix: str) -> str:
@@ -216,9 +208,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
 
         job_id = None
         remote_pid = None
-        if self._should_move_files():
-            logger.debug("Pushing start script")
-            script_path = self.channel.push_file(script_path, self.channel.script_dir)
 
         logger.debug("Launching")
         # We need to capture the exit code and the streams, so we put them in files. We also write
@@ -254,9 +243,6 @@ class LocalProvider(ExecutionProvider, RepresentationMixin):
 
         return job_id
 
-    def _should_move_files(self):
-        return (self.move_files is None and not isinstance(self.channel, LocalChannel)) or (self.move_files)
-
     def cancel(self, job_ids):
         ''' Cancels the jobs specified by a list of job ids
 

parsl/providers/lsf/lsf.py

@@ -68,7 +68,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
         :class:`~parsl.launchers.SingleNodeLauncher` (the default),
         :class:`~parsl.launchers.SrunLauncher`, or
         :class:`~parsl.launchers.AprunLauncher`
-    move_files : Optional[Bool]: should files be moved? by default, Parsl will try to move files.
     bsub_redirection: Bool
         Should a redirection symbol "<" be included when submitting jobs, i.e., Bsub < job_script.
     request_by_nodes: Bool
@@ -92,7 +91,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
                  project=None,
                  queue=None,
                  cmd_timeout=120,
-                 move_files=True,
                  bsub_redirection=False,
                  request_by_nodes=True,
                  launcher=SingleNodeLauncher()):
@@ -112,7 +110,6 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
         self.queue = queue
         self.cores_per_block = cores_per_block
         self.cores_per_node = cores_per_node
-        self.move_files = move_files
         self.bsub_redirection = bsub_redirection
         self.request_by_nodes = request_by_nodes
 
@@ -230,17 +227,10 @@ class LSFProvider(ClusterProvider, RepresentationMixin):
         logger.debug("Writing submit script")
         self._write_submit_script(template_string, script_path, job_name, job_config)
 
-        if self.move_files:
-            logger.debug("moving files")
-            channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
-        else:
-            logger.debug("not moving files")
-            channel_script_path = script_path
-
         if self.bsub_redirection:
-            cmd = "bsub < {0}".format(channel_script_path)
+            cmd = "bsub < {0}".format(script_path)
         else:
-            cmd = "bsub {0}".format(channel_script_path)
+            cmd = "bsub {0}".format(script_path)
         retcode, stdout, stderr = super().execute_wait(cmd)
 
         job_id = None

parsl/providers/pbspro/pbspro.py

@@ -183,15 +183,13 @@ class PBSProProvider(TorqueProvider):
         logger.debug("Writing submit script")
         self._write_submit_script(self.template_string, script_path, job_name, job_config)
 
-        channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
-
         submit_options = ''
         if self.queue is not None:
             submit_options = '{0} -q {1}'.format(submit_options, self.queue)
         if self.account is not None:
             submit_options = '{0} -A {1}'.format(submit_options, self.account)
 
-        launch_cmd = "qsub {0} {1}".format(submit_options, channel_script_path)
+        launch_cmd = "qsub {0} {1}".format(submit_options, script_path)
         retcode, stdout, stderr = self.execute_wait(launch_cmd)
 
         job_id = None

parsl/providers/slurm/slurm.py

@@ -110,7 +110,6 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         :class:`~parsl.launchers.SingleNodeLauncher` (the default),
         :class:`~parsl.launchers.SrunLauncher`, or
         :class:`~parsl.launchers.AprunLauncher`
-    move_files : Optional[Bool]: should files be moved? by default, Parsl will try to move files.
     """
 
     @typeguard.typechecked
@@ -134,7 +133,6 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                  worker_init: str = '',
                  cmd_timeout: int = 10,
                  exclusive: bool = True,
-                 move_files: bool = True,
                  launcher: Launcher = SingleNodeLauncher()):
         label = 'slurm'
         super().__init__(label,
@@ -152,7 +150,6 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         self.cores_per_node = cores_per_node
         self.mem_per_node = mem_per_node
         self.exclusive = exclusive
-        self.move_files = move_files
         self.account = account
         self.qos = qos
         self.constraint = constraint
@@ -308,14 +305,7 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
         logger.debug("Writing submit script")
         self._write_submit_script(template_string, script_path, job_name, job_config)
 
-        if self.move_files:
-            logger.debug("moving files")
-            channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
-        else:
-            logger.debug("not moving files")
-            channel_script_path = script_path
-
-        retcode, stdout, stderr = self.execute_wait("sbatch {0}".format(channel_script_path))
+        retcode, stdout, stderr = self.execute_wait("sbatch {0}".format(script_path))
 
         if retcode == 0:
             for line in stdout.split('\n'):

parsl/providers/torque/torque.py

@@ -189,15 +189,13 @@ class TorqueProvider(ClusterProvider, RepresentationMixin):
         logger.debug("Writing submit script")
         self._write_submit_script(self.template_string, script_path, job_name, job_config)
 
-        channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
-
         submit_options = ''
         if self.queue is not None:
             submit_options = '{0} -q {1}'.format(submit_options, self.queue)
         if self.account is not None:
             submit_options = '{0} -A {1}'.format(submit_options, self.account)
 
-        launch_cmd = "qsub {0} {1}".format(submit_options, channel_script_path)
+        launch_cmd = "qsub {0} {1}".format(submit_options, script_path)
         retcode, stdout, stderr = self.execute_wait(launch_cmd)
 
         job_id = None

parsl/serialize/facade.py

@@ -87,16 +87,16 @@ def pack_res_spec_apply_message(func: Any, args: Any, kwargs: Any, resource_spec
     return pack_apply_message(func, args, (kwargs, resource_specification), buffer_threshold=buffer_threshold)
 
 
-def unpack_apply_message(packed_buffer: bytes, user_ns: Any = None, copy: Any = False) -> List[Any]:
+def unpack_apply_message(packed_buffer: bytes) -> List[Any]:
     """ Unpack and deserialize function and parameters
     """
     return [deserialize(buf) for buf in unpack_buffers(packed_buffer)]
 
 
-def unpack_res_spec_apply_message(packed_buffer: bytes, user_ns: Any = None, copy: Any = False) -> List[Any]:
+def unpack_res_spec_apply_message(packed_buffer: bytes) -> List[Any]:
     """ Unpack and deserialize function, parameters, and resource_specification
     """
-    func, args, (kwargs, resource_spec) = unpack_apply_message(packed_buffer, user_ns=user_ns, copy=copy)
+    func, args, (kwargs, resource_spec) = unpack_apply_message(packed_buffer)
     return [func, args, kwargs, resource_spec]
 
 

parsl/tests/configs/htex_local.py

@@ -10,6 +10,7 @@ def fresh_config():
         executors=[
             HighThroughputExecutor(
                 label="htex_local",
+                loopback_address="::1",
                 worker_debug=True,
                 cores_per_worker=1,
                 encrypted=True,

parsl/tests/configs/htex_local_alternate.py

@@ -62,7 +62,6 @@ def fresh_config():
         retries=2,
         monitoring=MonitoringHub(
                         hub_address="localhost",
-                        hub_port=55055,
                         monitoring_debug=False,
                         resource_monitoring_interval=1,
         ),

parsl/tests/configs/local_threads_monitoring.py

@@ -5,7 +5,6 @@ from parsl.monitoring import MonitoringHub
 config = Config(executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
                 monitoring=MonitoringHub(
                     hub_address="localhost",
-                    hub_port=55055,
                     resource_monitoring_interval=3,
                 )
                 )

parsl/tests/manual_tests/test_udp_simple.py

@@ -15,7 +15,6 @@ def local_setup():
         ],
         monitoring=MonitoringHub(
             hub_address="127.0.0.1",
-            hub_port=55055,
             logging_level=logging.INFO,
             resource_monitoring_interval=10))
 

parsl/tests/test_execute_task.py

@@ -0,0 +1,29 @@
+import os
+
+import pytest
+
+from parsl.executors.execute_task import execute_task
+from parsl.serialize.facade import pack_res_spec_apply_message
+
+
+def addemup(*args: int, name: str = "apples"):
+    total = sum(args)
+    return f"{total} {name}"
+
+
+@pytest.mark.local
+def test_execute_task():
+    args = (1, 2, 3)
+    kwargs = {"name": "boots"}
+    buff = pack_res_spec_apply_message(addemup, args, kwargs, {})
+    res = execute_task(buff)
+    assert res == addemup(*args, **kwargs)
+
+
+@pytest.mark.local
+def test_execute_task_resource_spec():
+    resource_spec = {"num_nodes": 2, "ranks_per_node": 2, "num_ranks": 4}
+    buff = pack_res_spec_apply_message(addemup, (1, 2), {}, resource_spec)
+    execute_task(buff)
+    for key, val in resource_spec.items():
+        assert os.environ[f"PARSL_{key.upper()}"] == str(val)

parsl/tests/test_htex/test_zmq_binding.py

@@ -87,7 +87,7 @@ def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):
 def test_interchange_binding_bad_address(cert_dir: Optional[str]):
     """Confirm that we raise a ZMQError when a bad address is supplied"""
     address = "550.0.0.0"
-    with pytest.raises(zmq.error.ZMQError):
+    with pytest.raises(ValueError):
         make_interchange(interchange_address=address, cert_dir=cert_dir)
 
 
@@ -103,4 +103,5 @@ def test_limited_interface_binding(cert_dir: Optional[str]):
 
     matched_conns = [conn for conn in conns if conn.laddr.port == ix.worker_result_port]
     assert len(matched_conns) == 1
-    assert matched_conns[0].laddr.ip == address
+    # laddr.ip can return ::ffff:127.0.0.1 when using IPv6
+    assert address in matched_conns[0].laddr.ip

parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py

@@ -37,7 +37,6 @@ def fresh_config(run_dir, strategy, db_url):
         strategy_period=0.1,
         monitoring=MonitoringHub(
                         hub_address="localhost",
-                        hub_port=55055,
                         logging_endpoint=db_url
         )
     )

parsl/tests/test_monitoring/test_stdouterr.py

@@ -37,7 +37,6 @@ def fresh_config(run_dir):
         strategy_period=0.1,
         monitoring=MonitoringHub(
                         hub_address="localhost",
-                        hub_port=55055,
         )
     )
 

parsl/tests/unit/test_address.py

@@ -0,0 +1,20 @@
+import pytest
+
+from parsl.addresses import tcp_url
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("address, port,expected", [
+    ("127.0.0.1", 55001, "tcp://127.0.0.1:55001"),
+    ("127.0.0.1", "55001", "tcp://127.0.0.1:55001"),
+    ("127.0.0.1", None, "tcp://127.0.0.1"),
+    ("::1", "55001", "tcp://[::1]:55001"),
+    ("::ffff:127.0.0.1", 55001, "tcp://[::ffff:127.0.0.1]:55001"),
+    ("::ffff:127.0.0.1", None, "tcp://::ffff:127.0.0.1"),
+    ("::ffff:127.0.0.1", None, "tcp://::ffff:127.0.0.1"),
+    ("*", None, "tcp://*"),
+])
+def test_tcp_url(address, port, expected):
+    """Confirm valid address generation"""
+    result = tcp_url(address, port)
+    assert result == expected

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.11.18'
+VERSION = '2024.12.02'

{parsl-2024.11.18.data → parsl-2024.12.2.data}/scripts/exec_parsl_function.py

@@ -94,7 +94,7 @@ def unpack_source_code_function(function_info, user_namespace):
 
 def unpack_byte_code_function(function_info, user_namespace):
     from parsl.serialize import unpack_apply_message
-    func, args, kwargs = unpack_apply_message(function_info["byte code"], user_namespace, copy=False)
+    func, args, kwargs = unpack_apply_message(function_info["byte code"])
     return (func, 'parsl_function_name', args, kwargs)
 
 

{parsl-2024.11.18.data → parsl-2024.12.2.data}/scripts/interchange.py

@@ -14,6 +14,7 @@ from typing import Any, Dict, List, NoReturn, Optional, Sequence, Set, Tuple, ca
 import zmq
 
 from parsl import curvezmq
+from parsl.addresses import tcp_url
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import ManagerLost, VersionMismatch
 from parsl.executors.high_throughput.manager_record import ManagerRecord
@@ -115,13 +116,13 @@ class Interchange:
         self.zmq_context = curvezmq.ServerContext(self.cert_dir)
         self.task_incoming = self.zmq_context.socket(zmq.DEALER)
         self.task_incoming.set_hwm(0)
-        self.task_incoming.connect("tcp://{}:{}".format(client_address, client_ports[0]))
+        self.task_incoming.connect(tcp_url(client_address, client_ports[0]))
         self.results_outgoing = self.zmq_context.socket(zmq.DEALER)
         self.results_outgoing.set_hwm(0)
-        self.results_outgoing.connect("tcp://{}:{}".format(client_address, client_ports[1]))
+        self.results_outgoing.connect(tcp_url(client_address, client_ports[1]))
 
         self.command_channel = self.zmq_context.socket(zmq.REP)
-        self.command_channel.connect("tcp://{}:{}".format(client_address, client_ports[2]))
+        self.command_channel.connect(tcp_url(client_address, client_ports[2]))
         logger.info("Connected to client")
 
         self.run_id = run_id
@@ -144,14 +145,14 @@ class Interchange:
             self.worker_task_port = self.worker_ports[0]
             self.worker_result_port = self.worker_ports[1]
 
-            self.task_outgoing.bind(f"tcp://{self.interchange_address}:{self.worker_task_port}")
-            self.results_incoming.bind(f"tcp://{self.interchange_address}:{self.worker_result_port}")
+            self.task_outgoing.bind(tcp_url(self.interchange_address, self.worker_task_port))
+            self.results_incoming.bind(tcp_url(self.interchange_address, self.worker_result_port))
 
         else:
-            self.worker_task_port = self.task_outgoing.bind_to_random_port(f"tcp://{self.interchange_address}",
+            self.worker_task_port = self.task_outgoing.bind_to_random_port(tcp_url(self.interchange_address),
                                                                            min_port=worker_port_range[0],
                                                                            max_port=worker_port_range[1], max_tries=100)
-            self.worker_result_port = self.results_incoming.bind_to_random_port(f"tcp://{self.interchange_address}",
+            self.worker_result_port = self.results_incoming.bind_to_random_port(tcp_url(self.interchange_address),
                                                                                 min_port=worker_port_range[0],
                                                                                 max_port=worker_port_range[1], max_tries=100)
 

{parsl-2024.11.18.data → parsl-2024.12.2.data}/scripts/process_worker_pool.py

@@ -22,7 +22,9 @@ import psutil
 import zmq
 
 from parsl import curvezmq
+from parsl.addresses import tcp_url
 from parsl.app.errors import RemoteExceptionWrapper
+from parsl.executors.execute_task import execute_task
 from parsl.executors.high_throughput.errors import WorkerLost
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     VALID_LAUNCHERS,
@@ -35,7 +37,7 @@ from parsl.executors.high_throughput.mpi_resource_management import (
 from parsl.executors.high_throughput.probe import probe_addresses
 from parsl.multiprocessing import SpawnContext
 from parsl.process_loggers import wrap_with_logs
-from parsl.serialize import serialize, unpack_res_spec_apply_message
+from parsl.serialize import serialize
 from parsl.version import VERSION as PARSL_VERSION
 
 HEARTBEAT_CODE = (2 ** 32) - 1
@@ -158,8 +160,8 @@ class Manager:
                 raise Exception("No viable address found")
             else:
                 logger.info("Connection to Interchange successful on {}".format(ix_address))
-                task_q_url = "tcp://{}:{}".format(ix_address, task_port)
-                result_q_url = "tcp://{}:{}".format(ix_address, result_port)
+                task_q_url = tcp_url(ix_address, task_port)
+                result_q_url = tcp_url(ix_address, result_port)
                 logger.info("Task url : {}".format(task_q_url))
                 logger.info("Result url : {}".format(result_q_url))
         except Exception:
@@ -590,45 +592,13 @@ def update_resource_spec_env_vars(mpi_launcher: str, resource_spec: Dict, node_i
         os.environ[key] = prefix_table[key]
 
 
-def execute_task(bufs, mpi_launcher: Optional[str] = None):
-    """Deserialize the buffer and execute the task.
-
-    Returns the result or throws exception.
-    """
-    user_ns = locals()
-    user_ns.update({'__builtins__': __builtins__})
-
-    f, args, kwargs, resource_spec = unpack_res_spec_apply_message(bufs, user_ns, copy=False)
-
-    for varname in resource_spec:
-        envname = "PARSL_" + str(varname).upper()
-        os.environ[envname] = str(resource_spec[varname])
-
-    if resource_spec.get("MPI_NODELIST"):
-        worker_id = os.environ['PARSL_WORKER_RANK']
-        nodes_for_task = resource_spec["MPI_NODELIST"].split(',')
-        logger.info(f"Launching task on provisioned nodes: {nodes_for_task}")
-        assert mpi_launcher
-        update_resource_spec_env_vars(mpi_launcher,
-                                      resource_spec=resource_spec,
-                                      node_info=nodes_for_task)
-    # We might need to look into callability of the function from itself
-    # since we change it's name in the new namespace
-    prefix = "parsl_"
-    fname = prefix + "f"
-    argname = prefix + "args"
-    kwargname = prefix + "kwargs"
-    resultname = prefix + "result"
-
-    user_ns.update({fname: f,
-                    argname: args,
-                    kwargname: kwargs,
-                    resultname: resultname})
-
-    code = "{0} = {1}(*{2}, **{3})".format(resultname, fname,
-                                           argname, kwargname)
-    exec(code, user_ns, user_ns)
-    return user_ns.get(resultname)
+def _init_mpi_env(mpi_launcher: str, resource_spec: Dict):
+    node_list = resource_spec.get("MPI_NODELIST")
+    if node_list is None:
+        return
+    nodes_for_task = node_list.split(',')
+    logger.info(f"Launching task on provisioned nodes: {nodes_for_task}")
+    update_resource_spec_env_vars(mpi_launcher=mpi_launcher, resource_spec=resource_spec, node_info=nodes_for_task)
 
 
 @wrap_with_logs(target="worker_log")
@@ -786,8 +756,10 @@ def worker(
             ready_worker_count.value -= 1
         worker_enqueued = False
 
+        _init_mpi_env(mpi_launcher=mpi_launcher, resource_spec=req["resource_spec"])
+
         try:
-            result = execute_task(req['buffer'], mpi_launcher=mpi_launcher)
+            result = execute_task(req['buffer'])
             serialized_result = serialize(result, buffer_threshold=1000000)
         except Exception as e:
             logger.info('Caught an exception: {}'.format(e))

{parsl-2024.11.18.dist-info → parsl-2024.12.2.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.11.18
+Version: 2024.12.2
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.11.18.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.12.02.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0