parsl 2024.6.10__py3-none-any.whl → 2024.6.24__py3-none-any.whl

parsl/app/app.py

@@ -66,8 +66,6 @@ class AppBase(metaclass=ABCMeta):
             self.kwargs['walltime'] = params['walltime'].default
         if 'parsl_resource_specification' in params:
             self.kwargs['parsl_resource_specification'] = params['parsl_resource_specification'].default
-        self.outputs = params['outputs'].default if 'outputs' in params else []
-        self.inputs = params['inputs'].default if 'inputs' in params else []
 
     @abstractmethod
     def __call__(self, *args: Any, **kwargs: Any) -> AppFuture:

parsl/app/bash.py

@@ -1,5 +1,5 @@
 import logging
-from functools import partial, update_wrapper
+from functools import partial
 from inspect import Parameter, signature
 
 from parsl.app.app import AppBase
@@ -123,11 +123,10 @@ class BashApp(AppBase):
             if sig.parameters[s].default is not Parameter.empty:
                 self.kwargs[s] = sig.parameters[s].default
 
-        # update_wrapper allows remote_side_bash_executor to masquerade as self.func
         # partial is used to attach the first arg the "func" to the remote_side_bash_executor
         # this is done to avoid passing a function type in the args which parsl.serializer
         # doesn't support
-        remote_fn = partial(update_wrapper(remote_side_bash_executor, self.func), self.func)
+        remote_fn = partial(remote_side_bash_executor, self.func)
         remote_fn.__name__ = self.func.__name__
         self.wrapped_remote_function = wrap_error(remote_fn)
 

parsl/channels/local/local.py

@@ -55,6 +55,7 @@ class LocalChannel(Channel, RepresentationMixin):
         current_env.update(envs)
 
         try:
+            logger.debug("Creating process with command '%s'", cmd)
             proc = subprocess.Popen(
                 cmd,
                 stdout=subprocess.PIPE,
@@ -64,12 +65,16 @@ class LocalChannel(Channel, RepresentationMixin):
                 shell=True,
                 preexec_fn=os.setpgrp
             )
+            logger.debug("Created process with pid %s. Performing communicate", proc.pid)
             (stdout, stderr) = proc.communicate(timeout=walltime)
             retcode = proc.returncode
+            logger.debug("Process %s returned %s", proc.pid, proc.returncode)
 
-        except Exception as e:
-            logger.warning("Execution of command '{}' failed due to \n{}".format(cmd, e))
+        except Exception:
+            logger.exception(f"Execution of command failed:\n{cmd}")
             raise
+        else:
+            logger.debug("Execution of command in process %s completed normally", proc.pid)
 
         return (retcode, stdout.decode("utf-8"), stderr.decode("utf-8"))
 

parsl/configs/ASPIRE1.py

@@ -4,6 +4,7 @@ from parsl.executors import HighThroughputExecutor
 from parsl.launchers import MpiRunLauncher
 from parsl.monitoring.monitoring import MonitoringHub
 from parsl.providers import PBSProProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
         executors=[
@@ -39,5 +40,6 @@ config = Config(
         strategy='simple',
         retries=3,
         app_cache=True,
-        checkpoint_mode='task_exit'
+        checkpoint_mode='task_exit',
+        usage_tracking=LEVEL_1,
 )

parsl/configs/Azure.py

@@ -8,6 +8,7 @@ from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.rsync import RSyncStaging
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import AzureProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 vm_reference = {
     # All fields below are required
@@ -33,5 +34,6 @@ config = Config(
                             FTPInTaskStaging(),
                             RSyncStaging(getpass.getuser() + "@" + address_by_query())],
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/ad_hoc.py

@@ -4,6 +4,7 @@ from parsl.channels import SSHChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import AdHocProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 user_opts: Dict[str, Dict[str, Any]]
 user_opts = {'adhoc':
@@ -33,4 +34,5 @@ config = Config(
     ],
     #  AdHoc Clusters should not be setup with scaling strategy.
     strategy='none',
+    usage_tracking=LEVEL_1,
 )

parsl/configs/bridges.py

@@ -3,6 +3,7 @@ from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Bridges at PSC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -34,5 +35,6 @@ config = Config(
                 cmd_timeout=120,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/cc_in2p3.py

@@ -2,6 +2,7 @@ from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import GridEngineProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -19,4 +20,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/ec2.py

@@ -1,6 +1,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import AWSProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -25,4 +26,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/expanse.py

@@ -2,6 +2,7 @@ from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -24,5 +25,6 @@ config = Config(
                 nodes_per_block=2,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/frontera.py

@@ -3,6 +3,7 @@ from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -32,4 +33,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/htex_local.py

@@ -2,6 +2,7 @@ from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -15,4 +16,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/illinoiscluster.py

@@ -2,6 +2,7 @@ from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of the Campus Cluster at UIUC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
@@ -25,4 +26,5 @@ config = Config(
                ),
           )
      ],
+     usage_tracking=LEVEL_1,
 )

parsl/configs/kubernetes.py

@@ -2,6 +2,7 @@ from parsl.addresses import address_by_route
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import KubernetesProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -36,5 +37,6 @@ config = Config(
                 max_blocks=10,
             ),
         ),
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/local_threads.py

@@ -1,4 +1,8 @@
 from parsl.config import Config
 from parsl.executors.threads import ThreadPoolExecutor
+from parsl.usage_tracking.levels import LEVEL_1
 
-config = Config(executors=[ThreadPoolExecutor()])
+config = Config(
+    executors=[ThreadPoolExecutor()],
+    usage_tracking=LEVEL_1,
+)

parsl/configs/midway.py

@@ -3,6 +3,7 @@ from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -28,4 +29,5 @@ config = Config(
             ),
         )
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/osg.py

@@ -1,6 +1,7 @@
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import CondorProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -26,5 +27,6 @@ python3 -m venv parsl_env; source parsl_env/bin/activate; python3 -m pip install
             worker_logdir_root='$OSG_WN_TMP',
             worker_ports=(31000, 31001)
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/polaris.py

@@ -3,6 +3,7 @@ from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import MpiExecLauncher
 from parsl.providers import PBSProProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 # There are three user parameters to change for the PBSProProvider:
 #  YOUR_ACCOUNT: Account to charge usage
@@ -34,5 +35,6 @@ config = Config(
                 cpus_per_node=64,
             ),
         ),
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/stampede2.py

@@ -4,6 +4,7 @@ from parsl.data_provider.globus import GlobusStaging
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -34,4 +35,5 @@ config = Config(
         )
 
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/summit.py

@@ -3,6 +3,7 @@ from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import JsrunLauncher
 from parsl.providers import LSFProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -26,4 +27,5 @@ config = Config(
         )
 
     ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/toss3_llnl.py

@@ -2,6 +2,7 @@ from parsl.config import Config
 from parsl.executors import FluxExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -24,5 +25,6 @@ config = Config(
                 cmd_timeout=120,
             ),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/vineex_local.py

@@ -2,6 +2,7 @@ import uuid
 
 from parsl.config import Config
 from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -15,5 +16,6 @@ config = Config(
             # To disable status reporting, comment out the project_name.
             manager_config=TaskVineManagerConfig(project_name="parsl-vine-" + str(uuid.uuid4())),
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/configs/wqex_local.py

@@ -2,6 +2,7 @@ import uuid
 
 from parsl.config import Config
 from parsl.executors import WorkQueueExecutor
+from parsl.usage_tracking.levels import LEVEL_1
 
 config = Config(
     executors=[
@@ -21,5 +22,6 @@ config = Config(
             # A shared filesystem is not needed when using Work Queue.
             shared_fs=False
         )
-    ]
+    ],
+    usage_tracking=LEVEL_1,
 )

parsl/executors/high_throughput/executor.py

@@ -1,13 +1,13 @@
 import logging
 import math
 import pickle
+import subprocess
 import threading
 import typing
 import warnings
 from collections import defaultdict
 from concurrent.futures import Future
 from dataclasses import dataclass
-from multiprocessing import Process
 from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import typeguard
@@ -18,7 +18,7 @@ from parsl.addresses import get_all_addresses
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.staging import Staging
 from parsl.executors.errors import BadMessage, ScalingFailed
-from parsl.executors.high_throughput import interchange, zmq_pipes
+from parsl.executors.high_throughput import zmq_pipes
 from parsl.executors.high_throughput.errors import CommandClientTimeoutError
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     VALID_LAUNCHERS,
@@ -26,7 +26,6 @@ from parsl.executors.high_throughput.mpi_prefix_composer import (
 )
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.jobs.states import TERMINAL_STATES, JobState, JobStatus
-from parsl.multiprocessing import ForkProcess
 from parsl.process_loggers import wrap_with_logs
 from parsl.providers import LocalProvider
 from parsl.providers.base import ExecutionProvider
@@ -305,7 +304,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         self._task_counter = 0
         self.worker_ports = worker_ports
         self.worker_port_range = worker_port_range
-        self.interchange_proc: Optional[Process] = None
+        self.interchange_proc: Optional[subprocess.Popen] = None
         self.interchange_port_range = interchange_port_range
         self.heartbeat_threshold = heartbeat_threshold
         self.heartbeat_period = heartbeat_period
@@ -520,38 +519,45 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
 
         logger.info("Queue management worker finished")
 
-    def _start_local_interchange_process(self):
+    def _start_local_interchange_process(self) -> None:
         """ Starts the interchange process locally
 
-        Starts the interchange process locally and uses an internal command queue to
+        Starts the interchange process locally and uses the command queue to
         get the worker task and result ports that the interchange has bound to.
         """
-        self.interchange_proc = ForkProcess(target=interchange.starter,
-                                            kwargs={"client_address": "127.0.0.1",
-                                                    "client_ports": (self.outgoing_q.port,
-                                                                     self.incoming_q.port,
-                                                                     self.command_client.port),
-                                                    "interchange_address": self.address,
-                                                    "worker_ports": self.worker_ports,
-                                                    "worker_port_range": self.worker_port_range,
-                                                    "hub_address": self.hub_address,
-                                                    "hub_zmq_port": self.hub_zmq_port,
-                                                    "logdir": self.logdir,
-                                                    "heartbeat_threshold": self.heartbeat_threshold,
-                                                    "poll_period": self.poll_period,
-                                                    "logging_level": logging.DEBUG if self.worker_debug else logging.INFO,
-                                                    "cert_dir": self.cert_dir,
-                                                    },
-                                            daemon=True,
-                                            name="HTEX-Interchange"
-                                            )
-        self.interchange_proc.start()
 
+        interchange_config = {"client_address": "127.0.0.1",
+                              "client_ports": (self.outgoing_q.port,
+                                               self.incoming_q.port,
+                                               self.command_client.port),
+                              "interchange_address": self.address,
+                              "worker_ports": self.worker_ports,
+                              "worker_port_range": self.worker_port_range,
+                              "hub_address": self.hub_address,
+                              "hub_zmq_port": self.hub_zmq_port,
+                              "logdir": self.logdir,
+                              "heartbeat_threshold": self.heartbeat_threshold,
+                              "poll_period": self.poll_period,
+                              "logging_level": logging.DEBUG if self.worker_debug else logging.INFO,
+                              "cert_dir": self.cert_dir,
+                              }
+
+        config_pickle = pickle.dumps(interchange_config)
+
+        self.interchange_proc = subprocess.Popen(b"interchange.py", stdin=subprocess.PIPE)
+        stdin = self.interchange_proc.stdin
+        assert stdin is not None, "Popen should have created an IO object (vs default None) because of PIPE mode"
+
+        logger.debug("Popened interchange process. Writing config object")
+        stdin.write(config_pickle)
+        stdin.flush()
+        logger.debug("Sent config object. Requesting worker ports")
         try:
             (self.worker_task_port, self.worker_result_port) = self.command_client.run("WORKER_PORTS", timeout_s=120)
         except CommandClientTimeoutError:
-            logger.error("Interchange has not completed initialization in 120s. Aborting")
+            logger.error("Interchange has not completed initialization. Aborting")
             raise Exception("Interchange failed to start")
+        logger.debug("Got worker ports")
 
     def _start_queue_management_thread(self):
         """Method to start the management thread as a daemon.
@@ -810,13 +816,12 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin, UsageIn
         logger.info("Attempting HighThroughputExecutor shutdown")
 
         self.interchange_proc.terminate()
-        self.interchange_proc.join(timeout=timeout)
-        if self.interchange_proc.is_alive():
+        try:
+            self.interchange_proc.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
             logger.info("Unable to terminate Interchange process; sending SIGKILL")
             self.interchange_proc.kill()
 
-        self.interchange_proc.close()
-
         logger.info("Finished HighThroughputExecutor shutdown attempt")
 
     def get_usage_information(self):

parsl/executors/high_throughput/interchange.py

@@ -672,13 +672,10 @@ def start_file_logger(filename: str, level: int = logging.DEBUG, format_string:
     logger.addHandler(handler)
 
 
-@wrap_with_logs(target="interchange")
-def starter(*args: Any, **kwargs: Any) -> None:
-    """Start the interchange process
-
-    The executor is expected to call this function. The args, kwargs match that of the Interchange.__init__
-    """
+if __name__ == "__main__":
     setproctitle("parsl: HTEX interchange")
-    # logger = multiprocessing.get_logger()
-    ic = Interchange(*args, **kwargs)
+
+    config = pickle.load(sys.stdin.buffer)
+
+    ic = Interchange(**config)
     ic.start()

parsl/executors/workqueue/executor.py

@@ -215,6 +215,13 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             This requires a version of Work Queue / cctools after commit
             874df524516441da531b694afc9d591e8b134b73 (release 7.5.0 is too early).
             Default is False.
+
+        scaling_cores_per_worker: int
+            When using Parsl scaling, this specifies the number of cores that a
+            worker is expected to have available for computation. Default 1. This
+            parameter can be ignored when using a fixed number of blocks, or when
+            using one task per worker (by omitting a ``cores`` resource
+            specifiation for each task).
     """
 
     radio_mode = "filesystem"
@@ -244,12 +251,14 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                  full_debug: bool = True,
                  worker_executable: str = 'work_queue_worker',
                  function_dir: Optional[str] = None,
-                 coprocess: bool = False):
+                 coprocess: bool = False,
+                 scaling_cores_per_worker: int = 1):
         BlockProviderExecutor.__init__(self, provider=provider,
                                        block_error_handler=True)
         if not _work_queue_enabled:
             raise OptionalModuleMissing(['work_queue'], "WorkQueueExecutor requires the work_queue module.")
 
+        self.scaling_cores_per_worker = scaling_cores_per_worker
         self.label = label
         self.task_queue = multiprocessing.Queue()  # type: multiprocessing.Queue
         self.collector_queue = multiprocessing.Queue()  # type: multiprocessing.Queue
@@ -469,6 +478,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Create a Future object and have it be mapped from the task ID in the tasks dictionary
         fu = Future()
         fu.parsl_executor_task_id = executor_task_id
+        assert isinstance(resource_specification, dict)
+        fu.resource_specification = resource_specification
         logger.debug("Getting tasks_lock to set WQ-level task entry")
         with self.tasks_lock:
             logger.debug("Got tasks_lock to set WQ-level task entry")
@@ -654,20 +665,29 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
     @property
     def outstanding(self) -> int:
-        """Count the number of outstanding tasks. This is inefficiently
+        """Count the number of outstanding slots required. This is inefficiently
         implemented and probably could be replaced with a counter.
         """
+        logger.debug("Calculating outstanding task slot load")
         outstanding = 0
+        tasks = 0  # only for log message...
         with self.tasks_lock:
             for fut in self.tasks.values():
                 if not fut.done():
-                    outstanding += 1
-        logger.debug(f"Counted {outstanding} outstanding tasks")
+                    # if a task does not specify a core count, Work Queue will allocate an entire
+                    # worker node to that task. That's approximated here by saying that it uses
+                    # scaling_cores_per_worker.
+                    resource_spec = getattr(fut, 'resource_specification', {})
+                    cores = resource_spec.get('cores', self.scaling_cores_per_worker)
+
+                    outstanding += cores
+                    tasks += 1
+        logger.debug(f"Counted {tasks} outstanding tasks with {outstanding} outstanding slots")
         return outstanding
 
     @property
     def workers_per_node(self) -> Union[int, float]:
-        return 1
+        return self.scaling_cores_per_worker
 
     def scale_in(self, count: int) -> List[str]:
         """Scale in method.

parsl/providers/kubernetes/kube.py

@@ -243,13 +243,13 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         for jid in to_poll_job_ids:
             phase = None
             try:
-                pod_status = self.kube_client.read_namespaced_pod_status(name=jid, namespace=self.namespace)
+                pod = self.kube_client.read_namespaced_pod(name=jid, namespace=self.namespace)
             except Exception:
                 logger.exception("Failed to poll pod {} status, most likely because pod was terminated".format(jid))
                 if self.resources[jid]['status'] is JobStatus(JobState.RUNNING):
                     phase = 'Unknown'
             else:
-                phase = pod_status.status.phase
+                phase = pod.status.phase
             if phase:
                 status = translate_table.get(phase, JobState.UNKNOWN)
                 logger.debug("Updating pod {} with status {} to parsl status {}".format(jid,
@@ -286,7 +286,7 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         # Create the environment variables and command to initiate IPP
         environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
 
-        launch_args = ["-c", "{0};".format(cmd_string)]
+        launch_args = ["-c", "{0}".format(cmd_string)]
 
         volume_mounts = []
         # Create mount paths for the volumes