parsl 2024.5.27__py3-none-any.whl → 2024.6.10__py3-none-any.whl

parsl/providers/condor/condor.py

@@ -2,20 +2,22 @@ import logging
 import os
 import re
 import time
+
 import typeguard
 
 from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
-from parsl.utils import RepresentationMixin
 from parsl.launchers import SingleNodeLauncher
 from parsl.launchers.base import Launcher
-from parsl.providers.condor.template import template_string
 from parsl.providers.cluster_provider import ClusterProvider
+from parsl.providers.condor.template import template_string
 from parsl.providers.errors import ScaleOutFailed
+from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)
 
 from typing import Dict, List, Optional
+
 from parsl.channels.base import Channel
 
 # See http://pages.cs.wisc.edu/~adesmet/status.html

parsl/providers/errors.py

@@ -1,7 +1,7 @@
-from parsl.errors import ParslError
-
 import warnings
 
+from parsl.errors import ParslError
+
 
 class ExecutionProviderException(ParslError):
     """ Base class for all exceptions

parsl/providers/googlecloud/googlecloud.py

@@ -1,8 +1,9 @@
 import atexit
 import logging
 import os
-from parsl.launchers import SingleNodeLauncher
+
 from parsl.jobs.states import JobState, JobStatus
+from parsl.launchers import SingleNodeLauncher
 
 logger = logging.getLogger(__name__)
 

parsl/providers/grid_engine/grid_engine.py

@@ -3,10 +3,10 @@ import os
 import time
 
 from parsl.channels import LocalChannel
+from parsl.jobs.states import JobState, JobStatus
+from parsl.launchers import SingleNodeLauncher
 from parsl.providers.cluster_provider import ClusterProvider
 from parsl.providers.grid_engine.template import template_string
-from parsl.launchers import SingleNodeLauncher
-from parsl.jobs.states import JobState, JobStatus
 from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)

parsl/providers/kubernetes/kube.py

@@ -1,17 +1,19 @@
 import logging
 import time
+
 from parsl.providers.kubernetes.template import template_string
 
 logger = logging.getLogger(__name__)
 
+from typing import Any, Dict, List, Optional, Tuple
+
+import typeguard
+
 from parsl.errors import OptionalModuleMissing
 from parsl.jobs.states import JobState, JobStatus
 from parsl.providers.base import ExecutionProvider
 from parsl.utils import RepresentationMixin
 
-import typeguard
-from typing import Any, Dict, List, Optional, Tuple
-
 try:
     from kubernetes import client, config
     _kubernetes_enabled = True
@@ -81,6 +83,10 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
     persistent_volumes: list[(str, str)]
         List of tuples describing persistent volumes to be mounted in the pod.
         The tuples consist of (PVC Name, Mount Directory).
+    service_account_name: str
+        Name of the service account to run the pod as.
+    annotations: Dict[str, str]
+        Annotations to set on the pod.
     """
     @typeguard.typechecked
     def __init__(self,
@@ -101,7 +107,9 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
                  group_id: Optional[str] = None,
                  run_as_non_root: bool = False,
                  secret: Optional[str] = None,
-                 persistent_volumes: List[Tuple[str, str]] = []) -> None:
+                 persistent_volumes: List[Tuple[str, str]] = [],
+                 service_account_name: Optional[str] = None,
+                 annotations: Optional[Dict[str, str]] = None) -> None:
         if not _kubernetes_enabled:
             raise OptionalModuleMissing(['kubernetes'],
                                         "Kubernetes provider requires kubernetes module and config.")
@@ -144,6 +152,8 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         self.group_id = group_id
         self.run_as_non_root = run_as_non_root
         self.persistent_volumes = persistent_volumes
+        self.service_account_name = service_account_name
+        self.annotations = annotations
 
         self.kube_client = client.CoreV1Api()
 
@@ -182,7 +192,9 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
                          pod_name=pod_name,
                          job_name=job_name,
                          cmd_string=formatted_cmd,
-                         volumes=self.persistent_volumes)
+                         volumes=self.persistent_volumes,
+                         service_account_name=self.service_account_name,
+                         annotations=self.annotations)
         self.resources[pod_name] = {'status': JobStatus(JobState.RUNNING)}
 
         return pod_name
@@ -251,7 +263,9 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
                     job_name,
                     port=80,
                     cmd_string=None,
-                    volumes=[]):
+                    volumes=[],
+                    service_account_name=None,
+                    annotations=None):
         """ Create a kubernetes pod for the job.
         Args:
               - image (string) : Docker image to launch
@@ -309,11 +323,12 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
                                                    claim_name=volume[0])))
 
         metadata = client.V1ObjectMeta(name=pod_name,
-                                       labels={"app": job_name})
+                                       labels={"app": job_name},
+                                       annotations=annotations)
         spec = client.V1PodSpec(containers=[container],
                                 image_pull_secrets=[secret],
-                                volumes=volume_defs
-                                )
+                                volumes=volume_defs,
+                                service_account_name=service_account_name)
 
         pod = client.V1Pod(spec=spec, metadata=metadata)
         api_response = self.kube_client.create_namespaced_pod(namespace=self.namespace,

parsl/providers/local/local.py

@@ -6,7 +6,11 @@ from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers.base import ExecutionProvider
-from parsl.providers.errors import SchedulerMissingArgs, ScriptPathError, SubmitException
+from parsl.providers.errors import (
+    SchedulerMissingArgs,
+    ScriptPathError,
+    SubmitException,
+)
 from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)

parsl/providers/lsf/lsf.py

@@ -1,7 +1,7 @@
-import os
-import time
 import logging
 import math
+import os
+import time
 
 from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus

parsl/providers/pbspro/pbspro.py

@@ -1,7 +1,7 @@
+import json
 import logging
 import os
 import time
-import json
 
 from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus

parsl/providers/slurm/slurm.py

@@ -1,12 +1,12 @@
-import os
-import math
-import time
 import logging
+import math
+import os
 import re
-import typeguard
-
+import time
 from typing import Optional
 
+import typeguard
+
 from parsl.channels import LocalChannel
 from parsl.channels.base import Channel
 from parsl.jobs.states import JobState, JobStatus
@@ -19,25 +19,29 @@ from parsl.utils import RepresentationMixin, wtime_to_minutes
 
 logger = logging.getLogger(__name__)
 
+# From https://slurm.schedmd.com/sacct.html#SECTION_JOB-STATE-CODES
 translate_table = {
-    'PD': JobState.PENDING,
-    'R': JobState.RUNNING,
-    'CA': JobState.CANCELLED,
-    'CF': JobState.PENDING,  # (configuring),
-    'CG': JobState.RUNNING,  # (completing),
-    'CD': JobState.COMPLETED,
-    'F': JobState.FAILED,  # (failed),
-    'TO': JobState.TIMEOUT,  # (timeout),
-    'NF': JobState.FAILED,  # (node failure),
-    'RV': JobState.FAILED,  # (revoked) and
-    'SE': JobState.FAILED   # (special exit state)
+    'PENDING': JobState.PENDING,
+    'RUNNING': JobState.RUNNING,
+    'CANCELLED': JobState.CANCELLED,
+    'COMPLETED': JobState.COMPLETED,
+    'FAILED': JobState.FAILED,
+    'NODE_FAIL': JobState.FAILED,
+    'BOOT_FAIL': JobState.FAILED,
+    'DEADLINE': JobState.TIMEOUT,
+    'TIMEOUT': JobState.TIMEOUT,
+    'REVOKED': JobState.FAILED,
+    'OUT_OF_MEMORY': JobState.FAILED,
+    'SUSPENDED': JobState.HELD,
+    'PREEMPTED': JobState.TIMEOUT,
+    'REQUEUED': JobState.PENDING
 }
 
 
 class SlurmProvider(ClusterProvider, RepresentationMixin):
     """Slurm Execution Provider
 
-    This provider uses sbatch to submit, squeue for status and scancel to cancel
+    This provider uses sbatch to submit, sacct for status and scancel to cancel
     jobs. The sbatch script to be used is created from a template file in this
     same module.
 
@@ -168,14 +172,16 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
             logger.debug('No active jobs, skipping status update')
             return
 
-        cmd = "squeue --noheader --format='%i %t' --job '{0}'".format(job_id_list)
+        # Using state%20 to get enough characters to not truncate output
+        # of the state. Without output can look like "<job_id>     CANCELLED+"
+        cmd = "sacct -X --noheader --format=jobid,state%20 --job '{0}'".format(job_id_list)
         logger.debug("Executing %s", cmd)
         retcode, stdout, stderr = self.execute_wait(cmd)
-        logger.debug("squeue returned %s %s", stdout, stderr)
+        logger.debug("sacct returned %s %s", stdout, stderr)
 
         # Execute_wait failed. Do no update
         if retcode != 0:
-            logger.warning("squeue failed with non-zero exit code {}".format(retcode))
+            logger.warning("sacct failed with non-zero exit code {}".format(retcode))
             return
 
         jobs_missing = set(self.resources.keys())
@@ -183,7 +189,10 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
             if not line:
                 # Blank line
                 continue
-            job_id, slurm_state = line.split()
+            # Sacct includes extra information in some outputs
+            # For example "<job_id> CANCELLED by <user_id>"
+            # This splits and ignores anything past the first two unpacked values
+            job_id, slurm_state, *ignore = line.split()
             if slurm_state not in translate_table:
                 logger.warning(f"Slurm status {slurm_state} is not recognized")
             status = translate_table.get(slurm_state, JobState.UNKNOWN)
@@ -193,13 +202,13 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
                                                          stderr_path=self.resources[job_id]['job_stderr_path'])
             jobs_missing.remove(job_id)
 
-        # squeue does not report on jobs that are not running. So we are filling in the
-        # blanks for missing jobs, we might lose some information about why the jobs failed.
+        # sacct can get job info after jobs have completed so this path shouldn't be hit
+        # log a warning if there are missing jobs for some reason
         for missing_job in jobs_missing:
-            logger.debug("Updating missing job {} to completed status".format(missing_job))
-            self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED,
-                                                              stdout_path=self.resources[missing_job]['job_stdout_path'],
-                                                              stderr_path=self.resources[missing_job]['job_stderr_path'])
+            logger.warning("Updating missing job {} to completed status".format(missing_job))
+            self.resources[missing_job]['status'] = JobStatus(
+                JobState.COMPLETED, stdout_path=self.resources[missing_job]['job_stdout_path'],
+                stderr_path=self.resources[missing_job]['job_stderr_path'])
 
     def submit(self, command: str, tasks_per_node: int, job_name="parsl.slurm") -> str:
         """Submit the command as a slurm job.

parsl/providers/torque/torque.py

@@ -5,8 +5,8 @@ import time
 from parsl.channels import LocalChannel
 from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import AprunLauncher
-from parsl.providers.torque.template import template_string
 from parsl.providers.cluster_provider import ClusterProvider
+from parsl.providers.torque.template import template_string
 from parsl.utils import RepresentationMixin
 
 logger = logging.getLogger(__name__)

parsl/serialize/__init__.py

@@ -1,6 +1,11 @@
-from parsl.serialize.facade import (serialize, deserialize, pack_apply_message,
-                                    unpack_apply_message, unpack_res_spec_apply_message,
-                                    pack_res_spec_apply_message)
+from parsl.serialize.facade import (
+    deserialize,
+    pack_apply_message,
+    pack_res_spec_apply_message,
+    serialize,
+    unpack_apply_message,
+    unpack_res_spec_apply_message,
+)
 
 __all__ = ['serialize',
            'deserialize',

parsl/serialize/base.py

@@ -1,7 +1,6 @@
+import logging
 from abc import abstractmethod
 from functools import cached_property
-import logging
-
 from typing import Any
 
 logger = logging.getLogger(__name__)

parsl/serialize/concretes.py

@@ -1,13 +1,14 @@
-import dill
 import functools
-import pickle
 import logging
+import pickle
 
-logger = logging.getLogger(__name__)
-from parsl.serialize.base import SerializerBase
+import dill
 
+logger = logging.getLogger(__name__)
 from typing import Any
 
+from parsl.serialize.base import SerializerBase
+
 
 class PickleSerializer(SerializerBase):
     """ Pickle serialization covers most python objects, with some notable exceptions:

parsl/serialize/proxystore.py

@@ -1,10 +1,11 @@
-import dill
 import io
 import typing as t
 
-from parsl.serialize.base import SerializerBase
+import dill
 from proxystore.store import Store
 
+from parsl.serialize.base import SerializerBase
+
 
 class ProxyStoreDeepPickler(dill.Pickler):
     """This class extends dill so that certain objects will be stored into

parsl/tests/__init__.py

@@ -1,5 +1,5 @@
-from parsl.dataflow.memoization import id_for_memo
 from parsl.data_provider.files import File
+from parsl.dataflow.memoization import id_for_memo
 
 
 @id_for_memo.register(File)

parsl/tests/configs/ad_hoc_cluster_htex.py

@@ -1,9 +1,9 @@
-from parsl.providers import AdHocProvider
+from typing import Any, Dict
+
 from parsl.channels import SSHChannel
-from parsl.executors import HighThroughputExecutor
 from parsl.config import Config
-
-from typing import Any, Dict
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AdHocProvider
 
 user_opts = {'adhoc':
              {'username': 'YOUR_USERNAME',

parsl/tests/configs/azure_single_node.py

@@ -1,13 +1,12 @@
 """Config for Azure"""
-from parsl.providers import AzureProvider
+import getpass
 
 from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.rsync import RSyncStaging
-
-import getpass
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AzureProvider
 
 # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
 # If you are a user copying-and-pasting this as an example, make sure to either

parsl/tests/configs/bridges.py

@@ -1,7 +1,8 @@
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
+
 from .user_opts import user_opts
 
 

parsl/tests/configs/cc_in2p3.py

@@ -1,7 +1,7 @@
-from parsl.config import Config
 from parsl.channels import LocalChannel
-from parsl.providers import GridEngineProvider
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import GridEngineProvider
 
 from .user_opts import user_opts
 

parsl/tests/configs/comet.py

@@ -1,7 +1,8 @@
 from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
 from parsl.providers import SlurmProvider
-from parsl.executors import HighThroughputExecutor
+
 from .user_opts import user_opts
 
 

parsl/tests/configs/ec2_single_node.py

@@ -11,10 +11,9 @@ Block {Min:0, init:1, Max:1}
 ==================
 
 """
-from parsl.providers import AWSProvider
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import AWSProvider
 
 # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
 # If you are a user copying-and-pasting this as an example, make sure to either

parsl/tests/configs/ec2_spot.py

@@ -1,7 +1,6 @@
-from parsl.providers import AWSProvider
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.providers import AWSProvider
 
 # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
 # If you are a user copying-and-pasting this as an example, make sure to either

parsl/tests/configs/flux_local.py

@@ -0,0 +1,11 @@
+from parsl.config import Config
+from parsl.executors import FluxExecutor
+
+
+def fresh_config():
+    return Config(
+        executors=[FluxExecutor()],
+    )
+
+
+config = fresh_config()

parsl/tests/configs/frontera.py

@@ -1,10 +1,11 @@
-from parsl.config import Config
 from parsl.channels import LocalChannel
-from parsl.providers import SlurmProvider
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 from .user_opts import user_opts
+
 """ This config assumes that it is used to launch parsl tasks from the login nodes
 of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
 """

parsl/tests/configs/htex_ad_hoc_cluster.py

@@ -1,9 +1,7 @@
-from parsl.providers import AdHocProvider
 from parsl.channels import SSHChannel
-from parsl.executors import HighThroughputExecutor
-
 from parsl.config import Config
-
+from parsl.executors import HighThroughputExecutor
+from parsl.providers import AdHocProvider
 from parsl.tests.configs.user_opts import user_opts
 
 config = Config(

parsl/tests/configs/htex_local.py

@@ -1,9 +1,8 @@
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 
 def fresh_config():

parsl/tests/configs/htex_local_alternate.py

@@ -15,23 +15,20 @@ will cause substantially different behaviour on whatever
 those timing parameters control.
 """
 
-# imports for monitoring:
-from parsl.monitoring import MonitoringHub
-
 import os
 
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
-
 from parsl.config import Config
-from parsl.executors import HighThroughputExecutor
-
-
-from parsl.data_provider.http import HTTPInTaskStaging
-from parsl.data_provider.ftp import FTPInTaskStaging
 from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.zip import ZipFileStaging
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SingleNodeLauncher
+
+# imports for monitoring:
+from parsl.monitoring import MonitoringHub
+from parsl.providers import LocalProvider
 
 working_dir = os.getcwd() + "/" + "test_htex_alternate"
 

parsl/tests/configs/htex_local_intask_staging.py

@@ -1,13 +1,11 @@
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
-from parsl.data_provider.http import HTTPInTaskStaging
-from parsl.data_provider.ftp import FTPInTaskStaging
-from parsl.data_provider.file_noop import NoOpFileStaging
-
 from parsl.config import Config
+from parsl.data_provider.file_noop import NoOpFileStaging
+from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 config = Config(
     executors=[

parsl/tests/configs/htex_local_rsync_staging.py

@@ -1,13 +1,11 @@
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
-from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.config import Config
 from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.http import HTTPInTaskStaging
 from parsl.data_provider.rsync import RSyncStaging
-
-from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 config = Config(
     executors=[

parsl/tests/configs/local_adhoc.py

@@ -1,5 +1,5 @@
-from parsl.config import Config
 from parsl.channels import LocalChannel
+from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.providers import AdHocProvider
 

parsl/tests/configs/local_radical.py

@@ -1,9 +1,7 @@
 import os
 
 from parsl.config import Config
-from parsl.executors.radical import RadicalPilotExecutor
-from parsl.executors.radical import ResourceConfig
-
+from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
 
 rpex_cfg = ResourceConfig()
 

parsl/tests/configs/local_radical_mpi.py

@@ -1,10 +1,10 @@
 import os
+
 from parsl.config import Config
 
 
 def fresh_config():
-    from parsl.executors.radical import ResourceConfig
-    from parsl.executors.radical import RadicalPilotExecutor
+    from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
 
     rpex_cfg = ResourceConfig()
     rpex_cfg.worker_type = "MPI"

parsl/tests/configs/midway.py

@@ -1,7 +1,7 @@
 from parsl.config import Config
-from parsl.providers import SlurmProvider
-from parsl.launchers import SrunLauncher
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SrunLauncher
+from parsl.providers import SlurmProvider
 
 from .user_opts import user_opts
 

parsl/tests/configs/nscc_singapore.py

@@ -1,8 +1,8 @@
-from parsl.providers import PBSProProvider
-from parsl.executors import HighThroughputExecutor
-from parsl.launchers import MpiRunLauncher
 from parsl.addresses import address_by_interface
 from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import MpiRunLauncher
+from parsl.providers import PBSProProvider
 
 from .user_opts import user_opts