parsl 2023.5.29__py3-none-any.whl → 2023.6.12__py3-none-any.whl

parsl/executors/workqueue/executor.py

@@ -122,8 +122,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             In this case, environment variables can be used to influence the
             choice of port, documented here:
             https://ccl.cse.nd.edu/software/manuals/api/html/work__queue_8h.html#a21714a10bcdfcf5c3bd44a96f5dcbda6
-
-            Default: 0.
+            Default: WORK_QUEUE_DEFAULT_PORT.
 
         env: dict{str}
             Dictionary that contains the environmental variables that
@@ -179,7 +178,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             invocations of an app have similar performance characteristics,
             this will provide a reasonable set of categories automatically.
 
-        max_retries: Optional[int]
+        max_retries: int
             Set the number of retries that Work Queue will make when a task
             fails. This is distinct from Parsl level retries configured in
             parsl.config.Config. Set to None to allow Work Queue to retry
@@ -234,7 +233,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                  autolabel: bool = False,
                  autolabel_window: int = 1,
                  autocategory: bool = True,
-                 max_retries: Optional[int] = 1,
+                 max_retries: int = 1,
                  init_command: str = "",
                  worker_options: str = "",
                  full_debug: bool = True,
@@ -261,7 +260,7 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         self.storage_access = storage_access
         self.use_cache = use_cache
         self.working_dir = working_dir
-        self.registered_files = set()  # type: Set[str]
+        self.registered_files: Set[str] = set()
         self.full_debug = full_debug
         self.source = True if pack else source
         self.pack = pack
@@ -473,7 +472,6 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
 
         logger.debug("Creating executor task {} for function {} with args {}".format(executor_task_id, func, args))
 
-        # Pickle the result into object to pass into message buffer
         function_file = self._path_in_task(executor_task_id, "function")
         result_file = self._path_in_task(executor_task_id, "result")
         map_file = self._path_in_task(executor_task_id, "map")
@@ -860,7 +858,11 @@ def _work_queue_submit_wait(*,
                     logger.debug("Sending executor task {} with command: {}".format(task.id, command_str))
                     t = wq.Task(command_str)
                 else:
-                    t = wq.RemoteTask("run_parsl_task", "parsl_coprocess", task.map_file, task.function_file, task.result_file)
+                    t = wq.RemoteTask("run_parsl_task",
+                                      "parsl_coprocess",
+                                      os.path.basename(task.map_file),
+                                      os.path.basename(task.function_file),
+                                      os.path.basename(task.result_file))
                     t.specify_exec_method("direct")
                     logger.debug("Sending executor task {} to coprocess".format(task.id))
 

parsl/launchers/base.py

@@ -0,0 +1,17 @@
+from abc import ABCMeta, abstractmethod
+
+from parsl.utils import RepresentationMixin
+
+
+class Launcher(RepresentationMixin, metaclass=ABCMeta):
+    """Launchers are basically wrappers for user submitted scripts as they
+    are submitted to a specific execution resource.
+    """
+    def __init__(self, debug: bool = True):
+        self.debug = debug
+
+    @abstractmethod
+    def __call__(self, command: str, tasks_per_node: int, nodes_per_block: int) -> str:
+        """ Wraps the command with the Launcher calls.
+        """
+        pass

parsl/launchers/launchers.py

@@ -1,25 +1,10 @@
-from abc import ABCMeta, abstractmethod
 import logging
 
-from parsl.utils import RepresentationMixin
+from parsl.launchers.base import Launcher
 
 logger = logging.getLogger(__name__)
 
 
-class Launcher(RepresentationMixin, metaclass=ABCMeta):
-    """Launchers are basically wrappers for user submitted scripts as they
-    are submitted to a specific execution resource.
-    """
-    def __init__(self, debug: bool = True):
-        self.debug = debug
-
-    @abstractmethod
-    def __call__(self, command: str, tasks_per_node: int, nodes_per_block: int) -> str:
-        """ Wraps the command with the Launcher calls.
-        """
-        pass
-
-
 class SimpleLauncher(Launcher):
     """ Does no wrapping. Just returns the command as-is
     """

parsl/monitoring/monitoring.py

@@ -139,7 +139,7 @@ class MonitoringHub(RepresentationMixin):
         self._dfk_channel = None  # type: Any
 
         if _db_manager_excepts:
-            raise(_db_manager_excepts)
+            raise _db_manager_excepts
 
         self.client_address = client_address
         self.client_port_range = client_port_range
@@ -170,12 +170,23 @@ class MonitoringHub(RepresentationMixin):
         self.logger.debug("Initializing ZMQ Pipes to client")
         self.monitoring_hub_active = True
 
-        comm_q = SizedQueue(maxsize=10)  # type: Queue[Union[Tuple[int, int], str]]
-        self.exception_q = SizedQueue(maxsize=10)  # type: Queue[Tuple[str, str]]
-        self.priority_msgs = SizedQueue()  # type: Queue[Tuple[Any, int]]
-        self.resource_msgs = SizedQueue()  # type: Queue[AddressedMonitoringMessage]
-        self.node_msgs = SizedQueue()  # type: Queue[AddressedMonitoringMessage]
-        self.block_msgs = SizedQueue()  # type:  Queue[AddressedMonitoringMessage]
+        comm_q: Queue[Union[Tuple[int, int], str]]
+        comm_q = SizedQueue(maxsize=10)
+
+        self.exception_q: Queue[Tuple[str, str]]
+        self.exception_q = SizedQueue(maxsize=10)
+
+        self.priority_msgs: Queue[Tuple[Any, int]]
+        self.priority_msgs = SizedQueue()
+
+        self.resource_msgs: Queue[AddressedMonitoringMessage]
+        self.resource_msgs = SizedQueue()
+
+        self.node_msgs: Queue[AddressedMonitoringMessage]
+        self.node_msgs = SizedQueue()
+
+        self.block_msgs: Queue[AddressedMonitoringMessage]
+        self.block_msgs = SizedQueue()
 
         self.router_proc = ForkProcess(target=router_starter,
                                        args=(comm_q, self.exception_q, self.priority_msgs, self.node_msgs, self.block_msgs, self.resource_msgs),
@@ -328,7 +339,7 @@ def filesystem_receiver(logdir: str, q: "queue.Queue[AddressedMonitoringMessage]
                 with open(full_path_filename, "rb") as f:
                     message = deserialize(f.read())
                 logger.info(f"Message received is: {message}")
-                assert(isinstance(message, tuple))
+                assert isinstance(message, tuple)
                 q.put(cast(AddressedMonitoringMessage, message))
                 os.remove(full_path_filename)
             except Exception:

parsl/monitoring/visualization/plots/default/workflow_plots.py

@@ -9,6 +9,22 @@ import networkx as nx
 from parsl.monitoring.visualization.utils import timestamp_to_int, num_to_timestamp, DB_DATE_FORMAT
 
 
+# gantt_colors must assign a color value for every state name defined
+# in parsl/dataflow/states.py
+gantt_colors = {'unsched': 'rgb(240, 240, 240)',
+                'pending': 'rgb(168, 168, 168)',
+                'launched': 'rgb(100, 255, 255)',
+                'running': 'rgb(0, 0, 255)',
+                'running_ended': 'rgb(64, 64, 255)',
+                'joining': 'rgb(128, 128, 255)',
+                'dep_fail': 'rgb(255, 128, 255)',
+                'failed': 'rgb(200, 0, 0)',
+                'exec_done': 'rgb(0, 200, 0)',
+                'memo_done': 'rgb(64, 200, 64)',
+                'fail_retryable': 'rgb(200, 128,128)'
+               }
+
+
 def task_gantt_plot(df_task, df_status, time_completed=None):
 
     # if the workflow is not recorded as completed, then assume
@@ -47,25 +63,9 @@ def task_gantt_plot(df_task, df_status, time_completed=None):
                               }
             parsl_tasks.extend([last_status_bar])
 
-    # colours must assign a colour value for every state name defined
-    # in parsl/dataflow/states.py
-
-    colors = {'unsched': 'rgb(240, 240, 240)',
-              'pending': 'rgb(168, 168, 168)',
-              'launched': 'rgb(100, 255, 255)',
-              'running': 'rgb(0, 0, 255)',
-              'running_ended': 'rgb(64, 64, 255)',
-              'joining': 'rgb(128, 128, 255)',
-              'dep_fail': 'rgb(255, 128, 255)',
-              'failed': 'rgb(200, 0, 0)',
-              'exec_done': 'rgb(0, 200, 0)',
-              'memo_done': 'rgb(64, 200, 64)',
-              'fail_retryable': 'rgb(200, 128,128)'
-             }
-
     fig = ff.create_gantt(parsl_tasks,
                           title="",
-                          colors=colors,
+                          colors=gantt_colors,
                           group_tasks=True,
                           show_colorbar=True,
                           index_col='Resource',
@@ -194,6 +194,20 @@ def total_tasks_plot(df_task, df_status, columns=20):
     return plot(fig, show_link=False, output_type="div", include_plotlyjs=False)
 
 
+dag_state_colors = {"unsched": (0, 'rgb(240, 240, 240)'),
+                    "pending": (1, 'rgb(168, 168, 168)'),
+                    "launched": (2, 'rgb(100, 255, 255)'),
+                    "running": (3, 'rgb(0, 0, 255)'),
+                    "dep_fail": (4, 'rgb(255, 128, 255)'),
+                    "failed": (5, 'rgb(200, 0, 0)'),
+                    "exec_done": (6, 'rgb(0, 200, 0)'),
+                    "memo_done": (7, 'rgb(64, 200, 64)'),
+                    "fail_retryable": (8, 'rgb(200, 128,128)'),
+                    "joining": (9, 'rgb(128, 128, 255)'),
+                    "running_ended": (10, 'rgb(64, 64, 255)')
+                   }
+
+
 def workflow_dag_plot(df_tasks, group_by_apps=True):
     G = nx.DiGraph(directed=True)
     nodes = df_tasks['task_id'].unique()
@@ -215,18 +229,7 @@ def workflow_dag_plot(df_tasks, group_by_apps=True):
         groups_list = {app: (i, None) for i, app in enumerate(
             df_tasks['task_func_name'].unique())}
     else:
-        groups_list = {"unsched": (0, 'rgb(240, 240, 240)'),
-                       "pending": (1, 'rgb(168, 168, 168)'),
-                       "launched": (2, 'rgb(100, 255, 255)'),
-                       "running": (3, 'rgb(0, 0, 255)'),
-                       "dep_fail": (4, 'rgb(255, 128, 255)'),
-                       "failed": (5, 'rgb(200, 0, 0)'),
-                       "exec_done": (6, 'rgb(0, 200, 0)'),
-                       "memo_done": (7, 'rgb(64, 200, 64)'),
-                       "fail_retryable": (8, 'rgb(200, 128,128)'),
-                       "joining": (9, 'rgb(128, 128, 255)'),
-                       "running_ended": (10, 'rgb(64, 64, 255)')
-                      }
+        groups_list = dag_state_colors
 
     node_traces = [...] * len(groups_list)
 

parsl/providers/cluster_provider.py

@@ -67,9 +67,9 @@ class ClusterProvider(ExecutionProvider):
         self.walltime = walltime
         self.cmd_timeout = cmd_timeout
         if not callable(self.launcher):
-            raise(BadLauncher(self.launcher,
+            raise BadLauncher(self.launcher,
                               "Launcher for executor: {} is of type: {}. Expects a parsl.launcher.launcher.Launcher or callable".format(
-                                  label, type(self.launcher))))
+                                  label, type(self.launcher)))
 
         self.script_dir = None
 

parsl/providers/condor/condor.py

@@ -8,6 +8,7 @@ from parsl.channels import LocalChannel
 from parsl.providers.base import JobState, JobStatus
 from parsl.utils import RepresentationMixin
 from parsl.launchers import SingleNodeLauncher
+from parsl.launchers.base import Launcher
 from parsl.providers.condor.template import template_string
 from parsl.providers.cluster_provider import ClusterProvider
 from parsl.providers.errors import ScaleOutFailed
@@ -16,7 +17,6 @@ logger = logging.getLogger(__name__)
 
 from typing import Dict, List, Optional
 from parsl.channels.base import Channel
-from parsl.launchers.launchers import Launcher
 
 # See http://pages.cs.wisc.edu/~adesmet/status.html
 translate_table = {

parsl/providers/kubernetes/kube.py

@@ -128,7 +128,8 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         self.kube_client = client.CoreV1Api()
 
         # Dictionary that keeps track of jobs, keyed on job_id
-        self.resources = {}  # type: Dict[object, Dict[str, Any]]
+        self.resources: Dict[object, Dict[str, Any]]
+        self.resources = {}
 
     def submit(self, cmd_string, tasks_per_node, job_name="parsl"):
         """ Submit a job

parsl/providers/slurm/slurm.py

@@ -10,7 +10,7 @@ from typing import Optional
 from parsl.channels import LocalChannel
 from parsl.channels.base import Channel
 from parsl.launchers import SingleNodeLauncher
-from parsl.launchers.launchers import Launcher
+from parsl.launchers.base import Launcher
 from parsl.providers.cluster_provider import ClusterProvider
 from parsl.providers.base import JobState, JobStatus
 from parsl.providers.slurm.template import template_string

parsl/tests/configs/taskvine_ex.py

@@ -0,0 +1,11 @@
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor
+
+from parsl.data_provider.http import HTTPInTaskStaging
+from parsl.data_provider.ftp import FTPInTaskStaging
+from parsl.data_provider.file_noop import NoOpFileStaging
+
+
+def fresh_config():
+    return Config(executors=[TaskVineExecutor(port=9000,
+                                              storage_access=[FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging()])])

parsl/tests/conftest.py

@@ -124,7 +124,7 @@ def load_dfk_session(request, pytestconfig):
 
         yield
 
-        if(parsl.dfk() != dfk):
+        if parsl.dfk() != dfk:
             raise RuntimeError("DFK changed unexpectedly during test")
         dfk.cleanup()
         parsl.clear()
@@ -158,16 +158,16 @@ def load_dfk_local_module(request, pytestconfig):
             assert isinstance(c, parsl.Config)
             dfk = parsl.load(c)
 
-        if(callable(local_setup)):
+        if callable(local_setup):
             local_setup()
 
         yield
 
-        if(callable(local_teardown)):
+        if callable(local_teardown):
             local_teardown()
 
-        if(local_config):
-            if(parsl.dfk() != dfk):
+        if local_config:
+            if parsl.dfk() != dfk:
                 raise RuntimeError("DFK changed unexpectedly during test")
             dfk.cleanup()
             parsl.clear()
@@ -271,7 +271,7 @@ def pytest_ignore_collect(path):
         return True
     elif 'manual_tests' in path.strpath:
         return True
-    elif 'workqueue_tests/test_scale' in path.strpath:
+    elif 'scaling_tests/test_scale' in path.strpath:
         return True
     else:
         return False

parsl/tests/scaling_tests/vineex_condor.py

@@ -0,0 +1,10 @@
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor
+from parsl.providers import CondorProvider
+
+config = Config(
+    executors=[TaskVineExecutor(port=50055,
+                                source=True,
+                                provider=CondorProvider(),
+        )]
+)

parsl/tests/scaling_tests/vineex_local.py

@@ -0,0 +1,10 @@
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor
+from parsl.providers import LocalProvider
+
+config = Config(
+    executors=[TaskVineExecutor(port=50055,
+                                source=True,
+                                provider=LocalProvider(),
+        )]
+)

parsl/tests/test_bash_apps/test_pipeline.py

@@ -52,7 +52,7 @@ def test_increment(depth=5):
     futs = {}
     for i in range(1, depth):
         print("Launching {0} with {1}".format(i, prev))
-        assert(isinstance(prev, DataFuture) or isinstance(prev, File))
+        assert isinstance(prev, DataFuture) or isinstance(prev, File)
         output = File("test{0}.txt".format(i))
         fu = increment(inputs=[prev],  # Depend on the future from previous call
                        # Name the file to be created here
@@ -62,7 +62,7 @@ def test_increment(depth=5):
         [prev] = fu.outputs
         futs[i] = prev
         print(prev.filepath)
-        assert(isinstance(prev, DataFuture))
+        assert isinstance(prev, DataFuture)
 
     for key in futs:
         if key > 0:

parsl/tests/test_error_handling/test_retry_handler.py

@@ -61,4 +61,4 @@ def test_retry():
     with pytest.raises(parsl.app.errors.BashExitFailure):
         fu.result()
 
-    assert(fu.exception().exitcode == 5)
+    assert fu.exception().exitcode == 5

parsl/tests/test_monitoring/test_viz_colouring.py

@@ -0,0 +1,17 @@
+import pytest
+from parsl.dataflow.states import States
+
+
+@pytest.mark.local
+def test_all_states_colored() -> None:
+    """This checks that the coloring tables in parsl-visualize contain
+    a color for each state defined in the task state enumeration.
+    """
+
+    # imports inside test because viz can't be imported in an environment
+    # with no monitoring installed
+    import parsl.monitoring.visualization.plots.default.workflow_plots as workflow_plots
+
+    for s in States:
+        assert s.name in workflow_plots.gantt_colors
+        assert s.name in workflow_plots.dag_state_colors

parsl/utils.py

@@ -55,7 +55,7 @@ def get_all_checkpoints(rundir: str = "runinfo") -> List[str]:
 
     """
 
-    if(not os.path.isdir(rundir)):
+    if not os.path.isdir(rundir):
         return []
 
     dirs = sorted(os.listdir(rundir))
@@ -99,7 +99,7 @@ def get_last_checkpoint(rundir: str = "runinfo") -> List[str]:
     last_runid = dirs[-1]
     last_checkpoint = os.path.abspath(f'{rundir}/{last_runid}/checkpoint')
 
-    if(not(os.path.isdir(last_checkpoint))):
+    if not os.path.isdir(last_checkpoint):
         return []
 
     return [last_checkpoint]

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2023.05.29'
+VERSION = '2023.06.12'

{parsl-2023.5.29.dist-info → parsl-2023.6.12.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.5.29
+Version: 2023.6.12
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.05.29.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.06.12.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -19,7 +19,7 @@ Requires-Python: >=3.7.0
 License-File: LICENSE
 Requires-Dist: pyzmq (>=17.1.2)
 Requires-Dist: typeguard (<3,>=2.10)
-Requires-Dist: typing-extensions
+Requires-Dist: typing-extensions (<5,>=4.6)
 Requires-Dist: types-paramiko
 Requires-Dist: types-requests
 Requires-Dist: types-six