parsl 2024.10.28__py3-none-any.whl → 2024.11.4__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -162,8 +162,8 @@ class DataFlowKernel:
         }
 
         if self.monitoring:
-            self.monitoring.send(MessageType.WORKFLOW_INFO,
-                                 workflow_info)
+            self.monitoring.send((MessageType.WORKFLOW_INFO,
+                                 workflow_info))
 
         if config.checkpoint_files is not None:
             checkpoints = self.load_checkpoints(config.checkpoint_files)
@@ -238,7 +238,7 @@ class DataFlowKernel:
     def _send_task_log_info(self, task_record: TaskRecord) -> None:
         if self.monitoring:
             task_log_info = self._create_task_log_info(task_record)
-            self.monitoring.send(MessageType.TASK_INFO, task_log_info)
+            self.monitoring.send((MessageType.TASK_INFO, task_log_info))
 
     def _create_task_log_info(self, task_record: TaskRecord) -> Dict[str, Any]:
         """
@@ -1295,12 +1295,12 @@ class DataFlowKernel:
 
         if self.monitoring:
             logger.info("Sending final monitoring message")
-            self.monitoring.send(MessageType.WORKFLOW_INFO,
+            self.monitoring.send((MessageType.WORKFLOW_INFO,
                                  {'tasks_failed_count': self.task_state_counts[States.failed],
                                   'tasks_completed_count': self.task_state_counts[States.exec_done],
                                   "time_began": self.time_began,
                                   'time_completed': self.time_completed,
-                                  'run_id': self.run_id, 'rundir': self.run_dir})
+                                  'run_id': self.run_id, 'rundir': self.run_dir}))
 
             logger.info("Terminating monitoring")
             self.monitoring.close()

parsl/executors/high_throughput/executor.py

@@ -63,7 +63,6 @@ DEFAULT_INTERCHANGE_LAUNCH_CMD = ["interchange.py"]
 
 GENERAL_HTEX_PARAM_DOCS = """provider : :class:`~parsl.providers.base.ExecutionProvider`
        Provider to access computation resources. Can be one of :class:`~parsl.providers.aws.aws.EC2Provider`,
-        :class:`~parsl.providers.cobalt.cobalt.Cobalt`,
         :class:`~parsl.providers.condor.condor.Condor`,
         :class:`~parsl.providers.googlecloud.googlecloud.GoogleCloud`,
         :class:`~parsl.providers.gridEngine.gridEngine.GridEngine`,

parsl/executors/high_throughput/mpi_resource_management.py

@@ -17,7 +17,6 @@ class Scheduler(Enum):
     Unknown = 0
     Slurm = 1
     PBS = 2
-    Cobalt = 3
 
 
 def get_slurm_hosts_list() -> List[str]:
@@ -37,13 +36,6 @@ def get_pbs_hosts_list() -> List[str]:
         return [line.strip() for line in f.readlines()]
 
 
-def get_cobalt_hosts_list() -> List[str]:
-    """Get list of COBALT hosts from envvar: COBALT_NODEFILE"""
-    nodefile_name = os.environ["COBALT_NODEFILE"]
-    with open(nodefile_name) as f:
-        return [line.strip() for line in f.readlines()]
-
-
 def get_nodes_in_batchjob(scheduler: Scheduler) -> List[str]:
     """Get nodelist from all supported schedulers"""
     nodelist = []
@@ -51,8 +43,6 @@ def get_nodes_in_batchjob(scheduler: Scheduler) -> List[str]:
         nodelist = get_slurm_hosts_list()
     elif scheduler == Scheduler.PBS:
         nodelist = get_pbs_hosts_list()
-    elif scheduler == Scheduler.Cobalt:
-        nodelist = get_cobalt_hosts_list()
     else:
         raise RuntimeError(f"mpi_mode does not support scheduler:{scheduler}")
     return nodelist
@@ -64,8 +54,6 @@ def identify_scheduler() -> Scheduler:
         return Scheduler.Slurm
     elif os.environ.get("PBS_NODEFILE"):
         return Scheduler.PBS
-    elif os.environ.get("COBALT_NODEFILE"):
-        return Scheduler.Cobalt
     else:
         return Scheduler.Unknown
 

parsl/monitoring/monitoring.py

@@ -7,13 +7,12 @@ import queue
 import time
 from multiprocessing import Event, Process
 from multiprocessing.queues import Queue
-from typing import TYPE_CHECKING, Any, Literal, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Literal, Optional, Tuple, Union, cast
 
 import typeguard
 
 from parsl.log_utils import set_file_logger
 from parsl.monitoring.errors import MonitoringHubStartError
-from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.radios import MultiprocessingQueueRadioSender
 from parsl.monitoring.router import router_starter
 from parsl.monitoring.types import TaggedMonitoringMessage
@@ -202,10 +201,9 @@ class MonitoringHub(RepresentationMixin):
 
         self.hub_zmq_port = zmq_port
 
-    # TODO: tighten the Any message format
-    def send(self, mtype: MessageType, message: Any) -> None:
-        logger.debug("Sending message type {}".format(mtype))
-        self.radio.send((mtype, message))
+    def send(self, message: TaggedMonitoringMessage) -> None:
+        logger.debug("Sending message type {}".format(message[0]))
+        self.radio.send(message)
 
     def close(self) -> None:
         logger.info("Terminating Monitoring Hub")

parsl/monitoring/radios.py

@@ -5,15 +5,11 @@ import socket
 import uuid
 from abc import ABCMeta, abstractmethod
 from multiprocessing.queues import Queue
-from typing import Optional
 
 import zmq
 
 from parsl.serialize import serialize
 
-_db_manager_excepts: Optional[Exception]
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -41,9 +37,8 @@ class FilesystemRadioSender(MonitoringRadioSender):
     the UDP radio, but should be much more reliable.
     """
 
-    def __init__(self, *, monitoring_url: str, source_id: int, timeout: int = 10, run_dir: str):
+    def __init__(self, *, monitoring_url: str, timeout: int = 10, run_dir: str):
         logger.info("filesystem based monitoring channel initializing")
-        self.source_id = source_id
         self.base_path = f"{run_dir}/monitor-fs-radio/"
         self.tmp_path = f"{self.base_path}/tmp"
         self.new_path = f"{self.base_path}/new"
@@ -70,19 +65,16 @@ class FilesystemRadioSender(MonitoringRadioSender):
 
 class HTEXRadioSender(MonitoringRadioSender):
 
-    def __init__(self, monitoring_url: str, source_id: int, timeout: int = 10):
+    def __init__(self, monitoring_url: str, timeout: int = 10):
         """
         Parameters
         ----------
 
         monitoring_url : str
             URL of the form <scheme>://<IP>:<PORT>
-        source_id : str
-            String identifier of the source
         timeout : int
             timeout, default=10s
         """
-        self.source_id = source_id
         logger.info("htex-based monitoring channel initialising")
 
     def send(self, message: object) -> None:
@@ -124,21 +116,18 @@ class HTEXRadioSender(MonitoringRadioSender):
 
 class UDPRadioSender(MonitoringRadioSender):
 
-    def __init__(self, monitoring_url: str, source_id: int, timeout: int = 10):
+    def __init__(self, monitoring_url: str, timeout: int = 10):
         """
         Parameters
         ----------
 
         monitoring_url : str
             URL of the form <scheme>://<IP>:<PORT>
-        source_id : str
-            String identifier of the source
         timeout : int
             timeout, default=10s
         """
         self.monitoring_url = monitoring_url
         self.sock_timeout = timeout
-        self.source_id = source_id
         try:
             self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
             self.port = int(port)

parsl/monitoring/remote.py

@@ -103,14 +103,12 @@ def monitor_wrapper(*,
 def get_radio(radio_mode: str, monitoring_hub_url: str, task_id: int, run_dir: str) -> MonitoringRadioSender:
     radio: MonitoringRadioSender
     if radio_mode == "udp":
-        radio = UDPRadioSender(monitoring_hub_url,
-                               source_id=task_id)
+        radio = UDPRadioSender(monitoring_hub_url)
     elif radio_mode == "htex":
-        radio = HTEXRadioSender(monitoring_hub_url,
-                                source_id=task_id)
+        radio = HTEXRadioSender(monitoring_hub_url)
     elif radio_mode == "filesystem":
         radio = FilesystemRadioSender(monitoring_url=monitoring_hub_url,
-                                      source_id=task_id, run_dir=run_dir)
+                                      run_dir=run_dir)
     else:
         raise RuntimeError(f"Unknown radio mode: {radio_mode}")
     return radio

parsl/providers/__init__.py

@@ -1,7 +1,6 @@
 # Cloud Providers
 from parsl.providers.aws.aws import AWSProvider
 from parsl.providers.azure.azure import AzureProvider
-from parsl.providers.cobalt.cobalt import CobaltProvider
 from parsl.providers.condor.condor import CondorProvider
 from parsl.providers.googlecloud.googlecloud import GoogleCloudProvider
 from parsl.providers.grid_engine.grid_engine import GridEngineProvider
@@ -15,7 +14,6 @@ from parsl.providers.slurm.slurm import SlurmProvider
 from parsl.providers.torque.torque import TorqueProvider
 
 __all__ = ['LocalProvider',
-           'CobaltProvider',
            'CondorProvider',
            'GridEngineProvider',
            'SlurmProvider',

parsl/providers/base.py

@@ -11,7 +11,7 @@ class ExecutionProvider(metaclass=ABCMeta):
     """Execution providers are responsible for managing execution resources
     that have a Local Resource Manager (LRM). For instance, campus clusters
     and supercomputers generally have LRMs (schedulers) such as Slurm,
-    Torque/PBS, Condor and Cobalt. Clouds, on the other hand, have API
+    Torque/PBS, and Condor. Clouds, on the other hand, have API
     interfaces that allow much more fine-grained composition of an execution
     environment. An execution provider abstracts these types of resources and
     provides a single uniform interface to them.

parsl/tests/conftest.py

@@ -163,6 +163,10 @@ def pytest_configure(config):
         'markers',
         'executor_supports_std_stream_tuples: Marks tests that require tuple support for stdout/stderr'
     )
+    config.addinivalue_line(
+        'markers',
+        'shared_fs: Marks tests that require a shared_fs between the workers are the test client'
+    )
 
 
 @pytest.fixture(autouse=True, scope='session')

parsl/tests/site_tests/site_config_selector.py

@@ -7,12 +7,7 @@ def fresh_config():
     hostname = os.getenv('PARSL_HOSTNAME', platform.uname().node)
     print("Loading config for {}".format(hostname))
 
-    if 'thetalogin' in hostname:
-        from parsl.tests.configs.theta import fresh_config
-        config = fresh_config()
-        print("Loading Theta config")
-
-    elif 'frontera' in hostname:
+    if 'frontera' in hostname:
         print("Loading Frontera config")
         from parsl.tests.configs.frontera import fresh_config
         config = fresh_config()

parsl/tests/test_bash_apps/test_basic.py

@@ -24,6 +24,7 @@ def foo(x, y, z=10, stdout=None, label=None):
     return f"echo {x} {y} {z}"
 
 
+@pytest.mark.shared_fs
 def test_command_format_1(tmpd_cwd):
     """Testing command format for BashApps"""
 
@@ -38,6 +39,7 @@ def test_command_format_1(tmpd_cwd):
     assert so_content == "1 4 10"
 
 
+@pytest.mark.shared_fs
 def test_auto_log_filename_format(caplog):
     """Testing auto log filename format for BashApps
     """
@@ -66,6 +68,7 @@ def test_auto_log_filename_format(caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.shared_fs
 def test_parallel_for(tmpd_cwd, n=3):
     """Testing a simple parallel for loop"""
     outdir = tmpd_cwd / "outputs/test_parallel"

parsl/tests/test_bash_apps/test_error_codes.py

@@ -58,6 +58,7 @@ test_matrix = {
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
+@pytest.mark.shared_fs
 def test_div_0(test_fn=div_0):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -73,6 +74,7 @@ def test_div_0(test_fn=div_0):
     os.remove('std.out')
 
 
+@pytest.mark.shared_fs
 def test_bash_misuse(test_fn=bash_misuse):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -87,6 +89,7 @@ def test_bash_misuse(test_fn=bash_misuse):
     os.remove('std.out')
 
 
+@pytest.mark.shared_fs
 def test_command_not_found(test_fn=command_not_found):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -103,6 +106,7 @@ def test_command_not_found(test_fn=command_not_found):
     return True
 
 
+@pytest.mark.shared_fs
 def test_not_executable(test_fn=not_executable):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()

parsl/tests/test_bash_apps/test_kwarg_storage.py

@@ -8,6 +8,7 @@ def foo(z=2, stdout=None):
     return f"echo {z}"
 
 
+@pytest.mark.shared_fs
 def test_command_format_1(tmpd_cwd):
     """Testing command format for BashApps
     """

parsl/tests/test_bash_apps/test_memoize.py

@@ -9,9 +9,7 @@ def fail_on_presence(outputs=()):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
+@pytest.mark.shared_fs
 def test_bash_memoization(tmpd_cwd, n=2):
     """Testing bash memoization
     """
@@ -29,9 +27,7 @@ def fail_on_presence_kw(outputs=(), foo=None):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
+@pytest.mark.shared_fs
 def test_bash_memoization_keywords(tmpd_cwd, n=2):
     """Testing bash memoization
     """

parsl/tests/test_bash_apps/test_memoize_ignore_args.py

@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 import parsl
 from parsl.app.app import bash_app
 
@@ -21,6 +23,7 @@ def no_checkpoint_stdout_app_ignore_args(stdout=None):
     return "echo X"
 
 
+@pytest.mark.shared_fs
 def test_memo_stdout(tmpd_cwd):
     path_x = tmpd_cwd / "test.memo.stdout.x"
 

parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py

@@ -29,6 +29,7 @@ def no_checkpoint_stdout_app(stdout=None):
     return "echo X"
 
 
+@pytest.mark.shared_fs
 def test_memo_stdout(tmpd_cwd):
     assert const_list_x == const_list_x_arg
 

parsl/tests/test_bash_apps/test_multiline.py

@@ -14,6 +14,7 @@ def multiline(inputs=(), outputs=(), stderr=None, stdout=None):
     """.format(inputs=inputs, outputs=outputs)
 
 
+@pytest.mark.shared_fs
 def test_multiline(tmpd_cwd):
     so, se = tmpd_cwd / "std.out", tmpd_cwd / "std.err"
     f = multiline(

parsl/tests/test_bash_apps/test_stdout.py

@@ -91,6 +91,7 @@ def test_bad_stderr_file():
 
 
 @pytest.mark.executor_supports_std_stream_tuples
+@pytest.mark.shared_fs
 def test_stdout_truncate(tmpd_cwd, caplog):
     """Testing truncation of prior content of stdout"""
 
@@ -110,6 +111,7 @@ def test_stdout_truncate(tmpd_cwd, caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.shared_fs
 def test_stdout_append(tmpd_cwd, caplog):
     """Testing appending to prior content of stdout (default open() mode)"""
 

parsl/tests/test_docs/test_from_slides.py

@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 from parsl.app.app import bash_app, python_app
 from parsl.data_provider.files import File
 
@@ -15,6 +17,7 @@ def cat(inputs=[]):
         return f.readlines()
 
 
+@pytest.mark.staging_required
 def test_slides():
     """Testing code snippet from slides """
 

parsl/tests/test_docs/test_kwargs.py

@@ -1,6 +1,8 @@
 """Functions used to explain kwargs"""
 from pathlib import Path
 
+import pytest
+
 from parsl import File, python_app
 
 
@@ -19,6 +21,7 @@ def test_inputs():
     assert reduce_future.result() == 6
 
 
+@pytest.mark.shared_fs
 def test_outputs(tmpd_cwd):
     @python_app()
     def write_app(message, outputs=()):

parsl/tests/test_monitoring/test_basic.py

@@ -42,6 +42,18 @@ def htex_udp_config():
     return c
 
 
+def htex_filesystem_config():
+    """This config will force filesystem radio"""
+    from parsl.tests.configs.htex_local_alternate import fresh_config
+    c = fresh_config()
+    assert len(c.executors) == 1
+
+    assert c.executors[0].radio_mode == "htex", "precondition: htex has a radio mode attribute, configured for htex radio"
+    c.executors[0].radio_mode = "filesystem"
+
+    return c
+
+
 def workqueue_config():
     from parsl.tests.configs.workqueue_ex import fresh_config
     c = fresh_config()
@@ -61,7 +73,7 @@ def taskvine_config():
 
 
 @pytest.mark.local
-@pytest.mark.parametrize("fresh_config", [htex_config, htex_udp_config, workqueue_config, taskvine_config])
+@pytest.mark.parametrize("fresh_config", [htex_config, htex_filesystem_config, htex_udp_config, workqueue_config, taskvine_config])
 def test_row_counts(tmpd_cwd, fresh_config):
     # this is imported here rather than at module level because
     # it isn't available in a plain parsl install, so this module

parsl/tests/test_python_apps/test_outputs.py

@@ -16,6 +16,7 @@ def double(x, outputs=[]):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
+@pytest.mark.shared_fs
 def test_launch_apps(tmpd_cwd, n=2):
     outdir = tmpd_cwd / "outputs"
     outdir.mkdir()

parsl/tests/test_regression/test_226.py

@@ -53,6 +53,7 @@ def test_get_dataframe():
     assert res.equals(data), 'Unexpected dataframe'
 
 
+@pytest.mark.shared_fs
 def test_bash_default_arg():
     if os.path.exists('std.out'):
         os.remove('std.out')

parsl/tests/test_staging/test_docs_1.py

@@ -12,6 +12,7 @@ def convert(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test():
     # create an remote Parsl file
     inp = File('ftp://ftp.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')

parsl/tests/test_staging/test_output_chain_filenames.py

@@ -1,5 +1,7 @@
 from concurrent.futures import Future
 
+import pytest
+
 from parsl import File
 from parsl.app.app import bash_app
 
@@ -14,6 +16,7 @@ def app2(inputs=(), outputs=(), stdout=None, stderr=None, mock=False):
     return f"echo '{inputs[0]}' > {outputs[0]}"
 
 
+@pytest.mark.shared_fs
 def test_behavior(tmpd_cwd):
     expected_path = str(tmpd_cwd / "simple-out.txt")
     app1_future = app1(

parsl/tests/test_staging/test_staging_ftp.py

@@ -15,6 +15,7 @@ def sort_strings(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_ftp():
     """Test staging for an ftp file
 

parsl/tests/test_staging/test_staging_https.py

@@ -48,6 +48,7 @@ def sort_strings_additional_executor(inputs=(), outputs=()):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_cleannet(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -68,6 +69,7 @@ def test_staging_https_local(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_kwargs(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -78,6 +80,7 @@ def test_staging_https_kwargs(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_args(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')

parsl/tests/test_staging/test_staging_stdout.py

@@ -15,6 +15,7 @@ def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
     return "echo hello ; echo goodbye >&2"
 
 
+@pytest.mark.staging_required
 def test_stdout_staging_file(tmpd_cwd, caplog):
     basename = str(tmpd_cwd) + "/stdout.txt"
     stdout_file = File("file://" + basename)
@@ -30,6 +31,7 @@ def test_stdout_staging_file(tmpd_cwd, caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.staging_required
 def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
     zipfile_name = str(tmpd_cwd) + "/staging.zip"
     stdout_relative_path = "somewhere/test-out.txt"

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.10.28'
+VERSION = '2024.11.04'

{parsl-2024.10.28.dist-info → parsl-2024.11.4.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.10.28
+Version: 2024.11.4
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.10.28.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.11.04.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0