parsl 2024.5.20__py3-none-any.whl → 2024.6.3__py3-none-any.whl

parsl/tests/test_checkpointing/test_task_exit.py

@@ -5,8 +5,8 @@ import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.utils import time_limited_open
 from parsl.tests.configs.local_threads_checkpoint_task_exit import config
+from parsl.utils import time_limited_open
 
 
 def local_setup():
@@ -16,7 +16,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app(cache=True)

parsl/tests/test_docs/test_from_slides.py

@@ -1,8 +1,8 @@
+import os
+
 from parsl.app.app import bash_app, python_app
 from parsl.data_provider.files import File
 
-import os
-
 
 @bash_app
 def echo(message, outputs=[]):

parsl/tests/test_docs/test_kwargs.py

@@ -1,7 +1,7 @@
 """Functions used to explain kwargs"""
 from pathlib import Path
 
-from parsl import python_app, File
+from parsl import File, python_app
 
 
 def test_inputs():

parsl/tests/test_docs/test_tutorial_1.py

@@ -1,12 +1,11 @@
 import argparse
-import parsl
 
 import pytest
 
+import parsl
 from parsl.app.app import bash_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_docs/test_workflow1.py

@@ -1,12 +1,12 @@
 import os
+
 import pytest
-import parsl
 
+import parsl
 from parsl.app.app import bash_app, python_app
 from parsl.data_provider.files import File
 from parsl.tests.configs.local_threads import config
 
-
 # parsl.set_stream_logger()
 
 

parsl/tests/test_docs/test_workflow2.py

@@ -5,7 +5,6 @@ import pytest
 from parsl.app.app import python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_error_handling/test_rand_fail.py

@@ -15,8 +15,8 @@ def local_config():
 
 @python_app
 def sleep_fail(sleep_dur, sleep_rand_max, fail_prob, inputs=[]):
-    import time
     import random
+    import time
 
     s = sleep_dur + random.randint(-sleep_rand_max, sleep_rand_max)
 
@@ -144,8 +144,8 @@ def test_deps(numtasks=10):
 
 @python_app
 def sleep_then_fail(sleep_dur=0.1):
-    import time
     import math
+    import time
     time.sleep(sleep_dur)
     math.ceil("Trigger TypeError")
     return 0

parsl/tests/test_error_handling/test_resource_spec.py

@@ -1,9 +1,11 @@
 import parsl
 from parsl.app.app import python_app
-from parsl.executors.errors import UnsupportedFeatureError, ExecutorError
 from parsl.executors import WorkQueueExecutor
-from parsl.executors.high_throughput.mpi_prefix_composer import InvalidResourceSpecification
+from parsl.executors.errors import ExecutorError, UnsupportedFeatureError
 from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    InvalidResourceSpecification,
+)
 
 
 @python_app

parsl/tests/test_error_handling/test_retries.py

@@ -1,5 +1,6 @@
 import argparse
 import os
+
 import pytest
 
 import parsl
@@ -15,8 +16,8 @@ def local_config():
 
 @python_app
 def sleep_then_fail(inputs=[], sleep_dur=0.1):
-    import time
     import math
+    import time
     time.sleep(sleep_dur)
     math.ceil("Trigger TypeError")
     return 0

parsl/tests/test_error_handling/test_retry_handler.py

@@ -1,4 +1,5 @@
 import os
+
 import pytest
 
 import parsl

parsl/tests/test_error_handling/test_retry_handler_failure.py

@@ -1,6 +1,7 @@
-import parsl
 import pytest
 
+import parsl
+
 
 @parsl.python_app
 def always_fails():

parsl/tests/test_error_handling/test_serialization_fail.py

@@ -1,8 +1,8 @@
 import pytest
 
 from parsl import python_app
-from parsl.tests.configs.htex_local import fresh_config
 from parsl.serialize.errors import SerializationError
+from parsl.tests.configs.htex_local import fresh_config
 
 
 def local_config():

parsl/tests/test_error_handling/test_wrap_with_logs.py

@@ -1,4 +1,5 @@
 import logging
+
 import pytest
 
 from parsl.process_loggers import wrap_with_logs

parsl/tests/test_flux.py

@@ -1,5 +1,5 @@
-import os
 import concurrent.futures as cf
+import os
 
 import pytest
 

parsl/tests/test_htex/test_basic.py

@@ -14,7 +14,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app

parsl/tests/test_htex/test_command_client_timeout.py

@@ -0,0 +1,66 @@
+import threading
+import time
+
+import pytest
+import zmq
+
+from parsl import curvezmq
+from parsl.executors.high_throughput.errors import (
+    CommandClientBadError,
+    CommandClientTimeoutError,
+)
+from parsl.executors.high_throughput.zmq_pipes import CommandClient
+
+# Time constant used for timeout tests: various delays and
+# timeouts will be appropriate multiples of this, but the
+# value of T itself should not matter too much as long as
+# it is big enough for zmq connections to happen successfully.
+T = 0.25
+
+
+@pytest.mark.local
+def test_command_not_sent() -> None:
+    """Tests timeout on command send.
+    """
+    # RFC6335 ephemeral port range
+    cc = CommandClient("127.0.0.1", (49152, 65535))
+
+    # cc will now wait for a connection, but we won't do anything to make the
+    # other side of the connection exist, so any command given to cc should
+    # timeout.
+
+    with pytest.raises(CommandClientTimeoutError):
+        cc.run("SOMECOMMAND", timeout_s=T)
+
+    cc.close()
+
+
+@pytest.mark.local
+def test_command_ignored() -> None:
+    """Tests timeout on command response.
+    Tests that we timeout after a response and that the command client
+    sets itself into a bad state.
+
+    This only tests sequential access to the command client, even though
+    htex makes multithreaded use of the command client: see issue #3376 about
+    that lack of thread safety.
+    """
+    # RFC6335 ephemeral port range
+    cc = CommandClient("127.0.0.1", (49152, 65535))
+
+    ic_ctx = curvezmq.ServerContext(None)
+    ic_channel = ic_ctx.socket(zmq.REP)
+    ic_channel.connect(f"tcp://127.0.0.1:{cc.port}")
+
+    with pytest.raises(CommandClientTimeoutError):
+        cc.run("SLOW_COMMAND", timeout_s=T)
+
+    req = ic_channel.recv_pyobj()
+    assert req == "SLOW_COMMAND", "Should have received command on interchange side"
+    assert not cc.ok, "CommandClient should have set itself to bad"
+
+    with pytest.raises(CommandClientBadError):
+        cc.run("ANOTHER_COMMAND")
+
+    cc.close()
+    ic_channel.close()

parsl/tests/test_htex/test_connected_blocks.py

@@ -1,7 +1,8 @@
-import parsl
 import pytest
-from parsl.executors import HighThroughputExecutor
+
+import parsl
 from parsl import Config
+from parsl.executors import HighThroughputExecutor
 from parsl.providers import LocalProvider
 
 

parsl/tests/test_htex/test_cpu_affinity_explicit.py

@@ -1,8 +1,10 @@
 import logging
 import os
-import parsl
-import pytest
 import random
+
+import pytest
+
+import parsl
 from parsl.tests.configs.htex_local import fresh_config
 
 logger = logging.getLogger(__name__)
@@ -18,6 +20,7 @@ def my_affinity():
 
 @pytest.mark.local
 @pytest.mark.multiple_cores_required
+@pytest.mark.skipif('sched_getaffinity' not in dir(os), reason='System does not support sched_setaffinity')
 def test_cpu_affinity_explicit():
     available_cores = os.sched_getaffinity(0)
 
@@ -37,16 +40,9 @@ def test_cpu_affinity_explicit():
     config.executors[0].max_workers_per_node = 1
 
     logger.debug(f"config: {config}")
-    # TODO: is there a `with` style for this, to properly deal with exceptions?
-
-    parsl.load(config)
-    try:
 
+    with parsl.load(config):
         worker_affinity = my_affinity().result()
         logger.debug(f"worker reported this affinity: {worker_affinity}")
         assert len(worker_affinity) == 1
         assert worker_affinity == set((single_core,))
-
-    finally:
-        parsl.dfk().cleanup()
-        parsl.clear()

parsl/tests/test_htex/test_disconnected_blocks.py

@@ -1,11 +1,13 @@
 import logging
-import parsl
+
 import pytest
-from parsl.executors import HighThroughputExecutor
+
+import parsl
 from parsl import Config
-from parsl.providers import LocalProvider
+from parsl.executors import HighThroughputExecutor
 from parsl.executors.errors import BadStateException
-from parsl.jobs.states import JobStatus, JobState
+from parsl.jobs.states import JobState, JobStatus
+from parsl.providers import LocalProvider
 
 
 def local_config():

parsl/tests/test_htex/test_drain.py

@@ -1,13 +1,13 @@
-import parsl
-import pytest
 import time
 
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
+import pytest
 
+import parsl
+from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 # this constant is used to scale some durations that happen
 # based around the expected drain period: the drain period

parsl/tests/test_htex/test_htex.py

@@ -4,8 +4,7 @@ from unittest import mock
 
 import pytest
 
-from parsl import curvezmq
-from parsl import HighThroughputExecutor
+from parsl import HighThroughputExecutor, curvezmq
 from parsl.multiprocessing import ForkProcess
 
 _MOCK_BASE = "parsl.executors.high_throughput.executor"

parsl/tests/test_htex/test_manager_failure.py

@@ -20,7 +20,6 @@ def load_config():
     yield
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app

parsl/tests/test_htex/test_managers_command.py

@@ -1,22 +1,18 @@
-import pytest
 import logging
-
 import sys
+
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.tests.configs.htex_local import fresh_config
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].poll_period = 1
     config.executors[0].max_workers_per_node = 1
-    parsl.load(config)
-
-
-def local_teardown():
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app

parsl/tests/test_htex/test_missing_worker.py

@@ -5,18 +5,12 @@ from parsl.app.app import python_app
 from parsl.tests.configs.htex_local import fresh_config
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].poll_period = 1
     config.executors[0].max_workers_per_node = 1
     config.executors[0].launch_cmd = "executable_that_hopefully_does_not_exist_1030509.py"
-    parsl.load(config)
-
-
-def local_teardown():
-
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app

parsl/tests/test_htex/test_multiple_disconnected_blocks.py

@@ -1,11 +1,13 @@
 import logging
-import parsl
+
 import pytest
-from parsl.executors import HighThroughputExecutor
+
+import parsl
 from parsl import Config
-from parsl.providers import LocalProvider
+from parsl.executors import HighThroughputExecutor
 from parsl.executors.errors import BadStateException
-from parsl.jobs.states import JobStatus, JobState
+from parsl.jobs.states import JobState, JobStatus
+from parsl.providers import LocalProvider
 
 
 def local_config():

parsl/tests/test_monitoring/test_app_names.py

@@ -2,10 +2,11 @@
 """
 
 import os
-import parsl
-import pytest
 import time
 
+import pytest
+
+import parsl
 from parsl.tests.configs.htex_local_alternate import fresh_config
 
 
@@ -61,7 +62,6 @@ def test_app_name(get_app, expected_name, expected_result, tmpd_cwd):
     assert app().result() == expected_result
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
     engine = sqlalchemy.create_engine(c.monitoring.logging_endpoint)
     with engine.begin() as connection:

parsl/tests/test_monitoring/test_basic.py

@@ -1,12 +1,12 @@
 import os
-import parsl
-import pytest
 import time
 
+import pytest
+
+import parsl
 from parsl import HighThroughputExecutor
 from parsl.config import Config
-from parsl.executors.taskvine import TaskVineExecutor
-from parsl.executors.taskvine import TaskVineManagerConfig
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
 from parsl.monitoring import MonitoringHub
 
 
@@ -66,8 +66,6 @@ def test_row_counts(tmpd_cwd, fresh_config):
     with parsl.load(config):
         assert this_app().result() == 5
 
-    parsl.clear()
-
     # at this point, we should find one row in the monitoring database.
 
     engine = sqlalchemy.create_engine(db_url)

parsl/tests/test_monitoring/test_db_locks.py

@@ -1,9 +1,11 @@
 import logging
 import os
-import parsl
-import pytest
 import time
 
+import pytest
+
+import parsl
+
 logger = logging.getLogger(__name__)
 
 
@@ -14,9 +16,10 @@ def this_app():
 
 @pytest.mark.local
 def test_row_counts():
-    from parsl.tests.configs.htex_local_alternate import fresh_config
     import sqlalchemy
     from sqlalchemy import text
+
+    from parsl.tests.configs.htex_local_alternate import fresh_config
     if os.path.exists("runinfo/monitoring.db"):
         logger.info("Monitoring database already exists - deleting")
         os.remove("runinfo/monitoring.db")
@@ -63,7 +66,6 @@ def test_row_counts():
 
     logger.info("cleaning up parsl")
     parsl.dfk().cleanup()
-    parsl.clear()
 
     # at this point, we should find data consistent with executing one
     # task in the database.

parsl/tests/test_monitoring/test_fuzz_zmq.py

@@ -1,11 +1,13 @@
 import logging
 import os
-import parsl
-import pytest
 import socket
 import time
+
+import pytest
 import zmq
 
+import parsl
+
 logger = logging.getLogger(__name__)
 
 
@@ -16,10 +18,11 @@ def this_app():
 
 @pytest.mark.local
 def test_row_counts():
-    from parsl.tests.configs.htex_local_alternate import fresh_config
     import sqlalchemy
     from sqlalchemy import text
 
+    from parsl.tests.configs.htex_local_alternate import fresh_config
+
     if os.path.exists("runinfo/monitoring.db"):
         logger.info("Monitoring database already exists - deleting")
         os.remove("runinfo/monitoring.db")
@@ -83,7 +86,6 @@ def test_row_counts():
 
     logger.info("cleaning up parsl")
     parsl.dfk().cleanup()
-    parsl.clear()
 
     # at this point, we should find one row in the monitoring database.
 

parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py

@@ -1,16 +1,16 @@
 import logging
 import os
-import parsl
-import pytest
 import time
 
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
+import pytest
 
+import parsl
+from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
 from parsl.monitoring import MonitoringHub
+from parsl.providers import LocalProvider
 
 
 def fresh_config(run_dir, strategy, db_url):
@@ -65,8 +65,6 @@ def test_row_counts(tmpd_cwd, strategy):
 
         this_app().result()
 
-    parsl.clear()
-
     engine = sqlalchemy.create_engine(db_url)
     with engine.begin() as connection:
 

parsl/tests/test_monitoring/test_incomplete_futures.py

@@ -1,11 +1,12 @@
 import logging
 import os
-import parsl
-import pytest
 import random
-
 from concurrent.futures import Future
 
+import pytest
+
+import parsl
+
 
 @parsl.python_app
 def this_app(inputs=()):
@@ -16,6 +17,7 @@ def this_app(inputs=()):
 def test_future_representation(tmpd_cwd):
     import sqlalchemy
     from sqlalchemy import text
+
     from parsl.tests.configs.htex_local_alternate import fresh_config
 
     monitoring_db = str(tmpd_cwd / "monitoring.db")
@@ -52,7 +54,6 @@ def test_future_representation(tmpd_cwd):
     # seconds, with the assumption "data will arrive in the DB within
     # 30 seconds, but probably much sooner".
     parsl.dfk().cleanup()
-    parsl.clear()
 
     engine = sqlalchemy.create_engine(monitoring_url)
     with engine.begin() as connection:

parsl/tests/test_monitoring/test_memoization_representation.py

@@ -1,9 +1,11 @@
 
 import logging
 import os
-import parsl
+
 import pytest
 
+import parsl
+
 logger = logging.getLogger(__name__)
 
 
@@ -16,6 +18,7 @@ def this_app(x):
 def test_hashsum():
     import sqlalchemy
     from sqlalchemy import text
+
     from parsl.tests.configs.htex_local_alternate import fresh_config
 
     if os.path.exists("runinfo/monitoring.db"):
@@ -47,7 +50,6 @@ def test_hashsum():
 
     logger.info("cleaning up parsl")
     parsl.dfk().cleanup()
-    parsl.clear()
 
     # at this point, we should find one row in the monitoring database.
 

parsl/tests/test_monitoring/test_stdouterr.py

@@ -3,16 +3,16 @@
 
 import logging
 import os
-import parsl
-import pytest
 import re
 import time
-
 from typing import Union
 
+import pytest
+
+import parsl
 from parsl.config import Config
-from parsl.data_provider.files import File
 from parsl.data_provider.data_manager import default_staging
+from parsl.data_provider.files import File
 from parsl.data_provider.staging import Staging
 from parsl.executors import HighThroughputExecutor
 from parsl.monitoring import MonitoringHub
@@ -103,8 +103,6 @@ def test_stdstream_to_monitoring(stdx, expected_stdx, stream, tmpd_cwd, caplog):
         kwargs = {stream: stdx}
         stdapp(**kwargs).result()
 
-    parsl.clear()
-
     engine = sqlalchemy.create_engine(c.monitoring.logging_endpoint)
     with engine.begin() as connection:
 

parsl/tests/test_monitoring/test_viz_colouring.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl.dataflow.states import States