parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_scaling/test_regression_3696_oscillation.py

@@ -0,0 +1,103 @@
+import math
+from unittest.mock import MagicMock
+
+import pytest
+
+from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.jobs.states import JobState, JobStatus
+from parsl.jobs.strategy import Strategy
+
+
+# the parameterize tuple consists of:
+# Input:
+#  * number of tasks to mock the load as
+#  * number of workers per node
+# Expected output:
+#  * the number of blocks we should expect to be launched
+#    in this situation
+#
+# This test will configure an executor, then run strategize
+# a few times, asserting that it converges to the correct
+# number of blocks without oscillating.
+@pytest.mark.local
+@pytest.mark.parametrize("ns", [(14, 48, 1),    # values from issue #3696
+
+                                (1, 1, 1),      # one task needs one block
+
+                                (100, 1, 20),   # many one-task blocks, hitting hard-coded max blocks
+
+                                (47, 48, 1),    # some edge cases around #3696 values
+                                (48, 48, 1),    # "
+                                (49, 48, 2),    # "
+                                (149, 50, 3)])  # "
+def test_htex_strategy_does_not_oscillate(ns):
+    """Check for oscillations in htex scaling.
+    In issue 3696, with a large number of workers per block
+    and a smaller number of active tasks, the htex scaling
+    strategy oscillates between 0 and 1 active block, rather
+    than converging to 1 active block.
+    """
+
+    n_tasks, n_workers, n_blocks = ns
+
+    s = Strategy(strategy='htex_auto_scale', max_idletime=0)
+
+    provider = MagicMock()
+    executor = MagicMock(spec=HighThroughputExecutor)
+
+    statuses = {}
+
+    executor.provider = provider
+    executor.outstanding = n_tasks
+    executor.status_facade = statuses
+    executor.workers_per_node = n_workers
+
+    provider.parallelism = 1
+    provider.init_blocks = 0
+    provider.min_blocks = 0
+    provider.max_blocks = 20
+    provider.nodes_per_block = 1
+
+    def scale_out(n):
+        for _ in range(n):
+            statuses[len(statuses)] = JobStatus(state=JobState.PENDING)
+
+    executor.scale_out_facade.side_effect = scale_out
+
+    def scale_in(n, max_idletime=None):
+        # find n PENDING jobs and set them to CANCELLED
+        for k in statuses:
+            if n == 0:
+                return
+            if statuses[k].state == JobState.PENDING:
+                statuses[k].state = JobState.CANCELLED
+                n -= 1
+
+    executor.scale_in_facade.side_effect = scale_in
+
+    s.add_executors([executor])
+
+    # In issue #3696, this first strategise does initial and load based
+    # scale outs, because n_tasks > n_workers*0
+    s.strategize([executor])
+
+    executor.scale_out_facade.assert_called()
+    assert len(statuses) == n_blocks, "Should have launched n_blocks"
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == n_blocks
+    # there might be several calls to scale_out_facade inside strategy,
+    # but the end effect should be that exactly one block is scaled out.
+
+    executor.scale_in_facade.assert_not_called()
+
+    # In issue #3696, this second strategize does a scale in, because n_tasks < n_workers*1
+    s.strategize([executor])
+
+    # assert that there should still be n_blocks pending blocks
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == n_blocks
+    # this assert fails due to issue #3696
+
+    # Now check scale in happens with 0 load
+    executor.outstanding = 0
+    s.strategize([executor])
+    executor.scale_in_facade.assert_called()
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == 0

parsl/tests/test_scaling/test_scale_down.py

@@ -4,13 +4,11 @@ import time
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +28,6 @@ def local_config():
                 max_workers_per_node=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     max_blocks=_max_blocks,
                     min_blocks=_min_blocks,

parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py

@@ -1,15 +1,13 @@
+from threading import Event
+
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-
-from threading import Event
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 _max_blocks = 5
 _min_blocks = 0
@@ -24,10 +22,9 @@ def local_config():
                 poll_period=100,
                 label="htex_local",
                 address="127.0.0.1",
-                max_workers=1,
+                max_workers_per_node=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     max_blocks=_max_blocks,
                     min_blocks=_min_blocks,
@@ -37,6 +34,7 @@ def local_config():
         ],
         max_idletime=0.5,
         strategy='htex_auto_scale',
+        strategy_period=0.1
     )
 
 
@@ -62,16 +60,6 @@ def waiting_app(ident: int, outputs=(), inputs=()):
 def test_scale_out(tmpd_cwd, try_assert):
     dfk = parsl.dfk()
 
-    # reconfigure scaling strategy to run faster than usual. This allows
-    # this test to complete faster - at time of writing 27s with default
-    # 5s strategy, vs XXXX with 0.5s strategy.
-
-    # check this attribute still exists, in the presence of ongoing
-    # development, so we have some belief that setting it will not be
-    # setting a now-ignored parameter.
-    assert hasattr(dfk.job_status_poller, 'interval')
-    dfk.job_status_poller.interval = 0.1
-
     num_managers = len(dfk.executors['htex_local'].connected_managers())
 
     assert num_managers == 0, "Expected 0 managers at start"

parsl/tests/test_scaling/test_scale_down_htex_unregistered.py

@@ -0,0 +1,71 @@
+import logging
+import time
+
+import pytest
+
+import parsl
+from parsl import File, python_app
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.jobs.states import TERMINAL_STATES, JobState
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
+
+logger = logging.getLogger(__name__)
+
+_max_blocks = 1
+_min_blocks = 0
+
+
+def local_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                heartbeat_period=1,
+                heartbeat_threshold=2,
+                poll_period=100,
+                label="htex_local",
+                address="127.0.0.1",
+                max_workers_per_node=1,
+                encrypted=True,
+                launch_cmd="sleep inf",
+                provider=LocalProvider(
+                    init_blocks=1,
+                    max_blocks=_max_blocks,
+                    min_blocks=_min_blocks,
+                    launcher=SingleNodeLauncher(),
+                ),
+            )
+        ],
+        max_idletime=0.5,
+        strategy='htex_auto_scale',
+        strategy_period=0.1
+    )
+
+
+# see issue #1885 for details of failures of this test.
+# at the time of issue #1885 this test was failing frequently
+# in CI.
+@pytest.mark.local
+def test_scaledown_with_register(try_assert):
+    dfk = parsl.dfk()
+    htex = dfk.executors['htex_local']
+
+    num_managers = len(htex.connected_managers())
+    assert num_managers == 0, "Expected 0 managers at start"
+
+    try_assert(lambda: len(htex.status()),
+               fail_msg="Expected 1 block at start")
+
+    s = htex.status()
+    assert s['0'].state == JobState.RUNNING, "Expected block to be in RUNNING"
+
+    def check_zero_blocks():
+        s = htex.status()
+        return len(s) == 1 and s['0'].state in TERMINAL_STATES
+
+    try_assert(
+        check_zero_blocks,
+        fail_msg="Expected 0 blocks after idle scaledown",
+        timeout_ms=15000,
+    )

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -0,0 +1,73 @@
+import random
+import threading
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+
+# we need some blocks, but it doesn't matter too much how many, as long
+# as they can all start up and get registered within the try_assert
+# timeout later on.
+BLOCK_COUNT = 3
+
+
+class AccumulatingLocalProvider(LocalProvider):
+    def __init__(self, *args, **kwargs):
+        # Use a list for submitted job IDs because if there are multiple
+        # submissions returning the same job ID, this test should count
+        # those...
+        self.submit_job_ids = []
+
+        # ... but there's no requirement, I think, that cancel must be called
+        # only once per job id. What matters here is that each job ID is
+        # cancelled at least once.
+        self.cancel_job_ids = set()
+
+        super().__init__(*args, **kwargs)
+
+    def submit(self, *args, **kwargs):
+        job_id = super().submit(*args, **kwargs)
+        self.submit_job_ids.append(job_id)
+        return job_id
+
+    def cancel(self, job_ids):
+        self.cancel_job_ids.update(job_ids)
+        return super().cancel(job_ids)
+
+
+@pytest.mark.local
+def test_shutdown_scalein_blocks(tmpd_cwd, try_assert):
+    """
+    This test scales up several blocks, and then checks that they are all
+    scaled in at DFK shutdown.
+    """
+    accumulating_provider = AccumulatingLocalProvider(
+        init_blocks=BLOCK_COUNT,
+        min_blocks=0,
+        max_blocks=0,
+        launcher=SimpleLauncher(),
+    )
+
+    htex = HighThroughputExecutor(
+               label="htex_local",
+               cores_per_worker=1,
+               provider=accumulating_provider
+           )
+
+    config = Config(
+        executors=[htex],
+        strategy='none',
+        strategy_period=0.1,
+        run_dir=str(tmpd_cwd)
+    )
+
+    with parsl.load(config):
+        # this will wait for everything to be scaled out fully
+        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT)
+
+    assert len(accumulating_provider.submit_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been launched"
+    assert len(accumulating_provider.cancel_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been scaled in"

parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py

@@ -0,0 +1,90 @@
+import os
+import signal
+import time
+
+import pytest
+import zmq
+
+import parsl
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+
+T_s = 1
+
+
+def fresh_config():
+    htex = HighThroughputExecutor(
+               heartbeat_period=1 * T_s,
+               heartbeat_threshold=3 * T_s,
+               label="htex_local",
+               worker_debug=True,
+               cores_per_worker=1,
+               encrypted=False,
+               provider=LocalProvider(
+                   init_blocks=0,
+                   min_blocks=0,
+                   max_blocks=0,
+                   launcher=SimpleLauncher(),
+               ),
+           )
+    c = Config(
+        executors=[htex],
+        strategy='none',
+        strategy_period=0.5,
+    )
+    return c, htex
+
+
+@parsl.python_app
+def app():
+    return 7
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("msg",
+                         (b'FuzzyByte\rSTREAM',  # not JSON
+                          b'{}',  # missing fields
+                          b'{"type":"heartbeat"}',  # regression test #3262
+                          )
+                         )
+def test_bad_messages(try_assert, msg):
+    """This tests that the interchange is resilient to a few different bad
+    messages: malformed messages caused by implementation errors, and
+    heartbeat messages from managers that are not registered.
+
+    The heartbeat test is a regression test for issues #3262, #3632
+    """
+
+    c, htex = fresh_config()
+
+    with parsl.load(c):
+
+        # send a bad message into the interchange on the task_outgoing worker
+        # channel, and then check that the interchange is still alive enough
+        # that we can scale out a block and run a task.
+
+        (task_port, result_port) = htex.command_client.run("WORKER_PORTS")
+
+        context = zmq.Context()
+        channel_timeout = 10000  # in milliseconds
+        task_channel = context.socket(zmq.DEALER)
+        task_channel.setsockopt(zmq.LINGER, 0)
+        task_channel.setsockopt(zmq.IDENTITY, b'testid')
+
+        task_channel.set_hwm(0)
+        task_channel.setsockopt(zmq.SNDTIMEO, channel_timeout)
+        task_channel.connect(f"tcp://localhost:{task_port}")
+
+        task_channel.send(msg)
+
+        # If the interchange exits, it's likely that this test will hang rather
+        # than raise an error, because the interchange interaction code
+        # assumes the interchange is always there.
+        # In the case of issue #3262, an exception message goes to stderr, and
+        # no error goes to the interchange log file.
+        htex.scale_out_facade(1)
+        try_assert(lambda: len(htex.connected_managers()) == 1, timeout_ms=10000)
+
+        assert app().result() == 7

parsl/tests/test_serialization/test_2555_caching_deserializer.py

@@ -1,6 +1,6 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_serialization/test_3495_deserialize_managerlost.py

@@ -0,0 +1,47 @@
+import os
+import signal
+
+import pytest
+
+import parsl
+from parsl import Config, HighThroughputExecutor
+from parsl.executors.high_throughput.errors import ManagerLost
+
+
+@parsl.python_app
+def get_manager_pgid():
+    import os
+    return os.getpgid(os.getpid())
+
+
+@parsl.python_app
+def lose_manager():
+    import os
+    import signal
+
+    manager_pid = os.getppid()
+    os.kill(manager_pid, signal.SIGSTOP)
+
+
+@pytest.mark.local
+def test_manager_lost_system_failure(tmpd_cwd):
+    hte = HighThroughputExecutor(
+        label="htex_local",
+        address="127.0.0.1",
+        max_workers_per_node=2,
+        cores_per_worker=1,
+        worker_logdir_root=str(tmpd_cwd),
+        heartbeat_period=1,
+        heartbeat_threshold=3,
+    )
+    c = Config(executors=[hte], strategy='simple', strategy_period=0.1)
+
+    with parsl.load(c):
+        manager_pgid = get_manager_pgid().result()
+        try:
+            with pytest.raises(ManagerLost):
+                lose_manager().result()
+        finally:
+            # Allow process to clean itself up
+            os.killpg(manager_pgid, signal.SIGCONT)
+            os.killpg(manager_pgid, signal.SIGTERM)

parsl/tests/test_serialization/test_basic.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl.serialize import serialize, deserialize
+
+from parsl.serialize import deserialize, serialize
 from parsl.serialize.concretes import DillSerializer, PickleSerializer
 
 

parsl/tests/test_serialization/test_htex_code_cache.py

@@ -1,10 +1,9 @@
-import parsl
-import pytest
-
 from typing import Any
 
-from parsl.serialize.facade import methods_for_code
+import pytest
 
+import parsl
+from parsl.serialize.facade import methods_for_code
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_serialization/test_pack_resource_spec.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl.serialize import unpack_res_spec_apply_message, pack_res_spec_apply_message
+
+from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
 
 
 def double(x: int, y: int = 2) -> int:

parsl/tests/test_serialization/test_proxystore_configured.py

@@ -1,19 +1,24 @@
 import logging
-import pytest
 import uuid
 
+import pytest
+
 import parsl
-from parsl.serialize.facade import additional_methods_for_deserialization, methods_for_data, register_method_for_data
+from parsl.serialize.facade import (
+    additional_methods_for_deserialization,
+    methods_for_data,
+    register_method_for_data,
+)
 from parsl.tests.configs.htex_local import fresh_config
 
-
 logger = logging.getLogger(__name__)
 
 
 def local_setup():
-    from parsl.serialize.proxystore import ProxyStoreSerializer
-    from proxystore.store import Store, register_store
     from proxystore.connectors.file import FileConnector
+    from proxystore.store import Store, register_store
+
+    from parsl.serialize.proxystore import ProxyStoreSerializer
 
     parsl.load(fresh_config())
 
@@ -38,7 +43,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
     methods_for_data.clear()
     methods_for_data.update(previous_methods)

parsl/tests/test_serialization/test_proxystore_impl.py

@@ -1,6 +1,7 @@
-import pytest
 import uuid
 
+import pytest
+
 
 def policy_example(o):
     """Example policy will proxy only lists."""
@@ -13,10 +14,11 @@ def test_proxystore_nonglobal():
     """
     # import in function, because proxystore is not importable in base parsl
     # installation.
-    from parsl.serialize.proxystore import ProxyStoreSerializer
+    from proxystore.connectors.file import FileConnector
     from proxystore.proxy import Proxy
     from proxystore.store import Store, register_store
-    from proxystore.connectors.file import FileConnector
+
+    from parsl.serialize.proxystore import ProxyStoreSerializer
 
     store = Store(name='parsl_store_' + str(uuid.uuid4()), connector=FileConnector(store_dir="/tmp"))
     register_store(store)

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -0,0 +1,64 @@
+import os
+import signal
+import time
+
+import pytest
+
+import parsl
+from parsl.tests.configs.htex_local_alternate import fresh_config
+
+# This is a very generous upper bound on expected shutdown time of target
+# process after receiving a signal, measured in seconds.
+PERMITTED_SHUTDOWN_TIME_S = 60
+
+
+@parsl.python_app
+def simple_app():
+    return True
+
+
+@pytest.mark.local
+def test_no_kills():
+    """This tests that we can create a monitoring-enabled DFK and shut it down."""
+
+    parsl.load(fresh_config())
+
+    assert parsl.dfk().monitoring is not None, "This test requires monitoring"
+
+    parsl.dfk().cleanup()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM, signal.SIGKILL, signal.SIGQUIT])
+@pytest.mark.parametrize("process_attr", ["router_proc", "dbm_proc"])
+def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
+    """This tests that we can kill a monitoring process and still have successful shutdown.
+    SIGINT emulates some racy behaviour when ctrl-C is pressed: that
+    monitoring processes receive a ctrl-C too, and so the other processes
+    need to be tolerant to monitoring processes arbitrarily exiting.
+    """
+
+    parsl.load(fresh_config())
+
+    dfk = parsl.dfk()
+
+    assert dfk.monitoring is not None, "Monitoring required"
+
+    target_proc = getattr(dfk.monitoring, process_attr)
+
+    assert target_proc is not None, "prereq: target process must exist"
+    assert target_proc.is_alive(), "prereq: target process must be alive"
+
+    target_pid = target_proc.pid
+    assert target_pid is not None, "prereq: target process must have a pid"
+
+    os.kill(target_pid, sig)
+
+    try_assert(lambda: not target_proc.is_alive(), timeout_ms=PERMITTED_SHUTDOWN_TIME_S * 1000)
+
+    # now we have broken one piece of the monitoring system, do some app
+    # execution and then shut down.
+
+    simple_app().result()
+
+    parsl.dfk().cleanup()

parsl/tests/test_staging/staging_provider.py

@@ -62,8 +62,8 @@ def make_stage_out_app(executor, dfk):
 
 
 def stage_out_noop(app_fu, inputs=[], _parsl_staging_inhibit=True):
-    import time
     import logging
+    import time
     logger = logging.getLogger(__name__)
     logger.info("stage_out_noop")
     time.sleep(1)
@@ -75,8 +75,8 @@ def make_stage_in_app(executor, dfk):
 
 
 def stage_in_noop(parent_fut=None, outputs=[], _parsl_staging_inhibit=True):
-    import time
     import logging
+    import time
     logger = logging.getLogger(__name__)
     logger.info("stage_in_noop")
     time.sleep(1)

parsl/tests/test_staging/test_1316.py

@@ -1,8 +1,9 @@
+import time
+
 import pytest
 
 import parsl
-import time
-from parsl import python_app, ThreadPoolExecutor
+from parsl import ThreadPoolExecutor, python_app
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.data_provider.staging import Staging
@@ -60,7 +61,6 @@ def test_1316_local_path_on_execution_side_sp2():
     assert not file.local_path, "The local_path on the submit side should not be set"
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local
@@ -83,4 +83,3 @@ def test_1316_local_path_setting_preserves_dependency_sp2():
     assert not file.local_path, "The local_path on the submit side should not be set"
 
     parsl.dfk().cleanup()
-    parsl.clear()