parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_htex/test_drain.py

@@ -0,0 +1,79 @@
+import time
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+
+# this constant is used to scale some durations that happen
+# based around the expected drain period: the drain period
+# is TIME_CONST seconds, and the single executed task will
+# last twice that many number of seconds.
+TIME_CONST = 4
+
+CONNECTED_MANAGERS_POLL_MS = 100
+
+
+def local_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                drain_period=TIME_CONST,
+                worker_debug=True,
+                cores_per_worker=1,
+                encrypted=True,
+                provider=LocalProvider(
+                    init_blocks=1,
+                    min_blocks=0,
+                    max_blocks=0,
+                    launcher=SimpleLauncher(),
+                ),
+            )
+        ],
+        strategy='none',
+        strategy_period=0.1
+    )
+
+
+@parsl.python_app
+def f(n):
+    import time
+    time.sleep(n)
+
+
+@pytest.mark.local
+def test_drain(try_assert):
+
+    htex = parsl.dfk().executors['htex_local']
+
+    # wait till we have a block running...
+
+    try_assert(lambda: len(htex.connected_managers()) == 1, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
+
+    managers = htex.connected_managers()
+    assert managers[0]['active'], "The manager should be active"
+    assert not managers[0]['draining'], "The manager should not be draining"
+
+    fut = f(TIME_CONST * 2)
+
+    time.sleep(TIME_CONST)
+
+    # this assert should happen *very fast* after the above delay...
+    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
+
+    # and the test task should still be running...
+    assert not fut.done(), "The test task should still be running"
+
+    fut.result()
+
+    # and now we should see the manager disappear...
+    # ... with strategy='none', this should be coming from draining but
+    # that information isn't immediately obvious from the absence in
+    # connected managers.
+    # As with the above draining assert, this should happen very fast after
+    # the task ends.
+    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)

parsl/tests/test_htex/test_htex.py

@@ -1,12 +1,12 @@
+import logging
 import pathlib
-import warnings
+from subprocess import Popen, TimeoutExpired
+from typing import Optional, Sequence
 from unittest import mock
 
 import pytest
 
-from parsl import curvezmq
-from parsl import HighThroughputExecutor
-from parsl.multiprocessing import ForkProcess
+from parsl import HighThroughputExecutor, curvezmq
 
 _MOCK_BASE = "parsl.executors.high_throughput.executor"
 
@@ -72,51 +72,77 @@ def test_htex_start_encrypted(
 @pytest.mark.local
 @pytest.mark.parametrize("started", (True, False))
 @pytest.mark.parametrize("timeout_expires", (True, False))
-@mock.patch(f"{_MOCK_BASE}.logger")
 def test_htex_shutdown(
-    mock_logger: mock.MagicMock,
     started: bool,
     timeout_expires: bool,
     htex: HighThroughputExecutor,
+    caplog
 ):
-    mock_ix_proc = mock.Mock(spec=ForkProcess)
+    mock_ix_proc = mock.Mock(spec=Popen)
 
     if started:
         htex.interchange_proc = mock_ix_proc
-        mock_ix_proc.is_alive.return_value = True
+
+    # This will, in the absence of any exit trigger, block forever if
+    # no timeout is given and if the interchange does not terminate.
+    # Raise an exception to report that, rather than actually block,
+    # and hope that nothing is catching that exception.
+
+    # this function implements the behaviour if the interchange has
+    # not received a termination call
+    def proc_wait_alive(timeout):
+        if timeout:
+            raise TimeoutExpired(cmd="mock-interchange", timeout=timeout)
+        else:
+            raise RuntimeError("This wait call would hang forever")
+
+    def proc_wait_terminated(timeout):
+        return 0
+
+    mock_ix_proc.wait.side_effect = proc_wait_alive
 
     if not timeout_expires:
         # Simulate termination of the Interchange process
         def kill_interchange(*args, **kwargs):
-            mock_ix_proc.is_alive.return_value = False
+            mock_ix_proc.wait.side_effect = proc_wait_terminated
 
         mock_ix_proc.terminate.side_effect = kill_interchange
 
-    htex.shutdown()
+    with caplog.at_level(logging.INFO):
+        htex.shutdown()
 
-    mock_logs = mock_logger.info.call_args_list
     if started:
         assert mock_ix_proc.terminate.called
-        assert mock_ix_proc.join.called
-        assert {"timeout": 10} == mock_ix_proc.join.call_args[1]
+        assert mock_ix_proc.wait.called
+        assert {"timeout": 10} == mock_ix_proc.wait.call_args[1]
         if timeout_expires:
-            assert "Unable to terminate Interchange" in mock_logs[1][0][0]
+            assert "Unable to terminate Interchange" in caplog.text
             assert mock_ix_proc.kill.called
-        assert "Attempting" in mock_logs[0][0][0]
-        assert "Finished" in mock_logs[-1][0][0]
+        assert "Attempting HighThroughputExecutor shutdown" in caplog.text
+        assert "Finished HighThroughputExecutor shutdown" in caplog.text
     else:
         assert not mock_ix_proc.terminate.called
-        assert not mock_ix_proc.join.called
-        assert "has not started" in mock_logs[0][0][0]
+        assert not mock_ix_proc.wait.called
+        assert "HighThroughputExecutor has not started" in caplog.text
 
 
 @pytest.mark.local
-def test_max_workers_per_node():
-    with pytest.warns(DeprecationWarning) as record:
-        htex = HighThroughputExecutor(max_workers_per_node=1, max_workers=2)
+@pytest.mark.parametrize("cmd", (None, "custom-launch-cmd"))
+def test_htex_worker_pool_launch_cmd(cmd: Optional[str]):
+    if cmd:
+        htex = HighThroughputExecutor(launch_cmd=cmd)
+        assert htex.launch_cmd == cmd
+    else:
+        htex = HighThroughputExecutor()
+        assert htex.launch_cmd.startswith("process_worker_pool.py")
 
-    warning_msg = "max_workers is deprecated"
-    assert any(warning_msg in str(warning.message) for warning in record)
 
-    # Ensure max_workers_per_node takes precedence
-    assert htex.max_workers_per_node == htex.max_workers == 1
+@pytest.mark.local
+@pytest.mark.parametrize("cmd", (None, ["custom", "launch", "cmd"]))
+def test_htex_interchange_launch_cmd(cmd: Optional[Sequence[str]]):
+    if cmd:
+        htex = HighThroughputExecutor(interchange_launch_cmd=cmd)
+        assert htex.interchange_launch_cmd == cmd
+    else:
+        htex = HighThroughputExecutor()
+        assert htex.interchange_launch_cmd == ["interchange.py"]

parsl/tests/test_htex/test_manager_failure.py

@@ -20,7 +20,6 @@ def load_config():
     yield
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app

parsl/tests/test_htex/test_manager_selector_by_block.py

@@ -0,0 +1,51 @@
+import time
+
+import pytest
+
+import parsl
+from parsl.app.app import bash_app, python_app
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.executors.high_throughput.manager_selector import (
+    BlockIdManagerSelector,
+    ManagerSelector,
+)
+from parsl.launchers import WrappedLauncher
+from parsl.providers import LocalProvider
+from parsl.usage_tracking.levels import LEVEL_1
+
+BLOCK_COUNT = 2
+
+
+@parsl.python_app
+def get_worker_pid():
+    import os
+    return os.environ.get('PARSL_WORKER_BLOCK_ID')
+
+
+@pytest.mark.local
+def test_block_id_selection(try_assert):
+    htex = HighThroughputExecutor(
+        label="htex_local",
+        max_workers_per_node=1,
+        manager_selector=BlockIdManagerSelector(),
+        provider=LocalProvider(
+            init_blocks=BLOCK_COUNT,
+            max_blocks=BLOCK_COUNT,
+            min_blocks=BLOCK_COUNT,
+        ),
+    )
+
+    config = Config(
+        executors=[htex],
+        usage_tracking=LEVEL_1,
+    )
+
+    with parsl.load(config):
+        blockids = []
+        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT, timeout_ms=20000)
+        for i in range(10):
+            future = get_worker_pid()
+            blockids.append(future.result())
+
+        assert all(blockid == "1" for blockid in blockids)

parsl/tests/test_htex/test_managers_command.py

@@ -0,0 +1,36 @@
+import logging
+import sys
+
+import pytest
+
+import parsl
+from parsl.app.app import python_app
+from parsl.tests.configs.htex_local import fresh_config
+
+
+def local_config():
+    config = fresh_config()
+    config.executors[0].poll_period = 1
+    config.executors[0].max_workers_per_node = 1
+    return config
+
+
+@python_app
+def dummy():
+    pass
+
+
+@pytest.mark.local
+def test_connected_managers():
+
+    # Run dummy function to ensure a manager is online
+    x = dummy()
+    assert x.result() is None
+    executor = parsl.dfk().executors['htex_local']
+    manager_info_list = executor.connected_managers()
+    assert len(manager_info_list) == 1
+    manager_info = manager_info_list[0]
+    assert 'python_version' in manager_info
+    assert 'parsl_version' in manager_info
+    assert manager_info['parsl_version'] == parsl.__version__
+    assert manager_info['python_version'] == f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"

parsl/tests/test_htex/test_missing_worker.py

@@ -5,18 +5,12 @@ from parsl.app.app import python_app
 from parsl.tests.configs.htex_local import fresh_config
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].poll_period = 1
     config.executors[0].max_workers_per_node = 1
     config.executors[0].launch_cmd = "executable_that_hopefully_does_not_exist_1030509.py"
-    parsl.load(config)
-
-
-def local_teardown():
-
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app
@@ -37,7 +31,3 @@ def test_that_it_fails():
         raise Exception("The app somehow ran without a valid worker")
 
     assert parsl.dfk().config.executors[0]._executor_bad_state.is_set()
-
-    # htex needs shutting down explicitly because dfk.cleanup() will not
-    # do that, as it is in bad state
-    parsl.dfk().config.executors[0].shutdown()

parsl/tests/test_htex/test_multiple_disconnected_blocks.py

@@ -1,11 +1,13 @@
 import logging
-import parsl
+
 import pytest
-from parsl.executors import HighThroughputExecutor
+
+import parsl
 from parsl import Config
-from parsl.providers import LocalProvider
+from parsl.executors import HighThroughputExecutor
 from parsl.executors.errors import BadStateException
-from parsl.jobs.states import JobStatus, JobState
+from parsl.jobs.states import JobState, JobStatus
+from parsl.providers import LocalProvider
 
 
 def local_config():
@@ -19,16 +21,14 @@ def local_config():
                 poll_period=100,
                 max_workers_per_node=1,
                 provider=LocalProvider(
-                    worker_init="conda deactivate; export PATH=''; which python; exit 0",
-                    init_blocks=2,
-                    max_blocks=4,
-                    min_blocks=0,
+                    worker_init="exit 0",
+                    init_blocks=2
                 ),
             )
         ],
         run_dir="/tmp/test_htex",
         max_idletime=0.5,
-        strategy='htex_auto_scale',
+        strategy='none',
     )
 
 

parsl/tests/test_htex/test_resource_spec_validation.py

@@ -0,0 +1,45 @@
+import queue
+from unittest import mock
+
+import pytest
+
+from parsl.executors import HighThroughputExecutor
+from parsl.executors.errors import InvalidResourceSpecification
+
+
+def double(x):
+    return x * 2
+
+
+@pytest.mark.local
+def test_submit_calls_validate():
+
+    htex = HighThroughputExecutor()
+    htex.outgoing_q = mock.Mock(spec=queue.Queue)
+    htex.validate_resource_spec = mock.Mock(spec=htex.validate_resource_spec)
+
+    res_spec = {}
+    htex.submit(double, res_spec, (5,), {})
+    htex.validate_resource_spec.assert_called()
+
+
+@pytest.mark.local
+def test_resource_spec_validation():
+    htex = HighThroughputExecutor()
+    ret_val = htex.validate_resource_spec({})
+    assert ret_val is None
+
+
+@pytest.mark.local
+def test_resource_spec_validation_one_key():
+    htex = HighThroughputExecutor()
+    ret_val = htex.validate_resource_spec({"priority": 2})
+    assert ret_val is None
+
+
+@pytest.mark.local
+def test_resource_spec_validation_bad_keys():
+    htex = HighThroughputExecutor()
+
+    with pytest.raises(InvalidResourceSpecification):
+        htex.validate_resource_spec({"num_nodes": 2})

parsl/tests/test_htex/test_zmq_binding.py

@@ -1,3 +1,4 @@
+import logging
 import pathlib
 from typing import Optional
 from unittest import mock
@@ -8,6 +9,24 @@ import zmq
 
 from parsl import curvezmq
 from parsl.executors.high_throughput.interchange import Interchange
+from parsl.executors.high_throughput.manager_selector import RandomManagerSelector
+
+
+def make_interchange(*, interchange_address: Optional[str], cert_dir: Optional[str]) -> Interchange:
+    return Interchange(interchange_address=interchange_address,
+                       cert_dir=cert_dir,
+                       client_address="127.0.0.1",
+                       client_ports=(50055, 50056, 50057),
+                       worker_ports=None,
+                       worker_port_range=(54000, 55000),
+                       hub_address=None,
+                       hub_zmq_port=None,
+                       heartbeat_threshold=60,
+                       logdir=".",
+                       logging_level=logging.INFO,
+                       manager_selector=RandomManagerSelector(),
+                       poll_period=10,
+                       run_id="test_run_id")
 
 
 @pytest.fixture
@@ -31,7 +50,7 @@ def test_interchange_curvezmq_sockets(
     mock_socket: mock.MagicMock, cert_dir: Optional[str], encrypted: bool
 ):
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=address, cert_dir=cert_dir)
     assert isinstance(ix.zmq_context, curvezmq.ServerContext)
     assert ix.zmq_context.encrypted is encrypted
     assert mock_socket.call_count == 5
@@ -40,7 +59,7 @@ def test_interchange_curvezmq_sockets(
 @pytest.mark.local
 @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 def test_interchange_binding_no_address(cert_dir: Optional[str]):
-    ix = Interchange(cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=None, cert_dir=cert_dir)
     assert ix.interchange_address == "*"
 
 
@@ -49,17 +68,18 @@ def test_interchange_binding_no_address(cert_dir: Optional[str]):
 def test_interchange_binding_with_address(cert_dir: Optional[str]):
     # Using loopback address
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=address, cert_dir=cert_dir)
     assert ix.interchange_address == address
 
 
+@pytest.mark.skip("This behaviour is possibly unexpected. See issue #3037")
 @pytest.mark.local
 @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):
     # Confirm that a ipv4 address is required
     address = "localhost"
     with pytest.raises(zmq.error.ZMQError):
-        Interchange(interchange_address=address, cert_dir=cert_dir)
+        make_interchange(interchange_address=address, cert_dir=cert_dir)
 
 
 @pytest.mark.local
@@ -67,8 +87,8 @@ def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):
 def test_interchange_binding_bad_address(cert_dir: Optional[str]):
     """Confirm that we raise a ZMQError when a bad address is supplied"""
     address = "550.0.0.0"
-    with pytest.raises(zmq.error.ZMQError):
-        Interchange(interchange_address=address, cert_dir=cert_dir)
+    with pytest.raises(ValueError):
+        make_interchange(interchange_address=address, cert_dir=cert_dir)
 
 
 @pytest.mark.local
@@ -76,11 +96,12 @@ def test_interchange_binding_bad_address(cert_dir: Optional[str]):
 def test_limited_interface_binding(cert_dir: Optional[str]):
     """When address is specified the worker_port would be bound to it rather than to 0.0.0.0"""
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=address, cert_dir=cert_dir)
     ix.worker_result_port
     proc = psutil.Process()
     conns = proc.connections(kind="tcp")
 
     matched_conns = [conn for conn in conns if conn.laddr.port == ix.worker_result_port]
     assert len(matched_conns) == 1
-    assert matched_conns[0].laddr.ip == address
+    # laddr.ip can return ::ffff:127.0.0.1 when using IPv6
+    assert address in matched_conns[0].laddr.ip

parsl/tests/test_monitoring/test_app_names.py

@@ -0,0 +1,86 @@
+"""Tests monitoring records app name under various decoration patterns.
+"""
+
+import os
+import time
+
+import pytest
+
+import parsl
+from parsl.tests.configs.htex_local_alternate import fresh_config
+
+
+@parsl.python_app
+def regular_decorated_app():
+    return 5
+
+
+@pytest.mark.local
+def get_regular_decorated_app():
+    return regular_decorated_app
+
+
+def for_decoration_later():
+    return 77
+
+
+def get_for_decoration_later():
+    return parsl.python_app(for_decoration_later)
+
+
+def get_decorated_closure():
+
+    r = 53
+
+    @parsl.python_app
+    def decorated_closure():
+        return r
+
+    return decorated_closure
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("get_app,expected_name,expected_result",
+                         [(get_regular_decorated_app, "regular_decorated_app", 5),
+                          (get_for_decoration_later, "for_decoration_later", 77),
+                          (get_decorated_closure, "decorated_closure", 53)
+                          ])
+def test_app_name(get_app, expected_name, expected_result, tmpd_cwd):
+
+    # this is imported here rather than at module level because
+    # it isn't available in a plain parsl install, so this module
+    # would otherwise fail to import and break even a basic test
+    # run.
+    import sqlalchemy
+
+    c = fresh_config()
+    c.run_dir = tmpd_cwd
+    c.monitoring.logging_endpoint = f"sqlite:///{tmpd_cwd}/monitoring.db"
+    parsl.load(c)
+
+    app = get_app()
+    assert app().result() == expected_result
+
+    parsl.dfk().cleanup()
+
+    engine = sqlalchemy.create_engine(c.monitoring.logging_endpoint)
+    with engine.begin() as connection:
+
+        def count_rows(table: str):
+            result = connection.execute(sqlalchemy.text(f"SELECT COUNT(*) FROM {table}"))
+            (c, ) = result.first()
+            return c
+
+        # one workflow...
+        assert count_rows("workflow") == 1
+
+        # ... with one task ...
+        assert count_rows("task") == 1
+
+        # ... that was tried once ...
+        assert count_rows("try") == 1
+
+        # ... and has the expected name.
+        result = connection.execute(sqlalchemy.text("SELECT task_func_name FROM task"))
+        (c, ) = result.first()
+        assert c == expected_name