parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_htex/test_drain.py

@@ -1,19 +1,20 @@
-import parsl
-import pytest
 import time
 
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
+import pytest
 
+import parsl
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 # this constant is used to scale some durations that happen
 # based around the expected drain period: the drain period
 # is TIME_CONST seconds, and the single executed task will
 # last twice that many number of seconds.
-TIME_CONST = 1
+TIME_CONST = 4
+
+CONNECTED_MANAGERS_POLL_MS = 100
 
 
 def local_config():
@@ -26,7 +27,6 @@ def local_config():
                 cores_per_worker=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     min_blocks=0,
                     max_blocks=0,
@@ -35,6 +35,7 @@ def local_config():
             )
         ],
         strategy='none',
+        strategy_period=0.1
     )
 
 
@@ -51,7 +52,7 @@ def test_drain(try_assert):
 
     # wait till we have a block running...
 
-    try_assert(lambda: len(htex.connected_managers()) == 1)
+    try_assert(lambda: len(htex.connected_managers()) == 1, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
 
     managers = htex.connected_managers()
     assert managers[0]['active'], "The manager should be active"
@@ -62,7 +63,7 @@ def test_drain(try_assert):
     time.sleep(TIME_CONST)
 
     # this assert should happen *very fast* after the above delay...
-    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500)
+    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
 
     # and the test task should still be running...
     assert not fut.done(), "The test task should still be running"
@@ -75,4 +76,4 @@ def test_drain(try_assert):
     # connected managers.
     # As with the above draining assert, this should happen very fast after
     # the task ends.
-    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500)
+    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)

parsl/tests/test_htex/test_htex.py

@@ -1,12 +1,12 @@
+import logging
 import pathlib
-import warnings
+from subprocess import Popen, TimeoutExpired
+from typing import Optional, Sequence
 from unittest import mock
 
 import pytest
 
-from parsl import curvezmq
-from parsl import HighThroughputExecutor
-from parsl.multiprocessing import ForkProcess
+from parsl import HighThroughputExecutor, curvezmq
 
 _MOCK_BASE = "parsl.executors.high_throughput.executor"
 
@@ -72,51 +72,77 @@ def test_htex_start_encrypted(
 @pytest.mark.local
 @pytest.mark.parametrize("started", (True, False))
 @pytest.mark.parametrize("timeout_expires", (True, False))
-@mock.patch(f"{_MOCK_BASE}.logger")
 def test_htex_shutdown(
-    mock_logger: mock.MagicMock,
     started: bool,
     timeout_expires: bool,
     htex: HighThroughputExecutor,
+    caplog
 ):
-    mock_ix_proc = mock.Mock(spec=ForkProcess)
+    mock_ix_proc = mock.Mock(spec=Popen)
 
     if started:
         htex.interchange_proc = mock_ix_proc
-        mock_ix_proc.is_alive.return_value = True
+
+    # This will, in the absence of any exit trigger, block forever if
+    # no timeout is given and if the interchange does not terminate.
+    # Raise an exception to report that, rather than actually block,
+    # and hope that nothing is catching that exception.
+
+    # this function implements the behaviour if the interchange has
+    # not received a termination call
+    def proc_wait_alive(timeout):
+        if timeout:
+            raise TimeoutExpired(cmd="mock-interchange", timeout=timeout)
+        else:
+            raise RuntimeError("This wait call would hang forever")
+
+    def proc_wait_terminated(timeout):
+        return 0
+
+    mock_ix_proc.wait.side_effect = proc_wait_alive
 
     if not timeout_expires:
         # Simulate termination of the Interchange process
         def kill_interchange(*args, **kwargs):
-            mock_ix_proc.is_alive.return_value = False
+            mock_ix_proc.wait.side_effect = proc_wait_terminated
 
         mock_ix_proc.terminate.side_effect = kill_interchange
 
-    htex.shutdown()
+    with caplog.at_level(logging.INFO):
+        htex.shutdown()
 
-    mock_logs = mock_logger.info.call_args_list
     if started:
         assert mock_ix_proc.terminate.called
-        assert mock_ix_proc.join.called
-        assert {"timeout": 10} == mock_ix_proc.join.call_args[1]
+        assert mock_ix_proc.wait.called
+        assert {"timeout": 10} == mock_ix_proc.wait.call_args[1]
         if timeout_expires:
-            assert "Unable to terminate Interchange" in mock_logs[1][0][0]
+            assert "Unable to terminate Interchange" in caplog.text
             assert mock_ix_proc.kill.called
-        assert "Attempting" in mock_logs[0][0][0]
-        assert "Finished" in mock_logs[-1][0][0]
+        assert "Attempting HighThroughputExecutor shutdown" in caplog.text
+        assert "Finished HighThroughputExecutor shutdown" in caplog.text
     else:
         assert not mock_ix_proc.terminate.called
-        assert not mock_ix_proc.join.called
-        assert "has not started" in mock_logs[0][0][0]
+        assert not mock_ix_proc.wait.called
+        assert "HighThroughputExecutor has not started" in caplog.text
 
 
 @pytest.mark.local
-def test_max_workers_per_node():
-    with pytest.warns(DeprecationWarning) as record:
-        htex = HighThroughputExecutor(max_workers_per_node=1, max_workers=2)
+@pytest.mark.parametrize("cmd", (None, "custom-launch-cmd"))
+def test_htex_worker_pool_launch_cmd(cmd: Optional[str]):
+    if cmd:
+        htex = HighThroughputExecutor(launch_cmd=cmd)
+        assert htex.launch_cmd == cmd
+    else:
+        htex = HighThroughputExecutor()
+        assert htex.launch_cmd.startswith("process_worker_pool.py")
 
-    warning_msg = "max_workers is deprecated"
-    assert any(warning_msg in str(warning.message) for warning in record)
 
-    # Ensure max_workers_per_node takes precedence
-    assert htex.max_workers_per_node == htex.max_workers == 1
+@pytest.mark.local
+@pytest.mark.parametrize("cmd", (None, ["custom", "launch", "cmd"]))
+def test_htex_interchange_launch_cmd(cmd: Optional[Sequence[str]]):
+    if cmd:
+        htex = HighThroughputExecutor(interchange_launch_cmd=cmd)
+        assert htex.interchange_launch_cmd == cmd
+    else:
+        htex = HighThroughputExecutor()
+        assert htex.interchange_launch_cmd == ["interchange.py"]

parsl/tests/test_htex/test_manager_failure.py

@@ -20,7 +20,6 @@ def load_config():
     yield
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app

parsl/tests/test_htex/test_manager_selector_by_block.py

@@ -0,0 +1,51 @@
+import time
+
+import pytest
+
+import parsl
+from parsl.app.app import bash_app, python_app
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.executors.high_throughput.manager_selector import (
+    BlockIdManagerSelector,
+    ManagerSelector,
+)
+from parsl.launchers import WrappedLauncher
+from parsl.providers import LocalProvider
+from parsl.usage_tracking.levels import LEVEL_1
+
+BLOCK_COUNT = 2
+
+
+@parsl.python_app
+def get_worker_pid():
+    import os
+    return os.environ.get('PARSL_WORKER_BLOCK_ID')
+
+
+@pytest.mark.local
+def test_block_id_selection(try_assert):
+    htex = HighThroughputExecutor(
+        label="htex_local",
+        max_workers_per_node=1,
+        manager_selector=BlockIdManagerSelector(),
+        provider=LocalProvider(
+            init_blocks=BLOCK_COUNT,
+            max_blocks=BLOCK_COUNT,
+            min_blocks=BLOCK_COUNT,
+        ),
+    )
+
+    config = Config(
+        executors=[htex],
+        usage_tracking=LEVEL_1,
+    )
+
+    with parsl.load(config):
+        blockids = []
+        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT, timeout_ms=20000)
+        for i in range(10):
+            future = get_worker_pid()
+            blockids.append(future.result())
+
+        assert all(blockid == "1" for blockid in blockids)

parsl/tests/test_htex/test_managers_command.py

@@ -0,0 +1,36 @@
+import logging
+import sys
+
+import pytest
+
+import parsl
+from parsl.app.app import python_app
+from parsl.tests.configs.htex_local import fresh_config
+
+
+def local_config():
+    config = fresh_config()
+    config.executors[0].poll_period = 1
+    config.executors[0].max_workers_per_node = 1
+    return config
+
+
+@python_app
+def dummy():
+    pass
+
+
+@pytest.mark.local
+def test_connected_managers():
+
+    # Run dummy function to ensure a manager is online
+    x = dummy()
+    assert x.result() is None
+    executor = parsl.dfk().executors['htex_local']
+    manager_info_list = executor.connected_managers()
+    assert len(manager_info_list) == 1
+    manager_info = manager_info_list[0]
+    assert 'python_version' in manager_info
+    assert 'parsl_version' in manager_info
+    assert manager_info['parsl_version'] == parsl.__version__
+    assert manager_info['python_version'] == f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"

parsl/tests/test_htex/test_missing_worker.py

@@ -5,18 +5,12 @@ from parsl.app.app import python_app
 from parsl.tests.configs.htex_local import fresh_config
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].poll_period = 1
     config.executors[0].max_workers_per_node = 1
     config.executors[0].launch_cmd = "executable_that_hopefully_does_not_exist_1030509.py"
-    parsl.load(config)
-
-
-def local_teardown():
-
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app
@@ -37,7 +31,3 @@ def test_that_it_fails():
         raise Exception("The app somehow ran without a valid worker")
 
     assert parsl.dfk().config.executors[0]._executor_bad_state.is_set()
-
-    # htex needs shutting down explicitly because dfk.cleanup() will not
-    # do that, as it is in bad state
-    parsl.dfk().config.executors[0].shutdown()

parsl/tests/test_htex/test_multiple_disconnected_blocks.py

@@ -1,11 +1,13 @@
 import logging
-import parsl
+
 import pytest
-from parsl.executors import HighThroughputExecutor
+
+import parsl
 from parsl import Config
-from parsl.providers import LocalProvider
+from parsl.executors import HighThroughputExecutor
 from parsl.executors.errors import BadStateException
-from parsl.jobs.states import JobStatus, JobState
+from parsl.jobs.states import JobState, JobStatus
+from parsl.providers import LocalProvider
 
 
 def local_config():
@@ -19,16 +21,14 @@ def local_config():
                 poll_period=100,
                 max_workers_per_node=1,
                 provider=LocalProvider(
-                    worker_init="conda deactivate; export PATH=''; which python; exit 0",
-                    init_blocks=2,
-                    max_blocks=4,
-                    min_blocks=0,
+                    worker_init="exit 0",
+                    init_blocks=2
                 ),
             )
         ],
         run_dir="/tmp/test_htex",
         max_idletime=0.5,
-        strategy='htex_auto_scale',
+        strategy='none',
     )
 
 

parsl/tests/test_htex/test_resource_spec_validation.py

@@ -0,0 +1,45 @@
+import queue
+from unittest import mock
+
+import pytest
+
+from parsl.executors import HighThroughputExecutor
+from parsl.executors.errors import InvalidResourceSpecification
+
+
+def double(x):
+    return x * 2
+
+
+@pytest.mark.local
+def test_submit_calls_validate():
+
+    htex = HighThroughputExecutor()
+    htex.outgoing_q = mock.Mock(spec=queue.Queue)
+    htex.validate_resource_spec = mock.Mock(spec=htex.validate_resource_spec)
+
+    res_spec = {}
+    htex.submit(double, res_spec, (5,), {})
+    htex.validate_resource_spec.assert_called()
+
+
+@pytest.mark.local
+def test_resource_spec_validation():
+    htex = HighThroughputExecutor()
+    ret_val = htex.validate_resource_spec({})
+    assert ret_val is None
+
+
+@pytest.mark.local
+def test_resource_spec_validation_one_key():
+    htex = HighThroughputExecutor()
+    ret_val = htex.validate_resource_spec({"priority": 2})
+    assert ret_val is None
+
+
+@pytest.mark.local
+def test_resource_spec_validation_bad_keys():
+    htex = HighThroughputExecutor()
+
+    with pytest.raises(InvalidResourceSpecification):
+        htex.validate_resource_spec({"num_nodes": 2})

parsl/tests/test_htex/test_zmq_binding.py

@@ -1,3 +1,4 @@
+import logging
 import pathlib
 from typing import Optional
 from unittest import mock
@@ -8,6 +9,24 @@ import zmq
 
 from parsl import curvezmq
 from parsl.executors.high_throughput.interchange import Interchange
+from parsl.executors.high_throughput.manager_selector import RandomManagerSelector
+
+
+def make_interchange(*, interchange_address: Optional[str], cert_dir: Optional[str]) -> Interchange:
+    return Interchange(interchange_address=interchange_address,
+                       cert_dir=cert_dir,
+                       client_address="127.0.0.1",
+                       client_ports=(50055, 50056, 50057),
+                       worker_ports=None,
+                       worker_port_range=(54000, 55000),
+                       hub_address=None,
+                       hub_zmq_port=None,
+                       heartbeat_threshold=60,
+                       logdir=".",
+                       logging_level=logging.INFO,
+                       manager_selector=RandomManagerSelector(),
+                       poll_period=10,
+                       run_id="test_run_id")
 
 
 @pytest.fixture
@@ -31,7 +50,7 @@ def test_interchange_curvezmq_sockets(
     mock_socket: mock.MagicMock, cert_dir: Optional[str], encrypted: bool
 ):
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=address, cert_dir=cert_dir)
     assert isinstance(ix.zmq_context, curvezmq.ServerContext)
     assert ix.zmq_context.encrypted is encrypted
     assert mock_socket.call_count == 5
@@ -40,7 +59,7 @@ def test_interchange_curvezmq_sockets(
 @pytest.mark.local
 @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 def test_interchange_binding_no_address(cert_dir: Optional[str]):
-    ix = Interchange(cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=None, cert_dir=cert_dir)
     assert ix.interchange_address == "*"
 
 
@@ -49,17 +68,18 @@ def test_interchange_binding_no_address(cert_dir: Optional[str]):
 def test_interchange_binding_with_address(cert_dir: Optional[str]):
     # Using loopback address
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=address, cert_dir=cert_dir)
     assert ix.interchange_address == address
 
 
+@pytest.mark.skip("This behaviour is possibly unexpected. See issue #3037")
 @pytest.mark.local
 @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):
     # Confirm that a ipv4 address is required
     address = "localhost"
     with pytest.raises(zmq.error.ZMQError):
-        Interchange(interchange_address=address, cert_dir=cert_dir)
+        make_interchange(interchange_address=address, cert_dir=cert_dir)
 
 
 @pytest.mark.local
@@ -67,8 +87,8 @@ def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):
 def test_interchange_binding_bad_address(cert_dir: Optional[str]):
     """Confirm that we raise a ZMQError when a bad address is supplied"""
     address = "550.0.0.0"
-    with pytest.raises(zmq.error.ZMQError):
-        Interchange(interchange_address=address, cert_dir=cert_dir)
+    with pytest.raises(ValueError):
+        make_interchange(interchange_address=address, cert_dir=cert_dir)
 
 
 @pytest.mark.local
@@ -76,11 +96,12 @@ def test_interchange_binding_bad_address(cert_dir: Optional[str]):
 def test_limited_interface_binding(cert_dir: Optional[str]):
     """When address is specified the worker_port would be bound to it rather than to 0.0.0.0"""
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
+    ix = make_interchange(interchange_address=address, cert_dir=cert_dir)
     ix.worker_result_port
     proc = psutil.Process()
     conns = proc.connections(kind="tcp")
 
     matched_conns = [conn for conn in conns if conn.laddr.port == ix.worker_result_port]
     assert len(matched_conns) == 1
-    assert matched_conns[0].laddr.ip == address
+    # laddr.ip can return ::ffff:127.0.0.1 when using IPv6
+    assert address in matched_conns[0].laddr.ip

parsl/tests/test_monitoring/test_app_names.py

@@ -2,10 +2,11 @@
 """
 
 import os
-import parsl
-import pytest
 import time
 
+import pytest
+
+import parsl
 from parsl.tests.configs.htex_local_alternate import fresh_config
 
 
@@ -61,13 +62,12 @@ def test_app_name(get_app, expected_name, expected_result, tmpd_cwd):
     assert app().result() == expected_result
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
     engine = sqlalchemy.create_engine(c.monitoring.logging_endpoint)
     with engine.begin() as connection:
 
         def count_rows(table: str):
-            result = connection.execute(f"SELECT COUNT(*) FROM {table}")
+            result = connection.execute(sqlalchemy.text(f"SELECT COUNT(*) FROM {table}"))
             (c, ) = result.first()
             return c
 
@@ -81,6 +81,6 @@ def test_app_name(get_app, expected_name, expected_result, tmpd_cwd):
         assert count_rows("try") == 1
 
         # ... and has the expected name.
-        result = connection.execute("SELECT task_func_name FROM task")
+        result = connection.execute(sqlalchemy.text("SELECT task_func_name FROM task"))
         (c, ) = result.first()
         assert c == expected_name

parsl/tests/test_monitoring/test_basic.py

@@ -1,10 +1,13 @@
-import logging
 import os
-import parsl
-import pytest
 import time
 
-logger = logging.getLogger(__name__)
+import pytest
+
+import parsl
+from parsl import HighThroughputExecutor
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
+from parsl.monitoring import MonitoringHub
 
 
 @parsl.python_app
@@ -18,34 +21,79 @@ def this_app():
     return 5
 
 
+# The below fresh configs are for use in parametrization, and should return
+# a configuration that is suitably configured for monitoring.
+
+def htex_config():
+    """This config will use htex's default htex-specific monitoring radio mode"""
+    from parsl.tests.configs.htex_local_alternate import fresh_config
+    return fresh_config()
+
+
+def htex_udp_config():
+    """This config will force UDP"""
+    from parsl.tests.configs.htex_local_alternate import fresh_config
+    c = fresh_config()
+    assert len(c.executors) == 1
+
+    assert c.executors[0].radio_mode == "htex", "precondition: htex has a radio mode attribute, configured for htex radio"
+    c.executors[0].radio_mode = "udp"
+
+    return c
+
+
+def htex_filesystem_config():
+    """This config will force filesystem radio"""
+    from parsl.tests.configs.htex_local_alternate import fresh_config
+    c = fresh_config()
+    assert len(c.executors) == 1
+
+    assert c.executors[0].radio_mode == "htex", "precondition: htex has a radio mode attribute, configured for htex radio"
+    c.executors[0].radio_mode = "filesystem"
+
+    return c
+
+
+def workqueue_config():
+    from parsl.tests.configs.workqueue_ex import fresh_config
+    c = fresh_config()
+    c.monitoring = MonitoringHub(
+                        hub_address="localhost",
+                        resource_monitoring_interval=1)
+    return c
+
+
+def taskvine_config():
+    c = Config(executors=[TaskVineExecutor(manager_config=TaskVineManagerConfig(port=9000),
+                                           worker_launch_method='provider')],
+
+               monitoring=MonitoringHub(hub_address="localhost",
+                                        resource_monitoring_interval=1))
+    return c
+
+
 @pytest.mark.local
-def test_row_counts():
+@pytest.mark.parametrize("fresh_config", [htex_config, htex_filesystem_config, htex_udp_config, workqueue_config, taskvine_config])
+def test_row_counts(tmpd_cwd, fresh_config):
     # this is imported here rather than at module level because
     # it isn't available in a plain parsl install, so this module
     # would otherwise fail to import and break even a basic test
     # run.
     import sqlalchemy
     from sqlalchemy import text
-    from parsl.tests.configs.htex_local_alternate import fresh_config
 
-    if os.path.exists("runinfo/monitoring.db"):
-        logger.info("Monitoring database already exists - deleting")
-        os.remove("runinfo/monitoring.db")
+    db_url = f"sqlite:///{tmpd_cwd}/monitoring.db"
 
-    logger.info("loading parsl")
-    parsl.load(fresh_config())
+    config = fresh_config()
+    config.run_dir = tmpd_cwd
+    config.monitoring.logging_endpoint = db_url
 
-    logger.info("invoking and waiting for result")
-    assert this_app().result() == 5
-
-    logger.info("cleaning up parsl")
-    parsl.dfk().cleanup()
-    parsl.clear()
+    with parsl.load(config):
+        assert this_app().result() == 5
 
     # at this point, we should find one row in the monitoring database.
 
-    logger.info("checking database content")
-    engine = sqlalchemy.create_engine("sqlite:///runinfo/monitoring.db")
+    engine = sqlalchemy.create_engine(db_url)
     with engine.begin() as connection:
 
         result = connection.execute(text("SELECT COUNT(*) FROM workflow"))
@@ -67,10 +115,12 @@ def test_row_counts():
         (c, ) = result.first()
         assert c == 0
 
-        # Two entries: one showing manager active, one inactive
-        result = connection.execute(text("SELECT COUNT(*) FROM node"))
-        (c, ) = result.first()
-        assert c == 2
+        if isinstance(config.executors[0], HighThroughputExecutor):
+            # The node table is specific to the HighThroughputExecutor
+            # Two entries: one showing manager active, one inactive
+            result = connection.execute(text("SELECT COUNT(*) FROM node"))
+            (c, ) = result.first()
+            assert c == 2
 
         # There should be one block polling status
         # local provider has a status_polling_interval of 5s
@@ -81,5 +131,3 @@ def test_row_counts():
         result = connection.execute(text("SELECT COUNT(*) FROM resource"))
         (c, ) = result.first()
         assert c >= 1
-
-    logger.info("all done")