parsl 2024.3.11__py3-none-any.whl → 2024.3.25__py3-none-any.whl

parsl/monitoring/router.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import os
+import socket
+import time
+import pickle
+import logging
+import zmq
+
+import queue
+
+from parsl.log_utils import set_file_logger
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
+
+from parsl.monitoring.message_type import MessageType
+from parsl.monitoring.types import AddressedMonitoringMessage, TaggedMonitoringMessage
+from typing import Optional, Tuple, Union
+
+
+logger = logging.getLogger(__name__)
+
+
+class MonitoringRouter:
+
+    def __init__(self,
+                 *,
+                 hub_address: str,
+                 udp_port: Optional[int] = None,
+                 zmq_port_range: Tuple[int, int] = (55050, 56000),
+
+                 monitoring_hub_address: str = "127.0.0.1",
+                 logdir: str = ".",
+                 run_id: str,
+                 logging_level: int = logging.INFO,
+                 atexit_timeout: int = 3    # in seconds
+                 ):
+        """ Initializes a monitoring configuration class.
+
+        Parameters
+        ----------
+        hub_address : str
+             The ip address at which the workers will be able to reach the Hub.
+        udp_port : int
+             The specific port at which workers will be able to reach the Hub via UDP. Default: None
+        zmq_port_range : tuple(int, int)
+             The MonitoringHub picks ports at random from the range which will be used by Hub.
+             Default: (55050, 56000)
+        logdir : str
+             Parsl log directory paths. Logs and temp files go here. Default: '.'
+        logging_level : int
+             Logging level as defined in the logging module. Default: logging.INFO
+        atexit_timeout : float, optional
+            The amount of time in seconds to terminate the hub without receiving any messages, after the last dfk workflow message is received.
+
+        """
+        os.makedirs(logdir, exist_ok=True)
+        self.logger = set_file_logger("{}/monitoring_router.log".format(logdir),
+                                      name="monitoring_router",
+                                      level=logging_level)
+        self.logger.debug("Monitoring router starting")
+
+        self.hub_address = hub_address
+        self.atexit_timeout = atexit_timeout
+        self.run_id = run_id
+
+        self.loop_freq = 10.0  # milliseconds
+
+        # Initialize the UDP socket
+        self.udp_sock = socket.socket(socket.AF_INET,
+                                      socket.SOCK_DGRAM,
+                                      socket.IPPROTO_UDP)
+
+        # We are trying to bind to all interfaces with 0.0.0.0
+        if not udp_port:
+            self.udp_sock.bind(('0.0.0.0', 0))
+            self.udp_port = self.udp_sock.getsockname()[1]
+        else:
+            self.udp_port = udp_port
+            try:
+                self.udp_sock.bind(('0.0.0.0', self.udp_port))
+            except Exception as e:
+                raise RuntimeError(f"Could not bind to udp_port {udp_port} because: {e}")
+        self.udp_sock.settimeout(self.loop_freq / 1000)
+        self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
+
+        self._context = zmq.Context()
+        self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
+        self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
+        self.zmq_receiver_channel.set_hwm(0)
+        self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
+        self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
+        self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
+                                                                               min_port=zmq_port_range[0],
+                                                                               max_port=zmq_port_range[1])
+
+    def start(self,
+              priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              node_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              block_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              resource_msgs: "queue.Queue[AddressedMonitoringMessage]") -> None:
+        try:
+            router_keep_going = True
+            while router_keep_going:
+                try:
+                    data, addr = self.udp_sock.recvfrom(2048)
+                    resource_msg = pickle.loads(data)
+                    self.logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
+                    resource_msgs.put((resource_msg, addr))
+                except socket.timeout:
+                    pass
+
+                try:
+                    dfk_loop_start = time.time()
+                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
+                        # note that nothing checks that msg really is of the annotated type
+                        msg: TaggedMonitoringMessage
+                        msg = self.zmq_receiver_channel.recv_pyobj()
+
+                        assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
+                        assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
+                        assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
+
+                        msg_0: AddressedMonitoringMessage
+                        msg_0 = (msg, 0)
+
+                        if msg[0] == MessageType.NODE_INFO:
+                            msg[1]['run_id'] = self.run_id
+                            node_msgs.put(msg_0)
+                        elif msg[0] == MessageType.RESOURCE_INFO:
+                            resource_msgs.put(msg_0)
+                        elif msg[0] == MessageType.BLOCK_INFO:
+                            block_msgs.put(msg_0)
+                        elif msg[0] == MessageType.TASK_INFO:
+                            priority_msgs.put(msg_0)
+                        elif msg[0] == MessageType.WORKFLOW_INFO:
+                            priority_msgs.put(msg_0)
+                            if 'exit_now' in msg[1] and msg[1]['exit_now']:
+                                router_keep_going = False
+                        else:
+                            # There is a type: ignore here because if msg[0]
+                            # is of the correct type, this code is unreachable,
+                            # but there is no verification that the message
+                            # received from zmq_receiver_channel.recv_pyobj() is actually
+                            # of that type.
+                            self.logger.error("Discarding message "  # type: ignore[unreachable]
+                                              f"from interchange with unknown type {msg[0].value}")
+                except zmq.Again:
+                    pass
+                except Exception:
+                    # This will catch malformed messages. What happens if the
+                    # channel is broken in such a way that it always raises
+                    # an exception? Looping on this would maybe be the wrong
+                    # thing to do.
+                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
+
+            self.logger.info("Monitoring router draining")
+            last_msg_received_time = time.time()
+            while time.time() - last_msg_received_time < self.atexit_timeout:
+                try:
+                    data, addr = self.udp_sock.recvfrom(2048)
+                    msg = pickle.loads(data)
+                    self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
+                    resource_msgs.put((msg, addr))
+                    last_msg_received_time = time.time()
+                except socket.timeout:
+                    pass
+
+            self.logger.info("Monitoring router finishing normally")
+        finally:
+            self.logger.info("Monitoring router finished")
+
+
+@wrap_with_logs
+def router_starter(comm_q: "queue.Queue[Union[Tuple[int, int], str]]",
+                   exception_q: "queue.Queue[Tuple[str, str]]",
+                   priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   node_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   block_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   resource_msgs: "queue.Queue[AddressedMonitoringMessage]",
+
+                   hub_address: str,
+                   udp_port: Optional[int],
+                   zmq_port_range: Tuple[int, int],
+
+                   logdir: str,
+                   logging_level: int,
+                   run_id: str) -> None:
+    setproctitle("parsl: monitoring router")
+    try:
+        router = MonitoringRouter(hub_address=hub_address,
+                                  udp_port=udp_port,
+                                  zmq_port_range=zmq_port_range,
+                                  logdir=logdir,
+                                  logging_level=logging_level,
+                                  run_id=run_id)
+    except Exception as e:
+        logger.error("MonitoringRouter construction failed.", exc_info=True)
+        comm_q.put(f"Monitoring router construction failed: {e}")
+    else:
+        comm_q.put((router.udp_port, router.zmq_receiver_port))
+
+        router.logger.info("Starting MonitoringRouter in router_starter")
+        try:
+            router.start(priority_msgs, node_msgs, block_msgs, resource_msgs)
+        except Exception as e:
+            router.logger.exception("router.start exception")
+            exception_q.put(('Hub', str(e)))

parsl/monitoring/visualization/plots/default/workflow_plots.py

@@ -27,6 +27,9 @@ gantt_colors = {'unsched': 'rgb(240, 240, 240)',
 
 def task_gantt_plot(df_task, df_status, time_completed=None):
 
+    if df_task.empty:
+        return None
+
     # if the workflow is not recorded as completed, then assume
     # that tasks should continue in their last state until now,
     # rather than the workflow end time.

parsl/monitoring/visualization/views.py

@@ -8,7 +8,8 @@ from parsl.monitoring.visualization.models import Workflow, Task, Status, db
 
 from parsl.monitoring.visualization.plots.default.workflow_plots import task_gantt_plot, task_per_app_plot, workflow_dag_plot
 from parsl.monitoring.visualization.plots.default.task_plots import time_series_memory_per_task_plot
-from parsl.monitoring.visualization.plots.default.workflow_resource_plots import resource_distribution_plot, resource_efficiency, worker_efficiency
+from parsl.monitoring.visualization.plots.default.workflow_resource_plots import (resource_distribution_plot,
+                                                                                  resource_efficiency, worker_efficiency)
 
 dummy = True
 

parsl/providers/cluster_provider.py

@@ -91,7 +91,7 @@ class ClusterProvider(ExecutionProvider):
               - configs (dict) : configs that get pushed into the template
 
         Returns:
-              - True: on success
+              - None
 
         Raises:
               SchedulerMissingArgs : If template is missing args
@@ -117,8 +117,6 @@ class ClusterProvider(ExecutionProvider):
             logger.error("Uncategorized error: %s", e)
             raise e
 
-        return True
-
     @abstractmethod
     def _status(self):
         pass

parsl/tests/configs/user_opts.py

@@ -60,7 +60,8 @@ user_opts = {
     #     'username': OSG_USERNAME,
     #     'script_dir': '/home/{}/parsl_scripts'.format(OSG_USERNAME),
     #     'scheduler_options': "",
-    #     'worker_init' : 'module load python/3.5.2; python3 -m venv parsl_env; source parsl_env/bin/activate; python3 -m pip install parsl==0.5.2'
+    #     'worker_init' : 'module load python/3.5.2; python3 -m venv parsl_env;
+    #                      source parsl_env/bin/activate; python3 -m pip install parsl==0.5.2'
     # },
     # 'swan': {
     #     'username': SWAN_USERNAME,

parsl/tests/test_htex/test_drain.py

@@ -0,0 +1,78 @@
+import parsl
+import pytest
+import time
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+# this constant is used to scale some durations that happen
+# based around the expected drain period: the drain period
+# is TIME_CONST seconds, and the single executed task will
+# last twice that many number of seconds.
+TIME_CONST = 1
+
+
+def local_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                drain_period=TIME_CONST,
+                worker_debug=True,
+                cores_per_worker=1,
+                encrypted=True,
+                provider=LocalProvider(
+                    channel=LocalChannel(),
+                    init_blocks=1,
+                    min_blocks=0,
+                    max_blocks=0,
+                    launcher=SimpleLauncher(),
+                ),
+            )
+        ],
+        strategy='none',
+    )
+
+
+@parsl.python_app
+def f(n):
+    import time
+    time.sleep(n)
+
+
+@pytest.mark.local
+def test_drain(try_assert):
+
+    htex = parsl.dfk().executors['htex_local']
+
+    # wait till we have a block running...
+
+    try_assert(lambda: len(htex.connected_managers()) == 1)
+
+    managers = htex.connected_managers()
+    assert managers[0]['active'], "The manager should be active"
+    assert not managers[0]['draining'], "The manager should not be draining"
+
+    fut = f(TIME_CONST * 2)
+
+    time.sleep(TIME_CONST)
+
+    # this assert should happen *very fast* after the above delay...
+    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500)
+
+    # and the test task should still be running...
+    assert not fut.done(), "The test task should still be running"
+
+    fut.result()
+
+    # and now we should see the manager disappear...
+    # ... with strategy='none', this should be coming from draining but
+    # that information isn't immediately obvious from the absence in
+    # connected managers.
+    # As with the above draining assert, this should happen very fast after
+    # the task ends.
+    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500)

parsl/tests/test_monitoring/test_app_names.py

@@ -0,0 +1,86 @@
+"""Tests monitoring records app name under various decoration patterns.
+"""
+
+import os
+import parsl
+import pytest
+import time
+
+from parsl.tests.configs.htex_local_alternate import fresh_config
+
+
+@parsl.python_app
+def regular_decorated_app():
+    return 5
+
+
+@pytest.mark.local
+def get_regular_decorated_app():
+    return regular_decorated_app
+
+
+def for_decoration_later():
+    return 77
+
+
+def get_for_decoration_later():
+    return parsl.python_app(for_decoration_later)
+
+
+def get_decorated_closure():
+
+    r = 53
+
+    @parsl.python_app
+    def decorated_closure():
+        return r
+
+    return decorated_closure
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("get_app,expected_name,expected_result",
+                         [(get_regular_decorated_app, "regular_decorated_app", 5),
+                          (get_for_decoration_later, "for_decoration_later", 77),
+                          (get_decorated_closure, "decorated_closure", 53)
+                          ])
+def test_app_name(get_app, expected_name, expected_result, tmpd_cwd):
+
+    # this is imported here rather than at module level because
+    # it isn't available in a plain parsl install, so this module
+    # would otherwise fail to import and break even a basic test
+    # run.
+    import sqlalchemy
+
+    c = fresh_config()
+    c.run_dir = tmpd_cwd
+    c.monitoring.logging_endpoint = f"sqlite:///{tmpd_cwd}/monitoring.db"
+    parsl.load(c)
+
+    app = get_app()
+    assert app().result() == expected_result
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+    engine = sqlalchemy.create_engine(c.monitoring.logging_endpoint)
+    with engine.begin() as connection:
+
+        def count_rows(table: str):
+            result = connection.execute(f"SELECT COUNT(*) FROM {table}")
+            (c, ) = result.first()
+            return c
+
+        # one workflow...
+        assert count_rows("workflow") == 1
+
+        # ... with one task ...
+        assert count_rows("task") == 1
+
+        # ... that was tried once ...
+        assert count_rows("try") == 1
+
+        # ... and has the expected name.
+        result = connection.execute("SELECT task_func_name FROM task")
+        (c, ) = result.first()
+        assert c == expected_name

parsl/tests/test_monitoring/test_fuzz_zmq.py

@@ -41,11 +41,11 @@ def test_row_counts():
 
     # dig out the interchange port...
     hub_address = parsl.dfk().hub_address
-    hub_interchange_port = parsl.dfk().hub_interchange_port
+    hub_zmq_port = parsl.dfk().hub_zmq_port
 
     # this will send a string to a new socket connection
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.connect((hub_address, hub_interchange_port))
+        s.connect((hub_address, hub_zmq_port))
         s.sendall(b'fuzzing\r')
 
     # this will send a non-object down the DFK's existing ZMQ connection

parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py

@@ -0,0 +1,82 @@
+import logging
+import os
+import parsl
+import pytest
+import time
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.monitoring import MonitoringHub
+
+
+def fresh_config(run_dir, strategy, db_url):
+    return Config(
+        run_dir=os.fspath(run_dir),
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                cores_per_worker=1,
+                encrypted=True,
+                provider=LocalProvider(
+                    channel=LocalChannel(),
+                    init_blocks=1,
+                    # min and max are set to 0 to ensure that we don't get
+                    # a block from ongoing strategy scaling, only from
+                    # init_blocks
+                    min_blocks=0,
+                    max_blocks=0,
+                    launcher=SimpleLauncher(),
+                ),
+            )
+        ],
+        strategy=strategy,
+        strategy_period=0.1,
+        monitoring=MonitoringHub(
+                        hub_address="localhost",
+                        hub_port=55055,
+                        logging_endpoint=db_url
+        )
+    )
+
+
+@parsl.python_app
+def this_app():
+    pass
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("strategy", ('none', 'simple', 'htex_auto_scale'))
+def test_row_counts(tmpd_cwd, strategy):
+    # this is imported here rather than at module level because
+    # it isn't available in a plain parsl install, so this module
+    # would otherwise fail to import and break even a basic test
+    # run.
+    import sqlalchemy
+    from sqlalchemy import text
+
+    db_url = f"sqlite:///{tmpd_cwd}/monitoring.db"
+    parsl.load(fresh_config(tmpd_cwd, strategy, db_url))
+
+    this_app().result()
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+    engine = sqlalchemy.create_engine(db_url)
+    with engine.begin() as connection:
+
+        result = connection.execute(text("SELECT COUNT(DISTINCT block_id) FROM block"))
+        (c, ) = result.first()
+        assert c == 1, "We should see a single block in this database"
+
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'PENDING'"))
+        (c, ) = result.first()
+        assert c == 1, "There should be a single pending status"
+
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'CANCELLED'"))
+        (c, ) = result.first()
+        assert c == 1, "There should be a single cancelled status"

parsl/tests/test_python_apps/test_context_manager.py

@@ -0,0 +1,40 @@
+import parsl
+from parsl.tests.configs.local_threads import fresh_config
+import pytest
+from parsl.errors import NoDataFlowKernelError
+
+
+@parsl.python_app
+def square(x):
+    return x * x
+
+
+@parsl.bash_app
+def foo(x, stdout='foo.stdout'):
+    return f"echo {x + 1}"
+
+
+def local_setup():
+    pass
+
+
+def local_teardown():
+    parsl.clear()
+
+
+@pytest.mark.local
+def test_within_context_manger():
+    config = fresh_config()
+    with parsl.load(config=config):
+        py_future = square(2)
+        assert py_future.result() == 4
+
+        bash_future = foo(1)
+        assert bash_future.result() == 0
+
+        with open('foo.stdout', 'r') as f:
+            assert f.read() == "2\n"
+
+    with pytest.raises(NoDataFlowKernelError) as excinfo:
+        square(2).result()
+    assert str(excinfo.value) == "Cannot submit to a DFK that has been cleaned up"

parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py

@@ -37,6 +37,7 @@ def local_config():
         ],
         max_idletime=0.5,
         strategy='htex_auto_scale',
+        strategy_period=0.1
     )
 
 
@@ -62,16 +63,6 @@ def waiting_app(ident: int, outputs=(), inputs=()):
 def test_scale_out(tmpd_cwd, try_assert):
     dfk = parsl.dfk()
 
-    # reconfigure scaling strategy to run faster than usual. This allows
-    # this test to complete faster - at time of writing 27s with default
-    # 5s strategy, vs XXXX with 0.5s strategy.
-
-    # check this attribute still exists, in the presence of ongoing
-    # development, so we have some belief that setting it will not be
-    # setting a now-ignored parameter.
-    assert hasattr(dfk.job_status_poller, 'interval')
-    dfk.job_status_poller.interval = 0.1
-
     num_managers = len(dfk.executors['htex_local'].connected_managers())
 
     assert num_managers == 0, "Expected 0 managers at start"

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -0,0 +1,65 @@
+import os
+import parsl
+import pytest
+import signal
+import time
+
+from parsl.tests.configs.htex_local_alternate import fresh_config
+
+# This is a very generous upper bound on expected shutdown time of target
+# process after receiving a signal, measured in seconds.
+PERMITTED_SHUTDOWN_TIME_S = 60
+
+
+@parsl.python_app
+def simple_app():
+    return True
+
+
+@pytest.mark.local
+def test_no_kills():
+    """This tests that we can create a monitoring-enabled DFK and shut it down."""
+
+    parsl.load(fresh_config())
+
+    assert parsl.dfk().monitoring is not None, "This test requires monitoring"
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM, signal.SIGKILL, signal.SIGQUIT])
+@pytest.mark.parametrize("process_attr", ["router_proc", "dbm_proc"])
+def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
+    """This tests that we can kill a monitoring process and still have successful shutdown.
+    SIGINT emulates some racy behaviour when ctrl-C is pressed: that
+    monitoring processes receive a ctrl-C too, and so the other processes
+    need to be tolerant to monitoring processes arbitrarily exiting.
+    """
+
+    parsl.load(fresh_config())
+
+    dfk = parsl.dfk()
+
+    assert dfk.monitoring is not None, "Monitoring required"
+
+    target_proc = getattr(dfk.monitoring, process_attr)
+
+    assert target_proc is not None, "prereq: target process must exist"
+    assert target_proc.is_alive(), "prereq: target process must be alive"
+
+    target_pid = target_proc.pid
+    assert target_pid is not None, "prereq: target process must have a pid"
+
+    os.kill(target_pid, sig)
+
+    try_assert(lambda: not target_proc.is_alive(), timeout_ms=PERMITTED_SHUTDOWN_TIME_S * 1000)
+
+    # now we have broken one piece of the monitoring system, do some app
+    # execution and then shut down.
+
+    simple_app().result()
+
+    parsl.dfk().cleanup()
+    parsl.clear()

parsl/utils.py

@@ -296,12 +296,12 @@ class Timer:
 
     """
 
-    def __init__(self, callback: Callable, *args: Any, interval: int = 5, name: Optional[str] = None) -> None:
+    def __init__(self, callback: Callable, *args: Any, interval: Union[float, int] = 5, name: Optional[str] = None) -> None:
         """Initialize the Timer object.
         We start the timer thread here
 
         KWargs:
-             - interval (int) : number of seconds between callback events
+             - interval (int or float) : number of seconds between callback events
              - name (str) : a base name to use when naming the started thread
         """
 

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.03.11'
+VERSION = '2024.03.25'