parsl 2024.2.12__py3-none-any.whl → 2024.2.26__py3-none-any.whl

parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -0,0 +1,145 @@
+import contextlib
+import logging
+import os
+import typing
+
+
+import pytest
+import unittest
+
+import parsl
+from parsl.app.app import python_app
+from parsl.tests.configs.htex_local import fresh_config
+from typing import Dict
+from parsl.executors.high_throughput.mpi_resource_management import (
+    get_pbs_hosts_list,
+    get_slurm_hosts_list,
+    get_nodes_in_batchjob,
+    identify_scheduler,
+)
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    validate_resource_spec,
+    InvalidResourceSpecification
+)
+
+EXECUTOR_LABEL = "MPI_TEST"
+
+
+def local_setup():
+    config = fresh_config()
+    config.executors[0].label = EXECUTOR_LABEL
+    config.executors[0].max_workers = 1
+    parsl.load(config)
+
+
+def local_teardown():
+    logging.warning("Exiting")
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+
+@python_app
+def double(x, resource_spec=None):
+    return x * 2
+
+
+@python_app
+def get_env_vars(parsl_resource_specification: Dict = {}) -> Dict:
+    import os
+
+    parsl_vars = {}
+    for key in os.environ:
+        if key.startswith("PARSL_"):
+            parsl_vars[key] = os.environ[key]
+    return parsl_vars
+
+
+@pytest.mark.local
+def test_resource_spec_env_vars():
+    resource_spec = {
+        "num_nodes": 4,
+        "ranks_per_node": 2,
+    }
+
+    assert double(5).result() == 10
+
+    future = get_env_vars(parsl_resource_specification=resource_spec)
+
+    result = future.result()
+    assert isinstance(result, Dict)
+    assert result["PARSL_NUM_NODES"] == str(resource_spec["num_nodes"])
+    assert result["PARSL_RANKS_PER_NODE"] == str(resource_spec["ranks_per_node"])
+
+
+@pytest.mark.local
+@unittest.mock.patch("subprocess.check_output", return_value=b"c203-031\nc203-032\n")
+def test_slurm_mocked_mpi_fetch(subprocess_check):
+    nodeinfo = get_slurm_hosts_list()
+    assert isinstance(nodeinfo, list)
+    assert len(nodeinfo) == 2
+
+
+@contextlib.contextmanager
+def add_to_path(path: os.PathLike) -> typing.Generator[None, None, None]:
+    old_path = os.environ["PATH"]
+    try:
+        os.environ["PATH"] += str(path)
+        yield
+    finally:
+        os.environ["PATH"] = old_path
+
+
+@pytest.mark.local
+@pytest.mark.skip
+def test_slurm_mpi_fetch():
+    logging.warning(f"Current pwd : {os.path.dirname(__file__)}")
+    with add_to_path(os.path.dirname(__file__)):
+        logging.warning(f"PATH: {os.environ['PATH']}")
+        nodeinfo = get_slurm_hosts_list()
+    logging.warning(f"Got : {nodeinfo}")
+
+
+@contextlib.contextmanager
+def mock_pbs_nodefile(nodefile: str = "pbs_nodefile") -> typing.Generator[None, None, None]:
+    cwd = os.path.abspath(os.path.dirname(__file__))
+    filename = os.path.join(cwd, "mocks", nodefile)
+    try:
+        os.environ["PBS_NODEFILE"] = filename
+        yield
+    finally:
+        del os.environ["PBS_NODEFILE"]
+
+
+@pytest.mark.local
+def test_get_pbs_hosts_list():
+    with mock_pbs_nodefile():
+        nodelist = get_pbs_hosts_list()
+        assert nodelist
+        assert len(nodelist) == 4
+
+
+@pytest.mark.local
+def test_top_level():
+    with mock_pbs_nodefile():
+        scheduler = identify_scheduler()
+        nodelist = get_nodes_in_batchjob(scheduler)
+        assert len(nodelist) > 0
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "resource_spec, exception",
+    (
+        ({"num_nodes": 2, "ranks_per_node": 1}, None),
+        ({"launcher_options": "--debug_foo"}, None),
+        ({"num_nodes": 2, "BAD_OPT": 1}, InvalidResourceSpecification),
+        ({}, None),
+    )
+)
+def test_resource_spec(resource_spec: Dict, exception):
+    if exception:
+        with pytest.raises(exception):
+            validate_resource_spec(resource_spec)
+    else:
+        result = validate_resource_spec(resource_spec)
+        assert result is None

parsl/tests/test_providers/test_cobalt_deprecation_warning.py

@@ -0,0 +1,16 @@
+import warnings
+import pytest
+from parsl.providers import CobaltProvider
+
+
+@pytest.mark.local
+def test_deprecation_warning():
+
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+
+        CobaltProvider()
+
+        assert len(w) == 1
+        assert issubclass(w[-1].category, DeprecationWarning)
+        assert "CobaltProvider" in str(w[-1].message)

parsl/tests/test_providers/test_pbspro_template.py

@@ -0,0 +1,28 @@
+import random
+
+from unittest import mock
+import pytest
+
+from parsl.channels import LocalChannel
+from parsl.providers import PBSProProvider
+
+
+@pytest.mark.local
+def test_submit_script_basic(tmp_path):
+    """Test slurm resources table"""
+
+    provider = PBSProProvider(
+        queue="debug", channel=LocalChannel(script_dir=tmp_path)
+    )
+    provider.script_dir = tmp_path
+    job_id = str(random.randint(55000, 59000))
+    provider.execute_wait = mock.Mock(spec=PBSProProvider.execute_wait)
+    provider.execute_wait.return_value = (0, job_id, "")
+    result_job_id = provider.submit("test", tasks_per_node=1)
+    assert job_id == result_job_id
+    provider.execute_wait.assert_called()
+    assert job_id in provider.resources
+
+    job_info = provider.resources[job_id]
+    assert "job_stdout_path" in job_info
+    assert "job_stderr_path" in job_info

parsl/tests/test_providers/test_slurm_template.py

@@ -0,0 +1,29 @@
+import logging
+import random
+
+from unittest import mock
+import pytest
+
+from parsl.channels import LocalChannel
+from parsl.providers import SlurmProvider
+
+
+@pytest.mark.local
+def test_submit_script_basic(tmp_path):
+    """Test slurm resources table"""
+
+    provider = SlurmProvider(
+        partition="debug", channel=LocalChannel(script_dir=tmp_path)
+    )
+    provider.script_dir = tmp_path
+    job_id = str(random.randint(55000, 59000))
+    provider.execute_wait = mock.MagicMock(spec=SlurmProvider.execute_wait)
+    provider.execute_wait.return_value = (0, f"Submitted batch job {job_id}", "")
+    result_job_id = provider.submit("test", tasks_per_node=1)
+    assert job_id == result_job_id
+    provider.execute_wait.assert_called()
+    assert job_id in provider.resources
+
+    job_info = provider.resources[job_id]
+    assert "job_stdout_path" in job_info
+    assert "job_stderr_path" in job_info

parsl/tests/test_radical/test_mpi_funcs.py

@@ -16,6 +16,7 @@ def some_mpi_func(msg, sleep, comm=None, parsl_resource_specification={}):
 apps = []
 
 
+@pytest.mark.skip("hangs in CI - waiting for resolution of issue #3029")
 @pytest.mark.local
 @pytest.mark.radical
 def test_radical_mpi(n=7):

parsl/tests/test_scaling/test_scale_down.py

@@ -44,16 +44,16 @@ def local_config():
 
 
 @python_app
-def waiting_app(ident: int, inputs=()):
+def waiting_app(ident: int, inputs=(), outputs=()):
     import pathlib
     import time
 
     # Approximate an Event by writing to files; the test logic will poll this file
-    with open(inputs[0], "a") as f:
+    with open(outputs[0], "a") as f:
         f.write(f"Ready: {ident}\n")
 
     # Similarly, use Event approximation (file check!) by polling.
-    may_finish_file = pathlib.Path(inputs[1])
+    may_finish_file = pathlib.Path(inputs[0])
     while not may_finish_file.exists():
         time.sleep(0.01)
 
@@ -74,9 +74,10 @@ def test_scale_out(tmpd_cwd, try_assert):
     ready_path = tmpd_cwd / "workers_ready"
     finish_path = tmpd_cwd / "workers_may_continue"
     ready_path.touch()
-    inputs = [File(str(ready_path)), File(str(finish_path))]
+    inputs = [File(finish_path)]
+    outputs = [File(ready_path)]
 
-    futs = [waiting_app(i, inputs=inputs) for i in range(ntasks)]
+    futs = [waiting_app(i, outputs=outputs, inputs=inputs) for i in range(ntasks)]
 
     while ready_path.read_text().count("\n") < _max_blocks:
         time.sleep(0.5)

parsl/tests/test_serialization/test_htex_code_cache.py

@@ -0,0 +1,57 @@
+import parsl
+import pytest
+
+from typing import Any
+
+from parsl.serialize.facade import methods_for_code
+
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def f(x):
+    return x + 1
+
+
+@pytest.mark.local
+def test_caching() -> None:
+    # for future serializer devs: if this is failing because you added another
+    # code serializer, you'll also probably need to re-think what is being tested
+    # about serialization caching here.
+    assert len(methods_for_code) == 1
+
+    serializer = methods_for_code[b'C2']
+
+    # force type to Any here because a serializer method coming from
+    # methods_for_code doesn't statically have any cache management
+    # methods on itself such as cache_clear or cache_info.
+    serialize_method: Any = serializer.serialize
+
+    serialize_method.cache_clear()
+
+    assert serialize_method.cache_info().hits == 0
+    assert serialize_method.cache_info().misses == 0
+    assert serialize_method.cache_info().currsize == 0
+
+    assert f(7).result() == 8
+
+    # the code serializer cache should now contain only a (probably wrapped) f ...
+    assert serialize_method.cache_info().currsize == 1
+
+    # ... which was not already in the cache.
+    assert serialize_method.cache_info().misses == 1
+    assert serialize_method.cache_info().hits == 0
+
+    assert f(100).result() == 101
+
+    # this time round, we should have got a single cache hit...
+    assert serialize_method.cache_info().hits == 1
+    assert serialize_method.cache_info().misses == 1
+    assert serialize_method.cache_info().currsize == 1
+
+    assert f(200).result() == 201
+
+    # this time round, we should have got another single cache hit...
+    assert serialize_method.cache_info().hits == 2
+    assert serialize_method.cache_info().misses == 1
+    assert serialize_method.cache_info().currsize == 1

parsl/tests/test_serialization/test_pack_resource_spec.py

@@ -0,0 +1,22 @@
+import pytest
+from parsl.serialize import unpack_res_spec_apply_message, pack_res_spec_apply_message
+
+
+def double(x: int, y: int = 2) -> int:
+    return x * y
+
+
+@pytest.mark.local
+def test_pack_and_unpack():
+    args = (5,)
+    kwargs = {'y': 10}
+    resource_spec = {'num_nodes': 4}
+    packed = pack_res_spec_apply_message(double, args, kwargs, resource_specification=resource_spec)
+
+    unpacked = unpack_res_spec_apply_message(packed)
+    assert len(unpacked) == 4
+    u_fn, u_args, u_kwargs, u_res_spec = unpacked
+    assert u_fn == double
+    assert u_args == args
+    assert u_kwargs == kwargs
+    assert u_res_spec == resource_spec

parsl/usage_tracking/usage.py

@@ -14,8 +14,13 @@ from parsl.version import VERSION as PARSL_VERSION
 
 logger = logging.getLogger(__name__)
 
+from typing import Callable
+from typing_extensions import ParamSpec
 
-def async_process(fn):
+P = ParamSpec("P")
+
+
+def async_process(fn: Callable[P, None]) -> Callable[P, None]:
     """ Decorator function to launch a function as a separate process """
 
     def run(*args, **kwargs):
@@ -27,41 +32,23 @@ def async_process(fn):
 
 
 @async_process
-def udp_messenger(domain_name, UDP_IP, UDP_PORT, sock_timeout, message):
+def udp_messenger(domain_name: str, UDP_PORT: int, sock_timeout: int, message: str) -> None:
     """Send UDP messages to usage tracker asynchronously
 
     This multiprocessing based messenger was written to overcome the limitations
-    of signalling/terminating a thread that is blocked on a system call. This
-    messenger is created as a separate process, and initialized with 2 queues,
-    to_send to receive messages to be sent to the internet.
+    of signalling/terminating a thread that is blocked on a system call.
 
     Args:
           - domain_name (str) : Domain name string
-          - UDP_IP (str) : IP address YYY.YYY.YYY.YYY
           - UDP_PORT (int) : UDP port to send out on
           - sock_timeout (int) : Socket timeout
-          - to_send (multiprocessing.Queue) : Queue of outgoing messages to internet
     """
     setproctitle("parsl: Usage tracking")
 
     try:
-        if message is None:
-            raise ValueError("message was none")
-
         encoded_message = bytes(message, "utf-8")
 
-        if encoded_message is None:
-            raise ValueError("utf-8 encoding of message failed")
-
-        if domain_name:
-            try:
-                UDP_IP = socket.gethostbyname(domain_name)
-            except Exception:
-                # (False, "Domain lookup failed, defaulting to {0}".format(UDP_IP))
-                pass
-
-        if UDP_IP is None:
-            raise Exception("UDP_IP is None")
+        UDP_IP = socket.gethostbyname(domain_name)
 
         if UDP_PORT is None:
             raise Exception("UDP_PORT is None")
@@ -88,14 +75,14 @@ class UsageTracker:
 
     """
 
-    def __init__(self, dfk, ip='52.3.111.203', port=50077,
+    def __init__(self, dfk, port=50077,
                  domain_name='tracking.parsl-project.org'):
         """Initialize usage tracking unless the user has opted-out.
 
         We will try to resolve the hostname specified in kwarg:domain_name
         and if that fails attempt to use the kwarg:ip. Determining the
-        IP and sending message is threaded to avoid slowing down DFK
-        initialization.
+        IP and sending message happens in an asynchronous processs to avoid
+        slowing down DFK initialization.
 
         Tracks usage stats by inspecting the internal state of the dfk.
 
@@ -103,18 +90,15 @@ class UsageTracker:
              - dfk (DFK object) : Data Flow Kernel object
 
         KWargs:
-             - ip (string) : IP address
              - port (int) : Port number, Default:50077
              - domain_name (string) : Domain name, will override IP
                   Default: tracking.parsl-project.org
         """
 
         self.domain_name = domain_name
-        self.ip = ip
         # The sock timeout will only apply to UDP send and not domain resolution
         self.sock_timeout = 5
         self.UDP_PORT = port
-        self.UDP_IP = None
         self.procs = []
         self.dfk = dfk
         self.config = self.dfk.config
@@ -147,16 +131,13 @@ class UsageTracker:
 
         return track
 
-    def construct_start_message(self):
+    def construct_start_message(self) -> str:
         """Collect preliminary run info at the start of the DFK.
 
         Returns :
               - Message dict dumped as json string, ready for UDP
         """
         message = {'uuid': self.uuid,
-                   'test': False,  # this field previously indicated if parsl
-                                   # was being run in test mode, and is
-                                   # retained for protocol compatibility
                    'parsl_v': self.parsl_version,
                    'python_v': self.python_version,
                    'os': platform.system(),
@@ -165,7 +146,7 @@ class UsageTracker:
 
         return json.dumps(message)
 
-    def construct_end_message(self):
+    def construct_end_message(self) -> str:
         """Collect the final run information at the time of DFK cleanup.
 
         Returns:
@@ -181,35 +162,23 @@ class UsageTracker:
                    'end': time.time(),
                    't_apps': app_count,
                    'sites': site_count,
-                   'c_time': None,
-                   'failed': app_fails,
-                   'test': False,  # see comment in construct_start_message
+                   'failed': app_fails
                    }
 
         return json.dumps(message)
 
-    def send_UDP_message(self, message):
+    def send_UDP_message(self, message: str) -> None:
         """Send UDP message."""
-        x = 0
         if self.tracking_enabled:
             try:
-                proc = udp_messenger(self.domain_name, self.UDP_IP, self.UDP_PORT, self.sock_timeout, message)
+                proc = udp_messenger(self.domain_name, self.UDP_PORT, self.sock_timeout, message)
                 self.procs.append(proc)
             except Exception as e:
                 logger.debug("Usage tracking failed: {}".format(e))
-        else:
-            x = -1
-
-        return x
 
-    def send_message(self) -> float:
+    def send_message(self) -> None:
         """Send message over UDP.
-
-        Returns:
-            time taken
         """
-        start = time.time()
-        message = None
         if not self.initialized:
             message = self.construct_start_message()
             self.initialized = True
@@ -217,11 +186,16 @@ class UsageTracker:
             message = self.construct_end_message()
 
         self.send_UDP_message(message)
-        end = time.time()
-
-        return end - start
 
-    def close(self):
-        """We terminate (SIGTERM) the processes added to the self.procs list """
+    def close(self, timeout: float = 10.0) -> None:
+        """First give each process one timeout period to finish what it is
+        doing, then kill it (SIGKILL). There's no softer SIGTERM step,
+        because that adds one join period of delay for what is almost
+        definitely either: going to behave broadly the same as to SIGKILL,
+        or won't respond to SIGTERM.
+        """
         for proc in self.procs:
-            proc.terminate()
+            proc.join(timeout=timeout)
+            if proc.is_alive():
+                logger.info("Usage tracking process did not end itself; sending SIGKILL")
+                proc.kill()

parsl/utils.py

@@ -305,59 +305,36 @@ class Timer:
              - name (str) : a base name to use when naming the started thread
         """
 
-        self.interval = interval
+        self.interval = max(0, interval)
         self.cb_args = args
         self.callback = callback
-        self._wake_up_time = time.time() + 1
 
         self._kill_event = threading.Event()
-        if name is None:
-            name = "Timer-Thread-{}".format(id(self))
-        else:
-            name = "{}-Timer-Thread-{}".format(name, id(self))
-        self._thread = threading.Thread(target=self._wake_up_timer, args=(self._kill_event,), name=name)
-        self._thread.daemon = True
+        tname = f"Timer-Thread-{id(self)}"
+        if name:
+            tname = f"{name}-{tname}"
+        self._thread = threading.Thread(
+            target=self._wake_up_timer, name=tname, daemon=True
+        )
         self._thread.start()
 
-    def _wake_up_timer(self, kill_event: threading.Event) -> None:
-        """Internal. This is the function that the thread will execute.
-        waits on an event so that the thread can make a quick exit when close() is called
-
-        Args:
-            - kill_event (threading.Event) : Event to wait on
-        """
-
-        # Sleep till time to wake up
-        while True:
-            prev = self._wake_up_time
-
-            # Waiting for the event returns True only when the event
-            # is set, usually by the parent thread
-            time_to_die = kill_event.wait(float(max(prev - time.time(), 0)))
-
-            if time_to_die:
-                return
-
-            if prev == self._wake_up_time:
-                self.make_callback()
-            else:
-                print("Sleeping a bit more")
+    def _wake_up_timer(self) -> None:
+        while not self._kill_event.wait(self.interval):
+            self.make_callback()
 
     def make_callback(self) -> None:
         """Makes the callback and resets the timer.
         """
-        self._wake_up_time = time.time() + self.interval
-
         try:
             self.callback(*self.cb_args)
         except Exception:
             logger.error("Callback threw an exception - logging and proceeding anyway", exc_info=True)
 
-    def close(self) -> None:
+    def close(self, timeout: Optional[float] = None) -> None:
         """Merge the threads and terminate.
         """
         self._kill_event.set()
-        self._thread.join()
+        self._thread.join(timeout=timeout)
 
 
 class AutoCancelTimer(threading.Timer):

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.02.12'
+VERSION = '2024.02.26'