parsl 2024.10.14__py3-none-any.whl → 2024.10.28__py3-none-any.whl

parsl/tests/test_htex/test_drain.py

@@ -13,7 +13,9 @@ from parsl.providers import LocalProvider
 # based around the expected drain period: the drain period
 # is TIME_CONST seconds, and the single executed task will
 # last twice that many number of seconds.
-TIME_CONST = 1
+TIME_CONST = 4
+
+CONNECTED_MANAGERS_POLL_MS = 100
 
 
 def local_config():
@@ -52,7 +54,7 @@ def test_drain(try_assert):
 
     # wait till we have a block running...
 
-    try_assert(lambda: len(htex.connected_managers()) == 1)
+    try_assert(lambda: len(htex.connected_managers()) == 1, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
 
     managers = htex.connected_managers()
     assert managers[0]['active'], "The manager should be active"
@@ -63,7 +65,7 @@ def test_drain(try_assert):
     time.sleep(TIME_CONST)
 
     # this assert should happen *very fast* after the above delay...
-    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500)
+    try_assert(lambda: htex.connected_managers()[0]['draining'], timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)
 
     # and the test task should still be running...
     assert not fut.done(), "The test task should still be running"
@@ -76,4 +78,4 @@ def test_drain(try_assert):
     # connected managers.
     # As with the above draining assert, this should happen very fast after
     # the task ends.
-    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500)
+    try_assert(lambda: len(htex.connected_managers()) == 0, timeout_ms=500, check_period_ms=CONNECTED_MANAGERS_POLL_MS)

parsl/tests/test_htex/test_manager_selector_by_block.py

@@ -0,0 +1,53 @@
+import time
+
+import pytest
+
+import parsl
+from parsl.app.app import bash_app, python_app
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.executors.high_throughput.manager_selector import (
+    BlockIdManagerSelector,
+    ManagerSelector,
+)
+from parsl.launchers import WrappedLauncher
+from parsl.providers import LocalProvider
+from parsl.usage_tracking.levels import LEVEL_1
+
+BLOCK_COUNT = 2
+
+
+@parsl.python_app
+def get_worker_pid():
+    import os
+    return os.environ.get('PARSL_WORKER_BLOCK_ID')
+
+
+@pytest.mark.local
+def test_block_id_selection(try_assert):
+    htex = HighThroughputExecutor(
+        label="htex_local",
+        max_workers_per_node=1,
+        manager_selector=BlockIdManagerSelector(),
+        provider=LocalProvider(
+            channel=LocalChannel(),
+            init_blocks=BLOCK_COUNT,
+            max_blocks=BLOCK_COUNT,
+            min_blocks=BLOCK_COUNT,
+        ),
+    )
+
+    config = Config(
+        executors=[htex],
+        usage_tracking=LEVEL_1,
+    )
+
+    with parsl.load(config):
+        blockids = []
+        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT, timeout_ms=20000)
+        for i in range(10):
+            future = get_worker_pid()
+            blockids.append(future.result())
+
+        assert all(blockid == "1" for blockid in blockids)

parsl/tests/test_htex/test_resource_spec_validation.py

@@ -30,6 +30,13 @@ def test_resource_spec_validation():
     assert ret_val is None
 
 
+@pytest.mark.local
+def test_resource_spec_validation_one_key():
+    htex = HighThroughputExecutor()
+    ret_val = htex.validate_resource_spec({"priority": 2})
+    assert ret_val is None
+
+
 @pytest.mark.local
 def test_resource_spec_validation_bad_keys():
     htex = HighThroughputExecutor()

parsl/tests/test_providers/test_kubernetes_provider.py

@@ -0,0 +1,102 @@
+import re
+from unittest import mock
+
+import pytest
+
+from parsl.providers.kubernetes.kube import KubernetesProvider
+from parsl.tests.test_utils.test_sanitize_dns import DNS_SUBDOMAIN_REGEX
+
+_MOCK_BASE = "parsl.providers.kubernetes.kube"
+
+
+@pytest.fixture(autouse=True)
+def mock_kube_config():
+    with mock.patch(f"{_MOCK_BASE}.config") as mock_config:
+        mock_config.load_kube_config.return_value = None
+        yield mock_config
+
+
+@pytest.fixture
+def mock_kube_client():
+    mock_client = mock.MagicMock()
+    with mock.patch(f"{_MOCK_BASE}.client.CoreV1Api") as mock_api:
+        mock_api.return_value = mock_client
+        yield mock_client
+
+
+@pytest.mark.local
+def test_submit_happy_path(mock_kube_client: mock.MagicMock):
+    image = "test-image"
+    namespace = "test-namespace"
+    cmd_string = "test-command"
+    volumes = [("test-volume", "test-mount-path")]
+    service_account_name = "test-service-account"
+    annotations = {"test-annotation": "test-value"}
+    max_cpu = 2
+    max_mem = "2Gi"
+    init_cpu = 1
+    init_mem = "1Gi"
+    provider = KubernetesProvider(
+        image=image,
+        persistent_volumes=volumes,
+        namespace=namespace,
+        service_account_name=service_account_name,
+        annotations=annotations,
+        max_cpu=max_cpu,
+        max_mem=max_mem,
+        init_cpu=init_cpu,
+        init_mem=init_mem,
+    )
+
+    job_name = "test.job.name"
+    job_id = provider.submit(cmd_string=cmd_string, tasks_per_node=1, job_name=job_name)
+
+    assert job_id in provider.resources
+    assert mock_kube_client.create_namespaced_pod.call_count == 1
+
+    call_args = mock_kube_client.create_namespaced_pod.call_args[1]
+    pod = call_args["body"]
+    container = pod.spec.containers[0]
+    volume = container.volume_mounts[0]
+
+    assert image == container.image
+    assert namespace == call_args["namespace"]
+    assert any(cmd_string in arg for arg in container.args)
+    assert volumes[0] == (volume.name, volume.mount_path)
+    assert service_account_name == pod.spec.service_account_name
+    assert annotations == pod.metadata.annotations
+    assert str(max_cpu) == container.resources.limits["cpu"]
+    assert max_mem == container.resources.limits["memory"]
+    assert str(init_cpu) == container.resources.requests["cpu"]
+    assert init_mem == container.resources.requests["memory"]
+    assert job_id == pod.metadata.labels["parsl-job-id"]
+    assert job_id == container.name
+    assert f"{job_name}.{job_id}" == pod.metadata.name
+
+
+@pytest.mark.local
+@mock.patch(f"{_MOCK_BASE}.KubernetesProvider._create_pod")
+@pytest.mark.parametrize("char", (".", "-"))
+def test_submit_pod_name_includes_job_id(mock_create_pod: mock.MagicMock, char: str):
+    provider = KubernetesProvider(image="test-image")
+
+    job_name = "a." * 121 + f"a{char}" + "a" * 9
+    assert len(job_name) == 253  # Max length for pod name
+    job_id = provider.submit(cmd_string="test-command", tasks_per_node=1, job_name=job_name)
+
+    expected_pod_name = job_name[:253 - len(job_id) - 2] + f".{job_id}"
+    actual_pod_name = mock_create_pod.call_args[1]["pod_name"]
+    assert re.match(DNS_SUBDOMAIN_REGEX, actual_pod_name)
+    assert expected_pod_name == actual_pod_name
+
+
+@pytest.mark.local
+@mock.patch(f"{_MOCK_BASE}.KubernetesProvider._create_pod")
+@mock.patch(f"{_MOCK_BASE}.logger")
+@pytest.mark.parametrize("job_name", ("", ".", "-", "a.-.a", "$$$"))
+def test_submit_invalid_job_name(mock_logger: mock.MagicMock, mock_create_pod: mock.MagicMock, job_name: str):
+    provider = KubernetesProvider(image="test-image")
+    job_id = provider.submit(cmd_string="test-command", tasks_per_node=1, job_name=job_name)
+    assert mock_logger.warning.call_count == 1
+    assert f"Invalid pod name '{job_name}' for job '{job_id}'" in mock_logger.warning.call_args[0][0]
+    assert f"parsl.kube.{job_id}" == mock_create_pod.call_args[1]["pod_name"]

parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py

@@ -0,0 +1,92 @@
+import os
+import signal
+import time
+
+import pytest
+import zmq
+
+import parsl
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+
+T_s = 1
+
+
+def fresh_config():
+    htex = HighThroughputExecutor(
+               heartbeat_period=1 * T_s,
+               heartbeat_threshold=3 * T_s,
+               label="htex_local",
+               worker_debug=True,
+               cores_per_worker=1,
+               encrypted=False,
+               provider=LocalProvider(
+                   channel=LocalChannel(),
+                   init_blocks=0,
+                   min_blocks=0,
+                   max_blocks=0,
+                   launcher=SimpleLauncher(),
+               ),
+           )
+    c = Config(
+        executors=[htex],
+        strategy='none',
+        strategy_period=0.5,
+    )
+    return c, htex
+
+
+@parsl.python_app
+def app():
+    return 7
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("msg",
+                         (b'FuzzyByte\rSTREAM',  # not JSON
+                          b'{}',  # missing fields
+                          b'{"type":"heartbeat"}',  # regression test #3262
+                          )
+                         )
+def test_bad_messages(try_assert, msg):
+    """This tests that the interchange is resilient to a few different bad
+    messages: malformed messages caused by implementation errors, and
+    heartbeat messages from managers that are not registered.
+
+    The heartbeat test is a regression test for issues #3262, #3632
+    """
+
+    c, htex = fresh_config()
+
+    with parsl.load(c):
+
+        # send a bad message into the interchange on the task_outgoing worker
+        # channel, and then check that the interchange is still alive enough
+        # that we can scale out a block and run a task.
+
+        (task_port, result_port) = htex.command_client.run("WORKER_PORTS")
+
+        context = zmq.Context()
+        channel_timeout = 10000  # in milliseconds
+        task_channel = context.socket(zmq.DEALER)
+        task_channel.setsockopt(zmq.LINGER, 0)
+        task_channel.setsockopt(zmq.IDENTITY, b'testid')
+
+        task_channel.set_hwm(0)
+        task_channel.setsockopt(zmq.SNDTIMEO, channel_timeout)
+        task_channel.connect(f"tcp://localhost:{task_port}")
+
+        task_channel.send(msg)
+
+        # If the interchange exits, it's likely that this test will hang rather
+        # than raise an error, because the interchange interaction code
+        # assumes the interchange is always there.
+        # In the case of issue #3262, an exception message goes to stderr, and
+        # no error goes to the interchange log file.
+        htex.scale_out_facade(1)
+        try_assert(lambda: len(htex.connected_managers()) == 1, timeout_ms=10000)
+
+        assert app().result() == 7

parsl/tests/test_serialization/test_3495_deserialize_managerlost.py

@@ -32,7 +32,7 @@ def test_manager_lost_system_failure(tmpd_cwd):
         cores_per_worker=1,
         worker_logdir_root=str(tmpd_cwd),
         heartbeat_period=1,
-        heartbeat_threshold=1,
+        heartbeat_threshold=3,
     )
     c = Config(executors=[hte], strategy='simple', strategy_period=0.1)
 

parsl/tests/test_utils/test_sanitize_dns.py

@@ -0,0 +1,76 @@
+import random
+import re
+
+import pytest
+
+from parsl.utils import sanitize_dns_label_rfc1123, sanitize_dns_subdomain_rfc1123
+
+# Ref: https://datatracker.ietf.org/doc/html/rfc1123
+DNS_LABEL_REGEX = r'^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$'
+DNS_SUBDOMAIN_REGEX = r'^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?(\.[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?)*$'
+
+test_labels = [
+    "example-label-123",           # Valid label
+    "EXAMPLE",                     # Case sensitivity
+    "!@#example*",                 # Remove invalid characters
+    "--leading-and-trailing--",    # Leading and trailing hyphens
+    "..leading.and.trailing..",    # Leading and tailing dots
+    "multiple..dots",              # Consecutive dots
+    "valid--label",                # Consecutive hyphens
+    "a" * random.randint(64, 70),  # Longer than 63 characters
+    f"{'a' * 62}-a",               # Trailing hyphen at max length
+]
+
+
+def _generate_test_subdomains(num_subdomains: int):
+    subdomains = []
+    for _ in range(num_subdomains):
+        num_labels = random.randint(1, 5)
+        labels = [test_labels[random.randint(0, num_labels - 1)] for _ in range(num_labels)]
+        subdomain = ".".join(labels)
+        subdomains.append(subdomain)
+    return subdomains
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", test_labels)
+def test_sanitize_dns_label_rfc1123(raw_string: str):
+    print(sanitize_dns_label_rfc1123(raw_string))
+    assert re.match(DNS_LABEL_REGEX, sanitize_dns_label_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", ("", "-", "@", "$$$"))
+def test_sanitize_dns_label_rfc1123_empty(raw_string: str):
+    with pytest.raises(ValueError) as e_info:
+        sanitize_dns_label_rfc1123(raw_string)
+    assert str(e_info.value) == f"Sanitized DNS label is empty for input '{raw_string}'"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", _generate_test_subdomains(10))
+def test_sanitize_dns_subdomain_rfc1123(raw_string: str):
+    assert re.match(DNS_SUBDOMAIN_REGEX, sanitize_dns_subdomain_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("char", ("-", "."))
+def test_sanitize_dns_subdomain_rfc1123_trailing_non_alphanumeric_at_max_length(char: str):
+    raw_string = (f"{'a' * 61}." * 4) + f".aaaa{char}a"
+    assert re.match(DNS_SUBDOMAIN_REGEX, sanitize_dns_subdomain_rfc1123(raw_string))
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("raw_string", ("", ".", "..."))
+def test_sanitize_dns_subdomain_rfc1123_empty(raw_string: str):
+    with pytest.raises(ValueError) as e_info:
+        sanitize_dns_subdomain_rfc1123(raw_string)
+    assert str(e_info.value) == f"Sanitized DNS subdomain is empty for input '{raw_string}'"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "raw_string", ("a" * 253, "a" * random.randint(254, 300)), ids=("254 chars", ">253 chars")
+)
+def test_sanitize_dns_subdomain_rfc1123_max_length(raw_string: str):
+    assert len(sanitize_dns_subdomain_rfc1123(raw_string)) <= 253

parsl/utils.py

@@ -1,6 +1,7 @@
 import inspect
 import logging
 import os
+import re
 import shlex
 import subprocess
 import threading
@@ -380,3 +381,80 @@ class AutoCancelTimer(threading.Timer):
         exc_tb: Optional[TracebackType]
     ) -> None:
         self.cancel()
+
+
+def sanitize_dns_label_rfc1123(raw_string: str) -> str:
+    """Convert input string to a valid RFC 1123 DNS label.
+
+    Parameters
+    ----------
+    raw_string : str
+        String to sanitize.
+
+    Returns
+    -------
+    str
+        Sanitized string.
+
+    Raises
+    ------
+    ValueError
+        If the string is empty after sanitization.
+    """
+    # Convert to lowercase and replace non-alphanumeric characters with hyphen
+    sanitized = re.sub(r'[^a-z0-9]', '-', raw_string.lower())
+
+    # Remove consecutive hyphens
+    sanitized = re.sub(r'-+', '-', sanitized)
+
+    # DNS label cannot exceed 63 characters
+    sanitized = sanitized[:63]
+
+    # Strip after trimming to avoid trailing hyphens
+    sanitized = sanitized.strip("-")
+
+    if not sanitized:
+        raise ValueError(f"Sanitized DNS label is empty for input '{raw_string}'")
+
+    return sanitized
+
+
+def sanitize_dns_subdomain_rfc1123(raw_string: str) -> str:
+    """Convert input string to a valid RFC 1123 DNS subdomain.
+
+    Parameters
+    ----------
+    raw_string : str
+        String to sanitize.
+
+    Returns
+    -------
+    str
+        Sanitized string.
+
+    Raises
+    ------
+    ValueError
+        If the string is empty after sanitization.
+    """
+    segments = raw_string.split('.')
+
+    sanitized_segments = []
+    for segment in segments:
+        if not segment:
+            continue
+        sanitized_segment = sanitize_dns_label_rfc1123(segment)
+        sanitized_segments.append(sanitized_segment)
+
+    sanitized = '.'.join(sanitized_segments)
+
+    # DNS subdomain cannot exceed 253 characters
+    sanitized = sanitized[:253]
+
+    # Strip after trimming to avoid trailing dots or hyphens
+    sanitized = sanitized.strip(".-")
+
+    if not sanitized:
+        raise ValueError(f"Sanitized DNS subdomain is empty for input '{raw_string}'")
+
+    return sanitized

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.10.14'
+VERSION = '2024.10.28'

{parsl-2024.10.14.data → parsl-2024.10.28.data}/scripts/interchange.py

@@ -66,7 +66,7 @@ class Interchange:
              If specified the interchange will only listen on this address for connections from workers
              else, it binds to all addresses.
 
-        client_ports : triple(int, int, int)
+        client_ports : tuple(int, int, int)
              The ports at which the client can be reached
 
         worker_ports : tuple(int, int)
@@ -104,7 +104,6 @@ class Interchange:
         os.makedirs(self.logdir, exist_ok=True)
 
         start_file_logger("{}/interchange.log".format(self.logdir), level=logging_level)
-        logger.propagate = False
         logger.debug("Initializing Interchange process")
 
         self.client_address = client_address
@@ -437,9 +436,13 @@ class Interchange:
                     logger.info(f"Manager {manager_id!r} has compatible Parsl version {msg['parsl_v']}")
                     logger.info(f"Manager {manager_id!r} has compatible Python version {msg['python_v'].rsplit('.', 1)[0]}")
             elif msg['type'] == 'heartbeat':
-                self._ready_managers[manager_id]['last_heartbeat'] = time.time()
-                logger.debug("Manager %r sent heartbeat via tasks connection", manager_id)
-                self.task_outgoing.send_multipart([manager_id, b'', PKL_HEARTBEAT_CODE])
+                manager = self._ready_managers.get(manager_id)
+                if manager:
+                    manager['last_heartbeat'] = time.time()
+                    logger.debug("Manager %r sent heartbeat via tasks connection", manager_id)
+                    self.task_outgoing.send_multipart([manager_id, b'', PKL_HEARTBEAT_CODE])
+                else:
+                    logger.warning("Received heartbeat via tasks connection for not-registered manager %r", manager_id)
             elif msg['type'] == 'drain':
                 self._ready_managers[manager_id]['draining'] = True
                 logger.debug("Manager %r requested drain", manager_id)

{parsl-2024.10.14.data → parsl-2024.10.28.data}/scripts/process_worker_pool.py

@@ -362,7 +362,7 @@ class Manager:
                 if tasks == HEARTBEAT_CODE:
                     logger.debug("Got heartbeat from interchange")
                 elif tasks == DRAINED_CODE:
-                    logger.info("Got fulled drained message from interchange - setting kill flag")
+                    logger.info("Got fully drained message from interchange - setting kill flag")
                     kill_event.set()
                 else:
                     task_recv_counter += len(tasks)
@@ -650,14 +650,6 @@ def worker(
     debug: bool,
     mpi_launcher: str,
 ):
-    """
-
-    Put request token into queue
-    Get task from task_queue
-    Pop request from queue
-    Put result into result_queue
-    """
-
     # override the global logger inherited from the __main__ process (which
     # usually logs to manager.log) with one specific to this worker.
     global logger

{parsl-2024.10.14.dist-info → parsl-2024.10.28.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.10.14
+Version: 2024.10.28
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.10.14.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.10.28.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0