parsl 2024.1.22__py3-none-any.whl → 2024.2.5__py3-none-any.whl

parsl/tests/test_htex/test_cpu_affinity_explicit.py

@@ -0,0 +1,52 @@
+import logging
+import os
+import parsl
+import pytest
+import random
+from parsl.tests.configs.htex_local import fresh_config
+
+logger = logging.getLogger(__name__)
+
+
+@parsl.python_app
+def my_affinity():
+    """an app that returns the affinity of the unix process it is currently in.
+    """
+    import os
+    return os.sched_getaffinity(0)
+
+
+@pytest.mark.local
+@pytest.mark.multiple_cores_required
+def test_cpu_affinity_explicit():
+    available_cores = os.sched_getaffinity(0)
+
+    logger.debug(f"Got these cores: {available_cores}")
+
+    assert len(available_cores) >= 2, "This test requires multiple cores. Run with '-k not multiple_cores' to skip"
+
+    cores_as_list = list(available_cores)
+
+    single_core = random.choice(cores_as_list)
+    affinity = f"list:{single_core}"
+
+    logger.debug(f"Will test with affinity for one worker, one core: {affinity}")
+
+    config = fresh_config()
+    config.executors[0].cpu_affinity = affinity
+    config.executors[0].max_workers = 1
+
+    logger.debug(f"config: {config}")
+    # TODO: is there a `with` style for this, to properly deal with exceptions?
+
+    parsl.load(config)
+    try:
+
+        worker_affinity = my_affinity().result()
+        logger.debug(f"worker reported this affinity: {worker_affinity}")
+        assert len(worker_affinity) == 1
+        assert worker_affinity == set((single_core,))
+
+    finally:
+        parsl.dfk().cleanup()
+        parsl.clear()

parsl/tests/test_htex/test_htex.py

@@ -0,0 +1,46 @@
+import pathlib
+
+import pytest
+
+from parsl import curvezmq
+from parsl import HighThroughputExecutor
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False))
+@pytest.mark.parametrize("cert_dir_provided", (True, False))
+def test_htex_start_encrypted(
+    encrypted: bool, cert_dir_provided: bool, tmpd_cwd: pathlib.Path
+):
+    htex = HighThroughputExecutor(encrypted=encrypted)
+    htex.run_dir = str(tmpd_cwd)
+    if cert_dir_provided:
+        provided_base_dir = tmpd_cwd / "provided"
+        provided_base_dir.mkdir()
+        cert_dir = curvezmq.create_certificates(provided_base_dir)
+        htex.cert_dir = cert_dir
+    else:
+        cert_dir = curvezmq.create_certificates(htex.logdir)
+
+    if not encrypted and cert_dir_provided:
+        with pytest.raises(AttributeError) as pyt_e:
+            htex.start()
+        assert "change cert_dir to None" in str(pyt_e.value)
+        return
+
+    htex.start()
+
+    assert htex.encrypted is encrypted
+    if encrypted:
+        assert htex.cert_dir == cert_dir
+        assert htex.outgoing_q.zmq_context.cert_dir == cert_dir
+        assert htex.incoming_q.zmq_context.cert_dir == cert_dir
+        assert htex.command_client.zmq_context.cert_dir == cert_dir
+        assert isinstance(htex.outgoing_q.zmq_context, curvezmq.ClientContext)
+        assert isinstance(htex.incoming_q.zmq_context, curvezmq.ClientContext)
+        assert isinstance(htex.command_client.zmq_context, curvezmq.ClientContext)
+    else:
+        assert htex.cert_dir is None
+        assert htex.outgoing_q.zmq_context.cert_dir is None
+        assert htex.incoming_q.zmq_context.cert_dir is None
+        assert htex.command_client.zmq_context.cert_dir is None

parsl/tests/test_htex/test_htex_zmq_binding.py

@@ -1,42 +1,82 @@
-import logging
+import pathlib
+from typing import Optional
+from unittest import mock
 
 import psutil
 import pytest
 import zmq
 
+from parsl import curvezmq
 from parsl.executors.high_throughput.interchange import Interchange
 
 
-def test_interchange_binding_no_address():
-    ix = Interchange()
+@pytest.fixture
+def encrypted(request: pytest.FixtureRequest):
+    if hasattr(request, "param"):
+        return request.param
+    return True
+
+
+@pytest.fixture
+def cert_dir(encrypted: bool, tmpd_cwd: pathlib.Path):
+    if not encrypted:
+        return None
+    return curvezmq.create_certificates(tmpd_cwd)
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+@mock.patch.object(curvezmq.ServerContext, "socket", return_value=mock.MagicMock())
+def test_interchange_curvezmq_sockets(
+    mock_socket: mock.MagicMock, cert_dir: Optional[str], encrypted: bool
+):
+    address = "127.0.0.1"
+    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
+    assert isinstance(ix.zmq_context, curvezmq.ServerContext)
+    assert ix.zmq_context.encrypted is encrypted
+    assert mock_socket.call_count == 5
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_interchange_binding_no_address(cert_dir: Optional[str]):
+    ix = Interchange(cert_dir=cert_dir)
     assert ix.interchange_address == "*"
 
 
-def test_interchange_binding_with_address():
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_interchange_binding_with_address(cert_dir: Optional[str]):
     # Using loopback address
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address)
+    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
     assert ix.interchange_address == address
 
 
-def test_interchange_binding_with_non_ipv4_address():
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):
     # Confirm that a ipv4 address is required
     address = "localhost"
     with pytest.raises(zmq.error.ZMQError):
-        Interchange(interchange_address=address)
+        Interchange(interchange_address=address, cert_dir=cert_dir)
 
 
-def test_interchange_binding_bad_address():
-    """ Confirm that we raise a ZMQError when a bad address is supplied"""
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_interchange_binding_bad_address(cert_dir: Optional[str]):
+    """Confirm that we raise a ZMQError when a bad address is supplied"""
     address = "550.0.0.0"
     with pytest.raises(zmq.error.ZMQError):
-        Interchange(interchange_address=address)
+        Interchange(interchange_address=address, cert_dir=cert_dir)
 
 
-def test_limited_interface_binding():
-    """ When address is specified the worker_port would be bound to it rather than to 0.0.0.0"""
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_limited_interface_binding(cert_dir: Optional[str]):
+    """When address is specified the worker_port would be bound to it rather than to 0.0.0.0"""
     address = "127.0.0.1"
-    ix = Interchange(interchange_address=address)
+    ix = Interchange(interchange_address=address, cert_dir=cert_dir)
     ix.worker_result_port
     proc = psutil.Process()
     conns = proc.connections(kind="tcp")

parsl/tests/test_python_apps/test_futures.py

@@ -42,10 +42,10 @@ def test_fut_case_1():
 
 
 @pytest.mark.staging_required
-def test_fut_case_2(tmp_path):
+def test_fut_case_2(tmpd_cwd):
     """Testing the behavior of DataFutures where there are no dependencies
     """
-    output_f = tmp_path / 'test_fut_case_2.txt'
+    output_f = tmpd_cwd / 'test_fut_case_2.txt'
     app_fu = delay_incr(1, delay=0.01, outputs=[File(str(output_f))])
     data_fu = app_fu.outputs[0]
 
@@ -69,13 +69,13 @@ def test_fut_case_3():
 
 
 @pytest.mark.staging_required
-def test_fut_case_4(tmp_path):
+def test_fut_case_4(tmpd_cwd):
     """Testing the behavior of DataFutures where there are dependencies
 
     The first call has a delay, and the second call depends on the first
     """
-    output_f1 = tmp_path / 'test_fut_case_4_f1.txt'
-    output_f2 = tmp_path / 'test_fut_case_4_f2.txt'
+    output_f1 = tmpd_cwd / 'test_fut_case_4_f1.txt'
+    output_f2 = tmpd_cwd / 'test_fut_case_4_f2.txt'
     app_1 = delay_incr(1, delay=0.01, outputs=[File(str(output_f1))])
     app_2 = delay_incr(app_1, delay=0.01, outputs=[File(str(output_f2))])
     data_2 = app_2.outputs[0]

parsl/tests/test_regression/test_97_parallelism_0.py

@@ -15,6 +15,7 @@ def local_config() -> Config:
                 label="htex_local",
                 worker_debug=True,
                 cores_per_worker=1,
+                encrypted=True,
                 provider=LocalProvider(
                     init_blocks=0,
                     min_blocks=0,

parsl/tests/test_scaling/test_block_error_handler.py

@@ -11,7 +11,7 @@ from functools import partial
 @pytest.mark.local
 def test_block_error_handler_false():
     mock = Mock()
-    htex = HighThroughputExecutor(block_error_handler=False)
+    htex = HighThroughputExecutor(block_error_handler=False, encrypted=True)
     assert htex.block_error_handler is noop_error_handler
     htex.set_bad_state_and_fail_all = mock
 
@@ -27,7 +27,7 @@ def test_block_error_handler_false():
 @pytest.mark.local
 def test_block_error_handler_mock():
     handler_mock = Mock()
-    htex = HighThroughputExecutor(block_error_handler=handler_mock)
+    htex = HighThroughputExecutor(block_error_handler=handler_mock, encrypted=True)
     assert htex.block_error_handler is handler_mock
 
     bad_jobs = {'1': JobStatus(JobState.FAILED),
@@ -43,6 +43,7 @@ def test_block_error_handler_mock():
 @pytest.mark.local
 def test_simple_error_handler():
     htex = HighThroughputExecutor(block_error_handler=simple_error_handler,
+                                  encrypted=True,
                                   provider=LocalProvider(init_blocks=3))
 
     assert htex.block_error_handler is simple_error_handler
@@ -76,7 +77,7 @@ def test_simple_error_handler():
 
 @pytest.mark.local
 def test_windowed_error_handler():
-    htex = HighThroughputExecutor(block_error_handler=windowed_error_handler)
+    htex = HighThroughputExecutor(block_error_handler=windowed_error_handler, encrypted=True)
     assert htex.block_error_handler is windowed_error_handler
 
     bad_state_mock = Mock()
@@ -110,7 +111,7 @@ def test_windowed_error_handler():
 
 @pytest.mark.local
 def test_windowed_error_handler_sorting():
-    htex = HighThroughputExecutor(block_error_handler=windowed_error_handler)
+    htex = HighThroughputExecutor(block_error_handler=windowed_error_handler, encrypted=True)
     assert htex.block_error_handler is windowed_error_handler
 
     bad_state_mock = Mock()
@@ -136,7 +137,7 @@ def test_windowed_error_handler_sorting():
 @pytest.mark.local
 def test_windowed_error_handler_with_threshold():
     error_handler = partial(windowed_error_handler, threshold=2)
-    htex = HighThroughputExecutor(block_error_handler=error_handler)
+    htex = HighThroughputExecutor(block_error_handler=error_handler, encrypted=True)
     assert htex.block_error_handler is error_handler
 
     bad_state_mock = Mock()

parsl/tests/test_scaling/test_regression_1621.py

@@ -49,6 +49,7 @@ def test_one_block(tmpd_cwd):
                 address="127.0.0.1",
                 worker_debug=True,
                 cores_per_worker=1,
+                encrypted=True,
                 provider=oneshot_provider,
                 worker_logdir_root=str(tmpd_cwd)
             )

parsl/tests/test_scaling/test_scale_down.py

@@ -28,6 +28,7 @@ def local_config():
                 label="htex_local",
                 address="127.0.0.1",
                 max_workers=1,
+                encrypted=True,
                 provider=LocalProvider(
                     channel=LocalChannel(),
                     init_blocks=0,

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.01.22'
+VERSION = '2024.02.05'

{parsl-2024.1.22.data → parsl-2024.2.5.data}/scripts/process_worker_pool.py

@@ -20,8 +20,8 @@ import multiprocessing
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
 
+from parsl import curvezmq
 from parsl.process_loggers import wrap_with_logs
-
 from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
@@ -63,7 +63,8 @@ class Manager:
                  heartbeat_period,
                  poll_period,
                  cpu_affinity,
-                 available_accelerators: Sequence[str]):
+                 available_accelerators: Sequence[str],
+                 cert_dir: Optional[str]):
         """
         Parameters
         ----------
@@ -118,6 +119,8 @@ class Manager:
         available_accelerators: list of str
             List of accelerators available to the workers.
 
+        cert_dir : str | None
+            Path to the certificate directory.
         """
 
         logger.info("Manager started")
@@ -137,15 +140,16 @@ class Manager:
             print("Failed to find a viable address to connect to interchange. Exiting")
             exit(5)
 
-        self.context = zmq.Context()
-        self.task_incoming = self.context.socket(zmq.DEALER)
+        self.cert_dir = cert_dir
+        self.zmq_context = curvezmq.ClientContext(self.cert_dir)
+        self.task_incoming = self.zmq_context.socket(zmq.DEALER)
         self.task_incoming.setsockopt(zmq.IDENTITY, uid.encode('utf-8'))
         # Linger is set to 0, so that the manager can exit even when there might be
         # messages in the pipe
         self.task_incoming.setsockopt(zmq.LINGER, 0)
         self.task_incoming.connect(task_q_url)
 
-        self.result_outgoing = self.context.socket(zmq.DEALER)
+        self.result_outgoing = self.zmq_context.socket(zmq.DEALER)
         self.result_outgoing.setsockopt(zmq.IDENTITY, uid.encode('utf-8'))
         self.result_outgoing.setsockopt(zmq.LINGER, 0)
         self.result_outgoing.connect(result_q_url)
@@ -468,7 +472,7 @@ class Manager:
 
         self.task_incoming.close()
         self.result_outgoing.close()
-        self.context.term()
+        self.zmq_context.term()
         delta = time.time() - start
         logger.info("process_worker_pool ran for {} seconds".format(delta))
         return
@@ -578,7 +582,7 @@ def worker(
     # If desired, set process affinity
     if cpu_affinity != "none":
         # Count the number of cores per worker
-        avail_cores = sorted(os.sched_getaffinity(0))  # Get the available processors
+        avail_cores = sorted(os.sched_getaffinity(0))  # Get the available threads
         cores_per_worker = len(avail_cores) // pool_size
         assert cores_per_worker > 0, "Affinity does not work if there are more workers than cores"
 
@@ -590,6 +594,23 @@ def worker(
             my_cores = avail_cores[cores_per_worker * cpu_worker_id:cores_per_worker * (cpu_worker_id + 1)]
         elif cpu_affinity == "alternating":
             my_cores = avail_cores[worker_id::pool_size]
+        elif cpu_affinity[0:4] == "list":
+            thread_ranks = cpu_affinity.split(":")[1:]
+            if len(thread_ranks) != pool_size:
+                raise ValueError("Affinity list {} has wrong number of thread ranks".format(cpu_affinity))
+            threads = thread_ranks[worker_id]
+            thread_list = threads.split(",")
+            my_cores = []
+            for tl in thread_list:
+                thread_range = tl.split("-")
+                if len(thread_range) == 1:
+                    my_cores.append(int(thread_range[0]))
+                elif len(thread_range) == 2:
+                    start_thread = int(thread_range[0])
+                    end_thread = int(thread_range[1]) + 1
+                    my_cores += list(range(start_thread, end_thread))
+                else:
+                    raise ValueError("Affinity list formatting is not expected {}".format(cpu_affinity))
         else:
             raise ValueError("Affinity strategy {} is not supported".format(cpu_affinity))
 
@@ -703,6 +724,8 @@ if __name__ == "__main__":
                         help="Enable logging at DEBUG level")
     parser.add_argument("-a", "--addresses", default='',
                         help="Comma separated list of addresses at which the interchange could be reached")
+    parser.add_argument("--cert_dir", required=True,
+                        help="Path to certificate directory.")
     parser.add_argument("-l", "--logdir", default="process_worker_pool_logs",
                         help="Process worker pool log directory")
     parser.add_argument("-u", "--uid", default=str(uuid.uuid4()).split('-')[-1],
@@ -729,7 +752,17 @@ if __name__ == "__main__":
                         help="Poll period used in milliseconds")
     parser.add_argument("-r", "--result_port", required=True,
                         help="REQUIRED: Result port for posting results to the interchange")
-    parser.add_argument("--cpu-affinity", type=str, choices=["none", "block", "alternating", "block-reverse"],
+
+    def strategyorlist(s: str):
+        allowed_strategies = ["none", "block", "alternating", "block-reverse"]
+        if s in allowed_strategies:
+            return s
+        elif s[0:4] == "list":
+            return s
+        else:
+            raise argparse.ArgumentTypeError("cpu-affinity must be one of {} or a list format".format(allowed_strategies))
+
+    parser.add_argument("--cpu-affinity", type=strategyorlist,
                         required=True,
                         help="Whether/how workers should control CPU affinity.")
     parser.add_argument("--available-accelerators", type=str, nargs="*",
@@ -746,6 +779,7 @@ if __name__ == "__main__":
 
         logger.info("Python version: {}".format(sys.version))
         logger.info("Debug logging: {}".format(args.debug))
+        logger.info("Certificates dir: {}".format(args.cert_dir))
         logger.info("Log dir: {}".format(args.logdir))
         logger.info("Manager ID: {}".format(args.uid))
         logger.info("Block ID: {}".format(args.block_id))
@@ -777,7 +811,8 @@ if __name__ == "__main__":
                           heartbeat_period=int(args.hb_period),
                           poll_period=int(args.poll),
                           cpu_affinity=args.cpu_affinity,
-                          available_accelerators=args.available_accelerators)
+                          available_accelerators=args.available_accelerators,
+                          cert_dir=None if args.cert_dir == "None" else args.cert_dir)
         manager.start()
 
     except Exception:

{parsl-2024.1.22.dist-info → parsl-2024.2.5.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.1.22
+Version: 2024.2.5
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.01.22.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.02.05.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -19,9 +19,8 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8.0
 License-File: LICENSE
 Requires-Dist: pyzmq >=17.1.2
-Requires-Dist: typeguard <3,>=2.10
+Requires-Dist: typeguard !=3.*,<5,>=2.10
 Requires-Dist: typing-extensions <5,>=4.6
-Requires-Dist: six
 Requires-Dist: globus-sdk
 Requires-Dist: dill
 Requires-Dist: tblib
@@ -35,7 +34,7 @@ Requires-Dist: pydot ; extra == 'all'
 Requires-Dist: networkx <2.6,>=2.5 ; extra == 'all'
 Requires-Dist: Flask >=1.0.2 ; extra == 'all'
 Requires-Dist: flask-sqlalchemy ; extra == 'all'
-Requires-Dist: pandas <3 ; extra == 'all'
+Requires-Dist: pandas <2.2 ; extra == 'all'
 Requires-Dist: plotly ; extra == 'all'
 Requires-Dist: python-daemon ; extra == 'all'
 Requires-Dist: boto3 ; extra == 'all'
@@ -90,7 +89,7 @@ Requires-Dist: pydot ; extra == 'visualization'
 Requires-Dist: networkx <2.6,>=2.5 ; extra == 'visualization'
 Requires-Dist: Flask >=1.0.2 ; extra == 'visualization'
 Requires-Dist: flask-sqlalchemy ; extra == 'visualization'
-Requires-Dist: pandas <3 ; extra == 'visualization'
+Requires-Dist: pandas <2.2 ; extra == 'visualization'
 Requires-Dist: plotly ; extra == 'visualization'
 Requires-Dist: python-daemon ; extra == 'visualization'
 Provides-Extra: workqueue