parsl 2024.1.29__tar.gz → 2024.2.5__tar.gz

{parsl-2024.1.29/parsl.egg-info → parsl-2024.2.5}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.1.29
+Version: 2024.2.5
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.01.29.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.02.05.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

parsl-2024.2.5/parsl/curvezmq.py

@@ -0,0 +1,205 @@
+import os
+from abc import ABCMeta, abstractmethod
+from typing import Optional, Tuple, Union
+
+import zmq
+import zmq.auth
+from zmq.auth.thread import ThreadAuthenticator
+
+
+def create_certificates(base_dir: Union[str, os.PathLike]):
+    """Create server and client certificates in a private directory.
+
+    This will overwrite existing certificate files.
+
+    Parameters
+    ----------
+    base_dir : str | os.PathLike
+        Parent directory of the private certificates directory.
+    """
+    cert_dir = os.path.join(base_dir, "certificates")
+    os.makedirs(cert_dir, mode=0o700, exist_ok=True)
+
+    zmq.auth.create_certificates(cert_dir, name="server")
+    zmq.auth.create_certificates(cert_dir, name="client")
+
+    return cert_dir
+
+
+def _load_certificate(
+    cert_dir: Union[str, os.PathLike], name: str
+) -> Tuple[bytes, bytes]:
+    if os.stat(cert_dir).st_mode & 0o777 != 0o700:
+        raise OSError(f"The certificates directory must be private: {cert_dir}")
+
+    # pyzmq creates secret key files with the '.key_secret' extension
+    # Ref: https://github.com/zeromq/pyzmq/blob/ae615d4097ccfbc6b5c17de60355cbe6e00a6065/zmq/auth/certs.py#L73
+    secret_key_file = os.path.join(cert_dir, f"{name}.key_secret")
+    public_key, secret_key = zmq.auth.load_certificate(secret_key_file)
+    if secret_key is None:
+        raise ValueError(f"No secret key found in {secret_key_file}")
+
+    return public_key, secret_key
+
+
+class BaseContext(metaclass=ABCMeta):
+    """Base CurveZMQ context"""
+
+    def __init__(self, cert_dir: Optional[Union[str, os.PathLike]]) -> None:
+        self.cert_dir = cert_dir
+        self._ctx = zmq.Context()
+
+    @property
+    def encrypted(self):
+        """Indicates whether encryption is enabled.
+
+        False (disabled) when self.cert_dir is set to None.
+        """
+        return self.cert_dir is not None
+
+    @property
+    def closed(self):
+        return self._ctx.closed
+
+    @abstractmethod
+    def socket(self, socket_type: int, *args, **kwargs) -> zmq.Socket:
+        """Create a socket associated with this context.
+
+        This method will apply all necessary certificates and socket options.
+
+        Parameters
+        ----------
+        socket_type : int
+            The socket type, which can be any of the 0MQ socket types: REQ, REP,
+            PUB, SUB, PAIR, DEALER, ROUTER, PULL, PUSH, etc.
+
+        args:
+            passed to the zmq.Context.socket method.
+
+        kwargs:
+            passed to the zmq.Context.socket method.
+        """
+        ...
+
+    def term(self):
+        """Terminate the context."""
+        self._ctx.term()
+
+    def destroy(self, linger: Optional[int] = None):
+        """Close all sockets associated with this context and then terminate
+        the context.
+
+        .. warning::
+
+            destroy involves calling ``zmq_close()``, which is **NOT** threadsafe.
+            If there are active sockets in other threads, this must not be called.
+
+        Parameters
+        ----------
+        linger : int, optional
+            If specified, set LINGER on sockets prior to closing them.
+        """
+        self._ctx.destroy(linger)
+
+    def recreate(self, linger: Optional[int] = None):
+        """Destroy then recreate the context.
+
+        Parameters
+        ----------
+        linger : int, optional
+            If specified, set LINGER on sockets prior to closing them.
+        """
+        self.destroy(linger)
+        self._ctx = zmq.Context()
+
+
+class ServerContext(BaseContext):
+    """CurveZMQ server context
+
+    We create server sockets via the `ctx.socket` method, which automatically
+    applies the necessary certificates and socket options.
+
+    We handle client certificate authentication in a separate dedicated thread.
+
+    Parameters
+    ----------
+    cert_dir : str | os.PathLike | None
+        Path to the certificate directory. Setting this to None will disable encryption.
+    """
+
+    def __init__(self, cert_dir: Optional[Union[str, os.PathLike]]) -> None:
+        super().__init__(cert_dir)
+        self.auth_thread = None
+        if self.encrypted:
+            self.auth_thread = self._start_auth_thread()
+
+    def __del__(self):
+        # Avoid issues in which the auth_thread attr was
+        # previously deleted
+        if getattr(self, "auth_thread", None):
+            self.auth_thread.stop()
+
+    def _start_auth_thread(self) -> ThreadAuthenticator:
+        auth_thread = ThreadAuthenticator(self._ctx)
+        auth_thread.start()
+        # Only allow certs that are in the cert dir
+        assert self.cert_dir  # For mypy
+        auth_thread.configure_curve(domain="*", location=self.cert_dir)
+        return auth_thread
+
+    def socket(self, socket_type: int, *args, **kwargs) -> zmq.Socket:
+        sock = self._ctx.socket(socket_type, *args, **kwargs)
+        if self.encrypted:
+            assert self.cert_dir  # For mypy
+            _, secret_key = _load_certificate(self.cert_dir, name="server")
+            try:
+                # Only the clients need the server's public key to encrypt
+                # messages and verify the server's identity.
+                # Ref: http://curvezmq.org/page:read-the-docs
+                sock.setsockopt(zmq.CURVE_SECRETKEY, secret_key)
+            except zmq.ZMQError as e:
+                raise ValueError("Invalid CurveZMQ key format") from e
+            sock.setsockopt(zmq.CURVE_SERVER, True)  # Must come before bind
+        return sock
+
+    def term(self):
+        if self.auth_thread:
+            self.auth_thread.stop()
+        super().term()
+
+    def destroy(self, linger: Optional[int] = None):
+        if self.auth_thread:
+            self.auth_thread.stop()
+        super().destroy(linger)
+
+    def recreate(self, linger: Optional[int] = None):
+        super().recreate(linger)
+        if self.auth_thread:
+            self.auth_thread = self._start_auth_thread()
+
+
+class ClientContext(BaseContext):
+    """CurveZMQ client context
+
+    We create client sockets via the `ctx.socket` method, which automatically
+    applies the necessary certificates and socket options.
+
+    Parameters
+    ----------
+    cert_dir : str | os.PathLike | None
+        Path to the certificate directory. Setting this to None will disable encryption.
+    """
+
+    def socket(self, socket_type: int, *args, **kwargs) -> zmq.Socket:
+        sock = self._ctx.socket(socket_type, *args, **kwargs)
+        if self.encrypted:
+            assert self.cert_dir  # For mypy
+            public_key, secret_key = _load_certificate(self.cert_dir, name="client")
+            server_public_key, _ = _load_certificate(self.cert_dir, name="server")
+            try:
+                sock.setsockopt(zmq.CURVE_PUBLICKEY, public_key)
+                sock.setsockopt(zmq.CURVE_SECRETKEY, secret_key)
+                sock.setsockopt(zmq.CURVE_SERVERKEY, server_public_key)
+            except zmq.ZMQError as e:
+                raise ValueError("Invalid CurveZMQ key format") from e
+        return sock

{parsl-2024.1.29 → parsl-2024.2.5}/parsl/dataflow/dflow.py

@@ -302,7 +302,7 @@ class DataFlowKernel:
             res = self._unwrap_remote_exception_wrapper(future)
 
         except Exception as e:
-            logger.debug("Task {} try {} failed".format(task_id, task_record['try_id']))
+            logger.info(f"Task {task_id} try {task_record['try_id']} failed with exception of type {type(e).__name__}")
             # We keep the history separately, since the future itself could be
             # tossed.
             task_record['fail_history'].append(repr(e))

{parsl-2024.1.29 → parsl-2024.2.5}/parsl/executors/high_throughput/executor.py

@@ -22,6 +22,7 @@ from parsl.executors.errors import (
     UnsupportedFeatureError
 )
 
+from parsl import curvezmq
 from parsl.executors.status_handling import BlockProviderExecutor
 from parsl.providers.base import ExecutionProvider
 from parsl.data_provider.staging import Staging
@@ -174,6 +175,9 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
 
     worker_logdir_root : string
         In case of a remote file system, specify the path to where logs will be kept.
+
+    encrypted : bool
+        Flag to enable/disable encryption (CurveZMQ). Default is False.
     """
 
     @typeguard.typechecked
@@ -199,7 +203,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
                  poll_period: int = 10,
                  address_probe_timeout: Optional[int] = None,
                  worker_logdir_root: Optional[str] = None,
-                 block_error_handler: Union[bool, Callable[[BlockProviderExecutor, Dict[str, JobStatus]], None]] = True):
+                 block_error_handler: Union[bool, Callable[[BlockProviderExecutor, Dict[str, JobStatus]], None]] = True,
+                 encrypted: bool = False):
 
         logger.debug("Initializing HighThroughputExecutor")
 
@@ -256,6 +261,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
         self.run_dir = '.'
         self.worker_logdir_root = worker_logdir_root
         self.cpu_affinity = cpu_affinity
+        self.encrypted = encrypted
+        self.cert_dir = None
 
         if not launch_cmd:
             launch_cmd = (
@@ -267,6 +274,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
                 "--poll {poll_period} "
                 "--task_port={task_port} "
                 "--result_port={result_port} "
+                "--cert_dir {cert_dir} "
                 "--logdir={logdir} "
                 "--block_id={{block_id}} "
                 "--hb_period={heartbeat_period} "
@@ -280,6 +288,16 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
 
     radio_mode = "htex"
 
+    @property
+    def logdir(self):
+        return "{}/{}".format(self.run_dir, self.label)
+
+    @property
+    def worker_logdir(self):
+        if self.worker_logdir_root is not None:
+            return "{}/{}".format(self.worker_logdir_root, self.label)
+        return self.logdir
+
     def initialize_scaling(self):
         """Compose the launch command and scale out the initial blocks.
         """
@@ -289,9 +307,6 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
         address_probe_timeout_string = ""
         if self.address_probe_timeout:
             address_probe_timeout_string = "--address_probe_timeout={}".format(self.address_probe_timeout)
-        worker_logdir = "{}/{}".format(self.run_dir, self.label)
-        if self.worker_logdir_root is not None:
-            worker_logdir = "{}/{}".format(self.worker_logdir_root, self.label)
 
         l_cmd = self.launch_cmd.format(debug=debug_opts,
                                        prefetch_capacity=self.prefetch_capacity,
@@ -306,7 +321,8 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
                                        heartbeat_period=self.heartbeat_period,
                                        heartbeat_threshold=self.heartbeat_threshold,
                                        poll_period=self.poll_period,
-                                       logdir=worker_logdir,
+                                       cert_dir=self.cert_dir,
+                                       logdir=self.worker_logdir,
                                        cpu_affinity=self.cpu_affinity,
                                        accelerators=" ".join(self.available_accelerators))
         self.launch_cmd = l_cmd
@@ -327,9 +343,25 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
     def start(self):
         """Create the Interchange process and connect to it.
         """
-        self.outgoing_q = zmq_pipes.TasksOutgoing("127.0.0.1", self.interchange_port_range)
-        self.incoming_q = zmq_pipes.ResultsIncoming("127.0.0.1", self.interchange_port_range)
-        self.command_client = zmq_pipes.CommandClient("127.0.0.1", self.interchange_port_range)
+        if self.encrypted and self.cert_dir is None:
+            logger.debug("Creating CurveZMQ certificates")
+            self.cert_dir = curvezmq.create_certificates(self.logdir)
+        elif not self.encrypted and self.cert_dir:
+            raise AttributeError(
+                "The certificates directory path attribute (cert_dir) is defined, but the "
+                "encrypted attribute is set to False. You must either change cert_dir to "
+                "None or encrypted to True."
+            )
+
+        self.outgoing_q = zmq_pipes.TasksOutgoing(
+            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+        )
+        self.incoming_q = zmq_pipes.ResultsIncoming(
+            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+        )
+        self.command_client = zmq_pipes.CommandClient(
+            curvezmq.ClientContext(self.cert_dir), "127.0.0.1", self.interchange_port_range
+        )
 
         self._queue_management_thread = None
         self._start_queue_management_thread()
@@ -450,10 +482,11 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
                                                     "worker_port_range": self.worker_port_range,
                                                     "hub_address": self.hub_address,
                                                     "hub_port": self.hub_port,
-                                                    "logdir": "{}/{}".format(self.run_dir, self.label),
+                                                    "logdir": self.logdir,
                                                     "heartbeat_threshold": self.heartbeat_threshold,
                                                     "poll_period": self.poll_period,
-                                                    "logging_level": logging.DEBUG if self.worker_debug else logging.INFO
+                                                    "logging_level": logging.DEBUG if self.worker_debug else logging.INFO,
+                                                    "cert_dir": self.cert_dir,
                                                     },
                                             daemon=True,
                                             name="HTEX-Interchange"
@@ -603,7 +636,7 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
     def workers_per_node(self) -> Union[int, float]:
         return self._workers_per_node
 
-    def scale_in(self, blocks=None, block_ids=[], force=True, max_idletime=None):
+    def scale_in(self, blocks, force=True, max_idletime=None):
         """Scale in the number of active blocks by specified amount.
 
         The scale in method here is very rude. It doesn't give the workers
@@ -618,55 +651,51 @@ class HighThroughputExecutor(BlockProviderExecutor, RepresentationMixin):
 
         force : Bool
              Used along with blocks to indicate whether blocks should be terminated by force.
+
              When force = True, we will kill blocks regardless of the blocks being busy
-             When force = False, Only idle blocks will be terminated.
-             If the # of ``idle_blocks`` < ``blocks``, the list of jobs marked for termination
-             will be in the range: 0 - ``blocks``.
+
+             When force = False, only idle blocks will be terminated.  If the
+             number of idle blocks < ``blocks``, then fewer than ``blocks``
+             blocks will be terminated.
 
         max_idletime: float
              A time to indicate how long a block can be idle.
              Used along with force = False to kill blocks that have been idle for that long.
 
-        block_ids : list
-             List of specific block ids to terminate. Optional
-
         Returns
         -------
-        List of job_ids marked for termination
+        List of block IDs scaled in
         """
-        logger.debug(f"Scale in called, blocks={blocks}, block_ids={block_ids}")
-        if block_ids:
-            block_ids_to_kill = block_ids
+        logger.debug(f"Scale in called, blocks={blocks}")
+        managers = self.connected_managers()
+        block_info = {}  # block id -> list( tasks, idle duration )
+        for manager in managers:
+            if not manager['active']:
+                continue
+            b_id = manager['block_id']
+            if b_id not in block_info:
+                block_info[b_id] = [0, float('inf')]
+            block_info[b_id][0] += manager['tasks']
+            block_info[b_id][1] = min(block_info[b_id][1], manager['idle_duration'])
+
+        sorted_blocks = sorted(block_info.items(), key=lambda item: (item[1][1], item[1][0]))
+        logger.debug(f"Scale in selecting from {len(sorted_blocks)} blocks")
+        if force is True:
+            block_ids_to_kill = [x[0] for x in sorted_blocks[:blocks]]
         else:
-            managers = self.connected_managers()
-            block_info = {}  # block id -> list( tasks, idle duration )
-            for manager in managers:
-                if not manager['active']:
-                    continue
-                b_id = manager['block_id']
-                if b_id not in block_info:
-                    block_info[b_id] = [0, float('inf')]
-                block_info[b_id][0] += manager['tasks']
-                block_info[b_id][1] = min(block_info[b_id][1], manager['idle_duration'])
-
-            sorted_blocks = sorted(block_info.items(), key=lambda item: (item[1][1], item[1][0]))
-            logger.debug(f"Scale in selecting from {len(sorted_blocks)} blocks")
-            if force is True:
-                block_ids_to_kill = [x[0] for x in sorted_blocks[:blocks]]
+            if not max_idletime:
+                block_ids_to_kill = [x[0] for x in sorted_blocks if x[1][0] == 0][:blocks]
             else:
-                if not max_idletime:
-                    block_ids_to_kill = [x[0] for x in sorted_blocks if x[1][0] == 0][:blocks]
-                else:
-                    block_ids_to_kill = []
-                    for x in sorted_blocks:
-                        if x[1][1] > max_idletime and x[1][0] == 0:
-                            block_ids_to_kill.append(x[0])
-                            if len(block_ids_to_kill) == blocks:
-                                break
-                logger.debug("Selected idle block ids to kill: {}".format(
-                    block_ids_to_kill))
-                if len(block_ids_to_kill) < blocks:
-                    logger.warning(f"Could not find enough blocks to kill: wanted {blocks} but only selected {len(block_ids_to_kill)}")
+                block_ids_to_kill = []
+                for x in sorted_blocks:
+                    if x[1][1] > max_idletime and x[1][0] == 0:
+                        block_ids_to_kill.append(x[0])
+                        if len(block_ids_to_kill) == blocks:
+                            break
+            logger.debug("Selected idle block ids to kill: {}".format(
+                block_ids_to_kill))
+            if len(block_ids_to_kill) < blocks:
+                logger.warning(f"Could not find enough blocks to kill: wanted {blocks} but only selected {len(block_ids_to_kill)}")
 
         # Hold the block
         for block_id in block_ids_to_kill:

{parsl-2024.1.29 → parsl-2024.2.5}/parsl/executors/high_throughput/interchange.py

@@ -16,6 +16,7 @@ import json
 
 from typing import cast, Any, Dict, NoReturn, Sequence, Set, Optional, Tuple
 
+from parsl import curvezmq
 from parsl.utils import setproctitle
 from parsl.version import VERSION as PARSL_VERSION
 from parsl.serialize import serialize as serialize_object
@@ -79,6 +80,7 @@ class Interchange:
                  logdir: str = ".",
                  logging_level: int = logging.INFO,
                  poll_period: int = 10,
+                 cert_dir: Optional[str] = None,
                  ) -> None:
         """
         Parameters
@@ -120,7 +122,10 @@ class Interchange:
         poll_period : int
              The main thread polling period, in milliseconds. Default: 10ms
 
+        cert_dir : str | None
+            Path to the certificate directory. Default: None
         """
+        self.cert_dir = cert_dir
         self.logdir = logdir
         os.makedirs(self.logdir, exist_ok=True)
 
@@ -134,15 +139,15 @@ class Interchange:
 
         logger.info("Attempting connection to client at {} on ports: {},{},{}".format(
             client_address, client_ports[0], client_ports[1], client_ports[2]))
-        self.context = zmq.Context()
-        self.task_incoming = self.context.socket(zmq.DEALER)
+        self.zmq_context = curvezmq.ServerContext(self.cert_dir)
+        self.task_incoming = self.zmq_context.socket(zmq.DEALER)
         self.task_incoming.set_hwm(0)
         self.task_incoming.connect("tcp://{}:{}".format(client_address, client_ports[0]))
-        self.results_outgoing = self.context.socket(zmq.DEALER)
+        self.results_outgoing = self.zmq_context.socket(zmq.DEALER)
         self.results_outgoing.set_hwm(0)
         self.results_outgoing.connect("tcp://{}:{}".format(client_address, client_ports[1]))
 
-        self.command_channel = self.context.socket(zmq.REP)
+        self.command_channel = self.zmq_context.socket(zmq.REP)
         self.command_channel.connect("tcp://{}:{}".format(client_address, client_ports[2]))
         logger.info("Connected to client")
 
@@ -155,9 +160,9 @@ class Interchange:
         self.worker_ports = worker_ports
         self.worker_port_range = worker_port_range
 
-        self.task_outgoing = self.context.socket(zmq.ROUTER)
+        self.task_outgoing = self.zmq_context.socket(zmq.ROUTER)
         self.task_outgoing.set_hwm(0)
-        self.results_incoming = self.context.socket(zmq.ROUTER)
+        self.results_incoming = self.zmq_context.socket(zmq.ROUTER)
         self.results_incoming.set_hwm(0)
 
         if self.worker_ports:
@@ -241,7 +246,8 @@ class Interchange:
     def _create_monitoring_channel(self) -> Optional[zmq.Socket]:
         if self.hub_address and self.hub_port:
             logger.info("Connecting to monitoring")
-            hub_channel = self.context.socket(zmq.DEALER)
+            # This is a one-off because monitoring is unencrypted
+            hub_channel = zmq.Context().socket(zmq.DEALER)
             hub_channel.set_hwm(0)
             hub_channel.connect("tcp://{}:{}".format(self.hub_address, self.hub_port))
             logger.info("Monitoring enabled and connected to hub")
@@ -379,6 +385,7 @@ class Interchange:
             self.expire_bad_managers(interesting_managers, hub_channel)
             self.process_tasks_to_send(interesting_managers)
 
+        self.zmq_context.destroy()
         delta = time.time() - start
         logger.info("Processed {} tasks in {} seconds".format(self.count, delta))
         logger.warning("Exiting")

{parsl-2024.1.29 → parsl-2024.2.5}/parsl/executors/high_throughput/process_worker_pool.py

@@ -20,8 +20,8 @@ import multiprocessing
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
 
+from parsl import curvezmq
 from parsl.process_loggers import wrap_with_logs
-
 from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
@@ -63,7 +63,8 @@ class Manager:
                  heartbeat_period,
                  poll_period,
                  cpu_affinity,
-                 available_accelerators: Sequence[str]):
+                 available_accelerators: Sequence[str],
+                 cert_dir: Optional[str]):
         """
         Parameters
         ----------
@@ -118,6 +119,8 @@ class Manager:
         available_accelerators: list of str
             List of accelerators available to the workers.
 
+        cert_dir : str | None
+            Path to the certificate directory.
         """
 
         logger.info("Manager started")
@@ -137,15 +140,16 @@ class Manager:
             print("Failed to find a viable address to connect to interchange. Exiting")
             exit(5)
 
-        self.context = zmq.Context()
-        self.task_incoming = self.context.socket(zmq.DEALER)
+        self.cert_dir = cert_dir
+        self.zmq_context = curvezmq.ClientContext(self.cert_dir)
+        self.task_incoming = self.zmq_context.socket(zmq.DEALER)
         self.task_incoming.setsockopt(zmq.IDENTITY, uid.encode('utf-8'))
         # Linger is set to 0, so that the manager can exit even when there might be
         # messages in the pipe
         self.task_incoming.setsockopt(zmq.LINGER, 0)
         self.task_incoming.connect(task_q_url)
 
-        self.result_outgoing = self.context.socket(zmq.DEALER)
+        self.result_outgoing = self.zmq_context.socket(zmq.DEALER)
         self.result_outgoing.setsockopt(zmq.IDENTITY, uid.encode('utf-8'))
         self.result_outgoing.setsockopt(zmq.LINGER, 0)
         self.result_outgoing.connect(result_q_url)
@@ -468,7 +472,7 @@ class Manager:
 
         self.task_incoming.close()
         self.result_outgoing.close()
-        self.context.term()
+        self.zmq_context.term()
         delta = time.time() - start
         logger.info("process_worker_pool ran for {} seconds".format(delta))
         return
@@ -720,6 +724,8 @@ if __name__ == "__main__":
                         help="Enable logging at DEBUG level")
     parser.add_argument("-a", "--addresses", default='',
                         help="Comma separated list of addresses at which the interchange could be reached")
+    parser.add_argument("--cert_dir", required=True,
+                        help="Path to certificate directory.")
     parser.add_argument("-l", "--logdir", default="process_worker_pool_logs",
                         help="Process worker pool log directory")
     parser.add_argument("-u", "--uid", default=str(uuid.uuid4()).split('-')[-1],
@@ -773,6 +779,7 @@ if __name__ == "__main__":
 
         logger.info("Python version: {}".format(sys.version))
         logger.info("Debug logging: {}".format(args.debug))
+        logger.info("Certificates dir: {}".format(args.cert_dir))
         logger.info("Log dir: {}".format(args.logdir))
         logger.info("Manager ID: {}".format(args.uid))
         logger.info("Block ID: {}".format(args.block_id))
@@ -804,7 +811,8 @@ if __name__ == "__main__":
                           heartbeat_period=int(args.hb_period),
                           poll_period=int(args.poll),
                           cpu_affinity=args.cpu_affinity,
-                          available_accelerators=args.available_accelerators)
+                          available_accelerators=args.available_accelerators,
+                          cert_dir=None if args.cert_dir == "None" else args.cert_dir)
         manager.start()
 
     except Exception: