parsl 2024.1.22__py3-none-any.whl → 2024.2.5__py3-none-any.whl

parsl/executors/high_throughput/zmq_pipes.py

@@ -4,24 +4,28 @@ import zmq
 import logging
 import threading
 
+from parsl import curvezmq
+
 logger = logging.getLogger(__name__)
 
 
 class CommandClient:
     """ CommandClient
     """
-    def __init__(self, ip_address, port_range):
+    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
         """
         Parameters
         ----------
 
+        zmq_context: curvezmq.ClientContext
+            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
         """
-        self.context = zmq.Context()
+        self.zmq_context = zmq_context
         self.ip_address = ip_address
         self.port_range = port_range
         self.port = None
@@ -33,7 +37,7 @@ class CommandClient:
 
         Upon recreating the socket, we bind to the same port.
         """
-        self.zmq_socket = self.context.socket(zmq.REQ)
+        self.zmq_socket = self.zmq_context.socket(zmq.REQ)
         self.zmq_socket.setsockopt(zmq.LINGER, 0)
         if self.port is None:
             self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(self.ip_address),
@@ -62,9 +66,7 @@ class CommandClient:
                 except zmq.ZMQError:
                     logger.exception("Potential ZMQ REQ-REP deadlock caught")
                     logger.info("Trying to reestablish context")
-                    self.zmq_socket.close()
-                    self.context.destroy()
-                    self.context = zmq.Context()
+                    self.zmq_context.recreate()
                     self.create_socket_and_bind()
                 else:
                     break
@@ -77,25 +79,27 @@ class CommandClient:
 
     def close(self):
         self.zmq_socket.close()
-        self.context.term()
+        self.zmq_context.term()
 
 
 class TasksOutgoing:
     """ Outgoing task queue from the executor to the Interchange
     """
-    def __init__(self, ip_address, port_range):
+    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
         """
         Parameters
         ----------
 
+        zmq_context: curvezmq.ClientContext
+            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
         """
-        self.context = zmq.Context()
-        self.zmq_socket = self.context.socket(zmq.DEALER)
+        self.zmq_context = zmq_context
+        self.zmq_socket = self.zmq_context.socket(zmq.DEALER)
         self.zmq_socket.set_hwm(0)
         self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(ip_address),
                                                         min_port=port_range[0],
@@ -127,26 +131,28 @@ class TasksOutgoing:
 
     def close(self):
         self.zmq_socket.close()
-        self.context.term()
+        self.zmq_context.term()
 
 
 class ResultsIncoming:
     """ Incoming results queue from the Interchange to the executor
     """
 
-    def __init__(self, ip_address, port_range):
+    def __init__(self, zmq_context: curvezmq.ClientContext, ip_address, port_range):
         """
         Parameters
         ----------
 
+        zmq_context: curvezmq.ClientContext
+            CurveZMQ client context used to create secure sockets
         ip_address: str
            IP address of the client (where Parsl runs)
         port_range: tuple(int, int)
            Port range for the comms between client and interchange
 
         """
-        self.context = zmq.Context()
-        self.results_receiver = self.context.socket(zmq.DEALER)
+        self.zmq_context = zmq_context
+        self.results_receiver = self.zmq_context.socket(zmq.DEALER)
         self.results_receiver.set_hwm(0)
         self.port = self.results_receiver.bind_to_random_port("tcp://{}".format(ip_address),
                                                               min_port=port_range[0],
@@ -160,4 +166,4 @@ class ResultsIncoming:
 
     def close(self):
         self.results_receiver.close()
-        self.context.term()
+        self.zmq_context.term()

parsl/executors/taskvine/manager.py

@@ -47,6 +47,10 @@ def _set_manager_attributes(m, config):
     if config.enable_peer_transfers:
         m.enable_peer_transfers()
 
+    # Set catalog report to parsl if project name exists
+    if m.name:
+        m.set_property("framework", "parsl")
+
 
 def _prepare_environment_serverless(manager_config, env_cache_dir, poncho_create_script):
     # Return path to a packaged poncho environment
@@ -203,7 +207,7 @@ def _taskvine_submit_wait(ready_task_queue=None,
             break
 
         # Submit tasks
-        while ready_task_queue.qsize() > 0 and not should_stop.is_set():
+        while ready_task_queue.qsize() > 0 or m.empty() and not should_stop.is_set():
             # Obtain task from ready_task_queue
             try:
                 task = ready_task_queue.get(timeout=1)
@@ -248,6 +252,22 @@ def _taskvine_submit_wait(ready_task_queue=None,
                                                                      poncho_env=poncho_env_path,
                                                                      init_command=manager_config.init_command,
                                                                      add_env=add_env)
+
+                    # Configure the library if provided
+                    if manager_config.library_config:
+                        lib_cores = manager_config.library_config.get('cores', None)
+                        lib_memory = manager_config.library_config.get('memory', None)
+                        lib_disk = manager_config.library_config.get('disk', None)
+                        lib_slots = manager_config.library_config.get('num_slots', None)
+                        if lib_cores:
+                            serverless_lib.set_cores(lib_cores)
+                        if lib_memory:
+                            serverless_lib.set_memory(lib_memory)
+                        if lib_disk:
+                            serverless_lib.set_disk(lib_disk)
+                        if lib_slots:
+                            serverless_lib.set_function_slots(lib_slots)
+
                     if poncho_env_path:
                         serverless_lib_env_file = m.declare_poncho(poncho_env_path, cache=True, peer_transfer=True)
                         serverless_lib.add_environment(serverless_lib_env_file)
@@ -377,48 +397,45 @@ def _taskvine_submit_wait(ready_task_queue=None,
 
         # If the queue is not empty wait on the TaskVine queue for a task
         task_found = True
-        if not m.empty():
-            while task_found and not should_stop.is_set():
-                # Obtain the task from the queue
-                t = m.wait(1)
-                if t is None:
-                    task_found = False
-                    continue
-                logger.debug('Found a task')
-                executor_task_id = vine_id_to_executor_task_id[str(t.id)][0]
-                vine_id_to_executor_task_id.pop(str(t.id))
-
-                # When a task is found
-                result_file = result_file_of_task_id.pop(executor_task_id)
-
-                logger.debug(f"completed executor task info: {executor_task_id}, {t.category}, {t.command}, {t.std_output}")
-
-                # A tasks completes 'succesfully' if it has result file.
-                # A check whether the Python object represented using this file can be
-                # deserialized happens later in the collector thread of the executor
-                # process.
-                logger.debug("Looking for result in {}".format(result_file))
-                if os.path.exists(result_file):
-                    logger.debug("Found result in {}".format(result_file))
-                    finished_task_queue.put_nowait(VineTaskToParsl(executor_id=executor_task_id,
-                                                                   result_received=True,
-                                                                   result_file=result_file,
-                                                                   reason=None,
-                                                                   status=t.exit_code))
-                # If a result file could not be generated, explain the
-                # failure according to taskvine error codes.
-                else:
-                    reason = _explain_taskvine_result(t)
-                    logger.debug("Did not find result in {}".format(result_file))
-                    logger.debug("Wrapper Script status: {}\nTaskVine Status: {}"
-                                 .format(t.exit_code, t.result))
-                    logger.debug("Task with executor id {} / vine id {} failed because:\n{}"
-                                 .format(executor_task_id, t.id, reason))
-                    finished_task_queue.put_nowait(VineTaskToParsl(executor_id=executor_task_id,
-                                                                   result_received=False,
-                                                                   result_file=None,
-                                                                   reason=reason,
-                                                                   status=t.exit_code))
+        while not m.empty() and task_found and not should_stop.is_set():
+            # Obtain the task from the queue
+            t = m.wait(1)
+            if t is None:
+                task_found = False
+                continue
+            logger.debug('Found a task')
+            executor_task_id = vine_id_to_executor_task_id[str(t.id)][0]
+            vine_id_to_executor_task_id.pop(str(t.id))
+
+            # When a task is found
+            result_file = result_file_of_task_id.pop(executor_task_id)
+
+            logger.debug(f"completed executor task info: {executor_task_id}, {t.category}, {t.command}, {t.std_output}")
+
+            # A tasks completes 'succesfully' if it has result file.
+            # A check whether the Python object represented using this file can be
+            # deserialized happens later in the collector thread of the executor
+            # process.
+            logger.debug("Looking for result in {}".format(result_file))
+            if os.path.exists(result_file):
+                logger.debug("Found result in {}".format(result_file))
+                finished_task_queue.put_nowait(VineTaskToParsl(executor_id=executor_task_id,
+                                                               result_received=True,
+                                                               result_file=result_file,
+                                                               reason=None,
+                                                               status=t.exit_code))
+            # If a result file could not be generated, explain the
+            # failure according to taskvine error codes.
+            else:
+                reason = _explain_taskvine_result(t)
+                logger.debug("Did not find result in {}".format(result_file))
+                logger.debug("Wrapper Script status: {}\nTaskVine Status: {}".format(t.exit_code, t.result))
+                logger.debug("Task with executor id {} / vine id {} failed because:\n{}".format(executor_task_id, t.id, reason))
+                finished_task_queue.put_nowait(VineTaskToParsl(executor_id=executor_task_id,
+                                                               result_received=False,
+                                                               result_file=None,
+                                                               reason=reason,
+                                                               status=t.exit_code))
 
     logger.debug("Exiting TaskVine Monitoring Process")
     return 0

parsl/executors/taskvine/manager_config.py

@@ -84,6 +84,19 @@ class TaskVineManagerConfig:
         forever.
         Default is 1.
 
+    library_config: Optional[dict]
+        Only and must specify when functions are executed in the serverless mode.
+        Configure the number of function slots and amount of resources
+        a library task can run. A library task is a stateful object that executes
+        functions in the serverless way. Accept the following keywords:
+        'num_slots', 'cores', 'memory (MBs)', 'disk (MBs)'.
+        Default is {'num_slots': 1, 'cores': None, 'memory': None, 'disk': None},
+        which will take all resources of a worker node and run at most 1 function
+        invocation at any given time.
+        E.g., {'num_slots': 4, 'cores': 16, 'memory': 16000, 'disk': 16000} will
+        reserve those resources to the library task to run at most 4 function
+        invocations.
+
     shared_fs: bool
         Whether workers will use a shared filesystem or not. If so, TaskVine
         will not track and transfer files for execution, in exchange for
@@ -159,6 +172,7 @@ class TaskVineManagerConfig:
     app_pack: bool = False
     extra_pkgs: Optional[list] = None
     max_retries: int = 1
+    library_config: Optional[dict] = None
 
     # Performance-specific settings
     shared_fs: bool = False

parsl/monitoring/monitoring.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import socket
 import time
@@ -11,7 +13,8 @@ import queue
 import parsl.monitoring.remote
 
 from parsl.multiprocessing import ForkProcess, SizedQueue
-from multiprocessing import Process, Queue
+from multiprocessing import Process
+from multiprocessing.queues import Queue
 from parsl.utils import RepresentationMixin
 from parsl.process_loggers import wrap_with_logs
 from parsl.utils import setproctitle
@@ -20,7 +23,7 @@ from parsl.serialize import deserialize
 
 from parsl.monitoring.message_type import MessageType
 from parsl.monitoring.types import AddressedMonitoringMessage, TaggedMonitoringMessage
-from typing import cast, Any, Callable, Dict, Optional, Sequence, Tuple, Union
+from typing import cast, Any, Callable, Dict, Optional, Sequence, Tuple, Union, TYPE_CHECKING
 
 _db_manager_excepts: Optional[Exception]
 
@@ -125,7 +128,10 @@ class MonitoringHub(RepresentationMixin):
              This will include environment information such as start time, hostname and block id,
              along with periodic resource usage of each task. Default: True
         resource_monitoring_interval : float
-             The time interval, in seconds, at which the monitoring records the resource usage of each task. Default: 30 seconds
+             The time interval, in seconds, at which the monitoring records the resource usage of each task.
+             If set to 0, only start and end information will be logged, and no periodic monitoring will
+             be made.
+             Default: 30 seconds
         """
 
         self.logger = logger
@@ -168,7 +174,19 @@ class MonitoringHub(RepresentationMixin):
         self.logger.debug("Initializing ZMQ Pipes to client")
         self.monitoring_hub_active = True
 
-        comm_q: Queue[Union[Tuple[int, int], str]]
+        # This annotation is incompatible with typeguard 4.x instrumentation
+        # of local variables: Queue is not subscriptable at runtime, as far
+        # as typeguard is concerned. The more general Queue annotation works,
+        # but does not restrict the contents of the Queue. Using TYPE_CHECKING
+        # here allows the stricter definition to be seen by mypy, and the
+        # simpler definition to be seen by typeguard. Hopefully at some point
+        # in the future, Queue will allow runtime subscripts.
+
+        if TYPE_CHECKING:
+            comm_q: Queue[Union[Tuple[int, int], str]]
+        else:
+            comm_q: Queue
+
         comm_q = SizedQueue(maxsize=10)
 
         self.exception_q: Queue[Tuple[str, str]]

parsl/monitoring/remote.py

@@ -43,7 +43,7 @@ def monitor_wrapper(f: Any,           # per app
                            radio_mode,
                            run_dir)
 
-        if monitor_resources:
+        if monitor_resources and sleep_dur > 0:
             # create the monitor process and start
             pp = ForkProcess(target=monitor,
                              args=(os.getpid(),

parsl/providers/errors.py

@@ -51,20 +51,18 @@ class SubmitException(ExecutionProviderException):
     '''Raised by the submit() method of a provider if there is an error in launching a job.
     '''
 
-    def __init__(self, job_name, message, stdout=None, stderr=None):
+    def __init__(self, job_name, message, stdout=None, stderr=None, retcode=None):
         self.job_name = job_name
         self.message = message
         self.stdout = stdout
         self.stderr = stderr
+        self.retcode = retcode
 
     @property
     def task_name(self) -> str:
         warnings.warn("task_name is deprecated; use .job_name instead. This will be removed after 2024-06.", DeprecationWarning)
         return self.job_name
 
-    def __str__(self):
+    def __str__(self) -> str:
         # TODO: make this more user-friendly
-        return "Cannot launch job {0}: {1}; stdout={2}, stderr={3}".format(self.job_name,
-                                                                           self.message,
-                                                                           self.stdout,
-                                                                           self.stderr)
+        return f"Cannot launch job {self.job_name}: {self.message}; recode={self.retcode}, stdout={self.stdout}, stderr={self.stderr}"

parsl/providers/slurm/slurm.py

@@ -13,6 +13,7 @@ from parsl.jobs.states import JobState, JobStatus
 from parsl.launchers import SingleNodeLauncher
 from parsl.launchers.base import Launcher
 from parsl.providers.cluster_provider import ClusterProvider
+from parsl.providers.errors import SubmitException
 from parsl.providers.slurm.template import template_string
 from parsl.utils import RepresentationMixin, wtime_to_minutes
 
@@ -194,7 +195,7 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
             logger.debug("Updating missing job {} to completed status".format(missing_job))
             self.resources[missing_job]['status'] = JobStatus(JobState.COMPLETED)
 
-    def submit(self, command, tasks_per_node, job_name="parsl.slurm"):
+    def submit(self, command: str, tasks_per_node: int, job_name="parsl.slurm") -> str:
         """Submit the command as a slurm job.
 
         Parameters
@@ -207,8 +208,8 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
             Name for the job
         Returns
         -------
-        None or str
-            If at capacity, returns None; otherwise, a string identifier for the job
+        job id : str
+            A string identifier for the job
         """
 
         scheduler_options = self.scheduler_options
@@ -254,21 +255,21 @@ class SlurmProvider(ClusterProvider, RepresentationMixin):
 
         retcode, stdout, stderr = self.execute_wait("sbatch {0}".format(channel_script_path))
 
-        job_id = None
         if retcode == 0:
             for line in stdout.split('\n'):
                 match = re.match(self.regex_job_id, line)
                 if match:
                     job_id = match.group("id")
                     self.resources[job_id] = {'job_id': job_id, 'status': JobStatus(JobState.PENDING)}
-                    break
+                    return job_id
             else:
                 logger.error("Could not read job ID from submit command standard output.")
                 logger.error("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
+                raise SubmitException(job_name, "Could not read job ID from submit command standard output", stdout=stdout, stderr=stderr, retcode=retcode)
         else:
             logger.error("Submit command failed")
             logger.error("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
-        return job_id
+            raise SubmitException(job_name, "Could not read job ID from submit command standard output", stdout=stdout, stderr=stderr, retcode=retcode)
 
     def cancel(self, job_ids):
         ''' Cancels the jobs specified by a list of job ids

parsl/tests/configs/ad_hoc_cluster_htex.py

@@ -18,6 +18,7 @@ config = Config(
             label='remote_htex',
             max_workers=2,
             worker_logdir_root=user_opts['adhoc']['script_dir'],
+            encrypted=True,
             provider=AdHocProvider(
                 # Command to be run before starting a worker, such as:
                 # 'module load Anaconda; source activate parsl_env'.

parsl/tests/configs/azure_single_node.py

@@ -40,6 +40,7 @@ config = Config(
             storage_access=[HTTPInTaskStaging(), FTPInTaskStaging(), RSyncStaging(getpass.getuser() + "@" + user_opts['public_ip'])],
             label='azure_single_node',
             address=user_opts['public_ip'],
+            encrypted=True,
             provider=provider
         )
     ]

parsl/tests/configs/bluewaters.py

@@ -14,6 +14,7 @@ def fresh_config():
                 cores_per_worker=1,
                 worker_debug=False,
                 max_workers=1,
+                encrypted=True,
                 provider=TorqueProvider(
                     queue='normal',
                     launcher=AprunLauncher(overrides="-b -- bwpy-environ --"),

parsl/tests/configs/bridges.py

@@ -14,6 +14,7 @@ def fresh_config():
                 # which compute nodes can communicate
                 # address=address_by_interface('bond0.144'),
                 max_workers=1,
+                encrypted=True,
                 provider=SlurmProvider(
                     user_opts['bridges']['partition'],  # Partition / QOS
                     nodes_per_block=2,

parsl/tests/configs/cc_in2p3.py

@@ -12,6 +12,7 @@ def fresh_config():
             HighThroughputExecutor(
                 label='cc_in2p3_htex',
                 max_workers=1,
+                encrypted=True,
                 provider=GridEngineProvider(
                     channel=LocalChannel(),
                     nodes_per_block=2,

parsl/tests/configs/comet.py

@@ -11,6 +11,7 @@ def fresh_config():
             HighThroughputExecutor(
                 label='Comet_HTEX_multinode',
                 max_workers=1,
+                encrypted=True,
                 provider=SlurmProvider(
                     'debug',
                     launcher=SrunLauncher(),

parsl/tests/configs/cooley_htex.py

@@ -18,6 +18,7 @@ config = Config(
             label="cooley_htex",
             worker_debug=False,
             cores_per_worker=1,
+            encrypted=True,
             provider=CobaltProvider(
                 queue='debug',
                 account=user_opts['cooley']['account'],

parsl/tests/configs/ec2_single_node.py

@@ -28,6 +28,7 @@ config = Config(
         HighThroughputExecutor(
             label='ec2_single_node',
             address=user_opts['public_ip'],
+            encrypted=True,
             provider=AWSProvider(
                 user_opts['ec2']['image_id'],
                 region=user_opts['ec2']['region'],

parsl/tests/configs/ec2_spot.py

@@ -15,6 +15,7 @@ config = Config(
         HighThroughputExecutor(
             label='ec2_single_node',
             address=user_opts['public_ip'],
+            encrypted=True,
             provider=AWSProvider(
                 user_opts['ec2']['image_id'],
                 region=user_opts['ec2']['region'],

parsl/tests/configs/frontera.py

@@ -16,6 +16,7 @@ def fresh_config():
             HighThroughputExecutor(
                 label="frontera_htex",
                 max_workers=1,
+                encrypted=True,
                 provider=SlurmProvider(
                     cmd_timeout=60,     # Add extra time for slow scheduler responses
                     channel=LocalChannel(),

parsl/tests/configs/htex_ad_hoc_cluster.py

@@ -13,6 +13,7 @@ config = Config(
             cores_per_worker=1,
             worker_debug=False,
             address=user_opts['public_ip'],
+            encrypted=True,
             provider=AdHocProvider(
                 move_files=False,
                 parallelism=1,

parsl/tests/configs/htex_local.py

@@ -13,6 +13,7 @@ def fresh_config():
                 label="htex_local",
                 worker_debug=True,
                 cores_per_worker=1,
+                encrypted=True,
                 provider=LocalProvider(
                     channel=LocalChannel(),
                     init_blocks=1,

parsl/tests/configs/htex_local_alternate.py

@@ -48,6 +48,7 @@ def fresh_config():
                 heartbeat_period=2,
                 heartbeat_threshold=5,
                 poll_period=100,
+                encrypted=True,
                 provider=LocalProvider(
                     channel=LocalChannel(),
                     init_blocks=0,

parsl/tests/configs/htex_local_intask_staging.py

@@ -15,6 +15,7 @@ config = Config(
             label="htex_Local",
             worker_debug=True,
             cores_per_worker=1,
+            encrypted=True,
             provider=LocalProvider(
                 channel=LocalChannel(),
                 init_blocks=1,

parsl/tests/configs/htex_local_rsync_staging.py

@@ -16,6 +16,7 @@ config = Config(
             worker_debug=True,
             cores_per_worker=1,
             working_dir="./rsync-workdir/",
+            encrypted=True,
             provider=LocalProvider(
                 channel=LocalChannel(),
                 init_blocks=1,

parsl/tests/configs/local_adhoc.py

@@ -9,6 +9,7 @@ def fresh_config():
         executors=[
             HighThroughputExecutor(
                 label='AdHoc',
+                encrypted=True,
                 provider=AdHocProvider(
                     channels=[LocalChannel(), LocalChannel()]
                 )

parsl/tests/configs/midway.py

@@ -13,6 +13,7 @@ def fresh_config():
                 label='Midway_HTEX_multinode',
                 worker_debug=False,
                 max_workers=1,
+                encrypted=True,
                 provider=SlurmProvider(
                     'broadwl',  # Partition name, e.g 'broadwl'
                     launcher=SrunLauncher(),

parsl/tests/configs/nscc_singapore.py

@@ -17,6 +17,7 @@ def fresh_config():
                 worker_debug=False,
                 max_workers=1,
                 address=address_by_interface('ib0'),
+                encrypted=True,
                 provider=PBSProProvider(
                     launcher=MpiRunLauncher(),
                     # string to prepend to #PBS blocks in the submit

parsl/tests/configs/osg_htex.py

@@ -14,6 +14,7 @@ config = Config(
         HighThroughputExecutor(
             label='OSG_HTEX',
             max_workers=1,
+            encrypted=True,
             provider=CondorProvider(
                 nodes_per_block=1,
                 init_blocks=4,

parsl/tests/configs/petrelkube.py

@@ -23,6 +23,7 @@ def fresh_config():
 
                 # Address for the pod worker to connect back
                 address=address_by_route(),
+                encrypted=True,
                 provider=KubernetesProvider(
                     namespace="dlhub-privileged",
 

parsl/tests/configs/summit.py

@@ -21,6 +21,7 @@ def fresh_config():
                 # address=address_by_interface('ib0'),  # This assumes Parsl is running on login node
                 worker_port_range=(50000, 55000),
                 max_workers=1,
+                encrypted=True,
                 provider=LSFProvider(
                     launcher=JsrunLauncher(),
                     walltime="00:10:00",

parsl/tests/configs/swan_htex.py

@@ -24,6 +24,7 @@ config = Config(
     executors=[
         HighThroughputExecutor(
             label='swan_htex',
+            encrypted=True,
             provider=TorqueProvider(
                 channel=SSHChannel(
                     hostname='swan.cray.com',

parsl/tests/configs/theta.py

@@ -12,6 +12,7 @@ def fresh_config():
             HighThroughputExecutor(
                 label='theta_local_htex_multinode',
                 max_workers=1,
+                encrypted=True,
                 provider=CobaltProvider(
                     queue=user_opts['theta']['queue'],
                     account=user_opts['theta']['account'],