parsl 2023.10.2__tar.gz → 2023.10.16__tar.gz

{parsl-2023.10.2/parsl.egg-info → parsl-2023.10.16}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.10.2
+Version: 2023.10.16
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.10.02.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.10.16.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/benchmark/perf.py

@@ -48,7 +48,7 @@ def performance(*, resources: dict, target_t: float):
 
         submitted_t = time.time()
         print(f"All {n} tasks submitted ... waiting for completion")
-        print(f"Submission took {submitted_t - start_t:.3f} seconds = {n/(submitted_t - start_t):.3f} tasks/second")
+        print(f"Submission took {submitted_t - start_t:.3f} seconds = {n / (submitted_t - start_t):.3f} tasks/second")
 
         for f in concurrent.futures.as_completed(fs):
             assert f.result() == 7
@@ -62,7 +62,7 @@ def performance(*, resources: dict, target_t: float):
         print(f"Runtime: actual {delta_t:.3f}s vs target {target_t}s")
         print(f"Tasks per second: {rate:.3f}")
 
-        n = int(target_t * rate)
+        n = max(1, int(target_t * rate))
 
         iteration += 1
 

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/executors/taskvine/exec_parsl_function.py

@@ -1,13 +1,15 @@
-from parsl.app.errors import RemoteExceptionWrapper
-from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
 import traceback
 import sys
+
 import pickle
+from parsl.app.errors import RemoteExceptionWrapper
+from parsl.data_provider.files import File
+from parsl.utils import get_std_fname_mode
+from parsl.serialize import deserialize
 
-# This scripts executes a parsl function which is pickled in 3 files:
+# This scripts executes a parsl function which is pickled in 4 files:
 #
-# exec_parsl_function.py map_file function_file result_file
+# exec_parsl_function.py map_file function_file argument_file result_file
 #
 # map_file: Contains a pickled dictionary that indicates which local_paths the
 #           parsl Files should take.
@@ -15,6 +17,8 @@ import pickle
 # function_file: Contains a pickle parsl function. Function might be serialized in advance.
 # See @parsl.serialize.concretes.py
 #
+# argument_file: Contains the serialized arguments to the function call.
+#
 # result_file: A file path, whose content will contain the result of the function, including any
 #              exception generated. Exceptions will be wrapped with RemoteExceptionWrapper.
 #
@@ -26,12 +30,6 @@ import pickle
 #
 
 
-def load_pickled_file(filename: str):
-    """ Load a pickled file and return its pickled object."""
-    with open(filename, "rb") as f_in:
-        return pickle.load(f_in)
-
-
 def dump_result_to_file(result_file: str, result_package):
     """ Dump a result to the given result file."""
     with open(result_file, "wb") as f_out:
@@ -78,9 +76,9 @@ def remap_all_files(mapping, fn_args, fn_kwargs):
             remap_location(mapping, maybe_file)
 
 
-def unpack_object(serialized_obj, user_namespace):
-    from parsl.serialize import deserialize
-    obj = deserialize(serialized_obj)
+def unpack_object_from_file(path):
+    with open(path, 'rb') as f:
+        obj = deserialize(f.read())
     return obj
 
 
@@ -115,26 +113,23 @@ def encode_byte_code_function(user_namespace, fn, fn_name, args_name, kwargs_nam
 def load_function(map_file, function_file, argument_file):
     # Decodes the function and its file arguments to be executed into
     # function_code, and updates a user namespace with the function name and
-    # the variable named result_name. When the function is executed, its result
+    # the variable named `result_name`. When the function is executed, its result
     # will be stored in this variable in the user namespace.
     # Returns (namespace, function_code, result_name)
 
-    # Create the namespace to isolate the function execution.
-    user_ns = locals()
-    user_ns.update({'__builtins__': __builtins__})
-
-    packed_function = load_pickled_file(function_file)
-    packed_argument = load_pickled_file(argument_file)
-
-    fn = unpack_object(packed_function, user_ns)
-    args_dict = unpack_object(packed_argument, user_ns)
+    fn = unpack_object_from_file(function_file)
+    args_dict = unpack_object_from_file(argument_file)
     fn_args = args_dict['args']
     fn_kwargs = args_dict['kwargs']
     fn_name = 'parsl_tmp_func_name'
 
-    mapping = load_pickled_file(map_file)
+    mapping = unpack_object_from_file(map_file)
     remap_all_files(mapping, fn_args, fn_kwargs)
 
+    # Create the namespace to isolate the function execution.
+    user_ns = locals()
+    user_ns.update({'__builtins__': __builtins__})
+
     (code, result_name) = encode_function(user_ns, fn, fn_name, fn_args, fn_kwargs)
 
     return (user_ns, code, result_name)

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/executors/taskvine/executor.py

@@ -11,15 +11,13 @@ import tempfile
 import hashlib
 import subprocess
 import os
-import pickle
 import queue
 import inspect
 import shutil
 import itertools
 import uuid
-from ctypes import c_bool
 from concurrent.futures import Future
-from typing import List, Optional, Union
+from typing import List, Optional, Union, Literal
 
 # Import Parsl constructs
 import parsl.utils as putils
@@ -71,12 +69,17 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             with respect to other executors.
             Default is "TaskVineExecutor".
 
-        worker_launch_method: str
+        worker_launch_method: Union[Literal['provider'], Literal['factory'], Literal['manual']]
             Choose to use Parsl provider, TaskVine factory, or
             manual user-provided workers to scale workers.
             Options are among {'provider', 'factory', 'manual'}.
             Default is 'factory'.
 
+        function_exec_mode: Union[Literal['regular'], Literal['serverless']]
+            Choose to execute functions with a regular fresh python process or a
+            pre-warmed forked python process.
+            Default is 'regular'.
+
         manager_config: TaskVineManagerConfig
             Configuration for the TaskVine manager. Default
 
@@ -98,7 +101,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
     @typeguard.typechecked
     def __init__(self,
                  label: str = "TaskVineExecutor",
-                 worker_launch_method: str = 'factory',
+                 worker_launch_method: Union[Literal['provider'], Literal['factory'], Literal['manual']] = 'factory',
+                 function_exec_mode: Union[Literal['regular'], Literal['serverless']] = 'regular',
                  manager_config: TaskVineManagerConfig = TaskVineManagerConfig(),
                  factory_config: TaskVineFactoryConfig = TaskVineFactoryConfig(),
                  provider: Optional[ExecutionProvider] = LocalProvider(init_blocks=1),
@@ -107,9 +111,6 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Set worker launch option for this executor
         if worker_launch_method == 'factory' or worker_launch_method == 'manual':
             provider = None
-        elif worker_launch_method != 'provider':
-            raise ExecutorError(self, "Worker launch option '{worker_launch_method}' \
-                                       is not supported.")
 
         # Initialize the parent class with the execution provider and block error handling enabled.
         # If provider is None, then no worker is launched via the provider method.
@@ -126,6 +127,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Executor configurations
         self.label = label
         self.worker_launch_method = worker_launch_method
+        self.function_exec_mode = function_exec_mode
         self.manager_config = manager_config
         self.factory_config = factory_config
         self.storage_access = storage_access
@@ -136,8 +138,8 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Queue to send finished tasks from TaskVine manager process to TaskVine executor process
         self._finished_task_queue: multiprocessing.Queue = multiprocessing.Queue()
 
-        # Value to signal whether the manager and factory processes should stop running
-        self._should_stop = multiprocessing.Value(c_bool, False)
+        # Event to signal whether the manager and factory processes should stop running
+        self._should_stop = multiprocessing.Event()
 
         # TaskVine manager process
         self._submit_process = None
@@ -267,10 +269,11 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Begin work
         self._submit_process.start()
 
-        # Run worker scaler either with Parsl provider or TaskVine factory
+        # Run worker scaler either with Parsl provider or TaskVine factory.
+        # Skip if workers are launched manually.
         if self.worker_launch_method == 'factory':
             self._factory_process.start()
-        else:
+        elif self.worker_launch_method == 'provider':
             self.initialize_scaling()
 
         self._collector_thread.start()
@@ -314,7 +317,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         logger.debug(f'Got resource specification: {resource_specification}')
 
         # Default execution mode of apps is regular
-        exec_mode = resource_specification.get('exec_mode', 'regular')
+        exec_mode = resource_specification.get('exec_mode', self.function_exec_mode)
 
         # Detect resources and features of a submitted Parsl app
         cores = None
@@ -391,9 +394,9 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
                 and result to be found at: {}".format(executor_task_id, function_file, argument_file, result_file))
 
         # Serialize function object and arguments, separately
-        self._serialize_object(function_file, func)
+        self._serialize_object_to_file(function_file, func)
         args_dict = {'args': args, 'kwargs': kwargs}
-        self._serialize_object(argument_file, args_dict)
+        self._serialize_object_to_file(argument_file, args_dict)
 
         # Construct the map file of local filenames at worker
         self._construct_map_file(map_file, input_files, output_files)
@@ -466,11 +469,13 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             if self.project_password_file:
                 self.provider.transfer_input_files.append(self.project_password_file)
 
-    def _serialize_object(self, path, obj):
+    def _serialize_object_to_file(self, path, obj):
         """Takes any object and serializes it to the file path."""
         serialized_obj = serialize(obj, buffer_threshold=1024 * 1024)
         with open(path, 'wb') as f_out:
-            pickle.dump(serialized_obj, f_out)
+            written = 0
+            while written < len(serialized_obj):
+                written += f_out.write(serialized_obj[written:])
 
     def _construct_map_file(self, map_file, input_files, output_files):
         """ Map local filepath of parsl files to the filenames at the execution worker.
@@ -485,8 +490,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
             else:
                 remote_name = local_name
             file_translation_map[local_name] = remote_name
-        with open(map_file, "wb") as f_out:
-            pickle.dump(file_translation_map, f_out)
+        self._serialize_object_to_file(map_file, file_translation_map)
 
     def _register_file(self, parsl_file):
         """Generates a tuple (parsl_file.filepath, stage, cache) to give to
@@ -592,7 +596,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         collector thread, which shuts down the TaskVine system submission.
         """
         logger.debug("TaskVine shutdown started")
-        self._should_stop.value = True
+        self._should_stop.set()
 
         # Remove the workers that are still going
         kill_ids = [self.blocks[block] for block in self.blocks.keys()]
@@ -618,7 +622,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """
         logger.debug("Starting Collector Thread")
         try:
-            while not self._should_stop.value:
+            while not self._should_stop.is_set():
                 if not self._submit_process.is_alive():
                     raise ExecutorError(self, "taskvine Submit Process is not alive")
 

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/executors/taskvine/factory.py

@@ -1,16 +1,24 @@
 import logging
-import time
 
 from parsl.process_loggers import wrap_with_logs
 from parsl.executors.taskvine.errors import TaskVineFactoryFailure
 
-from ndcctools.taskvine import Factory
+# This try except clause prevents import errors
+# when TaskVine is not used in Parsl.
+try:
+    from ndcctools.taskvine import Factory
+    taskvine_available = True
+except ImportError:
+    taskvine_available = False
 
 logger = logging.getLogger(__name__)
 
 
 @wrap_with_logs
 def _taskvine_factory(should_stop, factory_config):
+    if not taskvine_available:
+        logger.debug("TaskVine package cannot be found. Please install the ndcctools package.")
+        return
     logger.debug("Starting TaskVine factory process")
 
     try:
@@ -54,11 +62,8 @@ def _taskvine_factory(should_stop, factory_config):
     if factory_config.batch_options:
         factory.batch_options = factory_config.batch_options
 
-    # setup factory context and sleep for a second in every loop to
-    # avoid wasting CPU
+    # run factory through Python context and wait for signal to stop.
     with factory:
-        while not should_stop.value:
-            time.sleep(1)
+        should_stop.wait()
 
     logger.debug("Exiting TaskVine factory process")
-    return 0

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/executors/taskvine/manager.py

@@ -106,6 +106,8 @@ def _prepare_environment_regular(m, manager_config, t, task, poncho_env_to_file,
                 env_tarball = str(uuid.uuid4()) + '.tar.gz'
                 logger.debug(f'Creating a poncho environment at {env_tarball} from conda environment {manager_config.env_pack}')
                 subprocess.run([poncho_create_script, manager_config.env_pack, env_tarball], stdout=subprocess.DEVNULL, check=True)
+            else:
+                env_tarball = manager_config.env_pack
             poncho_env_file = m.declare_poncho(env_tarball, cache=True, peer_transfer=True)
             poncho_env_to_file[manager_config.env_pack] = poncho_env_file
         else:
@@ -194,15 +196,15 @@ def _taskvine_submit_wait(ready_task_queue=None,
 
     logger.debug("Entering main loop of TaskVine manager")
 
-    while not should_stop.value:
-        # Monitor the task queue
+    while not should_stop.is_set():
+        # Check if executor process is still running
         ppid = os.getppid()
         if ppid != orig_ppid:
-            logger.debug("new Process")
+            logger.debug("Executor process is detected to have exited. Exiting..")
             break
 
         # Submit tasks
-        while ready_task_queue.qsize() > 0 and not should_stop.value:
+        while ready_task_queue.qsize() > 0 and not should_stop.is_set():
             # Obtain task from ready_task_queue
             try:
                 task = ready_task_queue.get(timeout=1)
@@ -377,7 +379,7 @@ def _taskvine_submit_wait(ready_task_queue=None,
         # If the queue is not empty wait on the TaskVine queue for a task
         task_found = True
         if not m.empty():
-            while task_found and not should_stop.value:
+            while task_found and not should_stop.is_set():
                 # Obtain the task from the queue
                 t = m.wait(1)
                 if t is None:

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/executors/taskvine/manager_config.py

@@ -2,7 +2,12 @@ import socket
 from dataclasses import dataclass
 from typing import Optional
 
-from ndcctools.taskvine.cvine import VINE_DEFAULT_PORT
+# This try except clause prevents import errors
+# when TaskVine is not used in Parsl.
+try:
+    from ndcctools.taskvine.cvine import VINE_DEFAULT_PORT
+except ImportError:
+    VINE_DEFAULT_PORT = 0   # use any available port.
 
 
 @dataclass

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/executors/threads.py

@@ -18,7 +18,7 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
 
     Parameters
     ----------
-    max_threads : int
+    max_threads : Optional[int]
         Number of threads. Default is 2.
     thread_name_prefix : string
         Thread name prefix
@@ -27,7 +27,7 @@ class ThreadPoolExecutor(ParslExecutor, RepresentationMixin):
     """
 
     @typeguard.typechecked
-    def __init__(self, label: str = 'threads', max_threads: int = 2,
+    def __init__(self, label: str = 'threads', max_threads: Optional[int] = 2,
                  thread_name_prefix: str = '', storage_access: Optional[List[Staging]] = None,
                  working_dir: Optional[str] = None):
         ParslExecutor.__init__(self)

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/tests/test_bash_apps/test_error_codes.py

@@ -74,7 +74,6 @@ def test_div_0(test_fn=div_0):
     print(os.listdir('.'))
     os.remove('std.err')
     os.remove('std.out')
-    return True
 
 
 @pytest.mark.issue363

{parsl-2023.10.2 → parsl-2023.10.16}/parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2023.10.02'
+VERSION = '2023.10.16'

{parsl-2023.10.2 → parsl-2023.10.16/parsl.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2023.10.2
+Version: 2023.10.16
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2023.10.02.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2023.10.16.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0

{parsl-2023.10.2 → parsl-2023.10.16}/parsl.egg-info/SOURCES.txt

@@ -326,7 +326,6 @@ parsl/tests/test_docs/test_from_slides.py
 parsl/tests/test_docs/test_tutorial_1.py
 parsl/tests/test_docs/test_workflow1.py
 parsl/tests/test_docs/test_workflow2.py
-parsl/tests/test_docs/test_workflow3.py
 parsl/tests/test_docs/test_workflow4.py
 parsl/tests/test_error_handling/__init__.py
 parsl/tests/test_error_handling/test_fail.py

parsl-2023.10.2/parsl/tests/test_docs/test_workflow3.py

@@ -1,23 +0,0 @@
-import parsl
-
-from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import config
-
-
-@python_app
-def generate(limit):
-    from random import randint
-    """Generate a random integer and return it"""
-    return randint(1, limit)
-
-
-def test_parallel_for(N=2):
-    """Test parallel workflows from docs on Composing workflows
-    """
-    rand_nums = []
-    for i in range(1, 5):
-        rand_nums.append(generate(i))
-
-    # wait for all apps to finish and collect the results
-    outputs = [i.result() for i in rand_nums]
-    return outputs