parsl 2023.5.29__py3-none-any.whl → 2023.6.12__py3-none-any.whl

parsl/addresses.py

@@ -113,7 +113,8 @@ def get_all_addresses() -> Set[str]:
             logger.exception("Ignoring failure to fetch address from interface {}".format(interface))
             pass
 
-    resolution_functions = [address_by_hostname, address_by_route, address_by_query]  # type: List[Callable[[], str]]
+    resolution_functions: List[Callable[[], str]]
+    resolution_functions = [address_by_hostname, address_by_route, address_by_query]
     for f in resolution_functions:
         try:
             s_addresses.add(f())

parsl/app/errors.py

@@ -1,6 +1,7 @@
 """Exceptions raised by Apps."""
 from functools import wraps
-from typing import Callable, List, Union, Any, TypeVar, Optional
+from typing import Callable, List, Optional, TypeVar, Union
+from typing_extensions import ParamSpec
 from types import TracebackType
 import logging
 from tblib import Traceback
@@ -132,28 +133,13 @@ class RemoteExceptionWrapper:
             return v
 
 
+P = ParamSpec('P')
 R = TypeVar('R')
 
-# There appears to be no solution to typing this without a mypy plugin.
-# The reason is because wrap_error maps a Callable[[X...], R] to a Callable[[X...], Union[R, R2]].
-# However, there is no provision in Python typing for pattern matching all possible types of
-# callable arguments. This is because Callable[] is, in the infinite wisdom of the typing module,
-# only used for callbacks: "There is no syntax to indicate optional or keyword arguments; such
-# function types are rarely used as callback types.".
-# The alternative supported by the typing module, of saying Callable[..., R] ->
-#   Callable[..., Union[R, R2]] results in no pattern matching between the first and second
-# ellipsis.
-# Yet another bogus solution that was here previously would simply define wrap_error as
-#   wrap_error(T) -> T, where T was a custom TypeVar. This obviously missed the fact that
-# the returned function had its return signature modified.
-# Ultimately, the best choice appears to be Callable[..., R] -> Callable[..., Union[R, ?Exception]],
-#  since it results in the correct type specification for the return value(s) while treating the
-#  arguments as Any.
-
-
-def wrap_error(func: Callable[..., R]) -> Callable[..., Union[R, RemoteExceptionWrapper]]:
+
+def wrap_error(func: Callable[P, R]) -> Callable[P, Union[R, RemoteExceptionWrapper]]:
     @wraps(func)
-    def wrapper(*args: object, **kwargs: object) -> Any:
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> Union[R, RemoteExceptionWrapper]:
         import sys
         from parsl.app.errors import RemoteExceptionWrapper
         try:

parsl/benchmark/perf.py

@@ -4,6 +4,8 @@ import time
 import concurrent.futures
 import parsl
 
+min_iterations = 2
+
 
 # TODO: factor with conftest.py where this is copy/pasted from?
 def load_dfk_from_config(filename):
@@ -34,7 +36,7 @@ def performance(*, resources: dict, target_t: float):
 
     iteration = 1
 
-    while delta_t < threshold_t:
+    while delta_t < threshold_t or iteration <= min_iterations:
         print(f"==== Iteration {iteration} ====")
         print(f"Will run {n} tasks to target {target_t} seconds runtime")
         start_t = time.time()

parsl/configs/vineex_local.py

@@ -0,0 +1,26 @@
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor
+
+import uuid
+
+config = Config(
+    executors=[
+        TaskVineExecutor(
+            label="parsl-vine-example",
+
+            # If a project_name is given, then TaskVine will periodically
+            # report its status and performance back to the global TaskVine catalog,
+            # which can be viewed here:  http://ccl.cse.nd.edu/software/taskvine/status
+
+            # To disable status reporting, comment out the project_name.
+            project_name="parsl-vine-" + str(uuid.uuid4()),
+
+            # The port number that TaskVine will listen on for connecting workers
+            # 0 means a random port.
+            port=0,
+
+            # A shared filesystem is not needed when using TaskVine.
+            shared_fs=False
+        )
+    ]
+)

parsl/data_provider/data_manager.py

@@ -16,7 +16,8 @@ logger = logging.getLogger(__name__)
 
 # these will be shared between all executors that do not explicitly
 # override, so should not contain executor-specific state
-default_staging = [NoOpFileStaging(), FTPSeparateTaskStaging(), HTTPSeparateTaskStaging()]  # type: List[Staging]
+default_staging: List[Staging]
+default_staging = [NoOpFileStaging(), FTPSeparateTaskStaging(), HTTPSeparateTaskStaging()]
 
 
 class DataManager:

parsl/data_provider/files.py

@@ -44,7 +44,7 @@ class File:
         self.netloc = parsed_url.netloc
         self.path = parsed_url.path
         self.filename = os.path.basename(self.path)
-        self.local_path = None  # type: Optional[str]
+        self.local_path: Optional[str] = None
 
     def cleancopy(self) -> "File":
         """Returns a copy of the file containing only the global immutable state,

parsl/data_provider/globus.py

@@ -94,7 +94,7 @@ class Globus:
         with 60 second timeout limit. If the task is ACTIVE after time runs out 'task_wait' returns False,
         and True otherwise.
         """
-        while not tc.task_wait(task['task_id'], 60, 15):
+        while not tc.task_wait(task['task_id'], timeout=60):
             task = tc.get_task(task['task_id'])
             # Get the last error Globus event
             events = tc.task_event_list(task['task_id'], num_results=1, filter='is_error:1')

parsl/dataflow/memoization.py

@@ -176,7 +176,7 @@ class Memoizer:
             - hash (str) : A unique hash string
         """
 
-        t = []  # type: List[bytes]
+        t: List[bytes] = []
 
         # if kwargs contains an outputs parameter, that parameter is removed
         # and normalised differently - with output_ref set to True.

parsl/executors/taskvine/__init__.py

@@ -0,0 +1,3 @@
+from parsl.executors.taskvine.executor import TaskVineExecutor
+
+__all__ = ['TaskVineExecutor']

parsl/executors/taskvine/errors.py

@@ -0,0 +1,22 @@
+from parsl.errors import ParslError
+from parsl.app.errors import AppException
+
+
+class TaskVineTaskFailure(AppException):
+    """A failure executing a task in taskvine
+
+    Contains:
+    reason(string)
+    status(int)
+    """
+
+    def __init__(self, reason, status):
+        self.reason = reason
+        self.status = status
+
+
+class TaskVineFailure(ParslError):
+    """A failure in the taskvine executor that prevented the task to be
+    executed.""
+    """
+    pass

parsl/executors/taskvine/exec_parsl_function.py

@@ -0,0 +1,207 @@
+from parsl.app.errors import RemoteExceptionWrapper
+from parsl.data_provider.files import File
+from parsl.utils import get_std_fname_mode
+import traceback
+import sys
+import pickle
+
+# This scripts executes a parsl function which is pickled in a file:
+#
+# exec_parsl_function.py map_file function_file result_file
+#
+# map_file: Contains a pickled dictionary that indicates which local_paths the
+#           parsl Files should take.
+#
+# function_file: Contains a pickle parsl function.
+#
+# result_file: It will contain the result of the function, including any
+#              exception generated. Exceptions will be wrapped with RemoteExceptionWrapper.
+#
+# Exit codes:
+# 0: The function was evaluated to completion. The result or any exception
+#    wrapped with RemoteExceptionWrapper were written to result_file.
+# anything else: There was an error that prevented writing to the result file altogether.
+#                The exit code corresponds to whatever the python interpreter gives.
+#
+
+
+def load_pickled_file(filename):
+    with open(filename, "rb") as f_in:
+        return pickle.load(f_in)
+
+
+def dump_result_to_file(result_file, result_package):
+    with open(result_file, "wb") as f_out:
+        pickle.dump(result_package, f_out)
+
+
+def remap_location(mapping, parsl_file):
+    if not isinstance(parsl_file, File):
+        return
+    # Below we rewrite .local_path when scheme != file only when the local_name
+    # was given by the main parsl process.  This is the case when scheme !=
+    # 'file' but .local_path (via filepath) is in mapping.
+    if parsl_file.scheme == 'file' or parsl_file.local_path:
+        master_location = parsl_file.filepath
+        if master_location in mapping:
+            parsl_file.local_path = mapping[master_location]
+
+
+def remap_list_of_files(mapping, maybe_files):
+    for maybe_file in maybe_files:
+        remap_location(mapping, maybe_file)
+
+
+def remap_all_files(mapping, fn_args, fn_kwargs):
+    # remap any positional argument given to the function that looks like a
+    # File
+    remap_list_of_files(mapping, fn_args)
+
+    # remap any keyword argument in the same way, but we need to treat
+    # "inputs" and "outputs" specially because they are lists, and
+    # "stdout" and "stderr", because they are not File's.
+    for kwarg, maybe_file in fn_kwargs.items():
+        if kwarg in ["inputs", "outputs"]:
+            remap_list_of_files(mapping, maybe_file)
+        if kwarg in ["stdout", "stderr"]:
+            if maybe_file:
+                (fname, mode) = get_std_fname_mode(kwarg, maybe_file)
+                if fname in mapping:
+                    fn_kwargs[kwarg] = (mapping[fname], mode)
+        else:
+            # Treat anything else as a possible File to be remapped.
+            remap_location(mapping, maybe_file)
+
+
+def unpack_function(function_info, user_namespace):
+    if "source code" in function_info:
+        return unpack_source_code_function(function_info, user_namespace)
+    elif "byte code" in function_info:
+        return unpack_byte_code_function(function_info, user_namespace)
+    else:
+        raise ValueError("Function file does not have a valid function representation.")
+
+
+def unpack_source_code_function(function_info, user_namespace):
+    source_code = function_info["source code"]
+    name = function_info["name"]
+    args = function_info["args"]
+    kwargs = function_info["kwargs"]
+    return (source_code, name, args, kwargs)
+
+
+def unpack_byte_code_function(function_info, user_namespace):
+    from parsl.serialize import unpack_apply_message
+    func, args, kwargs = unpack_apply_message(function_info["byte code"], user_namespace, copy=False)
+    return (func, 'parsl_function_name', args, kwargs)
+
+
+def encode_function(user_namespace, fn, fn_name, fn_args, fn_kwargs):
+    # Returns a tuple (code, result_name)
+    # code can be exec in the user_namespace to produce result_name.
+    prefix = "parsl_"
+    args_name = prefix + "args"
+    kwargs_name = prefix + "kwargs"
+    result_name = prefix + "result"
+
+    # Add variables to the namespace to make function call
+    user_namespace.update({args_name: fn_args,
+                           kwargs_name: fn_kwargs,
+                           result_name: result_name})
+
+    if isinstance(fn, str):
+        code = encode_source_code_function(user_namespace, fn, fn_name, args_name, kwargs_name, result_name)
+    elif callable(fn):
+        code = encode_byte_code_function(user_namespace, fn, fn_name, args_name, kwargs_name, result_name)
+    else:
+        raise ValueError("Function object does not look like a function.")
+
+    return (code, result_name)
+
+
+def encode_source_code_function(user_namespace, fn, fn_name, args_name, kwargs_name, result_name):
+    # We drop the first line as it names the parsl decorator used (i.e., @python_app)
+    source = fn.split('\n')[1:]
+    fn_app = "{0} = {1}(*{2}, **{3})".format(result_name, fn_name, args_name, kwargs_name)
+
+    source.append(fn_app)
+
+    code = "\n".join(source)
+    return code
+
+
+def encode_byte_code_function(user_namespace, fn, fn_name, args_name, kwargs_name, result_name):
+    user_namespace.update({fn_name: fn})
+    code = "{0} = {1}(*{2}, **{3})".format(result_name, fn_name, args_name, kwargs_name)
+    return code
+
+
+def load_function(map_file, function_file):
+    # Decodes the function and its file arguments to be executed into
+    # function_code, and updates a user namespace with the function name and
+    # the variable named result_name. When the function is executed, its result
+    # will be stored in this variable in the user namespace.
+    # Returns (namespace, function_code, result_name)
+
+    # Create the namespace to isolate the function execution.
+    user_ns = locals()
+    user_ns.update({'__builtins__': __builtins__})
+
+    function_info = load_pickled_file(function_file)
+
+    (fn, fn_name, fn_args, fn_kwargs) = unpack_function(function_info, user_ns)
+
+    mapping = load_pickled_file(map_file)
+    remap_all_files(mapping, fn_args, fn_kwargs)
+
+    (code, result_name) = encode_function(user_ns, fn, fn_name, fn_args, fn_kwargs)
+
+    return (user_ns, code, result_name)
+
+
+def execute_function(namespace, function_code, result_name):
+    # On executing the function inside the namespace, its result will be in a
+    # variable named result_name.
+
+    exec(function_code, namespace, namespace)
+    result = namespace.get(result_name)
+
+    return result
+
+
+if __name__ == "__main__":
+    try:
+        # parse the three required command line arguments:
+        # map_file: contains a pickled dictionary to map original names to
+        #           names at the execution site.
+        # function_file: contains the pickled parsl function to execute.
+        # result_file: any output (including exceptions) will be written to
+        #              this file.
+        try:
+            (map_file, function_file, result_file) = sys.argv[1:]
+        except ValueError:
+            print("Usage:\n\t{} function result mapping\n".format(sys.argv[0]))
+            raise
+
+        try:
+            (namespace, function_code, result_name) = load_function(map_file, function_file)
+        except Exception:
+            print("There was an error setting up the function for execution.")
+            raise
+
+        try:
+            result = execute_function(namespace, function_code, result_name)
+        except Exception:
+            print("There was an error executing the function.")
+            raise
+    except Exception:
+        traceback.print_exc()
+        result = RemoteExceptionWrapper(*sys.exc_info())
+
+    # Write out function result to the result file
+    try:
+        dump_result_to_file(result_file, result)
+    except Exception:
+        print("Could not write to result file.")
+        traceback.print_exc()
+        sys.exit(1)