executorlib 1.5.1__tar.gz → 1.5.2__tar.gz

{executorlib-1.5.1 → executorlib-1.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.5.1
+Version: 1.5.2
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -56,12 +56,12 @@ Requires-Dist: ipython<=9.0.2,>=7.33.0; extra == 'all'
 Requires-Dist: mpi4py<=4.0.1,>=3.1.4; extra == 'all'
 Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'all'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
-Requires-Dist: pysqa==0.2.6; extra == 'all'
+Requires-Dist: pysqa==0.2.7; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.14.0,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.14.0,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==0.2.6; extra == 'cluster'
+Requires-Dist: pysqa==0.2.7; extra == 'cluster'
 Provides-Extra: graph
 Requires-Dist: networkx<=3.4.2,>=2.8.8; extra == 'graph'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'

{executorlib-1.5.1 → executorlib-1.5.2}/executorlib/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.5.1'
-__version_tuple__ = version_tuple = (1, 5, 1)
+__version__ = version = '1.5.2'
+__version_tuple__ = version_tuple = (1, 5, 2)

{executorlib-1.5.1 → executorlib-1.5.2}/executorlib/task_scheduler/file/shared.py

@@ -52,7 +52,6 @@ class FutureItem:
 
 def execute_tasks_h5(
     future_queue: queue.Queue,
-    cache_directory: str,
     execute_function: Callable,
     resource_dict: dict,
     terminate_function: Optional[Callable] = None,
@@ -65,7 +64,6 @@ def execute_tasks_h5(
 
     Args:
         future_queue (queue.Queue): The queue containing the tasks.
-        cache_directory (str): The directory to store the HDF5 files.
         resource_dict (dict): A dictionary of resources required by the task. With the following keys:
                               - cores (int): number of MPI cores to be used for each function call
                               - cwd (str/None): current working directory where the parallel python task is executed
@@ -81,6 +79,7 @@ def execute_tasks_h5(
     """
     memory_dict: dict = {}
     process_dict: dict = {}
+    cache_dir_dict: dict = {}
     file_name_dict: dict = {}
     while True:
         task_dict = None
@@ -104,6 +103,7 @@ def execute_tasks_h5(
                 {k: v for k, v in resource_dict.items() if k not in task_resource_dict}
             )
             cache_key = task_resource_dict.pop("cache_key", None)
+            cache_directory = os.path.abspath(task_resource_dict.pop("cache_directory"))
             task_key, data_dict = serialize_funct_h5(
                 fn=task_dict["fn"],
                 fn_args=task_args,
@@ -147,11 +147,14 @@ def execute_tasks_h5(
                     cache_directory, task_key + "_o.h5"
                 )
                 memory_dict[task_key] = task_dict["future"]
+                cache_dir_dict[task_key] = cache_directory
             future_queue.task_done()
         else:
             memory_dict = {
                 key: _check_task_output(
-                    task_key=key, future_obj=value, cache_directory=cache_directory
+                    task_key=key,
+                    future_obj=value,
+                    cache_directory=cache_dir_dict[key],
                 )
                 for key, value in memory_dict.items()
                 if not value.done()

{executorlib-1.5.1 → executorlib-1.5.2}/executorlib/task_scheduler/file/task_scheduler.py

@@ -1,4 +1,3 @@
-import os
 from threading import Thread
 from typing import Callable, Optional
 
@@ -27,7 +26,6 @@ except ImportError:
 class FileTaskScheduler(TaskSchedulerBase):
     def __init__(
         self,
-        cache_directory: str = "executorlib_cache",
         resource_dict: Optional[dict] = None,
         execute_function: Callable = execute_with_pysqa,
         terminate_function: Optional[Callable] = None,
@@ -39,10 +37,10 @@ class FileTaskScheduler(TaskSchedulerBase):
         Initialize the FileExecutor.
 
         Args:
-            cache_directory (str, optional): The directory to store cache files. Defaults to "executorlib_cache".
             resource_dict (dict): A dictionary of resources required by the task. With the following keys:
                               - cores (int): number of MPI cores to be used for each function call
                               - cwd (str/None): current working directory where the parallel python task is executed
+                              - cache_directory (str): The directory to store cache files.
             execute_function (Callable, optional): The function to execute tasks. Defaults to execute_in_subprocess.
             terminate_function (Callable, optional): The function to terminate the tasks.
             pysqa_config_directory (str, optional): path to the pysqa config directory (only for pysqa based backend).
@@ -53,6 +51,7 @@ class FileTaskScheduler(TaskSchedulerBase):
         default_resource_dict = {
             "cores": 1,
             "cwd": None,
+            "cache_directory": "executorlib_cache",
         }
         if resource_dict is None:
             resource_dict = {}
@@ -61,13 +60,10 @@ class FileTaskScheduler(TaskSchedulerBase):
         )
         if execute_function == execute_in_subprocess and terminate_function is None:
             terminate_function = terminate_subprocess
-        cache_directory_path = os.path.abspath(cache_directory)
-        os.makedirs(cache_directory_path, exist_ok=True)
         self._process_kwargs = {
+            "resource_dict": resource_dict,
             "future_queue": self._future_queue,
             "execute_function": execute_function,
-            "cache_directory": cache_directory_path,
-            "resource_dict": resource_dict,
             "terminate_function": terminate_function,
             "pysqa_config_directory": pysqa_config_directory,
             "backend": backend,
@@ -82,11 +78,11 @@ class FileTaskScheduler(TaskSchedulerBase):
 
 
 def create_file_executor(
+    resource_dict: dict,
     max_workers: Optional[int] = None,
     backend: str = "flux_submission",
     max_cores: Optional[int] = None,
     cache_directory: Optional[str] = None,
-    resource_dict: Optional[dict] = None,
     flux_executor=None,
     flux_executor_pmi_mode: Optional[str] = None,
     flux_executor_nesting: bool = False,
@@ -97,8 +93,6 @@ def create_file_executor(
     init_function: Optional[Callable] = None,
     disable_dependencies: bool = False,
 ):
-    if cache_directory is None:
-        cache_directory = "executorlib_cache"
     if block_allocation:
         raise ValueError(
             "The option block_allocation is not available with the pysqa based backend."
@@ -107,6 +101,8 @@ def create_file_executor(
         raise ValueError(
             "The option to specify an init_function is not available with the pysqa based backend."
         )
+    if cache_directory is not None:
+        resource_dict["cache_directory"] = cache_directory
     check_flux_executor_pmi_mode(flux_executor_pmi_mode=flux_executor_pmi_mode)
     check_max_workers_and_cores(max_cores=max_cores, max_workers=max_workers)
     check_hostname_localhost(hostname_localhost=hostname_localhost)
@@ -114,7 +110,6 @@ def create_file_executor(
     check_nested_flux_executor(nested_flux_executor=flux_executor_nesting)
     check_flux_log_files(flux_log_files=flux_log_files)
     return FileTaskScheduler(
-        cache_directory=cache_directory,
         resource_dict=resource_dict,
         pysqa_config_directory=pysqa_config_directory,
         backend=backend.split("_submission")[0],

{executorlib-1.5.1 → executorlib-1.5.2}/executorlib/task_scheduler/interactive/shared.py

@@ -160,7 +160,6 @@ def _execute_task_with_cache(
         resource_dict=task_dict.get("resource_dict", {}),
         cache_key=cache_key,
     )
-    os.makedirs(cache_directory, exist_ok=True)
     file_name = os.path.abspath(os.path.join(cache_directory, task_key + "_o.h5"))
     if file_name not in get_cache_files(cache_directory=cache_directory):
         f = task_dict.pop("future")

{executorlib-1.5.1 → executorlib-1.5.2}/pyproject.toml

@@ -53,12 +53,12 @@ graphnotebook = [
 ]
 mpi = ["mpi4py>=3.1.4,<=4.0.1"]
 cluster = [
-    "pysqa==0.2.6",
+    "pysqa==0.2.7",
     "h5py>=3.6.0,<=3.14.0",
 ]
 all = [
     "mpi4py>=3.1.4,<=4.0.1",
-    "pysqa==0.2.6",
+    "pysqa==0.2.7",
     "h5py>=3.6.0,<=3.14.0",
     "pygraphviz>=1.10,<=1.14",
     "networkx>=2.8.8,<=3.4.2",