executorlib 1.9.4__tar.gz → 1.10.1__tar.gz

{executorlib-1.9.4 → executorlib-1.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.9.4
+Version: 1.10.1
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -45,8 +45,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Physics
-Requires-Python: <3.14,>3.9
+Requires-Python: <3.15,>3.9
 Requires-Dist: cloudpickle<=3.1.2,>=2.0.0
 Requires-Dist: pyzmq<=27.1.0,>=25.0.0
 Provides-Extra: all
@@ -56,12 +57,12 @@ Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'all'
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
 Requires-Dist: pydantic<=2.13.4,>=2.5.3; extra == 'all'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
-Requires-Dist: pysqa==0.4.2; extra == 'all'
+Requires-Dist: pysqa==0.4.3; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==0.4.2; extra == 'cluster'
+Requires-Dist: pysqa==0.4.3; extra == 'cluster'
 Provides-Extra: graph
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'
@@ -218,6 +219,25 @@ given allocation. Even when [SLURM](https://slurm.schedmd.com) is used as primar
 recommended to use [SLURM with flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux) 
 as hierarchical job scheduler within the allocations. 
 
+## Which Executor should I use?
+executorlib provides five `Executor` classes. They all share the same `submit()` / `map()` interface and only differ in
+*where* the Python functions are executed and *how* the resources are requested. A common workflow is to develop and 
+test with the `SingleNodeExecutor` on a laptop and then switch to one of the HPC executors by changing only the class 
+name:
+
+| Executor | Where it runs | Scheduler command | Best for |
+|---|---|---|---|
+| [`SingleNodeExecutor`](https://executorlib.readthedocs.io/en/latest/1-single-node.html) | laptop, workstation or single compute node | `subprocess` | developing and testing a workflow |
+| [`SlurmClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm) | HPC login node | `sbatch` (one job per function) | long-running functions that should outlive the Python session |
+| [`SlurmJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm) | inside a SLURM allocation | `srun` (job steps) | many functions within one existing allocation |
+| [`FluxClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux) | HPC login node or Flux instance | `flux submit` | long-running functions; disconnecting and reconnecting |
+| [`FluxJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux) | inside a Flux allocation | `flux run` | high-throughput execution of many short functions |
+
+The **Cluster** executors submit each Python function as an individual job and communicate via the file system, so the
+Python process which created the executor can be closed and the results reloaded later. The **Job** executors run inside
+an existing allocation and communicate via sockets, which has lower overhead and is the better choice for many short
+function calls.
+
 ## Documentation
 * [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
   * [Minimal](https://executorlib.readthedocs.io/en/latest/installation.html#minimal)
@@ -243,6 +263,9 @@ as hierarchical job scheduler within the allocations.
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
   * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux)
+  * [Disconnecting and Reconnecting](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#disconnecting-and-reconnecting)
+  * [Combine both](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#combine-both)
+  * [Cleaning Cache](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#cleaning-cache)
 * [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm)
   * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
@@ -250,6 +273,12 @@ as hierarchical job scheduler within the allocations.
 * [Application](https://executorlib.readthedocs.io/en/latest/application.html)
   * [GPAW](https://executorlib.readthedocs.io/en/latest/4-1-gpaw.html)
   * [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/4-2-quantum-espresso.html)
+* [Integration](https://executorlib.readthedocs.io/en/latest/coupling.html)
+  * [emcee](https://executorlib.readthedocs.io/en/latest/coupling.html#emcee)
+  * [pipefunc](https://executorlib.readthedocs.io/en/latest/coupling.html#pipefunc)
+  * [omp4py](https://executorlib.readthedocs.io/en/latest/coupling.html#omp4py)
+  * [pylammpsmpi](https://executorlib.readthedocs.io/en/latest/coupling.html#pylammpsmpi)
+  * [General Pattern](https://executorlib.readthedocs.io/en/latest/coupling.html#general-pattern)
 * [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
   * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
   * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)

{executorlib-1.9.4 → executorlib-1.10.1}/README.md

@@ -141,6 +141,25 @@ given allocation. Even when [SLURM](https://slurm.schedmd.com) is used as primar
 recommended to use [SLURM with flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux) 
 as hierarchical job scheduler within the allocations. 
 
+## Which Executor should I use?
+executorlib provides five `Executor` classes. They all share the same `submit()` / `map()` interface and only differ in
+*where* the Python functions are executed and *how* the resources are requested. A common workflow is to develop and 
+test with the `SingleNodeExecutor` on a laptop and then switch to one of the HPC executors by changing only the class 
+name:
+
+| Executor | Where it runs | Scheduler command | Best for |
+|---|---|---|---|
+| [`SingleNodeExecutor`](https://executorlib.readthedocs.io/en/latest/1-single-node.html) | laptop, workstation or single compute node | `subprocess` | developing and testing a workflow |
+| [`SlurmClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm) | HPC login node | `sbatch` (one job per function) | long-running functions that should outlive the Python session |
+| [`SlurmJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm) | inside a SLURM allocation | `srun` (job steps) | many functions within one existing allocation |
+| [`FluxClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux) | HPC login node or Flux instance | `flux submit` | long-running functions; disconnecting and reconnecting |
+| [`FluxJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux) | inside a Flux allocation | `flux run` | high-throughput execution of many short functions |
+
+The **Cluster** executors submit each Python function as an individual job and communicate via the file system, so the
+Python process which created the executor can be closed and the results reloaded later. The **Job** executors run inside
+an existing allocation and communicate via sockets, which has lower overhead and is the better choice for many short
+function calls.
+
 ## Documentation
 * [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
   * [Minimal](https://executorlib.readthedocs.io/en/latest/installation.html#minimal)
@@ -166,6 +185,9 @@ as hierarchical job scheduler within the allocations.
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
   * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux)
+  * [Disconnecting and Reconnecting](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#disconnecting-and-reconnecting)
+  * [Combine both](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#combine-both)
+  * [Cleaning Cache](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#cleaning-cache)
 * [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm)
   * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
@@ -173,6 +195,12 @@ as hierarchical job scheduler within the allocations.
 * [Application](https://executorlib.readthedocs.io/en/latest/application.html)
   * [GPAW](https://executorlib.readthedocs.io/en/latest/4-1-gpaw.html)
   * [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/4-2-quantum-espresso.html)
+* [Integration](https://executorlib.readthedocs.io/en/latest/coupling.html)
+  * [emcee](https://executorlib.readthedocs.io/en/latest/coupling.html#emcee)
+  * [pipefunc](https://executorlib.readthedocs.io/en/latest/coupling.html#pipefunc)
+  * [omp4py](https://executorlib.readthedocs.io/en/latest/coupling.html#omp4py)
+  * [pylammpsmpi](https://executorlib.readthedocs.io/en/latest/coupling.html#pylammpsmpi)
+  * [General Pattern](https://executorlib.readthedocs.io/en/latest/coupling.html#general-pattern)
 * [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
   * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
   * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)

{executorlib-1.9.4 → executorlib-1.10.1}/pyproject.toml

@@ -16,7 +16,7 @@ authors = [
 readme = "README.md"
 license = { file = "LICENSE" }
 keywords = ["high performance computing", "hpc", "task scheduler", "slurm", "flux-framework", "executor", "python", "parallel computing", "distributed computing", "scientific computing", "concurrent.futures", "mpi4py", "gpu"]
-requires-python = ">3.9, <3.14"
+requires-python = ">3.9, <3.15"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Topic :: Scientific/Engineering :: Physics",
@@ -27,6 +27,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
     "cloudpickle>=2.0.0,<=3.1.2",
@@ -52,7 +53,7 @@ graphnotebook = [
 ]
 mpi = ["mpi4py>=3.1.4,<=4.1.2"]
 cluster = [
-    "pysqa==0.4.2",
+    "pysqa==0.4.3",
     "h5py>=3.6.0,<=3.16.0",
 ]
 validation = [
@@ -60,7 +61,7 @@ validation = [
 ]
 all = [
     "mpi4py>=3.1.4,<=4.1.2",
-    "pysqa==0.4.2",
+    "pysqa==0.4.3",
     "h5py>=3.6.0,<=3.16.0",
     "pydantic>=2.5.3,<=2.13.4",
     "pygraphviz>=1.10,<=1.14",

{executorlib-1.9.4 → executorlib-1.10.1}/src/executorlib/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.9.4'
-__version_tuple__ = version_tuple = (1, 9, 4)
+__version__ = version = '1.10.1'
+__version_tuple__ = version_tuple = (1, 10, 1)
 
 __commit_id__ = commit_id = None

{executorlib-1.9.4 → executorlib-1.10.1}/src/executorlib/backend/interactive_serial.py

@@ -12,7 +12,7 @@ from executorlib.standalone.interactive.communication import (
 )
 
 
-def main(argument_lst: Optional[list[str]] = None):
+def evaluate_cmd(argument_lst: Optional[list[str]] = None):
     """
     The main function of the program.
 
@@ -90,4 +90,4 @@ def main(argument_lst: Optional[list[str]] = None):
 
 
 if __name__ == "__main__":
-    main(argument_lst=sys.argv)
+    evaluate_cmd(argument_lst=sys.argv)

{executorlib-1.9.4 → executorlib-1.10.1}/src/executorlib/standalone/hdf.py

@@ -38,7 +38,10 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
                     with contextlib.suppress(ValueError):
                         fname.create_dataset(
                             name="/" + group_dict[data_key],
-                            data=np.void(cloudpickle.dumps(data_value)),
+                            data=np.frombuffer(
+                                cloudpickle.dumps(data_value), dtype=np.uint8
+                            ),
+                            compression="gzip",
                         )
 
 
@@ -55,26 +58,26 @@ def load(file_name: str) -> dict:
     with h5py.File(file_name, "r") as hdf:
         data_dict = {}
         if "function" in hdf:
-            data_dict["fn"] = cloudpickle.loads(np.void(hdf["/function"]))
+            data_dict["fn"] = cloudpickle.loads(hdf["/function"][()].tobytes())
         else:
             raise TypeError("Function not found in HDF5 file.")
         if "input_args" in hdf:
-            data_dict["args"] = cloudpickle.loads(np.void(hdf["/input_args"]))
+            data_dict["args"] = cloudpickle.loads(hdf["/input_args"][()].tobytes())
         else:
             data_dict["args"] = ()
         if "input_kwargs" in hdf:
-            data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"]))
+            data_dict["kwargs"] = cloudpickle.loads(hdf["/input_kwargs"][()].tobytes())
         else:
             data_dict["kwargs"] = {}
         if "resource_dict" in hdf:
             data_dict["resource_dict"] = cloudpickle.loads(
-                np.void(hdf["/resource_dict"])
+                hdf["/resource_dict"][()].tobytes()
             )
         else:
             data_dict["resource_dict"] = {}
         if "error_log_file" in hdf:
             data_dict["error_log_file"] = cloudpickle.loads(
-                np.void(hdf["/error_log_file"])
+                hdf["/error_log_file"][()].tobytes()
             )
         return data_dict
 
@@ -93,9 +96,9 @@ def get_output(file_name: str) -> tuple[bool, bool, Any]:
     def get_output_helper(file_name: str) -> tuple[bool, bool, Any]:
         with h5py.File(file_name, "r") as hdf:
             if "output" in hdf:
-                return True, True, cloudpickle.loads(np.void(hdf["/output"]))
+                return True, True, cloudpickle.loads(hdf["/output"][()].tobytes())
             elif "error" in hdf:
-                return True, False, cloudpickle.loads(np.void(hdf["/error"]))
+                return True, False, cloudpickle.loads(hdf["/error"][()].tobytes())
             else:
                 return False, False, None
 
@@ -122,7 +125,7 @@ def get_runtime(file_name: str) -> float:
     """
     with h5py.File(file_name, "r") as hdf:
         if "runtime" in hdf:
-            return cloudpickle.loads(np.void(hdf["/runtime"]))
+            return cloudpickle.loads(hdf["/runtime"][()].tobytes())
         else:
             return 0.0
 
@@ -140,7 +143,7 @@ def get_queue_id(file_name: Optional[str]) -> Optional[int]:
     if file_name is not None and os.path.exists(file_name):
         with h5py.File(file_name, "r") as hdf:
             if "queue_id" in hdf:
-                return cloudpickle.loads(np.void(hdf["/queue_id"]))
+                return cloudpickle.loads(hdf["/queue_id"][()].tobytes())
     return None
 
 
@@ -230,7 +233,7 @@ def _get_content_of_file(file_name: str) -> dict:
     """
     with h5py.File(file_name, "r") as hdf:
         return {
-            key: cloudpickle.loads(np.void(hdf["/" + key]))
+            key: cloudpickle.loads(hdf["/" + key][()].tobytes())
             for key in group_dict.values()
             if key in hdf
         }

{executorlib-1.9.4 → executorlib-1.10.1}/src/executorlib/task_scheduler/interactive/spawner_pysqa.py

@@ -189,7 +189,7 @@ class PysqaSpawner(BaseSpawner):
         if self._cwd is not None:
             working_directory = os.path.join(self._cwd, hash)
         else:
-            working_directory = os.path.abspath(hash)
+            working_directory = os.path.join(os.path.abspath("executorlib_cache"), hash)
         set_current_directory_in_environment()
         return queue_adapter.submit_job(
             command=" ".join(self.generate_command(command_lst=command_lst)),
@@ -242,8 +242,6 @@ def create_pysqa_block_allocation_scheduler(
         executor_kwargs["cwd"] = os.path.abspath(cache_directory)
     if cache_directory is not None:
         executor_kwargs["cache_directory"] = os.path.abspath(cache_directory)
-    else:
-        executor_kwargs["cache_directory"] = os.path.abspath(".")
     executor_kwargs["hostname_localhost"] = hostname_localhost
     executor_kwargs["log_obj_size"] = log_obj_size
     executor_kwargs["pmi_mode"] = pmi_mode