executorlib 1.9.4__py3-none-any.whl → 1.10.1__py3-none-any.whl

executorlib/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.9.4'
-__version_tuple__ = version_tuple = (1, 9, 4)
+__version__ = version = '1.10.1'
+__version_tuple__ = version_tuple = (1, 10, 1)
 
 __commit_id__ = commit_id = None

executorlib/backend/interactive_serial.py

@@ -12,7 +12,7 @@ from executorlib.standalone.interactive.communication import (
 )
 
 
-def main(argument_lst: Optional[list[str]] = None):
+def evaluate_cmd(argument_lst: Optional[list[str]] = None):
     """
     The main function of the program.
 
@@ -90,4 +90,4 @@ def main(argument_lst: Optional[list[str]] = None):
 
 
 if __name__ == "__main__":
-    main(argument_lst=sys.argv)
+    evaluate_cmd(argument_lst=sys.argv)

executorlib/standalone/hdf.py

@@ -38,7 +38,10 @@ def dump(file_name: Optional[str], data_dict: dict) -> None:
                     with contextlib.suppress(ValueError):
                         fname.create_dataset(
                             name="/" + group_dict[data_key],
-                            data=np.void(cloudpickle.dumps(data_value)),
+                            data=np.frombuffer(
+                                cloudpickle.dumps(data_value), dtype=np.uint8
+                            ),
+                            compression="gzip",
                         )
 
 
@@ -55,26 +58,26 @@ def load(file_name: str) -> dict:
     with h5py.File(file_name, "r") as hdf:
         data_dict = {}
         if "function" in hdf:
-            data_dict["fn"] = cloudpickle.loads(np.void(hdf["/function"]))
+            data_dict["fn"] = cloudpickle.loads(hdf["/function"][()].tobytes())
         else:
             raise TypeError("Function not found in HDF5 file.")
         if "input_args" in hdf:
-            data_dict["args"] = cloudpickle.loads(np.void(hdf["/input_args"]))
+            data_dict["args"] = cloudpickle.loads(hdf["/input_args"][()].tobytes())
         else:
             data_dict["args"] = ()
         if "input_kwargs" in hdf:
-            data_dict["kwargs"] = cloudpickle.loads(np.void(hdf["/input_kwargs"]))
+            data_dict["kwargs"] = cloudpickle.loads(hdf["/input_kwargs"][()].tobytes())
         else:
             data_dict["kwargs"] = {}
         if "resource_dict" in hdf:
             data_dict["resource_dict"] = cloudpickle.loads(
-                np.void(hdf["/resource_dict"])
+                hdf["/resource_dict"][()].tobytes()
             )
         else:
             data_dict["resource_dict"] = {}
         if "error_log_file" in hdf:
             data_dict["error_log_file"] = cloudpickle.loads(
-                np.void(hdf["/error_log_file"])
+                hdf["/error_log_file"][()].tobytes()
             )
         return data_dict
 
@@ -93,9 +96,9 @@ def get_output(file_name: str) -> tuple[bool, bool, Any]:
     def get_output_helper(file_name: str) -> tuple[bool, bool, Any]:
         with h5py.File(file_name, "r") as hdf:
             if "output" in hdf:
-                return True, True, cloudpickle.loads(np.void(hdf["/output"]))
+                return True, True, cloudpickle.loads(hdf["/output"][()].tobytes())
             elif "error" in hdf:
-                return True, False, cloudpickle.loads(np.void(hdf["/error"]))
+                return True, False, cloudpickle.loads(hdf["/error"][()].tobytes())
             else:
                 return False, False, None
 
@@ -122,7 +125,7 @@ def get_runtime(file_name: str) -> float:
     """
     with h5py.File(file_name, "r") as hdf:
         if "runtime" in hdf:
-            return cloudpickle.loads(np.void(hdf["/runtime"]))
+            return cloudpickle.loads(hdf["/runtime"][()].tobytes())
         else:
             return 0.0
 
@@ -140,7 +143,7 @@ def get_queue_id(file_name: Optional[str]) -> Optional[int]:
     if file_name is not None and os.path.exists(file_name):
         with h5py.File(file_name, "r") as hdf:
             if "queue_id" in hdf:
-                return cloudpickle.loads(np.void(hdf["/queue_id"]))
+                return cloudpickle.loads(hdf["/queue_id"][()].tobytes())
     return None
 
 
@@ -230,7 +233,7 @@ def _get_content_of_file(file_name: str) -> dict:
     """
     with h5py.File(file_name, "r") as hdf:
         return {
-            key: cloudpickle.loads(np.void(hdf["/" + key]))
+            key: cloudpickle.loads(hdf["/" + key][()].tobytes())
             for key in group_dict.values()
             if key in hdf
         }

executorlib/task_scheduler/interactive/spawner_pysqa.py

@@ -189,7 +189,7 @@ class PysqaSpawner(BaseSpawner):
         if self._cwd is not None:
             working_directory = os.path.join(self._cwd, hash)
         else:
-            working_directory = os.path.abspath(hash)
+            working_directory = os.path.join(os.path.abspath("executorlib_cache"), hash)
         set_current_directory_in_environment()
         return queue_adapter.submit_job(
             command=" ".join(self.generate_command(command_lst=command_lst)),
@@ -242,8 +242,6 @@ def create_pysqa_block_allocation_scheduler(
         executor_kwargs["cwd"] = os.path.abspath(cache_directory)
     if cache_directory is not None:
         executor_kwargs["cache_directory"] = os.path.abspath(cache_directory)
-    else:
-        executor_kwargs["cache_directory"] = os.path.abspath(".")
     executor_kwargs["hostname_localhost"] = hostname_localhost
     executor_kwargs["log_obj_size"] = log_obj_size
     executor_kwargs["pmi_mode"] = pmi_mode

{executorlib-1.9.4.dist-info → executorlib-1.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: executorlib
-Version: 1.9.4
+Version: 1.10.1
 Summary: Up-scale python functions for high performance computing (HPC) with executorlib.
 Project-URL: Homepage, https://github.com/pyiron/executorlib
 Project-URL: Documentation, https://executorlib.readthedocs.io
@@ -45,8 +45,9 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Physics
-Requires-Python: <3.14,>3.9
+Requires-Python: <3.15,>3.9
 Requires-Dist: cloudpickle<=3.1.2,>=2.0.0
 Requires-Dist: pyzmq<=27.1.0,>=25.0.0
 Provides-Extra: all
@@ -56,12 +57,12 @@ Requires-Dist: mpi4py<=4.1.2,>=3.1.4; extra == 'all'
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'all'
 Requires-Dist: pydantic<=2.13.4,>=2.5.3; extra == 'all'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'all'
-Requires-Dist: pysqa==0.4.2; extra == 'all'
+Requires-Dist: pysqa==0.4.3; extra == 'all'
 Provides-Extra: cache
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cache'
 Provides-Extra: cluster
 Requires-Dist: h5py<=3.16.0,>=3.6.0; extra == 'cluster'
-Requires-Dist: pysqa==0.4.2; extra == 'cluster'
+Requires-Dist: pysqa==0.4.3; extra == 'cluster'
 Provides-Extra: graph
 Requires-Dist: networkx<=3.6.1,>=2.8.8; extra == 'graph'
 Requires-Dist: pygraphviz<=1.14,>=1.10; extra == 'graph'
@@ -218,6 +219,25 @@ given allocation. Even when [SLURM](https://slurm.schedmd.com) is used as primar
 recommended to use [SLURM with flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux) 
 as hierarchical job scheduler within the allocations. 
 
+## Which Executor should I use?
+executorlib provides five `Executor` classes. They all share the same `submit()` / `map()` interface and only differ in
+*where* the Python functions are executed and *how* the resources are requested. A common workflow is to develop and 
+test with the `SingleNodeExecutor` on a laptop and then switch to one of the HPC executors by changing only the class 
+name:
+
+| Executor | Where it runs | Scheduler command | Best for |
+|---|---|---|---|
+| [`SingleNodeExecutor`](https://executorlib.readthedocs.io/en/latest/1-single-node.html) | laptop, workstation or single compute node | `subprocess` | developing and testing a workflow |
+| [`SlurmClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm) | HPC login node | `sbatch` (one job per function) | long-running functions that should outlive the Python session |
+| [`SlurmJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm) | inside a SLURM allocation | `srun` (job steps) | many functions within one existing allocation |
+| [`FluxClusterExecutor`](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux) | HPC login node or Flux instance | `flux submit` | long-running functions; disconnecting and reconnecting |
+| [`FluxJobExecutor`](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#flux) | inside a Flux allocation | `flux run` | high-throughput execution of many short functions |
+
+The **Cluster** executors submit each Python function as an individual job and communicate via the file system, so the
+Python process which created the executor can be closed and the results reloaded later. The **Job** executors run inside
+an existing allocation and communicate via sockets, which has lower overhead and is the better choice for many short
+function calls.
+
 ## Documentation
 * [Installation](https://executorlib.readthedocs.io/en/latest/installation.html)
   * [Minimal](https://executorlib.readthedocs.io/en/latest/installation.html#minimal)
@@ -243,6 +263,9 @@ as hierarchical job scheduler within the allocations.
 * [HPC Cluster Executor](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#slurm)
   * [Flux](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#flux)
+  * [Disconnecting and Reconnecting](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#disconnecting-and-reconnecting)
+  * [Combine both](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#combine-both)
+  * [Cleaning Cache](https://executorlib.readthedocs.io/en/latest/2-hpc-cluster.html#cleaning-cache)
 * [HPC Job Executor](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html)
   * [SLURM](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm)
   * [SLURM with Flux](https://executorlib.readthedocs.io/en/latest/3-hpc-job.html#slurm-with-flux)
@@ -250,6 +273,12 @@ as hierarchical job scheduler within the allocations.
 * [Application](https://executorlib.readthedocs.io/en/latest/application.html)
   * [GPAW](https://executorlib.readthedocs.io/en/latest/4-1-gpaw.html)
   * [Quantum Espresso](https://executorlib.readthedocs.io/en/latest/4-2-quantum-espresso.html)
+* [Integration](https://executorlib.readthedocs.io/en/latest/coupling.html)
+  * [emcee](https://executorlib.readthedocs.io/en/latest/coupling.html#emcee)
+  * [pipefunc](https://executorlib.readthedocs.io/en/latest/coupling.html#pipefunc)
+  * [omp4py](https://executorlib.readthedocs.io/en/latest/coupling.html#omp4py)
+  * [pylammpsmpi](https://executorlib.readthedocs.io/en/latest/coupling.html#pylammpsmpi)
+  * [General Pattern](https://executorlib.readthedocs.io/en/latest/coupling.html#general-pattern)
 * [Trouble Shooting](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html)
   * [Filesystem Usage](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#filesystem-usage)
   * [Firewall Issues](https://executorlib.readthedocs.io/en/latest/trouble_shooting.html#firewall-issues)

{executorlib-1.9.4.dist-info → executorlib-1.10.1.dist-info}/RECORD

@@ -1,11 +1,11 @@
 executorlib/__init__.py,sha256=IY_DkT706LPIv2_iIe3whrQNGoDzaDFYBvXyUepoZpw,4780
-executorlib/_version.py,sha256=VdLE9NNjOWcm4_Kh_W9w_lFN3bh0oesrwyTL3nQnUKg,520
+executorlib/_version.py,sha256=9cYIW_mnApkxjl7hrsLPCkpAtl6R-GXOqmSgy2KFfGA,522
 executorlib/api.py,sha256=F6ibDJnVcPGxxPe-UScFJHprWHiVsETGhgAvS1T-9ZI,1365
 executorlib/backend/__init__.py,sha256=2b_iwds4VUrJe92nKtQXkoGxkJiXeEDSlbvCRN_-de8,456
 executorlib/backend/cache_parallel.py,sha256=ldb2iYzrhcXJZ0LcH75FpTLO_6F51mleU9JGAilF00k,2020
 executorlib/backend/cache_serial.py,sha256=9T3bAxVivGxvzHnXmvysyEPP4sw--VCwXCn2Mdkt0-o,177
 executorlib/backend/interactive_parallel.py,sha256=HimGnWOdTXInC3o7W4wMUSTTlj6MsMPLJLkcNTaK_SQ,3869
-executorlib/backend/interactive_serial.py,sha256=NxlSX566rQM-pHiy2rQja2rr3lFwdnv0EvIyXk8H7co,2982
+executorlib/backend/interactive_serial.py,sha256=ZV_VGcnCAk7N-VcoJKYEIG0OGUUwR4KFWoHG7MVBA3g,2998
 executorlib/executor/__init__.py,sha256=oR2ABLAMYV_TNs1eG-3-MAKHFS7OKB-1KOzCqKNbtWA,310
 executorlib/executor/base.py,sha256=UKJViNoshthAgF2I4FoK0nZT0FL7WgkrzWFYhRiU1H4,6957
 executorlib/executor/flux.py,sha256=g8K1GMfn9KMM4udj3LdmDMahavPBBCOusYVG36W1hAY,40779
@@ -15,7 +15,7 @@ executorlib/standalone/__init__.py,sha256=n_Q4NUFu35Fn0qlWIhmoC_FyUkmFGlWn1cNc_7
 executorlib/standalone/batched.py,sha256=o3TtsFJir8nP1O_NZFfW0Q78CoZAFgHMyroWlg520nE,1446
 executorlib/standalone/command.py,sha256=4Xndog84zJdku1ke1DwdUamkWvzRrMz-VDCi5cRXi0M,6548
 executorlib/standalone/error.py,sha256=aaMQoRsoLzLCbYT-E4LZxGM9pm_AhjMesRjVt0d_LDk,714
-executorlib/standalone/hdf.py,sha256=nE2mTH1U8O7sm2TPJkfRU0uEgWJOhLnYxI6yZord8Eo,7623
+executorlib/standalone/hdf.py,sha256=dRePOCYXbPG_i__ASUcD4qs5AAU2fPbP5o_5ytwaOjw,7805
 executorlib/standalone/inputcheck.py,sha256=kSWUzqdMOhILYXW4Isumn1EGH3SegxPUMWRkUWfEleU,10041
 executorlib/standalone/queue.py,sha256=KFEBFQXGvcaf5qcuCjT5kaZg7ucWOTGOtrD96tIBlq0,589
 executorlib/standalone/scheduler.py,sha256=LhCuIdaw9BjrP3GerbHUIqwsvxERdtH5JQoeyAqASIU,2325
@@ -42,9 +42,9 @@ executorlib/task_scheduler/interactive/dependency_plot.py,sha256=iZBZ9UMDeYOMFjD
 executorlib/task_scheduler/interactive/onetoone.py,sha256=cTXgTXR9kKzzlx6EfGumMsEm7DUa7BD9JSeBpdPiupI,13312
 executorlib/task_scheduler/interactive/shared.py,sha256=-krntmZFOOE5U0HdFwuU8JZPqZQmPW4-cePDWuvkTgM,6305
 executorlib/task_scheduler/interactive/spawner_flux.py,sha256=M7CPWvzY7MDkplJ4awymh7WbMYwKsDPTKxA2Op9wfNo,7075
-executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=96NqUfoIBUFd7KjY02dpIL9ZoS_u5beRuNw_1FUKH6Q,10393
+executorlib/task_scheduler/interactive/spawner_pysqa.py,sha256=i-E9MOK9_aiUigrrGFMLs_m0y3VhrEgu95_PXCOg-hg,10352
 executorlib/task_scheduler/interactive/spawner_slurm.py,sha256=K5mj63PE8FYbkw9WWm4uEjoA5q0HM42z1d2wjAmO5YA,3713
-executorlib-1.9.4.dist-info/METADATA,sha256=4augHmAbGQ-7H0r4Zh6vaFvTh-yTwZMSyuLVYsp7-Qo,18249
-executorlib-1.9.4.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-executorlib-1.9.4.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
-executorlib-1.9.4.dist-info/RECORD,,
+executorlib-1.10.1.dist-info/METADATA,sha256=cK6UBcUFueR_PY8sOJC9D6xJ0gBNTJ7oQyVSLlFi4A8,20966
+executorlib-1.10.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+executorlib-1.10.1.dist-info/licenses/LICENSE,sha256=NkrhccIxPXCbbvp_upyVojt7DNTwZIco3z9wsYpSLqU,1511
+executorlib-1.10.1.dist-info/RECORD,,