labmate 0.10.4__tar.gz → 0.10.5__tar.gz

{labmate-0.10.4/labmate.egg-info → labmate-0.10.5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: labmate
-Version: 0.10.4
+Version: 0.10.5
 Summary: Data management library to save data and plots to hdf5 files
 Home-page: https://github.com/kyrylo-gr/labmate
 Author: kyrylo.gr | LKB-OMQ
@@ -21,6 +21,17 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: check-manifest; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
 Requires-Dist: sphinx_rtd_theme; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Labmate. The mate that simplifies data management in your lab.
 

labmate-0.10.5/docs/starting_guide/advanced_examples.md

@@ -0,0 +1,74 @@
+# Advanced example
+
+## Mode understanding
+
+Modes can be set by providing `mode=..` keyword or explicitly by specifying `read_only` and `overwrite` options.
+
+'w' mode is the same as
+
+```python
+>>> sd = DH5('somedata.h5', read_only=False)
+```
+
+'a' mode is the same as
+
+```python
+>>> sd = DH5('somedata.h5', read_only=False, overwrite=False)
+```
+
+'r' mode is the same as
+
+```python
+>>> sd = DH5('somedata.h5', read_only=True)
+```
+
+DH5.open_overwrite method is the same as
+
+```python
+>>> sd = DH5('somedata.h5', read_only=False, overwrite=True)
+or
+>>> sd = DH5('somedata.h5', mode='w', overwrite=True)
+```
+
+## Mirroring acquisitions with a custom backend
+
+`AcquisitionManager` accepts an optional `AcquisitionBackend`
+implementation. Backends run in the background every time an acquisition is
+saved and can be used to duplicate the generated HDF5 file (and the companion
+Python/configuration files that Labmate writes next to it) to another location.
+
+The example below mirrors every saved acquisition into a second directory. It
+does so by copying the HDF5 file and any file that shares the same stem (e.g.
+`*_CELL.py`, copied configuration files, …) to the mirror folder.
+
+```python
+from pathlib import Path
+import shutil
+
+from labmate.acquisition import AcquisitionBackend
+from labmate.acquisition_notebook import AcquisitionAnalysisManager
+
+
+class MirrorDirectoryBackend(AcquisitionBackend):
+    def __init__(self, mirror_root: str):
+        self._mirror_root = Path(mirror_root)
+
+    def save_snapshot(self, acquisition):
+        source_prefix = Path(acquisition.filepath)
+        destination_dir = self._mirror_root / source_prefix.parent.name
+        destination_dir.mkdir(parents=True, exist_ok=True)
+
+        # Copy the main HDF5 file and any auxiliary files generated by Labmate.
+        for source_path in source_prefix.parent.glob(f"{source_prefix.name}*"):
+            shutil.copy2(source_path, destination_dir / source_path.name)
+
+
+backend = MirrorDirectoryBackend("/mnt/labmate-mirror")
+aqm = AcquisitionAnalysisManager("/data/labmate", backend=backend)
+```
+
+When `save_acquisition` (or `save_acquisition` on `AcquisitionAnalysisManager`)
+is called, Labmate will asynchronously copy the freshly written files to the
+mirror directory. Should you need to hydrate new local acquisitions from an
+existing mirror, implement `load_snapshot` in the same backend and perform the
+copy in the opposite direction.

{labmate-0.10.4 → labmate-0.10.5}/labmate/__config__.py

@@ -1,3 +1,3 @@
 """This file contains all package constants."""
 
-__version__ = "0.10.4"
+__version__ = "0.10.5"

{labmate-0.10.4 → labmate-0.10.5}/labmate/acquisition/__init__.py

@@ -1,6 +1,7 @@
 # flake8: noqa: F401
-from .acquisition_manager import AcquisitionManager
-from .acquisition_loop import AcquisitionLoop
 from .acquisition_data import NotebookAcquisitionData
+from .acquisition_loop import AcquisitionLoop
+from .acquisition_manager import AcquisitionManager
+from .backend import AcquisitionBackend
 from .analysis_data import AnalysisData, FigureProtocol
 from .analysis_loop import AnalysisLoop

{labmate-0.10.4 → labmate-0.10.5}/labmate/acquisition/acquisition_data.py

@@ -179,3 +179,21 @@ class NotebookAcquisitionData(DH5):
         if step is None:
             step = self.current_step
         self._cells[step] = cell
+
+    def backend_save(self) -> None:
+        """Hook for acquisition backends to serialise custom payloads.
+
+        Custom backends can override this hook to expose data that should be
+        included when persisting to a remote store.
+        """
+
+        pass
+
+    def backend_load(self) -> None:
+        """Hook for acquisition backends to hydrate from remote state.
+
+        Custom backends can override this hook to restore local state after a
+        backend has fetched remote payloads.
+        """
+
+        pass

{labmate-0.10.4 → labmate-0.10.5}/labmate/acquisition/acquisition_manager.py

@@ -1,5 +1,15 @@
 import os
-from typing import Dict, List, NamedTuple, Optional, Tuple, Union
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    List,
+    NamedTuple,
+    Optional,
+    Tuple,
+    Union,
+    Iterable,
+)
 
 from dh5 import jsn
 from dh5.path import Path
@@ -9,6 +19,9 @@ from ..utils import get_timestamp
 from ..utils.file_read import read_file, read_files
 from .acquisition_data import NotebookAcquisitionData
 
+if TYPE_CHECKING:
+    from .backend import AcquisitionBackend
+
 
 class AcquisitionTmpData(NamedTuple):
     """Temporary data that stores inside temp.json."""
@@ -40,6 +53,8 @@ class AcquisitionManager:
 
     cell: Optional[str] = None
 
+    _backend: Optional[Tuple["AcquisitionBackend", ...]] = None
+
     def __init__(
         self,
         data_directory: Optional[Union[str, Path]] = None,
@@ -47,6 +62,9 @@ class AcquisitionManager:
         config_files: Optional[List[str]] = None,
         save_files: Optional[bool] = None,
         save_on_edit: Optional[bool] = None,
+        backend: Optional[
+            Union["AcquisitionBackend", Iterable["AcquisitionBackend"]]
+        ] = None,
     ):
         if save_files is not None:
             self._save_files = save_files
@@ -58,6 +76,14 @@ class AcquisitionManager:
         self._acquisition_tmp_data = None
         self._once_saved = False
 
+        self._backend = (
+            tuple(backend)
+            if isinstance(backend, Iterable)
+            else (backend,)
+            if backend is not None
+            else None
+        )
+
         self.config_files = []
         self.config_files_eval = {}
         self._configs_last_modified = []
@@ -198,6 +224,48 @@ class AcquisitionManager:
     def _get_configs_last_modified(self) -> List[float]:
         return [os.path.getmtime(file) for file in self.config_files]
 
+    def _schedule_backend_save(
+        self, acquisition: NotebookAcquisitionData
+    ) -> Optional[Future]:
+        if self._backend is None:
+            return None
+
+        executor = ThreadPoolExecutor(max_workers=1)
+
+        def save_snapshot():
+            if self._backend is None:
+                return
+            for backend in self._backend:
+                backend.save_snapshot(acquisition)
+
+        def shutdown_executor(future: Future):
+            executor.shutdown(wait=False)
+
+        future = executor.submit(save_snapshot)
+        future.add_done_callback(shutdown_executor)
+        return future
+
+    def _schedule_backend_load(
+        self, acquisition: NotebookAcquisitionData
+    ) -> Optional[Future]:
+        if self._backend is None:
+            return None
+
+        executor = ThreadPoolExecutor(max_workers=1)
+
+        def load_snapshot():
+            if self._backend is None:
+                return
+            for backend in self._backend:
+                backend.load_snapshot(acquisition)
+
+        def shutdown_executor(future: Future):
+            executor.shutdown(wait=False)
+
+        future = executor.submit(load_snapshot)
+        future.add_done_callback(shutdown_executor)
+        return future
+
     def new_acquisition(
         self, name: str, cell: Optional[str] = None, save_on_edit: Optional[bool] = None
     ) -> NotebookAcquisitionData:
@@ -256,7 +324,7 @@ class AcquisitionManager:
         configs = configs if configs else None
         save_on_edit = save_on_edit if save_on_edit is not None else self._save_on_edit
 
-        return NotebookAcquisitionData(
+        acquisition = NotebookAcquisitionData(
             filepath=str(filepath),
             configs=configs,
             cell=cell or self.cell,
@@ -264,6 +332,10 @@ class AcquisitionManager:
             save_on_edit=save_on_edit,
             save_files=self._save_files,
         )
+        # TODO: chech if this gives the expected behaviour
+        self._schedule_backend_load(acquisition)
+
+        return acquisition
 
     @property
     def current_acquisition(self) -> NotebookAcquisitionData:
@@ -301,7 +373,7 @@ class AcquisitionManager:
 
         save_on_edit = save_on_edit if save_on_edit is not None else self._save_on_edit
 
-        return NotebookAcquisitionData(
+        acquisition = NotebookAcquisitionData(
             filepath=str(filepath),
             configs=configs,
             cell=cell,
@@ -311,6 +383,10 @@ class AcquisitionManager:
             experiment_name=acquisition_tmp_data.experiment_name,
         )
 
+        self._schedule_backend_load(acquisition)
+
+        return acquisition
+
     def save_acquisition(self, update_: bool = True, /, **kwds) -> "AcquisitionManager":
         acq_data = self.current_acquisition
         if acq_data is None:
@@ -329,4 +405,8 @@ class AcquisitionManager:
         if acq_data.save_on_edit is False:
             acq_data.save()
         self._once_saved = True
+        self._schedule_backend_save(acq_data)
         return self
+
+    def close(self) -> None:
+        pass

labmate-0.10.5/labmate/acquisition/backend.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:  # pragma: no cover - imported for typing only
+    from .acquisition_data import NotebookAcquisitionData
+
+
+class AcquisitionBackend:
+    """Lightweight backend interface used by :class:`AcquisitionManager`."""
+
+    def save_snapshot(self, acquisition: "NotebookAcquisitionData") -> None:
+        """Persist a snapshot of the acquisition remotely."""
+
+        pass
+
+    def load_snapshot(self, acquisition: "NotebookAcquisitionData") -> None:
+        """Hydrate the acquisition from a remote snapshot."""
+
+        pass
+
+
+__all__ = ["AcquisitionBackend"]

{labmate-0.10.4 → labmate-0.10.5}/labmate/acquisition_notebook/acquisition_analysis_manager.py

@@ -21,7 +21,7 @@ from . import display_widget
 
 if TYPE_CHECKING:
     from dh5.path import Path
-
+    from ..acquisition.backend import AcquisitionBackend
     from ..acquisition import FigureProtocol
     from ..acquisition.config_file import ConfigFile
 
@@ -83,6 +83,9 @@ class AcquisitionAnalysisManager(AcquisitionManager):
         save_on_edit_analysis: Optional[bool] = None,
         save_fig_inside_h5: bool = False,
         shell: Any = True,
+        backend: Optional[
+            Union["AcquisitionBackend", Iterable["AcquisitionBackend"]]
+        ] = None,
     ):
         """
         AcquisitionAnalysisManager.
@@ -131,6 +134,7 @@ class AcquisitionAnalysisManager(AcquisitionManager):
             config_files=config_files,
             save_files=save_files,
             save_on_edit=save_on_edit,
+            backend=backend,
         )
 
     @property
@@ -220,6 +224,10 @@ class AcquisitionAnalysisManager(AcquisitionManager):
                 fig = fig or fig_or_name
         self.save_fig_only(fig=fig, name=name, **kwds)
         self.save_analysis_cell(name=name, cell=cell)
+
+        if self.current_acquisition is not None:
+            self._schedule_backend_save(self.current_acquisition)
+
         if self._connected_widgets:
             display_widget.display_widgets(
                 self._connected_widgets,
@@ -419,8 +427,8 @@ class AcquisitionAnalysisManager(AcquisitionManager):
             (not self._is_old_data)
             and (self.shell is not None)
             and (
-                "acquisition_cell(" in self.shell.last_execution_result.info.raw_cell
-                and not self.shell.last_execution_result.success
+                "acquisition_cell(" in self.shell.last_execution_result.info.raw_cell  # type: ignore
+                and not self.shell.last_execution_result.success  # type: ignore
             )
         ):
             raise ChildProcessError(

{labmate-0.10.4 → labmate-0.10.5/labmate.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: labmate
-Version: 0.10.4
+Version: 0.10.5
 Summary: Data management library to save data and plots to hdf5 files
 Home-page: https://github.com/kyrylo-gr/labmate
 Author: kyrylo.gr | LKB-OMQ
@@ -21,6 +21,17 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: check-manifest; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
 Requires-Dist: sphinx_rtd_theme; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Labmate. The mate that simplifies data management in your lab.
 

{labmate-0.10.4 → labmate-0.10.5}/labmate.egg-info/SOURCES.txt

@@ -48,6 +48,7 @@ labmate/acquisition/acquisition_loop.py
 labmate/acquisition/acquisition_manager.py
 labmate/acquisition/analysis_data.py
 labmate/acquisition/analysis_loop.py
+labmate/acquisition/backend.py
 labmate/acquisition/config_file.py
 labmate/acquisition/custom_lint.py
 labmate/acquisition_notebook/__init__.py

labmate-0.10.4/docs/starting_guide/advanced_examples.md

@@ -1,31 +0,0 @@
-# Advanced example
-
-## Mode understanding
-
-Modes can be set by providing `mode=..` keyword or explicitly by specifying `read_only` and `overwrite` options.
-
-'w' mode is the same as
-
-```python
->>> sd = DH5('somedata.h5', read_only=False)
-```
-
-'a' mode is the same as
-
-```python
->>> sd = DH5('somedata.h5', read_only=False, overwrite=False)
-```
-
-'r' mode is the same as
-
-```python
->>> sd = DH5('somedata.h5', read_only=True)
-```
-
-DH5.open_overwrite method is the same as
-
-```python
->>> sd = DH5('somedata.h5', read_only=False, overwrite=True)
-or
->>> sd = DH5('somedata.h5', mode='w', overwrite=True)
-```