labmate 0.10.4__py3-none-any.whl → 0.10.6__py3-none-any.whl

labmate/__config__.py

@@ -1,3 +1,3 @@
 """This file contains all package constants."""
 
-__version__ = "0.10.4"
+__version__ = "0.10.6"

labmate/acquisition/__init__.py

@@ -1,6 +1,7 @@
 # flake8: noqa: F401
-from .acquisition_manager import AcquisitionManager
-from .acquisition_loop import AcquisitionLoop
 from .acquisition_data import NotebookAcquisitionData
+from .acquisition_loop import AcquisitionLoop
+from .acquisition_manager import AcquisitionManager
 from .analysis_data import AnalysisData, FigureProtocol
 from .analysis_loop import AnalysisLoop
+from .backend import AcquisitionBackend

labmate/acquisition/acquisition_data.py

@@ -179,3 +179,21 @@ class NotebookAcquisitionData(DH5):
         if step is None:
             step = self.current_step
         self._cells[step] = cell
+
+    def backend_save(self) -> None:
+        """Hook for acquisition backends to serialise custom payloads.
+
+        Custom backends can override this hook to expose data that should be
+        included when persisting to a remote store.
+        """
+
+        pass
+
+    def backend_load(self) -> None:
+        """Hook for acquisition backends to hydrate from remote state.
+
+        Custom backends can override this hook to restore local state after a
+        backend has fetched remote payloads.
+        """
+
+        pass

labmate/acquisition/acquisition_loop.py

@@ -96,9 +96,7 @@ class AcquisitionLoop(DH5):
     ) -> Iterator:
         """Return np.arange(start, stop, step) given a start, stop and step."""
 
-    def __call__(
-        self, *args, iterable: Optional[Iterable] = None, **kwds
-    ) -> Optional[Iterator]:
+    def __call__(self, *args, iterable: Optional[Iterable] = None, **kwds) -> Optional[Iterator]:
         """Append_data or arange.
 
         If kwds are provided then is same as calling append_data(kwds),
@@ -115,10 +113,11 @@ class AcquisitionLoop(DH5):
             return None
 
         if iterable is None:
-            if isinstance(args[0], (int, float, np.int_, np.floating)):  # type: ignore
-                iterable = np.arange(*args)
-            else:
-                iterable = args[0]
+            iterable = (
+                np.arange(*args)
+                if isinstance(args[0], (int, float, np.int_, np.floating))  # type: ignore
+                else args[0]
+            )
 
         if iterable is None:
             raise ValueError("You should provide an iterable")
@@ -166,8 +165,8 @@ class AcquisitionLoop(DH5):
                 if len(key_shape) < len(self[key].shape):
                     raise ValueError(
                         f"Object {key} hasn't the same shape as before. Now it's"
-                        f" {key_shape[len(shape):]},"
-                        f" but before it was {self[key].shape[len(shape):]}."
+                        f" {key_shape[len(shape) :]},"
+                        f" but before it was {self[key].shape[len(shape) :]}."
                     )
                 if len(key_shape) > len(self[key].shape):
                     raise ValueError(
@@ -178,9 +177,7 @@ class AcquisitionLoop(DH5):
                 self[key] = SyncNp(
                     np.pad(
                         self[key],
-                        pad_width=tuple(
-                            (0, i - j) for i, j in zip(key_shape, self[key].shape)
-                        ),
+                        pad_width=tuple((0, i - j) for i, j in zip(key_shape, self[key].shape)),
                     )
                 )
                 self[key][iteration] = value
@@ -201,9 +198,7 @@ class AcquisitionLoop(DH5):
     ):
         if length is None:
             if not hasattr(iterable, "__len__"):
-                raise TypeError(
-                    "Iterable should has __len__ method or length should be provided"
-                )
+                raise TypeError("Iterable should has __len__ method or length should be provided")
             length = len(iterable)  # type: ignore
 
         def loop_iter(array, length):
@@ -283,9 +278,7 @@ class AcquisitionLoop(DH5):
         """
         if key is None:
             if not self._save_indexes:
-                raise ValueError(
-                    "As indexes are not saved with the Loop, key should be provided."
-                )
+                raise ValueError("As indexes are not saved with the Loop, key should be provided.")
             key = f"__index_{self._level}__"
 
         iteration = tuple(self._iteration[: self._level])

labmate/acquisition/acquisition_manager.py

@@ -1,5 +1,16 @@
 import os
-from typing import Dict, List, NamedTuple, Optional, Tuple, Union
+from concurrent.futures import Future, ThreadPoolExecutor
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    NamedTuple,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from dh5 import jsn
 from dh5.path import Path
@@ -10,6 +21,10 @@ from ..utils.file_read import read_file, read_files
 from .acquisition_data import NotebookAcquisitionData
 
 
+if TYPE_CHECKING:
+    from .backend import AcquisitionBackend
+
+
 class AcquisitionTmpData(NamedTuple):
     """Temporary data that stores inside temp.json."""
 
@@ -40,13 +55,16 @@ class AcquisitionManager:
 
     cell: Optional[str] = None
 
+    _backend: Optional[Tuple["AcquisitionBackend", ...]] = None
+
     def __init__(
         self,
-        data_directory: Optional[Union[str, Path]] = None,
+        data_directory: Optional[Union[str, Path, Any]] = None,
         *,
         config_files: Optional[List[str]] = None,
         save_files: Optional[bool] = None,
         save_on_edit: Optional[bool] = None,
+        backend: Optional[Union["AcquisitionBackend", Iterable["AcquisitionBackend"]]] = None,
     ):
         if save_files is not None:
             self._save_files = save_files
@@ -58,12 +76,24 @@ class AcquisitionManager:
         self._acquisition_tmp_data = None
         self._once_saved = False
 
+        self._backend = (
+            tuple(backend)
+            if isinstance(backend, Iterable)
+            else (backend,)
+            if backend is not None
+            else None
+        )
+
         self.config_files = []
         self.config_files_eval = {}
         self._configs_last_modified = []
 
         if data_directory is not None:
-            self.data_directory = Path(data_directory)
+            self.data_directory = (
+                Path(str(data_directory))
+                if not isinstance(data_directory, Path)
+                else data_directory
+            )
         elif "ACQUISITION_DIR" in os.environ:
             self.data_directory = Path(os.environ["ACQUISITION_DIR"])
         else:
@@ -108,13 +138,9 @@ class AcquisitionManager:
                 configs: dict of configurations files to save
                 directory: directory where the data is stored
         """
-        acquisition_tmp_data = self._acquisition_tmp_data or self.get_temp_data(
-            self.temp_file_path
-        )
+        acquisition_tmp_data = self._acquisition_tmp_data or self.get_temp_data(self.temp_file_path)
         if acquisition_tmp_data is None:
-            raise ValueError(
-                "You should create a new acquisition. It will create temp.json file."
-            )
+            raise ValueError("You should create a new acquisition. It will create temp.json file.")
         return acquisition_tmp_data
 
     @acquisition_tmp_data.setter
@@ -136,12 +162,10 @@ class AcquisitionManager:
             filename = [str(filename)]
 
         self.config_files = list(filename)
-        self._config_files_names_to_path = {
-            os.path.basename(file): file for file in self.config_files
-        }
+        self._config_files_names_to_path = {Path(file).name: file for file in self.config_files}
 
         for config_file in self.config_files:
-            if not os.path.exists(config_file):
+            if not Path(config_file).exists():
                 raise ValueError(f"Configuration file at {config_file} does not exist")
 
         return self
@@ -151,7 +175,7 @@ class AcquisitionManager:
             raise ValueError(
                 "Configuration file should be specified before with set_config_file function"
             )
-        self.config_files_eval[os.path.basename(file)] = module
+        self.config_files_eval[Path(file).name] = module
 
     def set_init_analyse_file(self, filename: Union[str, Path]) -> None:
         if not isinstance(filename, Path):
@@ -168,22 +192,18 @@ class AcquisitionManager:
         if not experiment_path.exists():
             experiment_path.makedirs()
         # Copy init_analyses code to the experiment directory if it does not exist
-        if self._init_code and not os.path.exists(experiment_path / "init_analyse.py"):
-            with open(
-                experiment_path / "init_analyse.py", "w", encoding="utf-8"
-            ) as file:
+        if self._init_code and not (experiment_path / "init_analyse.py").exists():
+            with open(experiment_path / "init_analyse.py", "w", encoding="utf-8") as file:
                 file.write(self._init_code)
 
-        filepath_original = filepath = (
-            experiment_path / f"{dic.time_stamp}__{dic.experiment_name}"
-        )
+        filepath_original = filepath = experiment_path / f"{dic.time_stamp}__{dic.experiment_name}"
 
         # If ignore existence is True, no check is required
         if ignore_existence:
             return filepath
         # If the file already exists, add a suffix to the name
         index = 1
-        while os.path.exists(filepath + ".h5"):
+        while (filepath + ".h5").exists():
             filepath = filepath_original + f"__{index}"
             index += 1
 
@@ -191,12 +211,50 @@ class AcquisitionManager:
 
     @staticmethod
     def get_temp_data(path: Path) -> Optional[AcquisitionTmpData]:
-        if not os.path.exists(path):
+        if not Path(path).exists():
             return None
         return AcquisitionTmpData(**jsn.read(path))
 
     def _get_configs_last_modified(self) -> List[float]:
-        return [os.path.getmtime(file) for file in self.config_files]
+        return [Path(file).stat().st_mtime for file in self.config_files]
+
+    def _schedule_backend_save(self, acquisition: NotebookAcquisitionData) -> Optional[Future]:
+        if self._backend is None:
+            return None
+
+        executor = ThreadPoolExecutor(max_workers=1)
+
+        def save_snapshot():
+            if self._backend is None:
+                return
+            for backend in self._backend:
+                backend.save_snapshot(acquisition)
+
+        def shutdown_executor(future: Future):
+            executor.shutdown(wait=False)
+
+        future = executor.submit(save_snapshot)
+        future.add_done_callback(shutdown_executor)
+        return future
+
+    def _schedule_backend_load(self, acquisition: NotebookAcquisitionData) -> Optional[Future]:
+        if self._backend is None:
+            return None
+
+        executor = ThreadPoolExecutor(max_workers=1)
+
+        def load_snapshot():
+            if self._backend is None:
+                return
+            for backend in self._backend:
+                backend.load_snapshot(acquisition)
+
+        def shutdown_executor(future: Future):
+            executor.shutdown(wait=False)
+
+        future = executor.submit(load_snapshot)
+        future.add_done_callback(shutdown_executor)
+        return future
 
     def new_acquisition(
         self, name: str, cell: Optional[str] = None, save_on_edit: Optional[bool] = None
@@ -209,9 +267,7 @@ class AcquisitionManager:
         self._configs_last_modified = self._get_configs_last_modified()
 
         if self.config_files_eval:
-            configs = append_values_from_modules_to_files(
-                configs, self.config_files_eval
-            )
+            configs = append_values_from_modules_to_files(configs, self.config_files_eval)
 
         dic = AcquisitionTmpData(
             experiment_name=name,
@@ -222,9 +278,7 @@ class AcquisitionManager:
 
         self.acquisition_tmp_data = dic
 
-        self._current_acquisition = self.get_acquisition(
-            replace=True, save_on_edit=save_on_edit
-        )
+        self._current_acquisition = self.get_acquisition(replace=True, save_on_edit=save_on_edit)
 
         return self.current_acquisition
 
@@ -238,9 +292,7 @@ class AcquisitionManager:
         configs = read_files(self.config_files)
 
         if self.config_files_eval:
-            configs = append_values_from_modules_to_files(
-                configs, self.config_files_eval
-            )
+            configs = append_values_from_modules_to_files(configs, self.config_files_eval)
 
         if name is None:
             name = self.current_experiment_name + "_item"
@@ -256,7 +308,7 @@ class AcquisitionManager:
         configs = configs if configs else None
         save_on_edit = save_on_edit if save_on_edit is not None else self._save_on_edit
 
-        return NotebookAcquisitionData(
+        acquisition = NotebookAcquisitionData(
             filepath=str(filepath),
             configs=configs,
             cell=cell or self.cell,
@@ -264,6 +316,10 @@ class AcquisitionManager:
             save_on_edit=save_on_edit,
             save_files=self._save_files,
         )
+        # TODO: chech if this gives the expected behaviour
+        self._schedule_backend_load(acquisition)
+
+        return acquisition
 
     @property
     def current_acquisition(self) -> NotebookAcquisitionData:
@@ -284,24 +340,20 @@ class AcquisitionManager:
 
     @property
     def current_experiment_name(self) -> str:
-        return (
-            self.acquisition_tmp_data.experiment_name
-        )  # self.current_acquisition.name
+        return self.acquisition_tmp_data.experiment_name  # self.current_acquisition.name
 
     def get_acquisition(
         self, replace: Optional[bool] = False, save_on_edit: Optional[bool] = None
     ) -> NotebookAcquisitionData:
         acquisition_tmp_data = self.acquisition_tmp_data
-        filepath = self.create_path_from_tmp_data(
-            acquisition_tmp_data, ignore_existence=True
-        )
+        filepath = self.create_path_from_tmp_data(acquisition_tmp_data, ignore_existence=True)
         configs = acquisition_tmp_data.configs
         configs = configs if configs else None
         cell = self.cell
 
         save_on_edit = save_on_edit if save_on_edit is not None else self._save_on_edit
 
-        return NotebookAcquisitionData(
+        acquisition = NotebookAcquisitionData(
             filepath=str(filepath),
             configs=configs,
             cell=cell,
@@ -311,6 +363,10 @@ class AcquisitionManager:
             experiment_name=acquisition_tmp_data.experiment_name,
         )
 
+        self._schedule_backend_load(acquisition)
+
+        return acquisition
+
     def save_acquisition(self, update_: bool = True, /, **kwds) -> "AcquisitionManager":
         acq_data = self.current_acquisition
         if acq_data is None:
@@ -329,4 +385,8 @@ class AcquisitionManager:
         if acq_data.save_on_edit is False:
             acq_data.save()
         self._once_saved = True
+        self._schedule_backend_save(acq_data)
         return self
+
+    def close(self) -> None:
+        pass

labmate/acquisition/analysis_data.py

@@ -12,6 +12,7 @@ from ..logger import logger
 from .analysis_loop import AnalysisLoop
 from .config_file import ConfigFile
 
+
 _T = TypeVar("_T", bound="AnalysisData")
 
 
@@ -115,7 +116,7 @@ class AnalysisData(DH5):
         self._save_files = save_files
         self._save_fig_inside_h5 = save_fig_inside_h5
 
-        self._default_config_files: Tuple[str, ...] = tuple()
+        self._default_config_files: Tuple[str, ...] = ()
         if "info" in self and "default_config_files" in self["info"]:
             self._default_config_files = tuple(self["info"]["default_config_files"])
 
@@ -158,9 +159,9 @@ class AnalysisData(DH5):
             logger.warning("Analysis cell is not set. Nothing to save")
             return self
 
-        self.unlock_data(cell_name_key).update({cell_name_key: code}).lock_data(
-            cell_name_key
-        ).save([cell_name_key])
+        self.unlock_data(cell_name_key).update({cell_name_key: code}).lock_data(cell_name_key).save(
+            [cell_name_key]
+        )
 
         if self._save_files:
             assert self.filepath, "You must set self.filepath before saving"
@@ -198,18 +199,14 @@ class AnalysisData(DH5):
             "inside_h5", False
         ):
             try:
-                raise NotImplementedError(
-                    "save_fig_inside_h5 is not implemented for the moment."
-                )
+                raise NotImplementedError("save_fig_inside_h5 is not implemented for the moment.")
                 # import pltsave
 
                 # data = pltsave.dumps(fig).to_json()
                 # self[f"figures/{fig_name}"] = data
                 # self.save([f"figures/{fig_name}"])
             except Exception as error:
-                logger.exception(
-                    "Failed to save the figure inside h5 file due to %s", error
-                )
+                logger.exception("Failed to save the figure inside h5 file due to %s", error)
         if tight_layout and hasattr(fig, "tight_layout"):
             fig.tight_layout()  # type: ignore
         if metadata is None:
@@ -261,14 +258,12 @@ class AnalysisData(DH5):
                 name = str(name)
             if not name.isnumeric() and name[0] != "_":
                 name = "_" + name
-            if os.path.splitext(name)[-1] == "":
+            if Path(name).suffix == "":  # check if name has no extension
                 name = f"{name}.{extensions}"
 
         return "FIG" + name
 
-    def parse_config(
-        self, config_files: Optional[Tuple[str, ...]] = None
-    ) -> "ConfigFile":
+    def parse_config(self, config_files: Optional[Tuple[str, ...]] = None) -> "ConfigFile":
         """Parse config files. If `config_files` are not provided takes `default_config_files`."""
 
         config_files = config_files or self._default_config_files
@@ -303,9 +298,7 @@ class AnalysisData(DH5):
             if key_value == "filename" or key_value == "file" or key_value == "f":
                 filename = os.path.split(self.filepath)[-1]
                 keys_with_values.append(
-                    utils.title_parsing.ValueForPrint(
-                        key_value, filename, key_units, key_format
-                    )
+                    utils.title_parsing.ValueForPrint(key_value, filename, key_units, key_format)
                 )
             elif key_value in self:
                 keys_with_values.append(
@@ -358,9 +351,7 @@ class AnalysisData(DH5):
                 )
 
             if config_file_name in self._parsed_configs:
-                self._parsed_configs[original_config_name] = self._parsed_configs[
-                    config_file_name
-                ]
+                self._parsed_configs[original_config_name] = self._parsed_configs[config_file_name]
                 return self._parsed_configs[config_file_name]
 
         else:
@@ -393,16 +384,14 @@ class AnalysisData(DH5):
         code: Optional[dict] = self.get("analysis_cells")
         if code is None:
             raise ValueError(
-                f"There is no field 'analysis_cells' inside the data file. "
+                "There is no field 'analysis_cells' inside the data file. "
                 f"Possible keys are {tuple(self.keys())}."
             )
 
         # if isinstance(code, bytes):
         #     code = code.decode()
         if name not in code:
-            raise KeyError(
-                f"Cannot get cell '{name}'. Possible cells are: {tuple(code.keys())}"
-            )
+            raise KeyError(f"Cannot get cell '{name}'. Possible cells are: {tuple(code.keys())}")
 
         code_str: str = code[name]
         if update_code:
@@ -426,7 +415,8 @@ class AnalysisData(DH5):
         return figures
 
         # raise NotImplementedError(
-        # "Not implemented for the moment. If you want to open an old figure. Use open_old_figs function")
+        # "Not implemented for the moment. If you want to open an old figure.
+        # Use open_old_figs function" )
 
     def pull(self, force_pull: bool = False):
         self._reset_attrs()

labmate/acquisition/analysis_loop.py

@@ -2,7 +2,7 @@
 It has mainly __iter__ method and __getitem__ method for slicing.
 """
 
-from typing import Any, Iterable, List, Optional, Tuple, Union
+from typing import Any, List, Optional, Sequence, Tuple, Union
 
 from dh5 import DH5
 
@@ -39,16 +39,14 @@ class AnalysisLoop(DH5):
 
     """
 
-    def __init__(
-        self, data: Optional[dict] = None, loop_shape: Optional[List[int]] = None
-    ):
+    def __init__(self, data: Optional[dict] = None, loop_shape: Optional[List[int]] = None):
         """Initialize an AnalysisLoop object.
 
         Args:
             data (Optional[dict]): A dictionary containing data to initialize the object with.
-            loop_shape (Optional[List[int]]): A list of integers representing the shape of the analysis loop.
+            loop_shape (list[int] | None): A list of ints representing the shape of the analysis loop.
                 If not provided, the shape is retrieved from the object's '__loop_shape__' attribute.
-        """
+        """  # noqa: E501
         super().__init__(data=data)
         if loop_shape is None:
             loop_shape = self.get("__loop_shape__")
@@ -86,7 +84,11 @@ class AnalysisLoop(DH5):
                     child_kwds[key] = value[index]
 
                 val = child_kwds[key]
-                if isinstance(val, (Iterable)) and len(val) == 1 and not isinstance(val[0], (Iterable)):  # type: ignore
+                if (
+                    isinstance(val, (Sequence))
+                    and len(val) == 1
+                    and not isinstance(val[0], (Sequence))
+                ):  # type: ignore
                     child_kwds[key] = val[0]  # type: ignore
 
             if len(self._loop_shape) > 1:

labmate/acquisition/backend.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:  # pragma: no cover - imported for typing only
+    from .acquisition_data import NotebookAcquisitionData
+
+
+class AcquisitionBackend:
+    """Lightweight backend interface used by :class:`AcquisitionManager`."""
+
+    def save_snapshot(self, acquisition: "NotebookAcquisitionData") -> None:
+        """Persist a snapshot of the acquisition remotely."""
+
+        pass
+
+    def load_snapshot(self, acquisition: "NotebookAcquisitionData") -> None:
+        """Hydrate the acquisition from a remote snapshot."""
+
+        pass
+
+
+__all__ = ["AcquisitionBackend"]

labmate/acquisition/config_file.py

@@ -1,5 +1,7 @@
 """ConfigFile class."""
+
 from typing import Any
+
 from .. import attrdict
 
 
@@ -43,8 +45,8 @@ class ConfigFile(attrdict.AttrDict):
             raise ValueError("Content is not defined")
 
         module = type(attrdict)("config_module")
-        cc = compile(self.content, "<string>", "exec")  # noqa: DUO110
-        eval(cc, module.__dict__)  # pylint: disable=W0123 # noqa: DUO104
+        cc = compile(self.content, "<string>", "exec")
+        eval(cc, module.__dict__)
         return module
 
     def eval_key(self, key) -> Any:
@@ -59,5 +61,5 @@ class ConfigFile(attrdict.AttrDict):
         """
         val = self.get(key)
         if val and val.value:
-            return eval(val.value)  # pylint: disable=W0123 # noqa: DUO104
+            return eval(val.value)
         return None

labmate/acquisition/custom_lint.py

@@ -5,6 +5,7 @@ from typing import Any, Dict
 
 from ..utils.lint import get_all_args_from_call
 
+
 SPECIAL_FUNCTIONS_LIST = []
 
 
@@ -25,9 +26,7 @@ def on_call_functions(node: ast.Call, db: Dict[str, Any]):
             h = hash(tuple(get_all_args_from_call(node)))
             data = db.setdefault("save_fig", [])
             if h in data:
-                errors.append(
-                    "save_fig with the save arguments is used more then ones."
-                )
+                errors.append("save_fig with the save arguments is used more then ones.")
 
             data.append(h)
     return errors

labmate/acquisition_notebook/__init__.py

@@ -1,7 +1,7 @@
 # flake8: noqa: F401
+from ..acquisition.acquisition_loop import AcquisitionLoop
+from ..acquisition.analysis_data import AnalysisData
 from .acquisition_analysis_manager import (
     AcquisitionAnalysisManager,
     AcquisitionAnalysisManagerDataOnly,
 )
-from ..acquisition.acquisition_loop import AcquisitionLoop
-from ..acquisition.analysis_data import AnalysisData