junifer 0.0.7.dev264__tar.gz → 0.0.7.dev289__tar.gz

{junifer-0.0.7.dev264 → junifer-0.0.7.dev289}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.7.dev264
+Version: 0.0.7.dev289
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.7.dev289/docs/changes/newsfragments/472.enh

@@ -0,0 +1 @@
+Simplify ``Storage`` interface and implementations by `Synchon Mandal`_

{junifer-0.0.7.dev264 → junifer-0.0.7.dev289}/docs/understanding/storage.rst

@@ -17,11 +17,11 @@ Storage is meant to be used inside the DataGrabber context but you can operate
 on them outside the context as long as the processed data is in the memory and
 the Python runtime has not garbage-collected it.
 
-The :ref:`Markers <marker>` are responsible for defining what *storage kind*
-(``matrix``, ``vector``, ``timeseries``, ``scalar_table``) they support for
-which :ref:`data type <data_types>` by overriding its ``get_output_type``
-method. The storage object in turn declares and provides implementation for
-specific *storage kind*. For example, :class:`.SQLiteFeatureStorage` supports
+The :ref:`Markers <marker>` are responsible for mapping the input
+:ref:`data type <data_types>` to its output :ref:`storage type <storage_types>`
+as shown :ref:`here <extending_markers_input_output>`.
+The storage object in turn declares and provides implementation for
+specific *storage type*. For example, :class:`.SQLiteFeatureStorage` supports
 saving ``matrix``, ``vector`` and ``timeseries`` via ``store_matrix``,
 ``store_vector`` and ``store_timeseries`` methods respectively.
 
@@ -57,6 +57,10 @@ Storage Types
      - A 2D square or non-square matrix of scalar values with column names
      - ``col_names``
      - :meth:`.BaseFeatureStorage.store_timeseries`
+   * - ``timeseries_2d``
+     - A 3D(2D+1D) square or non-square matrix of scalar values with column names across sessions
+     - ``col_names``
+     - :meth:`.BaseFeatureStorage.store_timeseries_2d`
    * - ``scalar_table``
      - | A 2D square or non-square matrix of scalar values with row name, column
        | name and row header column name
@@ -83,4 +87,4 @@ Storage Interfaces
    * - :class:`.HDF5FeatureStorage`
      - ``.hdf5``
      - HDF5
-     - ``matrix``, ``vector``, ``timeseries``, ``scalar_table``
+     - ``matrix``, ``vector``, ``timeseries``, ``timeseries_2d``, ``scalar_table``

{junifer-0.0.7.dev264 → junifer-0.0.7.dev289}/junifer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.7.dev264'
-__version_tuple__ = version_tuple = (0, 0, 7, 'dev264')
+__version__ = version = '0.0.7.dev289'
+__version_tuple__ = version_tuple = (0, 0, 7, 'dev289')
 
-__commit_id__ = commit_id = 'g6eb48995c'
+__commit_id__ = commit_id = 'g5abcacf68'

{junifer-0.0.7.dev264 → junifer-0.0.7.dev289}/junifer/storage/base.py

@@ -5,14 +5,14 @@
 # License: AGPL
 
 from abc import ABC, abstractmethod
-from collections.abc import Iterable
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, ClassVar, Optional, Union
 
 import numpy as np
 import pandas as pd
 
-from ..utils import raise_error
+from ..utils import logger, raise_error
 from .utils import process_meta
 
 
@@ -22,44 +22,47 @@ __all__ = ["BaseFeatureStorage"]
 class BaseFeatureStorage(ABC):
     """Abstract base class for feature storage.
 
-    For every interface that is required, one needs to provide a concrete
+    For every storage, one needs to provide a concrete
     implementation of this abstract class.
 
     Parameters
     ----------
     uri : str or pathlib.Path
         The path to the storage.
-    storage_types : str or list of str
-        The available storage types for the class.
     single_output : bool, optional
         Whether to have single output (default True).
 
     Raises
     ------
-    ValueError
-        If required storage type(s) is(are) missing from ``storage_types``.
+    AttributeError
+        If the storage does not have `_STORAGE_TYPES` attribute.
 
     """
 
+    _STORAGE_TYPES: ClassVar[Sequence[str]]
+
     def __init__(
         self,
         uri: Union[str, Path],
-        storage_types: Union[list[str], str],
         single_output: bool = True,
     ) -> None:
-        self.uri = uri
-        # Convert storage_types to list
-        if not isinstance(storage_types, list):
-            storage_types = [storage_types]
-        # Check if required inputs are found
-        if any(x not in self.get_valid_inputs() for x in storage_types):
-            wrong_storage_types = [
-                x for x in storage_types if x not in self.get_valid_inputs()
-            ]
+        # Check for missing storage types attribute
+        if not hasattr(self, "_STORAGE_TYPES"):
             raise_error(
-                f"{self.__class__.__name__} cannot store {wrong_storage_types}"
+                msg="Missing `_STORAGE_TYPES` for the storage",
+                klass=AttributeError,
+            )
+        # Convert str to Path
+        if not isinstance(uri, Path):
+            uri = Path(uri)
+        self.uri = uri
+        # Create parent directories if not present
+        if not self.uri.parent.exists():
+            logger.info(
+                f"Output directory: '{self.uri.parent.resolve()}' "
+                "does not exist, creating now"
             )
-        self._valid_inputs = storage_types
+            self.uri.parent.mkdir(parents=True, exist_ok=True)
         self.single_output = single_output
 
     def get_valid_inputs(self) -> list[str]:
@@ -69,13 +72,10 @@ class BaseFeatureStorage(ABC):
         -------
         list of str
             The list of storage types that can be used as input for this
-            storage interface.
+            storage.
 
         """
-        raise_error(
-            msg="Concrete classes need to implement get_valid_inputs().",
-            klass=NotImplementedError,
-        )
+        return list(self._STORAGE_TYPES)
 
     def validate(self, input_: list[str]) -> None:
         """Validate the input to the pipeline step.
@@ -91,11 +91,12 @@ class BaseFeatureStorage(ABC):
             If the ``input_`` is invalid.
 
         """
-        if not any(x in input_ for x in self._valid_inputs):
+        # self._STORAGE_TYPES should be there already
+        if not any(x in input_ for x in self._STORAGE_TYPES):
             raise_error(
                 "Input does not have the required data."
                 f"\t Input: {input}"
-                f"\t Required (any of): {self._valid_inputs}"
+                f"\t Required (any of): {self._STORAGE_TYPES}"
             )
 
     @abstractmethod
@@ -113,7 +114,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement list_features().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
     def read(
@@ -141,7 +142,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement read().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
     def read_df(
@@ -167,7 +168,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement read_df().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
     def store_metadata(self, meta_md5: str, element: dict, meta: dict) -> None:
@@ -186,7 +187,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement store_metadata().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     def store(self, kind: str, **kwargs) -> None:
         """Store extracted features data.
@@ -206,14 +207,16 @@ class BaseFeatureStorage(ABC):
         """
         # Do the check before calling the abstract methods, otherwise the
         # meta might be stored even if the data is not stored.
-        if kind not in self._valid_inputs:
+        if kind not in self._STORAGE_TYPES:
             raise_error(
                 msg=f"I don't know how to store {kind}.",
                 klass=ValueError,
             )
+        # Process and store metadata
         t_meta = kwargs.pop("meta")
         meta_md5, t_meta, t_element = process_meta(t_meta)
         self.store_metadata(meta_md5=meta_md5, element=t_element, meta=t_meta)
+        # Store data
         if kind == "matrix":
             self.store_matrix(meta_md5=meta_md5, element=t_element, **kwargs)
         elif kind == "timeseries":
@@ -236,8 +239,8 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
-        row_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
+        row_names: Optional[Sequence[str]] = None,
         matrix_kind: str = "full",
         diagonal: bool = True,
     ) -> None:
@@ -251,9 +254,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The matrix data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
-        row_names : str, optional
+        row_names : list-like of str, optional
             The row labels (default None).
         matrix_kind : str, optional
             The kind of matrix:
@@ -278,7 +281,7 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: Union[np.ndarray, list],
-        col_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
     ) -> None:
         """Store vector.
 
@@ -290,7 +293,7 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray or list
             The vector data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
 
         """
@@ -304,7 +307,7 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
     ) -> None:
         """Store timeseries.
 
@@ -316,7 +319,7 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
 
         """
@@ -330,8 +333,8 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
-        row_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
+        row_names: Optional[Sequence[str]] = None,
     ) -> None:
         """Store 2D timeseries.
 
@@ -343,9 +346,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
-        row_names : list or tuple of str, optional
+        row_names : list-like of str, optional
             The row labels (default None).
 
         """
@@ -359,8 +362,8 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
-        row_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
+        row_names: Optional[Sequence[str]] = None,
         row_header_col_name: Optional[str] = "feature",
     ) -> None:
         """Store table with scalar values.
@@ -373,9 +376,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
-        row_names : str, optional
+        row_names : list-like of str, optional
             The row labels (default None).
         row_header_col_name : str, optional
             The column name for the row header column (default "feature").
@@ -392,7 +395,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement collect().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     def __str__(self) -> str:
         """Represent object as string.