junifer 0.0.7.dev264__tar.gz → 0.0.7.dev294__tar.gz

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.7.dev264
+Version: 0.0.7.dev294
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.7.dev294/docs/changes/newsfragments/472.enh

@@ -0,0 +1 @@
+Simplify ``Storage`` interface and implementations by `Synchon Mandal`_

junifer-0.0.7.dev294/docs/changes/newsfragments/476.enh

@@ -0,0 +1 @@
+Simplify :class:`.PipelineStepMixin` interface by `Synchon Mandal`_

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/docs/understanding/storage.rst

@@ -17,11 +17,11 @@ Storage is meant to be used inside the DataGrabber context but you can operate
 on them outside the context as long as the processed data is in the memory and
 the Python runtime has not garbage-collected it.
 
-The :ref:`Markers <marker>` are responsible for defining what *storage kind*
-(``matrix``, ``vector``, ``timeseries``, ``scalar_table``) they support for
-which :ref:`data type <data_types>` by overriding its ``get_output_type``
-method. The storage object in turn declares and provides implementation for
-specific *storage kind*. For example, :class:`.SQLiteFeatureStorage` supports
+The :ref:`Markers <marker>` are responsible for mapping the input
+:ref:`data type <data_types>` to its output :ref:`storage type <storage_types>`
+as shown :ref:`here <extending_markers_input_output>`.
+The storage object in turn declares and provides implementation for
+specific *storage type*. For example, :class:`.SQLiteFeatureStorage` supports
 saving ``matrix``, ``vector`` and ``timeseries`` via ``store_matrix``,
 ``store_vector`` and ``store_timeseries`` methods respectively.
 
@@ -57,6 +57,10 @@ Storage Types
      - A 2D square or non-square matrix of scalar values with column names
      - ``col_names``
      - :meth:`.BaseFeatureStorage.store_timeseries`
+   * - ``timeseries_2d``
+     - A 3D(2D+1D) square or non-square matrix of scalar values with column names across sessions
+     - ``col_names``
+     - :meth:`.BaseFeatureStorage.store_timeseries_2d`
    * - ``scalar_table``
      - | A 2D square or non-square matrix of scalar values with row name, column
        | name and row header column name
@@ -83,4 +87,4 @@ Storage Interfaces
    * - :class:`.HDF5FeatureStorage`
      - ``.hdf5``
      - HDF5
-     - ``matrix``, ``vector``, ``timeseries``, ``scalar_table``
+     - ``matrix``, ``vector``, ``timeseries``, ``timeseries_2d``, ``scalar_table``

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.0.7.dev264'
-__version_tuple__ = version_tuple = (0, 0, 7, 'dev264')
+__version__ = version = '0.0.7.dev294'
+__version_tuple__ = version_tuple = (0, 0, 7, 'dev294')
 
-__commit_id__ = commit_id = 'g6eb48995c'
+__commit_id__ = commit_id = 'gbb87fb782'

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/datareader/default.py

@@ -56,23 +56,6 @@ class DefaultDataReader(PipelineStepMixin, UpdateMetaMixin):
         # Nothing to validate, any input is fine
         return input
 
-    def get_output_type(self, input_type: str) -> str:
-        """Get output type.
-
-        Parameters
-        ----------
-        input_type : str
-            The data type input to the reader.
-
-        Returns
-        -------
-        str
-            The data type output by the reader.
-
-        """
-        # It will output the same type of data as the input
-        return input_type
-
     def _fit_transform(
         self,
         input: dict[str, dict],

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/datareader/tests/test_default_reader.py

@@ -30,7 +30,6 @@ def test_DefaultDataReader_validation(type_) -> None:
     """
     reader = DefaultDataReader()
     assert reader.validate_input(type_) == type_
-    assert reader.get_output_type(type_) == type_
     assert reader.validate(type_) == type_
 
 

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/pipeline/pipeline_step_mixin.py

@@ -53,25 +53,6 @@ class PipelineStepMixin:
             klass=NotImplementedError,
         )  # pragma: no cover
 
-    def get_output_type(self, input_type: str) -> str:
-        """Get output type.
-
-        Parameters
-        ----------
-        input_type : str
-            The data type input to the marker.
-
-        Returns
-        -------
-        str
-            The storage type output by the marker.
-
-        """
-        raise_error(
-            msg="Concrete classes need to implement get_output_type().",
-            klass=NotImplementedError,
-        )  # pragma: no cover
-
     def _fit_transform(
         self,
         input: dict[str, dict],
@@ -223,8 +204,9 @@ class PipelineStepMixin:
                     for val in self._MARKER_INOUT_MAPPINGS[t_input].values()
                 }
             )
+        # Only for datareader and preprocessor
         else:
-            outputs = [self.get_output_type(t_input) for t_input in fit_input]
+            outputs = fit_input
         return outputs
 
     def fit_transform(

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/pipeline/tests/test_pipeline_step_mixin.py

@@ -29,9 +29,6 @@ def test_PipelineStepMixin_correct_dependencies() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -50,9 +47,6 @@ def test_PipelineStepMixin_incorrect_dependencies() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -75,9 +69,6 @@ def test_PipelineStepMixin_correct_ext_dependencies() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -101,9 +92,6 @@ def test_PipelineStepMixin_ext_deps_correct_commands() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -129,9 +117,6 @@ def test_PipelineStepMixin_ext_deps_incorrect_commands() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -153,9 +138,6 @@ def test_PipelineStepMixin_incorrect_ext_dependencies() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -185,9 +167,6 @@ def test_PipelineStepMixin_correct_conditional_dependencies() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -214,9 +193,6 @@ def test_PipelineStepMixin_incorrect_conditional_dependencies() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 
@@ -249,9 +225,6 @@ def test_PipelineStepMixin_correct_conditional_ext_dependencies() -> None:
         def validate_input(self, input: list[str]) -> list[str]:
             return input
 
-        def get_output_type(self, input_type: str) -> str:
-            return input_type
-
         def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
             return {"input": input}
 

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/preprocess/base.py

@@ -115,23 +115,6 @@ class BasePreprocessor(ABC, PipelineStepMixin, UpdateMetaMixin):
         """
         return list(self._VALID_DATA_TYPES)
 
-    def get_output_type(self, input_type: str) -> str:
-        """Get output type.
-
-        Parameters
-        ----------
-        input_type : str
-            The data type input to the preprocessor.
-
-        Returns
-        -------
-        str
-            The data type output by the preprocessor.
-
-        """
-        # Does not add any new keys
-        return input_type
-
     @abstractmethod
     def preprocess(
         self,

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py

@@ -65,13 +65,6 @@ def test_fMRIPrepConfoundRemover_get_valid_inputs() -> None:
     assert confound_remover.get_valid_inputs() == ["BOLD"]
 
 
-def test_fMRIPrepConfoundRemover_get_output_type() -> None:
-    """Test fMRIPrepConfoundRemover get_output_type."""
-    confound_remover = fMRIPrepConfoundRemover()
-    # Confound remover works in place
-    assert confound_remover.get_output_type("BOLD") == "BOLD"
-
-
 def test_fMRIPrepConfoundRemover__map_adhoc_to_fmriprep() -> None:
     """Test fMRIPrepConfoundRemover adhoc to fmriprep spec mapping."""
     confound_remover = fMRIPrepConfoundRemover()

{junifer-0.0.7.dev264 → junifer-0.0.7.dev294}/junifer/storage/base.py

@@ -5,14 +5,14 @@
 # License: AGPL
 
 from abc import ABC, abstractmethod
-from collections.abc import Iterable
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any, ClassVar, Optional, Union
 
 import numpy as np
 import pandas as pd
 
-from ..utils import raise_error
+from ..utils import logger, raise_error
 from .utils import process_meta
 
 
@@ -22,44 +22,47 @@ __all__ = ["BaseFeatureStorage"]
 class BaseFeatureStorage(ABC):
     """Abstract base class for feature storage.
 
-    For every interface that is required, one needs to provide a concrete
+    For every storage, one needs to provide a concrete
     implementation of this abstract class.
 
     Parameters
     ----------
     uri : str or pathlib.Path
         The path to the storage.
-    storage_types : str or list of str
-        The available storage types for the class.
     single_output : bool, optional
         Whether to have single output (default True).
 
     Raises
     ------
-    ValueError
-        If required storage type(s) is(are) missing from ``storage_types``.
+    AttributeError
+        If the storage does not have `_STORAGE_TYPES` attribute.
 
     """
 
+    _STORAGE_TYPES: ClassVar[Sequence[str]]
+
     def __init__(
         self,
         uri: Union[str, Path],
-        storage_types: Union[list[str], str],
         single_output: bool = True,
     ) -> None:
-        self.uri = uri
-        # Convert storage_types to list
-        if not isinstance(storage_types, list):
-            storage_types = [storage_types]
-        # Check if required inputs are found
-        if any(x not in self.get_valid_inputs() for x in storage_types):
-            wrong_storage_types = [
-                x for x in storage_types if x not in self.get_valid_inputs()
-            ]
+        # Check for missing storage types attribute
+        if not hasattr(self, "_STORAGE_TYPES"):
             raise_error(
-                f"{self.__class__.__name__} cannot store {wrong_storage_types}"
+                msg="Missing `_STORAGE_TYPES` for the storage",
+                klass=AttributeError,
+            )
+        # Convert str to Path
+        if not isinstance(uri, Path):
+            uri = Path(uri)
+        self.uri = uri
+        # Create parent directories if not present
+        if not self.uri.parent.exists():
+            logger.info(
+                f"Output directory: '{self.uri.parent.resolve()}' "
+                "does not exist, creating now"
             )
-        self._valid_inputs = storage_types
+            self.uri.parent.mkdir(parents=True, exist_ok=True)
         self.single_output = single_output
 
     def get_valid_inputs(self) -> list[str]:
@@ -69,13 +72,10 @@ class BaseFeatureStorage(ABC):
         -------
         list of str
             The list of storage types that can be used as input for this
-            storage interface.
+            storage.
 
         """
-        raise_error(
-            msg="Concrete classes need to implement get_valid_inputs().",
-            klass=NotImplementedError,
-        )
+        return list(self._STORAGE_TYPES)
 
     def validate(self, input_: list[str]) -> None:
         """Validate the input to the pipeline step.
@@ -91,11 +91,12 @@ class BaseFeatureStorage(ABC):
             If the ``input_`` is invalid.
 
         """
-        if not any(x in input_ for x in self._valid_inputs):
+        # self._STORAGE_TYPES should be there already
+        if not any(x in input_ for x in self._STORAGE_TYPES):
             raise_error(
                 "Input does not have the required data."
                 f"\t Input: {input}"
-                f"\t Required (any of): {self._valid_inputs}"
+                f"\t Required (any of): {self._STORAGE_TYPES}"
             )
 
     @abstractmethod
@@ -113,7 +114,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement list_features().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
     def read(
@@ -141,7 +142,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement read().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
     def read_df(
@@ -167,7 +168,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement read_df().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
     def store_metadata(self, meta_md5: str, element: dict, meta: dict) -> None:
@@ -186,7 +187,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement store_metadata().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     def store(self, kind: str, **kwargs) -> None:
         """Store extracted features data.
@@ -206,14 +207,16 @@ class BaseFeatureStorage(ABC):
         """
         # Do the check before calling the abstract methods, otherwise the
         # meta might be stored even if the data is not stored.
-        if kind not in self._valid_inputs:
+        if kind not in self._STORAGE_TYPES:
             raise_error(
                 msg=f"I don't know how to store {kind}.",
                 klass=ValueError,
             )
+        # Process and store metadata
         t_meta = kwargs.pop("meta")
         meta_md5, t_meta, t_element = process_meta(t_meta)
         self.store_metadata(meta_md5=meta_md5, element=t_element, meta=t_meta)
+        # Store data
         if kind == "matrix":
             self.store_matrix(meta_md5=meta_md5, element=t_element, **kwargs)
         elif kind == "timeseries":
@@ -236,8 +239,8 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
-        row_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
+        row_names: Optional[Sequence[str]] = None,
         matrix_kind: str = "full",
         diagonal: bool = True,
     ) -> None:
@@ -251,9 +254,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The matrix data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
-        row_names : str, optional
+        row_names : list-like of str, optional
             The row labels (default None).
         matrix_kind : str, optional
             The kind of matrix:
@@ -278,7 +281,7 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: Union[np.ndarray, list],
-        col_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
     ) -> None:
         """Store vector.
 
@@ -290,7 +293,7 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray or list
             The vector data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
 
         """
@@ -304,7 +307,7 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
     ) -> None:
         """Store timeseries.
 
@@ -316,7 +319,7 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
 
         """
@@ -330,8 +333,8 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
-        row_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
+        row_names: Optional[Sequence[str]] = None,
     ) -> None:
         """Store 2D timeseries.
 
@@ -343,9 +346,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
-        row_names : list or tuple of str, optional
+        row_names : list-like of str, optional
             The row labels (default None).
 
         """
@@ -359,8 +362,8 @@ class BaseFeatureStorage(ABC):
         meta_md5: str,
         element: dict,
         data: np.ndarray,
-        col_names: Optional[Iterable[str]] = None,
-        row_names: Optional[Iterable[str]] = None,
+        col_names: Optional[Sequence[str]] = None,
+        row_names: Optional[Sequence[str]] = None,
         row_header_col_name: Optional[str] = "feature",
     ) -> None:
         """Store table with scalar values.
@@ -373,9 +376,9 @@ class BaseFeatureStorage(ABC):
             The element as a dictionary.
         data : numpy.ndarray
             The timeseries data to store.
-        col_names : list or tuple of str, optional
+        col_names : list-like of str, optional
             The column labels (default None).
-        row_names : str, optional
+        row_names : list-like of str, optional
             The row labels (default None).
         row_header_col_name : str, optional
             The column name for the row header column (default "feature").
@@ -392,7 +395,7 @@ class BaseFeatureStorage(ABC):
         raise_error(
             msg="Concrete classes need to implement collect().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     def __str__(self) -> str:
         """Represent object as string.