junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/storage/utils.py

@@ -10,8 +10,10 @@ from collections.abc import Sequence
 from importlib.metadata import PackageNotFoundError, version
 
 import numpy as np
+from pydantic import validate_call
 
-from ..utils.logging import logger, raise_error
+from ..utils import raise_error
+from .base import MatrixKind, logger
 
 
 __all__ = [
@@ -72,10 +74,17 @@ def _meta_hash(meta: dict) -> str:
     logger.debug(f"Hashing metadata: {meta}")
     if "dependencies" not in meta:
         raise_error("The metadata must contain the key 'dependencies'")
-    # Convert dependencies set into {dependency: version} dictionary
+    # Convert dependencies set into {dependency: version} dictionary, with
+    # junifer, so that the features computed with different versions of
+    # junifer have different hashes
     meta["dependencies"] = {
-        dep: get_dependency_version(dep) for dep in meta["dependencies"]
+        dep: get_dependency_version(dep)
+        for dep in {*meta["dependencies"], "junifer"}
     }
+    # Remove datadir from datagrabber only if datalad-based
+    if meta.get("datagrabber") is not None:
+        if meta["datagrabber"].get("uri") is not None:
+            _ = meta["datagrabber"].pop("datadir")
     meta_md5 = hashlib.md5(
         json.dumps(meta, sort_keys=True).encode("utf-8")
     ).hexdigest()
@@ -86,8 +95,13 @@ def _meta_hash(meta: dict) -> str:
 def process_meta(meta: dict) -> tuple[str, dict, dict]:
     """Process the metadata for storage.
 
-    It removes the key "element" and adds the "_element_keys" with the keys
-    used to index the element.
+    It removes the key "element" (the element being processed) and keeps
+    "_element_keys", the keys of the element that the data depends on (set by
+    the DataGrabber, e.g., only the subject for data that is the same for all
+    the tasks of a subject). If not set, all the keys of the element are used.
+    The element of the data, with only these keys, is the one to store, so the
+    data that does not depend on a key is stored once instead of once for
+    each of its values.
 
     Parameters
     ----------
@@ -101,12 +115,13 @@ def process_meta(meta: dict) -> tuple[str, dict, dict]:
     dict
         The processed metadata for storage.
     dict
-        The element.
+        The element of the data, with the keys in "_element_keys".
 
     Raises
     ------
     ValueError
-        If ``meta`` is None or if it does not contain the key "element".
+        If ``meta=None``, if it does not contain the key "element" or if
+        "_element_keys" has keys that are not in the element.
 
     """
     if meta is None:
@@ -124,7 +139,18 @@ def process_meta(meta: dict) -> tuple[str, dict, dict]:
     if "type" not in t_meta:
         raise_error(msg="`meta` must contain the key 'type'")
 
-    t_meta["_element_keys"] = list(element.keys())
+    # The keys of the element that the data depends on
+    element_keys = t_meta.get("_element_keys") or list(element.keys())
+    missing = [k for k in element_keys if k not in element]
+    if missing:
+        raise_error(
+            msg=(
+                f"The keys {missing} of `meta['_element_keys']` are not in "
+                f"the element: {element}"
+            )
+        )
+    t_meta["_element_keys"] = list(element_keys)
+    element = {k: element[k] for k in element_keys}
     type_ = t_meta["type"]
     name = t_meta["marker"]["name"]
     t_meta["name"] = f"{type_}_{name}"
@@ -163,8 +189,9 @@ def element_to_prefix(element: dict) -> str:
     return f"{prefix}_"
 
 
+@validate_call
 def store_matrix_checks(
-    matrix_kind: str,
+    matrix_kind: MatrixKind,
     diagonal: bool,
     data_shape: tuple[int, int],
     row_names_len: int,
@@ -174,15 +201,10 @@ def store_matrix_checks(
 
     Parameters
     ----------
-    matrix_kind : {"triu", "tril", "full"}
-        The kind of matrix:
-
-        * ``triu`` : store upper triangular only
-        * ``tril`` : store lower triangular
-        * ``full`` : full matrix
-
+    matrix_kind : :enum:`.MatrixKind`, optional
+        The matrix kind.
     diagonal : bool
-        Whether to store the diagonal. If ``matrix_kind`` is "full",
+        Whether to store the diagonal. If ``matrix_kind=MatrixKind.Full``,
         setting this to False will raise an error.
     data_shape : tuple of int and int
         The shape of the matrix data to store.
@@ -194,16 +216,12 @@ def store_matrix_checks(
     Raises
     ------
     ValueError
-        If the matrix kind is invalid
-        If the diagonal is False and the matrix kind is "full"
-        If the matrix kind is "triu" or "tril" and the matrix is not square
-        If the number of row names does not match the number of rows
-        If the number of column names does not match the number of columns
+        If ``diagonal=False`` and ``matrix_kind="full"`` or
+        if ``matrix_kind`` is "triu" or "tril" and the matrix is not square or
+        if the number of row names does not match the number of rows
+        If the number of column names does not match the number of columns.
 
     """
-    # Matrix kind validation
-    if matrix_kind not in ("triu", "tril", "full"):
-        raise_error(msg=f"Invalid kind {matrix_kind}", klass=ValueError)
     # Diagonal validation
     if diagonal is False and matrix_kind not in ["triu", "tril"]:
         raise_error(
@@ -250,9 +268,9 @@ def store_timeseries_2d_checks(
     Raises
     ------
     ValueError
-        If the data is not a 3D array (timepoints, rows, columns)
-        If the number of row names does not match the number of rows
-        If the number of column names does not match the number of columns
+        If the data is not a 3D array (timepoints, rows, columns) or
+        if the number of row names does not match the number of rows or
+        if the number of column names does not match the number of columns.
 
     """
     # Data validation
@@ -280,7 +298,7 @@ def matrix_to_vector(
     data: np.ndarray,
     col_names: Sequence[str],
     row_names: Sequence[str],
-    matrix_kind: str,
+    matrix_kind: MatrixKind,
     diagonal: bool,
 ) -> tuple[np.ndarray, list[str]]:
     """Convert matrix to vector based on parameters.
@@ -293,13 +311,8 @@ def matrix_to_vector(
         The column labels.
     row_names : list-like of str
         The row labels.
-    matrix_kind : str
-        The kind of matrix:
-
-        * ``triu`` : store upper triangular only
-        * ``tril`` : store lower triangular
-        * ``full`` : full matrix
-
+    matrix_kind : :enum:`.MatrixKind`
+        The matrix kind.
     diagonal : bool
         Whether to store the diagonal.
 
@@ -310,6 +323,11 @@ def matrix_to_vector(
     list of str
         The column labels.
 
+    Raises
+    ------
+    ValueError
+        If the matrix kind is invalid.
+
     """
     # Prepare data indexing based on matrix kind
     if matrix_kind == "triu":
@@ -318,17 +336,19 @@ def matrix_to_vector(
     elif matrix_kind == "tril":
         k = 0 if diagonal is True else -1
         data_idx = np.tril_indices(data.shape[0], k=k)
-    else:  # full
+    elif matrix_kind == "full":  # full
         data_idx = (
             np.repeat(np.arange(data.shape[0]), data.shape[1]),
             np.tile(np.arange(data.shape[1]), data.shape[0]),
         )
+    else:
+        raise_error(f"Invalid matrix kind: {matrix_kind}")
     # Subset data as 1D
     flat_data = data[data_idx]
     # Generate flat 1D row X column names
     columns = [
         f"{row_names[i]}~{col_names[j]}"
-        for i, j in zip(data_idx[0], data_idx[1])
+        for i, j in zip(data_idx[0], data_idx[1], strict=False)
     ]
 
     return flat_data, columns

junifer/testing/__init__.pyi

@@ -1,4 +1,4 @@
-__all__ = ["datagrabbers", "registry", "get_testing_data"]
+__all__ = ["datagrabbers", "registry", "config_override", "get_testing_data"]
 
 from . import datagrabbers, registry
-from .utils import get_testing_data
+from .utils import config_override, get_testing_data