junifer 0.0.5.dev242__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/datagrabber/pattern_validation_mixin.py

@@ -3,8 +3,8 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Dict, List
 
+from ..typing import DataGrabberPatterns
 from ..utils import logger, raise_error, warn_with_log
 
 
@@ -33,10 +33,12 @@ PATTERNS_SCHEMA = {
                 "mandatory": ["pattern", "format"],
                 "optional": ["mappings"],
             },
+            "reference": {"mandatory": ["pattern"], "optional": []},
+            "prewarp_space": {"mandatory": [], "optional": []},
         },
     },
     "Warp": {
-        "mandatory": ["pattern", "src", "dst"],
+        "mandatory": ["pattern", "src", "dst", "warper"],
         "optional": {},
     },
     "VBM_GM": {
@@ -72,7 +74,7 @@ PATTERNS_SCHEMA = {
 class PatternValidationMixin:
     """Mixin class for pattern validation."""
 
-    def _validate_types(self, types: List[str]) -> None:
+    def _validate_types(self, types: list[str]) -> None:
         """Validate the types.
 
         Parameters
@@ -95,8 +97,8 @@ class PatternValidationMixin:
 
     def _validate_replacements(
         self,
-        replacements: List[str],
-        patterns: Dict[str, Dict[str, str]],
+        replacements: list[str],
+        patterns: DataGrabberPatterns,
         partial_pattern_ok: bool,
     ) -> None:
         """Validate the replacements.
@@ -132,45 +134,57 @@ class PatternValidationMixin:
 
         if any(not isinstance(x, str) for x in replacements):
             raise_error(
-                msg="`replacements` must be a list of strings.",
+                msg="`replacements` must be a list of strings",
                 klass=TypeError,
             )
 
+        # Make a list of all patterns recursively
+        all_patterns = []
+        for dtype_val in patterns.values():
+            # Conditional for list dtype vals like Warp
+            if isinstance(dtype_val, list):
+                for entry in dtype_val:
+                    all_patterns.append(entry.get("pattern", ""))
+            else:
+                all_patterns.append(dtype_val.get("pattern", ""))
+        # Check for stray replacements
         for x in replacements:
-            if all(
-                x not in y
-                for y in [
-                    data_type_val.get("pattern", "")
-                    for data_type_val in patterns.values()
-                ]
-            ):
+            if all(x not in y for y in all_patterns):
                 if partial_pattern_ok:
                     warn_with_log(
                         f"Replacement: `{x}` is not part of any pattern, "
                         "things might not work as expected if you are unsure "
-                        "of what you are doing"
+                        "of what you are doing."
                     )
                 else:
                     raise_error(
-                        msg=f"Replacement: {x} is not part of any pattern."
+                        msg=f"Replacement: `{x}` is not part of any pattern"
                     )
 
         # Check that at least one pattern has all the replacements
         at_least_one = False
-        for data_type_val in patterns.values():
-            if all(
-                x in data_type_val.get("pattern", "") for x in replacements
-            ):
-                at_least_one = True
+        for dtype_val in patterns.values():
+            # Conditional for list dtype vals like Warp
+            if isinstance(dtype_val, list):
+                for entry in dtype_val:
+                    if all(
+                        x in entry.get("pattern", "") for x in replacements
+                    ):
+                        at_least_one = True
+            else:
+                if all(
+                    x in dtype_val.get("pattern", "") for x in replacements
+                ):
+                    at_least_one = True
         if not at_least_one and not partial_pattern_ok:
             raise_error(
-                msg="At least one pattern must contain all replacements."
+                msg="At least one pattern must contain all replacements"
             )
 
     def _validate_mandatory_keys(
         self,
-        keys: List[str],
-        schema: List[str],
+        keys: list[str],
+        schema: list[str],
         data_type: str,
         partial_pattern_ok: bool = False,
     ) -> None:
@@ -207,7 +221,7 @@ class PatternValidationMixin:
                     warn_with_log(
                         f"Mandatory key: `{key}` not found for {data_type}, "
                         "things might not work as expected if you are unsure "
-                        "of what you are doing"
+                        "of what you are doing."
                     )
                 else:
                     raise_error(
@@ -215,10 +229,10 @@ class PatternValidationMixin:
                         klass=KeyError,
                     )
             else:
-                logger.debug(f"Mandatory key: `{key}` found for {data_type}")
+                logger.debug(f"Mandatory key: `{key}` found for {data_type}.")
 
     def _identify_stray_keys(
-        self, keys: List[str], schema: List[str], data_type: str
+        self, keys: list[str], schema: list[str], data_type: str
     ) -> None:
         """Identify stray keys.
 
@@ -249,9 +263,9 @@ class PatternValidationMixin:
 
     def validate_patterns(
         self,
-        types: List[str],
-        replacements: List[str],
-        patterns: Dict[str, Dict[str, str]],
+        types: list[str],
+        replacements: list[str],
+        patterns: DataGrabberPatterns,
         partial_pattern_ok: bool = False,
     ) -> None:
         """Validate the patterns.
@@ -298,87 +312,185 @@ class PatternValidationMixin:
                 msg="`patterns` must contain all `types`", klass=ValueError
             )
         # Check against schema
-        for data_type_key, data_type_val in patterns.items():
+        for dtype_key, dtype_val in patterns.items():
             # Check if valid data type is provided
-            if data_type_key not in PATTERNS_SCHEMA:
+            if dtype_key not in PATTERNS_SCHEMA:
                 raise_error(
-                    f"Unknown data type: {data_type_key}, "
+                    f"Unknown data type: {dtype_key}, "
                     f"should be one of: {list(PATTERNS_SCHEMA.keys())}"
                 )
-            # Check mandatory keys for data type
-            self._validate_mandatory_keys(
-                keys=list(data_type_val),
-                schema=PATTERNS_SCHEMA[data_type_key]["mandatory"],
-                data_type=data_type_key,
-                partial_pattern_ok=partial_pattern_ok,
-            )
-            # Check optional keys for data type
-            for optional_key, optional_val in PATTERNS_SCHEMA[data_type_key][
-                "optional"
-            ].items():
-                if optional_key not in data_type_val:
-                    logger.debug(
-                        f"Optional key: `{optional_key}` missing for "
-                        f"{data_type_key}"
-                    )
-                else:
-                    logger.debug(
-                        f"Optional key: `{optional_key}` found for "
-                        f"{data_type_key}"
-                    )
-                    # Set nested type name for easier access
-                    nested_data_type = f"{data_type_key}.{optional_key}"
-                    nested_mandatory_keys_schema = PATTERNS_SCHEMA[
-                        data_type_key
-                    ]["optional"][optional_key]["mandatory"]
-                    nested_optional_keys_schema = PATTERNS_SCHEMA[
-                        data_type_key
-                    ]["optional"][optional_key]["optional"]
-                    # Check mandatory keys for nested type
+            # Conditional for list dtype vals like Warp
+            if isinstance(dtype_val, list):
+                for idx, entry in enumerate(dtype_val):
+                    # Check mandatory keys for data type
                     self._validate_mandatory_keys(
-                        keys=list(optional_val["mandatory"]),
-                        schema=nested_mandatory_keys_schema,
-                        data_type=nested_data_type,
+                        keys=list(entry),
+                        schema=PATTERNS_SCHEMA[dtype_key]["mandatory"],
+                        data_type=f"{dtype_key}.{idx}",
                         partial_pattern_ok=partial_pattern_ok,
                     )
-                    # Check optional keys for nested type
-                    for nested_optional_key in nested_optional_keys_schema:
-                        if nested_optional_key not in optional_val["optional"]:
+                    # Check optional keys for data type
+                    for optional_key, optional_val in PATTERNS_SCHEMA[
+                        dtype_key
+                    ]["optional"].items():
+                        if optional_key not in entry:
                             logger.debug(
-                                f"Optional key: `{nested_optional_key}` "
-                                f"missing for {nested_data_type}"
+                                f"Optional key: `{optional_key}` missing for "
+                                f"{dtype_key}.{idx}"
                             )
                         else:
                             logger.debug(
-                                f"Optional key: `{nested_optional_key}` found "
-                                f"for {nested_data_type}"
+                                f"Optional key: `{optional_key}` found for "
+                                f"{dtype_key}.{idx}"
+                            )
+                            # Set nested type name for easier access
+                            nested_dtype = f"{dtype_key}.{idx}.{optional_key}"
+                            nested_mandatory_keys_schema = PATTERNS_SCHEMA[
+                                dtype_key
+                            ]["optional"][optional_key]["mandatory"]
+                            nested_optional_keys_schema = PATTERNS_SCHEMA[
+                                dtype_key
+                            ]["optional"][optional_key]["optional"]
+                            # Check mandatory keys for nested type
+                            self._validate_mandatory_keys(
+                                keys=list(optional_val["mandatory"]),
+                                schema=nested_mandatory_keys_schema,
+                                data_type=nested_dtype,
+                                partial_pattern_ok=partial_pattern_ok,
+                            )
+                            # Check optional keys for nested type
+                            for (
+                                nested_optional_key
+                            ) in nested_optional_keys_schema:
+                                if (
+                                    nested_optional_key
+                                    not in optional_val["optional"]
+                                ):
+                                    logger.debug(
+                                        f"Optional key: "
+                                        f"`{nested_optional_key}` missing for "
+                                        f"{nested_dtype}"
+                                    )
+                                else:
+                                    logger.debug(
+                                        f"Optional key: "
+                                        f"`{nested_optional_key}` found for "
+                                        f"{nested_dtype}"
+                                    )
+                            # Check stray key for nested data type
+                            self._identify_stray_keys(
+                                keys=(
+                                    optional_val["mandatory"]
+                                    + optional_val["optional"]
+                                ),
+                                schema=(
+                                    nested_mandatory_keys_schema
+                                    + nested_optional_keys_schema
+                                ),
+                                data_type=nested_dtype,
                             )
-                    # Check stray key for nested data type
+                    # Check stray key for data type
                     self._identify_stray_keys(
-                        keys=optional_val["mandatory"]
-                        + optional_val["optional"],
-                        schema=nested_mandatory_keys_schema
-                        + nested_optional_keys_schema,
-                        data_type=nested_data_type,
+                        keys=list(entry.keys()),
+                        schema=(
+                            PATTERNS_SCHEMA[dtype_key]["mandatory"]
+                            + list(
+                                PATTERNS_SCHEMA[dtype_key]["optional"].keys()
+                            )
+                        ),
+                        data_type=dtype_key,
                     )
-            # Check stray key for data type
-            self._identify_stray_keys(
-                keys=list(data_type_val.keys()),
-                schema=(
-                    PATTERNS_SCHEMA[data_type_key]["mandatory"]
-                    + list(PATTERNS_SCHEMA[data_type_key]["optional"].keys())
-                ),
-                data_type=data_type_key,
-            )
-            # Wildcard check in patterns
-            if "}*" in data_type_val.get("pattern", ""):
-                raise_error(
-                    msg=(
-                        f"`{data_type_key}.pattern` must not contain `*` "
-                        "following a replacement"
+                    # Wildcard check in patterns
+                    if "}*" in entry.get("pattern", ""):
+                        raise_error(
+                            msg=(
+                                f"`{dtype_key}.pattern` must not contain `*` "
+                                "following a replacement"
+                            ),
+                            klass=ValueError,
+                        )
+            else:
+                # Check mandatory keys for data type
+                self._validate_mandatory_keys(
+                    keys=list(dtype_val),
+                    schema=PATTERNS_SCHEMA[dtype_key]["mandatory"],
+                    data_type=dtype_key,
+                    partial_pattern_ok=partial_pattern_ok,
+                )
+                # Check optional keys for data type
+                for optional_key, optional_val in PATTERNS_SCHEMA[dtype_key][
+                    "optional"
+                ].items():
+                    if optional_key not in dtype_val:
+                        logger.debug(
+                            f"Optional key: `{optional_key}` missing for "
+                            f"{dtype_key}."
+                        )
+                    else:
+                        logger.debug(
+                            f"Optional key: `{optional_key}` found for "
+                            f"{dtype_key}."
+                        )
+                        # Set nested type name for easier access
+                        nested_dtype = f"{dtype_key}.{optional_key}"
+                        nested_mandatory_keys_schema = PATTERNS_SCHEMA[
+                            dtype_key
+                        ]["optional"][optional_key]["mandatory"]
+                        nested_optional_keys_schema = PATTERNS_SCHEMA[
+                            dtype_key
+                        ]["optional"][optional_key]["optional"]
+                        # Check mandatory keys for nested type
+                        self._validate_mandatory_keys(
+                            keys=list(optional_val["mandatory"]),
+                            schema=nested_mandatory_keys_schema,
+                            data_type=nested_dtype,
+                            partial_pattern_ok=partial_pattern_ok,
+                        )
+                        # Check optional keys for nested type
+                        for nested_optional_key in nested_optional_keys_schema:
+                            if (
+                                nested_optional_key
+                                not in optional_val["optional"]
+                            ):
+                                logger.debug(
+                                    f"Optional key: `{nested_optional_key}` "
+                                    f"missing for {nested_dtype}"
+                                )
+                            else:
+                                logger.debug(
+                                    f"Optional key: `{nested_optional_key}` "
+                                    f"found for {nested_dtype}"
+                                )
+                        # Check stray key for nested data type
+                        self._identify_stray_keys(
+                            keys=(
+                                optional_val["mandatory"]
+                                + optional_val["optional"]
+                            ),
+                            schema=(
+                                nested_mandatory_keys_schema
+                                + nested_optional_keys_schema
+                            ),
+                            data_type=nested_dtype,
+                        )
+                # Check stray key for data type
+                self._identify_stray_keys(
+                    keys=list(dtype_val.keys()),
+                    schema=(
+                        PATTERNS_SCHEMA[dtype_key]["mandatory"]
+                        + list(PATTERNS_SCHEMA[dtype_key]["optional"].keys())
                     ),
-                    klass=ValueError,
+                    data_type=dtype_key,
                 )
+                # Wildcard check in patterns
+                if "}*" in dtype_val.get("pattern", ""):
+                    raise_error(
+                        msg=(
+                            f"`{dtype_key}.pattern` must not contain `*` "
+                            "following a replacement"
+                        ),
+                        klass=ValueError,
+                    )
 
         # Validate replacements
         self._validate_replacements(

junifer/datagrabber/tests/test_datalad_base.py

@@ -3,20 +3,21 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import warnings
 from pathlib import Path
-from typing import Type
 
 import datalad.api as dl
 import pytest
 
 from junifer.datagrabber import DataladDataGrabber
+from junifer.utils import config
 
 
 _testing_dataset = {
     "example_bids": {
         "uri": "https://gin.g-node.org/juaml/datalad-example-bids",
-        "commit": "b87897cbe51bf0ee5514becaa5c7dd76491db5ad",
-        "id": "8fddff30-6993-420a-9d1e-b5b028c59468",
+        "commit": "3f288c8725207ae0c9b3616e093e78cda192b570",
+        "id": "582b9696-f13f-42e4-9587-b4e62aa2a8e7",
     },
     "example_bids_ses": {
         "uri": "https://gin.g-node.org/juaml/datalad-example-bids-ses",
@@ -27,7 +28,7 @@ _testing_dataset = {
 
 
 @pytest.fixture
-def concrete_datagrabber() -> Type[DataladDataGrabber]:
+def concrete_datagrabber() -> type[DataladDataGrabber]:
     """Return a concrete datalad-based DataGrabber.
 
     Returns
@@ -69,7 +70,7 @@ def concrete_datagrabber() -> Type[DataladDataGrabber]:
 
 
 def test_DataladDataGrabber_install_errors(
-    tmp_path: Path, concrete_datagrabber: Type
+    tmp_path: Path, concrete_datagrabber: type
 ) -> None:
     """Test DataladDataGrabber install errors / warnings.
 
@@ -95,6 +96,12 @@ def test_DataladDataGrabber_install_errors(
     with pytest.raises(ValueError, match=r"different ID"):
         with dg:
             pass
+    # Set config to skip id check and test
+    config.set(key="datagrabber.skipidcheck", val=True)
+    with dg:
+        pass
+    # Reset config
+    config.delete("datagrabber.skipidcheck")
 
     elem1_t1w = datadir / "example_bids/sub-01/anat/sub-01_T1w.nii.gz"
     elem1_t1w.unlink()
@@ -105,10 +112,18 @@ def test_DataladDataGrabber_install_errors(
     with pytest.warns(RuntimeWarning, match=r"one file is not clean"):
         with dg:
             pass
+    # Set config to skip dirty check and test
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        config.set(key="datagrabber.skipdirtycheck", val=True)
+        with dg:
+            pass
+        # Reset config
+        config.delete("datagrabber.skipdirtycheck")
 
 
 def test_DataladDataGrabber_clone_cleanup(
-    tmp_path: Path, concrete_datagrabber: Type
+    tmp_path: Path, concrete_datagrabber: type
 ) -> None:
     """Test DataladDataGrabber clone and remove.
 
@@ -157,7 +172,7 @@ def test_DataladDataGrabber_clone_cleanup(
 
 
 def test_DataladDataGrabber_clone_create_cleanup(
-    concrete_datagrabber: Type,
+    concrete_datagrabber: type,
 ) -> None:
     """Test DataladDataGrabber tempdir clone and remove.
 
@@ -203,7 +218,7 @@ def test_DataladDataGrabber_clone_create_cleanup(
 
 
 def test_DataladDataGrabber_previously_cloned(
-    tmp_path: Path, concrete_datagrabber: Type
+    tmp_path: Path, concrete_datagrabber: type
 ) -> None:
     """Test DataladDataGrabber on cloned dataset.
 
@@ -247,7 +262,7 @@ def test_DataladDataGrabber_previously_cloned(
         meta = elem1["BOLD"]["meta"]
         assert "datagrabber" in meta
         assert "datalad_dirty" in meta["datagrabber"]
-        assert meta["datagrabber"]["datalad_dirty"] is False
+        assert meta["datagrabber"]["datalad_dirty"] is True
         assert "datalad_commit_id" in meta["datagrabber"]
         assert meta["datagrabber"]["datalad_commit_id"] == commit
         assert "datalad_id" in meta["datagrabber"]
@@ -272,7 +287,7 @@ def test_DataladDataGrabber_previously_cloned(
 
 
 def test_DataladDataGrabber_previously_cloned_and_get(
-    tmp_path: Path, concrete_datagrabber: Type
+    tmp_path: Path, concrete_datagrabber: type
 ) -> None:
     """Test DataladDataGrabber on cloned dataset with files present.
 
@@ -327,7 +342,7 @@ def test_DataladDataGrabber_previously_cloned_and_get(
         meta = elem1["BOLD"]["meta"]
         assert "datagrabber" in meta
         assert "datalad_dirty" in meta["datagrabber"]
-        assert meta["datagrabber"]["datalad_dirty"] is False
+        assert meta["datagrabber"]["datalad_dirty"] is True
         assert "datalad_commit_id" in meta["datagrabber"]
         assert meta["datagrabber"]["datalad_commit_id"] == commit
         assert "datalad_id" in meta["datagrabber"]
@@ -355,7 +370,7 @@ def test_DataladDataGrabber_previously_cloned_and_get(
 
 
 def test_DataladDataGrabber_previously_cloned_and_get_dirty(
-    tmp_path: Path, concrete_datagrabber: Type
+    tmp_path: Path, concrete_datagrabber: type
 ) -> None:
     """Test DataladDataGrabber on a dirty cloned dataset.
 

junifer/datagrabber/tests/test_dmcc13_benchmark.py

@@ -3,7 +3,7 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import pytest
 
@@ -116,7 +116,12 @@ def test_DMCC13Benchmark(
             data_file_names.extend(
                 [
                     "sub-01_desc-preproc_T1w.nii.gz",
-                    "sub-01_from-MNI152NLin2009cAsym_to-T1w_mode-image_xfm.h5",
+                    [
+                        "sub-01_from-MNI152NLin2009cAsym_to-T1w"
+                        "_mode-image_xfm.h5",
+                        "sub-01_from-T1w_to-MNI152NLin2009cAsym"
+                        "_mode-image_xfm.h5",
+                    ],
                 ]
             )
         else:
@@ -127,14 +132,26 @@ def test_DMCC13Benchmark(
         for data_type, data_file_name in zip(data_types, data_file_names):
             # Assert data type
             assert data_type in out
-            # Assert data file path exists
-            assert out[data_type]["path"].exists()
-            # Assert data file path is a file
-            assert out[data_type]["path"].is_file()
-            # Assert data file name
-            assert out[data_type]["path"].name == data_file_name
-            # Assert metadata
-            assert "meta" in out[data_type]
+            # Conditional for Warp
+            if data_type == "Warp":
+                for idx, fname in enumerate(data_file_name):
+                    # Assert data file path exists
+                    assert out[data_type][idx]["path"].exists()
+                    # Assert data file path is a file
+                    assert out[data_type][idx]["path"].is_file()
+                    # Assert data file name
+                    assert out[data_type][idx]["path"].name == fname
+                    # Assert metadata
+                    assert "meta" in out[data_type][idx]
+            else:
+                # Assert data file path exists
+                assert out[data_type]["path"].exists()
+                # Assert data file path is a file
+                assert out[data_type]["path"].is_file()
+                # Assert data file name
+                assert out[data_type]["path"].name == data_file_name
+                # Assert metadata
+                assert "meta" in out[data_type]
 
         # Check BOLD nested data types
         for type_, file_name in zip(
@@ -201,7 +218,7 @@ def test_DMCC13Benchmark(
     ],
 )
 def test_DMCC13Benchmark_partial_data_access(
-    types: Union[str, List[str]],
+    types: Union[str, list[str]],
     native_t1w: bool,
 ) -> None:
     """Test DMCC13Benchmark DataGrabber partial data access.