junifer 0.0.6.dev266__tar.gz → 0.0.6.dev285__tar.gz

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: junifer
-Version: 0.0.6.dev266
+Version: 0.0.6.dev285
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.6.dev285/docs/changes/newsfragments/401.feature

@@ -0,0 +1 @@
+Introduce :class:`junifer.utils.ConfigManager` singleton class to manage global configuration by `Fede Raimondo`_

junifer-0.0.6.dev285/docs/changes/newsfragments/403.enh

@@ -0,0 +1 @@
+Use :func:`junifer.utils.ConfigManager` to skip time-consuming dataset ID and dirty checks for :class:`.DataladDataGrabber` by `Fede Raimondo`_

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/docs/conf.py

@@ -96,6 +96,7 @@ nitpick_ignore_regex = [
     ("py:obj", "sqlalchemy.engine.Engine"),  # ignore sqlalchemy
     ("py:class", "pipeline.Pipeline"),  # nilearn
     ("py:obj", "neurokit2.*"),  # ignore neurokit2
+    ("py:obj", "datalad.*"),  # ignore datalad
 ]
 
 # -- Options for HTML output -------------------------------------------------

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.6.dev266'
-__version_tuple__ = version_tuple = (0, 0, 6, 'dev266')
+__version__ = version = '0.0.6.dev285'
+__version_tuple__ = version_tuple = (0, 0, 6, 'dev285')

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/api/functions.py

@@ -24,7 +24,7 @@ from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike
 from ..utils import logger, raise_error, yaml
 
 
-__all__ = ["run", "collect", "queue", "reset", "list_elements"]
+__all__ = ["collect", "list_elements", "queue", "reset", "run"]
 
 
 def _get_datagrabber(datagrabber_config: dict) -> DataGrabberLike:

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/cli/parser.py

@@ -15,7 +15,7 @@ import pandas as pd
 from ..utils import logger, raise_error, warn_with_log, yaml
 
 
-__all__ = ["parse_yaml", "parse_elements"]
+__all__ = ["parse_elements", "parse_yaml"]
 
 
 def parse_yaml(filepath: Union[str, Path]) -> dict:  # noqa: C901

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/data/masks/_masks.py

@@ -35,7 +35,7 @@ if TYPE_CHECKING:
     from nibabel.nifti1 import Nifti1Image
 
 
-__all__ = ["compute_brain_mask", "MaskRegistry"]
+__all__ = ["MaskRegistry", "compute_brain_mask"]
 
 
 # Path to the masks

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/data/template_spaces.py

@@ -16,7 +16,7 @@ from ..utils import logger, raise_error
 from .utils import closest_resolution
 
 
-__all__ = ["get_xfm", "get_template"]
+__all__ = ["get_template", "get_xfm"]
 
 
 def get_xfm(

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/datagrabber/datalad_base.py

@@ -17,7 +17,7 @@ from datalad.support.exceptions import IncompleteResultsError
 from datalad.support.gitrepo import GitRepo
 
 from ..pipeline import WorkDirManager
-from ..utils import logger, raise_error, warn_with_log
+from ..utils import config, logger, raise_error, warn_with_log
 from .base import BaseDataGrabber
 
 
@@ -143,27 +143,49 @@ class DataladDataGrabber(BaseDataGrabber):
         """
         return super().datadir / self._rootdir
 
-    def _get_dataset_id_remote(self) -> str:
+    def _get_dataset_id_remote(self) -> tuple[str, bool]:
         """Get the dataset ID from the remote.
 
         Returns
         -------
         str
             The dataset ID.
+        bool
+            Whether the dataset is dirty.
+
+        Raises
+        ------
+        ValueError
+            If the dataset ID cannot be obtained from the remote.
 
         """
         remote_id = None
+        is_dirty = False
         with tempfile.TemporaryDirectory() as tmpdir:
-            logger.debug(f"Querying {self.uri} for dataset ID")
-            repo = GitRepo.clone(
-                self.uri, path=tmpdir, clone_options=["-n", "--depth=1"]
+            if not config.get("datagrabber.skipidcheck", False):
+                logger.debug(f"Querying {self.uri} for dataset ID")
+                repo = GitRepo.clone(
+                    self.uri, path=tmpdir, clone_options=["-n", "--depth=1"]
+                )
+                repo.checkout(name=".datalad/config", options=["HEAD"])
+                remote_id = repo.config.get("datalad.dataset.id", None)
+                logger.debug(f"Got remote dataset ID = {remote_id}")
+
+                if not config.get("datagrabber.skipdirtycheck", False):
+                    is_dirty = repo.dirty
+                else:
+                    logger.debug("Skipping dirty check")
+                    is_dirty = False
+            else:
+                logger.debug("Skipping dataset ID check")
+                remote_id = self._dataset.id
+                is_dirty = False
+            logger.debug(
+                f"Remote dataset is {'' if is_dirty else 'not'} dirty"
             )
-            repo.checkout(name=".datalad/config", options=["HEAD"])
-            remote_id = repo.config.get("datalad.dataset.id", None)
-            logger.debug(f"Got remote dataset ID = {remote_id}")
         if remote_id is None:
             raise_error("Could not get dataset ID from remote")
-        return remote_id
+        return remote_id, is_dirty
 
     def _dataset_get(self, out: dict) -> dict:
         """Get the dataset found from the path in ``out``.
@@ -178,6 +200,11 @@ class DataladDataGrabber(BaseDataGrabber):
         dict
             The unmodified input dictionary.
 
+        Raises
+        ------
+        datalad.support.exceptions.IncompleteResultsError
+            If there is a datalad-related problem while fetching data.
+
         """
         to_get = []
         for type_val in out.values():
@@ -230,6 +257,8 @@ class DataladDataGrabber(BaseDataGrabber):
         ------
         ValueError
             If the dataset is already installed but with a different ID.
+        datalad.support.exceptions.IncompleteResultsError
+            If there is a datalad-related problem while cloning dataset.
 
         """
         isinstalled = dl.Dataset(self._datadir).is_installed()
@@ -238,23 +267,23 @@ class DataladDataGrabber(BaseDataGrabber):
             self._got_files = []
             self._dataset: dl.Dataset = dl.Dataset(self._datadir)
 
-            remote_id = self._get_dataset_id_remote()
+            # Check if dataset is already installed with a different ID
+            remote_id, is_dirty = self._get_dataset_id_remote()
             if remote_id != self._dataset.id:
                 raise_error(
                     "Dataset already installed but with a different "
                     f"ID: {self._dataset.id} (local) != {remote_id} (remote)"
                 )
 
-            # Check for dirty datasets:
-            status = self._dataset.status()
-            if any(x["state"] != "clean" for x in status):
-                self.datalad_dirty = True
+            # Conditional reporting on dataset dirtiness
+            self.datalad_dirty = is_dirty
+            if self.datalad_dirty:
                 warn_with_log(
-                    "At least one file is not clean, Junifer will "
-                    "consider this dataset as dirty."
+                    "At least one file is not clean, "
+                    f"marking dataset (id: {self._dataset.id}) as dirty."
                 )
             else:
-                logger.debug("Dataset is clean")
+                logger.debug(f"Dataset (id: {self._dataset.id}) is clean")
 
         else:
             logger.debug(f"Installing dataset {self.uri} to {self._datadir}")

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/datagrabber/tests/test_datalad_base.py

@@ -3,12 +3,14 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import warnings
 from pathlib import Path
 
 import datalad.api as dl
 import pytest
 
 from junifer.datagrabber import DataladDataGrabber
+from junifer.utils import config
 
 
 _testing_dataset = {
@@ -94,6 +96,12 @@ def test_DataladDataGrabber_install_errors(
     with pytest.raises(ValueError, match=r"different ID"):
         with dg:
             pass
+    # Set config to skip id check and test
+    config.set(key="datagrabber.skipidcheck", val=True)
+    with dg:
+        pass
+    # Reset config
+    config.delete("datagrabber.skipidcheck")
 
     elem1_t1w = datadir / "example_bids/sub-01/anat/sub-01_T1w.nii.gz"
     elem1_t1w.unlink()
@@ -104,6 +112,14 @@ def test_DataladDataGrabber_install_errors(
     with pytest.warns(RuntimeWarning, match=r"one file is not clean"):
         with dg:
             pass
+    # Set config to skip dirty check and test
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        config.set(key="datagrabber.skipdirtycheck", val=True)
+        with dg:
+            pass
+        # Reset config
+        config.delete("datagrabber.skipdirtycheck")
 
 
 def test_DataladDataGrabber_clone_cleanup(
@@ -246,7 +262,7 @@ def test_DataladDataGrabber_previously_cloned(
         meta = elem1["BOLD"]["meta"]
         assert "datagrabber" in meta
         assert "datalad_dirty" in meta["datagrabber"]
-        assert meta["datagrabber"]["datalad_dirty"] is False
+        assert meta["datagrabber"]["datalad_dirty"] is True
         assert "datalad_commit_id" in meta["datagrabber"]
         assert meta["datagrabber"]["datalad_commit_id"] == commit
         assert "datalad_id" in meta["datagrabber"]
@@ -326,7 +342,7 @@ def test_DataladDataGrabber_previously_cloned_and_get(
         meta = elem1["BOLD"]["meta"]
         assert "datagrabber" in meta
         assert "datalad_dirty" in meta["datagrabber"]
-        assert meta["datagrabber"]["datalad_dirty"] is False
+        assert meta["datagrabber"]["datalad_dirty"] is True
         assert "datalad_commit_id" in meta["datagrabber"]
         assert meta["datagrabber"]["datalad_commit_id"] == commit
         assert "datalad_id" in meta["datagrabber"]

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/stats.py

@@ -13,7 +13,7 @@ from scipy.stats.mstats import winsorize
 from .utils import logger, raise_error
 
 
-__all__ = ["get_aggfunc_by_name", "count", "winsorized_mean", "select"]
+__all__ = ["count", "get_aggfunc_by_name", "select", "winsorized_mean"]
 
 
 def get_aggfunc_by_name(

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/typing/__init__.pyi

@@ -9,6 +9,7 @@ __all__ = [
     "ExternalDependencies",
     "MarkerInOutMappings",
     "DataGrabberPatterns",
+    "ConfigVal",
 ]
 
 from ._typing import (
@@ -22,4 +23,5 @@ from ._typing import (
     ExternalDependencies,
     MarkerInOutMappings,
     DataGrabberPatterns,
+    ConfigVal,
 )

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/typing/_typing.py

@@ -19,16 +19,17 @@ if TYPE_CHECKING:
 
 
 __all__ = [
+    "ConditionalDependencies",
+    "ConfigVal",
     "DataGrabberLike",
-    "PreprocessorLike",
-    "MarkerLike",
-    "StorageLike",
-    "PipelineComponent",
+    "DataGrabberPatterns",
     "Dependencies",
-    "ConditionalDependencies",
     "ExternalDependencies",
     "MarkerInOutMappings",
-    "DataGrabberPatterns",
+    "MarkerLike",
+    "PipelineComponent",
+    "PreprocessorLike",
+    "StorageLike",
 ]
 
 
@@ -60,3 +61,4 @@ MarkerInOutMappings = MutableMapping[str, MutableMapping[str, str]]
 DataGrabberPatterns = dict[
     str, Union[dict[str, str], Sequence[dict[str, str]]]
 ]
+ConfigVal = Union[bool, int, float]

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/utils/__init__.pyi

@@ -1,15 +1,18 @@
 __all__ = [
     "make_executable",
     "configure_logging",
+    "config",
     "logger",
     "raise_error",
     "warn_with_log",
     "run_ext_cmd",
     "deep_update",
     "yaml",
+    "ConfigManager",
 ]
 
 from .fs import make_executable
 from .logging import configure_logging, logger, raise_error, warn_with_log
+from ._config import config, ConfigManager
 from .helpers import run_ext_cmd, deep_update
 from ._yaml import yaml

junifer-0.0.6.dev285/junifer/utils/_config.py

@@ -0,0 +1,110 @@
+"""Provide junifer global configuration."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import os
+from typing import Optional
+
+from ..typing import ConfigVal
+from .logging import logger
+from .singleton import Singleton
+
+
+__all__ = ["ConfigManager", "config"]
+
+
+class ConfigManager(metaclass=Singleton):
+    """Manage configuration parameters.
+
+    Attributes
+    ----------
+    _config : dict
+        Configuration parameters.
+
+    """
+
+    def __init__(self) -> None:
+        """Initialize the class."""
+        self._config = {}
+        # Initial setup from process env
+        self._reload()
+
+    def _reload(self) -> None:
+        """Reload env vars."""
+        for t_var in os.environ:
+            if t_var.startswith("JUNIFER_"):
+                # Set correct type
+                var_value = os.environ[t_var]
+                # bool
+                if var_value.lower() == "true":
+                    var_value = True
+                elif var_value.lower() == "false":
+                    var_value = False
+                # numeric
+                else:
+                    try:
+                        var_value = int(var_value)
+                    except ValueError:
+                        try:
+                            var_value = float(var_value)
+                        except ValueError:
+                            pass
+                # Set value
+                var_name = (
+                    t_var.replace("JUNIFER_", "").lower().replace("_", ".")
+                )
+                logger.debug(
+                    f"Setting `{var_name}` from environment to "
+                    f"`{var_value}` (type: {type(var_value)})"
+                )
+                self._config[var_name] = var_value
+
+    def get(self, key: str, default: Optional[ConfigVal] = None) -> ConfigVal:
+        """Get configuration parameter.
+
+        Parameters
+        ----------
+        key : str
+            The configuration key to get.
+        default : bool or int or float or None, optional
+            The default value to return if the key is not found (default None).
+
+        Returns
+        -------
+        bool or int or float
+            The configuration value.
+
+        """
+        return self._config.get(key, default)
+
+    def set(self, key: str, val: ConfigVal) -> None:
+        """Set configuration parameter.
+
+        Parameters
+        ----------
+        key : str
+            The configuration key to set.
+        val : bool or int or float
+            The value to set ``key`` to.
+
+        """
+        logger.debug(f"Setting `{key}` to `{val}` (type: {type(val)})")
+        self._config[key] = val
+
+    def delete(self, key: str) -> None:
+        """Delete configuration parameter.
+
+        Parameters
+        ----------
+        key : str
+            The configuration key to delete.
+
+        """
+        logger.debug(f"Deleting `{key}` from config")
+        _ = self._config.pop(key)
+
+
+# Initialize here to access from anywhere
+config = ConfigManager()

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/utils/helpers.py

@@ -10,7 +10,7 @@ import sys
 from .logging import logger, raise_error
 
 
-__all__ = ["run_ext_cmd", "deep_update"]
+__all__ = ["deep_update", "run_ext_cmd"]
 
 
 def run_ext_cmd(name: str, cmd: list[str]) -> None:

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer/utils/singleton.py

@@ -8,7 +8,7 @@ from abc import ABCMeta
 from typing import Any, ClassVar
 
 
-__all__ = ["Singleton", "ABCSingleton"]
+__all__ = ["ABCSingleton", "Singleton"]
 
 
 class Singleton(type):

junifer-0.0.6.dev285/junifer/utils/tests/test_config.py

@@ -0,0 +1,59 @@
+"""Provide tests for ConfigManager."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import os
+
+import pytest
+
+from junifer.typing import ConfigVal
+from junifer.utils import config
+from junifer.utils._config import ConfigManager
+
+
+def test_config_manager_singleton() -> None:
+    """Test that ConfigManager is a singleton."""
+    config_mgr_1 = ConfigManager()
+    config_mgr_2 = ConfigManager()
+    assert id(config_mgr_1) == id(config_mgr_2)
+
+
+def test_config_manager() -> None:
+    """Test config operations for ConfigManager."""
+    # Get non-existing with default
+    assert config.get(key="scooby") is None
+    # Set
+    config.set(key="scooby", val=True)
+    # Get existing
+    assert config.get("scooby")
+    # Delete
+    config.delete("scooby")
+    # Get non-existing with default
+    assert config.get(key="scooby") is None
+
+
+@pytest.mark.parametrize(
+    "val, expected_val",
+    [("TRUE", True), ("FALSE", False), ("1", 1), ("0.0", 0.0)],
+)
+def test_config_manager_env_reload(val: str, expected_val: ConfigVal) -> None:
+    """Test config parsing from env reload.
+
+    Parameters
+    ----------
+    val : str
+        The parametrized values.
+    expected_val : bool or int or float
+        The parametrized expected value.
+
+    """
+    # Set env var
+    os.environ["JUNIFER_TESTME"] = val
+    # Check
+    config._reload()
+    assert config.get("testme") == expected_val
+    # Cleanup
+    del os.environ["JUNIFER_TESTME"]
+    config.delete("testme")

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: junifer
-Version: 0.0.6.dev266
+Version: 0.0.6.dev285
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

{junifer-0.0.6.dev266 → junifer-0.0.6.dev285}/junifer.egg-info/SOURCES.txt

@@ -95,6 +95,8 @@ docs/changes/newsfragments/391.misc
 docs/changes/newsfragments/392.feature
 docs/changes/newsfragments/393.misc
 docs/changes/newsfragments/398.bugfix
+docs/changes/newsfragments/401.feature
+docs/changes/newsfragments/403.enh
 docs/extending/coordinates.rst
 docs/extending/datagrabber.rst
 docs/extending/dependencies.rst
@@ -468,12 +470,14 @@ junifer/typing/__init__.pyi
 junifer/typing/_typing.py
 junifer/utils/__init__.py
 junifer/utils/__init__.pyi
+junifer/utils/_config.py
 junifer/utils/_yaml.py
 junifer/utils/fs.py
 junifer/utils/helpers.py
 junifer/utils/logging.py
 junifer/utils/py.typed
 junifer/utils/singleton.py
+junifer/utils/tests/test_config.py
 junifer/utils/tests/test_fs.py
 junifer/utils/tests/test_helpers.py
 junifer/utils/tests/test_logging.py