junifer 0.0.4.dev11__tar.gz → 0.0.4.dev25__tar.gz

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: junifer
-Version: 0.0.4.dev11
+Version: 0.0.4.dev25
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.4.dev25/docs/changes/newsfragments/248.change

@@ -0,0 +1 @@
+Add ``feature_md5`` argument to :func:`.read_transform()` by `Synchon Mandal`_

junifer-0.0.4.dev25/junifer/_version.py

@@ -0,0 +1,4 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+__version__ = version = '0.0.4.dev25'
+__version_tuple__ = version_tuple = (0, 0, 4, 'dev25')

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer/api/cli.py

@@ -8,7 +8,7 @@ import pathlib
 import subprocess
 import sys
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Dict, List, Tuple, Union
 
 import click
 
@@ -32,27 +32,48 @@ from .utils import (
 )
 
 
-def _parse_elements(element: str, config: Dict) -> Union[List, None]:
+def _parse_elements(element: Tuple[str], config: Dict) -> Union[List, None]:
     """Parse elements from cli.
 
     Parameters
     ----------
-    element : str
-        The element to operate on.
+    element : tuple of str
+        The element(s) to operate on.
     config : dict
         The configuration to operate using.
 
     Returns
     -------
-    list
-        The element(s) as list.
+    list or None
+        The element(s) as list or None.
+
+    Raises
+    ------
+    ValueError
+        If no element is found either in the command-line options or
+        the configuration file.
+
+    Warns
+    -----
+    RuntimeWarning
+        If elements are specified both via the command-line options and
+        the configuration file.
 
     """
     logger.debug(f"Parsing elements: {element}")
+    # Early return None to continue with all elements
     if len(element) == 0:
         return None
-    # TODO: If len == 1, check if its a file, then parse elements from file
-    elements = [tuple(x.split(",")) if "," in x else x for x in element]
+    # Check if the element is a file for single element;
+    # if yes, then parse elements from it
+    if len(element) == 1 and Path(element[0]).resolve().is_file():
+        fetched_element = _parse_elements_file(Path(element[0]).resolve())
+    else:
+        fetched_element = element
+    # Process multi-keyed elements
+    elements = [
+        tuple(x.split(",")) if "," in x else x for x in fetched_element
+    ]
     logger.debug(f"Parsed elements: {elements}")
     if elements is not None and "elements" in config:
         warn_with_log(
@@ -61,19 +82,41 @@ def _parse_elements(element: str, config: Dict) -> Union[List, None]:
             "over the configuration file. That is, the elements specified "
             "in the command line will be used. The elements specified in "
             "the configuration file will be ignored. To remove this warning, "
-            'please remove the "elements" item from the configuration file.'
+            "please remove the `elements` item from the configuration file."
         )
     elif elements is None:
+        # Check in config
         elements = config.get("elements", None)
         if elements is None:
             raise_error(
-                "The 'elements' key is set in the configuration, but its value"
-                " is 'None'. It is likely that there is an empty 'elements' "
+                "The `elements` key is set in the configuration, but its value"
+                " is `None`. It is likely that there is an empty `elements` "
                 "section in the yaml configuration file."
             )
     return elements
 
 
+def _parse_elements_file(filepath: Path) -> List[str]:
+    """Parse elements from file.
+
+    Parameters
+    ----------
+    filepath : pathlib.Path
+        The path to the element file.
+
+    Returns
+    -------
+    list of str
+        The element(s) as list.
+
+    """
+    # Read file
+    with open(filepath) as file:
+        raw_elements = file.readlines()
+    # Strip whitespace and return
+    return [element.strip() for element in raw_elements]
+
+
 def _validate_verbose(
     ctx: click.Context, param: str, value: str
 ) -> Union[str, int]:
@@ -133,7 +176,9 @@ def cli() -> None:  # pragma: no cover
     callback=_validate_verbose,
     default="info",
 )
-def run(filepath: click.Path, element: str, verbose: Union[str, int]) -> None:
+def run(
+    filepath: click.Path, element: Tuple[str], verbose: Union[str, int]
+) -> None:
     """Run command for CLI.
 
     \f
@@ -142,8 +187,8 @@ def run(filepath: click.Path, element: str, verbose: Union[str, int]) -> None:
     ----------
     filepath : click.Path
         The filepath to the configuration file.
-    element : str
-        The element to operate using.
+    element : tuple of str
+        The element to operate on.
     verbose : click.Choice
         The verbosity level: warning, info or debug (default "info").
 

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer/api/functions.py

@@ -163,7 +163,9 @@ def run(
     # Fit elements
     with datagrabber_object:
         if elements is not None:
-            for t_element in elements:
+            for t_element in datagrabber_object.filter(
+                elements  # type: ignore
+            ):
                 mc.fit(datagrabber_object[t_element])
         else:
             for t_element in datagrabber_object:

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer/api/tests/test_cli.py

@@ -35,7 +35,16 @@ runner = CliRunner()
 def test_run_and_collect_commands(
     tmp_path: Path, elements: Tuple[str, ...]
 ) -> None:
-    """Test run and collect commands."""
+    """Test run and collect commands.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    elements : tuple of str
+        The elements to operate on.
+
+    """
     # Get test config
     infile = Path(__file__).parent / "data" / "gmd_mean.yaml"
     # Read test config
@@ -74,6 +83,57 @@ def test_run_and_collect_commands(
     assert collect_result.exit_code == 0
 
 
+@pytest.mark.parametrize(
+    "elements",
+    [
+        "sub-01",
+        "sub-01\nsub-02",
+    ],
+)
+def test_run_using_element_file(tmp_path: Path, elements: str) -> None:
+    """Test run command using element file.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    elements : str
+        The elements to write to the element file.
+
+    """
+    # Create test file
+    test_file_path = tmp_path / "elements.txt"
+    with open(test_file_path, "w") as f:
+        f.write(elements)
+
+    # Get test config
+    infile = Path(__file__).parent / "data" / "gmd_mean.yaml"
+    # Read test config
+    contents = yaml.load(infile)
+    # Working directory
+    workdir = tmp_path / "workdir"
+    contents["workdir"] = str(workdir.resolve())
+    # Output directory
+    outdir = tmp_path / "outdir"
+    # Storage
+    contents["storage"]["uri"] = str(outdir.resolve())
+    # Write new test config
+    outfile = tmp_path / "in.yaml"
+    yaml.dump(contents, stream=outfile)
+    # Run command arguments
+    run_args = [
+        str(outfile.absolute()),
+        "--verbose",
+        "debug",
+        "--element",
+        str(test_file_path.resolve()),
+    ]
+    # Invoke run command
+    run_result = runner.invoke(run, run_args)
+    # Check
+    assert run_result.exit_code == 0
+
+
 def test_wtf_short() -> None:
     """Test short version of wtf command."""
     # Invoke wtf command

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer/datagrabber/base.py

@@ -58,12 +58,12 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         """
         yield from self.get_elements()
 
-    def __getitem__(self, element: Union[str, Tuple]) -> Dict[str, Dict]:
+    def __getitem__(self, element: Union[str, Tuple[str]]) -> Dict[str, Dict]:
         """Enable indexing support.
 
         Parameters
         ----------
-        element : str or tuple
+        element : str or tuple of str
             The element to be indexed.
 
         Returns
@@ -117,6 +117,51 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         """
         return self._datadir
 
+    def filter(self, selection: List[Union[str, Tuple[str]]]) -> Iterator:
+        """Filter elements to be grabbed.
+
+        Parameters
+        ----------
+        selection : list of str or tuple
+            The list of partial element key values to filter using.
+
+        Yields
+        ------
+        object
+            An element that can be indexed by the DataGrabber.
+
+        """
+
+        def filter_func(element: Union[str, Tuple[str]]) -> bool:
+            """Filter element based on selection.
+
+            Parameters
+            ----------
+            element : str or tuple of str
+                The element to be filtered.
+
+            Returns
+            -------
+            bool
+                If the element passes the filter or not.
+
+            """
+            # Convert element to tuple
+            if not isinstance(element, tuple):
+                element = (element,)
+            # Filter based on selection kind
+            if isinstance(selection[0], str):
+                for opt in selection:
+                    if opt in element:
+                        return True
+            elif isinstance(selection[0], tuple):
+                for opt in selection:
+                    if set(opt).issubset(element):
+                        return True
+            return False
+
+        yield from filter(filter_func, self.get_elements())
+
     @abstractmethod
     def get_element_keys(self) -> List[str]:
         """Get element keys.
@@ -136,7 +181,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         )
 
     @abstractmethod
-    def get_elements(self) -> List:
+    def get_elements(self) -> List[Union[str, Tuple[str]]]:
         """Get elements.
 
         Returns

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer/datagrabber/tests/test_base.py

@@ -67,3 +67,60 @@ def test_BaseDataGrabber() -> None:
 
     with pytest.raises(NotImplementedError):
         dg.get_item(subject=1)  # type: ignore
+
+
+def test_BaseDataGrabber_filter_single() -> None:
+    """Test single-keyed element filter for BaseDataGrabber."""
+
+    # Create concrete class
+    class FilterDataGrabber(BaseDataGrabber):
+        def get_item(self, subject):
+            return {"BOLD": {}}
+
+        def get_elements(self):
+            return ["sub01", "sub02", "sub03"]
+
+        def get_element_keys(self):
+            return ["subject"]
+
+    dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"])
+    with dg:
+        assert "sub01" in list(dg.filter(["sub01"]))
+        assert "sub02" not in list(dg.filter(["sub01"]))
+
+
+def test_BaseDataGrabber_filter_multi() -> None:
+    """Test multi-keyed element filter for BaseDataGrabber."""
+
+    # Create concrete class
+    class FilterDataGrabber(BaseDataGrabber):
+        def get_item(self, subject):
+            return {"BOLD": {}}
+
+        def get_elements(self):
+            return [
+                ("sub01", "rest"),
+                ("sub01", "movie"),
+                ("sub02", "rest"),
+                ("sub02", "movie"),
+                ("sub03", "rest"),
+                ("sub03", "movie"),
+            ]
+
+        def get_element_keys(self):
+            return ["subject", "task"]
+
+    dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"])
+    with dg:
+        assert ("sub01", "rest") in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )
+        assert ("sub01", "movie") not in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )
+        assert ("sub02", "rest") not in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )
+        assert ("sub02", "movie") not in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer/onthefly/read_transform.py

@@ -17,8 +17,9 @@ if TYPE_CHECKING:
 
 def read_transform(
     storage: Type["BaseFeatureStorage"],
-    feature_name: str,
     transform: str,
+    feature_name: Optional[str] = None,
+    feature_md5: Optional[str] = None,
     transform_args: Optional[Tuple] = None,
     transform_kw_args: Optional[Dict] = None,
 ) -> pd.DataFrame:
@@ -28,11 +29,13 @@ def read_transform(
     ----------
     storage : storage-like
         The storage class, for example, SQLiteFeatureStorage.
-    feature_name : str
-        Name of the feature to read.
     transform : str
         The kind of transform formatted as ``<package>_<function>``,
         for example, ``bctpy_degrees_und``.
+    feature_name : str, optional
+        Name of the feature to read (default None).
+    feature_md5 : str, optional
+        MD5 hash of the feature to read (default None).
     transform_args : tuple, optional
         The positional arguments for the callable of ``transform``
         (default None).
@@ -62,7 +65,9 @@ def read_transform(
     transform_kw_args = transform_kw_args or {}
 
     # Read storage
-    stored_data = storage.read(feature_name=feature_name)  # type: ignore
+    stored_data = storage.read(
+        feature_name=feature_name, feature_md5=feature_md5
+    )  # type: ignore
     # Retrieve package and function
     package, func_str = transform.split("_", 1)
     # Condition for package
@@ -105,7 +110,8 @@ def read_transform(
         # Apply function and store subject-wise
         output_list = []
         logger.debug(
-            f"Computing '{package}.{func_str}' for feature {feature_name} ..."
+            f"Computing '{package}.{func_str}' for feature "
+            f"{feature_name or feature_md5} ..."
         )
         for subject in range(stored_data["data"].shape[2]):
             output = func(
@@ -119,7 +125,8 @@ def read_transform(
         idx_df = pd.DataFrame(data=stored_data["element"])
         # Create multiindex from dataframe
         logger.debug(
-            f"Generating pandas.MultiIndex for feature {feature_name} ..."
+            "Generating pandas.MultiIndex for feature "
+            f"{feature_name or feature_md5} ..."
         )
         data_idx = pd.MultiIndex.from_frame(df=idx_df)
 

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: junifer
-Version: 0.0.4.dev11
+Version: 0.0.4.dev25
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

{junifer-0.0.4.dev11 → junifer-0.0.4.dev25}/junifer.egg-info/SOURCES.txt

@@ -58,6 +58,7 @@ docs/changes/contributors.inc
 docs/changes/newsfragments/.gitignore
 docs/changes/newsfragments/233.bugfix
 docs/changes/newsfragments/247.doc
+docs/changes/newsfragments/248.change
 docs/extending/coordinates.rst
 docs/extending/datagrabber.rst
 docs/extending/extension.rst

junifer-0.0.4.dev11/junifer/_version.py

@@ -1,4 +0,0 @@
-# file generated by setuptools_scm
-# don't change, don't track in version control
-__version__ = version = '0.0.4.dev11'
-__version_tuple__ = version_tuple = (0, 0, 4, 'dev11')