nimare 0.4.2__py3-none-any.whl

nimare/extract/utils.py

@@ -0,0 +1,295 @@
+"""Utility functions for the extract module."""
+
+from __future__ import division
+
+import logging
+import os
+import os.path as op
+
+import numpy as np
+import pandas as pd
+import requests
+from fuzzywuzzy import fuzz
+
+from nimare.utils import _uk_to_us
+
+LGR = logging.getLogger(__name__)
+
+
+def get_data_dirs(data_dir=None):
+    """Return the directories in which NiMARE looks for data.
+
+    .. versionadded:: 0.0.2
+
+    This is typically useful for the end-user to check where the data is
+    downloaded and stored.
+
+    Parameters
+    ----------
+    data_dir: :obj:`pathlib.Path` or :obj:`str`, optional
+        Path of the data directory. Used to force data storage in a specified
+        location. Default: None
+
+    Returns
+    -------
+    paths : :obj:`list` of :obj:`str`
+        Paths of the dataset directories.
+
+    Notes
+    -----
+    Taken from Nilearn.
+    This function retrieves the datasets directories using the following
+    priority :
+
+    1. defaults system paths
+    2. the keyword argument data_dir
+    3. the global environment variable NIMARE_SHARED_DATA
+    4. the user environment variable NIMARE_DATA
+    5. nimare_data in the user home folder
+    """
+    # We build an array of successive paths by priority
+    # The boolean indicates if it is a pre_dir: in that case, we won't add the
+    # dataset name to the path.
+    paths = []
+
+    # Check data_dir which force storage in a specific location
+    if data_dir is not None:
+        paths.extend(str(data_dir).split(os.pathsep))
+
+    # If data_dir has not been specified, then we crawl default locations
+    if data_dir is None:
+        global_data = os.getenv("NIMARE_SHARED_DATA")
+        if global_data is not None:
+            paths.extend(global_data.split(os.pathsep))
+
+        local_data = os.getenv("NIMARE_DATA")
+        if local_data is not None:
+            paths.extend(local_data.split(os.pathsep))
+
+        paths.append(os.path.expanduser("~/.nimare"))
+    return paths
+
+
+def _get_dataset_dir(dataset_name, data_dir=None, default_paths=None):
+    """Create if necessary and returns data directory of given dataset.
+
+    .. versionadded:: 0.0.2
+
+    Parameters
+    ----------
+    dataset_name : :obj:`str`
+        The unique name of the dataset.
+    data_dir : :obj:`pathlib.Path` or :obj:`str`, optional
+        Path of the data directory. Used to force data storage in a specified
+        location. Default: None
+    default_paths : :obj:`list` of :obj:`str`, optional
+        Default system paths in which the dataset may already have been
+        installed by a third party software. They will be checked first.
+
+    Returns
+    -------
+    data_dir : :obj:`str`
+        Path of the given dataset directory.
+
+    Notes
+    -----
+    Taken from Nilearn.
+    This function retrieves the datasets directory (or data directory) using
+    the following priority :
+    1. defaults system paths
+    2. the keyword argument data_dir
+    3. the global environment variable NIMARE_SHARED_DATA
+    4. the user environment variable NIMARE_DATA
+    5. nimare_data in the user home folder
+    """
+    paths = []
+    # Search possible data-specific system paths
+    if default_paths is not None:
+        for default_path in default_paths:
+            paths.extend([(d, True) for d in str(default_path).split(os.pathsep)])
+
+    paths.extend([(d, False) for d in get_data_dirs(data_dir=data_dir)])
+
+    LGR.debug(f"Dataset search paths: {paths}")
+
+    # Check if the dataset exists somewhere
+    for path, is_pre_dir in paths:
+        if not is_pre_dir:
+            path = os.path.join(path, dataset_name)
+
+        if os.path.islink(path):
+            # Resolve path
+            path = readlinkabs(path)
+
+        if os.path.exists(path) and os.path.isdir(path):
+            LGR.info(f"Dataset found in {path}\n")
+            return path
+
+    # If not, create a folder in the first writeable directory
+    errors = []
+    for path, is_pre_dir in paths:
+        if not is_pre_dir:
+            path = os.path.join(path, dataset_name)
+
+        if not os.path.exists(path):
+            try:
+                os.makedirs(path)
+                LGR.info(f"Dataset created in {path}")
+                return path
+            except Exception as exc:
+                short_error_message = getattr(exc, "strerror", str(exc))
+                errors.append(f"\n -{path} ({short_error_message})")
+
+    raise OSError(
+        "NiMARE tried to store the dataset in the following directories, but: " + "".join(errors)
+    )
+
+
+def readlinkabs(link):
+    """Return an absolute path for the destination of a symlink.
+
+    .. versionadded:: 0.0.2
+
+    From nilearn.
+    """
+    path = os.readlink(link)
+    if os.path.isabs(path):
+        return path
+    return os.path.join(os.path.dirname(link), path)
+
+
+def _download_zipped_file(url, filename=None):
+    """Download from a URL to a file.
+
+    .. versionadded:: 0.0.2
+
+    """
+    if filename is None:
+        data_dir = op.abspath(op.getcwd())
+        filename = op.join(data_dir, url.split("/")[-1])
+    # NOTE the stream=True parameter
+    req = requests.get(url, stream=True)
+    with open(filename, "wb") as f_obj:
+        for chunk in req.iter_content(chunk_size=1024):
+            if chunk:  # filter out keep-alive new chunks
+                f_obj.write(chunk)
+    return filename
+
+
+def _longify(df):
+    """Expand comma-separated lists of aliases in DataFrame into separate rows.
+
+    .. versionadded:: 0.0.2
+
+    """
+    reduced = df[["id", "name", "alias"]]
+    rows = []
+    for index, row in reduced.iterrows():
+        if isinstance(row["alias"], str) and "," in row["alias"]:
+            aliases = row["alias"].split(", ") + [row["name"]]
+        else:
+            aliases = [row["name"]]
+
+        for alias in aliases:
+            rows.append([row["id"], row["name"].lower(), alias.lower()])
+    out_df = pd.DataFrame(columns=["id", "name", "alias"], data=rows)
+    out_df = out_df.replace("", np.nan)
+    return out_df
+
+
+def _get_ratio(tup):
+    """Get fuzzy ratio.
+
+    .. versionadded:: 0.0.2
+
+    """
+    if all(isinstance(t, str) for t in tup):
+        return fuzz.ratio(tup[0], tup[1])
+    else:
+        return 100
+
+
+def _gen_alt_forms(term):
+    """Generate a list of alternate forms for a given term.
+
+    .. versionadded:: 0.0.2
+
+    """
+    if not isinstance(term, str) or len(term) == 0:
+        return [None]
+
+    alt_forms = []
+    # For one alternate form, put contents of parentheses at beginning of term
+    if "(" in term:
+        prefix = term[term.find("(") + 1 : term.find(")")]
+        temp_term = term.replace(f"({prefix})", "").replace("  ", " ")
+        alt_forms.append(temp_term)
+        alt_forms.append(f"{prefix} {temp_term}")
+    else:
+        prefix = ""
+
+    # Remove extra spaces
+    alt_forms = [s.strip() for s in alt_forms]
+
+    # Allow plurals
+    # temp = [s+'s' for s in alt_forms]
+    # temp += [s+'es' for s in alt_forms]
+    # alt_forms += temp
+
+    # Remove words "task" and/or "paradigm"
+    alt_forms += [term.replace(" task", "") for term in alt_forms]
+    alt_forms += [term.replace(" paradigm", "") for term in alt_forms]
+
+    # Remove duplicates
+    alt_forms = list(set(alt_forms))
+    return alt_forms
+
+
+def _get_concept_reltype(relationship, direction):
+    """Convert two-part relationship info to more parsimonious representation.
+
+    .. versionadded:: 0.0.2
+
+    The two part representation includes relationship type and direction.
+    """
+    new_rel = None
+    if relationship == "PARTOF":
+        if direction == "child":
+            new_rel = "hasPart"
+        elif direction == "parent":
+            new_rel = "isPartOf"
+    elif relationship == "KINDOF":
+        if direction == "child":
+            new_rel = "hasKind"
+        elif direction == "parent":
+            new_rel = "isKindOf"
+    return new_rel
+
+
+def _expand_df(df):
+    """Add alternate forms to DataFrame, then sort DataFrame by alias length and similarity.
+
+    .. versionadded:: 0.0.2
+
+    Sorting by alias length is done for order of extraction from text. Sorting by similarity to
+    original name is done in order to select most appropriate term to associate with alias.
+    """
+    df = df.copy()
+    df["alias"] = df["alias"].apply(_uk_to_us)
+    new_rows = []
+    for index, row in df.iterrows():
+        alias = row["alias"]
+        alt_forms = _gen_alt_forms(alias)
+        for alt_form in alt_forms:
+            temp_row = row.copy()
+            temp_row["alias"] = alt_form
+            new_rows.append(temp_row.tolist())
+    alt_df = pd.DataFrame(columns=df.columns, data=new_rows)
+    df = pd.concat((df, alt_df), axis=0)
+    # Sort by name length and similarity of alternate form to preferred term
+    # For example, "task switching" the concept should take priority over the
+    # "task switching" version of the "task-switching" task.
+    df["length"] = df["alias"].str.len()
+    df["ratio"] = df[["alias", "name"]].apply(_get_ratio, axis=1)
+    df = df.sort_values(by=["length", "ratio"], ascending=[False, False])
+    return df

nimare/generate.py

@@ -0,0 +1,331 @@
+"""Utilities for generating data for testing."""
+
+from itertools import zip_longest
+
+import numpy as np
+import sparse
+
+from nimare.dataset import Dataset
+from nimare.io import convert_neurovault_to_dataset
+from nimare.meta.utils import compute_ale_ma, get_ale_kernel
+from nimare.transforms import ImageTransformer
+from nimare.utils import get_template, mm2vox, vox2mm
+
+# defaults for creating a neurovault dataset
+NEUROVAULT_IDS = (8836, 8838, 8893, 8895, 8892, 8891, 8962, 8894, 8956, 8854, 9000)
+CONTRAST_OF_INTEREST = {"animal": "as-Animal"}
+
+
+def create_coordinate_dataset(
+    foci=1,
+    foci_percentage="100%",
+    fwhm=10,
+    sample_size=30,
+    n_studies=30,
+    n_noise_foci=0,
+    seed=None,
+    space="MNI",
+):
+    """Generate coordinate based dataset for meta analysis.
+
+    .. versionadded:: 0.0.4
+
+    Parameters
+    ----------
+    foci : :obj:`int` or :obj:`list`
+        The number of foci to be generated per study or the
+        x,y,z coordinates of the ground truth foci. (Default=1)
+    foci_percentage : :obj:`float`
+        Percentage of studies where the foci appear. (Default="100%")
+    fwhm : :obj:`float`
+        Full width at half maximum (fwhm) to define the probability
+        spread of the foci. (Default=10)
+    sample_size : :obj:`int` or :obj:`list`
+        Either mean number of participants in each study
+        or a list specifying the sample size for each
+        study. If a list of two numbers and n_studies is
+        not two, then the first number will represent a lower
+        bound and the second number will represent an upper bound
+        of a uniform sample. (Default=30)
+    n_studies : :obj:`int`
+        Number of studies to generate. (Default=30)
+    n_noise_foci : :obj:`int`
+        Number of foci considered to be noise in each study. (Default=0)
+    seed : :obj:`int` or None
+        Random state to reproducibly initialize random numbers.
+        If seed is None, then the random state will try to be initialized
+        with data from /dev/urandom (or the Windows analogue) if available
+        or will initialize from the clock otherwise. (Default=None)
+    space : :obj:`str`
+        The template space the coordinates are reported in. (Default='MNI')
+
+    Returns
+    -------
+    ground_truth_foci : :obj:`list`
+        generated foci in xyz (mm) coordinates
+    dataset : :class:`~nimare.dataset.Dataset`
+    """
+    # set random state
+    rng = np.random.RandomState(seed=seed)
+
+    # check foci argument
+    if not isinstance(foci, int) and not _array_like(foci):
+        raise ValueError("foci must be a positive integer or array like")
+
+    # check foci_percentage argument
+    if (
+        (not isinstance(foci_percentage, (float, str)))
+        or (isinstance(foci_percentage, str) and foci_percentage[-1] != "%")
+        or (isinstance(foci_percentage, float) and not (0.0 <= foci_percentage <= 1.0))
+    ):
+        raise ValueError(
+            "foci_percentage must be a string (example '96%') or a float between 0 and 1"
+        )
+
+    # check sample_size argument
+    if _array_like(sample_size) and len(sample_size) != n_studies and len(sample_size) != 2:
+        raise ValueError("sample_size must be the same length as n_studies or list of 2 items")
+    elif not _array_like(sample_size) and not isinstance(sample_size, int):
+        raise ValueError("sample_size must be array like or integer")
+
+    # check space argument
+    if space != "MNI":
+        raise NotImplementedError("Only coordinates for the MNI atlas has been defined")
+
+    # process foci_percentage argument
+    if isinstance(foci_percentage, str) and foci_percentage[-1] == "%":
+        foci_percentage = float(foci_percentage[:-1]) / 100
+
+    # process sample_size argument
+    if isinstance(sample_size, int):
+        sample_size = [sample_size] * n_studies
+    elif _array_like(sample_size) and len(sample_size) == 2 and n_studies != 2:
+        sample_size_lower_limit = sample_size[0]
+        sample_size_upper_limit = sample_size[1]
+        sample_size = rng.randint(sample_size_lower_limit, sample_size_upper_limit, size=n_studies)
+
+    ground_truth_foci, foci_dict = _create_foci(
+        foci, foci_percentage, fwhm, n_studies, n_noise_foci, rng, space
+    )
+
+    source_dict = _create_source(foci_dict, sample_size, space)
+    dataset = Dataset(source_dict)
+
+    return ground_truth_foci, dataset
+
+
+def create_neurovault_dataset(
+    collection_ids=NEUROVAULT_IDS,
+    contrasts=CONTRAST_OF_INTEREST,
+    img_dir=None,
+    map_type_conversion=None,
+    **dset_kwargs,
+):
+    """Download images from NeuroVault and use them to create a dataset.
+
+    .. versionadded:: 0.0.8
+
+    This function will also attempt to generate Z images for any contrasts
+    for which this is possible.
+
+    Parameters
+    ----------
+    collection_ids : :obj:`list` of :obj:`int` or :obj:`dict`, optional
+        A list of collections on neurovault specified by their id.
+        The collection ids can accessed through the neurovault API
+        (i.e., https://neurovault.org/api/collections) or
+        their main website (i.e., https://neurovault.org/collections).
+        For example, in this URL https://neurovault.org/collections/8836/,
+        `8836` is the collection id.
+        collection_ids can also be a dictionary whose keys are the informative
+        study name and the values are collection ids to give the collections
+        human readable names in the dataset.
+    contrasts : :obj:`dict`, optional
+        Dictionary whose keys represent the name of the contrast in
+        the dataset and whose values represent a regular expression that would
+        match the names represented in NeuroVault.
+        For example, under the ``Name`` column in this URL
+        https://neurovault.org/collections/8836/,
+        a valid contrast could be "as-Animal", which will be called "animal" in the created
+        dataset if the contrasts argument is ``{'animal': "as-Animal"}``.
+    img_dir : :obj:`str` or None, optional
+        Base path to save all the downloaded images, by default the images
+        will be saved to a temporary directory with the prefix "neurovault"
+    map_type_conversion : :obj:`dict` or None, optional
+        Dictionary whose keys are what you expect the `map_type` name to
+        be in neurovault and the values are the name of the respective
+        statistic map in a nimare dataset. Default = None.
+    **dset_kwargs : keyword arguments passed to Dataset
+        Keyword arguments to pass in when creating the Dataset object.
+        see :obj:`~nimare.dataset.Dataset` for details.
+
+    Returns
+    -------
+    :obj:`~nimare.dataset.Dataset`
+        Dataset object containing experiment information from neurovault.
+    """
+    dataset = convert_neurovault_to_dataset(
+        collection_ids, contrasts, img_dir, map_type_conversion, **dset_kwargs
+    )
+    transformer = ImageTransformer(target="z")
+    dataset = transformer.transform(dataset)
+
+    return dataset
+
+
+def _create_source(foci, sample_sizes, space="MNI"):
+    """Create dictionary according to nimads(ish) specification.
+
+    .. versionadded:: 0.0.4
+
+    Parameters
+    ----------
+    foci : :obj:`dict`
+        A dictionary of foci in xyz (mm) coordinates whose keys represent
+        different studies.
+    sample_sizes : :obj:`list`
+        The sample size for each study
+    space : :obj:`str`
+        The template space the coordinates are reported in. (Default='MNI')
+
+    Returns
+    -------
+    source : :obj:`dict`
+        study information in nimads format
+    """
+    source = {}
+    for sample_size, (study, study_foci) in zip(sample_sizes, foci.items()):
+        source[f"study-{study}"] = {
+            "contrasts": {
+                "1": {
+                    "coords": {
+                        "space": space,
+                        "x": [c[0] for c in study_foci],
+                        "y": [c[1] for c in study_foci],
+                        "z": [c[2] for c in study_foci],
+                    },
+                    "metadata": {"sample_sizes": [sample_size]},
+                }
+            }
+        }
+
+    return source
+
+
+def _create_foci(foci, foci_percentage, fwhm, n_studies, n_noise_foci, rng, space):
+    """Generate study specific foci.
+
+    .. versionadded:: 0.0.4
+
+    Parameters
+    ----------
+    foci : :obj:`int` or :obj:`list`
+        The number of foci to be generated per study or the
+        x,y,z coordinates of the ground truth foci.
+    foci_percentage : :obj:`float`
+        Percentage of studies where the foci appear.
+    fwhm : :obj:`float`
+        Full width at half maximum (fwhm) to define the probability
+        spread of the foci.
+    n_studies : :obj:`int`
+        Number of n_studies to generate.
+    n_noise_foci : :obj:`int`
+        Number of foci considered to be noise in each study.
+    rng : :class:`numpy.random.RandomState`
+        Random state to reproducibly initialize random numbers.
+    space : :obj:`str`
+        The template space the coordinates are reported in.
+
+    Returns
+    -------
+    ground_truth_foci : :obj:`list`
+        List of 3-item tuples containing x, y, z coordinates
+        of the ground truth foci or an empty list if
+        there are no ground_truth_foci.
+    foci_dict : :obj:`dict`
+        Dictionary with keys representing the study, and
+        whose values represent the study specific foci.
+    """
+    # convert foci_percentage to float between 0 and 1
+    if isinstance(foci_percentage, str) and foci_percentage[-1] == "%":
+        foci_percentage = float(foci_percentage[:-1]) / 100
+
+    if space == "MNI":
+        template_img = get_template(space="mni152_2mm", mask="brain")
+
+    # use a template to find all "valid" coordinates
+    template_data = template_img.get_fdata()
+    possible_ijks = np.argwhere(template_data)
+
+    # number of "convergent" foci each study should report
+    if isinstance(foci, int):
+        foci_idxs = np.unique(rng.choice(range(possible_ijks.shape[0]), foci, replace=True))
+        # if there are no foci_idxs, give a dummy coordinate (0, 0, 0)
+        ground_truth_foci_ijks = possible_ijks[foci_idxs] if foci_idxs.size else np.array([[]])
+    elif isinstance(foci, list):
+        ground_truth_foci_ijks = np.array([mm2vox(coord, template_img.affine) for coord in foci])
+
+    # create a probability map for each peak
+    kernel = get_ale_kernel(template_img, fwhm)[1]
+    foci_prob_maps = {
+        tuple(peak): compute_ale_ma(template_img, np.atleast_2d(peak), kernel=kernel).reshape(
+            template_data.shape
+        )
+        for peak in ground_truth_foci_ijks
+        if peak.size
+    }
+
+    # get study specific instances of each foci
+    signal_studies = int(round(foci_percentage * n_studies))
+    signal_ijks = {
+        peak: sparse.argwhere(prob_map)[
+            rng.choice(
+                sparse.argwhere(prob_map).shape[0],
+                size=signal_studies,
+                replace=True,
+                p=(prob_map[prob_map.nonzero()] / sum(prob_map[prob_map.nonzero()])).todense(),
+            )
+        ]
+        for peak, prob_map in foci_prob_maps.items()
+    }
+
+    # reshape foci coordinates to be study specific
+    paired_signal_ijks = (
+        np.transpose(np.array(list(signal_ijks.values())), axes=(1, 0, 2))
+        if signal_ijks
+        else (None,)
+    )
+
+    foci_dict = {}
+    for study_signal_ijks, study in zip_longest(paired_signal_ijks, range(n_studies)):
+        if study_signal_ijks is None:
+            study_signal_ijks = np.array([[]])
+            n_noise_foci = max(1, n_noise_foci)
+
+        if n_noise_foci > 0:
+            noise_ijks = possible_ijks[
+                rng.choice(possible_ijks.shape[0], n_noise_foci, replace=True)
+            ]
+
+            # add the noise foci ijks to the existing signal ijks
+            foci_ijks = (
+                np.unique(np.vstack([study_signal_ijks, noise_ijks]), axis=0)
+                if np.any(study_signal_ijks)
+                else noise_ijks
+            )
+        else:
+            foci_ijks = study_signal_ijks
+
+        # transform ijk voxel coordinates to xyz mm coordinates
+        foci_xyzs = [vox2mm(ijk, template_img.affine) for ijk in foci_ijks]
+        foci_dict[study] = foci_xyzs
+
+    ground_truth_foci_xyz = [
+        tuple(vox2mm(ijk, template_img.affine)) for ijk in ground_truth_foci_ijks if np.any(ijk)
+    ]
+    return ground_truth_foci_xyz, foci_dict
+
+
+def _array_like(obj):
+    """Test if obj is array-like."""
+    return isinstance(obj, (list, tuple, np.ndarray))