nimare 0.4.2__py3-none-any.whl

nimare/annotate/lda.py

@@ -0,0 +1,147 @@
+"""Topic modeling with latent Dirichlet allocation."""
+
+import numpy as np
+import pandas as pd
+from sklearn.decomposition import LatentDirichletAllocation
+
+from nimare.annotate.text import generate_counts
+from nimare.base import NiMAREBase
+from nimare.utils import _check_ncores
+
+
+class LDAModel(NiMAREBase):
+    """Generate a latent Dirichlet allocation (LDA) topic model.
+
+    This class is a light wrapper around scikit-learn tools for tokenization and LDA.
+
+    Parameters
+    ----------
+    n_topics : :obj:`int`
+        Number of topics for topic model. This corresponds to the model's ``n_components``
+        parameter. Must be an integer >= 1.
+    max_iter : :obj:`int`, optional
+        Maximum number of iterations to use during model fitting. Default = 1000.
+    alpha : :obj:`float` or None, optional
+        The ``alpha`` value for the model. This corresponds to the model's ``doc_topic_prior``
+        parameter. Default is None, which evaluates to ``1 / n_topics``,
+        as was used in :footcite:t:`poldrack2012discovering`.
+    beta : :obj:`float` or None, optional
+        The ``beta`` value for the model. This corresponds to the model's ``topic_word_prior``
+        parameter. If None, it evaluates to ``1 / n_topics``.
+        Default is 0.001, which was used in :footcite:t:`poldrack2012discovering`.
+    text_column : :obj:`str`, optional
+        The source of text to use for the model. This should correspond to an existing column
+        in the :py:attr:`~nimare.dataset.Dataset.texts` attribute. Default is "abstract".
+    n_cores : :obj:`int`, optional
+        Number of cores to use for parallelization.
+        If <=0, defaults to using all available cores.
+        Default is 1.
+
+    Attributes
+    ----------
+    model : :obj:`~sklearn.decomposition.LatentDirichletAllocation`
+
+    Notes
+    -----
+    Latent Dirichlet allocation was first developed in :footcite:t:`blei2003latent`,
+    and was first applied to neuroimaging articles in :footcite:t:`poldrack2012discovering`.
+
+    References
+    ----------
+    .. footbibliography::
+
+    See Also
+    --------
+    :class:`~sklearn.feature_extraction.text.CountVectorizer`: Used to build a vocabulary of terms
+        and their associated counts from texts in the ``self.text_column`` of the Dataset's
+        ``texts`` attribute.
+    :class:`~sklearn.decomposition.LatentDirichletAllocation`: Used to train the LDA model.
+    """
+
+    def __init__(
+        self, n_topics, max_iter=1000, alpha=None, beta=0.001, text_column="abstract", n_cores=1
+    ):
+        self.n_topics = n_topics
+        self.max_iter = max_iter
+        self.alpha = alpha
+        self.beta = beta
+        self.text_column = text_column
+        self.n_cores = _check_ncores(n_cores)
+
+        self.model = LatentDirichletAllocation(
+            n_components=n_topics,
+            max_iter=max_iter,
+            learning_method="batch",
+            doc_topic_prior=alpha,
+            topic_word_prior=beta,
+            n_jobs=n_cores,
+        )
+
+    def fit(self, dset):
+        """Fit the LDA topic model to text from a Dataset.
+
+        Parameters
+        ----------
+        dset : :obj:`~nimare.dataset.Dataset`
+            A Dataset with, at minimum, text available in the ``self.text_column`` column of its
+            :py:attr:`~nimare.dataset.Dataset.texts` attribute.
+
+        Returns
+        -------
+        dset : :obj:`~nimare.dataset.Dataset`
+            A new Dataset with an updated :py:attr:`~nimare.dataset.Dataset.annotations` attribute.
+
+        Attributes
+        ----------
+        distributions_ : :obj:`dict`
+            A dictionary containing additional distributions produced by the model, including:
+
+                -   ``p_topic_g_word``: :obj:`numpy.ndarray` of shape (n_topics, n_tokens)
+                    containing the topic-term weights for the model.
+                -   ``p_topic_g_word_df``: :obj:`pandas.DataFrame` of shape (n_topics, n_tokens)
+                    containing the topic-term weights for the model.
+        """
+        counts_df = generate_counts(
+            dset.texts,
+            text_column=self.text_column,
+            tfidf=False,
+            max_df=len(dset.ids) - 2,
+            min_df=2,
+        )
+        vocabulary = counts_df.columns.to_numpy()
+        count_values = counts_df.values
+        study_ids = counts_df.index.tolist()
+
+        doc_topic_weights = self.model.fit_transform(count_values)
+        topic_word_weights = self.model.components_
+
+        # Get top 3 words for each topic for annotation
+        sorted_weights_idxs = np.argsort(-topic_word_weights, axis=1)
+        top_tokens = [
+            "_".join(vocabulary[sorted_weights_idxs[topic_i, :]][:3])
+            for topic_i in range(self.n_topics)
+        ]
+        topic_names = [
+            f"LDA{self.n_topics}__{i + 1}_{top_tokens[i]}" for i in range(self.n_topics)
+        ]
+
+        doc_topic_weights_df = pd.DataFrame(
+            index=study_ids,
+            columns=topic_names,
+            data=doc_topic_weights,
+        )
+        topic_word_weights_df = pd.DataFrame(
+            index=topic_names,
+            columns=vocabulary,
+            data=topic_word_weights,
+        )
+        self.distributions_ = {
+            "p_topic_g_word": topic_word_weights,
+            "p_topic_g_word_df": topic_word_weights_df,
+        }
+
+        annotations = dset.annotations.copy()
+        annotations = pd.merge(annotations, doc_topic_weights_df, left_on="id", right_index=True)
+        new_dset = dset.copy()
+        new_dset.annotations = annotations
+        return new_dset

nimare/annotate/text.py

@@ -0,0 +1,75 @@
+"""Text extraction tools."""
+
+import logging
+import os.path as op
+
+import pandas as pd
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+
+from nimare.utils import get_resource_path
+
+LGR = logging.getLogger(__name__)
+
+
+def generate_counts(text_df, text_column="abstract", tfidf=True, min_df=50, max_df=0.5):
+    """Generate tf-idf weights for unigrams/bigrams derived from textual data.
+
+    Parameters
+    ----------
+    text_df : (D x 2) :obj:`pandas.DataFrame`
+        A DataFrame with two columns ('id' and 'text'). D = document.
+
+    Returns
+    -------
+    weights_df : (D x T) :obj:`pandas.DataFrame`
+        A DataFrame where the index is 'id' and the columns are the
+        unigrams/bigrams derived from the data. D = document. T = term.
+    """
+    if text_column not in text_df.columns:
+        raise ValueError(f"Column '{text_column}' not found in DataFrame")
+
+    # Remove rows with empty text cells
+    orig_ids = text_df["id"].tolist()
+    text_df = text_df.fillna("")
+    keep_ids = text_df.loc[text_df[text_column] != "", "id"]
+    text_df = text_df.loc[text_df["id"].isin(keep_ids)]
+
+    if len(keep_ids) != len(orig_ids):
+        LGR.info(f"Retaining {len(keep_ids)}/{len(orig_ids)} studies")
+
+    ids = text_df["id"].tolist()
+    text = text_df[text_column].tolist()
+    stoplist = op.join(get_resource_path(), "neurosynth_stoplist.txt")
+    with open(stoplist, "r") as fo:
+        stop_words = fo.read().splitlines()
+
+    if tfidf:
+        vectorizer = TfidfVectorizer(
+            min_df=min_df,
+            max_df=max_df,
+            ngram_range=(1, 2),
+            vocabulary=None,
+            stop_words=stop_words,
+        )
+    else:
+        vectorizer = CountVectorizer(
+            min_df=min_df,
+            max_df=max_df,
+            ngram_range=(1, 2),
+            vocabulary=None,
+            stop_words=stop_words,
+        )
+    weights = vectorizer.fit_transform(text).toarray()
+
+    if hasattr(vectorizer, "get_feature_names_out"):
+        # scikit-learn >= 1.0.0
+        names = vectorizer.get_feature_names_out()
+    else:
+        # scikit-learn < 1.0.0
+        # To remove when we drop support for 3.6 and increase minimum sklearn version to 1.0.0.
+        names = vectorizer.get_feature_names()
+
+    names = [str(name) for name in names]
+    weights_df = pd.DataFrame(weights, columns=names, index=ids)
+    weights_df.index.name = "id"
+    return weights_df

nimare/annotate/utils.py

@@ -0,0 +1,87 @@
+"""Utility functions for ontology tools."""
+
+import numpy as np
+import pandas as pd
+
+
+def _generate_weights(rel_df, weights):
+    """Create an IDxID DataFrame linking weight value to each relationship type.
+
+    .. versionadded:: 0.0.2
+
+    Parameters
+    ----------
+    rel_df : (X x 3) :obj:`pandas.DataFrame`
+        DataFrame with three columns: input, output, and rel_type
+        (relationship type).
+    weights : :obj:`dict`
+        Dictionary defining relationship weights. Each relationship type is a
+        key and the associated value is the weight to use for that kind of
+        relationship.
+
+    Returns
+    -------
+    expanded_df : :obj:`pandas.DataFrame`
+        Square DataFrame where rows correspond to input items, columns
+        correspond to output items, and cells have the weights associated with
+        the particular input/output relationship.
+
+    Notes
+    -----
+    For example, if weights is {'partOf': 1}, the resulting expanded_df will
+    have a value of 1 for all cells where the input item (row) is a part of the
+    output item (column), and will have zeroes for all other cells.
+    """
+    # Override isSelf weight
+    weights["isSelf"] = 1
+
+    # Hierarchical expansion
+    def get_weight(rel_type):
+        weight = weights.get(rel_type, 0)
+        return weight
+
+    t_df = rel_df.copy()
+    t_df["rel_type"] = t_df["rel_type"].apply(get_weight)
+    weights_df = t_df.pivot_table(
+        index="input", columns="output", values="rel_type", aggfunc=np.max
+    )
+    weights_df = weights_df.fillna(0)
+    out_not_in = list(set(t_df["output"].values) - set(t_df["input"].values))
+    in_not_out = list(set(t_df["input"].values) - set(t_df["output"].values))
+
+    new_cols = pd.DataFrame(
+        columns=in_not_out,
+        index=weights_df.index,
+        data=np.zeros((weights_df.shape[0], len(in_not_out))),
+    )
+    weights_df = pd.concat((weights_df, new_cols), axis=1)
+    new_rows = pd.DataFrame(
+        columns=weights_df.columns,
+        index=out_not_in,
+        data=np.zeros((len(out_not_in), weights_df.shape[1])),
+    )
+    weights_df = pd.concat((weights_df, new_rows), axis=0)
+    all_cols = sorted(weights_df.columns.tolist())
+    weights_df = weights_df.loc[all_cols, :]
+    weights_df = weights_df.loc[:, all_cols]
+
+    # expanding the hierarchical expansion to all related terms
+    # this way, a single dot product will apply counts to all layers
+    expanded_df = weights_df.copy()
+    mat = weights_df.values
+
+    for i, val in enumerate(weights_df.index):
+        row = np.zeros((1, weights_df.shape[0]))
+        row[0, i] = 1  # identity
+        temp = np.zeros((1, weights_df.shape[0]))
+
+        while not np.array_equal(temp != 0, row != 0):
+            temp = np.copy(row)
+            row = np.dot(row, mat)
+
+            # Constrain weights to <=1.
+            # Hopefully this won't mess with weights <1,
+            # but will also prevent weights from adding to one another.
+            row[row > 1] = 1
+        expanded_df.loc[val] = np.squeeze(row)
+    return expanded_df

nimare/base.py

@@ -0,0 +1,217 @@
+"""Base classes for NiMARE."""
+
+import gzip
+import inspect
+import logging
+import pickle
+from abc import ABCMeta
+from collections import defaultdict
+
+from nilearn._utils import CacheMixin
+
+LGR = logging.getLogger(__name__)
+
+
+class NiMAREBase(CacheMixin, metaclass=ABCMeta):
+    """Base class for NiMARE.
+
+    This class contains a few features that are useful throughout the library:
+
+    - Custom __repr__ method for printing the object.
+    - get_params from scikit-learn, with which parameters provided at __init__ can be viewed.
+    - set_params from scikit-learn, with which parameters provided at __init__ can be overwritten.
+      I'm not sure that this is actually used or useable in NiMARE.
+    - save to save the object to a Pickle file.
+    - load to load an instance of the object from a Pickle file.
+
+    TODO: Actually write/refactor class methods. They mostly come directly from sklearn
+    https://github.com/scikit-learn/scikit-learn/blob/
+    2a1e9686eeb203f5fddf44fd06414db8ab6a554a/sklearn/base.py#L141
+    """
+
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        """Show basic NiMARE class representation.
+
+        Specifically, this shows the name of the class, along with any parameters
+        that are **not** set to the default.
+        """
+        # Get default parameter values for the object
+        signature = inspect.signature(self.__init__)
+        defaults = {
+            k: v.default
+            for k, v in signature.parameters.items()
+            if v.default is not inspect.Parameter.empty
+        }
+
+        # Eliminate any sub-parameters (e.g., parameters for a Estimator's KernelTransformer),
+        # as well as default values
+        params = self.get_params()
+        params = {k: v for k, v in params.items() if "__" not in k}
+        params = {k: v for k, v in params.items() if defaults.get(k) != v}
+
+        # Convert to strings
+        param_strs = []
+        for k, v in params.items():
+            if isinstance(v, str):
+                # Wrap string values in single quotes
+                param_str = f"{k}='{v}'"
+            else:
+                # Keep everything else as-is based on its own repr
+                param_str = f"{k}={v}"
+            param_strs.append(param_str)
+
+        rep = f"{self.__class__.__name__}({', '.join(param_strs)})"
+        return rep
+
+    @classmethod
+    def _get_param_names(cls):
+        """Get parameter names for the estimator."""
+        # fetch the constructor or the original constructor before
+        # deprecation wrapping if any
+        init = getattr(cls.__init__, "deprecated_original", cls.__init__)
+        if init is object.__init__:
+            # No explicit constructor to introspect
+            return []
+
+        # introspect the constructor arguments to find the model parameters
+        # to represent
+        init_signature = inspect.signature(init)
+        # Consider the constructor parameters excluding 'self'
+        parameters = [
+            p
+            for p in init_signature.parameters.values()
+            if p.name != "self" and p.kind != p.VAR_KEYWORD
+        ]
+        for p in parameters:
+            if p.kind == p.VAR_POSITIONAL:
+                raise RuntimeError(
+                    "scikit-learn estimators should always "
+                    "specify their parameters in the signature"
+                    " of their __init__ (no varargs)."
+                    " %s with constructor %s doesn't "
+                    " follow this convention." % (cls, init_signature)
+                )
+        # Extract and sort argument names excluding 'self'
+        return sorted([p.name for p in parameters])
+
+    def get_params(self, deep=True):
+        """Get parameters for this estimator.
+
+        Parameters
+        ----------
+        deep : :obj:`bool`, default=True
+            If True, will return the parameters for this estimator and
+            contained subobjects that are estimators.
+
+        Returns
+        -------
+        params : :obj:`dict`
+            Parameter names mapped to their values.
+        """
+        out = dict()
+        for key in self._get_param_names():
+            value = getattr(self, key, None)
+            if deep and hasattr(value, "get_params"):
+                deep_items = value.get_params().items()
+                out.update((key + "__" + k, val) for k, val in deep_items)
+            out[key] = value
+        return out
+
+    def set_params(self, **params):
+        """Set the parameters of this estimator.
+
+        The method works on simple estimators as well as on nested objects
+        (such as pipelines). The latter have parameters of the form
+        ``<component>__<parameter>`` so that it's possible to update each
+        component of a nested object.
+
+        Returns
+        -------
+        self
+        """
+        if not params:
+            # Simple optimization to gain speed (inspect is slow)
+            return self
+        valid_params = self.get_params(deep=True)
+
+        nested_params = defaultdict(dict)  # grouped by prefix
+        for key, value in params.items():
+            key, delim, sub_key = key.partition("__")
+            if key not in valid_params:
+                raise ValueError(
+                    "Invalid parameter %s for estimator %s. "
+                    "Check the list of available parameters "
+                    "with `estimator.get_params().keys()`." % (key, self)
+                )
+
+            if delim:
+                nested_params[key][sub_key] = value
+            else:
+                setattr(self, key, value)
+                valid_params[key] = value
+
+        for key, sub_params in nested_params.items():
+            valid_params[key].set_params(**sub_params)
+
+        return self
+
+    def save(self, filename, compress=True):
+        """Pickle the class instance to the provided file.
+
+        Parameters
+        ----------
+        filename : :obj:`str`
+            File to which object will be saved.
+        compress : :obj:`bool`, optional
+            If True, the file will be compressed with gzip. Otherwise, the
+            uncompressed version will be saved. Default = True.
+        """
+        if compress:
+            with gzip.GzipFile(filename, "wb") as file_object:
+                pickle.dump(self, file_object)
+        else:
+            with open(filename, "wb") as file_object:
+                pickle.dump(self, file_object)
+
+    @classmethod
+    def load(cls, filename, compressed=True):
+        """Load a pickled class instance from file.
+
+        Parameters
+        ----------
+        filename : :obj:`str`
+            Name of file containing object.
+        compressed : :obj:`bool`, default=True
+            If True, the file is assumed to be compressed and gzip will be used
+            to load it. Otherwise, it will assume that the file is not
+            compressed. Default = True.
+
+        Returns
+        -------
+        obj : class object
+            Loaded class object.
+        """
+        if compressed:
+            try:
+                with gzip.GzipFile(filename, "rb") as file_object:
+                    obj = pickle.load(file_object)
+            except UnicodeDecodeError:
+                # Need to try this for python3
+                with gzip.GzipFile(filename, "rb") as file_object:
+                    obj = pickle.load(file_object, encoding="latin")
+        else:
+            try:
+                with open(filename, "rb") as file_object:
+                    obj = pickle.load(file_object)
+            except UnicodeDecodeError:
+                # Need to try this for python3
+                with open(filename, "rb") as file_object:
+                    obj = pickle.load(file_object, encoding="latin")
+
+        if not isinstance(obj, cls):
+            raise IOError(f"Pickled object must be {cls}, not {type(obj)}")
+
+        return obj

nimare/cli.py

@@ -0,0 +1,124 @@
+"""Command-line interfaces for common workflows."""
+
+import argparse
+import os.path as op
+
+from nimare.io import convert_neurosynth_to_json, convert_sleuth_to_json
+from nimare.workflows.macm import macm_workflow
+
+
+def _is_valid_file(parser, arg):
+    """Check if argument is existing file."""
+    if not op.isfile(arg) and arg is not None:
+        parser.error(f"The file {arg} does not exist!")
+
+    return arg
+
+
+def _get_parser():
+    """Parse command line inputs for NiMARE.
+
+    Returns
+    -------
+    parser.parse_args() : argparse dict
+    """
+    parser = argparse.ArgumentParser(prog="nimare")
+    subparsers = parser.add_subparsers(help="NiMARE workflows")
+
+    # MACM
+    macm_parser = subparsers.add_parser(
+        "macm",
+        help=(
+            "Run a meta-analytic coactivation modeling (MACM) "
+            "analysis using activation likelihood estimation "
+            "(ALE) on a NiMARE dataset file and a target mask."
+        ),
+    )
+    macm_parser.set_defaults(func=macm_workflow)
+    macm_parser.add_argument(
+        "dataset_file", type=lambda x: _is_valid_file(parser, x), help=("Dataset file to analyze.")
+    )
+    macm_parser.add_argument(
+        "--mask",
+        "--mask_file",
+        dest="mask_file",
+        type=lambda x: _is_valid_file(parser, x),
+        help=("Mask file"),
+        required=True,
+    )
+    macm_parser.add_argument(
+        "--output_dir",
+        dest="output_dir",
+        metavar="PATH",
+        type=str,
+        help=("Output directory."),
+        default=".",
+    )
+    macm_parser.add_argument(
+        "--prefix", dest="prefix", type=str, help=("Common prefix for output maps."), default=""
+    )
+    macm_parser.add_argument(
+        "--n_iters",
+        dest="n_iters",
+        type=int,
+        help=("Number of iterations for permutation testing."),
+        default=5000,
+    )
+    macm_parser.add_argument(
+        "--v_thr",
+        dest="v_thr",
+        type=float,
+        help=("Voxel p-value threshold used to create clusters."),
+        default=0.001,
+    )
+    macm_parser.add_argument(
+        "--n_cores",
+        dest="n_cores",
+        type=int,
+        default=1,
+        help=("Number of processes to use for meta-analysis. If -1, use all available cores."),
+    )
+
+    # Conversion workflows
+    sleuth2nimare_parser = subparsers.add_parser(
+        "sleuth2nimare", help=("Convert a Sleuth text file to a NiMARE json file.")
+    )
+    sleuth2nimare_parser.set_defaults(func=convert_sleuth_to_json)
+    sleuth2nimare_parser.add_argument(
+        "text_file",
+        type=lambda x: _is_valid_file(parser, x),
+        help=("Sleuth text file to convert."),
+    )
+    sleuth2nimare_parser.add_argument("out_file", type=str, help=("Output file."))
+
+    neurosynth2nimare_parser = subparsers.add_parser(
+        "neurosynth2nimare", help=("Convert a Neurosynth text file to a NiMARE json file.")
+    )
+    neurosynth2nimare_parser.set_defaults(func=convert_neurosynth_to_json)
+    neurosynth2nimare_parser.add_argument(
+        "text_file",
+        type=lambda x: _is_valid_file(parser, x),
+        help=("Neurosynth text file to convert."),
+    )
+    neurosynth2nimare_parser.add_argument("out_file", type=str, help=("Output file."))
+    neurosynth2nimare_parser.add_argument(
+        "--annotations_file",
+        metavar="FILE",
+        type=lambda x: _is_valid_file(parser, x),
+        help=("Optional annotations (features) file."),
+        default=None,
+    )
+
+    return parser
+
+
+def _main(argv=None):
+    """Run NiMARE CLI entrypoint."""
+    options = _get_parser().parse_args(argv)
+    args = vars(options).copy()
+    args.pop("func")
+    options.func(**args)
+
+
+if __name__ == "__main__":
+    _main()