nimare 0.4.2__py3-none-any.whl

nimare/tests/test_diagnostics.py

@@ -0,0 +1,168 @@
+"""Tests for the nimare.diagnostics module."""
+
+import os.path as op
+
+import pytest
+from nilearn.input_data import NiftiLabelsMasker
+
+from nimare import diagnostics
+from nimare.meta import cbma, ibma
+from nimare.tests.utils import get_test_data_path
+
+
+@pytest.mark.parametrize(
+    "estimator,meta_type,n_samples,target_image,voxel_thresh",
+    [
+        (cbma.ALE, "cbma", "onesample", "z", 1.65),
+        (cbma.MKDADensity, "cbma", "onesample", "z", 1.65),
+        (cbma.KDA, "cbma", "onesample", "z", 1.65),
+        (cbma.MKDAChi2, "cbma", "twosample", "z_desc-uniformity", 1.65),
+        (ibma.Fishers, "ibma", "onesample", "z", 0.1),
+        (ibma.Stouffers, "ibma", "onesample", "z", 0.1),
+        (ibma.WeightedLeastSquares, "ibma", "onesample", "z", 0.1),
+        (ibma.DerSimonianLaird, "ibma", "onesample", "z", 0.1),
+        (ibma.Hedges, "ibma", "onesample", "z", 0.1),
+        # (ibma.SampleSizeBasedLikelihood, "ibma", "onesample", "z"),
+        # (ibma.VarianceBasedLikelihood, "ibma", "onesample", "z"),
+        # (ibma.PermutedOLS, "ibma", "onesample", "z"),
+    ],
+)
+def test_jackknife_smoke(
+    testdata_ibma,
+    testdata_cbma_full,
+    estimator,
+    meta_type,
+    n_samples,
+    target_image,
+    voxel_thresh,
+):
+    """Smoke test the Jackknife method."""
+    dset1 = testdata_cbma_full.slice(testdata_cbma_full.ids[:10])
+    dset2 = testdata_cbma_full.slice(testdata_cbma_full.ids[10:])
+
+    meta = estimator()
+    testdata = testdata_ibma if meta_type == "ibma" else testdata_cbma_full
+    res = meta.fit(dset1, dset2) if n_samples == "twosample" else meta.fit(testdata)
+
+    jackknife = diagnostics.Jackknife(target_image=target_image, voxel_thresh=voxel_thresh)
+    results = jackknife.transform(res)
+
+    image_name = "_".join(target_image.split("_")[1:])
+    image_name = f"_{image_name}" if image_name else image_name
+
+    # For ibma.WeightedLeastSquares we have both positive and negative tail combined.
+    contribution_table = (
+        results.tables[f"{target_image}_diag-Jackknife_tab-counts"]
+        if estimator == ibma.WeightedLeastSquares
+        else results.tables[f"{target_image}_diag-Jackknife_tab-counts_tail-positive"]
+    )
+
+    clusters_table = results.tables[f"{target_image}_tab-clust"]
+    label_maps = results.maps[f"label{image_name}_tail-positive"]
+    ids_ = meta.inputs_["id"] if n_samples == "onesample" else meta.inputs_["id1"]
+
+    assert contribution_table.shape[0] == len(ids_)
+    assert clusters_table.shape[0] >= contribution_table.shape[1] - 1
+    assert len(label_maps) > 0
+
+
+def test_jackknife_with_zero_clusters(testdata_cbma_full):
+    """Ensure that Jackknife will work with zero clusters."""
+    meta = cbma.ALE()
+    res = meta.fit(testdata_cbma_full)
+
+    jackknife = diagnostics.Jackknife(target_image="z", voxel_thresh=10)
+    results = jackknife.transform(res)
+
+    contribution_table = results.tables["z_diag-Jackknife_tab-counts"]
+    clusters_table = results.tables["z_tab-clust"]
+    label_maps = results.maps["label_tail-positive"]
+    assert contribution_table is None
+    assert clusters_table.empty
+    assert not label_maps
+
+
+def test_jackknife_with_custom_masker_smoke(testdata_ibma):
+    """Ensure that Jackknife will work with NiftiLabelsMaskers.
+
+    CBMAs don't work with NiftiLabelsMaskers and VarianceBasedLikelihood takes ~1 minute,
+    which is too long for a single test, so I'm just using SampleSizeBasedLikelihood.
+    """
+    atlas = op.join(get_test_data_path(), "test_pain_dataset", "atlas.nii.gz")
+    masker = NiftiLabelsMasker(atlas)
+
+    meta = ibma.SampleSizeBasedLikelihood(mask=masker)
+    res = meta.fit(testdata_ibma)
+
+    jackknife = diagnostics.Jackknife(target_image="z", voxel_thresh=0.5)
+    results = jackknife.transform(res)
+    contribution_table = results.tables["z_diag-Jackknife_tab-counts_tail-positive"]
+    assert contribution_table.shape[0] == len(meta.inputs_["id"])
+
+    # A Jackknife with a target_image that isn't present in the MetaResult raises a ValueError.
+    with pytest.raises(ValueError):
+        jackknife = diagnostics.Jackknife(target_image="doggy", voxel_thresh=0.5)
+        jackknife.transform(res)
+
+
+@pytest.mark.parametrize(
+    "estimator,meta_type,n_samples,target_image",
+    [
+        (cbma.ALE, "cbma", "onesample", "z"),
+        (cbma.MKDADensity, "cbma", "onesample", "z"),
+        (cbma.KDA, "cbma", "onesample", "z"),
+        (cbma.MKDAChi2, "cbma", "twosample", "z_desc-uniformity"),
+        (ibma.Stouffers, "ibma", "onesample", "z"),
+    ],
+)
+def test_focuscounter_smoke(
+    testdata_ibma,
+    testdata_cbma_full,
+    estimator,
+    meta_type,
+    n_samples,
+    target_image,
+):
+    """Smoke test the FocusCounter method."""
+    dset1 = testdata_cbma_full.slice(testdata_cbma_full.ids[:10])
+    dset2 = testdata_cbma_full.slice(testdata_cbma_full.ids[10:])
+
+    meta = estimator()
+    testdata = testdata_ibma if meta_type == "ibma" else testdata_cbma_full
+    res = meta.fit(dset1, dset2) if n_samples == "twosample" else meta.fit(testdata)
+
+    counter = diagnostics.FocusCounter(target_image=target_image, voxel_thresh=1.65)
+    if meta_type == "ibma":
+        with pytest.raises(ValueError):
+            counter.transform(res)
+    else:
+        results = counter.transform(res)
+
+        image_name = "_".join(target_image.split("_")[1:])
+        image_name = f"_{image_name}" if image_name else image_name
+
+        contribution_table = results.tables[
+            f"{target_image}_diag-FocusCounter_tab-counts_tail-positive"
+        ]
+        clusters_table = results.tables[f"{target_image}_tab-clust"]
+        label_maps = results.maps[f"label{image_name}_tail-positive"]
+        ids_ = meta.inputs_["id"] if n_samples == "onesample" else meta.inputs_["id1"]
+
+        assert contribution_table.shape[0] == len(ids_)
+        assert clusters_table.shape[0] >= contribution_table.shape[1] - 1
+        assert len(label_maps) > 0
+
+
+def test_focusfilter(testdata_laird):
+    """Ensure that the FocusFilter removes out-of-mask coordinates.
+
+    The Laird dataset contains 16 foci outside of the MNI brain mask, which the filter should
+    remove.
+    """
+    n_coordinates_all = testdata_laird.coordinates.shape[0]
+    ffilter = diagnostics.FocusFilter()
+    filtered_dset = ffilter.transform(testdata_laird)
+    n_coordinates_filtered = filtered_dset.coordinates.shape[0]
+    assert n_coordinates_all == 1117
+    assert n_coordinates_filtered == 1101
+    assert n_coordinates_filtered <= n_coordinates_all

nimare/tests/test_estimator_performance.py

@@ -0,0 +1,385 @@
+"""Test estimator, kerneltransformer, and multiple comparisons corrector performance."""
+
+import os
+from contextlib import ExitStack as does_not_raise
+
+import numpy as np
+import pytest
+
+from nimare.correct import FDRCorrector, FWECorrector
+from nimare.generate import create_coordinate_dataset
+from nimare.meta import ale, kernel, mkda
+from nimare.results import MetaResult
+from nimare.tests.utils import _check_p_values, _create_signal_mask, _transform_res
+from nimare.utils import mm2vox
+
+# set significance levels used for testing.
+ALPHA = 0.05
+
+if os.environ.get("CIRCLECI"):
+    N_CORES = 1
+else:
+    N_CORES = -1
+
+
+# PRECOMPUTED FIXTURES
+# --------------------
+
+
+##########################################
+# random state
+##########################################
+@pytest.fixture(scope="session")
+def random():
+    """Set random state for the tests."""
+    np.random.seed(1939)
+
+
+##########################################
+# simulated dataset(s)
+##########################################
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param(
+            {
+                "foci": 5,
+                "fwhm": 10.0,
+                "n_studies": 40,
+                "sample_size": 30,
+                "n_noise_foci": 20,
+                "seed": 1939,
+            },
+            id="normal_data",
+        )
+    ],
+)
+def simulatedata_cbma(request):
+    """Set the simulated CBMA data according to parameters."""
+    return request.param["fwhm"], create_coordinate_dataset(**request.param)
+
+
+##########################################
+# signal and non-signal masks
+##########################################
+@pytest.fixture(scope="session")
+def signal_masks(simulatedata_cbma):
+    """Define masks of signal and non-signal for performance evaluation."""
+    _, (ground_truth_foci, dataset) = simulatedata_cbma
+    ground_truth_foci_ijks = [
+        tuple(mm2vox(focus, dataset.masker.mask_img.affine)) for focus in ground_truth_foci
+    ]
+    return _create_signal_mask(np.array(ground_truth_foci_ijks), dataset.masker.mask_img)
+
+
+##########################################
+# meta-analysis estimators
+##########################################
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param(ale.ALE, id="ale"),
+        pytest.param(mkda.MKDADensity, id="mkda"),
+        pytest.param(mkda.KDA, id="kda"),
+    ],
+)
+def meta_est(request):
+    """Define meta-analysis estimators for tests."""
+    return request.param
+
+
+##########################################
+# meta-analysis estimator parameters
+##########################################
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param({"null_method": "montecarlo", "n_iters": 100}, id="montecarlo"),
+        pytest.param({"null_method": "approximate"}, id="approximate"),
+        pytest.param(
+            {"null_method": "reduced_montecarlo", "n_iters": 100}, id="reduced_montecarlo"
+        ),
+    ],
+)
+def meta_params(request):
+    """Define meta-analysis Estimator parameters for tests."""
+    return request.param
+
+
+##########################################
+# meta-analysis kernels
+##########################################
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param(kernel.ALEKernel, id="ale_kernel"),
+        pytest.param(kernel.MKDAKernel, id="mkda_kernel"),
+        pytest.param(kernel.KDAKernel, id="kda_kernel"),
+    ],
+)
+def kern(request):
+    """Define kernel transformers for tests."""
+    return request.param
+
+
+##########################################
+# multiple comparison correctors (testing)
+##########################################
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param(FWECorrector(method="bonferroni"), id="fwe_bonferroni"),
+        pytest.param(
+            FWECorrector(method="montecarlo", voxel_thresh=ALPHA, n_iters=100, n_cores=N_CORES),
+            id="fwe_montecarlo",
+        ),
+        pytest.param(FDRCorrector(method="indep", alpha=ALPHA), id="fdr_indep"),
+        pytest.param(FDRCorrector(method="negcorr", alpha=ALPHA), id="fdr_negcorr"),
+    ],
+)
+def corr(request):
+    """Define multiple comparisons correctors for tests."""
+    return request.param
+
+
+##########################################
+# multiple comparison correctors (smoke)
+##########################################
+@pytest.fixture(
+    scope="session",
+    params=[
+        pytest.param(FWECorrector(method="bonferroni"), id="fwe_bonferroni"),
+        pytest.param(
+            FWECorrector(method="montecarlo", voxel_thresh=ALPHA, n_iters=2, n_cores=1),
+            id="fwe_montecarlo",
+        ),
+        pytest.param(FDRCorrector(method="indep", alpha=ALPHA), id="fdr_indep"),
+        pytest.param(FDRCorrector(method="negcorr", alpha=ALPHA), id="fdr_negcorr"),
+    ],
+)
+def corr_small(request):
+    """Define multiple comparisons correctors for tests."""
+    return request.param
+
+
+###########################################
+# all meta-analysis estimator/kernel combos
+###########################################
+@pytest.fixture(scope="session")
+def meta(simulatedata_cbma, meta_est, kern, meta_params):
+    """Define estimator/kernel combinations for tests."""
+    fwhm, (_, _) = simulatedata_cbma
+    if kern == kernel.KDAKernel or kern == kernel.MKDAKernel:
+        kern = kern(r=fwhm / 2)
+    else:
+        kern = kern()
+
+    # instantiate meta-analysis estimator
+    return meta_est(kern, **meta_params)
+
+
+###########################################
+# meta-analysis estimator results
+###########################################
+@pytest.fixture(scope="session")
+def meta_res(simulatedata_cbma, meta, random):
+    """Define estimators for tests."""
+    _, (_, dataset) = simulatedata_cbma
+    # CHECK IF META/KERNEL WORK TOGETHER
+    ####################################
+    meta_expectation = does_not_raise()
+
+    with meta_expectation:
+        res = meta.fit(dataset)
+    # if creating the result failed (expected), do not continue
+    if isinstance(meta_expectation, type(pytest.raises(ValueError))):
+        pytest.xfail("this meta-analysis & kernel combo fails")
+    # instantiate meta-analysis estimator
+    return res
+
+
+###########################################
+# corrected results (testing)
+###########################################
+@pytest.fixture(scope="session")
+def meta_cres(meta, meta_res, corr, random):
+    """Define corrected results for tests."""
+    return _transform_res(meta, meta_res, corr)
+
+
+###########################################
+# corrected results (smoke)
+###########################################
+@pytest.fixture(scope="session")
+def meta_cres_small(meta, meta_res, corr_small, random):
+    """Define corrected results for tests."""
+    return _transform_res(meta, meta_res, corr_small)
+
+
+# --------------
+# TEST FUNCTIONS
+# --------------
+
+
+@pytest.mark.performance_smoke
+def test_meta_fit_smoke(meta_res):
+    """Smoke test for meta-analytic estimator fit."""
+    assert isinstance(meta_res, MetaResult)
+
+
+@pytest.mark.performance_estimators
+def test_meta_fit_performance(meta_res, signal_masks, simulatedata_cbma):
+    """Test meta-analytic estimator fit performance."""
+    _, (ground_truth_foci, _) = simulatedata_cbma
+    mask = meta_res.masker.mask_img
+    ground_truth_foci_ijks = [tuple(mm2vox(focus, mask.affine)) for focus in ground_truth_foci]
+    sig_idx, nonsig_idx = [
+        meta_res.masker.transform(img).astype(bool).squeeze() for img in signal_masks
+    ]
+
+    # all estimators generate p-values
+    p_array = meta_res.get_map("p", return_type="array")
+
+    # poor performer(s)
+    if (
+        isinstance(meta_res.estimator, ale.ALE)
+        and isinstance(meta_res.estimator.kernel_transformer, kernel.KDAKernel)
+        and meta_res.estimator.get_params().get("null_method") == "approximate"
+    ):
+        good_sensitivity = True
+        good_specificity = False
+    elif (
+        isinstance(meta_res.estimator, ale.ALE)
+        and isinstance(meta_res.estimator.kernel_transformer, kernel.KDAKernel)
+        and "montecarlo" in meta_res.estimator.get_params().get("null_method")
+    ):
+        good_sensitivity = False
+        good_specificity = True
+    elif (
+        isinstance(meta_res.estimator, ale.ALE)
+        and isinstance(meta_res.estimator.kernel_transformer, kernel.KDAKernel)
+        and "montecarlo" in meta_res.estimator.get_params().get("null_method")
+    ):
+        good_sensitivity = False
+        good_specificity = True
+    elif (
+        isinstance(meta_res.estimator, ale.ALE)
+        and isinstance(meta_res.estimator.kernel_transformer, kernel.KDAKernel)
+        and meta_res.estimator.get_params().get("null_method") == "approximate"
+    ):
+        good_sensitivity = True
+        good_specificity = False
+    elif (
+        isinstance(meta_res.estimator, mkda.MKDADensity)
+        and isinstance(meta_res.estimator.kernel_transformer, kernel.ALEKernel)
+        and meta_res.estimator.get_params().get("null_method") != "reduced_montecarlo"
+    ):
+        good_sensitivity = False
+        good_specificity = True
+    else:
+        good_sensitivity = True
+        good_specificity = True
+
+    _check_p_values(
+        p_array,
+        meta_res.masker,
+        sig_idx,
+        nonsig_idx,
+        ALPHA,
+        ground_truth_foci_ijks,
+        n_iters=None,
+        good_sensitivity=good_sensitivity,
+        good_specificity=good_specificity,
+    )
+
+
+@pytest.mark.performance_smoke
+def test_corr_transform_smoke(meta_cres_small):
+    """Smoke test for corrector transform."""
+    assert isinstance(meta_cres_small, MetaResult)
+
+
+@pytest.mark.performance_correctors
+def test_corr_transform_performance(meta_cres, corr, signal_masks, simulatedata_cbma):
+    """Test corrector transform performance."""
+    _, (ground_truth_foci, _) = simulatedata_cbma
+    mask = meta_cres.masker.mask_img
+    ground_truth_foci_ijks = [tuple(mm2vox(focus, mask.affine)) for focus in ground_truth_foci]
+    sig_idx, nonsig_idx = [
+        meta_cres.masker.transform(img).astype(bool).squeeze() for img in signal_masks
+    ]
+
+    p_array = meta_cres.maps.get("p")
+    if p_array is None or corr.method == "montecarlo":
+        p_array = 10 ** -meta_cres.maps.get("logp_level-voxel_corr-FWE_method-montecarlo")
+
+    n_iters = corr.parameters.get("n_iters")
+
+    # ALE with MKDA kernel with montecarlo correction
+    # combination gives poor performance
+    if (
+        isinstance(meta_cres.estimator, ale.ALE)
+        and isinstance(meta_cres.estimator.kernel_transformer, kernel.MKDAKernel)
+        and meta_cres.estimator.get_params().get("null_method") == "approximate"
+        and corr.method != "montecarlo"
+    ):
+        good_sensitivity = True
+        good_specificity = False
+    elif (
+        isinstance(meta_cres.estimator, ale.ALE)
+        and isinstance(meta_cres.estimator.kernel_transformer, kernel.MKDAKernel)
+        and "montecarlo" in meta_cres.estimator.get_params().get("null_method")
+    ):
+        good_sensitivity = False
+        good_specificity = True
+    elif (
+        isinstance(meta_cres.estimator, ale.ALE)
+        and isinstance(meta_cres.estimator.kernel_transformer, kernel.MKDAKernel)
+        and meta_cres.estimator.get_params().get("null_method") == "approximate"
+        and corr.method == "montecarlo"
+    ):
+        good_sensitivity = False
+        good_specificity = True
+    elif (
+        isinstance(meta_cres.estimator, ale.ALE)
+        and isinstance(meta_cres.estimator.kernel_transformer, kernel.KDAKernel)
+        and (
+            "montecarlo" in meta_cres.estimator.get_params().get("null_method")
+            or (
+                meta_cres.estimator.get_params().get("null_method") == "approximate"
+                and corr.method == "montecarlo"
+            )
+        )
+    ):
+        good_sensitivity = False
+        good_specificity = True
+    elif (
+        isinstance(meta_cres.estimator, ale.ALE)
+        and isinstance(meta_cres.estimator.kernel_transformer, kernel.KDAKernel)
+        and meta_cres.estimator.get_params().get("null_method") == "approximate"
+    ):
+        good_sensitivity = True
+        good_specificity = False
+    elif (
+        isinstance(meta_cres.estimator, mkda.MKDADensity)
+        and isinstance(meta_cres.estimator.kernel_transformer, kernel.ALEKernel)
+        and meta_cres.estimator.get_params().get("null_method") != "reduced_montecarlo"
+        and corr.method != "montecarlo"
+    ):
+        good_sensitivity = False
+        good_specificity = True
+    else:
+        good_sensitivity = True
+        good_specificity = True
+
+    _check_p_values(
+        p_array,
+        meta_cres.masker,
+        sig_idx,
+        nonsig_idx,
+        ALPHA,
+        ground_truth_foci_ijks,
+        n_iters=n_iters,
+        good_sensitivity=good_sensitivity,
+        good_specificity=good_specificity,
+    )

nimare/tests/test_extract.py

@@ -0,0 +1,46 @@
+"""Test nimare.extract."""
+
+import os
+from glob import glob
+
+import nimare
+
+
+def test_fetch_neurosynth(tmp_path_factory):
+    """Smoke test for extract.fetch_neurosynth.
+
+    Taken from the Neurosynth Python package.
+    """
+    tmpdir = tmp_path_factory.mktemp("test_fetch_neurosynth")
+    data_files = nimare.extract.fetch_neurosynth(
+        data_dir=tmpdir,
+        version="7",
+        overwrite=False,
+        source="abstract",
+        vocab="terms",
+    )
+    files = glob(os.path.join(tmpdir, "neurosynth", "*"))
+    assert len(files) == 4
+
+    # One set of files found
+    assert isinstance(data_files, list)
+    assert len(data_files) == 1
+
+
+def test_fetch_neuroquery(tmp_path_factory):
+    """Smoke test for extract.fetch_neuroquery."""
+    tmpdir = tmp_path_factory.mktemp("test_fetch_neuroquery")
+    data_files = nimare.extract.fetch_neuroquery(
+        data_dir=tmpdir,
+        version="1",
+        overwrite=False,
+        source="abstract",
+        vocab="neuroquery7547",
+        type="count",
+    )
+    files = glob(os.path.join(tmpdir, "neuroquery", "*"))
+    assert len(files) == 4
+
+    # One set of files found
+    assert isinstance(data_files, list)
+    assert len(data_files) == 1