brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/metrics/error_consistency/metric.py

@@ -0,0 +1,93 @@
+import itertools
+
+import numpy as np
+
+from brainscore_core.metrics import Metric, Score
+from brainscore_vision.metric_helpers.transformations import apply_aggregate
+
+
+class ErrorConsistency(Metric):
+    """
+    Computes the error consistency using :meth:`~brainscore.metrics.error_consistency.cohens_kappa`
+    implemented in Geirhos et al., 2020 https://arxiv.org/abs/2006.16736
+    """
+
+    def __call__(self, source, target) -> Score:
+        assert len(set(source['stimulus_id'].values)) == len(set(target['stimulus_id'].values))
+        # https://github.com/bethgelab/model-vs-human/blob/745046c4d82ff884af618756bd6a5f47b6f36c45/modelvshuman/plotting/analyses.py#L161
+        subject_scores = []
+        for subject in self.extract_subjects(target):
+            for condition in sorted(set(target['condition'].values)):
+                this_source = source.sel(condition=condition)
+                this_target = target.sel(subject=subject, condition=condition)
+                subject_score = self.compare_single_subject(this_source, this_target)
+                subject_score = subject_score.expand_dims('subject').expand_dims('condition')
+                subject_score['subject'] = [subject]
+                subject_score['condition'] = [condition]
+                subject_scores.append(subject_score)
+        subject_scores = Score.merge(*subject_scores)
+        subject_scores = apply_aggregate(aggregate_fnc=self.aggregate, values=subject_scores)
+        return subject_scores
+
+    @classmethod
+    def aggregate(cls, scores):
+        mean = scores.mean('condition').mean('subject')
+        # This deviates from the original paper which did not deal with scores < 0
+        # (neither for a single subject/condition, nor in the aggregate).
+        # We here avoid negative scores, so that we comply with all 0 <= scores <= 1.
+        score = abs(mean)
+        # estimate error; but note that the original paper did not have error estimates
+        score.attrs['error'] = scores.std(['condition', 'subject'])
+        return score
+
+    def ceiling(self, assembly):
+        """
+        Computes subject consistency by comparing each subject to all other subjects.
+        """
+        subjects = self.extract_subjects(assembly)
+        subject_scores = []
+        for subject1, subject2 in itertools.combinations(subjects, 2):
+            for condition in sorted(set(assembly['condition'].values)):
+                subject1_assembly = assembly.sel(subject=subject1, condition=condition)
+                subject2_assembly = assembly.sel(subject=subject2, condition=condition)
+                pairwise_score = self.compare_single_subject(subject1_assembly, subject2_assembly)
+                pairwise_score = pairwise_score.expand_dims('subject').expand_dims('condition')
+                pairwise_score['subject_left'] = 'subject', [subject1]
+                pairwise_score['subject_right'] = 'subject', [subject2]
+                pairwise_score['condition'] = [condition]
+                subject_scores.append(Score(pairwise_score))
+        subject_scores = Score.merge(*subject_scores)
+        subject_scores = apply_aggregate(aggregate_fnc=self.aggregate, values=subject_scores)
+        return subject_scores
+
+    def extract_subjects(self, assembly):
+        return list(sorted(set(assembly['subject'].values)))
+
+    def compare_single_subject(self, source, target):
+        assert len(source['stimulus_id']) == len(target['stimulus_id'])
+        source = source.sortby('stimulus_id')
+        target = target.sortby('stimulus_id')
+        assert all(source['stimulus_id'].values == target['stimulus_id'].values)
+
+        correct_source = source.values == source['truth'].values
+        correct_target = target.values == target['truth'].values
+        accuracy_source = np.mean(correct_source)
+        accuracy_target = np.mean(correct_target)
+
+        expected_consistency = accuracy_source * accuracy_target + (1 - accuracy_source) * (1 - accuracy_target)
+        observed_consistency = (correct_source == correct_target).sum() / len(target)
+        error_consistency = cohens_kappa(expected_consistency=expected_consistency,
+                                         observed_consistency=observed_consistency)
+        error_consistency = error_consistency
+        return Score(error_consistency)
+
+
+def cohens_kappa(expected_consistency, observed_consistency):
+    """
+    Computes the error consistency using Cohen's Kappa.
+    Cohen, 1960 https://doi.org/10.1177%2F001316446002000104
+    """
+    # https://github.com/bethgelab/model-vs-human/blob/745046c4d82ff884af618756bd6a5f47b6f36c45/modelvshuman/plotting/analyses.py#L147-L158
+    assert 0.0 <= expected_consistency <= 1.0
+    assert 0.0 <= observed_consistency <= 1.0
+    return (observed_consistency - expected_consistency) / (1.0 - expected_consistency)

brainscore_vision/metrics/error_consistency/test.py

@@ -0,0 +1,39 @@
+import numpy as np
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_metric
+
+
+def test_score():
+    assembly = _make_data()
+    metric = load_metric('error_consistency')
+    score = metric(assembly.sel(subject='A'), assembly)
+    assert score == approx(0.2)
+
+
+def test_has_error():
+    assembly = _make_data()
+    metric = load_metric('error_consistency')
+    score = metric(assembly.sel(subject='A'), assembly)
+    assert hasattr(score, 'error')
+
+
+def test_raw_subjects():
+    assembly = _make_data()
+    metric = load_metric('error_consistency')
+    score = metric(assembly.sel(subject='A'), assembly)
+    subject_scores = score.raw.mean('condition')
+    assert subject_scores.sel(subject='A') == 1
+
+
+def _make_data():
+    return BehavioralAssembly(['dog', 'cat', 'chair', 'cat', 'dog', 'dog', 'dog', 'dog', 'chair',  # subject A
+                               'cat', 'cat', 'chair', 'cat', 'dog', 'cat', 'chair', 'cat', 'cat',  # subject B
+                               'dog', 'cat', 'chair', 'dog', 'cat', 'chair', 'dog', 'cat', 'chair'  # subject C
+                               ],
+                              coords={'stimulus_id': ('presentation', np.resize(np.arange(9), 9 * 3)),
+                                      'truth': ('presentation', np.resize(['dog', 'cat', 'chair'], 9 * 3)),
+                                      'condition': ('presentation', np.resize([1, 1, 1, 2, 2, 2, 3, 3, 3], 9 * 3)),
+                                      'subject': ('presentation', ['A'] * 9 + ['B'] * 9 + ['C'] * 9)},
+                              dims=['presentation'])

brainscore_vision/metrics/i1i2/__init__.py

@@ -0,0 +1,16 @@
+from brainscore_vision import metric_registry
+from .metric import I1, I2n
+
+metric_registry['i1'] = I1
+metric_registry['i2n'] = I2n
+
+BIBTEX = """@article{rajalingham2018large,
+  title={Large-scale, high-resolution comparison of the core visual object recognition behavior of humans, monkeys, and state-of-the-art deep artificial neural networks},
+  author={Rajalingham, Rishi and Issa, Elias B and Bashivan, Pouya and Kar, Kohitij and Schmidt, Kailyn and DiCarlo, James J},
+  journal={Journal of Neuroscience},
+  volume={38},
+  number={33},
+  pages={7255--7269},
+  year={2018},
+  publisher={Soc Neuroscience}
+}"""

brainscore_vision/metrics/i1i2/metric.py

@@ -0,0 +1,299 @@
+import itertools
+import logging
+from collections import Counter
+
+import numpy as np
+import scipy.stats
+import xarray
+from numpy.random.mtrand import RandomState
+from scipy.stats import pearsonr
+
+from brainio.assemblies import walk_coords, DataAssembly
+from brainscore_core.metrics import Metric, Score
+from brainscore_vision.metric_helpers.transformations import apply_aggregate
+from brainscore_vision.utils import fullname
+
+
+# only the implementation for i2n has been verified.
+
+def I1(*args, **kwargs):
+    return _I(*args, collapse_distractors=True, normalize=False, **kwargs)
+
+
+def I1n(*args, **kwargs):
+    return _I(*args, collapse_distractors=True, normalize=True, **kwargs)
+
+
+def I2(*args, **kwargs):
+    return _I(*args, collapse_distractors=False, normalize=False, **kwargs)
+
+
+def I2n(*args, **kwargs):
+    return _I(*args, collapse_distractors=False, normalize=True, **kwargs)
+
+
+def O1(*args, **kwargs):
+    return _O(*args, collapse_distractors=True, normalize=False, **kwargs)
+
+
+def O2(*args, **kwargs):
+    return _O(*args, collapse_distractors=False, normalize=False, **kwargs)
+
+
+def _o2(assembly):
+    i2n = I2n()
+    false_alarm_rates = i2n.target_distractor_scores(assembly)
+    false_alarm_rates_object = false_alarm_rates.groupby('truth').mean('presentation')
+    false_alarm_rates_object = false_alarm_rates_object.rename({'truth': 'task_left', 'choice': 'task_right'})
+    # hit rates are one minus the flipped false alarm rates, e.g. HR(dog, cat) = 1 - FAR(cat, dog)
+    hit_rates_object = 1 - false_alarm_rates_object.rename({'task_left': 'task_right', 'task_right': 'task_left'})
+    dprime_false_alarms_rates_object = xarray.apply_ufunc(i2n.z_score, false_alarm_rates_object)
+    dprime_hit_rates_object = xarray.apply_ufunc(i2n.z_score, hit_rates_object)
+    o2 = dprime_hit_rates_object - dprime_false_alarms_rates_object
+    return o2
+
+
+class _Behavior_Metric(Metric):
+    """
+    Rajalingham & Issa et al., 2018 http://www.jneurosci.org/content/early/2018/07/13/JNEUROSCI.0388-18.2018
+    modified by Schrimpf & Kubilius et al., 2018 https://www.biorxiv.org/content/early/2018/09/05/407007:
+        - Rajalingham et al. generated model trials by using different train-test splits.
+            This implementation fixes the train-test split, and thus computes a fixed response matrix without trials.
+        - for computing dprime scores, Rajalingham et al. computed the false-alarms rate across the flat vector of all
+            distractor images. This implementation computes the false-alarms rate per object, and then takes the mean.
+    """
+
+    def __init__(self, collapse_distractors, normalize=False, repetitions=2):
+        super().__init__()
+        self._collapse_distractors = collapse_distractors
+        self._normalize = normalize
+        self._repetitions = repetitions
+        self._logger = logging.getLogger(fullname(self))
+
+    def __call__(self, source_probabilities, target) -> Score:
+        return self._repeat(lambda random_state:
+                            self._call_single(source_probabilities, target, random_state=random_state))
+
+    def _call_single(self, source_probabilities, target, random_state):
+        self.add_source_meta(source_probabilities, target)
+        source_response_matrix = self.target_distractor_scores(source_probabilities)
+        source_response_matrix = self.dprimes(source_response_matrix)
+        if self._collapse_distractors:
+            source_response_matrix = self.collapse_distractors(source_response_matrix)
+
+        target_half = self.generate_halves(target, random_state=random_state)[0]
+        target_response_matrix = self.build_response_matrix_from_responses(target_half)
+        target_response_matrix = self.dprimes(target_response_matrix)
+        if self._collapse_distractors:
+            target_response_matrix = self.collapse_distractors(target_response_matrix)
+
+        correlation = self.correlate(source_response_matrix, target_response_matrix,
+                                     collapse_distractors=self._collapse_distractors)
+        return correlation
+
+    def ceiling(self, assembly, skipna=False):
+        return self._repeat(lambda random_state:
+                            self.compute_ceiling(assembly, random_state=random_state, skipna=skipna))
+
+    def compute_ceiling(self, assembly, random_state, skipna=False):
+        dprime_halves = []
+        for half in self.generate_halves(assembly, random_state=random_state):
+            half = self.build_response_matrix_from_responses(half)
+            half = self.dprimes(half)
+            if self._collapse_distractors:
+                half = self.collapse_distractors(half)
+            dprime_halves.append(half)
+        return self.correlate(*dprime_halves, skipna=skipna, collapse_distractors=self._collapse_distractors)
+
+    def build_response_matrix_from_responses(self, responses):
+        num_choices = [(image_id, choice) for image_id, choice in
+                       zip(responses['stimulus_id'].values, responses.values)]
+        num_choices = Counter(num_choices)
+        num_objects = [[(image_id, sample_obj), (image_id, dist_obj)] for image_id, sample_obj, dist_obj in zip(
+            responses['stimulus_id'].values, responses['sample_obj'].values, responses['dist_obj'].values)]
+        num_objects = Counter(itertools.chain(*num_objects))
+
+        choices = np.unique(responses)
+        image_ids, indices = np.unique(responses['stimulus_id'], return_index=True)
+        truths = responses['truth'].values[indices]
+        image_dim = responses['stimulus_id'].dims
+        coords = {**{coord: (dims, value) for coord, dims, value in walk_coords(responses)},
+                  **{'choice': ('choice', choices)}}
+        coords = {coord: (dims, value if dims != image_dim else value[indices])  # align image_dim coords with indices
+                  for coord, (dims, value) in coords.items()}
+        response_matrix = np.zeros((len(image_ids), len(choices)))
+        for (image_index, image_id), (choice_index, choice) in itertools.product(
+                enumerate(image_ids), enumerate(choices)):
+            if truths[image_index] == choice:  # object == choice, ignore
+                p = np.nan
+            else:
+                # divide by number of times where object was one of the two choices (target or distractor)
+                p = (num_choices[(image_id, choice)] / num_objects[(image_id, choice)]) \
+                    if num_objects[(image_id, choice)] > 0 else np.nan
+            response_matrix[image_index, choice_index] = p
+        response_matrix = DataAssembly(response_matrix, coords=coords, dims=responses.dims + ('choice',))
+        return response_matrix
+
+    def dprimes(self, response_matrix, cap=5):
+        dprime_scores = self.dprime(response_matrix)
+        dprime_scores_clipped = dprime_scores.clip(-cap, cap)
+        dprime_scores_clipped = type(dprime_scores)(dprime_scores_clipped)  # make sure type is preserved
+        if not self._normalize:
+            return dprime_scores_clipped
+        else:
+            dprime_scores_normalized = self.subtract_mean(dprime_scores_clipped)
+            return dprime_scores_normalized
+
+    def target_distractor_scores(self, object_probabilities):
+        cached_object_probabilities = self._build_index(object_probabilities, ['stimulus_id', 'choice'])
+
+        def apply(p_choice, stimulus_id, truth, choice, **_):
+            if truth == choice:  # object == choice, ignore
+                return np.nan
+            # probability that something else was chosen rather than object (p_choice == p_distractor after above check)
+            p_object = cached_object_probabilities[(stimulus_id, truth)]
+            p = p_choice / (p_choice + p_object)
+            return p
+
+        result = object_probabilities.multi_dim_apply(['stimulus_id', 'choice'], apply)
+        return result
+
+    def z_score(self, value):
+        return scipy.stats.norm.ppf(value)
+
+    def subtract_mean(self, scores):
+        result = scores.multi_dim_apply(['truth', 'choice'], lambda group, **_: group - np.nanmean(group))
+        return result
+
+    @classmethod
+    def aggregate(cls, scores):
+        score = scores.mean('split')
+        score.attrs['error'] = scores.std('split')
+        return score
+
+    def generate_halves(self, assembly, random_state):
+        indices = list(range(len(assembly)))
+        random_state.shuffle(indices)
+        halves = assembly[indices[:int(len(indices) / 2)]], assembly[indices[int(len(indices) / 2):]]
+        return halves
+
+    def _build_index(self, assembly, coords):
+        np.testing.assert_array_equal(list(itertools.chain(*[assembly[coord].dims for coord in coords])), assembly.dims)
+        aligned_coords = itertools.product(*[assembly[coord].values for coord in coords])
+        result = {}
+        for coord_values, value in zip(aligned_coords, assembly.values.flatten()):
+            if coord_values in result:  # if there are duplicates, make it a list
+                previous_value = result[coord_values]
+                previous_value = (previous_value if isinstance(previous_value, list) else [previous_value])
+                result[coord_values] = previous_value + [value]
+            else:
+                result[coord_values] = value
+        return result
+
+    def _repeat(self, func):
+        random_state = self._initialize_random_state()
+        repetitions = list(range(self._repetitions))
+        scores = [func(random_state=random_state) for repetition in repetitions]
+        score = Score(scores, coords={'split': repetitions}, dims=['split'])
+        return apply_aggregate(self.aggregate, score)
+
+    def _initialize_random_state(self):
+        return RandomState(seed=0)  # fix the seed here so that the same halves are produced for score and ceiling
+
+    def add_source_meta(self, source_probabilities, target):
+        image_meta = {image_id: meta_value for image_id, meta_value in
+                      zip(target['stimulus_id'].values, target['truth'].values)}
+        meta_values = [image_meta[image_id] for image_id in source_probabilities['stimulus_id'].values]
+        source_probabilities['truth'] = 'presentation', meta_values
+
+
+class _I(_Behavior_Metric):
+    """
+    Rajalingham & Issa et al., 2018 http://www.jneurosci.org/content/early/2018/07/13/JNEUROSCI.0388-18.2018
+    modified by Schrimpf & Kubilius et al., 2018 https://www.biorxiv.org/content/early/2018/09/05/407007:
+        - Rajalingham et al. generated model trials by using different train-test splits.
+            This implementation fixes the train-test split, and thus computes a fixed response matrix without trials.
+        - for computing dprime scores, Rajalingham et al. computed the false-alarms rate across the flat vector of all
+            distractor images. This implementation computes the false-alarms rate per object, and then takes the mean.
+    """
+
+    def collapse_distractors(self, response_matrix):
+        return response_matrix.mean(dim='choice', skipna=True)
+
+    def dprime(self, response_matrix):
+        truth_choice_values = self._build_index(response_matrix, ['truth', 'choice'])
+
+        def apply(false_alarms_rate_images, choice, truth, **_):
+            hit_rate = 1 - false_alarms_rate_images
+            inverse_choice = truth_choice_values[(choice, truth)]
+            false_alarms_rate_objects = np.nanmean(inverse_choice)
+            dprime = self.z_score(hit_rate) - self.z_score(false_alarms_rate_objects)
+            return dprime
+
+        result = response_matrix.multi_dim_apply(['stimulus_id', 'choice'], apply)
+        return result
+
+    @classmethod
+    def correlate(cls, source_response_matrix, target_response_matrix, skipna=False, collapse_distractors=False):
+        # align
+        if collapse_distractors:
+            source_response_matrix = source_response_matrix.sortby('stimulus_id')
+            target_response_matrix = target_response_matrix.sortby('stimulus_id')
+        else:
+            source_response_matrix = source_response_matrix.sortby('stimulus_id').sortby('choice')
+            target_response_matrix = target_response_matrix.sortby('stimulus_id').sortby('choice')
+            assert all(source_response_matrix['choice'].values == target_response_matrix['choice'].values)
+        assert all(source_response_matrix['stimulus_id'].values == target_response_matrix['stimulus_id'].values)
+        # flatten and mask out NaNs
+        source, target = source_response_matrix.values.flatten(), target_response_matrix.values.flatten()
+        non_nan = ~np.isnan(target)
+        non_nan = np.logical_and(non_nan, (~np.isnan(source) if skipna else 1))
+        source, target = source[non_nan], target[non_nan]
+        assert not any(np.isnan(source))
+        correlation, p = pearsonr(source, target)
+        return correlation
+
+
+class _O(_Behavior_Metric):
+    """
+    Rajalingham & Issa et al., 2018 http://www.jneurosci.org/content/early/2018/07/13/JNEUROSCI.0388-18.2018
+    modified by Schrimpf & Kubilius et al., 2018 https://www.biorxiv.org/content/early/2018/09/05/407007:
+        - Rajalingham et al. generated model trials by using different train-test splits.
+            This implementation fixes the train-test split, and thus computes a fixed response matrix without trials.
+        - for computing dprime scores, Rajalingham et al. computed the false-alarms rate across the flat vector of all
+            distractor images. This implementation computes the false-alarms rate per object, and then takes the mean.
+    """
+
+    def collapse_distractors(self, response_matrix):
+        return response_matrix.mean(dim='task_left', skipna=True)
+
+    def dprime(self, response_matrix):
+        false_alarm_rates_object = response_matrix.groupby('truth').mean('presentation')
+        false_alarm_rates_object = false_alarm_rates_object.rename({'truth': 'task_left', 'choice': 'task_right'})
+        hit_rates_object = 1 - false_alarm_rates_object.rename({'task_left': 'task_right', 'task_right': 'task_left'})
+
+        dprime_false_alarms_rates_object = xarray.apply_ufunc(self.z_score, false_alarm_rates_object)
+        dprime_hit_rates_object = xarray.apply_ufunc(self.z_score, hit_rates_object)
+
+        result = dprime_hit_rates_object - dprime_false_alarms_rates_object
+        return result
+
+    @classmethod
+    def correlate(cls, source_response_matrix, target_response_matrix, skipna=False, collapse_distractors=False):
+        # align
+        if collapse_distractors:
+            source_response_matrix = source_response_matrix.sortby('task_right')
+            target_response_matrix = target_response_matrix.sortby('task_right')
+        else:
+            source_response_matrix = source_response_matrix.sortby('task_right').sortby('task_left')
+            target_response_matrix = target_response_matrix.sortby('task_right').sortby('task_left')
+            assert all(source_response_matrix['task_left'].values == target_response_matrix['task_left'].values)
+        assert all(source_response_matrix['task_right'].values == target_response_matrix['task_right'].values)
+        # flatten and mask out NaNs
+        source, target = source_response_matrix.values.flatten(), target_response_matrix.values.flatten()
+        non_nan = ~np.isnan(target)
+        non_nan = np.logical_and(non_nan, (~np.isnan(source) if skipna else 1))
+        source, target = source[non_nan], target[non_nan]
+        assert not any(np.isnan(source))
+        correlation, p = pearsonr(source, target)
+        return correlation

brainscore_vision/metrics/i1i2/requirements.txt

@@ -0,0 +1,2 @@
+numpy
+scipy

brainscore_vision/metrics/i1i2/test.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+import pytest
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_metric
+from brainscore_vision.benchmarks.rajalingham2018.benchmarks.benchmark import load_assembly
+
+
+@pytest.mark.private_access
+class TestI2N:
+    @pytest.mark.parametrize(['model', 'expected_score'], [
+        ('alexnet', .253),
+        ('resnet34', .37787),
+        ('resnet18', .3638),
+    ])
+    def test_model(self, model, expected_score):
+        # assemblies
+        objectome = load_assembly()
+        probabilities = Path(__file__).parent / 'test_resources' / f'{model}-probabilities.nc'
+        probabilities = BehavioralAssembly.from_files(
+            probabilities,
+            stimulus_set_identifier=objectome.attrs['stimulus_set_identifier'],
+            stimulus_set=objectome.attrs['stimulus_set'])
+        # metric
+        i2n = load_metric('i2n')
+        score = i2n(probabilities, objectome)
+        assert score == approx(expected_score, abs=0.005), f"expected {expected_score}, but got {score}"
+
+    def test_ceiling(self):
+        objectome = load_assembly()
+        i2n = load_metric('i2n')
+        ceiling = i2n.ceiling(objectome)
+        assert ceiling == approx(.4786, abs=.0064)
+        assert ceiling.attrs['error'] == approx(.00537, abs=.0015)

brainscore_vision/metrics/internal_consistency/__init__.py

@@ -0,0 +1,8 @@
+from brainscore_vision import metric_registry
+from .ceiling import InternalConsistency
+
+from brainscore_vision.metric_helpers.temporal import PerTime
+
+
+metric_registry['internal_consistency'] = InternalConsistency
+metric_registry['internal_consistency_temporal'] = lambda *args, **kwargs: PerTime(InternalConsistency(*args, **kwargs))

brainscore_vision/metrics/internal_consistency/ceiling.py

@@ -0,0 +1,127 @@
+import scipy.stats
+import xarray as xr
+from tqdm import tqdm
+
+from brainio.assemblies import walk_coords, DataAssembly
+from brainscore_core import Metric, Score
+from brainscore_vision.metric_helpers import Defaults as XarrayDefaults
+from brainscore_vision.metric_helpers.transformations import CrossValidationSingle
+from brainscore_vision.metric_helpers.xarray_utils import XarrayCorrelation
+from brainscore_vision.metrics import Ceiling
+
+
+class _SplitHalvesConsistency(Ceiling):
+    """
+    Computes the consistency within an assembly with repetitions,
+    by splitting the assembly in halves and computing consistency between the two halves.
+    """
+
+    class Defaults:
+        split_coord = 'repetition'
+
+    def __init__(self, consistency_metric, aggregate=None,
+                 split_coord=Defaults.split_coord, cross_validation_kwargs=None):
+        correction = SpearmanBrownCorrection()
+        self._consistency = self.SplitHalfWrapper(split_coord=split_coord,
+                                                  consistency_metric=consistency_metric, correction=correction)
+        self._aggregate = aggregate
+        cross_validation_defaults = dict(train_size=0.5, split_coord=split_coord,
+                                         stratification_coord=None, unique_split_values=True)
+        cross_validation_kwargs = {**cross_validation_defaults, **(cross_validation_kwargs or {})}
+        self._cross_validation = CrossValidationSingle(**cross_validation_kwargs)
+
+    def __call__(self, assembly):
+        return self._cross_validation(assembly, apply=self._consistency, aggregate=self._aggregate)
+
+    class SplitHalfWrapper:
+        def __init__(self, split_coord, consistency_metric: Metric, correction):
+            self._split_coord = split_coord
+            self._consistency_metric = consistency_metric
+            self._correction = correction
+
+        def __call__(self, half1, half2):
+            half1, half2 = self._average_repetitions(half1), self._average_repetitions(half2)
+            consistency = self._consistency_metric(half1, half2)
+            corrected_consistency = self._correction(consistency, n=2)
+            return corrected_consistency
+
+        def _average_repetitions(self, assembly):
+            repetition_dims = assembly[self._split_coord].dims
+            nonrepetition_coords = [coord for coord, dims, values in walk_coords(assembly)
+                                    if dims == repetition_dims and coord != self._split_coord]
+            average = assembly.multi_groupby(nonrepetition_coords).mean(dim=repetition_dims)
+            return average
+
+
+class PearsonCorrelation:
+    """
+    Computes the Pearson r between two halves of an assembly.
+    """
+
+    def __init__(self, stimulus_coord=XarrayDefaults.stimulus_coord,
+                 neuroid_dim=XarrayDefaults.neuroid_dim, neuroid_coord=XarrayDefaults.neuroid_coord):
+        correlation = scipy.stats.pearsonr
+        self._correlation = XarrayCorrelation(correlation, correlation_coord=stimulus_coord,
+                                              neuroid_coord=neuroid_coord)
+        self._neuroid_dim = neuroid_dim
+
+    def __call__(self, half1, half2):
+        return self._correlation(half1, half2)
+
+    def aggregate(self, scores):
+        return scores.median(dim=self._neuroid_dim)
+
+
+class InternalConsistency(Ceiling):
+    def __init__(self,
+                 consistency_metric: Metric = None,
+                 aggregate=None,
+                 split_coord=_SplitHalvesConsistency.Defaults.split_coord):
+        """
+        Creates a class to estimate the ceiling for a given assembly, based on split-half consistencies.
+
+        :param consistency_metric: The metric to compare two halves. :class:`~.PearsonCorrelation` by default.
+        :param aggregate: An optional function to aggregate the output of the `consistency_metric`.
+        :param split_coord: Over which coordinate to split to obtain the two halves.
+        """
+        if not consistency_metric:
+            consistency_metric = PearsonCorrelation(
+                stimulus_coord=XarrayDefaults.stimulus_coord, neuroid_dim=XarrayDefaults.neuroid_dim,
+                neuroid_coord=XarrayDefaults.neuroid_coord)
+            aggregate = consistency_metric.aggregate
+        self._consistency = _SplitHalvesConsistency(consistency_metric=consistency_metric,
+                                                    aggregate=aggregate,
+                                                    split_coord=split_coord)
+
+    def __call__(self, assembly: DataAssembly) -> Score:
+        return self._consistency(assembly)
+
+
+class SpearmanBrownCorrection:
+    """
+    Applies Spearman-Brown correction to all passed values.
+    """
+
+    def __call__(self, correlations, n):
+        return xr.apply_ufunc(lambda correlation: self.correct(correlation, n), correlations)
+
+    def correct(self, correlation, n):
+        return n * correlation / (1 + (n - 1) * correlation)
+
+
+class TemporalCeiling:
+    def __init__(self, ceiling):
+        """
+        :param ceiling: the ceiling to use per time-bin
+        """
+        self.ceiling = ceiling
+
+    def __call__(self, assembly: DataAssembly) -> Score:
+        ceilings = []
+        for time_bin in tqdm(assembly['time_bin'].values, desc='time-ceiling'):
+            ceiling = self.ceiling(assembly.sel(time_bin=time_bin))
+            ceiling = ceiling.expand_dims('time_bin')
+            ceiling['time_bin'] = [str(time_bin)]
+            ceilings.append(ceiling)
+        ceiling = Score.merge(*ceilings)
+        return ceiling

brainscore_vision/metrics/internal_consistency/requirements.txt

@@ -0,0 +1 @@
+scipy