brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/metrics/accuracy_distance/metric.py

@@ -0,0 +1,109 @@
+import itertools
+
+import numpy as np
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_core import Metric
+from brainscore_vision.metrics import Score
+from brainscore_vision.metric_helpers.transformations import apply_aggregate
+
+
+class AccuracyDistance(Metric):
+    """
+    Computes the accuracy distance using the relative distance between the
+    source and target accuracies, adjusted for the maximum possible
+    difference between the two accuracies. By default, the distance is computed
+    from a single accuracy score on the entire BehavioralAssembly. However,
+    the distance can also be computed on a condition-wise basis using the
+    'variables' argument. The advantage of the condition-wise approach is that
+    it can separate two models with identical overall accuracy if one exhibits a
+    more target-like pattern of performance across conditions.
+    """
+    def __call__(self, source: BehavioralAssembly, target:
+            BehavioralAssembly, variables: tuple=()) -> Score:
+        """Target should be the entire BehavioralAssembly, containing truth values."""
+
+        subjects = self.extract_subjects(target)
+        subject_scores = []
+        for subject in subjects:
+            subject_assembly = target.sel(subject=subject)
+
+            # compute single score across the entire dataset
+            if len(variables) == 0:
+                subject_score = self.compare_single_subject(source, subject_assembly)
+
+            # compute scores for each condition, then average
+            else:
+                cond_scores = []
+
+                # get iterator across all combinations of variables
+                if len(variables) == 1:
+                    conditions = set(subject_assembly[variables[0]].values)
+                    conditions = [[c] for c in conditions]  # to mimic itertools.product
+                else:
+                    conditions = itertools.product(
+                        *[set(subject_assembly[v].values) for v in variables])
+
+                # loop over conditions and compute scores
+                for cond in conditions:
+                    indexers = {v: cond[i] for i, v in enumerate(variables)}
+                    subject_cond_assembly = subject_assembly.sel(**indexers)
+                    source_cond_assembly = source.sel(**indexers)
+                    # to accomodate unbalanced designs, skip combinations of
+                    # variables that don't exist in both assemblies
+                    if len(subject_cond_assembly) and len(source_cond_assembly):
+                        cond_scores.append(self.compare_single_subject(
+                            source_cond_assembly, subject_cond_assembly))
+                subject_score = Score(np.mean(cond_scores))
+
+            subject_score = subject_score.expand_dims('subject')
+            subject_score['subject'] = 'subject', [subject]
+            subject_scores.append(subject_score)
+
+        subject_scores = Score.merge(*subject_scores)
+        subject_scores = apply_aggregate(aggregate_fnc=self.aggregate, values=subject_scores)
+        return subject_scores
+
+    @classmethod
+    def aggregate(cls, scores):
+        mean = scores.mean('subject')
+        score = abs(mean)  # avoid negative scores just in case
+        score.attrs['error'] = scores.std('subject')
+        return score
+
+    def compare_single_subject(self, source: BehavioralAssembly, target: BehavioralAssembly):
+        source = source.sortby('stimulus_id')
+        target = target.sortby('stimulus_id')
+        assert (target['stimulus_id'].values == source['stimulus_id'].values).all()
+
+        # .flatten() because models return lists of lists, and here we compare subject-by-subject
+        source_correct = source.values.flatten() == target['truth'].values
+        target_correct = target.values == target['truth'].values
+        source_mean = sum(source_correct) / len(source_correct)
+        target_mean = sum(target_correct) / len(target_correct)
+
+        maximum_distance = np.max([1 - target_mean, target_mean])
+        # get the proportion of the distance between the source and target accuracies, adjusted for the maximum possible
+        # difference between the two accuracies
+        relative_distance = 1 - np.abs(source_mean - target_mean) / maximum_distance
+
+        return Score(relative_distance)
+
+    def ceiling(self, assembly):
+        subjects = self.extract_subjects(assembly)
+        subject_scores = []
+        for subject1, subject2 in itertools.combinations(subjects, 2):
+            subject1_assembly = assembly.sel(subject=subject1)
+            subject2_assembly = assembly.sel(subject=subject2)
+            pairwise_score = self.compare_single_subject(subject1_assembly, subject2_assembly)
+            pairwise_score = pairwise_score.expand_dims('subject')
+            pairwise_score['subject_left'] = 'subject', [subject1]
+            pairwise_score['subject_right'] = 'subject', [subject2]
+            subject_scores.append(Score(pairwise_score))
+
+        subject_scores = Score.merge(*subject_scores)
+        subject_scores = apply_aggregate(aggregate_fnc=self.aggregate, values=subject_scores)
+        return subject_scores
+
+    def extract_subjects(self, assembly):
+        return list(sorted(set(assembly['subject'].values)))

brainscore_vision/metrics/accuracy_distance/test.py

@@ -0,0 +1,57 @@
+import numpy as np
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_metric
+
+
+def test_score():
+    assembly = _make_data()
+    metric = load_metric('accuracy_distance')
+    score = metric(assembly.sel(subject='A'), assembly)
+    assert score == approx(0.74074074)
+
+
+def test_score_single_variable():
+    assembly = _make_data()
+    metric = load_metric('accuracy_distance')
+    score = metric(assembly.sel(subject='C'), assembly, ('condition',))
+    assert score == approx(0.55555556)
+
+
+def test_score_multi_variable():
+    assembly = _make_data()
+    metric = load_metric('accuracy_distance')
+    score = metric(assembly.sel(subject='C'), assembly, ('condition','animacy'))
+    assert score == approx(0.55555556)
+
+
+def test_has_error():
+    assembly = _make_data()
+    metric = load_metric('accuracy_distance')
+    score = metric(assembly.sel(subject='A'), assembly)
+    assert hasattr(score, 'error')
+
+
+def test_raw_subjects():
+    assembly = _make_data()
+    metric = load_metric('accuracy_distance')
+    score = metric(assembly.sel(subject='A'), assembly)
+    subject_scores = score.raw
+    assert subject_scores.sel(subject='A') == 1.
+
+
+def _make_data():
+    # subject A is 5 / 9 = 0.55...% correct
+    # subject B is 4 / 9 = 0.44...% correct
+    # subject C is 9 / 9 = 100% correct
+    return BehavioralAssembly(['dog', 'cat', 'chair', 'cat', 'dog', 'dog', 'dog', 'dog', 'chair',  # subject A
+                               'cat', 'cat', 'chair', 'cat', 'dog', 'cat', 'chair', 'cat', 'cat',  # subject B
+                               'dog', 'cat', 'chair', 'dog', 'cat', 'chair', 'dog', 'cat', 'chair'  # subject C
+                               ],
+                              coords={'stimulus_id': ('presentation', np.resize(np.arange(9), 9 * 3)),
+                                      'truth': ('presentation', np.resize(['dog', 'cat', 'chair'], 9 * 3)),
+                                      'condition': ('presentation', np.resize([1, 1, 1, 2, 2, 2, 3, 3, 3], 9 * 3)),
+                                      'animacy': ('presentation', np.resize(['animate', 'animate', 'inanimate'], 9 * 3)),
+                                      'subject': ('presentation', ['A'] * 9 + ['B'] * 9 + ['C'] * 9)},
+                              dims=['presentation'])

brainscore_vision/metrics/baker_accuracy_delta/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import metric_registry
+from .metric import BakerAccuracyDelta
+
+metric_registry['baker_accuracy_delta'] = BakerAccuracyDelta

brainscore_vision/metrics/baker_accuracy_delta/metric.py

@@ -0,0 +1,94 @@
+from typing import List
+
+import numpy as np
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_core import Metric, Score
+
+# controls how many half-splits are averaged together to get human delta.
+HUMAN_SPLITS = 100
+
+
+class BakerAccuracyDelta(Metric):
+    def __init__(self, image_types: list):
+        self.image_types = image_types
+
+    def __call__(self, source: BehavioralAssembly, target: BehavioralAssembly):
+        model_delta = get_model_delta(source, self.image_types)
+        scores = []
+
+        # calculate score over average of 100 sub splits of human delta
+        for i in range(HUMAN_SPLITS):
+            # grab one half of the subjeects
+            random_state = np.random.RandomState(i)
+            num_subjects = len(set(target["subject"].values))
+            half1_subjects = random_state.choice(range(1, num_subjects), (num_subjects // 2), replace=False)
+            half1 = target[{'presentation': [subject in half1_subjects for subject in target['subject'].values]}]
+
+            human_delta = get_human_delta(half1, self.image_types)
+            score = max((1 - ((np.abs(human_delta - model_delta)) / human_delta)), 0)
+            scores.append(score)
+        score = np.mean(scores)
+        error = np.std(scores)
+
+        score = Score(score)
+        score.attrs['error'] = error
+        score.attrs['raw'] = scores
+        return score
+
+    def compute_ceiling(self, source: BehavioralAssembly, target: BehavioralAssembly, image_types: List[str]) -> float:
+        half_1_delta = get_human_delta(target=source, image_types=image_types)
+        half_2_delta = get_human_delta(target=target, image_types=image_types)
+        ceiling = max((1 - ((np.abs(half_1_delta - half_2_delta)) / half_2_delta)), 0)
+        return ceiling
+
+
+def extract_subjects(assembly):
+    return list(sorted(set(assembly['subject'].values)))
+
+
+def get_human_delta(target, image_types):
+    # calculate human accuracies for [whole, condition]
+    condition_scores_human = {}
+
+    # for whole condition, and other condition (frankenstein or fragmented)
+    for image_type in image_types:
+        scores = []
+
+        # get per subject accuracy
+        for subject in extract_subjects(target):
+            this_target = target.sel(image_type=image_type, subject=subject)
+            correct_count = np.count_nonzero(this_target.values)
+            accuracy = correct_count / len(this_target)
+            scores.append(accuracy)
+        condition_scores_human[image_type] = scores
+
+    # calculate 16-human pairwise accuracy:
+    condition = image_types[1]
+    delta_vector = [a_i - b_i for a_i, b_i in zip(condition_scores_human["w"], condition_scores_human[condition])]
+
+    # return mean of delta vector. This is equal to the mean of half of subject deltas (random half)
+    return np.mean(delta_vector)
+
+
+def get_model_delta(source, image_types):
+    condition_scores_model = []
+
+    # for whole condition, and other condition (frank or frag)
+    for image_type in image_types:
+
+        # raw network accuracy per category
+        scores = []
+        for category in sorted(set(source['animal'].values)):
+            this_source = source.sel(animal=category, image_type=image_type)
+            correct_count = (this_source.values == category).sum()
+            accuracy = correct_count / len(this_source[0])
+            scores.append(accuracy)
+
+        # overall accuracy, averaged over 9 categories
+        overall = np.mean(scores)
+        condition_scores_model.append(overall)
+
+    # return difference between whole and condition
+    model_delta = condition_scores_model[0] - condition_scores_model[1]
+    return model_delta

brainscore_vision/metrics/baker_accuracy_delta/requirements.txt

@@ -0,0 +1 @@
+numpy

brainscore_vision/metrics/baker_accuracy_delta/test.py

@@ -0,0 +1 @@
+# tests part of /benchmarks/baker2022/test.py, omitted here

brainscore_vision/metrics/cka/__init__.py

@@ -0,0 +1,14 @@
+from brainscore_vision import metric_registry
+from .metric import CKA, CKACrossValidated
+
+metric_registry['cka'] = CKA
+metric_registry['cka_cv'] = CKACrossValidated
+
+BIBTEX = """@inproceedings{kornblith2019similarity,
+  title={Similarity of neural network representations revisited},
+  author={Kornblith, Simon and Norouzi, Mohammad and Lee, Honglak and Hinton, Geoffrey},
+  booktitle={International conference on machine learning},
+  pages={3519--3529},
+  year={2019},
+  organization={PMLR}
+}"""

brainscore_vision/metrics/cka/metric.py

@@ -0,0 +1,105 @@
+import math
+
+import numpy as np
+
+from brainio.assemblies import DataAssembly
+from brainscore_core import Metric
+from brainscore_vision.metric_helpers import Defaults as XarrayDefaults
+from brainscore_vision.metric_helpers.transformations import TestOnlyCrossValidation, apply_aggregate
+from brainscore_vision.metrics import Score
+
+
+class CKACrossValidated(Metric):
+    """
+    Computes a cross-validated similarity index for the similarity between two assemblies
+    with centered kernel alignment (CKA).
+
+    Kornblith et al., 2019 http://proceedings.mlr.press/v97/kornblith19a/kornblith19a.pdf
+    """
+
+    def __init__(self, comparison_coord=XarrayDefaults.stimulus_coord, crossvalidation_kwargs=None):
+        self._metric = CKA(comparison_coord=comparison_coord)
+        crossvalidation_defaults = dict(test_size=.9)  # leave 10% out
+        crossvalidation_kwargs = {**crossvalidation_defaults, **(crossvalidation_kwargs or {})}
+        self._cross_validation = TestOnlyCrossValidation(**crossvalidation_kwargs)
+
+    def __call__(self, assembly1: DataAssembly, assembly2: DataAssembly) -> Score:
+        return self._cross_validation(assembly1, assembly2, apply=self._metric)
+
+
+class CKA(Metric):
+    """
+    Computes a similarity index for the similarity between two assemblies with centered kernel alignment (CKA).
+
+    Kornblith et al., 2019 http://proceedings.mlr.press/v97/kornblith19a/kornblith19a.pdf
+    """
+
+    def __init__(self, comparison_coord=XarrayDefaults.stimulus_coord):
+        self._comparison_coord = comparison_coord
+
+    def __call__(self, assembly1: DataAssembly, assembly2: DataAssembly) -> Score:
+        """
+        :param brainscore.assemblies.NeuroidAssembly assembly1:
+        :param brainscore.assemblies.NeuroidAssembly assembly2:
+        :return: brainscore.assemblies.DataAssembly
+        """
+        # ensure value order
+        assembly1 = assembly1.sortby(self._comparison_coord)
+        assembly2 = assembly2.sortby(self._comparison_coord)
+        assert (assembly1[self._comparison_coord].values == assembly2[self._comparison_coord].values).all()
+        # ensure dimensions order
+        dims = assembly1[self._comparison_coord].dims
+        np.testing.assert_array_equal(assembly2[self._comparison_coord].dims, dims)
+        assembly1 = assembly1.transpose(*(list(dims) + [dim for dim in assembly1.dims if dim not in dims]))
+        assembly2 = assembly2.transpose(*(list(dims) + [dim for dim in assembly2.dims if dim not in dims]))
+        similarity = linear_CKA(assembly1, assembly2)
+        return Score(similarity)
+
+
+def centering(K):
+    n = K.shape[0]
+    unit = np.ones([n, n])
+    I = np.eye(n)
+    H = I - unit / n
+
+    return np.dot(np.dot(H, K), H)
+    # HKH are the same with KH, KH is the first centering, H(KH) do the second time,
+    # results are the sme with one time centering
+    # return np.dot(H, K)  # KH
+
+
+def rbf(X, sigma=None):
+    GX = np.dot(X, X.T)
+    KX = np.diag(GX) - GX + (np.diag(GX) - GX).T
+    if sigma is None:
+        mdist = np.median(KX[KX != 0])
+        sigma = math.sqrt(mdist)
+    KX *= - 0.5 / (sigma * sigma)
+    KX = np.exp(KX)
+    return KX
+
+
+def kernel_HSIC(X, Y, sigma):
+    return np.sum(centering(rbf(X, sigma)) * centering(rbf(Y, sigma)))
+
+
+def linear_HSIC(X, Y):
+    L_X = np.dot(X, X.T)
+    L_Y = np.dot(Y, Y.T)
+    return np.sum(centering(L_X) * centering(L_Y))
+
+
+def linear_CKA(X, Y):
+    hsic = linear_HSIC(X, Y)
+    var1 = np.sqrt(linear_HSIC(X, X))
+    var2 = np.sqrt(linear_HSIC(Y, Y))
+
+    return hsic / (var1 * var2)
+
+
+def kernel_CKA(X, Y, sigma=None):
+    hsic = kernel_HSIC(X, Y, sigma)
+    var1 = np.sqrt(kernel_HSIC(X, X, sigma))
+    var2 = np.sqrt(kernel_HSIC(Y, Y, sigma))
+
+    return hsic / (var1 * var2)

brainscore_vision/metrics/cka/test.py

@@ -0,0 +1,28 @@
+import numpy as np
+from numpy.random import RandomState
+from pytest import approx
+
+from brainio.assemblies import DataAssembly
+from brainscore_vision import load_metric
+
+
+def _make_data():
+    rnd = RandomState(1)
+    return DataAssembly(rnd.uniform(size=(300, 10)), coords={
+        'stimulus_id': ('presentation', np.arange(300)), 'category': ('presentation', ['dummy'] * 300),
+        'neuroid_id': ('neuroid', np.arange(10)), 'region': ('neuroid', ['Vdummy'] * 10)},
+                        dims=['presentation', 'neuroid'])
+
+
+def test_identity():
+    cka = load_metric('cka')
+    assembly = _make_data()
+    similarity = cka(assembly, assembly)
+    assert similarity == approx(1)
+
+
+def test_crossvalidated_has_error():
+    assembly = _make_data()
+    ckacv = load_metric('cka_cv', crossvalidation_kwargs=dict(stratification_coord=None))
+    similarity = ckacv(assembly, assembly)
+    assert hasattr(similarity, 'error')

brainscore_vision/metrics/dimensionality/__init__.py

@@ -0,0 +1,13 @@
+from brainscore_vision import metric_registry
+from .metric import Dimensionality
+
+metric_registry['factor_dimensionality'] = Dimensionality
+
+BIBTEX = """@inproceedings{esser2020disentangling,
+  title={A disentangling invertible interpretation network for explaining latent representations},
+  author={Esser, Patrick and Rombach, Robin and Ommer, Bjorn},
+  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+  pages={9223--9232},
+  year={2020},
+  url={https://arxiv.org/abs/2004.13166}
+}"""

brainscore_vision/metrics/dimensionality/metric.py

@@ -0,0 +1,45 @@
+import numpy as np
+from brainscore_core import Metric, Score
+
+
+class Dimensionality(Metric):
+    def __init__(self, factor):
+        self.factor = factor #factor to return
+
+    def __call__(self, assembly1, assembly2) -> Score:
+        #each assembly should have 3 dimensions: (factor, sample_num, neuron_i)
+        dims_percent = self._dim_est(assembly1,assembly2)
+        factor_dim = dims_percent[self.factor]
+        return Score(factor_dim)
+
+    def _dim_est(self, za, zb):
+        score_by_factor = dict()
+        zall = np.concatenate([za,zb], 0)
+        mean = np.mean(zall, 0, keepdims=True) #mean and variance per neuron
+        var = np.sum(np.mean((zall-mean)*(zall-mean), 0))
+        for f in set(za["factor"].values):
+            za_factor = za[za.factor == f].values
+            zb_factor = zb[zb.factor == f].values
+            mean_f_factor = 0.5*(np.mean(za_factor, 0, keepdims=True)+np.mean(zb_factor, 0, keepdims=True))
+            cov_f_factor = np.mean((za_factor-mean_f_factor)*(zb_factor-mean_f_factor), 0)
+            raw_score_f_factor = np.sum(cov_f_factor)
+            score_by_factor[f] = raw_score_f_factor/var
+
+        assert "residual" not in score_by_factor
+        score_by_factor["residual"] = 1.0 #by default
+        score_names = score_by_factor.keys()
+        scores = np.fromiter(score_by_factor.values(),dtype=float)
+        dims_percent = self._softmax_dim(score_names,scores,za.shape[1])
+        return dims_percent
+
+    def _softmax_dim(self, score_names, scores, N):
+        m = np.max(scores)
+        e = np.exp(scores-m)
+        softmaxed = e / np.sum(e)
+        dim = N
+        dims = [int(s*dim) for s in softmaxed]
+        dims[-1] = dim - sum(dims[:-1])
+        dims_percent = {}
+        for name, i in zip(score_names,range(len(scores))):
+            dims_percent[name] = dims[i] / sum(dims)
+        return dims_percent

brainscore_vision/metrics/distribution_similarity/__init__.py

@@ -0,0 +1,14 @@
+from brainscore_vision import metric_registry
+from .metric import BootstrapDistributionSimilarity, NeuronalPropertyCeiling
+
+metric_registry['ks_similarity'] = BootstrapDistributionSimilarity
+metric_registry['property_ceiling'] = NeuronalPropertyCeiling
+
+BIBTEX = """@article{marques2021multi,
+  title={Multi-scale hierarchical neural network models that bridge from single neurons in the primate primary visual cortex to object recognition behavior},
+  author={Marques, Tiago and Schrimpf, Martin and DiCarlo, James J},
+  journal={bioRxiv},
+  pages={2021--03},
+  year={2021},
+  publisher={Cold Spring Harbor Laboratory}
+}"""

brainscore_vision/metrics/distribution_similarity/metric.py

@@ -0,0 +1,84 @@
+import numpy as np
+
+from brainio.assemblies import PropertyAssembly
+from brainscore_core.metrics import Metric, Score
+
+NSAMPLES = 1000
+
+
+def ks_similarity(p, q):
+    z1 = np.zeros_like(p)
+    z2 = np.zeros_like(p)
+    z1[0] = 1
+    z2[-1] = 1
+
+    kolm_raw = np.max(np.abs(np.cumsum(p) - np.cumsum(q)))
+    kolm_max1 = np.max(np.abs(np.cumsum(z1) - np.cumsum(q)))
+    kolm_max2 = np.max(np.abs(np.cumsum(z2) - np.cumsum(q)))
+    kolm_max = np.max(np.array([kolm_max1, kolm_max2]))
+    kolm_ceiled = kolm_raw / kolm_max
+
+    return 1 - kolm_ceiled
+
+
+class BootstrapDistributionSimilarity(Metric):
+    def __init__(self, property_name, similarity_func=ks_similarity, ns=NSAMPLES):
+        self.similarity_func = similarity_func
+        self.property_name = property_name
+        self.ns = ns
+
+    def __call__(self, model_property, data_property):
+        bins = data_property.attrs[self.property_name + '_bins']
+        data_property = data_property.loc[:, self.property_name].values
+        model_property = model_property.loc[:, self.property_name].values
+
+        data_property[data_property < bins[0]] = bins[0]
+        data_property[data_property > bins[-1]] = bins[-1]
+        model_property[model_property < bins[0]] = bins[0]
+        model_property[model_property > bins[-1]] = bins[-1]
+
+        n_neurons = data_property.shape[0]
+
+        data_hist = np.histogram(data_property, bins=bins)[0]
+        data_hist = data_hist / data_hist.sum()
+
+        model_hist = np.zeros((self.ns, data_hist.shape[0]))
+        dist_similarity = np.zeros(self.ns)
+
+        for s in range(self.ns):
+            sample = np.random.choice(model_property, n_neurons)
+            model_hist[s, :] = np.histogram(sample, bins=bins)[0]
+            model_hist[s, :] = model_hist[s, :] / model_hist[s, :].sum()
+            dist_similarity[s] = self.similarity_func(model_hist[s, :], data_hist)
+
+        if np.isnan(dist_similarity).sum() > NSAMPLES / 10:
+            center = 0
+            error = 0
+        else:
+            center = np.nanmean(dist_similarity)
+            error = np.nanstd(dist_similarity)
+
+        score = Score(center)
+        score.attrs['error'] = error
+        score.attrs[Score.RAW_VALUES_KEY] = dist_similarity
+
+        self.data_hist = data_hist
+        self.model_hist = model_hist
+        self.bins = bins
+
+        return score
+
+    def get_dists(self):
+        return self.data_hist, self.model_hist, self.bins
+
+
+class NeuronalPropertyCeiling:
+    def __init__(self, similarity_metric: Metric):
+        self.similarity_metric = similarity_metric
+
+    def __call__(self, assembly: PropertyAssembly) -> Score:
+        self.assembly = assembly
+        return self._ceiling(self.similarity_metric.property_name)
+
+    def _ceiling(self, identifier):
+        return self.similarity_metric(self.assembly, self.assembly)

brainscore_vision/metrics/distribution_similarity/test.py

@@ -0,0 +1,10 @@
+from pytest import approx
+
+from brainscore_vision import load_metric, load_dataset
+
+
+def test_identity_circularvariance():
+    assembly = load_dataset('Ringach2002')
+    metric = load_metric('ks_similarity', property_name='circular_variance')
+    score = metric(assembly, assembly)
+    assert score == approx(1, abs=0.05)

brainscore_vision/metrics/error_consistency/__init__.py

@@ -0,0 +1,13 @@
+from brainscore_vision import metric_registry
+from .metric import ErrorConsistency
+
+metric_registry['error_consistency'] = ErrorConsistency
+
+BIBTEX = """@article{geirhos2020beyond,
+  title={Beyond accuracy: quantifying trial-by-trial behaviour of CNNs and humans by measuring error consistency},
+  author={Geirhos, Robert and Meding, Kristof and Wichmann, Felix A},
+  journal={Advances in Neural Information Processing Systems},
+  volume={33},
+  pages={13890--13902},
+  year={2020}
+}"""