brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/metric_helpers/transformations.py

@@ -0,0 +1,379 @@
+import itertools
+import logging
+import math
+from collections import OrderedDict
+
+import numpy as np
+import xarray as xr
+from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit, KFold, StratifiedKFold
+from tqdm import tqdm
+
+from brainio.assemblies import DataAssembly
+from brainio.transform import subset
+from brainscore_vision.metric_helpers.utils import unique_ordered
+from brainscore_vision.metrics import Score
+from brainscore_vision.utils import fullname
+
+
+def apply_aggregate(aggregate_fnc, values):
+    """
+    Applies the aggregate while keeping the raw values in the attrs.
+    If raw values are already present, keeps them, else they are added.
+    """
+    score = aggregate_fnc(values)
+    if Score.RAW_VALUES_KEY not in score.attrs:
+        # check if the raw values are already in the values.
+        # if yes, they didn't get copied to the aggregate score and we use those as the "rawest" values.
+        raw = values if Score.RAW_VALUES_KEY not in values.attrs else values.attrs[Score.RAW_VALUES_KEY]
+        score.attrs[Score.RAW_VALUES_KEY] = raw
+    return score
+
+
+class Transformation(object):
+    """
+    Transforms an incoming assembly into parts/combinations thereof,
+    yields them for further processing,
+    and packages the results back together.
+    """
+
+    def __call__(self, *args, apply, aggregate=None, **kwargs):
+        values = self._run_pipe(*args, apply=apply, **kwargs)
+
+        score = apply_aggregate(aggregate, values) if aggregate is not None else values
+        score = apply_aggregate(self.aggregate, score)
+        return score
+
+    def _run_pipe(self, *args, apply, **kwargs):
+        generator = self.pipe(*args, **kwargs)
+        for vals in generator:
+            y = apply(*vals)
+            done = generator.send(y)
+            if done:
+                break
+        result = next(generator)
+        return result
+
+    def pipe(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def _get_result(self, *args, done):
+        """
+        Yields the `*args` for further processing by coroutines
+        and waits for the result to be sent back.
+        :param args: transformed values
+        :param bool done: whether this is the last transformation and the next `yield` is the combined result
+        :return: the result from processing by the coroutine
+        """
+        result = yield args  # yield the values to coroutine
+        yield done  # wait for coroutine to send back similarity and inform whether result is ready to be returned
+        return result
+
+    def aggregate(self, score):
+        return Score(score)
+
+
+class Alignment(Transformation):
+    class Defaults:
+        order_dimensions = ('presentation', 'neuroid')
+        alignment_dim = 'stimulus_id'
+        repeat = False
+
+    def __init__(self, order_dimensions=Defaults.order_dimensions, alignment_dim=Defaults.alignment_dim,
+                 repeat=Defaults.repeat):
+        self._order_dimensions = order_dimensions
+        self._alignment_dim = alignment_dim
+        self._repeat = repeat
+        self._logger = logging.getLogger(fullname(self))
+
+    def pipe(self, source_assembly, target_assembly):
+        self._logger.debug("Aligning by {} and {}, {} repeats".format(
+            self._order_dimensions, self._alignment_dim, "with" if self._repeat else "no"))
+        source_assembly = self.align(source_assembly, target_assembly)
+        self._logger.debug("Sorting by {}".format(self._alignment_dim))
+        source_assembly, target_assembly = self.sort(source_assembly), self.sort(target_assembly)
+        result = yield from self._get_result(source_assembly, target_assembly, done=True)
+        yield result
+
+    def align(self, source_assembly, target_assembly):
+        dimensions = list(self._order_dimensions) + list(set(source_assembly.dims) - set(self._order_dimensions))
+        source_assembly = source_assembly.transpose(*dimensions)
+        return subset(source_assembly, target_assembly, subset_dims=[self._alignment_dim], repeat=self._repeat)
+
+    def sort(self, assembly):
+        return assembly.sortby(self._alignment_dim)
+
+
+class CartesianProduct(Transformation):
+    """
+    Splits an incoming assembly along all dimensions that similarity is not computed over
+    as well as along dividing coords that denote separate parts of the assembly.
+    """
+
+    def __init__(self, dividers=()):
+        super(CartesianProduct, self).__init__()
+        self._dividers = dividers or ()
+        self._logger = logging.getLogger(fullname(self))
+
+    def dividers(self, assembly, dividing_coords):
+        """
+        divide data along dividing coords and non-central dimensions,
+        i.e. dimensions that the metric is not computed over
+        """
+        non_matched_coords = [coord for coord in dividing_coords if not hasattr(assembly, coord)]
+        assert not non_matched_coords, f"{non_matched_coords} not found in assembly"
+        choices = {coord: unique_ordered(assembly[coord].values) for coord in dividing_coords}
+        combinations = [dict(zip(choices, values)) for values in itertools.product(*choices.values())]
+        return combinations
+
+    def pipe(self, assembly):
+        """
+        :param brainscore.assemblies.NeuroidAssembly assembly:
+        :return: brainscore.assemblies.DataAssembly
+        """
+        dividers = self.dividers(assembly, dividing_coords=self._dividers)
+        scores = []
+        progress = tqdm(enumerate_done(dividers), total=len(dividers), desc='cartesian product')
+        for i, divider, done in progress:
+            progress.set_description(str(divider))
+            divided_assembly = assembly.multisel(**divider)
+            # squeeze dimensions if necessary
+            for divider_coord in divider:
+                dims = assembly[divider_coord].dims
+                assert len(dims) == 1
+                if dims[0] in divided_assembly.dims and len(divided_assembly[dims[0]]) == 1:
+                    divided_assembly = divided_assembly.squeeze(dims[0])
+            result = yield from self._get_result(divided_assembly, done=done)
+
+            for coord_name, coord_value in divider.items():
+                # expand and set coordinate value. If the underlying raw values already contain that coordinate
+                # (e.g. as part of a MultiIndex), don't create and set new dimension on raw values.
+                if not hasattr(result, 'raw'):  # no raw values anyway
+                    kwargs = {}
+                elif not hasattr(result.raw, coord_name):  # doesn't have values yet, we can set (True by default)
+                    kwargs = {}
+                else:  # has raw values but already has coord in them --> need to prevent setting
+                    # this expects the result to accept `_apply_raw` in its `expand_dims` method
+                    # which is true for our `Score` class, but not a standard `DataAssembly`.
+                    kwargs = dict(_apply_raw=False)
+                result = result.expand_dims(coord_name, **kwargs)
+                result.__setitem__(coord_name, [coord_value], **kwargs)
+            scores.append(result)
+        scores = Score.merge(*scores)
+        yield scores
+
+
+class Split:
+    class Defaults:
+        splits = 10
+        train_size = .9
+        split_coord = 'stimulus_id'
+        stratification_coord = 'object_name'  # cross-validation across images, balancing objects
+        unique_split_values = False
+        random_state = 1
+
+    def __init__(self,
+                 splits=Defaults.splits, train_size=None, test_size=None,
+                 split_coord=Defaults.split_coord, stratification_coord=Defaults.stratification_coord, kfold=False,
+                 unique_split_values=Defaults.unique_split_values, random_state=Defaults.random_state,
+                 preprocess_indices=None):
+        super().__init__()
+        if train_size is None and test_size is None:
+            train_size = self.Defaults.train_size
+        if kfold:
+            assert (train_size is None or train_size == self.Defaults.train_size) and test_size is None
+            if stratification_coord:
+                self._split = StratifiedKFold(n_splits=splits, shuffle=True, random_state=random_state)
+            else:
+                self._split = KFold(n_splits=splits, shuffle=True, random_state=random_state)
+        else:
+            if stratification_coord:
+                self._split = StratifiedShuffleSplit(
+                    n_splits=splits, train_size=train_size, test_size=test_size, random_state=random_state)
+            else:
+                self._split = ShuffleSplit(
+                    n_splits=splits, train_size=train_size, test_size=test_size, random_state=random_state)
+        self._split_coord = split_coord
+        self._stratification_coord = stratification_coord
+        self._unique_split_values = unique_split_values
+        self._preprocess_indices=preprocess_indices
+
+        self._logger = logging.getLogger(fullname(self))
+
+    @property
+    def do_stratify(self):
+        return bool(self._stratification_coord)
+
+    def build_splits(self, assembly):
+        cross_validation_values, indices = extract_coord(assembly, self._split_coord, unique=self._unique_split_values)
+        data_shape = np.zeros(len(cross_validation_values))
+        args = [assembly[self._stratification_coord].values[indices]] if self.do_stratify else []
+        splits = self._split.split(data_shape, *args)
+        return cross_validation_values, list(splits)
+
+    @classmethod
+    def aggregate(cls, values):
+        score = apply_aggregate(lambda scores: scores.mean('split'), values)
+        score.attrs['error'] = standard_error_of_the_mean(values, 'split')
+        return score
+
+
+def extract_coord(assembly, coord, unique=False):
+    if not unique:
+        coord_values = assembly[coord].values
+        indices = list(range(len(coord_values)))
+    else:
+        # need unique values for when e.g. repetitions are heavily redundant and splits would yield equal unique values
+        coord_values, indices = np.unique(assembly[coord].values, return_index=True)
+    dims = assembly[coord].dims
+    assert len(dims) == 1
+    extracted_assembly = xr.DataArray(coord_values, coords={coord: coord_values}, dims=[coord])
+    extracted_assembly = extracted_assembly.stack(**{dims[0]: (coord,)})
+    return extracted_assembly if not unique else extracted_assembly, indices
+
+
+class TestOnlyCrossValidationSingle:
+    def __init__(self, *args, **kwargs):
+        self._cross_validation = CrossValidationSingle(*args, **kwargs)
+
+    def __call__(self, *args, apply, **kwargs):
+        apply_wrapper = lambda train, test: apply(test)
+        return self._cross_validation(*args, apply=apply_wrapper, **kwargs)
+
+
+class TestOnlyCrossValidation:
+    def __init__(self, *args, **kwargs):
+        self._cross_validation = CrossValidation(*args, **kwargs)
+
+    def __call__(self, *args, apply, **kwargs):
+        apply_wrapper = lambda train1, train2, test1, test2: apply(test1, test2)
+        return self._cross_validation(*args, apply=apply_wrapper, **kwargs)
+
+
+class CrossValidationSingle(Transformation):
+    def __init__(self,
+                 splits=Split.Defaults.splits, train_size=None, test_size=None,
+                 split_coord=Split.Defaults.split_coord, stratification_coord=Split.Defaults.stratification_coord,
+                 unique_split_values=Split.Defaults.unique_split_values, random_state=Split.Defaults.random_state):
+        super().__init__()
+        self._split = Split(splits=splits, split_coord=split_coord,
+                            stratification_coord=stratification_coord, unique_split_values=unique_split_values,
+                            train_size=train_size, test_size=test_size, random_state=random_state)
+        self._logger = logging.getLogger(fullname(self))
+
+    def pipe(self, assembly):
+        """
+        :param assembly: the assembly to cross-validate over
+        """
+        cross_validation_values, splits = self._split.build_splits(assembly)
+
+        split_scores = []
+        for split_iterator, (train_indices, test_indices), done \
+                in tqdm(enumerate_done(splits), total=len(splits), desc='cross-validation'):
+            train_values, test_values = cross_validation_values[train_indices], cross_validation_values[test_indices]
+            train = subset(assembly, train_values, dims_must_match=False)
+            test = subset(assembly, test_values, dims_must_match=False)
+
+            split_score = yield from self._get_result(train, test, done=done)
+            split_score = split_score.expand_dims('split')
+            split_score['split'] = [split_iterator]
+            split_scores.append(split_score)
+
+        split_scores = Score.merge(*split_scores)
+        yield split_scores
+
+    def aggregate(self, score):
+        return self._split.aggregate(score)
+
+
+class CrossValidation(Transformation):
+    """
+    Performs multiple splits over a source and target assembly.
+    No guarantees are given for data-alignment, use the metadata.
+    """
+
+    def __init__(self, *args, split_coord=Split.Defaults.split_coord,
+                 stratification_coord=Split.Defaults.stratification_coord,
+                 preprocess_indices=None, **kwargs):
+        self._split_coord = split_coord
+        self._stratification_coord = stratification_coord
+        self._split = Split(*args, split_coord=split_coord, stratification_coord=stratification_coord, **kwargs)
+        self._logger = logging.getLogger(fullname(self))
+        self._preprocess_indices = preprocess_indices
+        
+
+    def pipe(self, source_assembly, target_assembly):
+        # check only for equal values, alignment is given by metadata
+        assert sorted(source_assembly[self._split_coord].values) == sorted(target_assembly[self._split_coord].values)
+        if self._split.do_stratify:
+            assert hasattr(source_assembly, self._stratification_coord), \
+                f"Expected stratification coordinate {self._stratification_coord}"
+            assert sorted(source_assembly[self._stratification_coord].values) == \
+                   sorted(target_assembly[self._stratification_coord].values)
+        cross_validation_values, splits = self._split.build_splits(target_assembly)
+
+        split_scores = []
+        for split_iterator, (train_indices, test_indices), done \
+                in tqdm(enumerate_done(splits), total=len(splits), desc='cross-validation'):
+            
+            if hasattr(self, '_preprocess_indices'):
+                if self._preprocess_indices is not None:
+                    train_indices, test_indices = self._preprocess_indices(train_indices, test_indices, source_assembly)
+
+            train_values, test_values = cross_validation_values[train_indices], cross_validation_values[test_indices]
+            train_source = subset(source_assembly, train_values, dims_must_match=False)
+            train_target = subset(target_assembly, train_values, dims_must_match=False)
+            assert len(train_source[self._split_coord]) == len(train_target[self._split_coord])
+            test_source = subset(source_assembly, test_values, dims_must_match=False)
+            test_target = subset(target_assembly, test_values, dims_must_match=False)
+            assert len(test_source[self._split_coord]) == len(test_target[self._split_coord])
+            split_score = yield from self._get_result(train_source, train_target, test_source, test_target,
+                                                      done=done)
+            split_score = split_score.expand_dims('split')
+            split_score['split'] = [split_iterator]
+            split_scores.append(split_score)
+
+        split_scores = Score.merge(*split_scores)
+        yield split_scores
+
+    def aggregate(self, score):
+        return self._split.aggregate(score)
+
+
+def standard_error_of_the_mean(values, dim):
+    return values.std(dim) / math.sqrt(len(values[dim]))
+
+
+def expand(assembly, target_dims):
+    def strip(coord):
+        stripped_coord = coord
+        if stripped_coord.endswith('_source'):
+            stripped_coord = stripped_coord[:-len('_source')]
+        if stripped_coord.endswith('_target'):
+            stripped_coord = stripped_coord[:-len('_target')]
+        return stripped_coord
+
+    def reformat_coord_values(coord, dims, values):
+        stripped_coord = strip(coord)
+
+        if stripped_coord in target_dims and len(values.shape) == 0:
+            values = np.array([values])
+            dims = [coord]
+        return dims, values
+
+    coords = {coord: reformat_coord_values(coord, values.dims, values.values)
+              for coord, values in assembly.coords.items()}
+    dim_shapes = OrderedDict((coord, values[1].shape)
+                             for coord, values in coords.items() if strip(coord) in target_dims)
+    shape = [_shape for shape in dim_shapes.values() for _shape in shape]
+    # prepare values for broadcasting by adding new dimensions
+    values = assembly.values
+    for _ in range(sum([dim not in assembly.dims for dim in dim_shapes])):
+        values = values[:, np.newaxis]
+    values = np.broadcast_to(values, shape)
+    return DataAssembly(values, coords=coords, dims=list(dim_shapes.keys()))
+
+
+def enumerate_done(values):
+    for i, val in enumerate(values):
+        done = i == len(values) - 1
+        yield i, val, done

brainscore_vision/metric_helpers/utils.py

@@ -0,0 +1,71 @@
+from collections import OrderedDict
+
+import numpy as np
+
+from brainio.assemblies import walk_coords
+
+
+def collect_coords(assembly, ignore_dims, rename_coords_list, kind):
+    coords = assembly.coords
+    filtered_coords = filter_coords(coords, ignore_dims=ignore_dims)
+    renamed_coords = rename_coords(filtered_coords, rename_coords_list=rename_coords_list, suffix=kind)
+    return renamed_coords
+
+
+def filter_coords(coords, ignore_dims):
+    def filter_func(coord_values):
+        coord, values = coord_values
+        if coord in ignore_dims:
+            return False
+        value_dims_ignore = [dim in ignore_dims for dim in values.dims]
+        if any(value_dims_ignore):
+            assert all(value_dims_ignore)
+            return False
+        return True
+
+    return dict(filter(filter_func, coords.items()))
+
+
+def rename_coords(coords, rename_coords_list, suffix):
+    coord_names = {coord: coord if coord not in rename_coords_list else coord + '-' + suffix for coord in coords}
+    return {coord_names[name]: (tuple(coord_names[dim] for dim in values.dims), values.values)
+            for name, values in coords.items()}
+
+
+def collect_dim_shapes(assembly, rename_dims_list, ignore_dims, kind):
+    dims = assembly.dims
+    filtered_dims = filter_dims(dims, ignore_dims=ignore_dims)
+    renamed_dims = rename_dims(filtered_dims, rename_dims_list=rename_dims_list, rename_suffix=kind)
+    shapes = OrderedDict((dim, assembly[dim].shape) for dim in filtered_dims)
+    renamed = OrderedDict((renamed_dim, shape) for renamed_dim, (_, shape) in zip(renamed_dims, shapes.items()))
+    return renamed
+
+
+def filter_dims(dims, ignore_dims):
+    return [dim for dim in dims if dim not in ignore_dims]
+
+
+def rename_dims(dims, rename_dims_list, rename_suffix):
+    return [dim if dim not in rename_dims_list else dim + '-' + rename_suffix for dim in dims]
+
+
+def get_modified_coords(assembly, modifier=lambda name, dims, values: (name, (dims, values))):
+    coords = {}
+    for name, dims, values in walk_coords(assembly):
+        name_dims_vals = modifier(name, dims, values)
+        if name_dims_vals is not None:
+            name, (dims, vals) = name_dims_vals
+            coords[name] = dims, vals
+    return coords
+
+
+def merge_dicts(dicts):
+    result = {}
+    for dict in dicts:
+        result = {**result, **dict}
+    return result
+
+
+def unique_ordered(a):
+    _, idx = np.unique(a, return_index=True)
+    return a[np.sort(idx)]

brainscore_vision/metric_helpers/xarray_utils.py

@@ -0,0 +1,151 @@
+import numpy as np
+import xarray as xr
+
+from brainio.assemblies import NeuroidAssembly, array_is_element, walk_coords
+from brainscore_vision.metric_helpers import Defaults
+from brainscore_vision.metrics import Score
+
+
+class XarrayRegression:
+    """
+    Adds alignment-checking, un- and re-packaging, and comparison functionality to a regression.
+    """
+
+    def __init__(self, regression, expected_dims=Defaults.expected_dims, neuroid_dim=Defaults.neuroid_dim,
+                 neuroid_coord=Defaults.neuroid_coord, stimulus_coord=Defaults.stimulus_coord):
+        self._regression = regression
+        self._expected_dims = expected_dims
+        self._neuroid_dim = neuroid_dim
+        self._neuroid_coord = neuroid_coord
+        self._stimulus_coord = stimulus_coord
+        self._target_neuroid_values = None
+
+    def fit(self, source, target):
+        source, target = self._align(source), self._align(target)
+        source, target = source.sortby(self._stimulus_coord), target.sortby(self._stimulus_coord)
+
+        self._regression.fit(source, target)
+
+        self._target_neuroid_values = {}
+        for name, dims, values in walk_coords(target):
+            if self._neuroid_dim in dims:
+                assert array_is_element(dims, self._neuroid_dim)
+                self._target_neuroid_values[name] = values
+
+    def predict(self, source):
+        source = self._align(source)
+        predicted_values = self._regression.predict(source)
+        prediction = self._package_prediction(predicted_values, source=source)
+        return prediction
+
+    def _package_prediction(self, predicted_values, source):
+        coords = {coord: (dims, values) for coord, dims, values in walk_coords(source)
+                  if not array_is_element(dims, self._neuroid_dim)}
+        # re-package neuroid coords
+        dims = source.dims
+        # if there is only one neuroid coordinate, it would get discarded and the dimension would be used as coordinate.
+        # to avoid this, we can build the assembly first and then stack on the neuroid dimension.
+        neuroid_level_dim = None
+        if len(self._target_neuroid_values) == 1:  # extract single key: https://stackoverflow.com/a/20145927/2225200
+            (neuroid_level_dim, _), = self._target_neuroid_values.items()
+            dims = [dim if dim != self._neuroid_dim else neuroid_level_dim for dim in dims]
+        for target_coord, target_value in self._target_neuroid_values.items():
+            # this might overwrite values which is okay
+            coords[target_coord] = (neuroid_level_dim or self._neuroid_dim), target_value
+        prediction = NeuroidAssembly(predicted_values, coords=coords, dims=dims)
+        if neuroid_level_dim:
+            prediction = prediction.stack(**{self._neuroid_dim: [neuroid_level_dim]})
+
+        return prediction
+
+    def _align(self, assembly):
+        assert set(assembly.dims) == set(self._expected_dims), \
+            f"Expected {set(self._expected_dims)}, but got {set(assembly.dims)}"
+        return assembly.transpose(*self._expected_dims)
+
+
+class XarrayCorrelation:
+    def __init__(self, correlation, correlation_coord=Defaults.stimulus_coord, neuroid_coord=Defaults.neuroid_coord):
+        self._correlation = correlation
+        self._correlation_coord = correlation_coord
+        self._neuroid_coord = neuroid_coord
+
+    def __call__(self, prediction, target):
+        # align
+        prediction = prediction.sortby([self._correlation_coord, self._neuroid_coord])
+        target = target.sortby([self._correlation_coord, self._neuroid_coord])
+        assert np.array(prediction[self._correlation_coord].values == target[self._correlation_coord].values).all()
+        assert np.array(prediction[self._neuroid_coord].values == target[self._neuroid_coord].values).all()
+        # compute correlation per neuroid
+        neuroid_dims = target[self._neuroid_coord].dims
+        assert len(neuroid_dims) == 1
+        correlations = []
+        for i, coord_value in enumerate(target[self._neuroid_coord].values):
+            target_neuroids = target.isel(**{neuroid_dims[0]: i})  # `isel` is about 10x faster than `sel`
+            prediction_neuroids = prediction.isel(**{neuroid_dims[0]: i})
+            r, p = self._correlation(target_neuroids, prediction_neuroids)
+            correlations.append(r)
+        # package
+        result = Score(correlations,
+                       coords={coord: (dims, values)
+                               for coord, dims, values in walk_coords(target) if dims == neuroid_dims},
+                       dims=neuroid_dims)
+        return result
+
+
+# ops that also applies to attrs (and attrs of attrs), which are xarrays
+def recursive_op(*arrs, op=lambda x:x):
+    # the attrs structure of each arr must be the same
+    val = op(*arrs)
+    attrs = arrs[0].attrs
+    for attr in attrs:
+        attr_val = arrs[0].attrs[attr]
+        if isinstance(attr_val, xr.DataArray):
+            attr_arrs = [arr.attrs[attr] for arr in arrs]
+            attr_val = recursive_op(*attr_arrs, op=op)
+        val.attrs[attr] = attr_val
+    return val
+
+
+# apply a callable to every slice of the xarray along the specified dimensions
+def apply_over_dims(callable, *asms, dims, njobs=-1):
+    asms = [asm.transpose(*dims, ...) for asm in asms]
+    sizes = [asms[0].sizes[dim] for dim in dims]
+
+    def apply_helper(sizes, dims, *asms):
+        xarr = []
+        attrs = {}
+        size = sizes[0]
+        rsizes = sizes[1:]
+        dim = dims[0]
+        rdims = dims[1:]
+
+        if len(sizes) == 1:
+            # parallel execution on the last applied dimension
+            from joblib import Parallel, delayed
+            results = Parallel(n_jobs=njobs)(delayed(callable)(*[asm.isel({dim:s}) for asm in asms]) for s in range(size))
+        else:
+            results = []
+            for s in range(size):
+                arr = apply_helper(rsizes, rdims, *[asm.isel({dim:s}) for asm in asms])
+                results.append(arr)
+
+        for arr in results:
+            if arr is not None:
+                for k,v in arr.attrs.items():
+                    assert isinstance(v, xr.DataArray)
+                    attrs.setdefault(k, []).append(v.expand_dims(dim))
+                xarr.append(arr)
+       
+        if not xarr:
+            return
+        else:
+            xarr = xr.concat(xarr, dim=dim)
+            attrs = {k: xr.concat(vs, dim=dim) for k,vs in attrs.items()}
+            xarr.coords[dim] = asms[0].coords[dim]
+            for k,v in attrs.items():
+                attrs[k].coords[dim] = asms[0].coords[dim]
+                xarr.attrs[k] = attrs[k]
+            return xarr
+
+    return apply_helper(sizes, dims, *asms)

brainscore_vision/metrics/__init__.py

@@ -0,0 +1,7 @@
+from brainio.assemblies import DataAssembly
+from brainscore_core import Score
+
+
+class Ceiling:
+    def __call__(self, assembly: DataAssembly) -> Score:
+        raise NotImplementedError()

brainscore_vision/metrics/accuracy/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import metric_registry
+from .metric import Accuracy
+
+metric_registry['accuracy'] = Accuracy

brainscore_vision/metrics/accuracy/metric.py

@@ -0,0 +1,16 @@
+import numpy as np
+
+from brainscore_core import Metric
+from brainscore_vision.metrics import Score
+
+
+class Accuracy(Metric):
+    def __call__(self, source, target) -> Score:
+        values = source == target
+        center = np.mean(values)
+        error = np.std(values)
+
+        score = Score(center)
+        score.attrs['error'] = error
+        score.attrs[Score.RAW_VALUES_KEY] = values
+        return score

brainscore_vision/metrics/accuracy/test.py

@@ -0,0 +1,11 @@
+import numpy as np
+from numpy.random import RandomState
+
+from brainio.assemblies import DataAssembly
+from brainscore_vision import load_metric
+
+
+def test_fifty_percent():
+    metric = load_metric('accuracy')
+    accuracy = metric(np.array([5, 3, 6, 7, 1, 8]), np.array([5, 2, 1, 7, 1, 9]))
+    assert accuracy == 0.5

brainscore_vision/metrics/accuracy_distance/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import metric_registry
+from .metric import AccuracyDistance
+
+metric_registry['accuracy_distance'] = AccuracyDistance