brainscore-vision 2.1__tar.gz

brainscore_vision-2.1/LICENSE

@@ -0,0 +1,11 @@
+
+MIT License
+
+Copyright (c) 2017, Martin Schrimpf and Jon Prescott-Roy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

brainscore_vision-2.1/MANIFEST.in

@@ -0,0 +1,10 @@
+include LICENSE
+include README.md
+include brainscore_vision/lookup.csv
+include brainscore_vision/model_helpers/brain_transformation/imagenet_classes.txt
+include brainscore_vision/model_helpers/check_submission/images/*.png
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+
+recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif *.csv

brainscore_vision-2.1/PKG-INFO

@@ -0,0 +1,153 @@
+Metadata-Version: 2.1
+Name: brainscore_vision
+Version: 2.1
+Summary: The Brain-Score library enables model comparisons to behavioral and neural experiments
+License: 
+        MIT License
+        
+        Copyright (c) 2017, Martin Schrimpf and Jon Prescott-Roy
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+        
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy<2
+Requires-Dist: brainscore_core
+Requires-Dist: result_caching
+Requires-Dist: importlib-metadata<5
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
+Requires-Dist: opencv-python
+Requires-Dist: h5py
+Requires-Dist: tqdm
+Requires-Dist: gitpython
+Requires-Dist: fire
+Requires-Dist: jupyter
+Requires-Dist: pybtex
+Requires-Dist: peewee
+Requires-Dist: psycopg2-binary
+Requires-Dist: networkx
+Requires-Dist: eva-decord
+Requires-Dist: psutil
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest_check; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-timeout; extra == "test"
+Requires-Dist: torch; extra == "test"
+Requires-Dist: torchvision; extra == "test"
+Requires-Dist: matplotlib; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+
+[![Build Status](https://app.travis-ci.com/brain-score/vision.svg?token=vqt7d2yhhpLGwHsiTZvT&branch=master)](https://app.travis-ci.com/brain-score/vision)
+[![Documentation Status](https://readthedocs.org/projects/brain-score/badge/?version=latest)](https://brain-score.readthedocs.io/en/latest/?badge=latest)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) 
+
+Brain-Score is a platform to evaluate computational models of brain function 
+on their match to brain measurements in primate vision. 
+The intent of Brain-Score is to adopt many (ideally all) the experimental benchmarks in the field
+for the purpose of model testing, falsification, and comparison.
+To that end, Brain-Score operationalizes experimental data into quantitative benchmarks 
+that any model candidate following the [`BrainModel`](brainscore_vision/model_interface.py) interface can be scored on.
+
+Note that you can only access a limited set of public benchmarks when running locally. To score a model on all benchmarks, submit it via the [brain-score.org website](http://www.brain-score.org).
+
+See the [documentation](https://brain-score.readthedocs.io) for more details, e.g. for submitting a [model](https://brain-score.readthedocs.io/en/latest/modules/model_tutorial.html) or [benchmark](https://brain-score.readthedocs.io/en/latest/modules/benchmark_tutorial.html) to Brain-Score. For a step-by-step walkthrough on submitting models to the Brain-Score website, see these [web tutorials](https://www.brain-score.org/tutorial/).
+
+See these [code examples](https://github.com/brain-score/vision/blob/master/examples) on scoring models, retrieving data, using and defining benchmarks and metrics.
+These [previous examples](https://github.com/brain-score/candidate_models/blob/master/examples/score-model.ipynb) might be helpful, but their usage has been deprecated after the 2.0 update.
+
+Brain-Score is made by and for the community. 
+To contribute, please [send in a pull request](https://github.com/brain-score/vision/pulls).
+
+
+## Local installation
+
+You will need Python = 3.11 and pip >= 18.1.
+
+`pip install git+https://github.com/brain-score/vision`
+
+Test if the installation is successful by scoring a model on a public benchmark:
+
+```python
+from brainscore_vision.benchmarks import public_benchmark_pool
+
+benchmark = public_benchmark_pool['dicarlo.MajajHong2015public.IT-pls']
+model = my_model()
+score = benchmark(model)
+
+# >  <xarray.Score ()>
+# >  array(0.07637264)
+# >  Attributes:
+# >  Attributes:
+# >      error:                 <xarray.Score ()>\narray(0.00548197)
+# >      raw:                   <xarray.Score ()>\narray(0.22545106)\nAttributes:\...
+# >      ceiling:               <xarray.DataArray ()>\narray(0.81579938)\nAttribut...
+# >      model_identifier:      my-model
+# >      benchmark_identifier:  dicarlo.MajajHong2015public.IT-pls
+```
+
+Some steps may take minutes because data has to be downloaded during first-time use.
+
+## Environment Variables
+
+| Variable               | Description                                                                                                                           |
+|------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
+| RESULTCACHING_HOME     | directory to cache results (benchmark ceilings) in, `~/.result_caching` by default (see https://github.com/brain-score/result_caching) |
+
+
+## License
+
+MIT license
+
+
+## Troubleshooting
+
+<details>
+<summary>`ValueError: did not find HDF5 headers` during netcdf4 installation</summary>
+pip seems to fail properly setting up the HDF5_DIR required by netcdf4.
+Use conda: `conda install netcdf4`
+</details>
+
+<details>
+<summary>repeated runs of a benchmark / model do not change the outcome even though code was changed</summary>
+results (scores, activations) are cached on disk using https://github.com/mschrimpf/result_caching.
+Delete the corresponding file or directory to clear the cache.
+</details>
+
+
+## CI environment
+
+Add CI related build commands to `test_setup.sh`. The script is executed in CI environment for unittests.
+
+
+## References
+
+If you use Brain-Score in your work, please cite 
+["Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?"](https://www.biorxiv.org/content/10.1101/407007v2) (technical) and 
+["Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence"](https://www.cell.com/neuron/fulltext/S0896-6273(20)30605-X) (perspective) 
+as well as the respective benchmark sources.
+
+```bibtex
+@article{SchrimpfKubilius2018BrainScore,
+  title={Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?},
+  author={Martin Schrimpf and Jonas Kubilius and Ha Hong and Najib J. Majaj and Rishi Rajalingham and Elias B. Issa and Kohitij Kar and Pouya Bashivan and Jonathan Prescott-Roy and Franziska Geiger and Kailyn Schmidt and Daniel L. K. Yamins and James J. DiCarlo},
+  journal={bioRxiv preprint},
+  year={2018},
+  url={https://www.biorxiv.org/content/10.1101/407007v2}
+}
+
+@article{Schrimpf2020integrative,
+  title={Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence},
+  author={Schrimpf, Martin and Kubilius, Jonas and Lee, Michael J and Murty, N Apurva Ratan and Ajemian, Robert and DiCarlo, James J},
+  journal={Neuron},
+  year={2020},
+  url={https://www.cell.com/neuron/fulltext/S0896-6273(20)30605-X}
+}
+```

brainscore_vision-2.1/README.md

@@ -0,0 +1,106 @@
+[![Build Status](https://app.travis-ci.com/brain-score/vision.svg?token=vqt7d2yhhpLGwHsiTZvT&branch=master)](https://app.travis-ci.com/brain-score/vision)
+[![Documentation Status](https://readthedocs.org/projects/brain-score/badge/?version=latest)](https://brain-score.readthedocs.io/en/latest/?badge=latest)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) 
+
+Brain-Score is a platform to evaluate computational models of brain function 
+on their match to brain measurements in primate vision. 
+The intent of Brain-Score is to adopt many (ideally all) the experimental benchmarks in the field
+for the purpose of model testing, falsification, and comparison.
+To that end, Brain-Score operationalizes experimental data into quantitative benchmarks 
+that any model candidate following the [`BrainModel`](brainscore_vision/model_interface.py) interface can be scored on.
+
+Note that you can only access a limited set of public benchmarks when running locally. To score a model on all benchmarks, submit it via the [brain-score.org website](http://www.brain-score.org).
+
+See the [documentation](https://brain-score.readthedocs.io) for more details, e.g. for submitting a [model](https://brain-score.readthedocs.io/en/latest/modules/model_tutorial.html) or [benchmark](https://brain-score.readthedocs.io/en/latest/modules/benchmark_tutorial.html) to Brain-Score. For a step-by-step walkthrough on submitting models to the Brain-Score website, see these [web tutorials](https://www.brain-score.org/tutorial/).
+
+See these [code examples](https://github.com/brain-score/vision/blob/master/examples) on scoring models, retrieving data, using and defining benchmarks and metrics.
+These [previous examples](https://github.com/brain-score/candidate_models/blob/master/examples/score-model.ipynb) might be helpful, but their usage has been deprecated after the 2.0 update.
+
+Brain-Score is made by and for the community. 
+To contribute, please [send in a pull request](https://github.com/brain-score/vision/pulls).
+
+
+## Local installation
+
+You will need Python = 3.11 and pip >= 18.1.
+
+`pip install git+https://github.com/brain-score/vision`
+
+Test if the installation is successful by scoring a model on a public benchmark:
+
+```python
+from brainscore_vision.benchmarks import public_benchmark_pool
+
+benchmark = public_benchmark_pool['dicarlo.MajajHong2015public.IT-pls']
+model = my_model()
+score = benchmark(model)
+
+# >  <xarray.Score ()>
+# >  array(0.07637264)
+# >  Attributes:
+# >  Attributes:
+# >      error:                 <xarray.Score ()>\narray(0.00548197)
+# >      raw:                   <xarray.Score ()>\narray(0.22545106)\nAttributes:\...
+# >      ceiling:               <xarray.DataArray ()>\narray(0.81579938)\nAttribut...
+# >      model_identifier:      my-model
+# >      benchmark_identifier:  dicarlo.MajajHong2015public.IT-pls
+```
+
+Some steps may take minutes because data has to be downloaded during first-time use.
+
+## Environment Variables
+
+| Variable               | Description                                                                                                                           |
+|------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
+| RESULTCACHING_HOME     | directory to cache results (benchmark ceilings) in, `~/.result_caching` by default (see https://github.com/brain-score/result_caching) |
+
+
+## License
+
+MIT license
+
+
+## Troubleshooting
+
+<details>
+<summary>`ValueError: did not find HDF5 headers` during netcdf4 installation</summary>
+pip seems to fail properly setting up the HDF5_DIR required by netcdf4.
+Use conda: `conda install netcdf4`
+</details>
+
+<details>
+<summary>repeated runs of a benchmark / model do not change the outcome even though code was changed</summary>
+results (scores, activations) are cached on disk using https://github.com/mschrimpf/result_caching.
+Delete the corresponding file or directory to clear the cache.
+</details>
+
+
+## CI environment
+
+Add CI related build commands to `test_setup.sh`. The script is executed in CI environment for unittests.
+
+
+## References
+
+If you use Brain-Score in your work, please cite 
+["Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?"](https://www.biorxiv.org/content/10.1101/407007v2) (technical) and 
+["Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence"](https://www.cell.com/neuron/fulltext/S0896-6273(20)30605-X) (perspective) 
+as well as the respective benchmark sources.
+
+```bibtex
+@article{SchrimpfKubilius2018BrainScore,
+  title={Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?},
+  author={Martin Schrimpf and Jonas Kubilius and Ha Hong and Najib J. Majaj and Rishi Rajalingham and Elias B. Issa and Kohitij Kar and Pouya Bashivan and Jonathan Prescott-Roy and Franziska Geiger and Kailyn Schmidt and Daniel L. K. Yamins and James J. DiCarlo},
+  journal={bioRxiv preprint},
+  year={2018},
+  url={https://www.biorxiv.org/content/10.1101/407007v2}
+}
+
+@article{Schrimpf2020integrative,
+  title={Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence},
+  author={Schrimpf, Martin and Kubilius, Jonas and Lee, Michael J and Murty, N Apurva Ratan and Ajemian, Robert and DiCarlo, James J},
+  journal={Neuron},
+  year={2020},
+  url={https://www.cell.com/neuron/fulltext/S0896-6273(20)30605-X}
+}
+```

brainscore_vision-2.1/brainscore_vision/__init__.py

@@ -0,0 +1,105 @@
+import logging
+from typing import Dict, Any, Union, Callable
+
+from brainio.assemblies import DataAssembly
+from brainio.stimuli import StimulusSet
+from brainscore_core.benchmarks import Benchmark
+from brainscore_core.metrics import Metric, Score
+from brainscore_core.plugin_management.conda_score import wrap_score
+from brainscore_core.plugin_management.import_plugin import import_plugin
+from brainscore_vision.metrics import Ceiling
+from brainscore_vision.model_interface import BrainModel
+
+_logger = logging.getLogger(__name__)
+
+data_registry: Dict[str, Callable[[], Union[DataAssembly, Any]]] = {}
+""" Pool of available data """
+
+stimulus_set_registry: Dict[str, Callable[[], StimulusSet]] = {}
+""" Pool of available stimulus sets"""
+
+metric_registry: Dict[str, Callable[[], Metric]] = {}
+""" Pool of available metrics """
+
+benchmark_registry: Dict[str, Callable[[], Benchmark]] = {}
+""" Pool of available benchmarks """
+
+model_registry: Dict[str, Callable[[], BrainModel]] = {}
+""" Pool of available models """
+
+
+def load_dataset(identifier: str) -> Union[DataAssembly, Any]:
+    import_plugin('brainscore_vision', 'data', identifier)
+
+    return data_registry[identifier]()
+
+
+def load_stimulus_set(identifier: str) -> Any:
+    # dataset and stimulus_set are kept in the same directory
+    import_plugin('brainscore_vision', 'data', identifier, registry_prefix='stimulus_set')
+
+    return stimulus_set_registry[identifier]()
+
+
+def load_metric(identifier: str, *args, **kwargs) -> Metric:
+    import_plugin('brainscore_vision', 'metrics', identifier)
+
+    return metric_registry[identifier](*args, **kwargs)
+
+
+def load_ceiling(identifier: str, *args, **kwargs) -> Ceiling:
+    """
+    Convenience method to load ceilings with the correct type -- internally uses :meth:`~brainscore_vision.load_metric`,
+    i.e. the ceiling has to be registered as a metric.
+    """
+    return load_metric(identifier, *args, **kwargs)
+
+
+def load_benchmark(identifier: str) -> Benchmark:
+    import_plugin('brainscore_vision', 'benchmarks', identifier)
+
+    return benchmark_registry[identifier]()
+
+
+def load_model(identifier: str) -> BrainModel:
+    import_plugin('brainscore_vision', 'models', identifier)
+
+    return model_registry[identifier]()
+
+
+def _run_score(model_identifier: str, benchmark_identifier: str) -> Score:
+    """
+    Score the model referenced by the `model_identifier` on the benchmark referenced by the `benchmark_identifier`.
+    """
+    model: BrainModel = load_model(model_identifier)
+    benchmark: Benchmark = load_benchmark(benchmark_identifier)
+    score: Score = benchmark(model)
+    score.attrs['model_identifier'] = model_identifier
+    score.attrs['benchmark_identifier'] = benchmark_identifier
+    try:  # attempt to look up the layer commitment if model uses a standard layer model
+        score.attrs['comment'] = f"layers: {model.layer_model.region_layer_map}"
+    except Exception:
+        pass
+    return score
+
+
+def score(model_identifier: str, benchmark_identifier: str, conda_active: bool = False) -> Score:
+    """
+    Score the model referenced by the `model_identifier` on the benchmark referenced by the `benchmark_identifier`.
+    The model needs to implement the :class:`~brainscore_vision.model_interface.BrainModel` interface
+    so that the benchmark can interact with it.
+    The benchmark will be looked up from the :data:`~brainscore_vision.benchmarks` and evaluates the model
+    (looked up from :data:`~brainscore_vision.models`) on how brain-like it is under that benchmark's
+    experimental paradigm, primate measurements, comparison metric, and ceiling.
+    This results in a quantitative
+    `Score <https://brain-score-core.readthedocs.io/en/latest/modules/metrics.html#brainscore_core.metrics.Score>`_
+    ranging from 0 (least brain-like) to 1 (most brain-like under this benchmark).
+
+    :param model_identifier: the identifier for the model
+    :param benchmark_identifier: the identifier for the benchmark to test the model against
+    :return: a Score of how brain-like the candidate model is under this benchmark. The score is normalized by
+        this benchmark's ceiling such that 1 means the model matches the data to ceiling level.
+    """
+    return wrap_score(__file__,
+                      model_identifier=model_identifier, benchmark_identifier=benchmark_identifier,
+                      score_function=_run_score, conda_active=conda_active)

brainscore_vision-2.1/brainscore_vision/__main__.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+import fire
+
+from brainscore_vision import score as _score_function
+
+
+def score(model_identifier: str, benchmark_identifier: str, conda_active: bool=False):
+    result = _score_function(model_identifier, benchmark_identifier, conda_active)
+    print(result)  # print instead of return because fire has issues with xarray objects
+
+
+def test_plugins(*args, **kwargs):
+    from brainscore_core.plugin_management.test_plugins import run_args as _core_test_plugins
+    root_directory = Path(__file__).parent
+    _core_test_plugins(*args, root_directory=root_directory, **kwargs)
+
+
+if __name__ == '__main__':
+    fire.Fire()

brainscore_vision-2.1/brainscore_vision/benchmark_helpers/__init__.py

@@ -0,0 +1,67 @@
+from typing import Union
+
+import numpy as np
+import hashlib
+
+from brainio.assemblies import NeuroidAssembly, DataAssembly
+from brainscore_core import Score
+from brainscore_vision.model_interface import BrainModel
+
+
+class PrecomputedFeatures(BrainModel):
+    def __init__(self, features: Union[DataAssembly, dict], visual_degrees):
+        """
+        :param features: The precomputed features. Either an assembly of features, indexable with `stimulus_id` or
+            a dictionary mapping from stimulus identifier to feature assemblies.
+        :param visual_degrees: Some visual degrees to use for the precomputed features. Since features are precomputed,
+            this should only affect the `place_on_screen` in the benchmark's __call__ method.
+        """
+        self.features = features
+        self._visual_degrees = visual_degrees
+
+    @property
+    def identifier(self) -> str:
+        # serialize the features to a string and create hash
+        features_data = str(self.features)
+        features_hash = hashlib.md5(features_data.encode('utf-8')).hexdigest()
+        return f"precomputed-{features_hash}"
+
+    def visual_degrees(self) -> int:
+        return self._visual_degrees
+
+    def start_task(self, task, fitting_stimuli=None):
+        pass
+
+    def start_recording(self, region, *args, **kwargs):
+        pass
+
+    def look_at(self, stimuli, number_of_trials=1):
+        features = self.features[stimuli.identifier] if isinstance(self.features, dict) else self.features
+        missing_stimulus_ids = set(stimuli['stimulus_id'].values) - set(features['stimulus_id'].values)
+        assert not missing_stimulus_ids, f"stored features do not contain stimulus_ids {missing_stimulus_ids}"
+        image_indices = [np.where(features['stimulus_id'].values == image_id)[0][0]
+                         for image_id in stimuli['stimulus_id'].values]
+        features = features.isel(presentation=image_indices)
+        assert all(features['stimulus_id'].values == stimuli['stimulus_id'].values)
+        return features
+
+
+def check_standard_format(assembly, nans_expected=False):
+    assert isinstance(assembly, NeuroidAssembly)
+    assert set(assembly.dims).issuperset({'presentation', 'neuroid'})
+    assert hasattr(assembly, 'stimulus_id')
+    assert hasattr(assembly, 'neuroid_id')
+    if not nans_expected:
+        assert not np.isnan(assembly).any()
+    assert 'stimulus_set_identifier' in assembly.attrs
+
+
+def bound_score(score: Score):
+    """
+    Force score value to be between 0 and 1.
+    If score is lower than 0, set to 0. If score is greater than 1, set to 1.
+    """
+    if score < 0:
+        score.loc[()] = 0
+    elif score > 1:
+        score.loc[()] = 1

brainscore_vision-2.1/brainscore_vision/benchmark_helpers/neural_common.py

@@ -0,0 +1,70 @@
+import numpy as np
+
+from brainio.assemblies import array_is_element, walk_coords
+from brainscore_core import Score
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.model_interface import BrainModel
+from .screen import place_on_screen
+
+
+class NeuralBenchmark(BenchmarkBase):
+    def __init__(self, identifier, assembly, similarity_metric, visual_degrees, number_of_trials, **kwargs):
+        super(NeuralBenchmark, self).__init__(identifier=identifier, **kwargs)
+        self._assembly = assembly
+        self._similarity_metric = similarity_metric
+        region = np.unique(self._assembly['region'])
+        assert len(region) == 1
+        self.region = region[0]
+        timebins = timebins_from_assembly(self._assembly)
+        self.timebins = timebins
+        self._visual_degrees = visual_degrees
+        self._number_of_trials = number_of_trials
+
+    def __call__(self, candidate: BrainModel):
+        candidate.start_recording(self.region, time_bins=self.timebins)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        source_assembly = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
+        if 'time_bin' in source_assembly.dims and source_assembly.sizes['time_bin'] == 1:
+            source_assembly = source_assembly.squeeze('time_bin')  # static case for these benchmarks
+        raw_score = self._similarity_metric(source_assembly, self._assembly)
+        ceiled_score = explained_variance(raw_score, self.ceiling)
+        return ceiled_score
+
+
+def timebins_from_assembly(assembly):
+    timebins = assembly['time_bin'].values
+    if 'time_bin' not in assembly.dims:
+        timebins = [timebins]  # only single time-bin
+    return timebins
+
+
+def explained_variance(score: Score, ceiling: Score) -> Score:
+    # ro(X, Y)
+    # = (r(X, Y) / sqrt(r(X, X) * r(Y, Y)))^2
+    # = (r(X, Y) / sqrt(r(Y, Y) * r(Y, Y)))^2  # assuming that r(Y, Y) ~ r(X, X) following Yamins 2014
+    # = (r(X, Y) / r(Y, Y))^2
+    r_square = np.power(score.values /
+                        ceiling.values, 2)
+    ceiled_score = Score(r_square)
+    if 'error' in score.attrs:
+        ceiled_score.attrs['error'] = score.attrs['error']
+    ceiled_score.attrs[Score.RAW_VALUES_KEY] = score
+    ceiled_score.attrs['ceiling'] = ceiling
+    return ceiled_score
+
+
+def average_repetition(assembly):
+    def avg_repr(assembly):
+        presentation_coords = [coord for coord, dims, values in walk_coords(assembly)
+                               if array_is_element(dims, 'presentation') and coord != 'repetition']
+        assembly = assembly.multi_groupby(presentation_coords).mean(dim='presentation', skipna=True)
+        return assembly
+
+    return apply_keep_attrs(assembly, avg_repr)
+
+def apply_keep_attrs(assembly, fnc):  # workaround to keeping attrs
+    attrs = assembly.attrs
+    assembly = fnc(assembly)
+    assembly.attrs = attrs
+    return assembly