brainscore-language 2.1.0__py3-none-any.whl

brainscore_language/__init__.py

@@ -0,0 +1,80 @@
+from typing import Dict, Any, Union, Callable
+
+from brainio.assemblies import DataAssembly
+from brainscore_core.benchmarks import Benchmark
+from brainscore_core.metrics import Score, Metric
+from brainscore_core.plugin_management.conda_score import wrap_score
+from brainscore_core.plugin_management.import_plugin import import_plugin
+from brainscore_language.artificial_subject import ArtificialSubject
+
+data_registry: Dict[str, Callable[[], Union[DataAssembly, Any]]] = {}
+""" Pool of available data """
+
+metric_registry: Dict[str, Callable[[], Metric]] = {}
+""" Pool of available metrics """
+
+benchmark_registry: Dict[str, Callable[[], Benchmark]] = {}
+""" Pool of available benchmarks """
+
+model_registry: Dict[str, Callable[[], ArtificialSubject]] = {}
+""" Pool of available models """
+
+
+def load_dataset(identifier: str) -> Union[DataAssembly, Any]:
+    import_plugin('brainscore_language', 'data', identifier)
+
+    return data_registry[identifier]()
+
+
+def load_metric(identifier: str, *args, **kwargs) -> Metric:
+    import_plugin('brainscore_language', 'metrics', identifier)
+
+    return metric_registry[identifier](*args, **kwargs)
+
+
+def load_benchmark(identifier: str) -> Benchmark:
+    import_plugin('brainscore_language', 'benchmarks', identifier)
+
+    return benchmark_registry[identifier]()
+
+
+def load_model(identifier: str) -> ArtificialSubject:
+    import_plugin('brainscore_language', 'models', identifier)
+
+    model = model_registry[identifier]()
+    model.identifier = identifier
+
+    return model
+
+def _run_score(model_identifier: str, benchmark_identifier: str) -> Score:
+    """
+    Score the model referenced by the `model_identifier` on the benchmark referenced by the `benchmark_identifier`.
+    """
+    model: ArtificialSubject = load_model(model_identifier)
+    benchmark: Benchmark = load_benchmark(benchmark_identifier)
+    score: Score = benchmark(model)
+    score.attrs['model_identifier'] = model_identifier
+    score.attrs['benchmark_identifier'] = benchmark_identifier
+    return score
+
+
+def score(model_identifier: str, benchmark_identifier: str, conda_active: bool=False) -> Score:
+    """
+    Score the model referenced by the `model_identifier` on the benchmark referenced by the `benchmark_identifier`.
+    The model needs to implement the :class:`~brainscore_language.artificial_subject.ArtificialSubject` interface
+    so that the benchmark can interact with it.
+    The benchmark will be looked up from the :data:`~brainscore_language.benchmarks` and evaluates the model
+    (looked up from :data:`~brainscore_language.models`) on how brain-like it is under that benchmark's
+    experimental paradigm, primate measurements, comparison metric, and ceiling.
+    This results in a quantitative
+    `Score <https://brain-score-core.readthedocs.io/en/latest/modules/metrics.html#brainscore_core.metrics.Score>`_
+    ranging from 0 (least brain-like) to 1 (most brain-like under this benchmark).
+    
+    :param model_identifier: the identifier for the model
+    :param benchmark_identifier: the identifier for the benchmark to test the model against
+    :return: a Score of how brain-like the candidate model is under this benchmark. The score is normalized by
+        this benchmark's ceiling such that 1 means the model matches the data to ceiling level.
+    """
+    return wrap_score(__file__,
+                      model_identifier=model_identifier, benchmark_identifier=benchmark_identifier,
+                      score_function=_run_score, conda_active=conda_active)

brainscore_language/__main__.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+
+import fire
+
+from brainscore_language import score as _score_function
+
+
+def score(model_identifier: str, benchmark_identifier: str, conda_active: bool=False):
+    result = _score_function(model_identifier, benchmark_identifier, conda_active)
+    print(result)  # print instead of return because fire has issues with xarray objects
+
+
+if __name__ == '__main__':
+    fire.Fire()

brainscore_language/artificial_subject.py

@@ -0,0 +1,140 @@
+from typing import Dict, List, Union
+
+from brainio.assemblies import DataAssembly
+
+
+class ArtificialSubject:
+    def identifier(self) -> str:
+        """
+        The unique identifier for this model.
+
+        :return: e.g. `'glove'`, or `'distilgpt2'`
+        """
+        raise NotImplementedError()
+
+    class Task:
+        """ task to perform """
+        # ideally we would define these as enums (also for RecordingTarget and RecordingType)
+        # but accessing enums gives weird Enum objects whereas this setup just gives a string
+
+        next_word = 'next_word'
+        """ 
+        Predict the next word from the preceding context. Output a :class:`~brainio.assemblies.BehavioralAssembly` with 
+        next-word predictions as the values and preceding stimulus in the `stimulus` coordinate.
+
+        Example:
+
+        Calling `digest_text(text = 'the quick brown')` could output 
+
+        .. code-block:: python
+
+           {'behavior': <xarray.BehavioralAssembly (presentation: 1)>
+                        array(['fox']), # the actual next word
+                        Coordinates:
+                          * presentation  (presentation) MultiIndex
+                          - stimulus      (presentation) object 'the quick brown'
+                          - stimulus_id   (presentation) int64 0}
+
+        Example:
+
+        Calling `digest_text(text = ['the quick brown', 'fox jumps', 'over the lazy'])` could output 
+
+        .. code-block:: python
+
+           {'behavior': <xarray.BehavioralAssembly (presentation: 3)>
+                        array(['fox', 'over', 'dog']), # the actual next words
+                        Coordinates:
+                          * presentation  (presentation) MultiIndex
+                          - stimulus      (presentation) object 'the quick brown' 'fox jumps', 'over the lazy'
+                          - stimulus_id   (presentation) int64 0 1 2}
+        """
+
+        reading_times = 'reading_times'
+        """ 
+        Output how long it took to read a given text, in milliseconds. 
+        Output a :class:`~brainio.assemblies.BehavioralAssembly` with reading times in milliseconds as the values and 
+        text in the `stimulus` coordinate.
+
+        Example:
+
+        Calling `digest_text(text = 'the quick brown')` could output 
+
+        .. code-block:: python
+
+           {'behavior': <xarray.BehavioralAssembly (presentation: 1)>
+                        array([329.15]), # reading time in milliseconds
+                        Coordinates:
+                          * presentation  (presentation) MultiIndex
+                          - stimulus      (presentation) object 'the quick brown'
+                          - stimulus_id   (presentation) int64 0}
+
+        Example:
+
+        Calling `digest_text(text = ['the quick brown', 'fox jumps', 'over the lazy'])` could output 
+
+        .. code-block:: python
+
+           {'behavior': <xarray.BehavioralAssembly (presentation: 3)>
+                        array([329.15, 337.53, 341.13]), # reading times in milliseconds
+                        Coordinates:
+                          * presentation  (presentation) MultiIndex
+                          - stimulus      (presentation) object 'the quick brown' 'fox jumps', 'over the lazy'
+                          - stimulus_id   (presentation) int64 0 1 2}
+        """
+
+    def start_behavioral_task(self, task: Task):
+        """
+        Instruct the subject to output behavior when queried with
+        :meth:`~brainscore_language.artificial_subject.ArtificialSubject.digest_text`. Calling
+        :meth:`~brainscore_language.artificial_subject.ArtificialSubject.digest_text` will then output a
+        :class:`~brainio.assemblies.BehavioralAssembly` in the `'behavior'` part of the output dict.
+
+        :param task: Which :class:`~brainscore_language.artificial_subject.ArtificialSubject.Task` to use.
+        """
+        raise NotImplementedError()
+
+    class RecordingTarget:
+        """ location to record from """
+        language_system = "language_system"
+        language_system_left_hemisphere = "language_system_left_hemisphere"
+        language_system_right_hemisphere = "language_system_right_hemisphere"
+
+    class RecordingType:
+        """ method of recording """
+
+        fMRI = "fMRI"
+        """ functional magnetic resonance imaging """
+        ECoG = "ECoG"
+        """ Electrocorticography """
+
+    def start_neural_recording(self, recording_target: RecordingTarget, recording_type: RecordingType):
+        """
+        Begin neural recordings from this subject. Calling
+        :meth:`~brainscore_language.artificial_subject.ArtificialSubject.digest_text` will then output a
+        :class:`~brainio.assemblies.NeuroidAssembly` in the `'neural'` part of the output dict.
+
+        :param recording_target: where in the language-involved parts of the brain to record from.
+        :param  recording_type: What recording method should be used.
+        """
+        raise NotImplementedError()
+
+    def digest_text(self, text: Union[str, List[str]]) -> Dict[str, DataAssembly]:
+        """
+        :param text: text to pass to the subject, either a single string (e.g. `"the quick brown fox jumped"`),
+            or a list of strings (e.g. `["the quick", "brown fox jumped"]`)
+        :return: a dictionary mapping from `'behavior'` and `'neural'` to a :class:`~brainio.assemblies.DataAssembly`,
+            if the subject was instructed to
+            :meth:`~brainscore_language.artificial_subject.ArtificialSubject.start_behavioral_task` and/or
+            :meth:`~brainscore_language.artificial_subject.ArtificialSubject.start_neural_recording` respectively.
+            The :class:`~brainio.assemblies.DataAssembly` always contains a `presentation` dimension, corresponding to
+            the presentation of text stimuli (including a `stimulus` coordinate for the text,
+            and a `stimulus_id` coordinate that uniquely identifies each stimulus).
+            Values along the `presentation` dimension are ordered in the same way as the input.
+            The `behavior` output has no other dimension, while the `neural` output has a `neuroid` dimension
+            (including a `neuroid_id` coordinate to uniquely identify each recording unit), and sometimes a
+            `time_bin` dimension (including `time_bin_start` and `time_bin_end`).
+            Note that model state is not saved between every time that `digest_text` is called. For keeping states,
+            between different inputs, `text` should be a list.
+
+        """
+        raise NotImplementedError()

brainscore_language/benchmark_helpers/__init__.py

@@ -0,0 +1,47 @@
+import numpy as np
+from typing import List
+
+from brainio.assemblies import walk_coords, array_is_element, DataAssembly
+
+
+def ci_error(samples, center, confidence=.95):
+    low, high = 100 * ((1 - confidence) / 2), 100 * (1 - ((1 - confidence) / 2))
+    confidence_below, confidence_above = np.nanpercentile(samples, low), np.nanpercentile(samples, high)
+    confidence_below, confidence_above = center - confidence_below, confidence_above - center
+    return confidence_below, confidence_above
+
+
+def manual_merge(*elements: List[DataAssembly], on='neuroid') -> DataAssembly:
+    """
+    Manually merge a set of assemblies where xarray's automated merge might fail.
+    This function likely covers only covers a small number of use-cases, and should thus be used with caution.
+    """
+    dims = elements[0].dims
+    assert all(element.dims == dims for element in elements[1:])
+    merge_index = dims.index(on)
+    # the coordinates in the merge index should have the same keys
+    assert _coords_match(elements, dim=on,
+                         match_values=False), f"coords in {[element[on] for element in elements]} do not match"
+    # all other dimensions, their coordinates and values should already align
+    for dim in set(dims) - {on}:
+        assert _coords_match(elements, dim=dim,
+                             match_values=True), f"coords in {[element[dim] for element in elements]} do not match"
+    # merge values without meta
+    merged_values = np.concatenate([element.values for element in elements], axis=merge_index)
+    # piece together with meta
+    result = type(elements[0])(merged_values, coords={
+        **{coord: (dims, values)
+           for coord, dims, values in walk_coords(elements[0])
+           if not array_is_element(dims, on)},
+        **{coord: (dims, np.concatenate([element[coord].values for element in elements]))
+           for coord, dims, _ in walk_coords(elements[0])
+           if array_is_element(dims, on)}}, dims=elements[0].dims)
+    return result
+
+
+def _coords_match(elements, dim, match_values=False):
+    """ Helper method for `manual_merge` """
+    first_coords = [(key, tuple(value)) if match_values else key for _, key, value in walk_coords(elements[0][dim])]
+    other_coords = [[(key, tuple(value)) if match_values else key for _, key, value in walk_coords(element[dim])]
+                    for element in elements[1:]]
+    return all(tuple(first_coords) == tuple(coords) for coords in other_coords)

brainscore_language/benchmarks/blank2014/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_language import benchmark_registry
+from .benchmark import Blank2014Linear
+
+benchmark_registry['Blank2014-linear'] = Blank2014Linear

brainscore_language/benchmarks/blank2014/benchmark.py

@@ -0,0 +1,50 @@
+import xarray as xr
+
+from brainscore_core.benchmarks import BenchmarkBase
+from brainscore_core.metrics import Score
+from brainscore_language import load_dataset, load_metric
+from brainscore_language.artificial_subject import ArtificialSubject
+from brainscore_language.benchmarks.blank2014.ceiling import ExtrapolationCeiling
+from brainscore_language.data.blank2014 import BIBTEX
+from brainscore_language.utils.ceiling import ceiling_normalize
+
+
+class Blank2014Linear(BenchmarkBase):
+    """
+    Evaluate model ability to predict neural activity in human language system functional regions of interest (fROIs)
+    in response to natural stories, recorded by Blank et al. 2014.
+    Alignment of neural activity between model and human subjects is evaluated via cross-validated linear predictivity.
+
+    This benchmark builds off the Blank2014 benchmark introduced in Schrimpf et al. 2021
+    (https://www.pnas.org/doi/10.1073/pnas.2105646118), but requires the model to have committed to neural readouts
+    (e.g. "layer 41 corresponds to the language system"), rather than testing every layer separately.
+    """
+
+    def __init__(self):
+        self.data = load_dataset('Blank2014.fROI')
+        self.metric = load_metric('linear_pearsonr')
+        ceiler = ExtrapolationCeiling()
+        ceiling = ceiler(assembly=self.data, metric=self.metric)
+        super(Blank2014Linear, self).__init__(
+            identifier='Blank2014-linear',
+            version=1,
+            parent='neural_language',
+            ceiling=ceiling,
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: ArtificialSubject) -> Score:
+        candidate.start_neural_recording(recording_target=ArtificialSubject.RecordingTarget.language_system,
+                                         recording_type=ArtificialSubject.RecordingType.fMRI)
+        stimuli = self.data['stimulus']
+        stories = self.data['story'].values
+        predictions = []
+        for story in sorted(set(stories)):  # go over individual stories, sorting to keep consistency across runs
+            story_indexer = [stimulus_story == story for stimulus_story in stories]
+            story_stimuli = stimuli[story_indexer]
+            story_predictions = candidate.digest_text(story_stimuli.values)['neural']
+            story_predictions['stimulus_id'] = 'presentation', story_stimuli['stimulus_id'].values
+            predictions.append(story_predictions)
+        predictions = xr.concat(predictions, dim='presentation')
+        raw_score = self.metric(predictions, self.data)
+        score = ceiling_normalize(raw_score, self.ceiling)
+        return score

brainscore_language/benchmarks/blank2014/ceiling.py

@@ -0,0 +1,204 @@
+import itertools
+import logging
+import numpy as np
+from numpy.random import RandomState
+from scipy.optimize import curve_fit
+from tqdm import tqdm, trange
+
+from brainio.assemblies import array_is_element, walk_coords, DataAssembly, merge_data_arrays
+from brainscore_core.metrics import Score
+from brainscore_language.benchmark_helpers import ci_error, manual_merge
+from brainscore_language.utils import fullname
+from brainscore_language.utils.transformations import apply_aggregate
+
+
+def v(x, v0, tau0):
+    return v0 * (1 - np.exp(-x / tau0))
+
+
+class ExtrapolationCeiling:
+    def __init__(self, subject_column='subject_id', extrapolation_dimension='neuroid', num_bootstraps=100):
+        self._logger = logging.getLogger(fullname(self))
+        self.subject_column = subject_column
+        self.holdout_ceiling = HoldoutSubjectCeiling(subject_column=subject_column)
+        self.extrapolation_dimension = extrapolation_dimension
+        self.num_bootstraps = num_bootstraps
+
+    def __call__(self, assembly, metric):
+        scores = self.collect(assembly=assembly, metric=metric)
+        return self.extrapolate(scores)
+
+    def collect(self, assembly, metric):
+        num_subjects = len(set(assembly[self.subject_column].values))
+        subject_subsamples = self.build_subject_subsamples(num_subjects)
+        scores = []
+        for num_subjects in tqdm(subject_subsamples, desc='num subjects'):
+            selection_combinations = self.iterate_subsets(assembly, num_subjects=num_subjects)
+            for selections, sub_assembly in tqdm(selection_combinations, desc='selections'):
+                score = self.holdout_ceiling(assembly=sub_assembly, metric=metric)
+                score = score.expand_dims('num_subjects')
+                score['num_subjects'] = [num_subjects]
+                for key, selection in selections.items():
+                    expand_dim = f'sub_{key}'
+                    score = score.expand_dims(expand_dim)
+                    score[expand_dim] = [str(selection)]
+                scores.append(score.raw)
+        scores = Score.merge(*scores)
+        assert hasattr(scores, 'neuroid_id')
+        return scores
+
+    def build_subject_subsamples(self, num_subjects):
+        return tuple(range(2, num_subjects + 1))
+
+    def iterate_subsets(self, assembly, num_subjects):
+        subjects = set(assembly[self.subject_column].values)
+        subject_combinations = list(itertools.combinations(sorted(subjects), num_subjects))
+        for sub_subjects in subject_combinations:
+            sub_assembly = assembly[{'neuroid': [subject in sub_subjects
+                                                 for subject in assembly[self.subject_column].values]}]
+            yield {self.subject_column: sub_subjects}, sub_assembly
+
+    def average_collected(self, scores):
+        return scores.median('neuroid')
+
+    def extrapolate(self, ceilings):
+        neuroid_ceilings, bootstrap_params, endpoint_xs = [], [], []
+        for i in trange(len(ceilings[self.extrapolation_dimension]),
+                        desc=f'{self.extrapolation_dimension} extrapolations'):
+            # extrapolate per-neuroid ceiling
+            neuroid_ceiling = ceilings.isel(**{self.extrapolation_dimension: [i]})
+            extrapolated_ceiling = self.extrapolate_neuroid(neuroid_ceiling.squeeze())
+            extrapolated_ceiling = self.add_neuroid_meta(extrapolated_ceiling, neuroid_ceiling)
+            neuroid_ceilings.append(extrapolated_ceiling)
+            # also keep track of bootstrapped parameters
+            neuroid_bootstrap_params = extrapolated_ceiling.bootstrapped_params
+            neuroid_bootstrap_params = self.add_neuroid_meta(neuroid_bootstrap_params, neuroid_ceiling)
+            bootstrap_params.append(neuroid_bootstrap_params)
+            # and endpoints
+            endpoint_x = self.add_neuroid_meta(extrapolated_ceiling.endpoint_x, neuroid_ceiling)
+            endpoint_xs.append(endpoint_x)
+        # merge and add meta
+        self._logger.debug("Merging neuroid ceilings")
+        neuroid_ceilings = manual_merge(*neuroid_ceilings, on=self.extrapolation_dimension)
+        neuroid_ceilings.attrs['raw'] = ceilings
+        self._logger.debug("Merging bootstrap params")
+        bootstrap_params = manual_merge(*bootstrap_params, on=self.extrapolation_dimension)
+        neuroid_ceilings.attrs['bootstrapped_params'] = bootstrap_params
+        self._logger.debug("Merging endpoints")
+        endpoint_xs = manual_merge(*endpoint_xs, on=self.extrapolation_dimension)
+        neuroid_ceilings.attrs['endpoint_x'] = endpoint_xs
+        # aggregate
+        ceiling = self.aggregate_neuroid_ceilings(neuroid_ceilings)
+        return ceiling
+
+    def add_neuroid_meta(self, target, source):
+        target = target.expand_dims(self.extrapolation_dimension)
+        for coord, dims, values in walk_coords(source):
+            if array_is_element(dims, self.extrapolation_dimension):
+                target[coord] = dims, values
+        return target
+
+    def aggregate_neuroid_ceilings(self, neuroid_ceilings):
+        ceiling = neuroid_ceilings.median(self.extrapolation_dimension)
+        ceiling.attrs['bootstrapped_params'] = neuroid_ceilings.bootstrapped_params.median(self.extrapolation_dimension)
+        ceiling.attrs['endpoint_x'] = neuroid_ceilings.endpoint_x.median(self.extrapolation_dimension)
+        ceiling.attrs['raw'] = neuroid_ceilings
+        return ceiling
+
+    def extrapolate_neuroid(self, ceilings):
+        # figure out how many extrapolation x points we have. E.g. for Pereira, not all combinations are possible
+        subject_subsamples = list(sorted(set(ceilings['num_subjects'].values)))
+        rng = RandomState(0)
+        bootstrap_params = []
+        for bootstrap in range(self.num_bootstraps):
+            bootstrapped_scores = []
+            for num_subjects in subject_subsamples:
+                num_scores = ceilings.sel(num_subjects=num_subjects)
+                # the sub_subjects dimension creates nans, get rid of those
+                num_scores = num_scores.dropna(f'sub_{self.subject_column}')
+                assert set(num_scores.dims) == {f'sub_{self.subject_column}', 'split'} or \
+                       set(num_scores.dims) == {f'sub_{self.subject_column}'}
+                # choose from subject subsets and the splits therein, with replacement for variance
+                choices = num_scores.values.flatten()
+                bootstrapped_score = rng.choice(choices, size=len(choices), replace=True)
+                bootstrapped_scores.append(np.mean(bootstrapped_score))
+
+            params = self.fit(subject_subsamples, bootstrapped_scores)
+            params = DataAssembly([params], coords={'bootstrap': [bootstrap], 'param': ['v0', 'tau0']},
+                                  dims=['bootstrap', 'param'])
+            bootstrap_params.append(params)
+        bootstrap_params = merge_data_arrays(bootstrap_params)
+        # find endpoint and error
+        asymptote_threshold = .0005
+        interpolation_xs = np.arange(1000)
+        ys = np.array([v(interpolation_xs, *params) for params in bootstrap_params.values
+                       if not np.isnan(params).any()])
+        median_ys = np.median(ys, axis=0)
+        diffs = np.diff(median_ys)
+        end_x = np.where(diffs < asymptote_threshold)[0].min()  # first x where increase smaller than threshold
+        # put together
+        center = np.median(np.array(bootstrap_params)[:, 0])
+        error_low, error_high = ci_error(ys[:, end_x], center=center)
+        score = Score(center)
+        score.attrs['error_low'] = error_low
+        score.attrs['error_high'] = error_high
+        score.attrs['raw'] = ceilings
+        score.attrs['bootstrapped_params'] = bootstrap_params
+        score.attrs['endpoint_x'] = DataAssembly(end_x)
+        return score
+
+    def fit(self, subject_subsamples, bootstrapped_scores):
+        params, pcov = curve_fit(v, subject_subsamples, bootstrapped_scores,
+                                 # v (i.e. max ceiling) is between 0 and 1, tau0 unconstrained
+                                 bounds=([0, -np.inf], [1, np.inf]))
+        return params
+
+
+class HoldoutSubjectCeiling:
+    def __init__(self, subject_column):
+        self.subject_column = subject_column
+        self._logger = logging.getLogger(fullname(self))
+
+    def __call__(self, assembly, metric):
+        subjects = set(assembly[self.subject_column].values)
+        scores = []
+        iterate_subjects = self.get_subject_iterations(subjects)
+        for subject in tqdm(iterate_subjects, desc='heldout subject'):
+            try:
+                subject_assembly = assembly[{'neuroid': [subject_value == subject
+                                                         for subject_value in assembly[self.subject_column].values]}]
+                # run subject pool as neural candidate
+                subject_pool = subjects - {subject}
+                pool_assembly = assembly[
+                    {'neuroid': [subject in subject_pool for subject in assembly[self.subject_column].values]}]
+                score = self.score(pool_assembly, subject_assembly, metric=metric)
+                # store scores
+                apply_raw = 'raw' in score.attrs and \
+                            not hasattr(score.raw, self.subject_column)  # only propagate if column not part of score
+                score = score.expand_dims(self.subject_column, _apply_raw=apply_raw)
+                score.__setitem__(self.subject_column, [subject], _apply_raw=apply_raw)
+                scores.append(score)
+            except NoOverlapException as e:
+                self._logger.debug(f"Ignoring no overlap {e}")
+                continue  # ignore
+            except ValueError as e:
+                if "Found array with" in str(e):
+                    self._logger.debug(f"Ignoring empty array {e}")
+                    continue
+                else:
+                    raise e
+
+        scores = Score.merge(*scores)
+        score = apply_aggregate(lambda scores: scores.mean(self.subject_column), scores)
+        score.attrs['error'] = scores.std(self.subject_column)
+        return scores
+
+    def get_subject_iterations(self, subjects):
+        return subjects  # iterate over all subjects
+
+    def score(self, pool_assembly, subject_assembly, metric):
+        return metric(pool_assembly, subject_assembly)
+
+
+class NoOverlapException(Exception):
+    pass

brainscore_language/benchmarks/blank2014/test.py

@@ -0,0 +1,75 @@
+import copy
+import numpy as np
+from numpy.random import RandomState
+from pytest import approx
+from typing import Callable, Union, List
+
+from brainio.assemblies import NeuroidAssembly
+from brainscore_language import ArtificialSubject, load_benchmark
+
+
+class TestBenchmark:
+    class DummyModel(ArtificialSubject):
+        def __init__(self, activity_for_text: Callable[[Union[str, List[str]]], NeuroidAssembly]):
+            self.activity_for_text = activity_for_text
+
+        def digest_text(self, stimuli):
+            neural_activity = self.activity_for_text(stimuli)
+            return {'neural': neural_activity}
+
+        def start_neural_recording(self, recording_target: ArtificialSubject.RecordingTarget,
+                                   recording_type: ArtificialSubject.RecordingType):
+            assert recording_target == ArtificialSubject.RecordingTarget.language_system
+            assert recording_type == ArtificialSubject.RecordingType.fMRI
+
+    def test_dummy_bad(self):
+        random_state = RandomState(0)
+
+        def activity_for_text(stimuli: Union[str, List[str]]) -> NeuroidAssembly:
+            num_stimuli = len(stimuli)
+            num_neuroids = 25
+            neural_activity = random_state.random(size=(num_stimuli, num_neuroids))  # presentation x neuroid
+            neural_activity = NeuroidAssembly(neural_activity,
+                                              coords={'stimulus_seq': ('presentation', np.arange(num_stimuli)),
+                                                      'stimulus_num': ('presentation', np.arange(num_stimuli)),
+                                                      'neuroid_id': ('neuroid', np.arange(num_neuroids)),
+                                                      'region': ('neuroid', ['some_region'] * num_neuroids)},
+                                              dims=['presentation', 'neuroid'])
+            neural_activity['stimulus'] = 'presentation', stimuli  # copy over
+            return neural_activity
+
+        benchmark = load_benchmark('Blank2014-linear')
+        dummy_model = TestBenchmark.DummyModel(activity_for_text=activity_for_text)
+        score = benchmark(dummy_model)
+        assert score == 0
+
+    def test_exact(self):
+        benchmark = load_benchmark('Blank2014-linear')
+        exact_data = copy.deepcopy(benchmark.data)
+
+        def activity_for_text(stimuli: Union[str, List[str]]) -> NeuroidAssembly:
+            passage_activity = exact_data[{'presentation': [
+                list(exact_data['stimulus'].values).index(stimulus) for stimulus in stimuli]}]
+            # remove stimulus_id and stimulus coordinates to not trip up benchmark
+            passage_activity = passage_activity.reset_index('presentation')
+            del passage_activity['stimulus_id']
+            passage_activity = NeuroidAssembly(passage_activity)  # index
+            return passage_activity
+
+        dummy_model = TestBenchmark.DummyModel(activity_for_text=activity_for_text)
+        score = benchmark(dummy_model)
+        assert score == approx(1)
+
+    def test_ceiling(self):
+        benchmark = load_benchmark(f'Blank2014-linear')
+        ceiling = benchmark.ceiling
+        assert ceiling == approx(.21026591, abs=.0005)
+
+    def test_ceiling_raw(self):
+        benchmark = load_benchmark(f'Blank2014-linear')
+        ceiling = benchmark.ceiling
+        assert hasattr(ceiling, 'raw')
+        assert set(ceiling.raw.dims) == {'neuroid'}
+        assert ceiling.raw.median() == ceiling
+        assert hasattr(ceiling.raw, 'raw')
+        assert set(ceiling.raw.raw.dims) == {'sub_subject_id', 'num_subjects', 'split', 'neuroid'}

brainscore_language/benchmarks/fedorenko2016/__init__.py

@@ -0,0 +1,5 @@
+from brainscore_language import benchmark_registry
+from .benchmark import Fedorenko2016_ridge, Fedorenko2016_linear
+
+benchmark_registry['Fedorenko2016-linear'] = Fedorenko2016_linear
+benchmark_registry['Fedorenko2016-ridge'] = Fedorenko2016_ridge