brainscore-vision 2.1__py3-none-any.whl

tests/test_metric_helpers/test_temporal.py

@@ -0,0 +1,80 @@
+import numpy as np
+import scipy.stats
+from pytest import approx
+from sklearn.linear_model import LinearRegression
+
+from brainio.assemblies import NeuroidAssembly
+from brainscore_vision.metric_helpers.xarray_utils import XarrayRegression, XarrayCorrelation
+from brainscore_vision.metric_helpers.temporal import PerTime, SpanTime, PerTimeRegression, SpanTimeRegression
+
+
+class TestMetricHelpers:
+    def test_pertime_ops(self):
+        jumbled_source = NeuroidAssembly(np.random.rand(500, 10, 20),
+                                         coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                 'image_meta': ('presentation', [0] * 500),
+                                                 'neuroid_id': ('neuroid', list(reversed(range(10)))),
+                                                 'neuroid_meta': ('neuroid', [0] * 10),
+                                                 'time_bin_start': ('time_bin', np.arange(0, 400, 20)),
+                                                 'time_bin_end': ('time_bin', np.arange(20, 420, 20))},
+                                         dims=['presentation', 'neuroid', 'time_bin'])
+        mean_neuroid = lambda arr: arr.mean('neuroid')
+        pertime_mean_neuroid = PerTime(mean_neuroid)
+        output = pertime_mean_neuroid(jumbled_source)
+        output = output.transpose('presentation', 'time_bin')
+        target = jumbled_source.transpose('presentation', 'time_bin', 'neuroid').mean('neuroid')
+        assert (output == approx(target)).all().item()
+
+    def test_spantime_ops(self):
+        jumbled_source = NeuroidAssembly(np.random.rand(500, 10, 20),
+                                         coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                 'image_meta': ('presentation', [0] * 500),
+                                                 'neuroid_id': ('neuroid', list(reversed(range(10)))),
+                                                 'neuroid_meta': ('neuroid', [0] * 10),
+                                                 'time_bin_start': ('time_bin', np.arange(0, 400, 20)),
+                                                 'time_bin_end': ('time_bin', np.arange(20, 420, 20))},
+                                         dims=['presentation', 'neuroid', 'time_bin'])
+        mean_presentation = lambda arr: arr.mean("presentation")
+        spantime_mean_presentation = SpanTime(mean_presentation)
+        output = spantime_mean_presentation(jumbled_source)
+        output = output.transpose('neuroid')
+        target = jumbled_source.transpose('presentation', 'time_bin', 'neuroid').mean('presentation').mean('time_bin')
+        assert (output == approx(target)).all().item()
+
+    def test_pertime_regression(self):
+        jumbled_source = NeuroidAssembly(np.random.rand(500, 10, 20),
+                                         coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                 'image_meta': ('presentation', [0] * 500),
+                                                 'neuroid_id': ('neuroid', list(reversed(range(10)))),
+                                                 'neuroid_meta': ('neuroid', [0] * 10),
+                                                 'time_bin_start': ('time_bin', np.arange(0, 400, 20)),
+                                                 'time_bin_end': ('time_bin', np.arange(20, 420, 20))},
+                                         dims=['presentation', 'neuroid', 'time_bin'])
+        target = jumbled_source.sortby(['stimulus_id', 'neuroid_id'])
+        pertime_regression = PerTimeRegression(XarrayRegression(LinearRegression()))
+        pertime_regression.fit(jumbled_source, target)
+        prediction = pertime_regression.predict(jumbled_source)
+        prediction = prediction.transpose(*target.dims)
+        # do not test for alignment of metadata - it is only important that the data is well-aligned with the metadata.
+        np.testing.assert_array_almost_equal(prediction.sortby(['stimulus_id', 'neuroid_id', 'time_bin']).values,
+                                             target.sortby(['stimulus_id', 'neuroid_id', 'time_bin']).values)
+
+
+    def test_spantime_regression(self):
+        jumbled_source = NeuroidAssembly(np.random.rand(500, 10, 20),
+                                         coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                 'image_meta': ('presentation', [0] * 500),
+                                                 'neuroid_id': ('neuroid', list(reversed(range(10)))),
+                                                 'neuroid_meta': ('neuroid', [0] * 10),
+                                                 'time_bin_start': ('time_bin', np.arange(0, 400, 20)),
+                                                 'time_bin_end': ('time_bin', np.arange(20, 420, 20))},
+                                         dims=['presentation', 'neuroid', 'time_bin'])
+        target = jumbled_source.sortby(['stimulus_id', 'neuroid_id'])
+        spantime_regression = SpanTimeRegression(XarrayRegression(LinearRegression()))
+        spantime_regression.fit(jumbled_source, target)
+        prediction = spantime_regression.predict(jumbled_source)
+        prediction = prediction.transpose(*target.dims)
+        # do not test for alignment of metadata - it is only important that the data is well-aligned with the metadata.
+        np.testing.assert_array_almost_equal(prediction.sortby(['stimulus_id', 'neuroid_id', 'time_bin']).values,
+                                             target.sortby(['stimulus_id', 'neuroid_id', 'time_bin']).values)
+

tests/test_metric_helpers/test_transformations.py

@@ -0,0 +1,171 @@
+import numpy as np
+
+from brainio.assemblies import NeuroidAssembly, DataAssembly
+from brainscore_core.metrics import Metric, Score
+from brainscore_vision.metric_helpers.transformations import CartesianProduct, CrossValidation, \
+    CrossValidationSingle
+
+
+class TestCrossValidationSingle:
+    class MetricPlaceholder(Metric):
+        def __init__(self):
+            super(TestCrossValidationSingle.MetricPlaceholder, self).__init__()
+            self.train_assemblies = []
+            self.test_assemblies = []
+
+        def __call__(self, train, test):
+            self.train_assemblies.append(train)
+            self.test_assemblies.append(test)
+            return DataAssembly(0)
+
+    def test_presentation_neuroid(self):
+        assembly = NeuroidAssembly(np.random.rand(500, 10),
+                                   coords={'stimulus_id': ('presentation', list(range(500))),
+                                           'image_meta': ('presentation', [0] * 500),
+                                           'neuroid_id': ('neuroid', list(range(10))),
+                                           'neuroid_meta': ('neuroid', [0] * 10)},
+                                   dims=['presentation', 'neuroid'])
+        cv = CrossValidationSingle(splits=10, stratification_coord=None)
+        metric = self.MetricPlaceholder()
+        score = cv(assembly, apply=metric)
+        assert len(metric.train_assemblies) == len(metric.test_assemblies) == 10
+        assert len(score.attrs['raw']['split']) == 10
+
+    def test_repeated_dim(self):
+        """
+        necessary for cross-validation over RDMs
+        """
+        assembly = NeuroidAssembly(np.random.rand(50, 50),
+                                   coords={'stimulus_id': ('presentation', list(range(50))),
+                                           'image_meta': ('presentation', [0] * 50)},
+                                   dims=['presentation', 'presentation'])
+        cv = CrossValidationSingle(splits=10, train_size=.9, stratification_coord=None)
+        metric = self.MetricPlaceholder()
+        score = cv(assembly, apply=metric)
+        assert len(metric.train_assemblies) == len(metric.test_assemblies) == 10
+        assert all(assembly.shape == (45, 45) for assembly in metric.train_assemblies)
+        assert all(assembly.shape == (5, 5) for assembly in metric.test_assemblies)
+        assert len(score.attrs['raw']['split']) == 10
+
+
+class TestCrossValidation:
+    class MetricPlaceholder(Metric):
+        def __init__(self):
+            super(TestCrossValidation.MetricPlaceholder, self).__init__()
+            self.train_source_assemblies = []
+            self.test_source_assemblies = []
+            self.train_target_assemblies = []
+            self.test_target_assemblies = []
+
+        def __call__(self, train_source, train_target, test_source, test_target):
+            assert sorted(train_source['stimulus_id'].values) == sorted(train_target['stimulus_id'].values)
+            assert sorted(test_source['stimulus_id'].values) == sorted(test_target['stimulus_id'].values)
+            self.train_source_assemblies.append(train_source)
+            self.train_target_assemblies.append(train_target)
+            self.test_source_assemblies.append(test_source)
+            self.test_target_assemblies.append(test_target)
+            return DataAssembly(0)
+
+    def test_misaligned(self):
+        jumbled_source = NeuroidAssembly(np.random.rand(500, 10),
+                                         coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                 'image_meta': ('presentation', [0] * 500),
+                                                 'neuroid_id': ('neuroid', list(reversed(range(10)))),
+                                                 'neuroid_meta': ('neuroid', [0] * 10)},
+                                         dims=['presentation', 'neuroid'])
+        target = jumbled_source.sortby(['stimulus_id', 'neuroid_id'])
+        cv = CrossValidation(splits=10, stratification_coord=None)
+        metric = self.MetricPlaceholder()
+        score = cv(jumbled_source, target, apply=metric)
+        assert len(metric.train_source_assemblies) == len(metric.test_source_assemblies) == \
+               len(metric.train_target_assemblies) == len(metric.test_target_assemblies) == 10
+        assert len(score.attrs['raw']) == 10
+
+
+class TestCartesianProduct:
+    def test_no_division3(self):
+        self._test_no_division_apply_manually(3)
+
+    def test_no_division100(self):
+        self._test_no_division_apply_manually(100)
+
+    def _test_no_division_apply_manually(self, num_values):
+        assembly = np.random.rand(num_values)
+        assembly = NeuroidAssembly(assembly, coords={'neuroid': list(range(len(assembly)))}, dims=['neuroid'])
+        transformation = CartesianProduct()
+        generator = transformation.pipe(assembly)
+        for divided_assembly in generator:  # should run only once
+            np.testing.assert_array_equal(assembly.values, divided_assembly[0])
+            done = generator.send(DataAssembly([0], coords={'split': [0]}, dims=['split']))
+            assert done
+            break
+        similarity = next(generator)
+        np.testing.assert_array_equal(similarity.shape, [1])
+        np.testing.assert_array_equal(similarity.dims, ['split'])
+        assert similarity[0] == 0
+
+    class MetricPlaceholder(Metric):
+        def __init__(self):
+            super(TestCartesianProduct.MetricPlaceholder, self).__init__()
+            self.assemblies = []
+
+        def __call__(self, assembly):
+            self.assemblies.append(assembly)
+            return DataAssembly([0])
+
+    def test_one_division(self):
+        assembly = np.random.rand(100, 3)
+        assembly = NeuroidAssembly(
+            assembly,
+            coords={'neuroid': list(range(len(assembly))), 'division_coord': list(range(assembly.shape[1]))},
+            dims=['neuroid', 'division_coord'])
+        transformation = CartesianProduct(dividers=['division_coord'])
+        placeholder = self.MetricPlaceholder()
+        transformation(assembly, apply=placeholder)
+        assert len(assembly['division_coord']) == len(placeholder.assemblies)
+        targets = [assembly.sel(division_coord=i) for i in assembly['division_coord'].values]
+        for target in targets:
+            match = any([actual == target] for actual in placeholder.assemblies)
+            assert match, "expected divided assembly not found: {target}"
+
+    def test_one_division_similarity_dim_last(self):
+        assembly = np.random.rand(3, 100)
+        assembly = NeuroidAssembly(
+            assembly,
+            coords={'neuroid': list(range(assembly.shape[1])), 'division_coord': list(range(assembly.shape[0]))},
+            dims=['division_coord', 'neuroid'])
+        transformation = CartesianProduct(dividers=['division_coord'])
+        placeholder = self.MetricPlaceholder()
+        transformation(assembly, apply=placeholder)
+        assert len(assembly['division_coord']) == len(placeholder.assemblies)
+        targets = [assembly.sel(division_coord=i) for i in assembly['division_coord'].values]
+        for target in targets:
+            match = any([actual == target] for actual in placeholder.assemblies)
+            assert match, "expected divided assembly not found: {target}"
+
+    def test_no_expand_raw_level(self):
+        assembly = np.random.rand(3, 100)
+        assembly = NeuroidAssembly(
+            assembly,
+            coords={'neuroid': list(range(assembly.shape[1])), 'division_coord': list(range(assembly.shape[0]))},
+            dims=['division_coord', 'neuroid'])
+        transformation = CartesianProduct(dividers=['division_coord'])
+
+        class RawMetricPlaceholder(Metric):
+            def __call__(self, assembly, *args, **kwargs):
+                result = Score([assembly.values[0]], dims=['dim'])
+                raw = Score(result.copy(), coords={
+                    'dim_id': ('dim', [assembly.values[1]]),
+                    'division_coord': ('dim', [assembly.values[2]])
+                })
+                result.attrs['raw'] = raw
+                return result
+
+        metric = RawMetricPlaceholder()
+        result = transformation(assembly, apply=metric)
+        assert result.dims == ("division_coord", "dim")
+        assert hasattr(result, 'raw')
+        assert result.raw.dims == ("dim",)
+        assert 'division_coord' not in result.raw.dims  # no dimension
+        assert hasattr(result.raw, 'division_coord')  # but a level
+        assert result.raw["dim"].variable.level_names == ["dim_id", "division_coord"]

tests/test_metric_helpers/test_xarray_utils.py

@@ -0,0 +1,85 @@
+import numpy as np
+import scipy.stats
+from pytest import approx
+from sklearn.linear_model import LinearRegression
+
+from brainio.assemblies import NeuroidAssembly
+from brainscore_vision.metric_helpers.xarray_utils import XarrayRegression, XarrayCorrelation
+
+
+class TestXarrayRegression:
+    def test_fitpredict_alignment(self):
+        jumbled_source = NeuroidAssembly(np.random.rand(500, 10),
+                                         coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                 'image_meta': ('presentation', [0] * 500),
+                                                 'neuroid_id': ('neuroid', list(reversed(range(10)))),
+                                                 'neuroid_meta': ('neuroid', [0] * 10)},
+                                         dims=['presentation', 'neuroid'])
+        target = jumbled_source.sortby(['stimulus_id', 'neuroid_id'])
+        regression = XarrayRegression(LinearRegression())
+        regression.fit(jumbled_source, target)
+        prediction = regression.predict(jumbled_source)
+        # do not test for alignment of metadata - it is only important that the data is well-aligned with the metadata.
+        np.testing.assert_array_almost_equal(prediction.sortby(['stimulus_id', 'neuroid_id']).values,
+                                             target.sortby(['stimulus_id', 'neuroid_id']).values)
+
+    def test_neuroid_single_coord(self):
+        jumbled_source = NeuroidAssembly(np.random.rand(500, 10),
+                                         coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                 'image_meta': ('presentation', [0] * 500),
+                                                 'neuroid_id': ('neuroid_id', list(reversed(range(10))))},
+                                         dims=['presentation', 'neuroid_id']).stack(neuroid=['neuroid_id'])
+        target = jumbled_source.sortby(['stimulus_id', 'neuroid_id'])
+        regression = XarrayRegression(LinearRegression())
+        regression.fit(jumbled_source, target)
+        prediction = regression.predict(jumbled_source)
+        assert set(prediction.dims) == {'presentation', 'neuroid'}
+        assert len(prediction['neuroid_id']) == 10
+
+
+class TestXarrayCorrelation:
+    def test_dimensions(self):
+        prediction = NeuroidAssembly(np.random.rand(500, 10),
+                                     coords={'stimulus_id': ('presentation', list(range(500))),
+                                             'image_meta': ('presentation', [0] * 500),
+                                             'neuroid_id': ('neuroid', list(range(10))),
+                                             'neuroid_meta': ('neuroid', [0] * 10)},
+                                     dims=['presentation', 'neuroid'])
+        correlation = XarrayCorrelation(lambda a, b: (1, 0))
+        score = correlation(prediction, prediction)
+        np.testing.assert_array_equal(score.dims, ['neuroid'])
+        np.testing.assert_array_equal(score.shape, [10])
+
+    def test_correlation(self):
+        prediction = NeuroidAssembly(np.random.rand(500, 10),
+                                     coords={'stimulus_id': ('presentation', list(range(500))),
+                                             'image_meta': ('presentation', [0] * 500),
+                                             'neuroid_id': ('neuroid', list(range(10))),
+                                             'neuroid_meta': ('neuroid', [0] * 10)},
+                                     dims=['presentation', 'neuroid'])
+        correlation = XarrayCorrelation(lambda a, b: (1, 0))
+        score = correlation(prediction, prediction)
+        assert all(score == approx(1))
+
+    def test_alignment(self):
+        jumbled_prediction = NeuroidAssembly(np.random.rand(500, 10),
+                                             coords={'stimulus_id': ('presentation', list(reversed(range(500)))),
+                                                     'image_meta': ('presentation', [0] * 500),
+                                                     'neuroid_id': ('neuroid', list(reversed(range(10)))),
+                                                     'neuroid_meta': ('neuroid', [0] * 10)},
+                                             dims=['presentation', 'neuroid'])
+        prediction = jumbled_prediction.sortby(['stimulus_id', 'neuroid_id'])
+        correlation = XarrayCorrelation(scipy.stats.pearsonr)
+        score = correlation(jumbled_prediction, prediction)
+        assert all(score == approx(1))
+
+    def test_neuroid_single_coord(self):
+        prediction = NeuroidAssembly(np.random.rand(500, 10),
+                                     coords={'stimulus_id': ('presentation', list(range(500))),
+                                             'image_meta': ('presentation', [0] * 500),
+                                             'neuroid_id': ('neuroid_id', list(range(10)))},
+                                     dims=['presentation', 'neuroid_id']).stack(neuroid=['neuroid_id'])
+        correlation = XarrayCorrelation(lambda a, b: (1, 0))
+        score = correlation(prediction, prediction)
+        np.testing.assert_array_equal(score.dims, ['neuroid'])
+        assert len(score['neuroid']) == 10

tests/test_model_helpers/__init__.py

@@ -0,0 +1,6 @@
+import logging
+import sys
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+for disable_logger in ['s3transfer', 'botocore', 'boto3', 'urllib3', 'peewee', 'PIL']:
+    logging.getLogger(disable_logger).setLevel(logging.WARNING)