brainscore-vision 2.1__py3-none-any.whl

tests/test_model_helpers/test_generic_plugin_tests.py

@@ -0,0 +1,310 @@
+from collections import namedtuple
+from typing import List, Tuple
+
+import pytest
+from numpy.random import RandomState
+
+from brainio.assemblies import BehavioralAssembly, NeuroidAssembly
+from brainio.stimuli import StimulusSet
+from brainscore_vision import BrainModel
+# Note that we cannot import `from brainscore_vision.model_helpers.generic_plugin_tests import test_*` directly
+# since this would expose the `test_*` methods during pytest test collection
+from brainscore_vision.model_helpers import generic_plugin_tests
+
+
+class TestIdentifier:
+    ModelClass = namedtuple("DummyModel", field_names=['identifier'])
+
+    def test_no_identifier_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(identifier=None)
+        with pytest.raises(AssertionError):
+            generic_plugin_tests.test_identifier(identifier='dummy')
+
+    def test_proper_identifier(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(identifier='dummy')
+        generic_plugin_tests.test_identifier('dummy')
+
+
+class TestVisualDegrees:
+    ModelClass = namedtuple("DummyModel", field_names=['visual_degrees'])
+
+    def test_proper_degrees(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(visual_degrees=lambda: 8)
+        generic_plugin_tests.test_visual_degrees('dummy')
+
+    def test_degrees_0_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(visual_degrees=lambda: 0)
+        with pytest.raises(AssertionError):
+            generic_plugin_tests.test_visual_degrees(identifier='dummy')
+
+    def test_degrees_None_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(visual_degrees=lambda: None)
+        with pytest.raises(TypeError):
+            generic_plugin_tests.test_visual_degrees(identifier='dummy')
+
+    def test_no_function_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = object()
+        with pytest.raises(AttributeError):
+            generic_plugin_tests.test_visual_degrees(identifier='dummy')
+
+
+class TestStartTaskOrRecording:
+    ModelClass = namedtuple("DummyModel", field_names=['start_task', 'start_recording'], defaults=[None, None])
+
+    def test_supports_all(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(start_task=lambda task, fitting_stimuli=None: None,
+                                                 start_recording=lambda recording_target, time_bins: None)
+        generic_plugin_tests.test_start_task_or_recording('dummy')
+
+    def test_supports_task_only(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(start_task=lambda task, fitting_stimuli=None: None)
+        generic_plugin_tests.test_start_task_or_recording('dummy')
+
+    def test_supports_task_label_only(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def start_task(task: BrainModel.Task, fitting_stimuli):
+            if task != BrainModel.Task.label:
+                raise NotImplementedError()
+
+        load_mock.return_value = self.ModelClass(start_task=start_task)
+        generic_plugin_tests.test_start_task_or_recording('dummy')
+
+    def test_supports_recording_only(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass(start_recording=lambda recording_target, time_bins: None)
+        generic_plugin_tests.test_start_task_or_recording('dummy')
+
+    def test_supports_recording_V1_only(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def start_recording(recording_target: BrainModel.RecordingTarget, time_bins):
+            if recording_target != BrainModel.RecordingTarget.V1:
+                raise NotImplementedError()
+
+        load_mock.return_value = self.ModelClass(start_recording=start_recording)
+        generic_plugin_tests.test_start_task_or_recording('dummy')
+
+    def test_supports_recording_IT_only(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def start_recording(recording_target: BrainModel.RecordingTarget, time_bins):
+            if recording_target != BrainModel.RecordingTarget.IT:
+                raise NotImplementedError()
+
+        load_mock.return_value = self.ModelClass(start_recording=start_recording)
+        generic_plugin_tests.test_start_task_or_recording('dummy')
+
+    def test_supports_None_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+        load_mock.return_value = self.ModelClass()
+        with pytest.raises(AssertionError):
+            generic_plugin_tests.test_start_task_or_recording('dummy')
+
+
+class TestLookAtBehaviorProbabilities:
+    class ModelMock(BrainModel):
+        def __init__(self, assembly_factory):
+            self.assembly_factory = assembly_factory
+            self.fitting_stimuli = None
+
+        def start_task(self, task: BrainModel.Task, fitting_stimuli=None):
+            assert task == BrainModel.Task.probabilities
+            assert fitting_stimuli is not None
+            self.fitting_stimuli = fitting_stimuli
+
+        def look_at(self, stimuli: StimulusSet, number_of_trials=1) -> BehavioralAssembly:
+            return self.assembly_factory(stimuli, self.fitting_stimuli)
+
+    def test_proper_format(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, fitting_stimuli: StimulusSet) -> BehavioralAssembly:
+            rng = RandomState(1)
+            choices = list(sorted(set(fitting_stimuli['image_label'])))
+            probabilities = rng.rand(len(stimuli), len(choices))
+            return BehavioralAssembly(probabilities,
+                                      coords={**{coord: ('presentation', stimuli[coord]) for coord in stimuli},
+                                              **{'choice': ('choice', choices)}},
+                                      dims=['presentation', 'choice'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        generic_plugin_tests.test_look_at_behavior_probabilities('dummy')
+
+    def test_None_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        load_mock.return_value = self.ModelMock(lambda stimuli, fitting_stimuli: None)
+        with pytest.raises(AttributeError):
+            generic_plugin_tests.test_look_at_behavior_probabilities('dummy')
+
+    def test_no_stimulus_id_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, fitting_stimuli: StimulusSet) -> BehavioralAssembly:
+            rng = RandomState(1)
+            choices = list(sorted(set(fitting_stimuli['image_label'])))
+            probabilities = rng.rand(len(stimuli), len(choices))
+            return BehavioralAssembly(probabilities,
+                                      coords={**{coord: ('presentation', stimuli[coord]) for coord in stimuli
+                                                 if coord != 'stimulus_id'},
+                                              **{'choice': ('choice', choices)}},
+                                      dims=['presentation', 'choice'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        with pytest.raises(KeyError):
+            generic_plugin_tests.test_look_at_behavior_probabilities('dummy')
+
+    def test_no_choice_dim_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, fitting_stimuli: StimulusSet) -> BehavioralAssembly:
+            rng = RandomState(1)
+            choices = list(sorted(set(fitting_stimuli['image_label'])))
+            # improperly output labels instead of probabilities
+            return BehavioralAssembly(rng.choice(choices, size=len(stimuli)),
+                                      coords={coord: ('presentation', stimuli[coord]) for coord in stimuli
+                                              if coord != 'stimulus_id'},
+                                      dims=['presentation'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        with pytest.raises(AssertionError):
+            generic_plugin_tests.test_look_at_behavior_probabilities('dummy')
+
+
+class TestLookAtNeuralV1:
+    class ModelMock(BrainModel):
+        def __init__(self, assembly_factory):
+            self.assembly_factory = assembly_factory
+            self.time_bins = None
+
+        def start_recording(self, recording_target: BrainModel.RecordingTarget, time_bins: List[Tuple[int, int]]):
+            assert recording_target == BrainModel.RecordingTarget.V1
+            assert len(time_bins) == 1
+            self.time_bins = time_bins
+
+        def look_at(self, stimuli: StimulusSet, number_of_trials=1) -> NeuroidAssembly:
+            return self.assembly_factory(stimuli, self.time_bins)
+
+    def test_proper_format_with_timebin(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, time_bins: List[Tuple[int, int]]) -> NeuroidAssembly:
+            num_units = 300
+            rng = RandomState(1)
+            neural_site_predictions = rng.rand(len(stimuli), num_units)
+            return NeuroidAssembly([neural_site_predictions],
+                                   coords={**{coord: ('presentation', stimuli[coord]) for coord in stimuli},
+                                           **{'neuroid_id': ('neuroid', [f"nid{i}" for i in range(num_units)]),
+                                              'region': ('neuroid', ['V1'] * num_units),
+                                              'layer': ('neuroid', ['dummylayer'] * num_units),
+                                              'time_bin_start': ('time_bin', [time_bins[0][0]]),
+                                              'time_bin_end': ('time_bin', [time_bins[0][1]]),
+                                              }},
+                                   dims=['time_bin', 'presentation', 'neuroid'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        generic_plugin_tests.test_look_at_neural_V1('dummy')
+
+    def test_proper_format_without_timebin(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, time_bins: List[Tuple[int, int]]) -> NeuroidAssembly:
+            num_units = 300
+            rng = RandomState(1)
+            neural_site_predictions = rng.rand(len(stimuli), num_units)
+            return NeuroidAssembly(neural_site_predictions,
+                                   coords={**{coord: ('presentation', stimuli[coord]) for coord in stimuli},
+                                           **{'neuroid_id': ('neuroid', [f"nid{i}" for i in range(num_units)]),
+                                              'region': ('neuroid', ['V1'] * num_units),
+                                              'layer': ('neuroid', ['dummylayer'] * num_units),
+                                              }},
+                                   dims=['presentation', 'neuroid'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        generic_plugin_tests.test_look_at_neural_V1('dummy')
+
+    def test_None_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        load_mock.return_value = self.ModelMock(lambda stimuli, time_bins: None)
+        with pytest.raises(TypeError):
+            generic_plugin_tests.test_look_at_neural_V1('dummy')
+
+    def test_no_stimulus_id_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, time_bins: List[Tuple[int, int]]) -> NeuroidAssembly:
+            num_units = 300
+            rng = RandomState(1)
+            neural_site_predictions = rng.rand(len(stimuli), num_units)
+            return NeuroidAssembly(neural_site_predictions,
+                                   coords={**{coord: ('presentation', stimuli[coord]) for coord in stimuli
+                                              if coord != 'stimulus_id'},
+                                           **{'neuroid_id': ('neuroid', [f"nid{i}" for i in range(num_units)]),
+                                              'region': ('neuroid', ['V1'] * num_units),
+                                              'layer': ('neuroid', ['dummylayer'] * num_units),
+                                              }},
+                                   dims=['presentation', 'neuroid'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        with pytest.raises(KeyError):
+            generic_plugin_tests.test_look_at_neural_V1('dummy')
+
+    def test_no_neuroid_id_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, time_bins: List[Tuple[int, int]]) -> NeuroidAssembly:
+            num_units = 300
+            rng = RandomState(1)
+            neural_site_predictions = rng.rand(len(stimuli), num_units)
+            return NeuroidAssembly(neural_site_predictions,
+                                   coords={**{coord: ('presentation', stimuli[coord]) for coord in stimuli},
+                                           **{'neuroid_num': ('neuroid', [f"nnum{i}" for i in range(num_units)]),
+                                              'region': ('neuroid', ['V1'] * num_units),
+                                              'layer': ('neuroid', ['dummylayer'] * num_units),
+                                              }},
+                                   dims=['presentation', 'neuroid'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        with pytest.raises(KeyError):
+            generic_plugin_tests.test_look_at_neural_V1('dummy')
+
+    def test_no_neuroid_dim_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, time_bins: List[Tuple[int, int]]) -> NeuroidAssembly:
+            rng = RandomState(1)
+            neural_site_predictions = rng.rand(len(stimuli))
+            return NeuroidAssembly(neural_site_predictions,
+                                   coords={coord: ('presentation', stimuli[coord]) for coord in stimuli},
+                                   dims=['presentation'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        with pytest.raises(KeyError):
+            generic_plugin_tests.test_look_at_neural_V1('dummy')
+
+    def test_no_neuroid_dim_with_timebin_fails(self, mocker):
+        load_mock = mocker.patch('brainscore_vision.model_helpers.generic_plugin_tests.load_model')
+
+        def assembly_maker(stimuli: StimulusSet, time_bins: List[Tuple[int, int]]) -> NeuroidAssembly:
+            rng = RandomState(1)
+            neural_site_predictions = rng.rand(len(stimuli))
+            return NeuroidAssembly([neural_site_predictions],
+                                   coords={**{coord: ('presentation', stimuli[coord]) for coord in stimuli},
+                                           **{'time_bin_start': ('time_bin', [time_bins[0][0]]),
+                                              'time_bin_end': ('time_bin', [time_bins[0][1]]),
+                                              }},
+                                   dims=['time_bin', 'presentation'])
+
+        load_mock.return_value = self.ModelMock(assembly_maker)
+        with pytest.raises(KeyError):
+            generic_plugin_tests.test_look_at_neural_V1('dummy')

tests/test_model_helpers/test_imports.py

@@ -0,0 +1,10 @@
+# noinspection PyUnresolvedReferences
+class TestImport:
+    def test_model_helpers(self):
+        import brainscore_vision.model_helpers
+
+    def test_activations_extractor(self):
+        from brainscore_vision.model_helpers.activations.core import ActivationsExtractorHelper
+
+    def test_model_commitment(self):
+        from brainscore_vision.model_helpers.brain_transformation import ModelCommitment

tests/test_model_helpers/test_s3.py

@@ -0,0 +1,38 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+from brainscore_vision.model_helpers import load_folder_from_s3, load_weight_file
+
+
+class TestLoadFolderFromS3:
+    save_directory = Path(__file__).parent / 'test_loader_folder_from_s3'        
+
+    @pytest.mark.private_access
+    def test_model_weights(self):
+        try:
+            filename_version_sha = [
+                ('.data-00000-of-00001', 'yYBXN7uf57Y70EdQLchC_dMU5KO6GkIi', 'fef2a64f8c591f5d7562677272c91dcf88989d53'),
+                ('.index', '7SJtRE0pdSahjajwjW5c_nVPVGBKn3q2', 'ba9e531243cb87de8562152ab8a22cdb3d218c3b'),
+                ('checkpoint', '7rAEuKAwsWAmjT9OvXMsPOaQkogaxMMC', 'a39e7f800a473d18781931c26f84eb3399bdb484')]
+            load_folder_from_s3(
+                bucket='brainscore-vision', folder_path='models/4o_model_submission/rcnn_cand_4o_weights_10ep',
+                filename_version_sha=filename_version_sha,
+                save_directory=str(self.save_directory))
+            assert self.save_directory.is_dir()
+            assert len(list(self.save_directory.glob('*'))) == 3
+            for filename, _, _ in filename_version_sha:
+                assert (self.save_directory / filename).is_file()
+        finally:
+            shutil.rmtree(self.save_directory, ignore_errors=True)
+
+    @pytest.mark.private_access
+    def test_load_weight_file(self):
+        relative_path = 'resnet18_lr0.05_epoch60/model_best.pth.tar'
+        local_path = load_weight_file(
+            bucket='brainscore-vision', relative_path=relative_path,
+            version_id='H1en67w4bNnXY_rJ2CXKzvzFpsqGm_MG', sha1='55a627678dd5670c945ae49dcbf83924c7772cd6'
+        )
+        assert local_path.is_file()
+        assert str(local_path).endswith(str(relative_path))

tests/test_models.py

@@ -0,0 +1,15 @@
+# ##
+# generic tests for model plugins
+# ##
+
+import pytest
+
+import brainscore_vision
+
+
+@pytest.mark.travis_slow
+@pytest.mark.parametrize('identifier',
+                         ['pixels', 'alexnet'])
+def test_has_identifier(identifier):
+    model = brainscore_vision.load_model(identifier)
+    assert model.identifier == identifier

tests/test_submission/mock_config.py

@@ -0,0 +1,3 @@
+from brainscore_vision.submission import config
+test_database = 'brainscore-ohio-test'
+config.get_database_secret = lambda: test_database

tests/test_submission/test_actions_helpers.py

@@ -0,0 +1,67 @@
+import pytest
+from subprocess import call
+
+from brainscore_vision.submission.actions_helpers import BASE_URL, get_pr_num_from_head, get_data, get_statuses_result, are_all_tests_passing, is_labeled_automerge, get_pr_head_from_github_event
+
+PR_HEAD_SHA = '209e6c81d39179fd161a1bd3a5845682170abfd2'
+PR_BRANCH_NAME = 'web_submission_11/add_plugins'
+
+
+def test_get_pr_num_from_head_pull_request(monkeypatch):
+    monkeypatch.setenv('GITHUB_EVENT_NAME', 'pull_request')
+    pr_num = get_pr_num_from_head(PR_BRANCH_NAME)
+    assert pr_num == 442
+
+def test_get_pr_num_from_head_non_pull_request(monkeypatch):
+    monkeypatch.setenv('GITHUB_EVENT_NAME', 'status')
+    pr_num = get_pr_num_from_head(PR_HEAD_SHA)
+    assert pr_num == 442
+
+def test_get_pr_head_status_event(monkeypatch, mocker):
+    monkeypatch.setenv('GITHUB_EVENT_NAME', 'status')
+    mock_status_json = {'branches': [{'name': 'master', 'commit': {'sha': 123}}, {'name': 'pr_branch', 'commit': {'sha': PR_HEAD_SHA}}]}
+    mocker.patch('brainscore_vision.submission.actions_helpers._load_event_file', return_value=mock_status_json)
+    assert get_pr_head_from_github_event() == PR_HEAD_SHA
+
+def test_get_pr_head_status_event_master_only(monkeypatch, mocker):
+    monkeypatch.setenv('GITHUB_EVENT_NAME', 'status')
+    mock_status_json = {'branches': [{'name': 'master', 'commit': {'sha': 123}}]}
+    mocker.patch('brainscore_vision.submission.actions_helpers._load_event_file', return_value=mock_status_json)
+    assert not get_pr_head_from_github_event()
+
+def test_get_pr_head_pull_request_event(monkeypatch):
+    monkeypatch.setenv('GITHUB_EVENT_NAME', 'pull_request')
+    monkeypatch.setenv('GITHUB_HEAD_REF', PR_BRANCH_NAME)
+    assert get_pr_head_from_github_event() == PR_BRANCH_NAME
+
+def test_get_statuses_result():
+    data = get_data(f"{BASE_URL}/statuses/{PR_HEAD_SHA}")
+    assert len(data) == 9
+
+def test_get_statuses_result():
+    data = get_data(f"{BASE_URL}/statuses/{PR_HEAD_SHA}")
+    jenkins_plugintests_result = get_statuses_result('Brain-Score Jenkins CI - plugin tests', data)
+    assert jenkins_plugintests_result == 'failure'
+
+def test_are_all_tests_passing():
+    results_dict = {'travis_branch_result': 'success',
+                'travis_pr_result': 'success',
+                'jenkins_plugintests_result': 'success',
+                'jenkins_unittests_result': 'success'}
+    success = are_all_tests_passing(results_dict)
+    assert success == True
+
+def test_one_test_failing():
+    results_dict = {'travis_branch_result': 'success',
+                    'travis_pr_result': 'failure',
+                    'jenkins_plugintests_result': 'success',
+                    'jenkins_unittests_result': 'success'}
+    success = are_all_tests_passing(results_dict)
+    assert success == False
+ 
+def test_is_labeled_automerge(mocker):
+    assert is_labeled_automerge(442) == True
+
+def test_is_not_labeled_automerge(mocker):
+    assert is_labeled_automerge(433) == False
+

tests/test_submission/test_db.py

@@ -0,0 +1,54 @@
+from datetime import datetime
+
+import pytest
+from brainscore_core.submission.database import connect_db
+from brainscore_core.submission.database_models import (
+    BenchmarkType,
+    Model,
+    User,
+    clear_schema,
+)
+
+
+def init_user():
+    User.create(id=1, email='test@brainscore.com', is_active=True, is_staff=False, is_superuser=False,
+                last_login=datetime.now(), password='abcde')
+    User.create(id=2, email='admin@brainscore.com', is_active=True, is_staff=True, is_superuser=True,
+                last_login=datetime.now(), password='abcdef')
+
+
+def init_benchmark_parents():
+    BenchmarkType.create(identifier='average_vision', order=0, owner_id=2)
+
+    BenchmarkType.create(identifier='behavior', parent='average_vision', order=1, owner_id=2)
+
+    BenchmarkType.create(identifier='neural_vision', parent='average_vision', order=0, owner_id=2)
+    BenchmarkType.create(identifier='V1', parent='neural_vision', order=0, owner_id=2)
+    BenchmarkType.create(identifier='V2', parent='neural_vision', order=1, owner_id=2)
+    BenchmarkType.create(identifier='V4', parent='neural_vision', order=2, owner_id=2)
+    BenchmarkType.create(identifier='IT', parent='neural_vision', order=3, owner_id=2)
+
+
+def init_models():
+    Model.create(name='dummy_model_1', owner=1, public=True)
+    Model.create(name='dummy_model_2', owner=2, public=True)
+    Model.create(name='dummy_model_3', owner=1, public=True)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+@pytest.mark.parametrize('database', ['brainscore-ohio-test'])  # test database
+def test_db(database):
+    connect_db(database)
+    clear_schema()
+    init_user()
+    init_benchmark_parents()
+    init_models()
+
+    user_entries = list(User.select())
+    model_entries = list(Model.select())
+    benchmark_type_entries = list(BenchmarkType.select())
+
+    assert len(user_entries) == 2
+    assert len(model_entries) == 3
+    assert len(benchmark_type_entries) == 7

tests/test_submission/test_endpoints.py

@@ -0,0 +1,125 @@
+import logging
+import time
+
+import numpy as np
+import pytest
+from pytest import approx
+
+""" the mock import has to be before importing endpoints so that the database is properly mocked """
+from .mock_config import test_database
+
+from brainscore_core.submission import database_models
+from brainscore_core.submission.database import connect_db
+from brainscore_core.submission.database_models import clear_schema, create_schema, drop_schema, create_tables
+from brainscore_vision.submission.endpoints import run_scoring
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.mark.private_access
+@pytest.mark.travis_slow
+class TestRunScoring:
+    @classmethod
+    def setup_class(cls):
+        logger.info('Connect to database')
+        connect_db(test_database)
+
+        # generate unique schema name and create
+        cls.schema_name = f'test_schema_{int(time.time())}'
+        create_schema(cls.schema_name)
+
+        # set schema for all models
+        database_models.PeeweeBase.set_schema(cls.schema_name)
+
+        # create tables in schema
+        create_tables()
+
+    @classmethod
+    def teardown_class(cls):
+        drop_schema(cls.schema_name)
+
+    def setup_method(self):
+        logger.info('Initialize database entries')
+        database_models.User.create(id=1, email='test@brainscore.com', password='abcde',
+                                    is_active=True, is_staff=False, is_superuser=False, last_login='2022-10-14 9:25:00')
+        database_models.User.create(id=2, email='admin@brainscore.com', password='abcdef',
+                                    is_active=True, is_staff=True, is_superuser=True, last_login='2024-03-26 6:24:00')
+
+    def teardown_method(self):
+        logger.info('Clean database')
+        clear_schema()
+
+    def test_successful_run(self):
+        args_dict = {'jenkins_id': 62, 'user_id': 1, 'model_type': 'brainmodel',
+                     'public': True, 'competition': None, 'specified_only': True,
+                     'new_models': ['alexnet'], 'new_benchmarks': ['MajajHong2015public.IT-pls']}
+        run_scoring(args_dict)
+        score_entries = database_models.Score.select()
+        score_entries = list(score_entries)
+        assert len(score_entries) == 1
+        score_entry = score_entries[0]
+        assert score_entry.score_ceiled == approx(.5079817, abs=0.005)
+        assert score_entry.comment.startswith('layers:')
+
+    def test_two_models_one_benchmark(self):
+        args_dict = {'jenkins_id': 62, 'user_id': 1, 'model_type': 'brainmodel',
+                     'public': True, 'competition': None, 'specified_only': True,
+                     'new_models': ['pixels', 'alexnet'],
+                     'new_benchmarks': ['MajajHong2015public.IT-pls']}
+        run_scoring(args_dict)
+        score_entries = database_models.Score.select()
+        assert len(score_entries) == 2
+        score_values = [entry.score_ceiled for entry in score_entries]
+        assert all(np.array(score_values) > 0)
+
+    def test_one_model_two_benchmarks(self):
+        args_dict = {'jenkins_id': 62, 'user_id': 1, 'model_type': 'brainmodel',
+                     'public': True, 'competition': None, 'specified_only': True,
+                     'new_models': ['alexnet'],
+                     'new_benchmarks': ['MajajHong2015public.IT-pls', 'Rajalingham2018-i2n']}
+        run_scoring(args_dict)
+        score_entries = database_models.Score.select()
+        assert len(score_entries) == 2
+        score_values = [entry.score_ceiled for entry in score_entries]
+        assert all(np.array(score_values) > 0)
+        score_MajajHong = database_models.Score.get(benchmark__benchmark_type_id='MajajHong2015.IT.public-pls')
+        assert score_MajajHong.score_ceiled == approx(.5079817, abs=0.005)
+        score_Rajalingham = database_models.Score.get(benchmark__benchmark_type_id='Rajalingham2018-i2n')
+        assert score_Rajalingham.score_ceiled == approx(.3701702, abs=0.005)
+
+    def test_two_models_two_benchmarks(self):
+        args_dict = {'jenkins_id': 62, 'user_id': 1, 'model_type': 'brainmodel',
+                     'public': True, 'competition': None, 'specified_only': True,
+                     'new_models': ['pixels', 'alexnet'],
+                     'new_benchmarks': ['MajajHong2015public.IT-pls', 'Rajalingham2018-i2n']}
+        run_scoring(args_dict)
+        score_entries = database_models.Score.select()
+        assert len(score_entries) == 4
+        score_values = [entry.score_ceiled for entry in score_entries]
+        assert all(np.array(score_values) > 0)
+
+    def test_benchmark_does_not_exist(self):
+        args_dict = {'jenkins_id': 62, 'user_id': 1, 'model_type': 'brainmodel',
+                     'public': True, 'competition': None, 'new_models': ['alexnet'],
+                     'new_benchmarks': ['idonotexist'], 'specified_only': True}
+        run_scoring(args_dict)
+        score_entries = database_models.Score.select()
+        assert len(score_entries) == 0
+
+    def test_competition_field_set(self):
+        args_dict = {'jenkins_id': 62, 'user_id': 1, 'model_type': 'brainmodel',
+                     'public': True, 'competition': 'cosyne2022', 'new_models': ['alexnet'],
+                     'new_benchmarks': ['MajajHong2015public.IT-pls'], 'specified_only': True}
+        run_scoring(args_dict)
+        model_entries = database_models.Model.select()
+        assert len(model_entries) == 1
+        assert model_entries[0].competition == "cosyne2022"
+
+    def test_competition_field_not_set(self):
+        args_dict = {'jenkins_id': 62, 'user_id': 1, 'model_type': 'brainmodel',
+                     'public': True, 'competition': None, 'new_models': ['alexnet'],
+                     'new_benchmarks': ['MajajHong2015public.IT-pls'], 'specified_only': True}
+        run_scoring(args_dict)
+        model_entries = database_models.Model.select()
+        assert len(model_entries) == 1
+        assert model_entries[0].competition is None

tests/test_utils.py

@@ -0,0 +1,21 @@
+from brainscore_vision.utils import recursive_dict_merge
+
+
+class TestRecursiveDictMerge:
+    def test_no_overlap(self):
+        dict1 = {'a': 1}
+        dict2 = {'b': 2}
+        merged = recursive_dict_merge(dict1, dict2)
+        assert {'a': 1, 'b': 2} == merged
+
+    def test_overlap1(self):
+        dict1 = {"foo": {"bar": 23, "blub": 42}, "flub": 17}
+        dict2 = {"foo": {"bar": 100}, "flub": {"flub2": 10}, "more": {"stuff": 111}}
+        merged = recursive_dict_merge(dict1, dict2)
+        assert {'foo': {'bar': 100, 'blub': 42}, 'flub': {'flub2': 10}, 'more': {'stuff': 111}} == merged
+
+    def test_overlap2(self):
+        dict1 = {"foo": {"bar": 100}, "flub": {"flub2": 10}, "more": {"stuff": 111}}
+        dict2 = {"foo": {"bar": 23, "blub": 42}, "flub": 17}
+        merged = recursive_dict_merge(dict1, dict2)
+        assert {'foo': {'bar': 23, 'blub': 42}, 'flub': 17, "more": {"stuff": 111}} == merged