brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/benchmarks/scialom2024/benchmark.py

@@ -0,0 +1,97 @@
+import numpy as np
+
+import brainscore_vision
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_dataset, load_metric
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.utils import LazyLoad
+
+BIBTEX = ""  # to appear in a future article
+
+
+class _Scialom2024BehavioralErrorConsistency(BenchmarkBase):
+    def __init__(self, dataset):
+        self._metric = load_metric('error_consistency')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        self._visual_degrees = 8
+        self._number_of_trials = 1
+
+        super(_Scialom2024BehavioralErrorConsistency, self).__init__(
+            identifier=f'Scialom2024_{dataset}-error_consistency', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='Scialom2024',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        labels = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
+        raw_score = self._metric(labels, self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+class _Scialom2024BehavioralAccuracyDistance(BenchmarkBase):
+    # behavioral benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('accuracy_distance')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        super(_Scialom2024BehavioralAccuracyDistance, self).__init__(
+            identifier=f'Scialom2024_{dataset}-behavioral_accuracy', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='Scialom2024',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly.stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._assembly.stimulus_set, number_of_trials=1)
+        raw_score = self._metric(labels, target=self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        # ensure score <= 1.0
+        if score.values > 1:
+            score = Score(np.array(1.))
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+class _Scialom2024EngineeringAccuracy(BenchmarkBase):
+    # engineering/ML benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('accuracy')
+        # no lazyload because needed in candidate.look_at()
+        self._stimulus_set = load_assembly(dataset).stimulus_set
+        super(_Scialom2024EngineeringAccuracy, self).__init__(
+            identifier=f'Scialom2024_{dataset}-engineering_accuracy', version=1,
+            ceiling_func=lambda: Score(1),
+            parent='Scialom2024-top1',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._stimulus_set, number_of_trials=10)
+        raw_score = self._metric(labels, target=self._stimulus_set['truth'].values)
+        ceiling = Score(np.array(1.))
+        score = raw_score / ceiling
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+def load_assembly(dataset: str) -> BehavioralAssembly:
+    assembly = brainscore_vision.load_dataset(f'Scialom2024_{dataset}')
+    return assembly

brainscore_vision/benchmarks/scialom2024/test.py

@@ -0,0 +1,162 @@
+from pathlib import Path
+
+import numpy as np
+import pytest
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import benchmark_registry, load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.data_helpers import s3
+
+
+@pytest.mark.parametrize('benchmark', [
+    'Scialom2024_rgbBehavioralAccuracyDistance',
+    'Scialom2024_contoursBehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-12BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-16BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-21BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-27BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-35BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-46BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-59BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-77BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-100BehavioralAccuracyDistance',
+    'Scialom2024_segments-12BehavioralAccuracyDistance',
+    'Scialom2024_segments-16BehavioralAccuracyDistance',
+    'Scialom2024_segments-21BehavioralAccuracyDistance',
+    'Scialom2024_segments-27BehavioralAccuracyDistance',
+    'Scialom2024_segments-35BehavioralAccuracyDistance',
+    'Scialom2024_segments-46BehavioralAccuracyDistance',
+    'Scialom2024_segments-59BehavioralAccuracyDistance',
+    'Scialom2024_segments-77BehavioralAccuracyDistance',
+    'Scialom2024_segments-100BehavioralAccuracyDistance',
+    'Scialom2024_phosphenes-allBehavioralErrorConsistency',
+    'Scialom2024_segments-allBehavioralErrorConsistency',
+    'Scialom2024_phosphenes-allBehavioralAccuracyDistance',
+    'Scialom2024_segments-allBehavioralAccuracyDistance'
+])
+def test_benchmark_registry(benchmark):
+    assert benchmark in benchmark_registry
+
+
+class TestBehavioral:
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, expected_ceiling', [
+        ('rgb', approx(0.98484, abs=0.001)),
+        ('contours', approx(0.97794, abs=0.001)),
+        ('phosphenes-12', approx(0.94213, abs=0.001)),
+        ('phosphenes-16', approx(0.90114, abs=0.001)),
+        ('phosphenes-21', approx(0.89134, abs=0.001)),
+        ('phosphenes-27', approx(0.77732, abs=0.001)),
+        ('phosphenes-35', approx(0.78466, abs=0.001)),
+        ('phosphenes-46', approx(0.78246, abs=0.001)),
+        ('phosphenes-59', approx(0.79745, abs=0.001)),
+        ('phosphenes-77', approx(0.83475, abs=0.001)),
+        ('phosphenes-100', approx(0.86073, abs=0.001)),
+        ('segments-12', approx(0.86167, abs=0.001)),
+        ('segments-16', approx(0.8202, abs=0.001)),
+        ('segments-21', approx(0.8079, abs=0.001)),
+        ('segments-27', approx(0.78282, abs=0.001)),
+        ('segments-35', approx(0.77724, abs=0.001)),
+        ('segments-46', approx(0.85391, abs=0.001)),
+        ('segments-59', approx(0.83410, abs=0.001)),
+        ('segments-77', approx(0.86227, abs=0.001)),
+        ('segments-100', approx(0.92517, abs=0.001)),  # all of the above are AccuracyDistance
+        ('phosphenes-allBehavioralErrorConsistency', approx(0.45755, abs=0.01)),
+        ('segments-allBehavioralErrorConsistency', approx(0.42529, abs=0.01)),
+        ('phosphenes-allBehavioralAccuracyDistance', approx(0.89533, abs=0.01)),
+        ('segments-allBehavioralAccuracyDistance', approx(0.89052, abs=0.01)),
+    ])
+    def test_dataset_ceiling(self, dataset, expected_ceiling):
+        if 'all' in dataset:
+            benchmark = f"Scialom2024_{dataset}"
+        else:
+            benchmark = f"Scialom2024_{dataset}BehavioralAccuracyDistance"
+        benchmark = load_benchmark(benchmark)
+        ceiling = benchmark.ceiling
+        assert ceiling == expected_ceiling
+
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, expected_raw_score', [
+        ('rgb', approx(0.92616, abs=0.001)),
+        ('contours', approx(0.25445, abs=0.001)),
+        ('phosphenes-12', approx(0.84177, abs=0.001)),
+        ('phosphenes-16', approx(0.77513, abs=0.001)),
+        ('phosphenes-21', approx(0.76437, abs=0.001)),
+        ('phosphenes-27', approx(0.56895, abs=0.001)),
+        ('phosphenes-35', approx(0.52008, abs=0.001)),
+        ('phosphenes-46', approx(0.29478, abs=0.001)),
+        ('phosphenes-59', approx(0.19022, abs=0.001)),
+        ('phosphenes-77', approx(0.13569, abs=0.001)),
+        ('phosphenes-100', approx(0.11234, abs=0.001)),
+        ('segments-12', approx(0.72937, abs=0.001)),
+        ('segments-16', approx(0.57043, abs=0.001)),
+        ('segments-21', approx(0.49300, abs=0.001)),
+        ('segments-27', approx(0.30014, abs=0.001)),
+        ('segments-35', approx(0.22442, abs=0.001)),
+        ('segments-46', approx(0.14312, abs=0.001)),
+        ('segments-59', approx(0.12072, abs=0.001)),
+        ('segments-77', approx(0.12996, abs=0.001)),
+        ('segments-100', approx(0.11540, abs=0.001)),  # all of the above are AccuracyDistance
+        ('phosphenes-all', approx(0.18057, abs=0.01)),  # alls are ErrorConsistency
+        ('segments-all', approx(0.15181, abs=0.01)),
+    ])
+    def test_model_8_degrees(self, dataset, expected_raw_score):
+        if 'all' in dataset:
+            benchmark = f"Scialom2024_{dataset}BehavioralErrorConsistency"
+        else:
+            benchmark = f"Scialom2024_{dataset}BehavioralAccuracyDistance"
+        benchmark = load_benchmark(benchmark)
+        nc_filename = dataset.upper() if dataset == 'rgb' else dataset
+        filename = f"resnet50_julios_Scialom2024_{nc_filename}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainscore-vision', remote_filepath=f'benchmarks/Scialom2024/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark(precomputed_features)
+        raw_score = score.raw
+        # division by ceiling <= 1 should result in higher score
+        assert score >= raw_score
+        assert raw_score == expected_raw_score
+
+
+class TestEngineering:
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, expected_accuracy', [
+        ('rgb', approx(0.91666, abs=0.001)),
+        ('contours', approx(0.25000, abs=0.001)),
+        ('phosphenes-12', approx(0.08333, abs=0.001)),
+        ('phosphenes-16', approx(0.08333, abs=0.001)),
+        ('phosphenes-21', approx(0.08333, abs=0.001)),
+        ('phosphenes-27', approx(0.08333, abs=0.001)),
+        ('phosphenes-35', approx(0.12500, abs=0.001)),
+        ('phosphenes-46', approx(0.08333, abs=0.001)),
+        ('phosphenes-59', approx(0.08333, abs=0.001)),
+        ('phosphenes-77', approx(0.08333, abs=0.001)),
+        ('phosphenes-100', approx(0.08333, abs=0.001)),
+        ('segments-12', approx(0.08333, abs=0.001)),
+        ('segments-16', approx(0.08333, abs=0.001)),
+        ('segments-21', approx(0.08333, abs=0.001)),
+        ('segments-27', approx(0.08333, abs=0.001)),
+        ('segments-35', approx(0.08333, abs=0.001)),
+        ('segments-46', approx(0.08333, abs=0.001)),
+        ('segments-59', approx(0.08333, abs=0.001)),
+        ('segments-77', approx(0.10416, abs=0.001)),
+        ('segments-100', approx(0.10416, abs=0.001))
+    ])
+    def test_accuracy(self, dataset, expected_accuracy):
+        benchmark = load_benchmark(f"Scialom2024_{dataset}EngineeringAccuracy")
+        nc_filename = dataset.upper() if dataset == 'rgb' else dataset
+        filename = f"resnet50_julios_Scialom2024_{nc_filename}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainio-brainscore', remote_filepath=f'/benchmarks/Scialom2024/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark(precomputed_features)
+        raw_score = score.raw
+        # division by ceiling <= 1 should result in higher score
+        assert score >= raw_score
+        assert raw_score == expected_accuracy

brainscore_vision/data/baker2022/__init__.py

@@ -0,0 +1,40 @@
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import data_registry, stimulus_set_registry, load_stimulus_set
+from brainscore_vision.data_helpers.s3 import load_assembly_from_s3, load_stimulus_set_from_s3
+
+
+# normal distortion:
+stimulus_set_registry['Baker2022_normal_distortion'] = lambda: load_stimulus_set_from_s3(
+    identifier='Baker2022_normal_distortion',
+    bucket="brainio-brainscore",
+    csv_sha1="17d4db7458a29a787d12bb29c34e91daef1872bf",
+    zip_sha1="2c726abaf081c8a9828269a559222f8c6eea0e4f",
+    csv_version_id="upenpJUf5t_sZpuqmQOkARfnDx1ITvi0",
+    zip_version_id="GO0K_jHqUMWVYcV2DoM4ka.HHv.KzrSf")
+
+data_registry['Baker2022_normal_distortion'] = lambda: load_assembly_from_s3(
+    identifier='Baker2022_normal_distortion',
+    version_id="7HACUgvwlGp_mIWb62BFzh8JlMwtDxpo",
+    sha1="46c79a48bf2dbd995a9305d8fbc03a134a852e17",
+    bucket="brainio-brainscore",
+    cls=BehavioralAssembly,
+    stimulus_set_loader=lambda: load_stimulus_set('Baker2022_normal_distortion'),
+)
+
+# inverted distortion:
+stimulus_set_registry['Baker2022_inverted_distortion'] = lambda: load_stimulus_set_from_s3(
+    identifier='Baker2022_inverted_distortion',
+    bucket="brainio-brainscore",
+    csv_sha1="91e452e4651024c2b382694edfcbc7bdc6c3189b",
+    zip_sha1="4740a096af994c2232350469c664e53796f17a05",
+    csv_version_id="yiKAKACwcvMygzX27DI3BSG5h599Eji_",
+    zip_version_id="ZAmZEaRp0S2txTdXjn3LBY_vcFXaALPr")
+
+data_registry['Baker2022_inverted_distortion'] = lambda: load_assembly_from_s3(
+    identifier='Baker2022_inverted_distortion',
+    version_id="7MV4uklu3pL0I9LB2xlHWvlnOjG7Flbv",
+    sha1="b76fb57b25a58ca68db78d188fd0a783e1dcaf73",
+    bucket="brainio-brainscore",
+    cls=BehavioralAssembly,
+    stimulus_set_loader=lambda: load_stimulus_set('Baker2022_inverted_distortion'),
+)

brainscore_vision/data/baker2022/data_packaging/inverted_distortion_data_assembly.py

@@ -0,0 +1,43 @@
+import numpy as np
+
+from brainio.assemblies import BehavioralAssembly
+from brainio.packaging import package_data_assembly
+import pandas as pd
+
+'''
+Experiment Information:
+https://www.sciencedirect.com/science/article/pii/S2589004222011853#sec9
+ - (Data/subjects are from experiment 1 in paper above)
+
+ - 32 subjects 
+ - 40 images for each condition (whole, fragmented, frankenstein) = 120 images/subject shown
+ - 120 * 32 = 3840 total images shown 
+ - However there are only 3706 trials total, for data issues (according to Baker)
+ - 9-way AFC, from set: {bear, bunny, cat, elephant, frog, lizard, tiger, turtle, wolf}
+ 
+'''
+
+# initial csv to dataframe processing:
+all_subjects = pd.read_csv('human_data/inverted.csv')
+
+
+# construct the assembly
+assembly = BehavioralAssembly(all_subjects['RSP'],
+                              coords={
+                                  'stimulus_id': ('presentation', all_subjects['FileName']),
+                                  'subject': ('presentation', all_subjects['Subj']),
+                                  'orientation': ('presentation', all_subjects['Inv']),
+                                  'condition': ('presentation', all_subjects['Config']),
+                                  'truth': ('presentation', all_subjects['Animal']),
+                                  'correct': ('presentation', all_subjects['RSP']),
+                              },
+                              dims=['presentation']
+                              )
+
+# give the assembly an identifier name
+assembly.name = 'Baker2022_inverted_distortion'
+
+# upload to S3
+package_data_assembly('brainio_brainscore', assembly, assembly_identifier=assembly.name,
+                      stimulus_set_identifier='Baker2022_inverted_distortion',
+                      assembly_class_name="BehavioralAssembly", bucket_name="brainio-brainscore")

brainscore_vision/data/baker2022/data_packaging/inverted_distortion_stimulus_set.py

@@ -0,0 +1,81 @@
+from pathlib import Path
+from brainio.stimuli import StimulusSet
+from brainio.packaging import package_stimulus_set
+import pandas as pd
+
+stimuli = []
+image_paths = {}
+stimuli_directory = 'images'
+
+
+'''
+Dataset Information:
+- From Baker 2022: https://www.sciencedirect.com/science/article/pii/S2589004222011853#sec9
+- 4320 trials total, across 12 subjects. Each subject saw the same 360 images, composed of:
+    - 90 inverted-whole, 90 inverted-frankenstein, 90-normal frankenstein, 90-normal whole.
+    - of those 4 subtypes, they are further broken into the same 10 images of the set of 9 categories below.
+
+Fields:
+1) ground_truth: the base object, in set above
+2) image_type:  a string in the set {w, f} for {whole, frankenstein} respectively.
+    Note: inverted only hase whole and frankenstein images, no fragmented 
+3) image_number: a number {1,2...40} indicates the image variation
+
+'''
+categories = ['bear', 'bunny', 'cat', 'elephant', 'frog', 'lizard', 'tiger', 'turtle', 'wolf']
+images_actually_shown = pd.read_csv('human_data/inverted.csv')
+images_actually_shown = set(images_actually_shown["FileName"].values)
+
+for filepath in Path(stimuli_directory).glob('*.jpg'):
+
+    # entire name of image file:
+    image_id = filepath.stem
+
+    if image_id in images_actually_shown:
+        import re
+        match = re.match(r"([a-z]+)([0-9]+)", image_id, re.I)
+        if match:
+            items = match.groups()
+        else:
+            items = ["", ""]
+
+        # ground truth
+        ground_truth = items[0]
+
+        # image_number:
+        image_number = items[1]
+
+        # parse the needed image type letter
+        if ground_truth in categories:
+            image_type = "w"
+        else:
+            image_type = ground_truth[0]
+            ground_truth = ground_truth[1:]
+
+        if "inv" in ground_truth:
+            ground_truth = ground_truth.replace("inv", "")
+        elif "nv" in ground_truth:
+            ground_truth = ground_truth.replace("nv", "")
+
+        image_paths[image_id] = filepath
+        stimuli.append({
+            'stimulus_id': image_id,
+            'animal': ground_truth,
+            'image_type': "w" if image_type is "i" else image_type,
+            'image_type_entire': "whole" if image_type is "w" else "frankenstein",
+            'image_number': image_number,
+            "orientation": "normal" if "inv" not in image_id else "inverted",
+        })
+    else:
+        pass
+
+stimuli = StimulusSet(stimuli)
+stimuli.stimulus_paths = image_paths
+
+
+# give the StimulusSet an identifier name
+stimuli.name = 'Baker2022_inverted_distortion'
+
+# upload to S3
+package_stimulus_set("brainio_brainscore", stimuli, stimulus_set_identifier=stimuli.name,
+                     bucket_name="brainio-brainscore")

brainscore_vision/data/baker2022/data_packaging/mapping.py

@@ -0,0 +1,60 @@
+import pandas as pd
+import re
+
+human_data = pd.read_csv('human_data_normal/human_data.csv')
+images_shown = pd.read_csv('human_data_normal/images_shown.csv')
+human_data["image_shown"] = "FILL"
+
+columns = {"Participants 1-7": [1,2,3,4,5,6,7],"Participants 8-13": [8,9,10,11,12,13],
+           "Participants 14-19": [14,15,16,17,18,19],"Participants 20-24": [20,21,22,23,24],
+           "Participants 25-28": [25,26,27,28,29], "Participants 29-32": [29,30,31,32]}
+categories = ['bear', 'bunny', 'cat', 'elephant', 'frog', 'lizard', 'tiger', 'turtle', 'wolf']
+mapping = {"o": "fragmented", "w": "whole", "f": "Frankenstein"}
+
+def get_parts(image_id):
+    match = re.match(r"([a-z]+)([0-9]+)", image_id, re.I)
+    if match:
+        items = match.groups()
+    else:
+        items = ["", ""]
+    # ground truth
+    ground_truth = items[0]
+    return ground_truth
+
+
+for i in range(0, len(human_data)):
+    subject_number = human_data["Subj"][i]
+    column_to_look_at = ""
+    for header in columns:
+        if subject_number in columns[header]:
+            column_to_look_at = header
+            break
+    condition = human_data["Frankensteinonfig"][i]
+    animal = human_data["Animal"][i]
+
+    # clear seen every subject
+    if i % 32 == 0:
+        seen = []
+
+    for image_name in images_shown[column_to_look_at]:
+
+        ground_truth = get_parts(image_name)
+        if ground_truth in categories:
+            image_type = "w"
+        else:
+            image_type = ground_truth[0]
+            ground_truth = ground_truth[1:]
+        image_type_entire = mapping[image_type]
+        if animal in image_name:
+            if condition == image_type_entire and image_name not in seen:
+                human_data["image_shown"][i] = image_name.replace(".jpg", "")
+                seen.append(image_name)
+                break
+
+human_data.to_csv('human_data_normal/human_data.csv')
+
+
+
+
+
+

brainscore_vision/data/baker2022/data_packaging/normal_distortion_data_assembly.py

@@ -0,0 +1,46 @@
+import numpy as np
+
+from brainio.assemblies import BehavioralAssembly
+from brainio.packaging import package_data_assembly
+import pandas as pd
+
+'''
+Experiment Information:
+ - https://www.sciencedirect.com/science/article/pii/S2589004222011853#sec9
+ - Data/subjects are from experiment 1 in paper above
+
+ - 32 subjects 
+ - 40 images for each condition (whole, fragmented, frankenstein) = 120 images/subject shown
+ - 120 * 32 = 3840 total images shown 
+ - However there are only 3706 trials total, for data issues (according to Baker)
+ - 9-way AFC, from set: {bear, bunny, cat, elephant, frog, lizard, tiger, turtle, wolf}
+ 
+'''
+
+# initial csv to dataframe processing:
+all_subjects = pd.read_csv('human_data/human_data.csv')
+
+# all images presented in this assembly are normal (non-inverted)
+all_subjects["orientation"] = "normal"
+
+# construct the assembly
+assembly = BehavioralAssembly(all_subjects['Correct?'],
+                              coords={
+                                  'stimulus_id': ('presentation', all_subjects['image_shown']),
+                                  'subject': ('presentation', all_subjects['Subj']),
+                                  'condition': ('presentation', all_subjects['Frankensteinonfig']),
+                                  'truth': ('presentation', all_subjects['Animal']),
+                                  'correct': ('presentation', all_subjects['Correct?']),
+                                  'orientation': ('presentation', all_subjects['orientation']),
+                              },
+                              dims=['presentation']
+                              )
+
+# give the assembly an identifier name
+assembly.name = 'Baker2022_normal_distortion'
+
+
+# upload to S3
+package_data_assembly('brainio_brainscore', assembly, assembly_identifier=assembly.name,
+                      stimulus_set_identifier='Baker2022_normal_distortion',
+                      assembly_class_name="BehavioralAssembly", bucket_name="brainio-brainscore")

brainscore_vision/data/baker2022/data_packaging/normal_distortion_stimulus_set.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+from brainio.stimuli import StimulusSet
+from brainio.packaging import package_stimulus_set
+import pandas as pd
+import re
+
+stimuli = []
+image_paths = {}
+stimuli_directory = 'images'
+
+
+'''
+Dataset Information:
+
+- From Baker 2022: https://www.sciencedirect.com/science/article/pii/S2589004222011853#sec9
+- 1800 images total: 1080 normal distortion (** this stimulus set **), 720 inverted distortion
+- normal/inverted distortion -> contains the three classes of distortion:
+    1) normal image
+    2) fragmented image
+    3) frankenstein image
+
+- For the normal distortion, there are 40 images/condition, with 9 categories: 
+    categories are: {bear, bunny, cat, elephant, frog, lizard, tiger, turtle, wolf}
+- For inverted distortion, there are a variable number of images/condition, but 720 total unique images
+
+Fields:
+
+1) ground_truth: the base object, in set above
+2) image_type:  a string in the set {w, f, o} for {whole, frankenstein, fragmented} respectively. 
+3) image_number: a number {1,2...40} indicates the image variation
+
+'''
+categories = ['bear', 'bunny', 'cat', 'elephant', 'frog', 'lizard', 'tiger', 'turtle', 'wolf']
+images_actually_shown = pd.read_csv('human_data/images_shown.csv')
+images_actually_shown = pd.concat([images_actually_shown[col] for col in images_actually_shown]).values
+
+for filepath in Path(stimuli_directory).glob('*.jpg'):
+
+    # entire name of image file:
+    image_id = filepath.stem
+
+    if f"{image_id}.jpg" in images_actually_shown:
+        match = re.match(r"([a-z]+)([0-9]+)", image_id, re.I)
+        if match:
+            items = match.groups()
+        else:
+            items = ["", ""]
+
+        # ground truth
+        ground_truth = items[0]
+
+        # image_number:
+        image_number = items[1]
+
+        # parse the needed image type letter
+        if ground_truth in categories:
+            image_type = "w"
+        else:
+            image_type = ground_truth[0]
+            ground_truth = ground_truth[1:]
+
+        if "inv" in ground_truth:
+            ground_truth = ground_truth.replace("inv", "")
+        elif "nv" in ground_truth:
+            ground_truth = ground_truth.replace("nv", "")
+
+        if image_type is "w":
+            image_type_entire = "whole"
+        elif image_type is "o":
+            image_type_entire = "fragmented"
+        else:
+            image_type_entire = "frankenstein"
+
+        image_paths[image_id] = filepath
+        stimuli.append({
+            'stimulus_id': image_id,
+            'animal': ground_truth,
+            'image_type': "w" if image_type is "i" else image_type,
+            'image_type_entire': image_type_entire,
+            'image_number': image_number,
+            "orientation": "normal" if "inv" not in image_id else "inverted",
+        })
+
+stimuli = StimulusSet(stimuli)
+
+stimuli.stimulus_paths = image_paths
+
+# remove all inverted stimuli
+stimuli = stimuli[stimuli["orientation"] == "normal"]
+stimuli.name = 'Baker2022_normal_distortion'  # give the StimulusSet an identifier name
+
+# upload to S3
+package_stimulus_set("brainio_brainscore", stimuli, stimulus_set_identifier=stimuli.name,
+                     bucket_name="brainio-brainscore")