brainscore-vision 2.1__py3-none-any.whl

brainscore_vision-2.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (74.1.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

brainscore_vision-2.1.dist-info/top_level.txt

@@ -0,0 +1,4 @@
+brainscore_vision
+docs
+migrations
+tests

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

@@ -0,0 +1,78 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../..'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'Brain-Score'
+copyright = '2024, Brain-Score Team'
+author = 'Brain-Score Team'
+
+# The full version, including alpha/beta/rc tags
+release = '2.0.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'recommonmark',
+    'sphinx.ext.viewcode',
+]
+autosummary_generate = True  # Turn on sphinx.ext.autosummary
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+source_suffix = ['.rst', '.md']
+
+# The master toctree document.
+master_doc = 'index'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+def skip(app, what, name, obj, would_skip, options):
+    if name in ("__init__", "__call__"):
+        return False
+    if name.startswith('test') or name.startswith('Test'):
+        return False
+    return would_skip
+
+
+def setup(app):
+    app.connect("autodoc-skip-member", skip)

docs/source/index.rst

@@ -0,0 +1,21 @@
+Brain-Score
+===========
+
+Brain-Score is a collection of benchmarks and models.
+The benchmarks consist of data and metrics that score any model on how brain-like it is.
+By following a unified :ref:`interface`, all models can be treated as an experimental subject and tested on all benchmarks.
+
+.. autofunction:: brainscore_vision.score
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   examples
+   modules/model_interface
+   modules/developer_clarifications.rst
+   modules/benchmarks
+   modules/metrics
+   modules/submission
+   modules/utils
+   modules/api_reference

docs/source/modules/api_reference.rst

@@ -0,0 +1,10 @@
+.. _api_reference:
+
+API Reference
+-------------
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   brainscore

docs/source/modules/benchmarks.rst

@@ -0,0 +1,8 @@
+.. _benchmarks:
+
+Benchmarks
+----------
+
+.. automodule:: brainscore_vision.benchmarks
+    :members:
+    :undoc-members:

docs/source/modules/developer_clarifications.rst

@@ -0,0 +1,36 @@
+.. _interface:
+
+************************
+Developer Clarifications
+************************
+
+The Following documentation stores commonly-asked developer questions. We hope this will be useful to
+anyone interested in contributing to Brain-Score's codebase or scientific workings.
+
+
+
+1. **For a given model, are activations different on each benchmark? How?**
+
+
+    Activations per model are generated based on benchmark stimuli; not every benchmark has unique stimuli. For most
+    model-benchmark pairs, activations will be different because stimuli will be different. The exceptions to this
+    are the benchmarks that use the same stimuli, such as the `MajajHong20215` family of benchmarks.
+
+2. **Result Caching**
+
+    Result Caching is a Brain-Score `repo <https://github.com/brain-score/result_caching>`_ that allows model activations (and other functions) to be cached
+    to disk, in order to speed up the process of rescoring models. It contains a decorator that can be attached to a function
+    right before it is defined. On the first run of that function, `result_caching` will save to disk the result of tha function
+    and will load that result from disk in future calls with the same parameters. All files are saved in the user's `~/result_caching`
+    folder, and they are persistent, as there is no garbage collection built in. You can deactivate
+    `result_caching` by simply setting the environment flag `RESULTCACHING_DISABLE` to `1`. Please see the link above
+    for more detailed documentation.
+
+3. **Model Mapping Procedure**
+
+    In general, there are different methods that are used in the Brain-Score code to instruct the model to "begin recording",
+    observe stimuli, and to generate scores. Models follow the `ModelCommitment` to conform to the `BrainModel` API. A
+    `BrainModel` is any model that has a `region_layer_map`. This allows the layers in the model to be mapped to layers in
+    the ventral visual stream, and is chosen by scoring models on the public version of a benchmark (the private
+    benchmark data is heldout for the BrainModel to be scored on). See the more technical docs
+    `here <https://brain-score.readthedocs.io/en/latest/modules/model_interface.html>`_  for additional notes.

docs/source/modules/metrics.rst

@@ -0,0 +1,8 @@
+.. _metrics:
+
+Metrics
+-------
+
+.. automodule:: brainscore_vision.metrics
+    :members:
+    :undoc-members:

docs/source/modules/model_interface.rst

@@ -0,0 +1,8 @@
+.. _interface:
+
+BrainModel interface
+--------------------
+
+.. autoclass:: brainscore_vision.model_interface.BrainModel
+    :members:
+    :undoc-members:

docs/source/modules/submission.rst

@@ -0,0 +1,112 @@
+Submission System
+------------------
+
+To submit a model or benchmark to the Brain-Score leaderboard, Brain-Score provides an automated submission system.
+This allows us to publicly score new models on existing benchmarks and score existing models against new benchmarks.
+
+.. image:: docs/source/modules/brainscore_submission.png
+    :width: 400px
+    :align: center
+    :height: 200px
+    :alt: submission system diagram
+
+Users can submit models or benchmarks as a zip file via the `Brain-Score website <#Brain-Score Website>`_ or as a pull request via GitHub.
+Appropriately configured zip file submissions are converted by `Jenkins <#Jenkins>`_ (`create_github_pr`) into a pull request. 
+If tests pass, the PR is automatically merged into the codebase by a GitHub Action (`automerge_plugin-only_prs`).
+If tests fail on a website submission, the submitter will be notified via email.
+Note: While we currently do not support private code submissions, this feature is under development.
+
+If a newly merged PR contains new models or benchmarks,
+a separate Github Action (`score_new_plugins`) triggers the scoring runs which are handled by Jenkins
+and run on the `OpenMind compute cluster <#Openmind>`_.
+
+
+Submission System Components
+############################
+
+- **Brain-Score Website**:
+
+    The `website <www.brain-score.org>`_ (`GitHub <https://github.com/brain-score/brain-score.web>`_) handles our front 
+    end. It is implemented using Django and also accesses the database instance.
+
+    The website is hosted via Amazon Elastic Beanstalk. There are two instances on AWS EB:
+
+    - `brain-score-web-dev <brain-score-web-dev.us-east-2.elasticbeanstalk.com>`_: This is our dev website environment.
+      It is the same as prod but uses the dev database.
+
+    - `brain-score-web-prod <brain-score-web-prod.us-east-2.elasticbeanstalk.com>`_: Our production branch
+        that uses the prod database. This is what end users see, as the above URL is forwarded to brain-score.org
+- **Jenkins**:
+
+    Jenkins is a continuous integration tool, which we use to automatically run 
+    project unittests and the scoring process for models of the brain. `Jenkins is running on Braintree <http://braintree.mit.edu:8080/>`_ 
+    - DiCarlo lab's internal server. Jenkins defines different jobs and executes different tasks. The task for a new submission is
+    triggered via the website and the unittest tasks are triggerd by GitHub web hooks. Once the jobs are triggered,
+    Jenkins runs a procedure to execute the tests or scoring and communicate the results back to the user (via email)
+    or back to GitHub.
+
+-  **OpenMind**
+
+    As scoring submissions is a computationally and memory expensive process, we cannot execute model scoring on small
+    machines. We submit jobs to Openmind, a computer cluster operated by MIT BCS. The big advantage of Openmind is its
+    queuing system, which allows to define detailed resource requirements. Jobs are executed once their requested
+    resources are available. The Jenkins related contents are stored on ``/om2/group/dicarlo/jenkins``. This directory
+    contains a script for model submission (`score_model.sh`), for unittests (`unittests_brainscore.sh`), and for plugin-specific
+    tests (`unittests_plugins.sh`). The scripts are executed in Openmind jobs and are responsible for fully installing a conda 
+    environment, executing the process, and shutting everything down again. For scoring runs, results are stored in the database 
+    and sent to the user via email. For unit and plugin tests, results are reported back to Github.
+
+
+- **Postgres database:**
+
+    Our database, hosted on Amazon AWS, contains all displayed score and submission data, along with much more user and
+    model data/metadata. Our AWS account contains three database instances:
+     
+    - Prod (brainscore-prod-ohio-cred): This database is used in production mode, containing real user's data. This
+      database should not be altered for development until features have been tested and vetted on Dev.
+    - Dev (brainscore-1-ohio-cred): A development database, which can be used to develop new database dependent
+      features. Nothing will break when the database schema here is changed; it is periodically updated to match Prod.
+    - Test (brainscore-ohio-test): The database used for executing tests. Jenkins also executes unittests of all
+      Brain-Score projects and should use this database for testing.
+
+    The names in parantheses are used in brain-score to load database credentials for the different databases.
+    Just change the name and another database is used. Databases are automatically snapshotted every 7 days, and
+    devs can restore snapshots at any time.
+
+
+
+
+
+
+What to do
+#################
+
+
+...when changing the database schema
+************************************
+The current schema is depicted `here
+<https://github.com/brain-score/brain-score/blob/master/brainscore_vision/docs/source/modules/db_schema.uml>`_.
+
+
+When the database schema has to be changed, use the `Brain-Score.web <https://github.com/brain-score/brain-score.web>`_
+project, along with django commands, to adjust the tables (in `benchmark/models.py`). The schema also has to be updated
+in `core <https://github.com/brain-score/core/blob/main/brainscore_core/submission/database_models.py>`_. Once changes
+are made locally, follow `these instructions <https://github.com/brain-score/brain-score.web/blob/master/deployment.md#to-deploy>`_ to
+apply those migrations to the correct databases. All needed changes to the database (dev or prod) should be done with
+Django via migrations. During development, work with the dev database (secret `brainscore-1-ohio-cred`); when your
+tests pass on the test database (`brainscore-ohio-test`) they are ready for the PR. Once the PR is approved and test
+cases run, the PR can be merged. Finally, apply those migrations to the prod database via the link above.
+
+
+...changing the submission process
+**********************************
+In addition to the main job for scoring submission (`score_plugins`), Jenkins contains a second job (`dev_score_plugins`),
+which can be used to test new submission code. It is also a good idea instead of checking out the Brain-Score master
+branch, as it is done in the default job, to checkout your development branch instead. This way you can run a whole
+submission without harming the "production" job. This is accomplished already by a duplicate of score_models.sh for dev,
+aptly named dev_score_models.sh. That script is what is run on Jenkins's dev environment. Once the development job runs
+successfully, the code can be merged to master and will be run "in production".
+
+Scoring Process Description
+###########################
+For scoring submitted files, we install the Brain-Score framework on Openmind and run the scoring process.

docs/source/modules/utils.rst

@@ -0,0 +1,22 @@
+Helpers and Utils
+-----------------
+
+.. automodule:: brainscore_vision.model_helpers
+    :members:
+    :undoc-members:
+
+.. automodule:: brainscore_vision.data_helpers
+    :members:
+    :undoc-members:
+
+.. automodule:: brainscore_vision.metric_helpers
+    :members:
+    :undoc-members:
+
+.. automodule:: brainscore_vision.benchmark_helpers
+    :members:
+    :undoc-members:
+
+.. automodule:: brainscore_vision.utils
+    :members:
+    :undoc-members:

migrations/2020-12-20_pkl_to_nc.py

@@ -0,0 +1,90 @@
+import pickle
+from pathlib import Path
+
+import boto3
+from brainio_base.assemblies import BehavioralAssembly
+from brainio_collection.packaging import write_netcdf
+
+
+local_pkl_names = [
+    'alexnet-probabilities.pkl',
+    'resnet34-probabilities.pkl',
+    'resnet18-probabilities.pkl'
+]
+
+
+s3_pkl_names = [
+    "alexnet-freemanziemba2013.aperture-private.pkl",
+    "alexnet-majaj2015.private-features.12.pkl",
+    "CORnetZ-rajalingham2018public.pkl",
+    "cornet_s-kar2019.pkl",
+    "alexnet-sanghavi2020-features.12.pkl",
+    "alexnet-sanghavijozwik2020-features.12.pkl",
+    "alexnet-sanghavimurty2020-features.12.pkl",
+    "alexnet-rajalingham2020-features.12.pkl",
+]
+
+
+def local_pkls():
+    target_dir_path = Path(__file__).parents[1] / "tests" / "test_metrics"
+    for pkl_name in local_pkl_names:
+        pkl_path = target_dir_path / pkl_name
+        nc_path = pkl_path.with_suffix(".nc")
+        if not nc_path.exists():
+            print(f"{nc_path} does not exist.  ")
+            with open(pkl_path, 'rb') as f:
+                unpickled = pickle.load(f)
+                #   write netcdf
+                sha1 = write_netcdf(BehavioralAssembly(unpickled["data"]), str(nc_path))
+        else:
+            print(f"{nc_path} already exists.  ")
+
+
+def s3_pkls():
+    session = boto3.session.Session(profile_name="dicarlolab_jjpr")
+    s3 = session.client("s3")
+    bucket_name = "brain-score-tests"
+    def exists(key):
+        try:
+            s3.head_object(Bucket=bucket_name, Key=key)
+            return True
+        except s3.exceptions.NoSuchKey:
+            return False
+
+    prefix_path = Path("tests", "test_benchmarks")
+    target_dir_path = Path(__file__).parent / "test_pkl"
+
+    for pkl_name in s3_pkl_names:
+        pkl_path = Path(pkl_name)
+        nc_path = pkl_path.with_suffix(".nc")
+        object_key_pkl = prefix_path / pkl_path
+        target_file_pkl = target_dir_path / pkl_path
+        target_file_nc = target_dir_path / nc_path
+        object_key_nc = prefix_path / nc_path
+
+        if not exists(str(object_key_nc)):
+            print(f"{object_key_nc} does not exist.  ")
+            if not target_file_nc.exists():
+                if not target_file_pkl.exists():
+                    #   fetch file
+                    s3.download_file(bucket_name, str(object_key_pkl), str(target_file_pkl))
+                #   unpickle
+                with open(target_file_pkl, 'rb') as f:
+                    unpickled = pickle.load(f)
+                    #   write netcdf
+                    sha1 = write_netcdf(unpickled["data"], str(target_file_nc))
+            #   upload
+            s3.upload_file(str(target_file_nc), bucket_name, str(object_key_nc))
+        else:
+            print(f"{object_key_nc} already exists.  ")
+
+
+def main():
+    # assert xarray is 0.12.3
+    local_pkls()
+    s3_pkls()
+
+
+if __name__ == '__main__':
+    main()
+

tests/__init__.py

@@ -0,0 +1,6 @@
+import logging
+import sys
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format='%(asctime)-15s %(levelname)s:%(name)s:%(message)s')
+for logger in ['peewee', 's3transfer', 'botocore', 'boto3', 'urllib3', 'PIL']:
+    logging.getLogger(logger).setLevel(logging.INFO)

tests/conftest.py

@@ -0,0 +1,26 @@
+import brainio
+import pytest
+
+from brainscore_vision.benchmark_helpers import screen
+
+
+@pytest.fixture
+def brainio_home(tmp_path, monkeypatch):
+    monkeypatch.setattr(brainio.fetch, "_local_data_path", str(tmp_path))
+    yield tmp_path
+
+
+@pytest.fixture
+def resultcaching_home(tmp_path, monkeypatch):
+    monkeypatch.setenv('RESULTCACHING_HOME', str(tmp_path))
+    yield tmp_path
+
+
+@pytest.fixture
+def brainscore_home(tmp_path, monkeypatch):
+    monkeypatch.setattr(screen, "root_path", tmp_path)
+    yield tmp_path
+
+
+
+

tests/test_benchmark_helpers/test_screen.py

@@ -0,0 +1,75 @@
+import numpy as np
+from PIL import Image
+from pathlib import Path
+from pytest import approx
+from tempfile import TemporaryDirectory
+
+from brainscore_vision.benchmark_helpers.screen import ImageConverter
+
+
+class TestImageConverter:
+    def test_identical(self):
+        image_path = Path(__file__).parent / "rgb1.jpg"
+        source_image = np.array(Image.open(image_path))
+        with TemporaryDirectory('screen') as target_dir:
+            image_converter = ImageConverter(target_dir=target_dir)
+            converted_image_path = image_converter.convert_image(image_path, source_degrees=8, target_degrees=8)
+            converted_image = np.array(Image.open(converted_image_path))
+        assert image_path == converted_image_path
+        assert (source_image == converted_image).all()
+
+    def test_crop_shape(self):
+        image_path = Path(__file__).parent / "rgb1.jpg"
+        with TemporaryDirectory('screen') as target_dir:
+            image_converter = ImageConverter(target_dir=target_dir)
+            converted_image_path = image_converter.convert_image(image_path, source_degrees=10, target_degrees=8)
+            source_image, converted_image = Image.open(image_path), Image.open(converted_image_path)
+        np.testing.assert_array_equal(converted_image.size, [1855 * .8, 1855 * .8])
+
+    def test_enlarge_shape(self):
+        image_path = Path(__file__).parent / "rgb1.jpg"
+        with TemporaryDirectory('screen') as target_dir:
+            image_converter = ImageConverter(target_dir=target_dir)
+            converted_image_path = image_converter.convert_image(image_path, source_degrees=10, target_degrees=12)
+            source_image, converted_image = Image.open(image_path), Image.open(converted_image_path)
+        np.testing.assert_array_equal(converted_image.size, [1855 * 1.2, 1855 * 1.2])
+
+    def test_enlarge_match(self):
+        image_path = Path(__file__).parent / "rgb1.jpg"
+        with TemporaryDirectory('screen') as target_dir:
+            image_converter = ImageConverter(target_dir=target_dir)
+            converted_image_path = image_converter.convert_image(image_path, source_degrees=10, target_degrees=12)
+            converted_image = Image.open(converted_image_path)
+        target_image = Image.open(Path(__file__).parent / "rgb1-10to12.png")
+        np.testing.assert_array_equal(converted_image, target_image)
+
+    def test_enlarge_gray_background(self):
+        image_path = Path(__file__).parent / "rgb1.jpg"
+        with TemporaryDirectory('screen') as target_dir:
+            image_converter = ImageConverter(target_dir=target_dir)
+            converted_image_path = image_converter.convert_image(image_path, source_degrees=8, target_degrees=10)
+            converted_image = Image.open(converted_image_path)
+
+        converted_image = np.array(converted_image)
+        amount_gray = 0
+        for index in np.ndindex(converted_image.shape[:2]):
+            color = converted_image[index]
+            gray = [128, 128, 128]
+            if (color == gray).all():
+                amount_gray += 1
+        assert amount_gray / converted_image.size == approx(.11658, abs=.0001)
+        assert amount_gray == approx(1_880_000, rel=.0005)  # depending on Pillow version for some reason
+
+    def test_enlarge_image_centered(self):
+        image_path = Path(__file__).parent / "rgb1.jpg"
+        with TemporaryDirectory('screen') as target_dir:
+            image_converter = ImageConverter(target_dir=target_dir)
+            converted_image_path = image_converter.convert_image(image_path, source_degrees=8, target_degrees=10)
+            converted_image = Image.open(converted_image_path)
+
+        converted_image = np.array(converted_image)
+        gray = [128, 128, 128]
+        assert (converted_image[48, 48] == gray).all()
+        assert (converted_image[224 - 48, 224 - 48] == gray).all()
+        assert (converted_image[48, 224 - 48] == gray).all()
+        assert (converted_image[224 - 48, 48] == gray).all()

tests/test_examples.py

@@ -0,0 +1,41 @@
+# These tests ensure that the example notebooks execute without errors
+
+import logging
+from pathlib import Path
+
+import nbformat
+import pytest
+from nbconvert.preprocessors import ExecutePreprocessor
+
+_logger = logging.getLogger(__name__)
+
+
+def run_notebook(notebook_path):
+    _logger.info(f"Running {notebook_path}")
+    with open(notebook_path, encoding="utf-8") as f:
+        nb = nbformat.read(f, as_version=4)
+
+    proc = ExecutePreprocessor(timeout=30 * 60, kernel_name='python3')
+    proc.allow_errors = True
+    proc.preprocess(nb, {'metadata': {'path': '/'}})
+
+    errors = []
+    for cell in nb.cells:
+        if 'outputs' in cell:
+            for output in cell['outputs']:
+                if output.output_type == 'error':
+                    errors.append(output)
+
+    return nb, errors
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize('filename', [
+    'score.ipynb',
+    'data_metrics_benchmarks.ipynb',
+    'models.ipynb',
+])
+def test_notebook(filename):
+    filepath = Path(__file__).parent / '..' / 'examples' / filename
+    nb, errors = run_notebook(filepath)
+    assert not errors, f"Encountered errors: {errors}"

tests/test_integration.py

@@ -0,0 +1,43 @@
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from pytest import approx
+
+
+def test_can_import():
+    # noinspection PyUnresolvedReferences
+    import brainscore_vision
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("model_identifier, benchmark_identifier, expected_score", [
+    ("pixels", "MajajHong2015public.IT-pls", approx(0.07637264, abs=0.0005)),
+])
+def test_model_benchmark_score(model_identifier, benchmark_identifier, expected_score):
+    from brainscore_vision import score
+    actual_score = score(model_identifier=model_identifier, benchmark_identifier=benchmark_identifier)
+    assert actual_score == expected_score
+
+@pytest.mark.memory_intense
+def test_model_benchmark_commandline_score():
+    process = subprocess.run(
+        [
+            sys.executable,
+            "brainscore_vision",
+            "score",
+            "--model_identifier=pixels",
+            "--benchmark_identifier=MajajHong2015public.IT-pls",
+        ],
+        cwd=Path(__file__).parent.parent,
+        capture_output=True,
+        text=True,
+    )
+    assert process.returncode == 0, "Process failed"
+    assert "error" not in process.stderr.lower()
+    output = process.stdout
+    assert "Score" in output
+    assert "0.00548197" in output
+    assert "<xarray.Score ()>\narray(0.07637264)" in output
+    assert "model_identifier:      pixels" in output
+    assert "benchmark_identifier:  MajajHong2015public.IT-pls" in output