brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/submission/actions_helpers.py

@@ -0,0 +1,153 @@
+"""
+Supports two GitHub Actions automerge workflows:
+1. automerge_plugin-only_prs
+2. check_if_pr_is_automergeable
+Uses GitHub API to support the following functions
+- Retrieve a PR Head (SHA or branch name) from the workflow trigger event
+- Retrieve a PR number from a PR Head (SHA or branch name)
+- Check test suite status (Travis, Jenkins)
+Final outputs are accessed by the action via print()
+"""
+
+import json
+import os
+import requests
+import sys
+import smtplib
+from typing import Union
+from email.mime.text import MIMEText
+
+BASE_URL = "https://api.github.com/repos/brain-score/vision"
+
+
+def get_data(url: str) -> dict:
+    r = requests.get(url)
+    assert r.status_code == 200, f'{r.status_code}: {r.reason}'
+    return r.json()
+
+def get_pr_num_from_head(pr_head) -> int:
+    """
+    Given either an SHA (status event) or a branch name (pull request event),
+    returns the number of the pull request with that head SHA or branch name.
+    """
+    event_type = os.environ["GITHUB_EVENT_NAME"]
+
+    if event_type == "pull_request":
+        query = f"repo:brain-score/vision type:pr head:{pr_head}"
+    else:
+        query = f"repo:brain-score/vision type:pr sha:{pr_head}"
+    url = f"https://api.github.com/search/issues?q={query}"
+    pull_requests = get_data(url)
+    assert pull_requests["total_count"] == 1, f'Expected one PR associated with this SHA but found none or more than one, cannot automerge'
+    pr_num = pull_requests["items"][0]["number"]
+
+    return pr_num
+
+def _load_event_file() -> dict:
+    with open(os.environ["GITHUB_EVENT_PATH"]) as f:
+        return json.load(f)
+    
+def get_pr_head_from_github_event() -> str:
+    """
+    Based on the event that triggered the workflow (status or pull_request),
+    returns either an SHA (status) or branch name (pull_request)
+    """
+    pr_head = None
+    event_type = os.environ["GITHUB_EVENT_NAME"]
+
+    if event_type == "status":
+        f = _load_event_file()
+        candidate_branches = [branch for branch in f["branches"] if branch["name"] != "master"]
+        if len(candidate_branches) == 1:
+            pr_head = candidate_branches[0]["commit"]["sha"]
+
+    elif event_type == "pull_request":
+        pr_head = os.environ["GITHUB_HEAD_REF"]
+
+    return pr_head
+
+def _get_end_time(d: dict) -> str:
+    return d['end_time']
+
+def _return_last_result(results: list) -> Union[str, None]:
+    if results:
+        last_result = max(results, key=_get_end_time)['result']
+    else:
+        last_result = None
+    return last_result
+
+def get_statuses_result(context: str, statuses_json: dict) -> str:
+    statuses = [{'end_time': status['updated_at'], 'result': status['state']} 
+                for status in statuses_json if status['context'] == context]
+    last_status_result = _return_last_result(statuses)
+    return last_status_result
+
+def are_all_tests_passing(test_results: dict) -> dict:
+    if any(result != "success" for result in test_results.values()):
+        return False
+    else:
+        return True
+        
+def any_tests_failing(test_results: dict) -> dict:
+    if any(result == "failure" for result in test_results.values()):
+        return True
+    else:
+        return False
+    
+def is_labeled_automerge(pr_num: int) -> bool:
+    label_data = get_data(f"{BASE_URL}/issues/{pr_num}/labels")
+    labeled_automerge = any(label['name'] in ('automerge', 'automerge-web') for label in label_data)
+    return labeled_automerge
+
+def send_failure_email(email: str, pr_number: str, mail_username: str, mail_password: str):
+    """ Send submitter an email if their web-submitted PR fails. """
+    body = "Your Brain-Score submission did not pass checks. " \
+           "Please review the test results and update the PR at " \
+           f"https://github.com/brain-score/vision/pull/{pr_number} " \
+           "or send in an updated submission via the website."
+    msg = MIMEText(body)
+    msg['Subject'] = "Brain-Score submission failed"
+    msg['From'] = "Brain-Score"
+    msg['To'] = email
+
+    # send email
+    with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp_server:
+        smtp_server.login(mail_username, mail_password)
+        smtp_server.sendmail(mail_username, email, msg.as_string())
+
+
+if __name__ == "__main__":
+    
+    pr_head = get_pr_head_from_github_event()
+    if not pr_head:
+        print("No PR head found. Exiting."); sys.exit()
+    pr_num = get_pr_num_from_head(pr_head)
+    
+    # GitHub Actions helpers
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "get_pr_head":
+            print(pr_head)
+        elif sys.argv[1] == "get_pr_num":
+            print(pr_num)
+        sys.exit()
+
+    # Check test results and ensure PR is automergeable
+    statuses_json = get_data(f"{BASE_URL}/statuses/{pr_head}")
+
+    results_dict = {'travis_pr_result': get_statuses_result('continuous-integration/travis', statuses_json),
+                    'jenkins_plugintests_result': get_statuses_result('Brain-Score Jenkins CI - plugin tests', statuses_json),
+                    'jenkins_unittests_result': get_statuses_result('Brain-Score Jenkins CI', statuses_json)}
+
+    tests_pass = are_all_tests_passing(results_dict)
+    tests_fail = any_tests_failing(results_dict)
+
+    if tests_pass:
+        if is_labeled_automerge(pr_num):
+            print(True)
+        else:
+            print("All tests pass but not labeled for automerge. Exiting.")
+    else:
+        if tests_fail and is_labeled_automerge(pr_num):
+            print(False)
+        else:
+            print(results_dict)

brainscore_vision/submission/config.py

@@ -0,0 +1,7 @@
+import os
+
+
+def get_database_secret() -> str:
+    secret = os.getenv('BSC_DATABASESECRET')
+    assert secret is not None, "Need to specify environment variable 'BSC_DATABASESECRET'"
+    return secret

brainscore_vision/submission/endpoints.py

@@ -0,0 +1,58 @@
+from typing import List, Union, Dict
+
+from brainscore_core import Score, Benchmark
+from brainscore_core.submission import RunScoringEndpoint, DomainPlugins
+from brainscore_core.submission.endpoints import make_argparser, resolve_models_benchmarks, get_user_id, \
+    send_email_to_submitter as send_email_to_submitter_core
+from brainscore_vision import load_model, load_benchmark, score
+from brainscore_vision.submission import config
+
+
+class VisionPlugins(DomainPlugins):
+    def load_model(self, model_identifier: str):
+        return load_model(model_identifier)
+
+    def load_benchmark(self, benchmark_identifier: str) -> Benchmark:
+        return load_benchmark(benchmark_identifier)
+
+    def score(self, model_identifier: str, benchmark_identifier: str) -> Score:
+        return score(model_identifier, benchmark_identifier)
+
+
+vision_plugins = VisionPlugins()
+run_scoring_endpoint = RunScoringEndpoint(vision_plugins, db_secret=config.get_database_secret())
+
+
+def run_scoring(args_dict: Dict[str, Union[str, List]]):
+    model_ids, benchmark_ids = resolve_models_benchmarks(domain="vision", args_dict=args_dict)
+
+    for benchmark in benchmark_ids:
+        for model in model_ids:
+            run_scoring_endpoint(domain="vision", jenkins_id=args_dict["jenkins_id"],
+                                 model_identifier=model, benchmark_identifier=benchmark,
+                                 user_id=args_dict["user_id"], model_type="brainmodel",
+                                 public=args_dict["public"], competition=args_dict["competition"])
+
+
+def send_email_to_submitter(uid: int, domain: str, pr_number: str,
+                            mail_username: str, mail_password: str):
+    send_email_to_submitter_core(uid=uid, domain=domain, pr_number=pr_number,
+                                 db_secret=config.get_database_secret(),
+                                 mail_username=mail_username, mail_password=mail_password)
+
+
+if __name__ == '__main__':
+    parser = make_argparser()
+    args, remaining_args = parser.parse_known_args()
+    args_dict = vars(args)
+
+    if 'user_id' not in args_dict or args_dict['user_id'] is None:
+        user_id = get_user_id(args_dict['author_email'], db_secret=config.get_database_secret())
+        args_dict['user_id'] = user_id
+
+    if args.fn == 'run_scoring':
+        run_scoring(args_dict)
+    elif args.fn == 'resolve_models_benchmarks':
+        resolve_models_benchmarks(domain="vision", args_dict=args_dict)
+    else:
+        raise ValueError(f'Invalid method: {args.fn}')

brainscore_vision/utils/__init__.py

@@ -0,0 +1,91 @@
+"""
+Provide generic helper classes.
+"""
+
+import copy
+
+
+def fullname(obj):
+    """ Resolve the full module-qualified name of an object. Typically used for logger naming. """
+    return obj.__module__ + "." + obj.__class__.__name__
+
+
+def map_fields(obj, func):
+    for field_name, field_value in vars(obj).items():
+        field_value = func(field_value)
+        setattr(obj, field_name, field_value)
+
+
+def combine_fields(objs, func):
+    if len(objs) == 0:
+        return objs
+    fields = list(vars(objs[0]).keys())
+    field_values = {field_name: [] for field_name in fields}
+    for obj in objs:
+        for field_name in fields:
+            field_value = getattr(obj, field_name)
+            field_values[field_name].append(field_value)
+    field_values = dict(map(
+        lambda field_name_values: (field_name_values[0], func(field_name_values[1])), field_values.items()))
+    ctr = objs[0].__class__
+    return ctr(**field_values)
+
+
+def recursive_dict_merge(dict1, dict2):
+    """
+    Merges dictionaries (of dictionaries).
+    Preference is given to the second dict, i.e. if a key occurs in both dicts, the value from `dict2` is used.
+    """
+    result = copy.deepcopy(dict1)
+    for key in dict2:
+        if key in dict1 and isinstance(dict1[key], dict) and isinstance(dict2[key], dict):
+            result[key] = recursive_dict_merge(dict1[key], dict2[key])
+        else:
+            result[key] = dict2[key]
+    return result
+
+
+class LazyLoad:
+    def __init__(self, load_fnc):
+        self.load_fnc = load_fnc
+        self.content = None
+
+    def __getattr__(self, name):
+        self._ensure_loaded()
+        return getattr(self.content, name)
+
+    def __setattr__(self, key, value):
+        if key not in ['content', 'load_fnc']:
+            self._ensure_loaded()
+            return setattr(self.content, key, value)
+        return super(LazyLoad, self).__setattr__(key, value)
+
+    def __getitem__(self, item):
+        self._ensure_loaded()
+        return self.content.__getitem__(item)
+
+    def __setitem__(self, key, value):
+        self._ensure_loaded()
+        return self.content.__setitem__(key, value)
+
+    def _ensure_loaded(self):
+        if self.content is None:
+            self.content = self.load_fnc()
+
+    def reload(self):
+        self.content = self.load_fnc()
+
+    def __call__(self, *args, **kwargs):
+        self._ensure_loaded()
+        return self.content(*args, **kwargs)
+
+    def __len__(self):
+        self._ensure_loaded()
+        return len(self.content)
+
+    @property
+    def __class__(self):
+        self._ensure_loaded()
+        return self.content.__class__
+
+

brainscore_vision-2.1.dist-info/LICENSE

@@ -0,0 +1,11 @@
+
+MIT License
+
+Copyright (c) 2017, Martin Schrimpf and Jon Prescott-Roy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

brainscore_vision-2.1.dist-info/METADATA

@@ -0,0 +1,152 @@
+Metadata-Version: 2.1
+Name: brainscore_vision
+Version: 2.1
+Summary: The Brain-Score library enables model comparisons to behavioral and neural experiments
+License: 
+        MIT License
+        
+        Copyright (c) 2017, Martin Schrimpf and Jon Prescott-Roy
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+        
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy<2
+Requires-Dist: brainscore-core
+Requires-Dist: result-caching
+Requires-Dist: importlib-metadata<5
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
+Requires-Dist: opencv-python
+Requires-Dist: h5py
+Requires-Dist: tqdm
+Requires-Dist: gitpython
+Requires-Dist: fire
+Requires-Dist: jupyter
+Requires-Dist: pybtex
+Requires-Dist: peewee
+Requires-Dist: psycopg2-binary
+Requires-Dist: networkx
+Requires-Dist: eva-decord
+Requires-Dist: psutil
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-check; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Requires-Dist: pytest-timeout; extra == "test"
+Requires-Dist: torch; extra == "test"
+Requires-Dist: torchvision; extra == "test"
+Requires-Dist: matplotlib; extra == "test"
+
+[![Build Status](https://app.travis-ci.com/brain-score/vision.svg?token=vqt7d2yhhpLGwHsiTZvT&branch=master)](https://app.travis-ci.com/brain-score/vision)
+[![Documentation Status](https://readthedocs.org/projects/brain-score/badge/?version=latest)](https://brain-score.readthedocs.io/en/latest/?badge=latest)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) 
+
+Brain-Score is a platform to evaluate computational models of brain function 
+on their match to brain measurements in primate vision. 
+The intent of Brain-Score is to adopt many (ideally all) the experimental benchmarks in the field
+for the purpose of model testing, falsification, and comparison.
+To that end, Brain-Score operationalizes experimental data into quantitative benchmarks 
+that any model candidate following the [`BrainModel`](brainscore_vision/model_interface.py) interface can be scored on.
+
+Note that you can only access a limited set of public benchmarks when running locally. To score a model on all benchmarks, submit it via the [brain-score.org website](http://www.brain-score.org).
+
+See the [documentation](https://brain-score.readthedocs.io) for more details, e.g. for submitting a [model](https://brain-score.readthedocs.io/en/latest/modules/model_tutorial.html) or [benchmark](https://brain-score.readthedocs.io/en/latest/modules/benchmark_tutorial.html) to Brain-Score. For a step-by-step walkthrough on submitting models to the Brain-Score website, see these [web tutorials](https://www.brain-score.org/tutorial/).
+
+See these [code examples](https://github.com/brain-score/vision/blob/master/examples) on scoring models, retrieving data, using and defining benchmarks and metrics.
+These [previous examples](https://github.com/brain-score/candidate_models/blob/master/examples/score-model.ipynb) might be helpful, but their usage has been deprecated after the 2.0 update.
+
+Brain-Score is made by and for the community. 
+To contribute, please [send in a pull request](https://github.com/brain-score/vision/pulls).
+
+
+## Local installation
+
+You will need Python = 3.11 and pip >= 18.1.
+
+`pip install git+https://github.com/brain-score/vision`
+
+Test if the installation is successful by scoring a model on a public benchmark:
+
+```python
+from brainscore_vision.benchmarks import public_benchmark_pool
+
+benchmark = public_benchmark_pool['dicarlo.MajajHong2015public.IT-pls']
+model = my_model()
+score = benchmark(model)
+
+# >  <xarray.Score ()>
+# >  array(0.07637264)
+# >  Attributes:
+# >  Attributes:
+# >      error:                 <xarray.Score ()>\narray(0.00548197)
+# >      raw:                   <xarray.Score ()>\narray(0.22545106)\nAttributes:\...
+# >      ceiling:               <xarray.DataArray ()>\narray(0.81579938)\nAttribut...
+# >      model_identifier:      my-model
+# >      benchmark_identifier:  dicarlo.MajajHong2015public.IT-pls
+```
+
+Some steps may take minutes because data has to be downloaded during first-time use.
+
+## Environment Variables
+
+| Variable               | Description                                                                                                                           |
+|------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
+| RESULTCACHING_HOME     | directory to cache results (benchmark ceilings) in, `~/.result_caching` by default (see https://github.com/brain-score/result_caching) |
+
+
+## License
+
+MIT license
+
+
+## Troubleshooting
+
+<details>
+<summary>`ValueError: did not find HDF5 headers` during netcdf4 installation</summary>
+pip seems to fail properly setting up the HDF5_DIR required by netcdf4.
+Use conda: `conda install netcdf4`
+</details>
+
+<details>
+<summary>repeated runs of a benchmark / model do not change the outcome even though code was changed</summary>
+results (scores, activations) are cached on disk using https://github.com/mschrimpf/result_caching.
+Delete the corresponding file or directory to clear the cache.
+</details>
+
+
+## CI environment
+
+Add CI related build commands to `test_setup.sh`. The script is executed in CI environment for unittests.
+
+
+## References
+
+If you use Brain-Score in your work, please cite 
+["Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?"](https://www.biorxiv.org/content/10.1101/407007v2) (technical) and 
+["Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence"](https://www.cell.com/neuron/fulltext/S0896-6273(20)30605-X) (perspective) 
+as well as the respective benchmark sources.
+
+```bibtex
+@article{SchrimpfKubilius2018BrainScore,
+  title={Brain-Score: Which Artificial Neural Network for Object Recognition is most Brain-Like?},
+  author={Martin Schrimpf and Jonas Kubilius and Ha Hong and Najib J. Majaj and Rishi Rajalingham and Elias B. Issa and Kohitij Kar and Pouya Bashivan and Jonathan Prescott-Roy and Franziska Geiger and Kailyn Schmidt and Daniel L. K. Yamins and James J. DiCarlo},
+  journal={bioRxiv preprint},
+  year={2018},
+  url={https://www.biorxiv.org/content/10.1101/407007v2}
+}
+
+@article{Schrimpf2020integrative,
+  title={Integrative Benchmarking to Advance Neurally Mechanistic Models of Human Intelligence},
+  author={Schrimpf, Martin and Kubilius, Jonas and Lee, Michael J and Murty, N Apurva Ratan and Ajemian, Robert and DiCarlo, James J},
+  journal={Neuron},
+  year={2020},
+  url={https://www.cell.com/neuron/fulltext/S0896-6273(20)30605-X}
+}
+```