flyvis 1.1.1__tar.gz

flyvis-1.1.1/.env.example

@@ -0,0 +1,11 @@
+# This is to permanently set the root directory for the flyvis data.
+# Uncomment the line below and set the path to the desired root directory.
+
+# FLYVIS_ROOT_DIR='data'
+
+# This is to change the timezone for the loggers.
+TIMEZONE=Europe/Berlin
+# TIMEZONE=America/New_York
+
+# This is to set the testing flag.
+TESTING=FALSE

flyvis-1.1.1/.github/workflows/auto_test.yml

@@ -0,0 +1,46 @@
+name: Tests
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install the repository in editable mode
+      run: pip install -e ".[dev]"
+
+    - name: Download pretrained models
+      run: flyvis download-pretrained
+
+    - name: Run tests with coverage
+      run: pytest -m "not require_large_download and not require_gpu" --cov=flyvis --cov-report=xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./coverage.xml
+        flags: unittests
+        name: codecov-umbrella
+        fail_ci_if_error: false

flyvis-1.1.1/.github/workflows/docs.yml

@@ -0,0 +1,58 @@
+name: Documentation
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'examples/**'
+      - 'scripts/**'
+      - 'mkdocs.yml'
+      - '.github/workflows/docs.yml'
+
+permissions:
+  contents: write
+
+jobs:
+  docs:
+    # Only run if commit message contains [build-docs]
+    if: "contains(github.event.head_commit.message, '[build-docs]')"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # fetch all history for proper versioning
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[docs]
+
+      # TODO: Enable this section when GPU runner is available
+      # - name: Convert notebooks to markdown
+      #   env:
+      #     TESTING: "true"
+      #     JUPYTER_CONFIG_DIR: "/tmp/jupyter-config"
+      #   run: |
+      #     # Convert notebooks to markdown
+      #     jupyter nbconvert --to markdown examples/*.ipynb --output-dir docs/examples/ --TagRemovePreprocessor.remove_cell_tags hide
+      #   # Note: This step requires GPU access. Currently notebooks should be converted locally
+      #   # and committed to the repository. When setting up GPU runner:
+      #   # 1. Use a runner with GPU support
+      #   # 2. Add necessary CUDA dependencies
+      #   # 3. Uncomment this section
+
+      - name: Build script documentation
+        run: |
+          python docs/script_to_md.py
+
+      - name: Deploy documentation
+        run: |
+          git config --global user.name "github-actions"
+          git config --global user.email "github-actions@github.com"
+          mkdocs gh-deploy --force

flyvis-1.1.1/.gitignore

@@ -0,0 +1,187 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+.vscode/
+
+/data/**
+!data/connectome/fib25-fib19_v2.2.json
+!/data/results
+
+*.code-workspace
+*.egg-info
+.pytest_cache
+.ruff_cache
+.fonts
+*.log
+**/__pycache__
+**/**/__pycache__
+.coverage
+**/version.py
+debug
+.env.dev
+prepare_api_reference.py
+profile_imports.py
+import_profile*
+.fonts/
+docs/site/
+*.dev
+dev

flyvis-1.1.1/.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.5.5
+  hooks:
+    - id: ruff
+      types_or: [ python]
+      args: [ --fix ]
+    - id: ruff-format
+      types_or: [ python]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.5.0
+  hooks:
+    - id: check-added-large-files
+      args: ['--maxkb=10240']
+    - id: check-merge-conflict
+    - id: check-toml
+    - id: check-yaml
+    - id: end-of-file-fixer
+    - id: mixed-line-ending
+      args: [--fix=lf]
+    - id: trailing-whitespace

flyvis-1.1.1/CHANGELOG.md

@@ -0,0 +1,46 @@
+# Changelog
+
+## [v1.1.1] - 2024-11-21
+
+### Distribution
+- Moved configuration files into package structure
+- Switched to `importlib.resources` for resource management
+- Removed MANIFEST.in in favor of pyproject.toml configuration
+- Enhanced root directory resolution logic
+
+### Infrastructure
+- Improved cluster management with better dry-run and slurm support
+
+### CLI
+- Enhanced command-line interface error handling
+- Improved argument parsing for multiple commands
+- Added `init_config` for config-based customization of network and training
+
+### Documentation
+- Updated package metadata for PyPI
+- Added project URLs to package configuration
+- Removed broken badges from README
+- Added explanations for hydra-config-based customization of network and training in `CLI Reference`
+
+## [v1.1.0] - 2024-11-20
+
+### Breaking
+- Renamed package from `flyvision` to `flyvis` for better consistency
+
+### Features
+- Added Command Line Interface `flyvis` for scripts
+- Improved test suite performance and coverage
+- Relaxed Python version dependency and added multi-Python version testing
+
+### Documentation
+- Improved docs
+- Updated project metadata for PyPI
+- Removed broken badges from README
+
+### Infrastructure
+- Removed strict version pins for better compatibility (particularly removed UMAP constraints)
+- Updated GitHub workflows
+- Updated README badges
+- Updated package-data handling
+
+[v1.1.0]: https://github.com/TuragaLab/flyvis/releases/tag/v1.1.0

flyvis-1.1.1/PKG-INFO

@@ -0,0 +1,165 @@
+Metadata-Version: 2.1
+Name: flyvis
+Version: 1.1.1
+Summary: Connectome and task-constrained vision models of the fruit fly.
+Project-URL: Homepage, https://github.com/TuragaLab/flyvis
+Project-URL: Documentation, https://turagalab.github.io/flyvis
+Project-URL: Source Code, https://github.com/TuragaLab/flyvis
+Project-URL: Bug Tracker, https://github.com/TuragaLab/flyvis/issues
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Adaptive Technologies
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Development Status :: 3 - Alpha
+Requires-Python: <3.13,>=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: ipython
+Requires-Dist: numba
+Requires-Dist: datamate
+Requires-Dist: scipy
+Requires-Dist: matplotlib
+Requires-Dist: networkx
+Requires-Dist: scikit-learn
+Requires-Dist: umap-learn
+Requires-Dist: ffmpeg-python
+Requires-Dist: pytest
+Requires-Dist: torch
+Requires-Dist: torchvision
+Requires-Dist: python-dotenv
+Requires-Dist: hydra-core
+Requires-Dist: toolz
+Requires-Dist: tqdm
+Requires-Dist: requests
+Requires-Dist: google-api-python-client
+Requires-Dist: xarray
+Requires-Dist: pandas
+Requires-Dist: joblib
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: jupyter; extra == "dev"
+Requires-Dist: papermill; extra == "dev"
+Requires-Dist: tabulate; extra == "dev"
+Requires-Dist: tqdm; extra == "dev"
+Requires-Dist: ipywidgets; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: markdown-include; extra == "docs"
+Requires-Dist: mkdocs-redirects; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.18; extra == "docs"
+Requires-Dist: nbconvert; extra == "docs"
+Requires-Dist: mike; extra == "docs"
+Requires-Dist: mkdocs-macros-plugin; extra == "docs"
+Requires-Dist: jupyter; extra == "docs"
+Requires-Dist: tabulate; extra == "docs"
+Requires-Dist: tqdm; extra == "docs"
+Requires-Dist: ipywidgets; extra == "docs"
+Provides-Extra: examples
+Requires-Dist: jupyter; extra == "examples"
+Requires-Dist: tabulate; extra == "examples"
+Requires-Dist: tqdm; extra == "examples"
+Requires-Dist: ipywidgets; extra == "examples"
+Provides-Extra: pretrained
+Requires-Dist: numba==0.55.2; extra == "pretrained"
+Requires-Dist: matplotlib==3.7.1; extra == "pretrained"
+Requires-Dist: scikit-learn==1.1.1; extra == "pretrained"
+Requires-Dist: umap-learn==0.5.3; extra == "pretrained"
+Requires-Dist: xarray==2023.7.0; extra == "pretrained"
+Requires-Dist: pandas==2.2.2; extra == "pretrained"
+Requires-Dist: joblib==1.3.2; extra == "pretrained"
+
+<h1>
+<p style="text-align:left;">
+    <img src="docs/docs/images/flyvis_logo_light@150ppi.webp" width="50%" alt="Flyvis Logo">
+</p>
+</h1>
+
+A connectome-constrained deep mechanistic network (DMN) model of the fruit fly visual system in PyTorch.
+
+- Explore connectome-constrained models of the fruit fly visual system.
+- Generate and test hypotheses about neural computations.
+- Try pretrained models on your data.
+- Develop custom models using our framework.
+
+Flyvis is our official implementation of
+[Lappalainen et al., "Connectome-constrained networks predict neural activity across the fly visual system." Nature (2024).](https://www.nature.com/articles/s41586-024-07939-3)
+
+## Documentation
+
+For detailed documentation, installation instructions, tutorials, and API reference, visit our [documentation website](https://turagalab.github.io/flyvis/).
+
+## Tutorials
+
+Explore our tutorials to get started with flyvis. You can read the prerun tutorials in the docs or try them yourself for a quick start in Google Colab:
+
+1. Explore the Connectome
+   - [Tutorial](https://turagalab.github.io/flyvis/examples/01_flyvision_connectome/)
+   - [Google Colab](https://colab.research.google.com/github/TuragaLab/flyvis/blob/main/examples/01_flyvision_connectome.ipynb)
+
+2. Train the Network on the Optic Flow Task
+   - [Tutorial](https://turagalab.github.io/flyvis/examples/02_flyvision_optic_flow_task/)
+   - [Google Colab](https://colab.research.google.com/github/TuragaLab/flyvis/blob/main/examples/02_flyvision_optic_flow_task.ipynb)
+
+3. Flash Responses
+   - [Tutorial](https://turagalab.github.io/flyvis/examples/03_flyvision_flash_responses/)
+   - [Google Colab](https://colab.research.google.com/github/TuragaLab/flyvis/blob/main/examples/03_flyvision_flash_responses.ipynb)
+
+4. Moving Edge Responses
+   - [Tutorial](https://turagalab.github.io/flyvis/examples/04_flyvision_moving_edge_responses/)
+   - [Google Colab](https://colab.research.google.com/github/TuragaLab/flyvis/blob/main/examples/04_flyvision_moving_edge_responses.ipynb)
+
+5. Ensemble Clustering
+   - [Tutorial](https://turagalab.github.io/flyvis/examples/05_flyvision_umap_and_clustering_models/)
+   - [Google Colab](https://colab.research.google.com/github/TuragaLab/flyvis/blob/main/examples/05_flyvision_umap_and_clustering_models.ipynb)
+
+6. Maximally Excitatory Stimuli
+   - [Tutorial](https://turagalab.github.io/flyvis/examples/06_flyvision_maximally_excitatory_stimuli/)
+   - [Google Colab](https://colab.research.google.com/github/TuragaLab/flyvis/blob/main/examples/06_flyvision_maximally_excitatory_stimuli.ipynb)
+
+7. Custom Stimuli
+   - [Tutorial](https://turagalab.github.io/flyvis/examples/07_flyvision_providing_custom_stimuli/)
+   - [Google Colab](https://colab.research.google.com/github/TuragaLab/flyvis/blob/main/examples/07_flyvision_providing_custom_stimuli.ipynb)
+
+
+## Main Results
+
+Find the notebooks for the main results in the documentation.
+
+- [Fig. 1: Connectome-constrained and task-optimized models of the fly visual system](https://turagalab.github.io/flyvis/examples/figure_01_fly_visual_system/)
+- [Fig. 2: Ensembles of DMNs predict tuning properties](https://turagalab.github.io/flyvis/examples/figure_02_simple_stimuli_responses/)
+- [Fig. 3: Cluster analysis of DMN ensembles enables hypothesis generation and suggests experimental tests](https://turagalab.github.io/flyvis/examples/figure_03_naturalistic_stimuli_responses/)
+- [Fig. 4: Task-optimal DMNs largely recapitulate known mechanisms of motion computation](https://turagalab.github.io/flyvis/examples/figure_04_mechanisms/)
+
+## Citation
+
+```
+@article{lappalainen2024connectome,
+	title = {Connectome-constrained networks predict neural activity across the fly visual system},
+	issn = {1476-4687},
+	url = {https://doi.org/10.1038/s41586-024-07939-3},
+	doi = {10.1038/s41586-024-07939-3},
+	journal = {Nature},
+	author = {Lappalainen, Janne K. and Tschopp, Fabian D. and Prakhya, Sridhama and McGill, Mason and Nern, Aljoscha and Shinomiya, Kazunori and Takemura, Shin-ya and Gruntman, Eyal and Macke, Jakob H. and Turaga, Srinivas C.},
+	month = sep,
+	year = {2024},
+}
+```
+
+## Links
+
+- [Nature Article](https://www.nature.com/articles/s41586-024-07939-3)
+- [Documentation](https://turagalab.github.io/flyvis/)
+
+## Correspondence
+
+For questions or inquiries, [please contact us.](mailto:janne.lappalainen@uni-tuebingen.de?cc=jakob.macke@uni-tuebingen.de,turagas@janelia.hhmi.org)

flyvis-1.1.1/docs/README.md

@@ -0,0 +1,53 @@
+# Documentation
+
+The documentation is available at: <https://turagalab.github.io/flyvis/>
+
+## Building the Documentation
+
+The documentation is built with [mkdocs](https://www.mkdocs.org/).
+
+### Convert examples to markdown:
+
+Only applies to making changes to the examples.
+
+1. Run all notebooks inplace:
+
+This takes a while. Make sure that recordings are precomputed and cached, i.e., run
+`scripts/record.py` beforehand for the required ensemble.
+
+```bash
+export TESTING=true  # optional, will not run animations to speed things up
+export JUPYTER_CONFIG_DIR=$(mktemp -d) # to avoid conflicts with notebook version and extensions
+
+for notebook in ../examples/*.ipynb; do
+    jupyter nbconvert --to notebook --execute "$notebook" --inplace
+done
+```
+
+2. Convert notebooks to markdown:
+
+```bash
+jupyter nbconvert --to markdown ../examples/*.ipynb --output-dir docs/examples/ --TagRemovePreprocessor.remove_cell_tags hide
+```
+
+### Build the scripts docs
+
+Only applies to making changes to the scripts.
+
+```bash
+python script_to_md.py
+```
+
+## Serve the docs locally
+
+```bash
+mkdocs serve
+```
+
+## Deploy the docs to GitHub
+
+See [mkdocs user guide](https://www.mkdocs.org/user-guide/deploying-your-docs/) for more details.
+
+```bash
+mkdocs gh-deploy
+```

flyvis-1.1.1/docs/docs/acknowledgements.md

@@ -0,0 +1,36 @@
+# Acknowledgements
+
+`flyvision` is developed at the [Macke Lab](https://www.mackelab.org) at the University of Tübingen and at the [Turaga Lab](https://www.janelia.org/lab/turaga-lab) at Janelia Research Campus.
+
+## Paper
+
+Acknowledgements for the scientific content can be found in the [paper](https://www.nature.com/articles/s41586-024-07939-3#Ack1).
+
+> Lappalainen J.K., Tschopp F.D., Prakhya S., McGill M., Nern A., Shinomiya K., Takemura S., Gruntman E., Macke J.H., Turaga S.C., Connectome-constrained networks predict neural activity across the fly visual system. Nature 1–9 (2024).
+
+### Funding
+
+> This project was supported by the HHMI. J.K.L. and J.H.M. were supported by the German Research Foundation (DFG) through Germany’s Excellence Strategy (EXC-Number 2064/1, Project number 390727645), the German Federal Ministry of Education and Research (BMBF; Tübingen AI Center, FKZ: 01IS18039A) and the European Union (ERC, DeepCoMechTome, 101089288).
+
+
+## Credits to colleagues
+
+We extend credits to Jan Funke for an early discussion on the IO code. Roman Vaxenburg for the function `is_inside_hex` for more fine-grained control over stimulus rendering.
+The whole new flyvis team at the Macke Lab for supporting the code cleanup and filling in tutorial notebooks.
+Gert-Jan Both for general feedback and for pointing towards xarray and joblib for pruning of labeled tensor code and custom caching.
+
+## Disclaimer on the usage of AI
+
+The majority of the code in `flyvision` is based on code that we wrote before the widespread use of AI in scientific software development. Since our preprint was realeased in early 2023, we used AI in the last months to help us with in depth documentation. We tested the code to the best of our ability, but it might still contain errors. Please help us improve the code by [opening an issue](https://github.com/TuragaLab/flyvis/issues) on Github.
+
+## Credits to open source software
+
+[PyTorch](https://pytorch.org/), [NumPy](https://numpy.org/), [xarray](https://xarray.dev/),
+[matplotlib](https://matplotlib.org/),[umap-learn](https://umap-learn.readthedocs.io/), [pandas](https://pandas.pydata.org/),
+[scikit-learn](https://scikit-learn.org/), [scipy](https://scipy.org/), [omegaconf](https://omegaconf.readthedocs.io/),
+[hydra](https://hydra.cc/), [networkx](https://networkx.org/), [toolz](https://toolz.readthedocs.io/en/latest/),
+[joblib](https://joblib.readthedocs.io/)
+
+## MIT License
+
+> Copyright (c) 2023 Janne K. Lappalainen, Fabian D. Tschopp, Mason McGill, Jakob H. Macke, Srinivas C. Turaga