unitrefine 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unitrefine-0.2.0/LICENSE +21 -0
- unitrefine-0.2.0/MANIFEST.in +5 -0
- unitrefine-0.2.0/PKG-INFO +153 -0
- unitrefine-0.2.0/README.md +109 -0
- unitrefine-0.2.0/pyproject.toml +96 -0
- unitrefine-0.2.0/setup.cfg +4 -0
- unitrefine-0.2.0/src/unitrefine/__init__.py +75 -0
- unitrefine-0.2.0/src/unitrefine/apply_models.py +111 -0
- unitrefine-0.2.0/src/unitrefine/cli.py +251 -0
- unitrefine-0.2.0/src/unitrefine/evaluation.py +457 -0
- unitrefine-0.2.0/src/unitrefine/figures.py +1458 -0
- unitrefine-0.2.0/src/unitrefine/import_labels.py +176 -0
- unitrefine-0.2.0/src/unitrefine/launch_sigui.py +138 -0
- unitrefine-0.2.0/src/unitrefine/launch_sigui_validate.py +243 -0
- unitrefine-0.2.0/src/unitrefine/launcher.py +239 -0
- unitrefine-0.2.0/src/unitrefine/main.py +2395 -0
- unitrefine-0.2.0/src/unitrefine/manage_models.py +193 -0
- unitrefine-0.2.0/src/unitrefine/metrics.py +146 -0
- unitrefine-0.2.0/src/unitrefine/model_selection.py +159 -0
- unitrefine-0.2.0/src/unitrefine/models_hub.py +118 -0
- unitrefine-0.2.0/src/unitrefine/phy_curation.py +306 -0
- unitrefine-0.2.0/src/unitrefine/phy_import.py +428 -0
- unitrefine-0.2.0/src/unitrefine/predictions.py +507 -0
- unitrefine-0.2.0/src/unitrefine/resources/curation_gui.jpg +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/fonts/Inter-Bold.ttf +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/fonts/Inter-LICENSE.txt +92 -0
- unitrefine-0.2.0/src/unitrefine/resources/fonts/Inter-Regular.ttf +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/logo.png +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/model_training_example.jpg +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/predict-gui.JPG +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/re-label.JPG +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/si-gui.JPG +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/train_and_load_gui.JPG +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/unit_types_example.jpg +0 -0
- unitrefine-0.2.0/src/unitrefine/resources/unitrefine_gui.JPG +0 -0
- unitrefine-0.2.0/src/unitrefine/retraining_history.py +220 -0
- unitrefine-0.2.0/src/unitrefine/sigui_progress.py +157 -0
- unitrefine-0.2.0/src/unitrefine/theme.py +254 -0
- unitrefine-0.2.0/src/unitrefine/train.py +622 -0
- unitrefine-0.2.0/src/unitrefine/widgets.py +961 -0
- unitrefine-0.2.0/tests/conftest.py +125 -0
- unitrefine-0.2.0/tests/sigui_harness.py +43 -0
- unitrefine-0.2.0/tests/test_active_learning.py +264 -0
- unitrefine-0.2.0/tests/test_cli.py +298 -0
- unitrefine-0.2.0/tests/test_default_models.py +84 -0
- unitrefine-0.2.0/tests/test_figures.py +130 -0
- unitrefine-0.2.0/tests/test_gui.py +122 -0
- unitrefine-0.2.0/tests/test_gui_missing_metrics.py +89 -0
- unitrefine-0.2.0/tests/test_import_labels.py +100 -0
- unitrefine-0.2.0/tests/test_logic.py +168 -0
- unitrefine-0.2.0/tests/test_phy_curation.py +206 -0
- unitrefine-0.2.0/tests/test_phy_import.py +229 -0
- unitrefine-0.2.0/tests/test_project.py +95 -0
- unitrefine-0.2.0/tests/test_sigui_labels.py +108 -0
- unitrefine-0.2.0/tests/test_training.py +113 -0
- unitrefine-0.2.0/tests/test_without_gui.py +62 -0
- unitrefine-0.2.0/unitrefine.egg-info/PKG-INFO +153 -0
- unitrefine-0.2.0/unitrefine.egg-info/SOURCES.txt +60 -0
- unitrefine-0.2.0/unitrefine.egg-info/dependency_links.txt +1 -0
- unitrefine-0.2.0/unitrefine.egg-info/entry_points.txt +4 -0
- unitrefine-0.2.0/unitrefine.egg-info/requires.txt +26 -0
- unitrefine-0.2.0/unitrefine.egg-info/top_level.txt +1 -0
unitrefine-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Anoushka Jain
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
# What the source package (sdist) on PyPI contains besides src/unitrefine: the tests with their helpers, so they
|
|
2
|
+
# can be run from it, but not the paper's notebooks, datasets and tutorials in UnitRefine/ (they are on GitHub)
|
|
3
|
+
recursive-include tests *.py
|
|
4
|
+
prune UnitRefine
|
|
5
|
+
prune UnitRefine.egg-info
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: unitrefine
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: UnitRefine is a machine-learning toolbox that simplifies spike sorting curation by reducing manual curation.
|
|
5
|
+
Author-email: Anoushka Jain <jain.anoushka3@gmail.com>, Chris Halcrow <chrishalcrow@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/anoushkajain/UnitRefine
|
|
8
|
+
Project-URL: Documentation, https://github.com/anoushkajain/UnitRefine#readme
|
|
9
|
+
Project-URL: Issues, https://github.com/anoushkajain/UnitRefine/issues
|
|
10
|
+
Project-URL: Preprint, https://www.biorxiv.org/content/10.1101/2025.03.30.645770v2
|
|
11
|
+
Project-URL: Models, https://huggingface.co/AnoushkaJain3
|
|
12
|
+
Keywords: spike sorting,curation,electrophysiology,machine learning,neuropixels,spikeinterface
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
21
|
+
Requires-Python: <3.14,>=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: numcodecs==0.15.1
|
|
25
|
+
Requires-Dist: numba>=0.61.0
|
|
26
|
+
Requires-Dist: pandas>=2.2.3
|
|
27
|
+
Requires-Dist: spikeinterface>=0.105.0
|
|
28
|
+
Requires-Dist: pyyaml>=6.0.2
|
|
29
|
+
Requires-Dist: scikit-learn==1.7
|
|
30
|
+
Requires-Dist: skops>=0.11.0
|
|
31
|
+
Requires-Dist: huggingface-hub>=0.35.0
|
|
32
|
+
Requires-Dist: s3fs>=2025.10.0
|
|
33
|
+
Requires-Dist: mtscomp>=1.0.2
|
|
34
|
+
Provides-Extra: gui
|
|
35
|
+
Requires-Dist: spikeinterface-gui>=0.14.0; extra == "gui"
|
|
36
|
+
Requires-Dist: pyqtgraph>=0.13.7; extra == "gui"
|
|
37
|
+
Requires-Dist: pyqt5>=5.15.11; extra == "gui"
|
|
38
|
+
Requires-Dist: pyqt5-qt5<5.15.18; (sys_platform == "win32" and platform_machine == "AMD64") and extra == "gui"
|
|
39
|
+
Requires-Dist: pyqt5-qt5>=5.15.0; sys_platform == "darwin" and extra == "gui"
|
|
40
|
+
Requires-Dist: pyqt5-qt5>=5.15.0; sys_platform == "linux" and extra == "gui"
|
|
41
|
+
Requires-Dist: matplotlib>=3.8; extra == "gui"
|
|
42
|
+
Requires-Dist: shap>=0.49.1; extra == "gui"
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# UnitRefine: a community toolbox for automated spike sorting curation
|
|
46
|
+
|
|
47
|
+
[](https://github.com/anoushkajain/UnitRefine/actions/workflows/tests.yml)
|
|
48
|
+
[](https://github.com/anoushkajain/UnitRefine/blob/main/LICENSE)
|
|
49
|
+
|
|
50
|
+
**UnitRefine** labels spike sorted units as **single units (SUA)**, **multi-units (MUA)** or **noise** with machine
|
|
51
|
+
learning classifiers trained on human curation, so you don't have to curate every unit by hand. It works with any
|
|
52
|
+
probe, species, brain region and spike sorter, and is part of [SpikeInterface](https://github.com/SpikeInterface/spikeinterface).
|
|
53
|
+
|
|
54
|
+
<p align="center">
|
|
55
|
+
<img src="https://raw.githubusercontent.com/anoushkajain/UnitRefine/main/src/unitrefine/resources/unit_types_example.jpg" width="500"/>
|
|
56
|
+
</p>
|
|
57
|
+
|
|
58
|
+
With UnitRefine you can:
|
|
59
|
+
|
|
60
|
+
- label your units with [pretrained models](#pretrained-models)
|
|
61
|
+
- train your own model on your labels, and improve it with active learning, in the [GUI](#the-gui)
|
|
62
|
+
- apply models to many recordings on a [computing cluster](https://github.com/anoushkajain/UnitRefine/blob/main/docs/hpc.md)
|
|
63
|
+
- share models on the [Hugging Face Hub](https://huggingface.co/AnoushkaJain3)
|
|
64
|
+
|
|
65
|
+
## Preprint
|
|
66
|
+
|
|
67
|
+
UnitRefine and its validation on mouse, naked mole rat, monkey and human recordings are described in our
|
|
68
|
+
[preprint](https://www.biorxiv.org/content/10.1101/2025.03.30.645770v2).
|
|
69
|
+
|
|
70
|
+
## The GUI
|
|
71
|
+
|
|
72
|
+
<p align="center">
|
|
73
|
+
<img src="https://raw.githubusercontent.com/anoushkajain/UnitRefine/main/docs/images/gui_window.png" width="700"/>
|
|
74
|
+
</p>
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
unitrefine --project_folder my_project
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
A new project comes with the pretrained models. Add your spike sorted data in section 3.3 (a Sorting Analyzer, or a
|
|
81
|
+
Kilosort output folder curated in Phy) and UnitRefine model labels
|
|
82
|
+
its units. To train your own model, label units in the
|
|
83
|
+
[SpikeInterface-GUI](https://github.com/SpikeInterface/spikeinterface-gui/) or import labels you already have.
|
|
84
|
+
|
|
85
|
+
See the **[GUI tutorial](https://github.com/anoushkajain/UnitRefine/blob/main/src/unitrefine/README.md)** for all steps.
|
|
86
|
+
|
|
87
|
+
## Pretrained models
|
|
88
|
+
|
|
89
|
+
A new project downloads these three models from the Hugging Face Hub:
|
|
90
|
+
|
|
91
|
+
| Model | Labels |
|
|
92
|
+
|-------|--------|
|
|
93
|
+
| [noise vs neural](https://huggingface.co/SpikeInterface/UnitRefine_noise_neural_classifier) + [SUA vs MUA](https://huggingface.co/SpikeInterface/UnitRefine_sua_mua_classifier), used together as a cascade | good (SUA), MUA, noise |
|
|
94
|
+
| [generalized SUA classifier](https://huggingface.co/AnoushkaJain3/UnitRefine-generalized-sua-classifier) | SUA, not SUA |
|
|
95
|
+
|
|
96
|
+
UnitRefine was trained and tested on these datasets ([more models](https://huggingface.co/AnoushkaJain3)):
|
|
97
|
+
|
|
98
|
+
| Dataset | Species | Probe | Spike sorter |
|
|
99
|
+
|---------|---------|-------|--------------|
|
|
100
|
+
| UnitRefine base dataset | Mouse | Neuropixels 1.0 | Kilosort 2.5 |
|
|
101
|
+
| International Brain Laboratory | Mouse | Neuropixels 1.0 | IBL sorter (PyKilosort 2.5) |
|
|
102
|
+
| Allen Institute | Mouse | Neuropixels 2.0 | Kilosort 4 |
|
|
103
|
+
| [Shirdhankar et al., 2025](https://doi.org/10.64898/2025.12.15.693140) | Naked mole rat | Neuropixels 2.0 | Kilosort 4 |
|
|
104
|
+
| [Chen et al., 2022](https://www.nature.com/articles/s41597-022-01180-1) | Rhesus macaque | Utah array | Kilosort 4 |
|
|
105
|
+
| [Gerken et al., 2025](https://elifesciences.org/reviewed-preprints/106758) | Human | Behnke–Fried electrodes | Combinato |
|
|
106
|
+
The curated datasets are on [figshare](https://figshare.com/articles/dataset/Curated_dataset/28282799).
|
|
107
|
+
|
|
108
|
+
## Installation
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
conda create -n unitrefine python=3.13
|
|
112
|
+
conda activate unitrefine
|
|
113
|
+
pip install uv
|
|
114
|
+
uv pip install "unitrefine[gui]"
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Without the GUI, e.g. on a computing cluster: `uv pip install unitrefine`. To install from the repository, with
|
|
118
|
+
conda only, or if installing fails, see [Installation](https://github.com/anoushkajain/UnitRefine/blob/main/docs/installation.md).
|
|
119
|
+
|
|
120
|
+
## Documentation
|
|
121
|
+
|
|
122
|
+
- [GUI tutorial](https://github.com/anoushkajain/UnitRefine/blob/main/src/unitrefine/README.md)
|
|
123
|
+
- [Tutorial notebooks](https://github.com/anoushkajain/UnitRefine/blob/main/docs/tutorials.md): apply and train models in Python, use a `.csv` of metrics, SHAP plots
|
|
124
|
+
- [Applying models on a computing cluster](https://github.com/anoushkajain/UnitRefine/blob/main/docs/hpc.md)
|
|
125
|
+
- [Installation](https://github.com/anoushkajain/UnitRefine/blob/main/docs/installation.md)
|
|
126
|
+
- [Automated curation in SpikeInterface](https://spikeinterface.readthedocs.io/en/latest/tutorials_custom_index.html#automated-curation-tutorials)
|
|
127
|
+
|
|
128
|
+
## Citation
|
|
129
|
+
|
|
130
|
+
If you find UnitRefine useful in your research, please cite:
|
|
131
|
+
|
|
132
|
+
```bibtex
|
|
133
|
+
@article{jain2025unitrefine,
|
|
134
|
+
title={UnitRefine: A Community Toolbox for Automated Spike Sorting Curation},
|
|
135
|
+
author={Jain, Anoushka and Greene, Robyn and Halcrow, Chris and Swann, Jake A and Kleinjohann, Alexander and Spurio, Federico and Graff, Severin and Pan-Vazquez, Alejandro and Kampa, Bj{\"o}rn and Gall, Juergen and Gr{\"u}n, Sonja and Winter, Olivier and Buccino, Alessio and Hennig, Matthias H and Musall, Simon},
|
|
136
|
+
journal={bioRxiv},
|
|
137
|
+
year={2025},
|
|
138
|
+
doi={10.1101/2025.03.30.645770}
|
|
139
|
+
}
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Acknowledgements
|
|
143
|
+
|
|
144
|
+
UnitRefine builds on [SpikeInterface](https://github.com/SpikeInterface/spikeinterface) and
|
|
145
|
+
[SpikeInterface-GUI](https://github.com/SpikeInterface/spikeinterface-gui/). Many thanks to Alessio, Sam, Zack and Joe
|
|
146
|
+
for their help and feedback, and to the whole SpikeInterface team.
|
|
147
|
+
|
|
148
|
+
- **Code refactoring and integration in SpikeInterface:** Chris Halcrow, Jake Swann, Robyn Greene, Sangeetha Nandakumar (IBOTS)
|
|
149
|
+
- **Model curators:** Nilufar Lahiji, Sacha Abou Rachid, Severin Graff, Luca Koenig, Natalia Babushkina, Simon Musall
|
|
150
|
+
- **Advisors and collaborators:** Alessio Buccino, Olivier Winter, Sonja Grün, Matthias Hennig, Simon Musall
|
|
151
|
+
|
|
152
|
+
Feedback and contributions are welcome: please [open an issue](https://github.com/anoushkajain/UnitRefine/issues) or a
|
|
153
|
+
pull request. UnitRefine is released under the MIT license.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# UnitRefine: a community toolbox for automated spike sorting curation
|
|
2
|
+
|
|
3
|
+
[](https://github.com/anoushkajain/UnitRefine/actions/workflows/tests.yml)
|
|
4
|
+
[](https://github.com/anoushkajain/UnitRefine/blob/main/LICENSE)
|
|
5
|
+
|
|
6
|
+
**UnitRefine** labels spike sorted units as **single units (SUA)**, **multi-units (MUA)** or **noise** with machine
|
|
7
|
+
learning classifiers trained on human curation, so you don't have to curate every unit by hand. It works with any
|
|
8
|
+
probe, species, brain region and spike sorter, and is part of [SpikeInterface](https://github.com/SpikeInterface/spikeinterface).
|
|
9
|
+
|
|
10
|
+
<p align="center">
|
|
11
|
+
<img src="https://raw.githubusercontent.com/anoushkajain/UnitRefine/main/src/unitrefine/resources/unit_types_example.jpg" width="500"/>
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
With UnitRefine you can:
|
|
15
|
+
|
|
16
|
+
- label your units with [pretrained models](#pretrained-models)
|
|
17
|
+
- train your own model on your labels, and improve it with active learning, in the [GUI](#the-gui)
|
|
18
|
+
- apply models to many recordings on a [computing cluster](https://github.com/anoushkajain/UnitRefine/blob/main/docs/hpc.md)
|
|
19
|
+
- share models on the [Hugging Face Hub](https://huggingface.co/AnoushkaJain3)
|
|
20
|
+
|
|
21
|
+
## Preprint
|
|
22
|
+
|
|
23
|
+
UnitRefine and its validation on mouse, naked mole rat, monkey and human recordings are described in our
|
|
24
|
+
[preprint](https://www.biorxiv.org/content/10.1101/2025.03.30.645770v2).
|
|
25
|
+
|
|
26
|
+
## The GUI
|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<img src="https://raw.githubusercontent.com/anoushkajain/UnitRefine/main/docs/images/gui_window.png" width="700"/>
|
|
30
|
+
</p>
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
unitrefine --project_folder my_project
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
A new project comes with the pretrained models. Add your spike sorted data in section 3.3 (a Sorting Analyzer, or a
|
|
37
|
+
Kilosort output folder curated in Phy) and UnitRefine model labels
|
|
38
|
+
its units. To train your own model, label units in the
|
|
39
|
+
[SpikeInterface-GUI](https://github.com/SpikeInterface/spikeinterface-gui/) or import labels you already have.
|
|
40
|
+
|
|
41
|
+
See the **[GUI tutorial](https://github.com/anoushkajain/UnitRefine/blob/main/src/unitrefine/README.md)** for all steps.
|
|
42
|
+
|
|
43
|
+
## Pretrained models
|
|
44
|
+
|
|
45
|
+
A new project downloads these three models from the Hugging Face Hub:
|
|
46
|
+
|
|
47
|
+
| Model | Labels |
|
|
48
|
+
|-------|--------|
|
|
49
|
+
| [noise vs neural](https://huggingface.co/SpikeInterface/UnitRefine_noise_neural_classifier) + [SUA vs MUA](https://huggingface.co/SpikeInterface/UnitRefine_sua_mua_classifier), used together as a cascade | good (SUA), MUA, noise |
|
|
50
|
+
| [generalized SUA classifier](https://huggingface.co/AnoushkaJain3/UnitRefine-generalized-sua-classifier) | SUA, not SUA |
|
|
51
|
+
|
|
52
|
+
UnitRefine was trained and tested on these datasets ([more models](https://huggingface.co/AnoushkaJain3)):
|
|
53
|
+
|
|
54
|
+
| Dataset | Species | Probe | Spike sorter |
|
|
55
|
+
|---------|---------|-------|--------------|
|
|
56
|
+
| UnitRefine base dataset | Mouse | Neuropixels 1.0 | Kilosort 2.5 |
|
|
57
|
+
| International Brain Laboratory | Mouse | Neuropixels 1.0 | IBL sorter (PyKilosort 2.5) |
|
|
58
|
+
| Allen Institute | Mouse | Neuropixels 2.0 | Kilosort 4 |
|
|
59
|
+
| [Shirdhankar et al., 2025](https://doi.org/10.64898/2025.12.15.693140) | Naked mole rat | Neuropixels 2.0 | Kilosort 4 |
|
|
60
|
+
| [Chen et al., 2022](https://www.nature.com/articles/s41597-022-01180-1) | Rhesus macaque | Utah array | Kilosort 4 |
|
|
61
|
+
| [Gerken et al., 2025](https://elifesciences.org/reviewed-preprints/106758) | Human | Behnke–Fried electrodes | Combinato |
|
|
62
|
+
The curated datasets are on [figshare](https://figshare.com/articles/dataset/Curated_dataset/28282799).
|
|
63
|
+
|
|
64
|
+
## Installation
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
conda create -n unitrefine python=3.13
|
|
68
|
+
conda activate unitrefine
|
|
69
|
+
pip install uv
|
|
70
|
+
uv pip install "unitrefine[gui]"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Without the GUI, e.g. on a computing cluster: `uv pip install unitrefine`. To install from the repository, with
|
|
74
|
+
conda only, or if installing fails, see [Installation](https://github.com/anoushkajain/UnitRefine/blob/main/docs/installation.md).
|
|
75
|
+
|
|
76
|
+
## Documentation
|
|
77
|
+
|
|
78
|
+
- [GUI tutorial](https://github.com/anoushkajain/UnitRefine/blob/main/src/unitrefine/README.md)
|
|
79
|
+
- [Tutorial notebooks](https://github.com/anoushkajain/UnitRefine/blob/main/docs/tutorials.md): apply and train models in Python, use a `.csv` of metrics, SHAP plots
|
|
80
|
+
- [Applying models on a computing cluster](https://github.com/anoushkajain/UnitRefine/blob/main/docs/hpc.md)
|
|
81
|
+
- [Installation](https://github.com/anoushkajain/UnitRefine/blob/main/docs/installation.md)
|
|
82
|
+
- [Automated curation in SpikeInterface](https://spikeinterface.readthedocs.io/en/latest/tutorials_custom_index.html#automated-curation-tutorials)
|
|
83
|
+
|
|
84
|
+
## Citation
|
|
85
|
+
|
|
86
|
+
If you find UnitRefine useful in your research, please cite:
|
|
87
|
+
|
|
88
|
+
```bibtex
|
|
89
|
+
@article{jain2025unitrefine,
|
|
90
|
+
title={UnitRefine: A Community Toolbox for Automated Spike Sorting Curation},
|
|
91
|
+
author={Jain, Anoushka and Greene, Robyn and Halcrow, Chris and Swann, Jake A and Kleinjohann, Alexander and Spurio, Federico and Graff, Severin and Pan-Vazquez, Alejandro and Kampa, Bj{\"o}rn and Gall, Juergen and Gr{\"u}n, Sonja and Winter, Olivier and Buccino, Alessio and Hennig, Matthias H and Musall, Simon},
|
|
92
|
+
journal={bioRxiv},
|
|
93
|
+
year={2025},
|
|
94
|
+
doi={10.1101/2025.03.30.645770}
|
|
95
|
+
}
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## Acknowledgements
|
|
99
|
+
|
|
100
|
+
UnitRefine builds on [SpikeInterface](https://github.com/SpikeInterface/spikeinterface) and
|
|
101
|
+
[SpikeInterface-GUI](https://github.com/SpikeInterface/spikeinterface-gui/). Many thanks to Alessio, Sam, Zack and Joe
|
|
102
|
+
for their help and feedback, and to the whole SpikeInterface team.
|
|
103
|
+
|
|
104
|
+
- **Code refactoring and integration in SpikeInterface:** Chris Halcrow, Jake Swann, Robyn Greene, Sangeetha Nandakumar (IBOTS)
|
|
105
|
+
- **Model curators:** Nilufar Lahiji, Sacha Abou Rachid, Severin Graff, Luca Koenig, Natalia Babushkina, Simon Musall
|
|
106
|
+
- **Advisors and collaborators:** Alessio Buccino, Olivier Winter, Sonja Grün, Matthias Hennig, Simon Musall
|
|
107
|
+
|
|
108
|
+
Feedback and contributions are welcome: please [open an issue](https://github.com/anoushkajain/UnitRefine/issues) or a
|
|
109
|
+
pull request. UnitRefine is released under the MIT license.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "unitrefine"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "UnitRefine is a machine-learning toolbox that simplifies spike sorting curation by reducing manual curation."
|
|
5
|
+
authors = [
|
|
6
|
+
{ name = "Anoushka Jain", email = "jain.anoushka3@gmail.com" },
|
|
7
|
+
{ name = "Chris Halcrow", email = "chrishalcrow@gmail.com" },
|
|
8
|
+
]
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11,<3.14"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
keywords = ["spike sorting", "curation", "electrophysiology", "machine learning", "neuropixels", "spikeinterface"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
|
23
|
+
]
|
|
24
|
+
# what applying models needs, also without a screen, e.g. on a computing cluster (unitrefine-apply, see docs/hpc.md)
|
|
25
|
+
dependencies = [
|
|
26
|
+
"numcodecs==0.15.1",
|
|
27
|
+
"numba>=0.61.0",
|
|
28
|
+
"pandas>=2.2.3",
|
|
29
|
+
"spikeinterface>=0.105.0",
|
|
30
|
+
"pyyaml>=6.0.2",
|
|
31
|
+
"scikit-learn==1.7",
|
|
32
|
+
"skops>=0.11.0",
|
|
33
|
+
"huggingface-hub>=0.35.0",
|
|
34
|
+
"s3fs>=2025.10.0",
|
|
35
|
+
# reading compressed IBL recordings (.cbin), e.g. for + Phy output
|
|
36
|
+
"mtscomp>=1.0.2",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
# the GUI: pip install "unitrefine[gui]". `uv sync` installs it too (see default-groups below)
|
|
41
|
+
gui = [
|
|
42
|
+
"spikeinterface-gui>=0.14.0",
|
|
43
|
+
"pyqtgraph>=0.13.7",
|
|
44
|
+
"pyqt5>=5.15.11",
|
|
45
|
+
"pyqt5-qt5<5.15.18; sys_platform == 'win32' and platform_machine == 'AMD64'",
|
|
46
|
+
"pyqt5-qt5>=5.15.0; sys_platform == 'darwin'",
|
|
47
|
+
"pyqt5-qt5>=5.15.0; sys_platform == 'linux'",
|
|
48
|
+
"matplotlib>=3.8",
|
|
49
|
+
"shap>=0.49.1",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[project.urls]
|
|
53
|
+
Homepage = "https://github.com/anoushkajain/UnitRefine"
|
|
54
|
+
Documentation = "https://github.com/anoushkajain/UnitRefine#readme"
|
|
55
|
+
Issues = "https://github.com/anoushkajain/UnitRefine/issues"
|
|
56
|
+
Preprint = "https://www.biorxiv.org/content/10.1101/2025.03.30.645770v2"
|
|
57
|
+
Models = "https://huggingface.co/AnoushkaJain3"
|
|
58
|
+
|
|
59
|
+
[build-system]
|
|
60
|
+
# 77: the license as `license = "MIT"`
|
|
61
|
+
requires = ["setuptools>=77"]
|
|
62
|
+
build-backend = "setuptools.build_meta"
|
|
63
|
+
|
|
64
|
+
[tool.setuptools]
|
|
65
|
+
# resources holds the images and fonts used by the GUI
|
|
66
|
+
packages = ["unitrefine", "unitrefine.resources", "unitrefine.resources.fonts"]
|
|
67
|
+
package-dir = {"unitrefine" = "src/unitrefine"}
|
|
68
|
+
|
|
69
|
+
[tool.setuptools.package-data]
|
|
70
|
+
"unitrefine.resources" = ["*"]
|
|
71
|
+
"unitrefine.resources.fonts" = ["*"]
|
|
72
|
+
|
|
73
|
+
[project.scripts]
|
|
74
|
+
unitrefine = "unitrefine.launcher:main"
|
|
75
|
+
# without the GUI, e.g. on a computing cluster (see src/unitrefine/cli.py)
|
|
76
|
+
unitrefine-apply = "unitrefine.cli:apply_main"
|
|
77
|
+
unitrefine-download-models = "unitrefine.cli:download_main"
|
|
78
|
+
|
|
79
|
+
[dependency-groups]
|
|
80
|
+
# for running the tests: uv run pytest
|
|
81
|
+
dev = ["pytest>=8"]
|
|
82
|
+
# so `uv sync` installs the GUI, as it always has. Without it: uv sync --no-group gui
|
|
83
|
+
gui = ["unitrefine[gui]"]
|
|
84
|
+
|
|
85
|
+
[tool.uv]
|
|
86
|
+
default-groups = ["dev", "gui"]
|
|
87
|
+
|
|
88
|
+
[tool.pytest.ini_options]
|
|
89
|
+
testpaths = ["tests"]
|
|
90
|
+
filterwarnings = [
|
|
91
|
+
# from the small synthetic recordings the tests make
|
|
92
|
+
"ignore::UserWarning:spikeinterface.*",
|
|
93
|
+
"ignore::DeprecationWarning",
|
|
94
|
+
]
|
|
95
|
+
|
|
96
|
+
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
# the version in pyproject.toml, e.g. to say which version a problem was found with
|
|
5
|
+
__version__ = version("unitrefine")
|
|
6
|
+
except PackageNotFoundError:
|
|
7
|
+
# run from the source folder without installing it
|
|
8
|
+
__version__ = "unknown"
|
|
9
|
+
|
|
10
|
+
# The SpikeInterface-GUI scripts run in their own process. They print these markers so the main
|
|
11
|
+
# window can show which step they are on while they open, and when their window is shown.
|
|
12
|
+
SIGUI_SHOWN_MARKER = "UNITREFINE_SIGUI_WINDOW_SHOWN"
|
|
13
|
+
SIGUI_STEP_MARKER = "UNITREFINE_SIGUI_STEP:"
|
|
14
|
+
# printed when labels could not be saved where they belong and were saved to a backup file instead
|
|
15
|
+
LABELS_BACKUP_MARKER = "UNITREFINE_LABELS_BACKUP:"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# The labels are passed to the SpikeInterface-GUI in curation format 2, which spikeinterface-gui handles
|
|
19
|
+
# correctly (also for units shown without a label) from this version on
|
|
20
|
+
MINIMUM_SIGUI_VERSION = "0.14.0"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def check_sigui_version():
|
|
24
|
+
"""
|
|
25
|
+
Stops the SpikeInterface-GUI script with a clear message if spikeinterface-gui is too old: older versions
|
|
26
|
+
would not save the labels set with the n/g/m keys correctly.
|
|
27
|
+
"""
|
|
28
|
+
import sys
|
|
29
|
+
from packaging.version import parse
|
|
30
|
+
import spikeinterface_gui
|
|
31
|
+
|
|
32
|
+
installed = getattr(spikeinterface_gui, "__version__", "0")
|
|
33
|
+
if parse(installed) < parse(MINIMUM_SIGUI_VERSION):
|
|
34
|
+
print(f"UnitRefine needs spikeinterface-gui {MINIMUM_SIGUI_VERSION} or newer, but {installed} is installed, "
|
|
35
|
+
f"which would not save your labels correctly. Please upgrade it, e.g. with "
|
|
36
|
+
f"'pip install --upgrade spikeinterface-gui' or 'uv sync'.", flush=True)
|
|
37
|
+
sys.exit(1)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def unwritable_file_message(path, error):
|
|
41
|
+
"""What to tell the user when a file can't be written, e.g. because it is read-only."""
|
|
42
|
+
advice = ("The file is read-only or open in another program. In File Explorer, right-click it, choose "
|
|
43
|
+
"Properties and untick 'Read-only' (or close the other program), then try again.")
|
|
44
|
+
return f"UnitRefine could not save {path}:\n{error}\n\n{advice}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def save_labels_or_backup(labels, path, write=None, **to_csv_kwargs):
|
|
48
|
+
"""
|
|
49
|
+
Saves the labels set in the SpikeInterface-GUI to `path` (with `write(labels, path)` if given). If that
|
|
50
|
+
fails, e.g. because the file is read-only, the labels are saved to a backup file in the home folder, so
|
|
51
|
+
they are not lost, and the main window is told where (LABELS_BACKUP_MARKER). Returns where they were saved.
|
|
52
|
+
"""
|
|
53
|
+
import time
|
|
54
|
+
from pathlib import Path
|
|
55
|
+
|
|
56
|
+
path = Path(path)
|
|
57
|
+
try:
|
|
58
|
+
if write is not None:
|
|
59
|
+
write(labels, path)
|
|
60
|
+
else:
|
|
61
|
+
labels.to_csv(path, **to_csv_kwargs)
|
|
62
|
+
return path
|
|
63
|
+
except OSError as error:
|
|
64
|
+
backup = Path.home() / f"UnitRefine_unsaved_{time.strftime('%Y%m%d_%H%M%S')}_{path.name}"
|
|
65
|
+
labels.to_csv(backup, **to_csv_kwargs)
|
|
66
|
+
print(unwritable_file_message(path, error), flush=True)
|
|
67
|
+
print(f"Your labels were saved to {backup} instead. Once {path.name} can be written, copy that file over it.",
|
|
68
|
+
flush=True)
|
|
69
|
+
print(f"{LABELS_BACKUP_MARKER}{backup}|{path}", flush=True)
|
|
70
|
+
return backup
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def report_sigui_step(step_name):
|
|
74
|
+
"""Called by the SpikeInterface-GUI scripts when they start a step, e.g. 'loading'."""
|
|
75
|
+
print(f"{SIGUI_STEP_MARKER}{step_name}", flush=True)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import PyQt5.QtWidgets as QtWidgets
|
|
4
|
+
from spikeinterface_gui.main import check_folder_is_analyzer
|
|
5
|
+
|
|
6
|
+
from unitrefine.predictions import find_analyzers
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ApplyModelsDialog(QtWidgets.QDialog):
|
|
10
|
+
"""Pick the analyzers to apply the selected models to, and the folder to save the predictions in."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, selection_text, default_output_folder, parent=None):
|
|
13
|
+
|
|
14
|
+
super().__init__(parent)
|
|
15
|
+
|
|
16
|
+
self.setWindowTitle("UNITREFINE: Apply models to analyzers")
|
|
17
|
+
self.setMinimumWidth(650)
|
|
18
|
+
|
|
19
|
+
layout = QtWidgets.QVBoxLayout(self)
|
|
20
|
+
|
|
21
|
+
selection_label = QtWidgets.QLabel(f"Models: {selection_text}")
|
|
22
|
+
selection_label.setWordWrap(True)
|
|
23
|
+
layout.addWidget(selection_label)
|
|
24
|
+
|
|
25
|
+
analyzers_label = QtWidgets.QLabel("Analyzers the models will be applied to:")
|
|
26
|
+
analyzers_label.setStyleSheet("font-weight: bold;")
|
|
27
|
+
layout.addWidget(analyzers_label)
|
|
28
|
+
|
|
29
|
+
self.analyzer_list = QtWidgets.QListWidget()
|
|
30
|
+
self.analyzer_list.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
|
|
31
|
+
layout.addWidget(self.analyzer_list)
|
|
32
|
+
|
|
33
|
+
buttons_layout = QtWidgets.QHBoxLayout()
|
|
34
|
+
add_button = QtWidgets.QPushButton("+ Add analyzer folder")
|
|
35
|
+
add_button.clicked.connect(self.add_analyzer_folder)
|
|
36
|
+
add_all_button = QtWidgets.QPushButton("+ Add all analyzers inside a folder")
|
|
37
|
+
add_all_button.setToolTip("Searches up to 3 folder levels down for Sorting Analyzer folders")
|
|
38
|
+
add_all_button.clicked.connect(self.add_analyzers_inside_folder)
|
|
39
|
+
remove_button = QtWidgets.QPushButton("Remove selected")
|
|
40
|
+
remove_button.clicked.connect(self.remove_selected)
|
|
41
|
+
buttons_layout.addWidget(add_button)
|
|
42
|
+
buttons_layout.addWidget(add_all_button)
|
|
43
|
+
buttons_layout.addWidget(remove_button)
|
|
44
|
+
layout.addLayout(buttons_layout)
|
|
45
|
+
|
|
46
|
+
output_label = QtWidgets.QLabel("Save the predictions (one csv per analyzer) in:")
|
|
47
|
+
output_label.setStyleSheet("font-weight: bold;")
|
|
48
|
+
layout.addWidget(output_label)
|
|
49
|
+
|
|
50
|
+
output_layout = QtWidgets.QHBoxLayout()
|
|
51
|
+
self.output_folder_form = QtWidgets.QLineEdit(str(default_output_folder))
|
|
52
|
+
browse_button = QtWidgets.QPushButton("Browse")
|
|
53
|
+
browse_button.clicked.connect(self.choose_output_folder)
|
|
54
|
+
output_layout.addWidget(self.output_folder_form)
|
|
55
|
+
output_layout.addWidget(browse_button)
|
|
56
|
+
layout.addLayout(output_layout)
|
|
57
|
+
|
|
58
|
+
button_box = QtWidgets.QDialogButtonBox()
|
|
59
|
+
self.apply_button = button_box.addButton("Apply", QtWidgets.QDialogButtonBox.AcceptRole)
|
|
60
|
+
button_box.addButton(QtWidgets.QDialogButtonBox.StandardButton.Cancel)
|
|
61
|
+
button_box.accepted.connect(self.accept_if_ready)
|
|
62
|
+
button_box.rejected.connect(self.reject)
|
|
63
|
+
layout.addWidget(button_box)
|
|
64
|
+
|
|
65
|
+
def add_paths(self, paths):
|
|
66
|
+
existing = set(self.analyzer_paths())
|
|
67
|
+
for path in paths:
|
|
68
|
+
if str(path) not in existing:
|
|
69
|
+
self.analyzer_list.addItem(str(path))
|
|
70
|
+
existing.add(str(path))
|
|
71
|
+
|
|
72
|
+
def add_analyzer_folder(self):
|
|
73
|
+
folder = QtWidgets.QFileDialog.getExistingDirectory(self, "Select a Sorting Analyzer folder")
|
|
74
|
+
if folder == "":
|
|
75
|
+
return
|
|
76
|
+
if check_folder_is_analyzer(folder):
|
|
77
|
+
self.add_paths([Path(folder)])
|
|
78
|
+
else:
|
|
79
|
+
print(f"Selected directory {folder} is not a SortingAnalyzer.")
|
|
80
|
+
|
|
81
|
+
def add_analyzers_inside_folder(self):
|
|
82
|
+
folder = QtWidgets.QFileDialog.getExistingDirectory(self, "Select a folder containing Sorting Analyzers")
|
|
83
|
+
if folder == "":
|
|
84
|
+
return
|
|
85
|
+
analyzers = find_analyzers(folder)
|
|
86
|
+
print(f"Found {len(analyzers)} Sorting Analyzers in {folder}.")
|
|
87
|
+
self.add_paths(analyzers)
|
|
88
|
+
|
|
89
|
+
def remove_selected(self):
|
|
90
|
+
for item in self.analyzer_list.selectedItems():
|
|
91
|
+
self.analyzer_list.takeItem(self.analyzer_list.row(item))
|
|
92
|
+
|
|
93
|
+
def choose_output_folder(self):
|
|
94
|
+
folder = QtWidgets.QFileDialog.getExistingDirectory(self, "Select the folder to save the predictions in")
|
|
95
|
+
if folder != "":
|
|
96
|
+
self.output_folder_form.setText(folder)
|
|
97
|
+
|
|
98
|
+
def analyzer_paths(self):
|
|
99
|
+
return [self.analyzer_list.item(row).text() for row in range(self.analyzer_list.count())]
|
|
100
|
+
|
|
101
|
+
def output_folder(self):
|
|
102
|
+
return self.output_folder_form.text().strip()
|
|
103
|
+
|
|
104
|
+
def accept_if_ready(self):
|
|
105
|
+
if len(self.analyzer_paths()) == 0:
|
|
106
|
+
QtWidgets.QMessageBox.warning(self, "No analyzers", "Add at least one analyzer to apply the models to.")
|
|
107
|
+
return
|
|
108
|
+
if self.output_folder() == "":
|
|
109
|
+
QtWidgets.QMessageBox.warning(self, "No output folder", "Choose a folder to save the predictions in.")
|
|
110
|
+
return
|
|
111
|
+
self.accept()
|