DARTsort 0.3.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dartsort-0.3.5/.github/workflows/ci.yml +52 -0
- dartsort-0.3.5/.github/workflows/ci_windows.yml +43 -0
- dartsort-0.3.5/.github/workflows/deploy.yml +56 -0
- dartsort-0.3.5/.gitignore +170 -0
- dartsort-0.3.5/PKG-INFO +93 -0
- dartsort-0.3.5/README.md +50 -0
- dartsort-0.3.5/environment.yml +18 -0
- dartsort-0.3.5/notebook/.keep +0 -0
- dartsort-0.3.5/notebook/basic_integration_test.py +137 -0
- dartsort-0.3.5/notebook/ci diag.py +573 -0
- dartsort-0.3.5/notebook/dartsort_dredge_demo.ipynb +437 -0
- dartsort-0.3.5/notebook/dartsort_dredge_demo.py +144 -0
- dartsort-0.3.5/notebook/data test.py +259 -0
- dartsort-0.3.5/notebook/dead reckoning.py +361 -0
- dartsort-0.3.5/notebook/denoise np1.py +356 -0
- dartsort-0.3.5/notebook/denoise np2.py +480 -0
- dartsort-0.3.5/notebook/exploratory.py +431 -0
- dartsort-0.3.5/notebook/geom debug.py +167 -0
- dartsort-0.3.5/notebook/hdbscan_clustering_5min.py +1294 -0
- dartsort-0.3.5/notebook/hybrid 10.py +1662 -0
- dartsort-0.3.5/notebook/hybrid cluster.py +475 -0
- dartsort-0.3.5/notebook/hybrid_11.py +1214 -0
- dartsort-0.3.5/notebook/initial training.py +162 -0
- dartsort-0.3.5/notebook/make hybrid.py +1376 -0
- dartsort-0.3.5/notebook/mgh.py +514 -0
- dartsort-0.3.5/notebook/np1 feats.py +1211 -0
- dartsort-0.3.5/notebook/np2 processing version c.py +321 -0
- dartsort-0.3.5/notebook/paperfigs.py +907 -0
- dartsort-0.3.5/notebook/pdists.py +641 -0
- dartsort-0.3.5/notebook/postcheck.py +803 -0
- dartsort-0.3.5/notebook/ptp vae.py +206 -0
- dartsort-0.3.5/notebook/re clust.py +893 -0
- dartsort-0.3.5/notebook/reg fig loop.py +723 -0
- dartsort-0.3.5/notebook/reg_sim_exp.py +493 -0
- dartsort-0.3.5/notebook/relocating.py +284 -0
- dartsort-0.3.5/notebook/relocating_all.py +182 -0
- dartsort-0.3.5/notebook/restats +0 -0
- dartsort-0.3.5/notebook/sortStinger1.py +1590 -0
- dartsort-0.3.5/notebook/sortStinger3.py +1541 -0
- dartsort-0.3.5/notebook/sortStinger4.py +1835 -0
- dartsort-0.3.5/notebook/sortStinger5.py +1586 -0
- dartsort-0.3.5/notebook/sortStinger6.py +1915 -0
- dartsort-0.3.5/notebook/spike farm.py +100 -0
- dartsort-0.3.5/notebook/ss_ensemble_sim.py +1965 -0
- dartsort-0.3.5/notebook/ss_ensemble_sim_drift.py +1928 -0
- dartsort-0.3.5/notebook/ss_sim.py +1965 -0
- dartsort-0.3.5/notebook/stan diagnostics.py +207 -0
- dartsort-0.3.5/notebook/storytelling c.py +686 -0
- dartsort-0.3.5/notebook/storytelling.py +684 -0
- dartsort-0.3.5/notebook/subtract_with_phaseshift_denoiser.py +260 -0
- dartsort-0.3.5/notebook/tests/debug_matching.py +454 -0
- dartsort-0.3.5/notebook/tests/test_agreement.py +367 -0
- dartsort-0.3.5/notebook/tests/test_amplitude_scaling.py +737 -0
- dartsort-0.3.5/notebook/tests/test_batch_optim.py +679 -0
- dartsort-0.3.5/notebook/tests/test_ibme.py +86 -0
- dartsort-0.3.5/notebook/tests/test_mm_initialization.py +142 -0
- dartsort-0.3.5/notebook/tests/test_normxcorr1d.py +63 -0
- dartsort-0.3.5/notebook/tests/test_reassignment.py +422 -0
- dartsort-0.3.5/notebook/tests/test_relocation.py +230 -0
- dartsort-0.3.5/notebook/thomas.py +1004 -0
- dartsort-0.3.5/pyproject.toml +73 -0
- dartsort-0.3.5/setup.cfg +4 -0
- dartsort-0.3.5/src/DARTsort.egg-info/PKG-INFO +93 -0
- dartsort-0.3.5/src/DARTsort.egg-info/SOURCES.txt +213 -0
- dartsort-0.3.5/src/DARTsort.egg-info/dependency_links.txt +1 -0
- dartsort-0.3.5/src/DARTsort.egg-info/entry_points.txt +2 -0
- dartsort-0.3.5/src/DARTsort.egg-info/requires.txt +37 -0
- dartsort-0.3.5/src/DARTsort.egg-info/top_level.txt +1 -0
- dartsort-0.3.5/src/dartsort/__init__.py +68 -0
- dartsort-0.3.5/src/dartsort/cli.py +105 -0
- dartsort-0.3.5/src/dartsort/clustering/__init__.py +3 -0
- dartsort-0.3.5/src/dartsort/clustering/cluster_util.py +392 -0
- dartsort-0.3.5/src/dartsort/clustering/clustering.py +765 -0
- dartsort-0.3.5/src/dartsort/clustering/clustering_features.py +202 -0
- dartsort-0.3.5/src/dartsort/clustering/density.py +1299 -0
- dartsort-0.3.5/src/dartsort/clustering/forward_backward.py +219 -0
- dartsort-0.3.5/src/dartsort/clustering/gmm/mixture.py +5790 -0
- dartsort-0.3.5/src/dartsort/clustering/gmm/stable_features.py +993 -0
- dartsort-0.3.5/src/dartsort/clustering/kmeans.py +448 -0
- dartsort-0.3.5/src/dartsort/clustering/merge.py +501 -0
- dartsort-0.3.5/src/dartsort/clustering/modes.py +214 -0
- dartsort-0.3.5/src/dartsort/clustering/refine_util.py +191 -0
- dartsort-0.3.5/src/dartsort/clustering/relocate.py +109 -0
- dartsort-0.3.5/src/dartsort/clustering/split.py +1285 -0
- dartsort-0.3.5/src/dartsort/config.py +288 -0
- dartsort-0.3.5/src/dartsort/detect/__init__.py +2 -0
- dartsort-0.3.5/src/dartsort/detect/detect.py +449 -0
- dartsort-0.3.5/src/dartsort/detect/detect_filters.py +37 -0
- dartsort-0.3.5/src/dartsort/evaluate/__init__.py +1 -0
- dartsort-0.3.5/src/dartsort/evaluate/analysis.py +594 -0
- dartsort-0.3.5/src/dartsort/evaluate/comparison.py +534 -0
- dartsort-0.3.5/src/dartsort/evaluate/hybrid_util.py +599 -0
- dartsort-0.3.5/src/dartsort/evaluate/noise_recording_tools.py +184 -0
- dartsort-0.3.5/src/dartsort/evaluate/sim_template_tools.py +842 -0
- dartsort-0.3.5/src/dartsort/evaluate/simkit.py +840 -0
- dartsort-0.3.5/src/dartsort/evaluate/simlib.py +270 -0
- dartsort-0.3.5/src/dartsort/localize/__init__.py +0 -0
- dartsort-0.3.5/src/dartsort/localize/localize_torch.py +291 -0
- dartsort-0.3.5/src/dartsort/localize/localize_util.py +313 -0
- dartsort-0.3.5/src/dartsort/main.py +657 -0
- dartsort-0.3.5/src/dartsort/peel/__init__.py +6 -0
- dartsort-0.3.5/src/dartsort/peel/grab.py +208 -0
- dartsort-0.3.5/src/dartsort/peel/matching.py +628 -0
- dartsort-0.3.5/src/dartsort/peel/matching_util/__init__.py +3 -0
- dartsort-0.3.5/src/dartsort/peel/matching_util/compressed_upsampled.py +718 -0
- dartsort-0.3.5/src/dartsort/peel/matching_util/drifty.py +791 -0
- dartsort-0.3.5/src/dartsort/peel/matching_util/matching_base.py +616 -0
- dartsort-0.3.5/src/dartsort/peel/matching_util/pairwise.py +359 -0
- dartsort-0.3.5/src/dartsort/peel/matching_util/pairwise_util.py +1317 -0
- dartsort-0.3.5/src/dartsort/peel/peel_base.py +1000 -0
- dartsort-0.3.5/src/dartsort/peel/running_template.py +1498 -0
- dartsort-0.3.5/src/dartsort/peel/subtract.py +1123 -0
- dartsort-0.3.5/src/dartsort/peel/threshold.py +471 -0
- dartsort-0.3.5/src/dartsort/peel/universal.py +191 -0
- dartsort-0.3.5/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/detect_np1.pt +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/detect_np2.pt +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/glasbey1024.npz +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
- dartsort-0.3.5/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
- dartsort-0.3.5/src/dartsort/templates/__init__.py +11 -0
- dartsort-0.3.5/src/dartsort/templates/get_templates.py +771 -0
- dartsort-0.3.5/src/dartsort/templates/postprocess_util.py +555 -0
- dartsort-0.3.5/src/dartsort/templates/realignment.py +466 -0
- dartsort-0.3.5/src/dartsort/templates/superres_util.py +204 -0
- dartsort-0.3.5/src/dartsort/templates/template_util.py +518 -0
- dartsort-0.3.5/src/dartsort/templates/templates.py +528 -0
- dartsort-0.3.5/src/dartsort/transform/__init__.py +2 -0
- dartsort-0.3.5/src/dartsort/transform/_multichan_denoiser_kit.py +650 -0
- dartsort-0.3.5/src/dartsort/transform/all_transformers.py +58 -0
- dartsort-0.3.5/src/dartsort/transform/amortized_localization.py +503 -0
- dartsort-0.3.5/src/dartsort/transform/amplitudes.py +197 -0
- dartsort-0.3.5/src/dartsort/transform/decollider.py +635 -0
- dartsort-0.3.5/src/dartsort/transform/enforce_decrease.py +158 -0
- dartsort-0.3.5/src/dartsort/transform/localize.py +86 -0
- dartsort-0.3.5/src/dartsort/transform/matching_denoiser.py +44 -0
- dartsort-0.3.5/src/dartsort/transform/pipeline.py +391 -0
- dartsort-0.3.5/src/dartsort/transform/single_channel_denoiser.py +153 -0
- dartsort-0.3.5/src/dartsort/transform/single_channel_templates.py +98 -0
- dartsort-0.3.5/src/dartsort/transform/supervised_denoiser.py +486 -0
- dartsort-0.3.5/src/dartsort/transform/temporal_pca.py +413 -0
- dartsort-0.3.5/src/dartsort/transform/transform_base.py +255 -0
- dartsort-0.3.5/src/dartsort/util/__init__.py +0 -0
- dartsort-0.3.5/src/dartsort/util/cli_util.py +179 -0
- dartsort-0.3.5/src/dartsort/util/data_util.py +1128 -0
- dartsort-0.3.5/src/dartsort/util/drift_util.py +1054 -0
- dartsort-0.3.5/src/dartsort/util/internal_config.py +1074 -0
- dartsort-0.3.5/src/dartsort/util/interpolation_util.py +1012 -0
- dartsort-0.3.5/src/dartsort/util/job_util.py +19 -0
- dartsort-0.3.5/src/dartsort/util/logging_util.py +91 -0
- dartsort-0.3.5/src/dartsort/util/main_util.py +235 -0
- dartsort-0.3.5/src/dartsort/util/more_operators.py +269 -0
- dartsort-0.3.5/src/dartsort/util/multiprocessing_util.py +211 -0
- dartsort-0.3.5/src/dartsort/util/nn_util.py +461 -0
- dartsort-0.3.5/src/dartsort/util/noise_util.py +1461 -0
- dartsort-0.3.5/src/dartsort/util/peel_util.py +223 -0
- dartsort-0.3.5/src/dartsort/util/py_util.py +213 -0
- dartsort-0.3.5/src/dartsort/util/registration_util.py +128 -0
- dartsort-0.3.5/src/dartsort/util/residual_util.py +30 -0
- dartsort-0.3.5/src/dartsort/util/sparse_util.py +81 -0
- dartsort-0.3.5/src/dartsort/util/spikeio.py +227 -0
- dartsort-0.3.5/src/dartsort/util/spiketorch.py +1198 -0
- dartsort-0.3.5/src/dartsort/util/testing_util/__init__.py +0 -0
- dartsort-0.3.5/src/dartsort/util/testing_util/matching_debug_util.py +517 -0
- dartsort-0.3.5/src/dartsort/util/testing_util/mixture_testing_util.py +291 -0
- dartsort-0.3.5/src/dartsort/util/torch_optimization_util.py +156 -0
- dartsort-0.3.5/src/dartsort/util/torch_util.py +75 -0
- dartsort-0.3.5/src/dartsort/util/universal_util.py +269 -0
- dartsort-0.3.5/src/dartsort/util/waveform_util.py +809 -0
- dartsort-0.3.5/src/dartsort/vis/__init__.py +19 -0
- dartsort-0.3.5/src/dartsort/vis/analysis_plots.py +392 -0
- dartsort-0.3.5/src/dartsort/vis/colors.py +36 -0
- dartsort-0.3.5/src/dartsort/vis/gt.py +392 -0
- dartsort-0.3.5/src/dartsort/vis/layout.py +149 -0
- dartsort-0.3.5/src/dartsort/vis/metrics_util.py +23 -0
- dartsort-0.3.5/src/dartsort/vis/mixture.py +1489 -0
- dartsort-0.3.5/src/dartsort/vis/over_time.py +514 -0
- dartsort-0.3.5/src/dartsort/vis/recanim.py +429 -0
- dartsort-0.3.5/src/dartsort/vis/scatterplots.py +779 -0
- dartsort-0.3.5/src/dartsort/vis/sorting.py +163 -0
- dartsort-0.3.5/src/dartsort/vis/unit.py +917 -0
- dartsort-0.3.5/src/dartsort/vis/unit_comparison.py +1017 -0
- dartsort-0.3.5/src/dartsort/vis/versus.py +300 -0
- dartsort-0.3.5/src/dartsort/vis/vismain.py +537 -0
- dartsort-0.3.5/src/dartsort/vis/waveforms.py +357 -0
- dartsort-0.3.5/tests/conftest.py +90 -0
- dartsort-0.3.5/tests/test_alignment.py +511 -0
- dartsort-0.3.5/tests/test_clustering.py +249 -0
- dartsort-0.3.5/tests/test_config.py +24 -0
- dartsort-0.3.5/tests/test_dartsort.py +158 -0
- dartsort-0.3.5/tests/test_data_util.py +71 -0
- dartsort-0.3.5/tests/test_detect.py +288 -0
- dartsort-0.3.5/tests/test_drift_util.py +305 -0
- dartsort-0.3.5/tests/test_drifty_matching.py +192 -0
- dartsort-0.3.5/tests/test_enforce_decrease.py +16 -0
- dartsort-0.3.5/tests/test_grab_and_featurize.py +141 -0
- dartsort-0.3.5/tests/test_kmeans.py +39 -0
- dartsort-0.3.5/tests/test_linalg.py +83 -0
- dartsort-0.3.5/tests/test_localize_torch.py +94 -0
- dartsort-0.3.5/tests/test_matching.py +824 -0
- dartsort-0.3.5/tests/test_modeling.py +464 -0
- dartsort-0.3.5/tests/test_relocate.py +147 -0
- dartsort-0.3.5/tests/test_simkit.py +235 -0
- dartsort-0.3.5/tests/test_spiketorch.py +203 -0
- dartsort-0.3.5/tests/test_subtract.py +447 -0
- dartsort-0.3.5/tests/test_templates.py +504 -0
- dartsort-0.3.5/tests/test_threshold.py +9 -0
- dartsort-0.3.5/tests/test_torch_optimization_util.py +41 -0
- dartsort-0.3.5/tests/test_transform.py +160 -0
- dartsort-0.3.5/tests/test_universal.py +9 -0
- dartsort-0.3.5/tests/test_util.py +91 -0
- dartsort-0.3.5/tests/test_waveform_util.py +171 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
- develop
|
|
8
|
+
pull_request:
|
|
9
|
+
branches:
|
|
10
|
+
- "*"
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
|
|
15
|
+
build:
|
|
16
|
+
runs-on: "ubuntu-latest"
|
|
17
|
+
defaults:
|
|
18
|
+
run:
|
|
19
|
+
shell: bash -l {0}
|
|
20
|
+
|
|
21
|
+
steps:
|
|
22
|
+
- name: Checkout
|
|
23
|
+
uses: actions/checkout@v4
|
|
24
|
+
with:
|
|
25
|
+
path: dartsort
|
|
26
|
+
|
|
27
|
+
- name: Get DREDge
|
|
28
|
+
uses: actions/checkout@v4
|
|
29
|
+
with:
|
|
30
|
+
repository: evarol/dredge
|
|
31
|
+
path: dredge
|
|
32
|
+
|
|
33
|
+
- uses: actions/setup-python@v5
|
|
34
|
+
with:
|
|
35
|
+
python-version: '3.x'
|
|
36
|
+
cache: 'pip' # caching pip dependencies
|
|
37
|
+
|
|
38
|
+
- name: Install dependencies
|
|
39
|
+
run: |
|
|
40
|
+
python -m pip install -U pip # seems necessary to make sure pip is my pip
|
|
41
|
+
pip install --extra-index-url https://download.pytorch.org/whl/cpu torch
|
|
42
|
+
pip install dredge/
|
|
43
|
+
pip install -e dartsort/[full]
|
|
44
|
+
|
|
45
|
+
- name: pytest
|
|
46
|
+
run: |
|
|
47
|
+
pytest --cache-clear --log-level=15 --durations=100 --cov=dartsort/src/dartsort --cov-config=dartsort/pyproject.toml --cov-report lcov --cov-report term dartsort/tests/
|
|
48
|
+
|
|
49
|
+
- name: Coveralls
|
|
50
|
+
uses: coverallsapp/github-action@master
|
|
51
|
+
with:
|
|
52
|
+
path-to-lcov: coverage.lcov
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: CI (Windows)
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
|
|
11
|
+
build:
|
|
12
|
+
runs-on: "windows-latest"
|
|
13
|
+
defaults:
|
|
14
|
+
run:
|
|
15
|
+
shell: bash -el {0}
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- name: Checkout
|
|
19
|
+
uses: actions/checkout@v4
|
|
20
|
+
with:
|
|
21
|
+
path: dartsort
|
|
22
|
+
|
|
23
|
+
- name: Get DREDge
|
|
24
|
+
uses: actions/checkout@v4
|
|
25
|
+
with:
|
|
26
|
+
repository: evarol/dredge
|
|
27
|
+
path: dredge
|
|
28
|
+
|
|
29
|
+
- uses: actions/setup-python@v5
|
|
30
|
+
with:
|
|
31
|
+
python-version: '3.x'
|
|
32
|
+
cache: 'pip' # caching pip dependencies
|
|
33
|
+
|
|
34
|
+
- name: Install dependencies
|
|
35
|
+
run: |
|
|
36
|
+
python -m pip install -U pip # seems necessary to make sure pip is my pip
|
|
37
|
+
pip install --extra-index-url https://download.pytorch.org/whl/cpu torch
|
|
38
|
+
pip install dredge/
|
|
39
|
+
pip install -e dartsort/[full]
|
|
40
|
+
|
|
41
|
+
- name: pytest
|
|
42
|
+
run: |
|
|
43
|
+
PYTHONFAULTHANDLER=1 pytest --cache-clear --maxfail=30 --log-level=15 dartsort/tests/
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
build:
|
|
11
|
+
name: Build distribution 📦
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v6
|
|
16
|
+
with:
|
|
17
|
+
persist-credentials: false
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v6
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.x"
|
|
22
|
+
- name: Install pypa/build
|
|
23
|
+
run: >-
|
|
24
|
+
python3 -m
|
|
25
|
+
pip install
|
|
26
|
+
build
|
|
27
|
+
--user
|
|
28
|
+
- name: Build a binary wheel and a source tarball
|
|
29
|
+
run: python3 -m build
|
|
30
|
+
- name: Store the distribution packages
|
|
31
|
+
uses: actions/upload-artifact@v5
|
|
32
|
+
with:
|
|
33
|
+
name: python-package-distributions
|
|
34
|
+
path: dist/
|
|
35
|
+
|
|
36
|
+
publish-to-pypi:
|
|
37
|
+
name: >-
|
|
38
|
+
Publish Python 🐍 distribution 📦 to PyPI
|
|
39
|
+
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
|
|
40
|
+
needs:
|
|
41
|
+
- build
|
|
42
|
+
runs-on: ubuntu-latest
|
|
43
|
+
environment:
|
|
44
|
+
name: pypi
|
|
45
|
+
url: https://pypi.org/p/dartsort
|
|
46
|
+
permissions:
|
|
47
|
+
id-token: write # IMPORTANT: mandatory for trusted publishing
|
|
48
|
+
|
|
49
|
+
steps:
|
|
50
|
+
- name: Download all the dists
|
|
51
|
+
uses: actions/download-artifact@v6
|
|
52
|
+
with:
|
|
53
|
+
name: python-package-distributions
|
|
54
|
+
path: dist/
|
|
55
|
+
- name: Publish distribution 📦 to PyPI
|
|
56
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
# from me
|
|
2
|
+
.jupyter/
|
|
3
|
+
**/*.ipynb
|
|
4
|
+
.DS_Store
|
|
5
|
+
data/*.h5
|
|
6
|
+
data/*.npy
|
|
7
|
+
data/spt*
|
|
8
|
+
data/*/
|
|
9
|
+
logs/*.log
|
|
10
|
+
**/*.c
|
|
11
|
+
notebook/.stan/*
|
|
12
|
+
figs/*
|
|
13
|
+
.idea
|
|
14
|
+
*.swp
|
|
15
|
+
*.sublime-workspace
|
|
16
|
+
*.sublime-project
|
|
17
|
+
|
|
18
|
+
## Core latex/pdflatex auxiliary files:
|
|
19
|
+
*.aux
|
|
20
|
+
*.lof
|
|
21
|
+
*.log
|
|
22
|
+
*.lot
|
|
23
|
+
*.fls
|
|
24
|
+
*.out
|
|
25
|
+
*.toc
|
|
26
|
+
*.fmt
|
|
27
|
+
*.fot
|
|
28
|
+
*.cb
|
|
29
|
+
*.cb2
|
|
30
|
+
.*.lb
|
|
31
|
+
|
|
32
|
+
## Bibliography auxiliary files (bibtex/biblatex/biber):
|
|
33
|
+
*.bbl
|
|
34
|
+
*.bcf
|
|
35
|
+
*.blg
|
|
36
|
+
*-blx.aux
|
|
37
|
+
*-blx.bib
|
|
38
|
+
*.run.xml
|
|
39
|
+
|
|
40
|
+
*.pdf
|
|
41
|
+
|
|
42
|
+
# Byte-compiled / optimized / DLL files
|
|
43
|
+
__pycache__/
|
|
44
|
+
*.py[cod]
|
|
45
|
+
*$py.class
|
|
46
|
+
|
|
47
|
+
# C extensions
|
|
48
|
+
*.so
|
|
49
|
+
|
|
50
|
+
# Distribution / packaging
|
|
51
|
+
.Python
|
|
52
|
+
build/
|
|
53
|
+
develop-eggs/
|
|
54
|
+
dist/
|
|
55
|
+
downloads/
|
|
56
|
+
eggs/
|
|
57
|
+
.eggs/
|
|
58
|
+
lib/
|
|
59
|
+
lib64/
|
|
60
|
+
parts/
|
|
61
|
+
sdist/
|
|
62
|
+
var/
|
|
63
|
+
wheels/
|
|
64
|
+
pip-wheel-metadata/
|
|
65
|
+
share/python-wheels/
|
|
66
|
+
*.egg-info/
|
|
67
|
+
.installed.cfg
|
|
68
|
+
*.egg
|
|
69
|
+
MANIFEST
|
|
70
|
+
|
|
71
|
+
# PyInstaller
|
|
72
|
+
# Usually these files are written by a python script from a template
|
|
73
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
74
|
+
*.manifest
|
|
75
|
+
*.spec
|
|
76
|
+
|
|
77
|
+
# Installer logs
|
|
78
|
+
pip-log.txt
|
|
79
|
+
pip-delete-this-directory.txt
|
|
80
|
+
|
|
81
|
+
# Unit test / coverage reports
|
|
82
|
+
htmlcov/
|
|
83
|
+
.tox/
|
|
84
|
+
.nox/
|
|
85
|
+
.coverage
|
|
86
|
+
.coverage.*
|
|
87
|
+
.cache
|
|
88
|
+
nosetests.xml
|
|
89
|
+
coverage.xml
|
|
90
|
+
*.cover
|
|
91
|
+
*.py,cover
|
|
92
|
+
.hypothesis/
|
|
93
|
+
.pytest_cache/
|
|
94
|
+
|
|
95
|
+
# Translations
|
|
96
|
+
*.mo
|
|
97
|
+
*.pot
|
|
98
|
+
|
|
99
|
+
# Django stuff:
|
|
100
|
+
*.log
|
|
101
|
+
local_settings.py
|
|
102
|
+
db.sqlite3
|
|
103
|
+
db.sqlite3-journal
|
|
104
|
+
|
|
105
|
+
# Flask stuff:
|
|
106
|
+
instance/
|
|
107
|
+
.webassets-cache
|
|
108
|
+
|
|
109
|
+
# Scrapy stuff:
|
|
110
|
+
.scrapy
|
|
111
|
+
|
|
112
|
+
# Sphinx documentation
|
|
113
|
+
docs/_build/
|
|
114
|
+
|
|
115
|
+
# PyBuilder
|
|
116
|
+
target/
|
|
117
|
+
|
|
118
|
+
# Jupyter Notebook
|
|
119
|
+
.ipynb_checkpoints
|
|
120
|
+
|
|
121
|
+
# IPython
|
|
122
|
+
profile_default/
|
|
123
|
+
ipython_config.py
|
|
124
|
+
|
|
125
|
+
# pyenv
|
|
126
|
+
.python-version
|
|
127
|
+
|
|
128
|
+
# pipenv
|
|
129
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
130
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
131
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
132
|
+
# install all needed dependencies.
|
|
133
|
+
#Pipfile.lock
|
|
134
|
+
|
|
135
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
|
136
|
+
__pypackages__/
|
|
137
|
+
|
|
138
|
+
# Celery stuff
|
|
139
|
+
celerybeat-schedule
|
|
140
|
+
celerybeat.pid
|
|
141
|
+
|
|
142
|
+
# SageMath parsed files
|
|
143
|
+
*.sage.py
|
|
144
|
+
|
|
145
|
+
# Environments
|
|
146
|
+
.env
|
|
147
|
+
.venv
|
|
148
|
+
env/
|
|
149
|
+
venv/
|
|
150
|
+
ENV/
|
|
151
|
+
env.bak/
|
|
152
|
+
venv.bak/
|
|
153
|
+
|
|
154
|
+
# Spyder project settings
|
|
155
|
+
.spyderproject
|
|
156
|
+
.spyproject
|
|
157
|
+
|
|
158
|
+
# Rope project settings
|
|
159
|
+
.ropeproject
|
|
160
|
+
|
|
161
|
+
# mkdocs documentation
|
|
162
|
+
/site
|
|
163
|
+
|
|
164
|
+
# mypy
|
|
165
|
+
.mypy_cache/
|
|
166
|
+
.dmypy.json
|
|
167
|
+
dmypy.json
|
|
168
|
+
|
|
169
|
+
# Pyre type checker
|
|
170
|
+
.pyre/
|
dartsort-0.3.5/PKG-INFO
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: DARTsort
|
|
3
|
+
Version: 0.3.5
|
|
4
|
+
Summary: DARTsort
|
|
5
|
+
Author-email: Charlie Windolf <ciw2107@columbia.edu>
|
|
6
|
+
Project-URL: Homepage, https://github.com/cwindolf/dartsort
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/cwindolf/dartsort/issues
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
Requires-Dist: cupy
|
|
14
|
+
Requires-Dist: h5py
|
|
15
|
+
Requires-Dist: linear_operator
|
|
16
|
+
Requires-Dist: numba
|
|
17
|
+
Requires-Dist: numpy>=1.20; python_version < "3.13"
|
|
18
|
+
Requires-Dist: numpy>=2.0.0; python_version >= "3.13"
|
|
19
|
+
Requires-Dist: opt-einsum
|
|
20
|
+
Requires-Dist: pandas
|
|
21
|
+
Requires-Dist: packaging
|
|
22
|
+
Requires-Dist: probeinterface
|
|
23
|
+
Requires-Dist: pydantic
|
|
24
|
+
Requires-Dist: scipy>=1.13
|
|
25
|
+
Requires-Dist: scikit-learn
|
|
26
|
+
Requires-Dist: spikeinterface>=0.101.2
|
|
27
|
+
Requires-Dist: torch>=2.0
|
|
28
|
+
Requires-Dist: tqdm
|
|
29
|
+
Provides-Extra: vis
|
|
30
|
+
Requires-Dist: matplotlib; extra == "vis"
|
|
31
|
+
Requires-Dist: seaborn; extra == "vis"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
35
|
+
Provides-Extra: full
|
|
36
|
+
Requires-Dist: matplotlib; extra == "full"
|
|
37
|
+
Requires-Dist: seaborn; extra == "full"
|
|
38
|
+
Requires-Dist: pytest; extra == "full"
|
|
39
|
+
Requires-Dist: pytest-cov; extra == "full"
|
|
40
|
+
Requires-Dist: ipywidgets; extra == "full"
|
|
41
|
+
Requires-Dist: ipykernel; extra == "full"
|
|
42
|
+
Requires-Dist: black; extra == "full"
|
|
43
|
+
|
|
44
|
+
[](https://github.com/cwindolf/dartsort/actions/)
|
|
45
|
+
[](https://coveralls.io/github/cwindolf/dartsort)
|
|
46
|
+
|
|
47
|
+
# dartsort
|
|
48
|
+
|
|
49
|
+
## :warning: Work in progress code repository
|
|
50
|
+
|
|
51
|
+
We do not currently recommend DARTsort for production spike sorting purposes. We are in the process of implementing a robust and documented pipeline in [`src/dartsort`](src/dartsort), and we will update this page accordingly.
|
|
52
|
+
|
|
53
|
+
A workflow described in our preprint (https://www.biorxiv.org/content/10.1101/2023.08.11.553023v1) is in [uhd_pipeline.py](scripts/uhd_pipeline.py), which is implemented using the legacy code in [`src/spike_psvae`](src/spike_psvae).
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
## Suggested install steps
|
|
57
|
+
|
|
58
|
+
If you don't already have Python and PyTorch 2 installed, we recommend doing this with the Miniforge distribution of `conda`. You can find info and installers for your platform [at Miniforge's GitHub repository](https://github.com/conda-forge/miniforge). After installing Miniforge, `conda` will be available on your computer for installing Python packages, as well as the newer and faster conda replacement tool `mamba`. We recommend using `mamba` instead of `conda` below, since the installation tends to be a lot faster with `mamba`.
|
|
59
|
+
|
|
60
|
+
To install DARTsort, first clone this GitHub repository.
|
|
61
|
+
|
|
62
|
+
After cloning the repository, create and activate the `mamba`/`conda` environment from the configuration file provided as follows:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
$ mamba env create -f environment.yml
|
|
66
|
+
$ mamba activate dartsort
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Next, visit https://pytorch.org/get-started/locally/ and follow the `PyTorch` install instructions for your specific OS and hardware needs.
|
|
70
|
+
We also need to install `linear_operator` from the `gpytorch` channel.
|
|
71
|
+
For example, on a Linux workstation or cluster with NVIDIA GPUs available, one might use (dropping in `mamba` for `conda` commands):
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Example -- see https://pytorch.org/get-started/locally/ to find your platform's command.
|
|
75
|
+
(dartsort) $ mamba install pytorch torchvision torchaudio pytorch-cuda=11.8 linear_operator -c pytorch -c nvidia -c gpytorch
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Finally, install the remaining `pip` dependencies and `dartsort` itself:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
(dartsort) $ pip install -r requirements-full.txt
|
|
82
|
+
(dartsort) $ pip install -e .
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
To enable DARTsort's default motion correction algorithm [DREDge](https://www.biorxiv.org/content/10.1101/2023.10.24.563768), clone [its GitHub repository](https://github.com/evarol/dredge), and then `cd dredge/` and install the DREDge package with `pip install -e .`.
|
|
86
|
+
|
|
87
|
+
Soon we will have a package on PyPI so that these last steps will be just a `pip install dartsort`.
|
|
88
|
+
|
|
89
|
+
To make sure everything is working:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
$ (dartsort) pytest tests/*
|
|
93
|
+
```
|
dartsort-0.3.5/README.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[](https://github.com/cwindolf/dartsort/actions/)
|
|
2
|
+
[](https://coveralls.io/github/cwindolf/dartsort)
|
|
3
|
+
|
|
4
|
+
# dartsort
|
|
5
|
+
|
|
6
|
+
## :warning: Work in progress code repository
|
|
7
|
+
|
|
8
|
+
We do not currently recommend DARTsort for production spike sorting purposes. We are in the process of implementing a robust and documented pipeline in [`src/dartsort`](src/dartsort), and we will update this page accordingly.
|
|
9
|
+
|
|
10
|
+
A workflow described in our preprint (https://www.biorxiv.org/content/10.1101/2023.08.11.553023v1) is in [uhd_pipeline.py](scripts/uhd_pipeline.py), which is implemented using the legacy code in [`src/spike_psvae`](src/spike_psvae).
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
## Suggested install steps
|
|
14
|
+
|
|
15
|
+
If you don't already have Python and PyTorch 2 installed, we recommend doing this with the Miniforge distribution of `conda`. You can find info and installers for your platform [at Miniforge's GitHub repository](https://github.com/conda-forge/miniforge). After installing Miniforge, `conda` will be available on your computer for installing Python packages, as well as the newer and faster conda replacement tool `mamba`. We recommend using `mamba` instead of `conda` below, since the installation tends to be a lot faster with `mamba`.
|
|
16
|
+
|
|
17
|
+
To install DARTsort, first clone this GitHub repository.
|
|
18
|
+
|
|
19
|
+
After cloning the repository, create and activate the `mamba`/`conda` environment from the configuration file provided as follows:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
$ mamba env create -f environment.yml
|
|
23
|
+
$ mamba activate dartsort
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Next, visit https://pytorch.org/get-started/locally/ and follow the `PyTorch` install instructions for your specific OS and hardware needs.
|
|
27
|
+
We also need to install `linear_operator` from the `gpytorch` channel.
|
|
28
|
+
For example, on a Linux workstation or cluster with NVIDIA GPUs available, one might use (dropping in `mamba` for `conda` commands):
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
# Example -- see https://pytorch.org/get-started/locally/ to find your platform's command.
|
|
32
|
+
(dartsort) $ mamba install pytorch torchvision torchaudio pytorch-cuda=11.8 linear_operator -c pytorch -c nvidia -c gpytorch
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Finally, install the remaining `pip` dependencies and `dartsort` itself:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
(dartsort) $ pip install -r requirements-full.txt
|
|
39
|
+
(dartsort) $ pip install -e .
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
To enable DARTsort's default motion correction algorithm [DREDge](https://www.biorxiv.org/content/10.1101/2023.10.24.563768), clone [its GitHub repository](https://github.com/evarol/dredge), and then `cd dredge/` and install the DREDge package with `pip install -e .`.
|
|
43
|
+
|
|
44
|
+
Soon we will have a package on PyPI so that these last steps will be just a `pip install dartsort`.
|
|
45
|
+
|
|
46
|
+
To make sure everything is working:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
$ (dartsort) pytest tests/*
|
|
50
|
+
```
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: dartsort
|
|
3
|
+
channels:
|
|
4
|
+
- conda-forge
|
|
5
|
+
dependencies:
|
|
6
|
+
- python=3.11
|
|
7
|
+
- h5py
|
|
8
|
+
- matplotlib
|
|
9
|
+
- numpy>=1.26
|
|
10
|
+
- pytest
|
|
11
|
+
- scipy>=1.13
|
|
12
|
+
- scikit-learn
|
|
13
|
+
- tqdm
|
|
14
|
+
- pandas
|
|
15
|
+
- linear_operator
|
|
16
|
+
- numba
|
|
17
|
+
- pydantic
|
|
18
|
+
- opt-einsum
|
|
File without changes
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# ---
|
|
2
|
+
# jupyter:
|
|
3
|
+
# jupytext:
|
|
4
|
+
# text_representation:
|
|
5
|
+
# extension: .py
|
|
6
|
+
# format_name: percent
|
|
7
|
+
# format_version: '1.3'
|
|
8
|
+
# jupytext_version: 1.14.5
|
|
9
|
+
# kernelspec:
|
|
10
|
+
# display_name: Python [conda env:a]
|
|
11
|
+
# language: python
|
|
12
|
+
# name: conda-env-a-py
|
|
13
|
+
# ---
|
|
14
|
+
|
|
15
|
+
# %%
|
|
16
|
+
# %load_ext autoreload
|
|
17
|
+
# %autoreload 2
|
|
18
|
+
|
|
19
|
+
# %%
|
|
20
|
+
import numpy as np
|
|
21
|
+
from spike_psvae import subtract
|
|
22
|
+
from one.api import ONE
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
# %%
|
|
26
|
+
import spikeinterface.full as si
|
|
27
|
+
|
|
28
|
+
# %%
|
|
29
|
+
one = ONE()
|
|
30
|
+
|
|
31
|
+
# %%
|
|
32
|
+
sdsc_base_path = Path("/mnt/sdceph/users/ibl/data")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# %%
|
|
36
|
+
def eid2sdscpath(eid):
|
|
37
|
+
pids, probes = one.eid2pid(eid)
|
|
38
|
+
alyx_base_path = one.eid2path(eid)
|
|
39
|
+
paths = {}
|
|
40
|
+
for pid, probe in zip(pids, probes):
|
|
41
|
+
rel_path = one.list_datasets(eid, f"raw_ephys_data/{probe}*ap.cbin")
|
|
42
|
+
assert len(rel_path) == 1
|
|
43
|
+
rel_path = Path(rel_path[0])
|
|
44
|
+
searchdir = sdsc_base_path / alyx_base_path.relative_to(one.cache_dir) / rel_path.parent
|
|
45
|
+
pattern = Path(rel_path.name).with_suffix(f".*.cbin")
|
|
46
|
+
glob = list(searchdir.glob(str(pattern)))
|
|
47
|
+
assert len(glob) == 1
|
|
48
|
+
paths[probe] = pid, glob[0]
|
|
49
|
+
assert paths[probe][1].exists()
|
|
50
|
+
return paths
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# %%
|
|
54
|
+
pid = "7d999a68-0215-4e45-8e6c-879c6ca2b771"
|
|
55
|
+
|
|
56
|
+
# %%
|
|
57
|
+
eid, probe = one.pid2eid(pid)
|
|
58
|
+
paths = eid2sdscpath(eid)
|
|
59
|
+
print(paths)
|
|
60
|
+
pid, cbin_path = paths[probe]
|
|
61
|
+
ch_path = list(cbin_path.parent.glob("*ap*.ch"))
|
|
62
|
+
assert len(ch_path) == 1
|
|
63
|
+
ch_path = ch_path[0]
|
|
64
|
+
meta_path = list(cbin_path.parent.glob("*ap*.meta"))
|
|
65
|
+
assert len(meta_path) == 1
|
|
66
|
+
meta_path = meta_path[0]
|
|
67
|
+
|
|
68
|
+
print("-" * 50)
|
|
69
|
+
rec = si.read_cbin_ibl(str(cbin_path.parent), cbin_file=str(cbin_path), ch_file=str(ch_path), meta_file=str(meta_path))
|
|
70
|
+
print(rec)
|
|
71
|
+
fs = int(rec.get_sampling_frequency())
|
|
72
|
+
|
|
73
|
+
# %%
|
|
74
|
+
rec = si.highpass_filter(rec)
|
|
75
|
+
rec = si.phase_shift(rec)
|
|
76
|
+
bad_channel_ids, channel_labels = si.detect_bad_channels(rec, num_random_chunks=100)
|
|
77
|
+
print(f"{bad_channel_ids=}")
|
|
78
|
+
rec = si.interpolate_bad_channels(rec, bad_channel_ids)
|
|
79
|
+
rec = si.highpass_spatial_filter(rec)
|
|
80
|
+
# we had been working with this before -- should switch to MAD,
|
|
81
|
+
# but we need to rethink the thresholds
|
|
82
|
+
rec = si.zscore(rec, mode="mean+std", num_chunks_per_segment=100, margin_frames=250*fs)
|
|
83
|
+
|
|
84
|
+
# %%
|
|
85
|
+
rec = rec.frame_slice(start_frame=rec.get_num_samples() // 2, end_frame=rec.get_num_samples() // 2 + 30 * 30000)
|
|
86
|
+
|
|
87
|
+
rec
|
|
88
|
+
|
|
89
|
+
rec_chansliced = rec.channel_slice(channel_ids=rec.channel_ids[[11, 13, 21, 65, 4, 2]])
|
|
90
|
+
|
|
91
|
+
rec_chansliced
|
|
92
|
+
|
|
93
|
+
geom = rec_chansliced.get_channel_locations()
|
|
94
|
+
geom
|
|
95
|
+
|
|
96
|
+
rec_ord = si.depth_order(rec_chansliced)
|
|
97
|
+
rec_ord
|
|
98
|
+
|
|
99
|
+
geom = rec_ord.get_channel_locations()
|
|
100
|
+
geom
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# !rm -rf /tmp/testfolder
|
|
105
|
+
|
|
106
|
+
rec_ord.("/tmp/testfolder", n_jobs=1)
|
|
107
|
+
|
|
108
|
+
# !ls /tmp/testfolder
|
|
109
|
+
|
|
110
|
+
xcoords = geom[:, 0]
|
|
111
|
+
ycoords = geom[:, 1]
|
|
112
|
+
kcoords = np.zeros_like(geom[:, 0])
|
|
113
|
+
chanMap = np.arange(len(geom))
|
|
114
|
+
|
|
115
|
+
chanMap
|
|
116
|
+
|
|
117
|
+
from scipy.io import savemat
|
|
118
|
+
|
|
119
|
+
savemat("/tmp/chanMap.mat", dict(xcoords=xcoords, ycoords=ycoords, cm))
|
|
120
|
+
|
|
121
|
+
# %%
|
|
122
|
+
sub_h5 = subtract.subtraction(
|
|
123
|
+
rec,
|
|
124
|
+
out_folder="/tmp/test",
|
|
125
|
+
thresholds=[12, 10, 8, 6, 5],
|
|
126
|
+
n_sec_pca=10,
|
|
127
|
+
save_subtracted_tpca_projs=False,
|
|
128
|
+
save_cleaned_tpca_projs=False,
|
|
129
|
+
save_denoised_tpca_projs=False,
|
|
130
|
+
n_jobs=1,
|
|
131
|
+
loc_workers=1,
|
|
132
|
+
overwrite=False,
|
|
133
|
+
n_sec_chunk=1,
|
|
134
|
+
device="cuda:1",
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
# %%
|