DARTsort 0.3.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. dartsort-0.3.5/.github/workflows/ci.yml +52 -0
  2. dartsort-0.3.5/.github/workflows/ci_windows.yml +43 -0
  3. dartsort-0.3.5/.github/workflows/deploy.yml +56 -0
  4. dartsort-0.3.5/.gitignore +170 -0
  5. dartsort-0.3.5/PKG-INFO +93 -0
  6. dartsort-0.3.5/README.md +50 -0
  7. dartsort-0.3.5/environment.yml +18 -0
  8. dartsort-0.3.5/notebook/.keep +0 -0
  9. dartsort-0.3.5/notebook/basic_integration_test.py +137 -0
  10. dartsort-0.3.5/notebook/ci diag.py +573 -0
  11. dartsort-0.3.5/notebook/dartsort_dredge_demo.ipynb +437 -0
  12. dartsort-0.3.5/notebook/dartsort_dredge_demo.py +144 -0
  13. dartsort-0.3.5/notebook/data test.py +259 -0
  14. dartsort-0.3.5/notebook/dead reckoning.py +361 -0
  15. dartsort-0.3.5/notebook/denoise np1.py +356 -0
  16. dartsort-0.3.5/notebook/denoise np2.py +480 -0
  17. dartsort-0.3.5/notebook/exploratory.py +431 -0
  18. dartsort-0.3.5/notebook/geom debug.py +167 -0
  19. dartsort-0.3.5/notebook/hdbscan_clustering_5min.py +1294 -0
  20. dartsort-0.3.5/notebook/hybrid 10.py +1662 -0
  21. dartsort-0.3.5/notebook/hybrid cluster.py +475 -0
  22. dartsort-0.3.5/notebook/hybrid_11.py +1214 -0
  23. dartsort-0.3.5/notebook/initial training.py +162 -0
  24. dartsort-0.3.5/notebook/make hybrid.py +1376 -0
  25. dartsort-0.3.5/notebook/mgh.py +514 -0
  26. dartsort-0.3.5/notebook/np1 feats.py +1211 -0
  27. dartsort-0.3.5/notebook/np2 processing version c.py +321 -0
  28. dartsort-0.3.5/notebook/paperfigs.py +907 -0
  29. dartsort-0.3.5/notebook/pdists.py +641 -0
  30. dartsort-0.3.5/notebook/postcheck.py +803 -0
  31. dartsort-0.3.5/notebook/ptp vae.py +206 -0
  32. dartsort-0.3.5/notebook/re clust.py +893 -0
  33. dartsort-0.3.5/notebook/reg fig loop.py +723 -0
  34. dartsort-0.3.5/notebook/reg_sim_exp.py +493 -0
  35. dartsort-0.3.5/notebook/relocating.py +284 -0
  36. dartsort-0.3.5/notebook/relocating_all.py +182 -0
  37. dartsort-0.3.5/notebook/restats +0 -0
  38. dartsort-0.3.5/notebook/sortStinger1.py +1590 -0
  39. dartsort-0.3.5/notebook/sortStinger3.py +1541 -0
  40. dartsort-0.3.5/notebook/sortStinger4.py +1835 -0
  41. dartsort-0.3.5/notebook/sortStinger5.py +1586 -0
  42. dartsort-0.3.5/notebook/sortStinger6.py +1915 -0
  43. dartsort-0.3.5/notebook/spike farm.py +100 -0
  44. dartsort-0.3.5/notebook/ss_ensemble_sim.py +1965 -0
  45. dartsort-0.3.5/notebook/ss_ensemble_sim_drift.py +1928 -0
  46. dartsort-0.3.5/notebook/ss_sim.py +1965 -0
  47. dartsort-0.3.5/notebook/stan diagnostics.py +207 -0
  48. dartsort-0.3.5/notebook/storytelling c.py +686 -0
  49. dartsort-0.3.5/notebook/storytelling.py +684 -0
  50. dartsort-0.3.5/notebook/subtract_with_phaseshift_denoiser.py +260 -0
  51. dartsort-0.3.5/notebook/tests/debug_matching.py +454 -0
  52. dartsort-0.3.5/notebook/tests/test_agreement.py +367 -0
  53. dartsort-0.3.5/notebook/tests/test_amplitude_scaling.py +737 -0
  54. dartsort-0.3.5/notebook/tests/test_batch_optim.py +679 -0
  55. dartsort-0.3.5/notebook/tests/test_ibme.py +86 -0
  56. dartsort-0.3.5/notebook/tests/test_mm_initialization.py +142 -0
  57. dartsort-0.3.5/notebook/tests/test_normxcorr1d.py +63 -0
  58. dartsort-0.3.5/notebook/tests/test_reassignment.py +422 -0
  59. dartsort-0.3.5/notebook/tests/test_relocation.py +230 -0
  60. dartsort-0.3.5/notebook/thomas.py +1004 -0
  61. dartsort-0.3.5/pyproject.toml +73 -0
  62. dartsort-0.3.5/setup.cfg +4 -0
  63. dartsort-0.3.5/src/DARTsort.egg-info/PKG-INFO +93 -0
  64. dartsort-0.3.5/src/DARTsort.egg-info/SOURCES.txt +213 -0
  65. dartsort-0.3.5/src/DARTsort.egg-info/dependency_links.txt +1 -0
  66. dartsort-0.3.5/src/DARTsort.egg-info/entry_points.txt +2 -0
  67. dartsort-0.3.5/src/DARTsort.egg-info/requires.txt +37 -0
  68. dartsort-0.3.5/src/DARTsort.egg-info/top_level.txt +1 -0
  69. dartsort-0.3.5/src/dartsort/__init__.py +68 -0
  70. dartsort-0.3.5/src/dartsort/cli.py +105 -0
  71. dartsort-0.3.5/src/dartsort/clustering/__init__.py +3 -0
  72. dartsort-0.3.5/src/dartsort/clustering/cluster_util.py +392 -0
  73. dartsort-0.3.5/src/dartsort/clustering/clustering.py +765 -0
  74. dartsort-0.3.5/src/dartsort/clustering/clustering_features.py +202 -0
  75. dartsort-0.3.5/src/dartsort/clustering/density.py +1299 -0
  76. dartsort-0.3.5/src/dartsort/clustering/forward_backward.py +219 -0
  77. dartsort-0.3.5/src/dartsort/clustering/gmm/mixture.py +5790 -0
  78. dartsort-0.3.5/src/dartsort/clustering/gmm/stable_features.py +993 -0
  79. dartsort-0.3.5/src/dartsort/clustering/kmeans.py +448 -0
  80. dartsort-0.3.5/src/dartsort/clustering/merge.py +501 -0
  81. dartsort-0.3.5/src/dartsort/clustering/modes.py +214 -0
  82. dartsort-0.3.5/src/dartsort/clustering/refine_util.py +191 -0
  83. dartsort-0.3.5/src/dartsort/clustering/relocate.py +109 -0
  84. dartsort-0.3.5/src/dartsort/clustering/split.py +1285 -0
  85. dartsort-0.3.5/src/dartsort/config.py +288 -0
  86. dartsort-0.3.5/src/dartsort/detect/__init__.py +2 -0
  87. dartsort-0.3.5/src/dartsort/detect/detect.py +449 -0
  88. dartsort-0.3.5/src/dartsort/detect/detect_filters.py +37 -0
  89. dartsort-0.3.5/src/dartsort/evaluate/__init__.py +1 -0
  90. dartsort-0.3.5/src/dartsort/evaluate/analysis.py +594 -0
  91. dartsort-0.3.5/src/dartsort/evaluate/comparison.py +534 -0
  92. dartsort-0.3.5/src/dartsort/evaluate/hybrid_util.py +599 -0
  93. dartsort-0.3.5/src/dartsort/evaluate/noise_recording_tools.py +184 -0
  94. dartsort-0.3.5/src/dartsort/evaluate/sim_template_tools.py +842 -0
  95. dartsort-0.3.5/src/dartsort/evaluate/simkit.py +840 -0
  96. dartsort-0.3.5/src/dartsort/evaluate/simlib.py +270 -0
  97. dartsort-0.3.5/src/dartsort/localize/__init__.py +0 -0
  98. dartsort-0.3.5/src/dartsort/localize/localize_torch.py +291 -0
  99. dartsort-0.3.5/src/dartsort/localize/localize_util.py +313 -0
  100. dartsort-0.3.5/src/dartsort/main.py +657 -0
  101. dartsort-0.3.5/src/dartsort/peel/__init__.py +6 -0
  102. dartsort-0.3.5/src/dartsort/peel/grab.py +208 -0
  103. dartsort-0.3.5/src/dartsort/peel/matching.py +628 -0
  104. dartsort-0.3.5/src/dartsort/peel/matching_util/__init__.py +3 -0
  105. dartsort-0.3.5/src/dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  106. dartsort-0.3.5/src/dartsort/peel/matching_util/drifty.py +791 -0
  107. dartsort-0.3.5/src/dartsort/peel/matching_util/matching_base.py +616 -0
  108. dartsort-0.3.5/src/dartsort/peel/matching_util/pairwise.py +359 -0
  109. dartsort-0.3.5/src/dartsort/peel/matching_util/pairwise_util.py +1317 -0
  110. dartsort-0.3.5/src/dartsort/peel/peel_base.py +1000 -0
  111. dartsort-0.3.5/src/dartsort/peel/running_template.py +1498 -0
  112. dartsort-0.3.5/src/dartsort/peel/subtract.py +1123 -0
  113. dartsort-0.3.5/src/dartsort/peel/threshold.py +471 -0
  114. dartsort-0.3.5/src/dartsort/peel/universal.py +191 -0
  115. dartsort-0.3.5/src/dartsort/pretrained/default_temporal_kernel.npy +0 -0
  116. dartsort-0.3.5/src/dartsort/pretrained/detect_np1.pt +0 -0
  117. dartsort-0.3.5/src/dartsort/pretrained/detect_np2.pt +0 -0
  118. dartsort-0.3.5/src/dartsort/pretrained/glasbey1024.npz +0 -0
  119. dartsort-0.3.5/src/dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  120. dartsort-0.3.5/src/dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  121. dartsort-0.3.5/src/dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  122. dartsort-0.3.5/src/dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  123. dartsort-0.3.5/src/dartsort/pretrained/single_chan_denoiser.pt +0 -0
  124. dartsort-0.3.5/src/dartsort/templates/__init__.py +11 -0
  125. dartsort-0.3.5/src/dartsort/templates/get_templates.py +771 -0
  126. dartsort-0.3.5/src/dartsort/templates/postprocess_util.py +555 -0
  127. dartsort-0.3.5/src/dartsort/templates/realignment.py +466 -0
  128. dartsort-0.3.5/src/dartsort/templates/superres_util.py +204 -0
  129. dartsort-0.3.5/src/dartsort/templates/template_util.py +518 -0
  130. dartsort-0.3.5/src/dartsort/templates/templates.py +528 -0
  131. dartsort-0.3.5/src/dartsort/transform/__init__.py +2 -0
  132. dartsort-0.3.5/src/dartsort/transform/_multichan_denoiser_kit.py +650 -0
  133. dartsort-0.3.5/src/dartsort/transform/all_transformers.py +58 -0
  134. dartsort-0.3.5/src/dartsort/transform/amortized_localization.py +503 -0
  135. dartsort-0.3.5/src/dartsort/transform/amplitudes.py +197 -0
  136. dartsort-0.3.5/src/dartsort/transform/decollider.py +635 -0
  137. dartsort-0.3.5/src/dartsort/transform/enforce_decrease.py +158 -0
  138. dartsort-0.3.5/src/dartsort/transform/localize.py +86 -0
  139. dartsort-0.3.5/src/dartsort/transform/matching_denoiser.py +44 -0
  140. dartsort-0.3.5/src/dartsort/transform/pipeline.py +391 -0
  141. dartsort-0.3.5/src/dartsort/transform/single_channel_denoiser.py +153 -0
  142. dartsort-0.3.5/src/dartsort/transform/single_channel_templates.py +98 -0
  143. dartsort-0.3.5/src/dartsort/transform/supervised_denoiser.py +486 -0
  144. dartsort-0.3.5/src/dartsort/transform/temporal_pca.py +413 -0
  145. dartsort-0.3.5/src/dartsort/transform/transform_base.py +255 -0
  146. dartsort-0.3.5/src/dartsort/util/__init__.py +0 -0
  147. dartsort-0.3.5/src/dartsort/util/cli_util.py +179 -0
  148. dartsort-0.3.5/src/dartsort/util/data_util.py +1128 -0
  149. dartsort-0.3.5/src/dartsort/util/drift_util.py +1054 -0
  150. dartsort-0.3.5/src/dartsort/util/internal_config.py +1074 -0
  151. dartsort-0.3.5/src/dartsort/util/interpolation_util.py +1012 -0
  152. dartsort-0.3.5/src/dartsort/util/job_util.py +19 -0
  153. dartsort-0.3.5/src/dartsort/util/logging_util.py +91 -0
  154. dartsort-0.3.5/src/dartsort/util/main_util.py +235 -0
  155. dartsort-0.3.5/src/dartsort/util/more_operators.py +269 -0
  156. dartsort-0.3.5/src/dartsort/util/multiprocessing_util.py +211 -0
  157. dartsort-0.3.5/src/dartsort/util/nn_util.py +461 -0
  158. dartsort-0.3.5/src/dartsort/util/noise_util.py +1461 -0
  159. dartsort-0.3.5/src/dartsort/util/peel_util.py +223 -0
  160. dartsort-0.3.5/src/dartsort/util/py_util.py +213 -0
  161. dartsort-0.3.5/src/dartsort/util/registration_util.py +128 -0
  162. dartsort-0.3.5/src/dartsort/util/residual_util.py +30 -0
  163. dartsort-0.3.5/src/dartsort/util/sparse_util.py +81 -0
  164. dartsort-0.3.5/src/dartsort/util/spikeio.py +227 -0
  165. dartsort-0.3.5/src/dartsort/util/spiketorch.py +1198 -0
  166. dartsort-0.3.5/src/dartsort/util/testing_util/__init__.py +0 -0
  167. dartsort-0.3.5/src/dartsort/util/testing_util/matching_debug_util.py +517 -0
  168. dartsort-0.3.5/src/dartsort/util/testing_util/mixture_testing_util.py +291 -0
  169. dartsort-0.3.5/src/dartsort/util/torch_optimization_util.py +156 -0
  170. dartsort-0.3.5/src/dartsort/util/torch_util.py +75 -0
  171. dartsort-0.3.5/src/dartsort/util/universal_util.py +269 -0
  172. dartsort-0.3.5/src/dartsort/util/waveform_util.py +809 -0
  173. dartsort-0.3.5/src/dartsort/vis/__init__.py +19 -0
  174. dartsort-0.3.5/src/dartsort/vis/analysis_plots.py +392 -0
  175. dartsort-0.3.5/src/dartsort/vis/colors.py +36 -0
  176. dartsort-0.3.5/src/dartsort/vis/gt.py +392 -0
  177. dartsort-0.3.5/src/dartsort/vis/layout.py +149 -0
  178. dartsort-0.3.5/src/dartsort/vis/metrics_util.py +23 -0
  179. dartsort-0.3.5/src/dartsort/vis/mixture.py +1489 -0
  180. dartsort-0.3.5/src/dartsort/vis/over_time.py +514 -0
  181. dartsort-0.3.5/src/dartsort/vis/recanim.py +429 -0
  182. dartsort-0.3.5/src/dartsort/vis/scatterplots.py +779 -0
  183. dartsort-0.3.5/src/dartsort/vis/sorting.py +163 -0
  184. dartsort-0.3.5/src/dartsort/vis/unit.py +917 -0
  185. dartsort-0.3.5/src/dartsort/vis/unit_comparison.py +1017 -0
  186. dartsort-0.3.5/src/dartsort/vis/versus.py +300 -0
  187. dartsort-0.3.5/src/dartsort/vis/vismain.py +537 -0
  188. dartsort-0.3.5/src/dartsort/vis/waveforms.py +357 -0
  189. dartsort-0.3.5/tests/conftest.py +90 -0
  190. dartsort-0.3.5/tests/test_alignment.py +511 -0
  191. dartsort-0.3.5/tests/test_clustering.py +249 -0
  192. dartsort-0.3.5/tests/test_config.py +24 -0
  193. dartsort-0.3.5/tests/test_dartsort.py +158 -0
  194. dartsort-0.3.5/tests/test_data_util.py +71 -0
  195. dartsort-0.3.5/tests/test_detect.py +288 -0
  196. dartsort-0.3.5/tests/test_drift_util.py +305 -0
  197. dartsort-0.3.5/tests/test_drifty_matching.py +192 -0
  198. dartsort-0.3.5/tests/test_enforce_decrease.py +16 -0
  199. dartsort-0.3.5/tests/test_grab_and_featurize.py +141 -0
  200. dartsort-0.3.5/tests/test_kmeans.py +39 -0
  201. dartsort-0.3.5/tests/test_linalg.py +83 -0
  202. dartsort-0.3.5/tests/test_localize_torch.py +94 -0
  203. dartsort-0.3.5/tests/test_matching.py +824 -0
  204. dartsort-0.3.5/tests/test_modeling.py +464 -0
  205. dartsort-0.3.5/tests/test_relocate.py +147 -0
  206. dartsort-0.3.5/tests/test_simkit.py +235 -0
  207. dartsort-0.3.5/tests/test_spiketorch.py +203 -0
  208. dartsort-0.3.5/tests/test_subtract.py +447 -0
  209. dartsort-0.3.5/tests/test_templates.py +504 -0
  210. dartsort-0.3.5/tests/test_threshold.py +9 -0
  211. dartsort-0.3.5/tests/test_torch_optimization_util.py +41 -0
  212. dartsort-0.3.5/tests/test_transform.py +160 -0
  213. dartsort-0.3.5/tests/test_universal.py +9 -0
  214. dartsort-0.3.5/tests/test_util.py +91 -0
  215. dartsort-0.3.5/tests/test_waveform_util.py +171 -0
@@ -0,0 +1,52 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+ - develop
8
+ pull_request:
9
+ branches:
10
+ - "*"
11
+ workflow_dispatch:
12
+
13
+ jobs:
14
+
15
+ build:
16
+ runs-on: "ubuntu-latest"
17
+ defaults:
18
+ run:
19
+ shell: bash -l {0}
20
+
21
+ steps:
22
+ - name: Checkout
23
+ uses: actions/checkout@v4
24
+ with:
25
+ path: dartsort
26
+
27
+ - name: Get DREDge
28
+ uses: actions/checkout@v4
29
+ with:
30
+ repository: evarol/dredge
31
+ path: dredge
32
+
33
+ - uses: actions/setup-python@v5
34
+ with:
35
+ python-version: '3.x'
36
+ cache: 'pip' # caching pip dependencies
37
+
38
+ - name: Install dependencies
39
+ run: |
40
+ python -m pip install -U pip # seems necessary to make sure pip is my pip
41
+ pip install --extra-index-url https://download.pytorch.org/whl/cpu torch
42
+ pip install dredge/
43
+ pip install -e dartsort/[full]
44
+
45
+ - name: pytest
46
+ run: |
47
+ pytest --cache-clear --log-level=15 --durations=100 --cov=dartsort/src/dartsort --cov-config=dartsort/pyproject.toml --cov-report lcov --cov-report term dartsort/tests/
48
+
49
+ - name: Coveralls
50
+ uses: coverallsapp/github-action@master
51
+ with:
52
+ path-to-lcov: coverage.lcov
@@ -0,0 +1,43 @@
1
+ name: CI (Windows)
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+
11
+ build:
12
+ runs-on: "windows-latest"
13
+ defaults:
14
+ run:
15
+ shell: bash -el {0}
16
+
17
+ steps:
18
+ - name: Checkout
19
+ uses: actions/checkout@v4
20
+ with:
21
+ path: dartsort
22
+
23
+ - name: Get DREDge
24
+ uses: actions/checkout@v4
25
+ with:
26
+ repository: evarol/dredge
27
+ path: dredge
28
+
29
+ - uses: actions/setup-python@v5
30
+ with:
31
+ python-version: '3.x'
32
+ cache: 'pip' # caching pip dependencies
33
+
34
+ - name: Install dependencies
35
+ run: |
36
+ python -m pip install -U pip # seems necessary to make sure pip is my pip
37
+ pip install --extra-index-url https://download.pytorch.org/whl/cpu torch
38
+ pip install dredge/
39
+ pip install -e dartsort/[full]
40
+
41
+ - name: pytest
42
+ run: |
43
+ PYTHONFAULTHANDLER=1 pytest --cache-clear --maxfail=30 --log-level=15 dartsort/tests/
@@ -0,0 +1,56 @@
1
+ name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+ build:
11
+ name: Build distribution 📦
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v6
16
+ with:
17
+ persist-credentials: false
18
+ - name: Set up Python
19
+ uses: actions/setup-python@v6
20
+ with:
21
+ python-version: "3.x"
22
+ - name: Install pypa/build
23
+ run: >-
24
+ python3 -m
25
+ pip install
26
+ build
27
+ --user
28
+ - name: Build a binary wheel and a source tarball
29
+ run: python3 -m build
30
+ - name: Store the distribution packages
31
+ uses: actions/upload-artifact@v5
32
+ with:
33
+ name: python-package-distributions
34
+ path: dist/
35
+
36
+ publish-to-pypi:
37
+ name: >-
38
+ Publish Python 🐍 distribution 📦 to PyPI
39
+ if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
40
+ needs:
41
+ - build
42
+ runs-on: ubuntu-latest
43
+ environment:
44
+ name: pypi
45
+ url: https://pypi.org/p/dartsort
46
+ permissions:
47
+ id-token: write # IMPORTANT: mandatory for trusted publishing
48
+
49
+ steps:
50
+ - name: Download all the dists
51
+ uses: actions/download-artifact@v6
52
+ with:
53
+ name: python-package-distributions
54
+ path: dist/
55
+ - name: Publish distribution 📦 to PyPI
56
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,170 @@
1
+ # from me
2
+ .jupyter/
3
+ **/*.ipynb
4
+ .DS_Store
5
+ data/*.h5
6
+ data/*.npy
7
+ data/spt*
8
+ data/*/
9
+ logs/*.log
10
+ **/*.c
11
+ notebook/.stan/*
12
+ figs/*
13
+ .idea
14
+ *.swp
15
+ *.sublime-workspace
16
+ *.sublime-project
17
+
18
+ ## Core latex/pdflatex auxiliary files:
19
+ *.aux
20
+ *.lof
21
+ *.log
22
+ *.lot
23
+ *.fls
24
+ *.out
25
+ *.toc
26
+ *.fmt
27
+ *.fot
28
+ *.cb
29
+ *.cb2
30
+ .*.lb
31
+
32
+ ## Bibliography auxiliary files (bibtex/biblatex/biber):
33
+ *.bbl
34
+ *.bcf
35
+ *.blg
36
+ *-blx.aux
37
+ *-blx.bib
38
+ *.run.xml
39
+
40
+ *.pdf
41
+
42
+ # Byte-compiled / optimized / DLL files
43
+ __pycache__/
44
+ *.py[cod]
45
+ *$py.class
46
+
47
+ # C extensions
48
+ *.so
49
+
50
+ # Distribution / packaging
51
+ .Python
52
+ build/
53
+ develop-eggs/
54
+ dist/
55
+ downloads/
56
+ eggs/
57
+ .eggs/
58
+ lib/
59
+ lib64/
60
+ parts/
61
+ sdist/
62
+ var/
63
+ wheels/
64
+ pip-wheel-metadata/
65
+ share/python-wheels/
66
+ *.egg-info/
67
+ .installed.cfg
68
+ *.egg
69
+ MANIFEST
70
+
71
+ # PyInstaller
72
+ # Usually these files are written by a python script from a template
73
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
74
+ *.manifest
75
+ *.spec
76
+
77
+ # Installer logs
78
+ pip-log.txt
79
+ pip-delete-this-directory.txt
80
+
81
+ # Unit test / coverage reports
82
+ htmlcov/
83
+ .tox/
84
+ .nox/
85
+ .coverage
86
+ .coverage.*
87
+ .cache
88
+ nosetests.xml
89
+ coverage.xml
90
+ *.cover
91
+ *.py,cover
92
+ .hypothesis/
93
+ .pytest_cache/
94
+
95
+ # Translations
96
+ *.mo
97
+ *.pot
98
+
99
+ # Django stuff:
100
+ *.log
101
+ local_settings.py
102
+ db.sqlite3
103
+ db.sqlite3-journal
104
+
105
+ # Flask stuff:
106
+ instance/
107
+ .webassets-cache
108
+
109
+ # Scrapy stuff:
110
+ .scrapy
111
+
112
+ # Sphinx documentation
113
+ docs/_build/
114
+
115
+ # PyBuilder
116
+ target/
117
+
118
+ # Jupyter Notebook
119
+ .ipynb_checkpoints
120
+
121
+ # IPython
122
+ profile_default/
123
+ ipython_config.py
124
+
125
+ # pyenv
126
+ .python-version
127
+
128
+ # pipenv
129
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
130
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
131
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
132
+ # install all needed dependencies.
133
+ #Pipfile.lock
134
+
135
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
136
+ __pypackages__/
137
+
138
+ # Celery stuff
139
+ celerybeat-schedule
140
+ celerybeat.pid
141
+
142
+ # SageMath parsed files
143
+ *.sage.py
144
+
145
+ # Environments
146
+ .env
147
+ .venv
148
+ env/
149
+ venv/
150
+ ENV/
151
+ env.bak/
152
+ venv.bak/
153
+
154
+ # Spyder project settings
155
+ .spyderproject
156
+ .spyproject
157
+
158
+ # Rope project settings
159
+ .ropeproject
160
+
161
+ # mkdocs documentation
162
+ /site
163
+
164
+ # mypy
165
+ .mypy_cache/
166
+ .dmypy.json
167
+ dmypy.json
168
+
169
+ # Pyre type checker
170
+ .pyre/
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.4
2
+ Name: DARTsort
3
+ Version: 0.3.5
4
+ Summary: DARTsort
5
+ Author-email: Charlie Windolf <ciw2107@columbia.edu>
6
+ Project-URL: Homepage, https://github.com/cwindolf/dartsort
7
+ Project-URL: Bug Tracker, https://github.com/cwindolf/dartsort/issues
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Operating System :: OS Independent
11
+ Requires-Python: >=3.11
12
+ Description-Content-Type: text/markdown
13
+ Requires-Dist: cupy
14
+ Requires-Dist: h5py
15
+ Requires-Dist: linear_operator
16
+ Requires-Dist: numba
17
+ Requires-Dist: numpy>=1.20; python_version < "3.13"
18
+ Requires-Dist: numpy>=2.0.0; python_version >= "3.13"
19
+ Requires-Dist: opt-einsum
20
+ Requires-Dist: pandas
21
+ Requires-Dist: packaging
22
+ Requires-Dist: probeinterface
23
+ Requires-Dist: pydantic
24
+ Requires-Dist: scipy>=1.13
25
+ Requires-Dist: scikit-learn
26
+ Requires-Dist: spikeinterface>=0.101.2
27
+ Requires-Dist: torch>=2.0
28
+ Requires-Dist: tqdm
29
+ Provides-Extra: vis
30
+ Requires-Dist: matplotlib; extra == "vis"
31
+ Requires-Dist: seaborn; extra == "vis"
32
+ Provides-Extra: dev
33
+ Requires-Dist: pytest; extra == "dev"
34
+ Requires-Dist: pytest-cov; extra == "dev"
35
+ Provides-Extra: full
36
+ Requires-Dist: matplotlib; extra == "full"
37
+ Requires-Dist: seaborn; extra == "full"
38
+ Requires-Dist: pytest; extra == "full"
39
+ Requires-Dist: pytest-cov; extra == "full"
40
+ Requires-Dist: ipywidgets; extra == "full"
41
+ Requires-Dist: ipykernel; extra == "full"
42
+ Requires-Dist: black; extra == "full"
43
+
44
+ [![badge](https://github.com/cwindolf/dartsort/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cwindolf/dartsort/actions/)
45
+ [![coveralls](https://coveralls.io/repos/github/cwindolf/dartsort/badge.svg?branch=main)](https://coveralls.io/github/cwindolf/dartsort)
46
+
47
+ # dartsort
48
+
49
+ ## :warning: Work in progress code repository
50
+
51
+ We do not currently recommend DARTsort for production spike sorting purposes. We are in the process of implementing a robust and documented pipeline in [`src/dartsort`](src/dartsort), and we will update this page accordingly.
52
+
53
+ A workflow described in our preprint (https://www.biorxiv.org/content/10.1101/2023.08.11.553023v1) is in [uhd_pipeline.py](scripts/uhd_pipeline.py), which is implemented using the legacy code in [`src/spike_psvae`](src/spike_psvae).
54
+
55
+
56
+ ## Suggested install steps
57
+
58
+ If you don't already have Python and PyTorch 2 installed, we recommend doing this with the Miniforge distribution of `conda`. You can find info and installers for your platform [at Miniforge's GitHub repository](https://github.com/conda-forge/miniforge). After installing Miniforge, `conda` will be available on your computer for installing Python packages, as well as the newer and faster conda replacement tool `mamba`. We recommend using `mamba` instead of `conda` below, since the installation tends to be a lot faster with `mamba`.
59
+
60
+ To install DARTsort, first clone this GitHub repository.
61
+
62
+ After cloning the repository, create and activate the `mamba`/`conda` environment from the configuration file provided as follows:
63
+
64
+ ```bash
65
+ $ mamba env create -f environment.yml
66
+ $ mamba activate dartsort
67
+ ```
68
+
69
+ Next, visit https://pytorch.org/get-started/locally/ and follow the `PyTorch` install instructions for your specific OS and hardware needs.
70
+ We also need to install `linear_operator` from the `gpytorch` channel.
71
+ For example, on a Linux workstation or cluster with NVIDIA GPUs available, one might use (dropping in `mamba` for `conda` commands):
72
+
73
+ ```bash
74
+ # Example -- see https://pytorch.org/get-started/locally/ to find your platform's command.
75
+ (dartsort) $ mamba install pytorch torchvision torchaudio pytorch-cuda=11.8 linear_operator -c pytorch -c nvidia -c gpytorch
76
+ ```
77
+
78
+ Finally, install the remaining `pip` dependencies and `dartsort` itself:
79
+
80
+ ```bash
81
+ (dartsort) $ pip install -r requirements-full.txt
82
+ (dartsort) $ pip install -e .
83
+ ```
84
+
85
+ To enable DARTsort's default motion correction algorithm [DREDge](https://www.biorxiv.org/content/10.1101/2023.10.24.563768), clone [its GitHub repository](https://github.com/evarol/dredge), and then `cd dredge/` and install the DREDge package with `pip install -e .`.
86
+
87
+ Soon we will have a package on PyPI so that these last steps will be just a `pip install dartsort`.
88
+
89
+ To make sure everything is working:
90
+
91
+ ```bash
92
+ $ (dartsort) pytest tests/*
93
+ ```
@@ -0,0 +1,50 @@
1
+ [![badge](https://github.com/cwindolf/dartsort/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cwindolf/dartsort/actions/)
2
+ [![coveralls](https://coveralls.io/repos/github/cwindolf/dartsort/badge.svg?branch=main)](https://coveralls.io/github/cwindolf/dartsort)
3
+
4
+ # dartsort
5
+
6
+ ## :warning: Work in progress code repository
7
+
8
+ We do not currently recommend DARTsort for production spike sorting purposes. We are in the process of implementing a robust and documented pipeline in [`src/dartsort`](src/dartsort), and we will update this page accordingly.
9
+
10
+ A workflow described in our preprint (https://www.biorxiv.org/content/10.1101/2023.08.11.553023v1) is in [uhd_pipeline.py](scripts/uhd_pipeline.py), which is implemented using the legacy code in [`src/spike_psvae`](src/spike_psvae).
11
+
12
+
13
+ ## Suggested install steps
14
+
15
+ If you don't already have Python and PyTorch 2 installed, we recommend doing this with the Miniforge distribution of `conda`. You can find info and installers for your platform [at Miniforge's GitHub repository](https://github.com/conda-forge/miniforge). After installing Miniforge, `conda` will be available on your computer for installing Python packages, as well as the newer and faster conda replacement tool `mamba`. We recommend using `mamba` instead of `conda` below, since the installation tends to be a lot faster with `mamba`.
16
+
17
+ To install DARTsort, first clone this GitHub repository.
18
+
19
+ After cloning the repository, create and activate the `mamba`/`conda` environment from the configuration file provided as follows:
20
+
21
+ ```bash
22
+ $ mamba env create -f environment.yml
23
+ $ mamba activate dartsort
24
+ ```
25
+
26
+ Next, visit https://pytorch.org/get-started/locally/ and follow the `PyTorch` install instructions for your specific OS and hardware needs.
27
+ We also need to install `linear_operator` from the `gpytorch` channel.
28
+ For example, on a Linux workstation or cluster with NVIDIA GPUs available, one might use (dropping in `mamba` for `conda` commands):
29
+
30
+ ```bash
31
+ # Example -- see https://pytorch.org/get-started/locally/ to find your platform's command.
32
+ (dartsort) $ mamba install pytorch torchvision torchaudio pytorch-cuda=11.8 linear_operator -c pytorch -c nvidia -c gpytorch
33
+ ```
34
+
35
+ Finally, install the remaining `pip` dependencies and `dartsort` itself:
36
+
37
+ ```bash
38
+ (dartsort) $ pip install -r requirements-full.txt
39
+ (dartsort) $ pip install -e .
40
+ ```
41
+
42
+ To enable DARTsort's default motion correction algorithm [DREDge](https://www.biorxiv.org/content/10.1101/2023.10.24.563768), clone [its GitHub repository](https://github.com/evarol/dredge), and then `cd dredge/` and install the DREDge package with `pip install -e .`.
43
+
44
+ Soon we will have a package on PyPI so that these last steps will be just a `pip install dartsort`.
45
+
46
+ To make sure everything is working:
47
+
48
+ ```bash
49
+ $ (dartsort) pytest tests/*
50
+ ```
@@ -0,0 +1,18 @@
1
+ ---
2
+ name: dartsort
3
+ channels:
4
+ - conda-forge
5
+ dependencies:
6
+ - python=3.11
7
+ - h5py
8
+ - matplotlib
9
+ - numpy>=1.26
10
+ - pytest
11
+ - scipy>=1.13
12
+ - scikit-learn
13
+ - tqdm
14
+ - pandas
15
+ - linear_operator
16
+ - numba
17
+ - pydantic
18
+ - opt-einsum
File without changes
@@ -0,0 +1,137 @@
1
+ # ---
2
+ # jupyter:
3
+ # jupytext:
4
+ # text_representation:
5
+ # extension: .py
6
+ # format_name: percent
7
+ # format_version: '1.3'
8
+ # jupytext_version: 1.14.5
9
+ # kernelspec:
10
+ # display_name: Python [conda env:a]
11
+ # language: python
12
+ # name: conda-env-a-py
13
+ # ---
14
+
15
+ # %%
16
+ # %load_ext autoreload
17
+ # %autoreload 2
18
+
19
+ # %%
20
+ import numpy as np
21
+ from spike_psvae import subtract
22
+ from one.api import ONE
23
+ from pathlib import Path
24
+
25
+ # %%
26
+ import spikeinterface.full as si
27
+
28
+ # %%
29
+ one = ONE()
30
+
31
+ # %%
32
+ sdsc_base_path = Path("/mnt/sdceph/users/ibl/data")
33
+
34
+
35
+ # %%
36
+ def eid2sdscpath(eid):
37
+ pids, probes = one.eid2pid(eid)
38
+ alyx_base_path = one.eid2path(eid)
39
+ paths = {}
40
+ for pid, probe in zip(pids, probes):
41
+ rel_path = one.list_datasets(eid, f"raw_ephys_data/{probe}*ap.cbin")
42
+ assert len(rel_path) == 1
43
+ rel_path = Path(rel_path[0])
44
+ searchdir = sdsc_base_path / alyx_base_path.relative_to(one.cache_dir) / rel_path.parent
45
+ pattern = Path(rel_path.name).with_suffix(f".*.cbin")
46
+ glob = list(searchdir.glob(str(pattern)))
47
+ assert len(glob) == 1
48
+ paths[probe] = pid, glob[0]
49
+ assert paths[probe][1].exists()
50
+ return paths
51
+
52
+
53
+ # %%
54
+ pid = "7d999a68-0215-4e45-8e6c-879c6ca2b771"
55
+
56
+ # %%
57
+ eid, probe = one.pid2eid(pid)
58
+ paths = eid2sdscpath(eid)
59
+ print(paths)
60
+ pid, cbin_path = paths[probe]
61
+ ch_path = list(cbin_path.parent.glob("*ap*.ch"))
62
+ assert len(ch_path) == 1
63
+ ch_path = ch_path[0]
64
+ meta_path = list(cbin_path.parent.glob("*ap*.meta"))
65
+ assert len(meta_path) == 1
66
+ meta_path = meta_path[0]
67
+
68
+ print("-" * 50)
69
+ rec = si.read_cbin_ibl(str(cbin_path.parent), cbin_file=str(cbin_path), ch_file=str(ch_path), meta_file=str(meta_path))
70
+ print(rec)
71
+ fs = int(rec.get_sampling_frequency())
72
+
73
+ # %%
74
+ rec = si.highpass_filter(rec)
75
+ rec = si.phase_shift(rec)
76
+ bad_channel_ids, channel_labels = si.detect_bad_channels(rec, num_random_chunks=100)
77
+ print(f"{bad_channel_ids=}")
78
+ rec = si.interpolate_bad_channels(rec, bad_channel_ids)
79
+ rec = si.highpass_spatial_filter(rec)
80
+ # we had been working with this before -- should switch to MAD,
81
+ # but we need to rethink the thresholds
82
+ rec = si.zscore(rec, mode="mean+std", num_chunks_per_segment=100, margin_frames=250*fs)
83
+
84
+ # %%
85
+ rec = rec.frame_slice(start_frame=rec.get_num_samples() // 2, end_frame=rec.get_num_samples() // 2 + 30 * 30000)
86
+
87
+ rec
88
+
89
+ rec_chansliced = rec.channel_slice(channel_ids=rec.channel_ids[[11, 13, 21, 65, 4, 2]])
90
+
91
+ rec_chansliced
92
+
93
+ geom = rec_chansliced.get_channel_locations()
94
+ geom
95
+
96
+ rec_ord = si.depth_order(rec_chansliced)
97
+ rec_ord
98
+
99
+ geom = rec_ord.get_channel_locations()
100
+ geom
101
+
102
+
103
+
104
+ # !rm -rf /tmp/testfolder
105
+
106
+ rec_ord.("/tmp/testfolder", n_jobs=1)
107
+
108
+ # !ls /tmp/testfolder
109
+
110
+ xcoords = geom[:, 0]
111
+ ycoords = geom[:, 1]
112
+ kcoords = np.zeros_like(geom[:, 0])
113
+ chanMap = np.arange(len(geom))
114
+
115
+ chanMap
116
+
117
+ from scipy.io import savemat
118
+
119
+ savemat("/tmp/chanMap.mat", dict(xcoords=xcoords, ycoords=ycoords, cm))
120
+
121
+ # %%
122
+ sub_h5 = subtract.subtraction(
123
+ rec,
124
+ out_folder="/tmp/test",
125
+ thresholds=[12, 10, 8, 6, 5],
126
+ n_sec_pca=10,
127
+ save_subtracted_tpca_projs=False,
128
+ save_cleaned_tpca_projs=False,
129
+ save_denoised_tpca_projs=False,
130
+ n_jobs=1,
131
+ loc_workers=1,
132
+ overwrite=False,
133
+ n_sec_chunk=1,
134
+ device="cuda:1",
135
+ )
136
+
137
+ # %%