DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,153 @@
1
+ import torch
2
+ from torch import nn
3
+
4
+ from ..util.py_util import resolve_path
5
+ from ..util.waveform_util import get_channels_in_probe, set_channels_in_probe
6
+ from .transform_base import BaseWaveformDenoiser
7
+
8
+ try:
9
+ from importlib.resources import files
10
+ except ImportError:
11
+ try:
12
+ from importlib_resources import files # type: ignore
13
+ except ImportError:
14
+ raise ValueError("Need python>=3.10 or pip install importlib_resources.")
15
+
16
+ default_pretrained_path = files("dartsort.pretrained")
17
+ default_pretrained_path = default_pretrained_path.joinpath("single_chan_denoiser.pt")
18
+
19
+
20
+ class SingleChannelWaveformDenoiser(BaseWaveformDenoiser):
21
+ default_name = "single_chan_denoiser"
22
+
23
+ def __init__(
24
+ self,
25
+ channel_index,
26
+ geom=None,
27
+ denoiser=None,
28
+ batch_size=1024,
29
+ in_place=False,
30
+ name=None,
31
+ name_prefix="",
32
+ n_epochs=None,
33
+ epoch_size=None,
34
+ ):
35
+ super().__init__(
36
+ channel_index=channel_index, name=name, name_prefix=name_prefix
37
+ )
38
+ self.batch_size = batch_size
39
+ self.in_place = in_place
40
+ if denoiser is None:
41
+ denoiser = SingleChannelDenoiser().load(default_pretrained_path)
42
+ self.denoiser = denoiser
43
+
44
+ @property
45
+ def device(self):
46
+ return self.denoiser.conv1[0].weight.device
47
+
48
+ def forward(self, waveforms, *, channels, **unused):
49
+ odev = waveforms.device
50
+ channels_in_probe, waveforms_in_probe = get_channels_in_probe(
51
+ waveforms, channels, self.channel_index.to(odev)
52
+ )
53
+
54
+ n_in_probe = len(waveforms_in_probe)
55
+ dev = self.device
56
+ for batch_start in range(0, n_in_probe, self.batch_size):
57
+ batch_end = min(batch_start + self.batch_size, n_in_probe)
58
+ waveforms_in_probe[batch_start:batch_end] = self.denoiser(
59
+ waveforms_in_probe[batch_start:batch_end].to(dev)
60
+ ).to(odev)
61
+
62
+ waveforms = set_channels_in_probe(
63
+ waveforms_in_probe,
64
+ waveforms,
65
+ channels_in_probe,
66
+ in_place=self.in_place,
67
+ )
68
+
69
+ return waveforms
70
+
71
+ @classmethod
72
+ def load_from_pt(
73
+ cls,
74
+ pretrained_path=default_pretrained_path,
75
+ clsname="SingleChannelDenoiser",
76
+ **kwargs,
77
+ ):
78
+ denoiser = dnclss[clsname]().load(pretrained_path)
79
+ return cls(denoiser=denoiser, **kwargs)
80
+
81
+
82
+ class SingleChannelDenoiser(nn.Module):
83
+ def __init__(
84
+ self,
85
+ n_filters=[16, 8],
86
+ filter_sizes=[5, 11],
87
+ spike_size=121,
88
+ ):
89
+ super().__init__()
90
+ self.conv1 = nn.Sequential(
91
+ nn.Conv1d(1, n_filters[0], filter_sizes[0]), nn.ReLU()
92
+ )
93
+ self.conv2 = nn.Sequential(
94
+ nn.Conv1d(n_filters[0], n_filters[1], filter_sizes[1]), nn.ReLU()
95
+ )
96
+ n_input_feat = n_filters[1] * (
97
+ spike_size - filter_sizes[0] - filter_sizes[1] + 2
98
+ )
99
+ self.out = nn.Linear(n_input_feat, spike_size)
100
+
101
+ def forward(self, x):
102
+ x = x[:, None]
103
+ x = self.conv1(x)
104
+ x = self.conv2(x)
105
+ x = x.view(x.shape[0], -1)
106
+ return self.out(x)
107
+
108
+ def load(self, pretrained_path=default_pretrained_path):
109
+ pretrained_path = resolve_path(pretrained_path)
110
+ checkpoint = torch.load(pretrained_path, map_location="cpu", weights_only=True)
111
+ self.load_state_dict(checkpoint)
112
+ self.eval()
113
+ self.requires_grad_(False)
114
+ return self
115
+
116
+
117
+ class FlexibleSingleChanDenoiser(nn.Module):
118
+ def __init__(
119
+ self,
120
+ pretrained_path=None,
121
+ n_filters=[32, 32, 32],
122
+ filter_sizes=[11, 11, 11],
123
+ spike_size=121,
124
+ ):
125
+ super().__init__()
126
+ nets = []
127
+ for inf, outf, s in zip([1, *n_filters], n_filters, filter_sizes):
128
+ nets.append(nn.Sequential(nn.Conv1d(inf, outf, s), nn.ReLU()))
129
+ # self.conv1 = nn.Sequential(nn.Conv1d(1, feat1, size1), nn.ReLU())
130
+ # self.conv2 = nn.Sequential(nn.Conv1d(feat1, feat2, size2), nn.ReLU())
131
+ self.conv = nn.Sequential(*nets)
132
+ n_input_feat = n_filters[-1] * (
133
+ spike_size - sum(filter_sizes) + len(filter_sizes)
134
+ )
135
+ self.out = nn.Linear(n_input_feat, spike_size)
136
+
137
+ def forward(self, x):
138
+ x = x[:, None]
139
+ x = self.conv(x)
140
+ # x = self.conv2(x)
141
+ x = x.view(x.shape[0], -1)
142
+ return self.out(x)
143
+
144
+ def load(self, pretrained_path):
145
+ checkpoint = torch.load(pretrained_path, map_location="cpu")
146
+ self.load_state_dict(checkpoint)
147
+ return self
148
+
149
+
150
+ dnclss = {
151
+ "FlexibleSingleChanDenoiser": FlexibleSingleChanDenoiser,
152
+ "SingleChannelDenoiser": SingleChannelDenoiser,
153
+ }
@@ -0,0 +1,98 @@
1
+ import numpy as np
2
+ import torch
3
+
4
+ from dartsort.util import waveform_util
5
+ from dartsort.util.universal_util import get_singlechan_centroids
6
+
7
+ from .transform_base import BaseWaveformModule
8
+
9
+
10
+ class SingleChannelTemplates(BaseWaveformModule):
11
+ """Utility class for computing single channel templates"""
12
+
13
+ default_name = "singlechan_templates"
14
+
15
+ def __init__(
16
+ self,
17
+ channel_index,
18
+ geom=None,
19
+ n_centroids=10,
20
+ pca_rank=8,
21
+ trough_offset_samples=42,
22
+ dtype=torch.float,
23
+ alignment_padding=20,
24
+ random_state=0,
25
+ kmeanspp_initial="random",
26
+ taper=True,
27
+ name=None,
28
+ name_prefix=None,
29
+ max_waveforms=None,
30
+ ):
31
+ super().__init__(
32
+ channel_index=channel_index, name=name, name_prefix=name_prefix
33
+ )
34
+
35
+ # main control
36
+ self.n_centroids = n_centroids
37
+ self.alignment_padding = alignment_padding
38
+ self.taper = taper
39
+ self.pca_rank = pca_rank
40
+ self.max_waveforms = max_waveforms
41
+
42
+ # input waveform details
43
+ self.channel_index = channel_index
44
+ self.trough_offset_samples = trough_offset_samples
45
+ self.alignment_padding = alignment_padding
46
+
47
+ # gizmos
48
+ self.random_state = random_state
49
+ self.kmeanspp_initial = kmeanspp_initial
50
+ self._needs_fit = True
51
+
52
+ def needs_fit(self):
53
+ return self._needs_fit
54
+
55
+ def initialize_spike_length_dependent_params(self):
56
+ assert self.spike_length_samples is not None
57
+ self.template_trough = self.trough_offset_samples - self.alignment_padding
58
+ self.template_length = self.spike_length_samples - 2 * self.alignment_padding
59
+ self.register_buffer(
60
+ "templates", torch.zeros((self.n_centroids, self.template_length))
61
+ )
62
+ self.to(self.channel_index.device)
63
+
64
+ def fit(self, recording, waveforms, *, channels, weights=None, **unused):
65
+ super().fit(recording, waveforms, channels=channels, weights=weights)
66
+ if weights is not None:
67
+ rg = np.random.default_rng(self.random_state)
68
+ weights = weights.numpy(force=True) if torch.is_tensor(weights) else weights
69
+ weights = weights.astype(np.float64)
70
+ weights = weights / weights.sum()
71
+ choices = rg.choice(len(weights), p=weights, size=self.max_waveforms)
72
+ choices = np.atleast_1d(choices)
73
+ choices.sort()
74
+ choices = torch.from_numpy(choices)
75
+ waveforms = waveforms[choices]
76
+ channels = channels[choices]
77
+ singlechan_waveforms = waveform_util.grab_main_channels_torch(
78
+ waveforms, channels, self.channel_index.to(device=waveforms.device)
79
+ )
80
+ assert singlechan_waveforms.ndim == 3
81
+ assert singlechan_waveforms.shape[2] == 1
82
+ singlechan_waveforms = singlechan_waveforms[:, :, 0]
83
+ singlechan_waveforms = singlechan_waveforms.to(self.b.channel_index.device)
84
+ templates = get_singlechan_centroids(
85
+ singlechan_waveforms=singlechan_waveforms,
86
+ trough_offset_samples=self.template_trough,
87
+ spike_length_samples=self.template_length,
88
+ alignment_padding=self.alignment_padding,
89
+ n_centroids=self.n_centroids,
90
+ pca_rank=self.pca_rank,
91
+ taper=self.taper,
92
+ taper_start=self.alignment_padding // 2,
93
+ taper_end=self.alignment_padding,
94
+ kmeanspp_initial=self.kmeanspp_initial,
95
+ random_seed=self.random_state,
96
+ )
97
+ self.b.templates[:] = templates
98
+ self._needs_fit = False