DARTsort 0.3.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. dartsort/__init__.py +68 -0
  2. dartsort/cli.py +105 -0
  3. dartsort/clustering/__init__.py +3 -0
  4. dartsort/clustering/cluster_util.py +392 -0
  5. dartsort/clustering/clustering.py +765 -0
  6. dartsort/clustering/clustering_features.py +202 -0
  7. dartsort/clustering/density.py +1299 -0
  8. dartsort/clustering/forward_backward.py +219 -0
  9. dartsort/clustering/gmm/mixture.py +5790 -0
  10. dartsort/clustering/gmm/stable_features.py +993 -0
  11. dartsort/clustering/kmeans.py +448 -0
  12. dartsort/clustering/merge.py +501 -0
  13. dartsort/clustering/modes.py +214 -0
  14. dartsort/clustering/refine_util.py +191 -0
  15. dartsort/clustering/relocate.py +109 -0
  16. dartsort/clustering/split.py +1285 -0
  17. dartsort/config.py +288 -0
  18. dartsort/detect/__init__.py +2 -0
  19. dartsort/detect/detect.py +449 -0
  20. dartsort/detect/detect_filters.py +37 -0
  21. dartsort/evaluate/__init__.py +1 -0
  22. dartsort/evaluate/analysis.py +594 -0
  23. dartsort/evaluate/comparison.py +534 -0
  24. dartsort/evaluate/hybrid_util.py +599 -0
  25. dartsort/evaluate/noise_recording_tools.py +184 -0
  26. dartsort/evaluate/sim_template_tools.py +842 -0
  27. dartsort/evaluate/simkit.py +840 -0
  28. dartsort/evaluate/simlib.py +270 -0
  29. dartsort/localize/__init__.py +0 -0
  30. dartsort/localize/localize_torch.py +291 -0
  31. dartsort/localize/localize_util.py +313 -0
  32. dartsort/main.py +657 -0
  33. dartsort/peel/__init__.py +6 -0
  34. dartsort/peel/grab.py +208 -0
  35. dartsort/peel/matching.py +628 -0
  36. dartsort/peel/matching_util/__init__.py +3 -0
  37. dartsort/peel/matching_util/compressed_upsampled.py +718 -0
  38. dartsort/peel/matching_util/drifty.py +791 -0
  39. dartsort/peel/matching_util/matching_base.py +616 -0
  40. dartsort/peel/matching_util/pairwise.py +359 -0
  41. dartsort/peel/matching_util/pairwise_util.py +1317 -0
  42. dartsort/peel/peel_base.py +1000 -0
  43. dartsort/peel/running_template.py +1498 -0
  44. dartsort/peel/subtract.py +1123 -0
  45. dartsort/peel/threshold.py +471 -0
  46. dartsort/peel/universal.py +191 -0
  47. dartsort/pretrained/default_temporal_kernel.npy +0 -0
  48. dartsort/pretrained/detect_np1.pt +0 -0
  49. dartsort/pretrained/detect_np2.pt +0 -0
  50. dartsort/pretrained/glasbey1024.npz +0 -0
  51. dartsort/pretrained/offset_trained_denoiser.pt +0 -0
  52. dartsort/pretrained/offset_trained_denoiser_added_temp.pt +0 -0
  53. dartsort/pretrained/offset_trained_denoiser_added_temp_boost.pt +0 -0
  54. dartsort/pretrained/original_single_chan_denoiser.pt +0 -0
  55. dartsort/pretrained/single_chan_denoiser.pt +0 -0
  56. dartsort/templates/__init__.py +11 -0
  57. dartsort/templates/get_templates.py +771 -0
  58. dartsort/templates/postprocess_util.py +555 -0
  59. dartsort/templates/realignment.py +466 -0
  60. dartsort/templates/superres_util.py +204 -0
  61. dartsort/templates/template_util.py +518 -0
  62. dartsort/templates/templates.py +528 -0
  63. dartsort/transform/__init__.py +2 -0
  64. dartsort/transform/_multichan_denoiser_kit.py +650 -0
  65. dartsort/transform/all_transformers.py +58 -0
  66. dartsort/transform/amortized_localization.py +503 -0
  67. dartsort/transform/amplitudes.py +197 -0
  68. dartsort/transform/decollider.py +635 -0
  69. dartsort/transform/enforce_decrease.py +158 -0
  70. dartsort/transform/localize.py +86 -0
  71. dartsort/transform/matching_denoiser.py +44 -0
  72. dartsort/transform/pipeline.py +391 -0
  73. dartsort/transform/single_channel_denoiser.py +153 -0
  74. dartsort/transform/single_channel_templates.py +98 -0
  75. dartsort/transform/supervised_denoiser.py +486 -0
  76. dartsort/transform/temporal_pca.py +413 -0
  77. dartsort/transform/transform_base.py +255 -0
  78. dartsort/util/__init__.py +0 -0
  79. dartsort/util/cli_util.py +179 -0
  80. dartsort/util/data_util.py +1128 -0
  81. dartsort/util/drift_util.py +1054 -0
  82. dartsort/util/internal_config.py +1074 -0
  83. dartsort/util/interpolation_util.py +1012 -0
  84. dartsort/util/job_util.py +19 -0
  85. dartsort/util/logging_util.py +91 -0
  86. dartsort/util/main_util.py +235 -0
  87. dartsort/util/more_operators.py +269 -0
  88. dartsort/util/multiprocessing_util.py +211 -0
  89. dartsort/util/nn_util.py +461 -0
  90. dartsort/util/noise_util.py +1461 -0
  91. dartsort/util/peel_util.py +223 -0
  92. dartsort/util/py_util.py +213 -0
  93. dartsort/util/registration_util.py +128 -0
  94. dartsort/util/residual_util.py +30 -0
  95. dartsort/util/sparse_util.py +81 -0
  96. dartsort/util/spikeio.py +227 -0
  97. dartsort/util/spiketorch.py +1198 -0
  98. dartsort/util/testing_util/__init__.py +0 -0
  99. dartsort/util/testing_util/matching_debug_util.py +517 -0
  100. dartsort/util/testing_util/mixture_testing_util.py +291 -0
  101. dartsort/util/torch_optimization_util.py +156 -0
  102. dartsort/util/torch_util.py +75 -0
  103. dartsort/util/universal_util.py +269 -0
  104. dartsort/util/waveform_util.py +809 -0
  105. dartsort/vis/__init__.py +19 -0
  106. dartsort/vis/analysis_plots.py +392 -0
  107. dartsort/vis/colors.py +36 -0
  108. dartsort/vis/gt.py +392 -0
  109. dartsort/vis/layout.py +149 -0
  110. dartsort/vis/metrics_util.py +23 -0
  111. dartsort/vis/mixture.py +1489 -0
  112. dartsort/vis/over_time.py +514 -0
  113. dartsort/vis/recanim.py +429 -0
  114. dartsort/vis/scatterplots.py +779 -0
  115. dartsort/vis/sorting.py +163 -0
  116. dartsort/vis/unit.py +917 -0
  117. dartsort/vis/unit_comparison.py +1017 -0
  118. dartsort/vis/versus.py +300 -0
  119. dartsort/vis/vismain.py +537 -0
  120. dartsort/vis/waveforms.py +357 -0
  121. dartsort-0.3.5.dist-info/METADATA +93 -0
  122. dartsort-0.3.5.dist-info/RECORD +125 -0
  123. dartsort-0.3.5.dist-info/WHEEL +5 -0
  124. dartsort-0.3.5.dist-info/entry_points.txt +2 -0
  125. dartsort-0.3.5.dist-info/top_level.txt +1 -0
@@ -0,0 +1,158 @@
1
+ import numpy as np
2
+ import torch
3
+ import torch.nn.functional as F
4
+ from dartsort.util import spiketorch
5
+ from scipy.spatial.distance import cdist
6
+
7
+ from .transform_base import BaseWaveformDenoiser
8
+
9
+
10
+ class EnforceDecrease(BaseWaveformDenoiser):
11
+ """A torch module for enforcing spatial decrease of amplitudes
12
+
13
+ Calling an instance of this class on a N,T,C batch of waveforms
14
+ will return a result whose peak-to-peak amplitudes decrease
15
+ as you move away from the detection channel.
16
+ """
17
+
18
+ default_name = "enforce_decrease"
19
+
20
+ def __init__(
21
+ self,
22
+ channel_index,
23
+ geom,
24
+ batch_size=1024,
25
+ name=None,
26
+ name_prefix="",
27
+ ):
28
+ super().__init__(
29
+ name=name,
30
+ name_prefix=name_prefix,
31
+ geom=geom,
32
+ channel_index=channel_index,
33
+ )
34
+ self.batch_size = batch_size
35
+ self.n_channels = self.b.geom.shape[0]
36
+
37
+ def needs_precompute(self):
38
+ return self.n_channels > 1 and not hasattr(self, "parents_index")
39
+
40
+ def precompute(self):
41
+ self.register_buffer(
42
+ "parents_index",
43
+ torch.tensor(
44
+ make_parents_index(
45
+ self.b.geom.numpy(force=True),
46
+ self.b.channel_index.numpy(force=True),
47
+ )
48
+ ),
49
+ persistent=False,
50
+ )
51
+
52
+ def forward(self, waveforms, *, channels, **unused):
53
+ """
54
+ enfdec = EnforceDecrease(geom, channel_index)
55
+ ...
56
+ dec_wfs, dec_ptps = enfdec(waveforms, maxchans)
57
+ """
58
+ if self.n_channels == 1:
59
+ return waveforms
60
+ n = waveforms.shape[0]
61
+ assert (n,) == channels.shape
62
+ assert waveforms.shape[2] == self.b.parents_index.shape[1]
63
+
64
+ # get peak to peak amplitudes -- (N, c) shaped
65
+ ptps = spiketorch.ptp(waveforms)
66
+ ptps = ptps.to(self.b.parents_index.device)
67
+
68
+ # pad with an extra channel to support indexing tricks
69
+ pad_ptps = F.pad(ptps, (0, 1), value=torch.inf)
70
+ parent_min_ptps = torch.zeros_like(ptps)
71
+ # get amplitudes of all parents for all channels -- (N, c, <=c-1)
72
+ # this is a gather_reduce, and it would be nice if torch had one.
73
+ # batching the following:
74
+ # parent_ptps = pad_ptps[
75
+ # torch.arange(n)[:, None, None],
76
+ # self.parents_index[channels],
77
+ # ]
78
+ # parent_min_ptps = parent_ptps.min(dim=2).values
79
+ for bs in range(0, n, self.batch_size):
80
+ be = min(n, bs + self.batch_size)
81
+ parent_ptps = pad_ptps[
82
+ torch.arange(bs, be)[:, None, None],
83
+ self.b.parents_index[channels[bs:be]],
84
+ ]
85
+ torch.amin(parent_ptps, dim=2, out=parent_min_ptps[bs:be])
86
+
87
+ # what would we need to multiply by to ensure my amp is <= all parents?
88
+ rescaling = parent_min_ptps.div_(ptps).clamp_(max=1.0)
89
+ rescaling = rescaling.to(waveforms.device)
90
+ decreasing_waveforms = waveforms.mul_(rescaling[:, None, :])
91
+ return decreasing_waveforms
92
+
93
+
94
+ def make_parents_index(geom, channel_index):
95
+ """Indexing helper for EnforceDecrease
96
+
97
+ Waveforms extracted according to channel_index enter into the enforce
98
+ decrease routine, and for each channel A in the waveform we want to find
99
+ the maximum amplitude of any channel which is A's ancestor in the tree
100
+ rooted at the detection channel. This function computes an indexing
101
+ structure to help with that.
102
+
103
+ If channel_index is (C, c)-shaped, where C is the full number of channels
104
+ and c is the number of channels extracted per waverform, then this function
105
+ returns a (C, c, <=c-1)-shaped array parents_index, where parents_index[a, b]
106
+ contains the (at most) c-1 parents (relatively indexed) of channel b
107
+ (relatively indexed) in the tree rooted at detection channel a.
108
+
109
+ For each site A, another site B is considered a parent relative to detection
110
+ channel C if it is inside the square whose diagonal is AC, which is tested
111
+ by <ACB < 45 deg and <CBA > 90 deg.
112
+
113
+ Here, relatively indexed means in the 0:c index space rather than the 0:C
114
+ index space.
115
+ """
116
+ C, c = channel_index.shape
117
+ assert geom.ndim == 2 and geom.shape[0] == C
118
+
119
+ parents_index = np.full((C, c, c - 1), c)
120
+ geom_pad = np.pad(geom, [(0, 1), (0, 0)], constant_values=np.nan)
121
+ for i in range(len(geom)):
122
+ g = geom_pad[channel_index[i]]
123
+ i_rel = np.flatnonzero(channel_index[i] == i)
124
+ assert i_rel.size == 1
125
+ i_rel = i_rel[0]
126
+
127
+ # detection channel = C
128
+ # candidate parent = B
129
+ # leaf = A
130
+ g_detect = geom[i][None]
131
+ g_rel = g - g_detect
132
+
133
+ # compute all angles ACB
134
+ cos_across_detect = 1.0 - cdist(g_rel, g_rel, metric="cosine")
135
+ cos_across_detect[i_rel, :] = cos_across_detect[:, i_rel] = 1
136
+
137
+ # compute all angles CBA
138
+ # these are segments BA (parent is on the second axis)
139
+ # child,parent
140
+ g_par_to_child = g[:, None] - g[None, :]
141
+ # segments BC (parent on first+only axis)
142
+ g_par_to_detect = g_detect - g
143
+ cos_across_parent = np.sum(g_par_to_detect[None] * g_par_to_child, axis=2)
144
+ cos_across_parent[:, i_rel] = -1
145
+ # don't need to normalize this one since we just care about sign of cosine
146
+
147
+ # check valid -- child on first axis, candidate parent on second
148
+ is_parent = cos_across_detect >= np.cos(np.pi / 4)
149
+ is_parent &= cos_across_parent < 0
150
+ for j in range(c):
151
+ my_parents = np.setdiff1d(np.flatnonzero(is_parent[j]), [j])
152
+ parents_index[i, j, : my_parents.size] = my_parents
153
+
154
+ # trim to largest number of parents
155
+ biggest = (parents_index < c).sum(axis=2).max()
156
+ parents_index = parents_index[:, :, :biggest]
157
+
158
+ return parents_index
@@ -0,0 +1,86 @@
1
+ from typing import cast
2
+
3
+ import torch
4
+
5
+ from dartsort.localize.localize_torch import localize_amplitude_vectors
6
+ from dartsort.util.spiketorch import ptp
7
+
8
+ from .transform_base import BaseWaveformFeaturizer
9
+
10
+
11
+ class Localization(BaseWaveformFeaturizer):
12
+ """Order of output columns: x, y, z_abs, alpha"""
13
+
14
+ default_name = "point_source_localizations"
15
+ shape = (4,)
16
+ dtype = torch.double
17
+
18
+ def __init__(
19
+ self,
20
+ channel_index,
21
+ geom,
22
+ radius=None,
23
+ n_channels_subset=None,
24
+ logbarrier=True,
25
+ amplitude_kind="peak",
26
+ localization_model="pointsource",
27
+ name=None,
28
+ name_prefix="",
29
+ batch_size=1024,
30
+ ):
31
+ assert amplitude_kind in ("peak", "ptp")
32
+ super().__init__(
33
+ geom=geom,
34
+ channel_index=channel_index,
35
+ name=name,
36
+ name_prefix=name_prefix,
37
+ )
38
+ self.amplitude_kind = amplitude_kind
39
+ self.radius = radius
40
+ self.n_channels_subset = n_channels_subset
41
+ self.logbarrier = logbarrier
42
+ self.localization_model = localization_model
43
+ self.batch_size = batch_size
44
+
45
+ def transform(self, waveforms, *, channels, **unused):
46
+ # get amplitude vectors
47
+ if self.amplitude_kind == "peak":
48
+ ampvecs = waveforms.abs().max(dim=1).values
49
+ elif self.amplitude_kind == "ptp":
50
+ ampvecs = ptp(waveforms, dim=1)
51
+ else:
52
+ assert False
53
+
54
+ n = len(ampvecs)
55
+ localizations = ampvecs.new_empty((n, 4), dtype=self.dtype)
56
+
57
+ with torch.enable_grad():
58
+ for batch_start in range(0, n, self.batch_size):
59
+ batch_end = min(n, batch_start + self.batch_size)
60
+ sl = slice(batch_start, batch_end)
61
+ loc_result = localize_amplitude_vectors(
62
+ ampvecs[sl],
63
+ self.geom,
64
+ channels[sl],
65
+ channel_index=self.channel_index,
66
+ radius=self.radius,
67
+ n_channels_subset=self.n_channels_subset,
68
+ logbarrier=self.logbarrier,
69
+ dtype=cast(torch.dtype, self.dtype),
70
+ model=self.localization_model,
71
+ )
72
+ localizations[sl, 0] = loc_result["x"]
73
+ if "y" in loc_result:
74
+ localizations[sl, 1] = loc_result["y"]
75
+ else:
76
+ localizations[sl, 1] = 0.0
77
+ localizations[sl, 2] = loc_result["z_abs"]
78
+ if "alpha" in loc_result:
79
+ localizations[sl, 3] = loc_result["alpha"]
80
+ else:
81
+ localizations[sl, 3] = 0.0
82
+
83
+ return {self.name: localizations}
84
+
85
+
86
+ PointSourceLocalization = Localization
@@ -0,0 +1,44 @@
1
+ import torch
2
+
3
+ from ..util.waveform_util import get_channels_in_probe, set_channels_in_probe
4
+ from .transform_base import BaseWaveformDenoiser
5
+
6
+
7
+ class DebugMatchingPursuitDenoiser(BaseWaveformDenoiser):
8
+ """This denoiser is used for testing purposes only."""
9
+
10
+ default_name = "pursuit_basis"
11
+
12
+ def __init__(self, channel_index, geom=None, basis=None, name=None, name_prefix=""):
13
+ super().__init__(
14
+ channel_index=channel_index, name=name, name_prefix=name_prefix
15
+ )
16
+ if basis is not None:
17
+ assert basis.ndim == 2
18
+ self.register_buffer("basis", basis)
19
+
20
+ def initialize_spike_length_dependent_params(self):
21
+ if not hasattr(self, "basis"):
22
+ # this path is only hit by test_transform
23
+ zbuf = torch.zeros((1, self.spike_length_samples)) # type: ignore
24
+ self.register_buffer("basis", zbuf)
25
+
26
+ def needs_fit(self) -> bool:
27
+ return not hasattr(self, "basis")
28
+
29
+ def _project_in_probe(self, waveforms_in_probe):
30
+ dev = waveforms_in_probe.device
31
+ assert self.basis is not None
32
+ dots = waveforms_in_probe.to(device=self.b.basis.device) @ self.basis.T
33
+ best = dots.abs().argmax(dim=1)
34
+ dots = dots.take_along_dim(best[:, None], dim=1)[:, 0]
35
+ denoised = self.b.basis.take_along_dim(best[:, None], dim=0)
36
+ denoised = denoised.mul_(dots.sign()[:, None])
37
+ return denoised.to(device=dev)
38
+
39
+ def forward(self, waveforms, *, channels, **unused):
40
+ channels_in_probe, waveforms_in_probe = get_channels_in_probe(
41
+ waveforms, channels, self.channel_index
42
+ )
43
+ waveforms_in_probe = self._project_in_probe(waveforms_in_probe)
44
+ return set_channels_in_probe(waveforms_in_probe, waveforms, channels_in_probe)
@@ -0,0 +1,391 @@
1
+ """A class which manages pipelines of denoisers and featurizers"""
2
+
3
+ import torch
4
+
5
+ from .transform_base import BaseWaveformModule, BaseWaveformFeaturizer
6
+ from ..util.data_util import SpikeDataset
7
+ from ..util.internal_config import WaveformConfig, FeaturizationConfig
8
+
9
+
10
+ class WaveformPipeline(torch.nn.Module):
11
+ def __init__(self, transformers: list[BaseWaveformModule], kwargs_to_store=None):
12
+ super().__init__()
13
+ check_unique_feature_names(transformers)
14
+ self.transformers: list[BaseWaveformModule] = torch.nn.ModuleList(transformers) # type: ignore
15
+ self.kwargs_to_store = kwargs_to_store
16
+
17
+ def __len__(self):
18
+ return len(self.transformers)
19
+
20
+ def __bool__(self):
21
+ # if I am False, I return empty dicts always.
22
+ return bool(len(self.transformers))
23
+
24
+ def get_extra_state(self):
25
+ if self.kwargs_to_store is None:
26
+ return {}
27
+ return dict(class_names_and_kwargs=self.kwargs_to_store)
28
+
29
+ def set_extra_state(self, state):
30
+ # needed so that load_state_dict doesn't complain if there's
31
+ # extra state in there.
32
+ pass
33
+
34
+ def spike_datasets(self) -> list[SpikeDataset]:
35
+ datasets = []
36
+ for transformer in self.transformers:
37
+ if transformer.is_featurizer:
38
+ assert isinstance(transformer, BaseWaveformFeaturizer)
39
+ datasets.extend(transformer.spike_datasets)
40
+ return datasets
41
+
42
+ @classmethod
43
+ def from_state_dict_pt(cls, geom, channel_index, state_dict_pt):
44
+ state_dict = torch.load(state_dict_pt)
45
+ extra_state = state_dict.get("_extra_state", {})
46
+ class_names_and_kwargs = extra_state.get("class_names_and_kwargs")
47
+ if class_names_and_kwargs is None:
48
+ raise ValueError(
49
+ "Can't load a featurization pipeline from state dict if it "
50
+ "wasn't initially created with from_config() or "
51
+ "from_class_names_and_kwargs(). Instead, you can instantiate "
52
+ "it the same way you originally did and use .load_state_dict() "
53
+ "directly. (You may want to just use torch.save() and load()!)"
54
+ )
55
+ self = cls.from_class_names_and_kwargs(
56
+ geom, channel_index, class_names_and_kwargs
57
+ )
58
+ self.precompute()
59
+ # strict=False is needed here, because some transformers don't have
60
+ # parameters initialized until their pre-load hooks are called, and
61
+ # the strict check happens before that...
62
+ self.load_state_dict(state_dict, strict=False)
63
+ return self
64
+
65
+ @classmethod
66
+ def from_class_names_and_kwargs(cls, geom, channel_index, class_names_and_kwargs):
67
+ from .all_transformers import transformers_by_class_name
68
+
69
+ channel_index = torch.as_tensor(channel_index)
70
+ geom = torch.as_tensor(geom)
71
+ probe_kw = dict(channel_index=channel_index, geom=geom)
72
+
73
+ transformers = []
74
+ for name, kwargs in class_names_and_kwargs:
75
+ transformer_cls = transformers_by_class_name[name]
76
+ if kwargs.get("pretrained_path") is not None:
77
+ transformer = transformer_cls.load_from_pt(**probe_kw, **kwargs)
78
+ else:
79
+ assert kwargs.pop("pretrained_path", None) is None
80
+ transformer = transformer_cls(**probe_kw, **kwargs)
81
+ transformers.append(transformer)
82
+
83
+ return cls(transformers, kwargs_to_store=class_names_and_kwargs)
84
+
85
+ @classmethod
86
+ def from_config(
87
+ cls,
88
+ featurization_cfg,
89
+ waveform_cfg,
90
+ recording=None,
91
+ geom=None,
92
+ channel_index=None,
93
+ sampling_frequency: int | float = 30_000,
94
+ ):
95
+ if geom is None:
96
+ from dartsort.util.waveform_util import make_channel_index
97
+
98
+ assert recording is not None
99
+ sampling_frequency = recording.sampling_frequency
100
+ geom = torch.tensor(recording.get_channel_locations())
101
+ channel_index = make_channel_index(
102
+ geom, featurization_cfg.extract_radius, to_torch=True
103
+ )
104
+ else:
105
+ assert recording is None
106
+ assert channel_index is not None
107
+ args = featurization_config_to_class_names_and_kwargs(
108
+ featurization_cfg, waveform_cfg, sampling_frequency=sampling_frequency
109
+ )
110
+ return cls.from_class_names_and_kwargs(geom, channel_index, args)
111
+
112
+ def needs_precompute(self):
113
+ return any(t.needs_precompute() for t in self.transformers)
114
+
115
+ def needs_fit(self):
116
+ return any(t.needs_fit() for t in self.transformers)
117
+
118
+ def forward(self, waveforms, **fixed_properties):
119
+ """
120
+ fixed_properties usually contains max_channels, and may contain other relevant
121
+ unchanging aspects of spikes like weights (used in fit()) or temporal shifts
122
+ (used in pca slice logic).
123
+ """
124
+ waveforms = torch.asarray(waveforms)
125
+ fixed_properties = {k: torch.asarray(v) for k, v in fixed_properties.items()}
126
+ assert waveforms.ndim == 3
127
+ for v in fixed_properties.values():
128
+ assert v.shape[0] == waveforms.shape[0]
129
+
130
+ features = {}
131
+
132
+ if not waveforms.shape[0]:
133
+ return waveforms, features
134
+
135
+ for transformer in self.transformers:
136
+ if transformer.is_featurizer and transformer.is_denoiser:
137
+ waveforms, new_features = transformer(waveforms, **fixed_properties)
138
+ features.update(new_features)
139
+ elif transformer.is_featurizer:
140
+ assert isinstance(transformer, BaseWaveformFeaturizer)
141
+ features.update(transformer.transform(waveforms, **fixed_properties))
142
+ elif transformer.is_denoiser:
143
+ waveforms = transformer(waveforms, **fixed_properties)
144
+
145
+ return waveforms, features
146
+
147
+ def fit(self, recording, waveforms, **fixed_properties):
148
+ waveforms = torch.asarray(waveforms)
149
+ fixed_properties = {k: torch.asarray(v) for k, v in fixed_properties.items()}
150
+ assert waveforms.ndim == 3
151
+ for v in fixed_properties.values():
152
+ assert v.shape[0] == waveforms.shape[0]
153
+
154
+ if not self.needs_fit():
155
+ return
156
+
157
+ for transformer in self.transformers:
158
+ if transformer.needs_fit():
159
+ transformer.train()
160
+ transformer.fit(
161
+ recording=recording, waveforms=waveforms, **fixed_properties
162
+ )
163
+ transformer.eval()
164
+ transformer.requires_grad_(False)
165
+
166
+ # if we're done already, stop before denoising
167
+ if not self.needs_fit():
168
+ break
169
+
170
+ if transformer.is_denoiser:
171
+ waveforms = transformer(waveforms, **fixed_properties)
172
+ if transformer.is_featurizer:
173
+ # result is tuple wfs, feats
174
+ waveforms = waveforms[0]
175
+ assert not waveforms.requires_grad
176
+
177
+ def precompute(self):
178
+ for transformer in self.transformers:
179
+ transformer.precompute()
180
+
181
+ def __iter__(self):
182
+ return iter(self.transformers)
183
+
184
+ def get_transformer(self, transformer_name):
185
+ for t in self.transformers:
186
+ if t.name == transformer_name:
187
+ return t
188
+ return None
189
+
190
+
191
+ def check_unique_feature_names(transformers):
192
+ fnames = []
193
+ for f in transformers:
194
+ if f.is_featurizer:
195
+ if f.is_multi:
196
+ fnames.extend(f.name)
197
+ else:
198
+ fnames.append(f.name)
199
+ if not len(fnames) == len(set(fnames)):
200
+ raise ValueError("Featurizer name collision in a WaveformPipeline")
201
+
202
+
203
+ def featurization_config_to_class_names_and_kwargs(
204
+ featurization_cfg: FeaturizationConfig,
205
+ waveform_cfg: WaveformConfig,
206
+ sampling_frequency: float = 30_000.0,
207
+ ):
208
+ """Convert this config into a list of waveform transformer classes and arguments
209
+
210
+ Used by WaveformPipeline.from_config(...) to construct WaveformPipelines
211
+ from FeaturizationConfig objects.
212
+ """
213
+ fc = featurization_cfg
214
+ if fc.skip:
215
+ return []
216
+
217
+ class_names_and_kwargs = []
218
+ do_feats = not fc.denoise_only
219
+
220
+ tos = waveform_cfg.trough_offset_samples(sampling_frequency)
221
+ sls = waveform_cfg.spike_length_samples(sampling_frequency)
222
+ tos_kw = dict(trough_offset_samples=tos)
223
+ sls_kw = dict(spike_length_samples=sls)
224
+
225
+ if do_feats and fc.save_input_voltages:
226
+ class_names_and_kwargs.append(
227
+ ("Voltage", {"name_prefix": fc.input_waveforms_name, **tos_kw})
228
+ )
229
+ if fc.save_input_waveforms:
230
+ class_names_and_kwargs.append(
231
+ ("Waveform", {"name_prefix": fc.input_waveforms_name, **sls_kw})
232
+ )
233
+ if do_feats and fc.learn_cleaned_tpca_basis:
234
+ class_names_and_kwargs.append(
235
+ (
236
+ "BaseTemporalPCA",
237
+ {
238
+ "rank": fc.tpca_rank,
239
+ "name_prefix": fc.input_waveforms_name,
240
+ "centered": False,
241
+ "max_waveforms": fc.tpca_max_waveforms,
242
+ "fit_radius": fc.tpca_fit_radius,
243
+ },
244
+ )
245
+ )
246
+
247
+ # logic for picking an efficient combo of tpcas and nn denoisers
248
+ class_names_and_kwargs.extend(_add_tpca_and_nn(featurization_cfg, waveform_cfg))
249
+
250
+ if fc.do_enforce_decrease:
251
+ class_names_and_kwargs.append(("EnforceDecrease", {}))
252
+ if fc.save_output_waveforms:
253
+ class_names_and_kwargs.append(
254
+ ("Waveform", {"name_prefix": fc.output_waveforms_name, **sls_kw})
255
+ )
256
+
257
+ if fc.save_output_tpca_projs:
258
+ class_names_and_kwargs.append(
259
+ (
260
+ "TemporalPCAFeaturizer",
261
+ {
262
+ "rank": fc.tpca_rank,
263
+ "name_prefix": fc.output_waveforms_name,
264
+ "centered": fc.tpca_centered,
265
+ "fit_radius": fc.tpca_fit_radius,
266
+ },
267
+ )
268
+ )
269
+
270
+ # logic for grabbing localizations and amplitude vectors
271
+ class_names_and_kwargs.extend(_add_localization_and_ampvec(featurization_cfg))
272
+
273
+ return class_names_and_kwargs
274
+
275
+
276
+ def _add_tpca_and_nn(fc, wc):
277
+ do_feats = not fc.denoise_only
278
+ more = []
279
+
280
+ # now, if there's no downstream waveform feature or output tpca feature,
281
+ # we can combine the tpca featurization and denoising into one step
282
+ # it's not exactly equivalent, since the denoiser would have run on the full
283
+ # time length, but it's close enough
284
+ combine = (
285
+ do_feats
286
+ and fc.save_input_tpca_projs
287
+ and not (
288
+ fc.do_nn_denoise or fc.save_output_waveforms or fc.save_output_tpca_projs
289
+ )
290
+ )
291
+ if combine or (do_feats and fc.save_input_tpca_projs):
292
+ tslice = fc.input_tpca_waveform_cfg.relative_slice(wc)
293
+ more.append(
294
+ (
295
+ "TemporalPCA",
296
+ {
297
+ "rank": fc.tpca_rank,
298
+ "name_prefix": fc.input_waveforms_name,
299
+ "centered": fc.tpca_centered,
300
+ "temporal_slice": tslice,
301
+ "max_waveforms": fc.tpca_max_waveforms,
302
+ "fit_radius": fc.tpca_fit_radius,
303
+ },
304
+ )
305
+ )
306
+
307
+ if combine:
308
+ # that was it, all in one as discussed above.
309
+ return more
310
+
311
+ if fc.do_nn_denoise:
312
+ more.append(
313
+ (
314
+ fc.nn_denoiser_class_name,
315
+ {
316
+ "pretrained_path": fc.nn_denoiser_pretrained_path,
317
+ "n_epochs": fc.nn_denoiser_train_epochs,
318
+ "epoch_size": fc.nn_denoiser_epoch_size,
319
+ **(fc.nn_denoiser_extra_kwargs or {}),
320
+ },
321
+ )
322
+ )
323
+ if fc.do_tpca_denoise:
324
+ more.append(
325
+ (
326
+ "TemporalPCADenoiser",
327
+ {
328
+ "rank": fc.tpca_rank,
329
+ "fit_radius": fc.tpca_fit_radius,
330
+ "centered": fc.tpca_centered,
331
+ "max_waveforms": fc.tpca_max_waveforms,
332
+ },
333
+ )
334
+ )
335
+
336
+ return more
337
+
338
+
339
+ def _add_localization_and_ampvec(fc):
340
+ do_feats = not fc.denoise_only
341
+ more = []
342
+
343
+ if do_feats and fc.do_localization and fc.nn_localization:
344
+ more.append(
345
+ (
346
+ "AmortizedLocalization",
347
+ {
348
+ "amplitude_kind": fc.localization_amplitude_type,
349
+ "localization_model": fc.localization_model,
350
+ "radius": fc.localization_radius,
351
+ "softmax_noise_floor": fc.localization_noise_floor,
352
+ },
353
+ )
354
+ )
355
+ if do_feats and fc.additional_com_localization:
356
+ more.append(
357
+ (
358
+ "Localization",
359
+ {
360
+ "amplitude_kind": fc.localization_amplitude_type,
361
+ "localization_model": "com",
362
+ "radius": fc.localization_radius,
363
+ "name": "com_localizations",
364
+ },
365
+ )
366
+ )
367
+
368
+ do_ptp_amp = do_feats and fc.save_amplitudes
369
+ do_peak_vec = do_feats and (
370
+ fc.do_localization
371
+ and fc.localization_amplitude_type == "peak"
372
+ and not fc.nn_localization
373
+ )
374
+ do_ptp_vec = do_feats and fc.save_amplitudes
375
+ do_logptt = do_feats and fc.save_amplitudes
376
+ do_any_amp = do_peak_vec or do_ptp_vec or do_ptp_amp or do_logptt
377
+ if do_any_amp or (do_feats and fc.save_all_amplitudes):
378
+ more.append(
379
+ (
380
+ "AmplitudeFeatures",
381
+ {
382
+ "name_prefix": fc.output_waveforms_name,
383
+ "ptp_max_amplitude": do_ptp_amp or fc.save_all_amplitudes,
384
+ "peak_amplitude_vectors": do_peak_vec or fc.save_all_amplitudes,
385
+ "ptp_amplitude_vectors": do_ptp_vec or fc.save_all_amplitudes,
386
+ "log_peak_to_trough": do_logptt or fc.save_all_amplitudes,
387
+ },
388
+ )
389
+ )
390
+
391
+ return more