ayar 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ayar/__init__.py +28 -0
- ayar/audit.py +165 -0
- ayar/bundle.py +328 -0
- ayar/cli.py +176 -0
- ayar/config.py +79 -0
- ayar/detectors/__init__.py +7 -0
- ayar/detectors/_pairs.py +35 -0
- ayar/detectors/drift.py +178 -0
- ayar/detectors/duplicate.py +73 -0
- ayar/detectors/overlap.py +101 -0
- ayar/examples.py +76 -0
- ayar/flags.py +144 -0
- ayar/io.py +135 -0
- ayar/render.py +108 -0
- ayar/scan.py +54 -0
- ayar/stats.py +112 -0
- ayar/synth.py +216 -0
- ayar/validate.py +126 -0
- ayar-0.1.0.dist-info/METADATA +214 -0
- ayar-0.1.0.dist-info/RECORD +23 -0
- ayar-0.1.0.dist-info/WHEEL +4 -0
- ayar-0.1.0.dist-info/entry_points.txt +2 -0
- ayar-0.1.0.dist-info/licenses/LICENSE +202 -0
ayar/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""ayar: assay your spike-sorted finding before you publish.
|
|
2
|
+
|
|
3
|
+
ayar reads the output of any spike sorter, flags sorting artifacts that can
|
|
4
|
+
manufacture or hide a result (duplicated units, overlap holes, drift-driven rate
|
|
5
|
+
changes), and re-computes *your* finding without the flagged units to tell you
|
|
6
|
+
whether it survives.
|
|
7
|
+
|
|
8
|
+
import ayar
|
|
9
|
+
bundle = ayar.load("kilosort_output/") # or a .npz bundle
|
|
10
|
+
report = ayar.scan(bundle, finding=my_analysis)
|
|
11
|
+
print(report)
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
__version__ = "0.1.0"
|
|
15
|
+
|
|
16
|
+
from .bundle import Bundle, Events # noqa: E402
|
|
17
|
+
from .config import DEFAULTS, Thresholds # noqa: E402
|
|
18
|
+
from .flags import Flag, NotChecked, Report, Severity # noqa: E402
|
|
19
|
+
from .io import from_arrays, from_phy, load # noqa: E402
|
|
20
|
+
from .scan import scan # noqa: E402
|
|
21
|
+
from .synth import SynthConfig, make_session # noqa: E402
|
|
22
|
+
from .validate import validate # noqa: E402
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"Bundle", "Events", "Thresholds", "DEFAULTS", "Flag", "NotChecked", "Report", "Severity",
|
|
26
|
+
"from_arrays", "from_phy", "load", "scan", "make_session", "SynthConfig", "validate",
|
|
27
|
+
"__version__",
|
|
28
|
+
]
|
ayar/audit.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""The finding audit: does the user's result survive removing the flagged units?
|
|
2
|
+
|
|
3
|
+
The user's finding is any function ``f(bundle) -> float | dict[str, float]``.
|
|
4
|
+
For each detector that raised flags, ayar removes the implicated units, recomputes
|
|
5
|
+
the finding, and compares the change with *random-drop controls*: the same number
|
|
6
|
+
of unflagged units removed at random. A change is only blamed on the artifact if
|
|
7
|
+
it is larger than what dropping arbitrary units does.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import importlib
|
|
13
|
+
import importlib.util
|
|
14
|
+
import math
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Callable
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
|
|
21
|
+
from .bundle import Bundle
|
|
22
|
+
from .config import Thresholds
|
|
23
|
+
from .flags import AuditRow, Flag
|
|
24
|
+
|
|
25
|
+
FindingFn = Callable[[Bundle], Any]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def units_to_drop(flag: Flag) -> set[int]:
|
|
29
|
+
"""The units whose removal neutralises a flag.
|
|
30
|
+
|
|
31
|
+
Pair flags list the unit to keep first, so the second unit is dropped
|
|
32
|
+
(for duplicates, the smaller copy). Single-unit flags drop the unit.
|
|
33
|
+
"""
|
|
34
|
+
if len(flag.units) >= 2:
|
|
35
|
+
return {int(flag.units[1])}
|
|
36
|
+
return {int(u) for u in flag.units}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def call_finding(fn: FindingFn, bundle: Bundle) -> dict[str, float]:
|
|
40
|
+
out = fn(bundle)
|
|
41
|
+
if isinstance(out, dict):
|
|
42
|
+
items = out.items()
|
|
43
|
+
else:
|
|
44
|
+
items = [("value", out)]
|
|
45
|
+
res: dict[str, float] = {}
|
|
46
|
+
for k, v in items:
|
|
47
|
+
try:
|
|
48
|
+
res[str(k)] = float(v)
|
|
49
|
+
except (TypeError, ValueError):
|
|
50
|
+
raise TypeError(
|
|
51
|
+
f"finding must return a number or a dict of numbers; {k!r} is {type(v).__name__}"
|
|
52
|
+
) from None
|
|
53
|
+
return res
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _rel(new: float, base: float) -> float:
|
|
57
|
+
if not (math.isfinite(new) and math.isfinite(base)):
|
|
58
|
+
return math.inf
|
|
59
|
+
return abs(new - base) / max(abs(base), 1e-12)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _flipped(new: float, base: float) -> bool:
|
|
63
|
+
return base != 0 and new != 0 and math.copysign(1, new) != math.copysign(1, base)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def audit(
|
|
67
|
+
bundle: Bundle,
|
|
68
|
+
finding: FindingFn,
|
|
69
|
+
flags: list[Flag],
|
|
70
|
+
th: Thresholds,
|
|
71
|
+
) -> tuple[dict[str, float], list[AuditRow]]:
|
|
72
|
+
baseline = call_finding(finding, bundle)
|
|
73
|
+
|
|
74
|
+
groups: dict[str, set[int]] = {}
|
|
75
|
+
for f in flags:
|
|
76
|
+
groups.setdefault(f.code, set()).update(units_to_drop(f))
|
|
77
|
+
if len(groups) > 1:
|
|
78
|
+
groups["ALL_FLAGS"] = set().union(*groups.values())
|
|
79
|
+
|
|
80
|
+
flagged_any = set().union(*groups.values()) if groups else set()
|
|
81
|
+
pool = np.array([int(u) for u in bundle.unit_ids if int(u) not in flagged_any])
|
|
82
|
+
rng = np.random.default_rng(th.audit_seed)
|
|
83
|
+
|
|
84
|
+
rows: list[AuditRow] = []
|
|
85
|
+
for name, drop in groups.items():
|
|
86
|
+
k = len(drop)
|
|
87
|
+
try:
|
|
88
|
+
value = call_finding(finding, bundle.without(drop))
|
|
89
|
+
except Exception as e: # the user's function may not cope with fewer units
|
|
90
|
+
for key, base in baseline.items():
|
|
91
|
+
rows.append(AuditRow(key, name, k, base, None, None, None, None, "failed",
|
|
92
|
+
f"finding raised {type(e).__name__}: {e}"))
|
|
93
|
+
continue
|
|
94
|
+
|
|
95
|
+
controls: list[dict[str, float]] = []
|
|
96
|
+
note = ""
|
|
97
|
+
if th.audit_controls > 0 and pool.size >= k and k > 0:
|
|
98
|
+
for _ in range(th.audit_controls):
|
|
99
|
+
pick = rng.choice(pool, size=k, replace=False)
|
|
100
|
+
try:
|
|
101
|
+
controls.append(call_finding(finding, bundle.without(pick.tolist())))
|
|
102
|
+
except Exception:
|
|
103
|
+
pass
|
|
104
|
+
elif th.audit_controls > 0:
|
|
105
|
+
note = f"no random-drop controls: only {pool.size} unflagged units for {k} drops"
|
|
106
|
+
|
|
107
|
+
for key, base in baseline.items():
|
|
108
|
+
new = value.get(key, math.nan)
|
|
109
|
+
rel = _rel(new, base)
|
|
110
|
+
ctrl_rel = [_rel(c[key], base) for c in controls if key in c]
|
|
111
|
+
ctrl_q = float(np.quantile(ctrl_rel, th.audit_control_quantile)) if ctrl_rel else None
|
|
112
|
+
flip_rate = (
|
|
113
|
+
float(np.mean([_flipped(c[key], base) for c in controls if key in c]))
|
|
114
|
+
if ctrl_rel else None
|
|
115
|
+
)
|
|
116
|
+
beyond = ctrl_q is None or rel > ctrl_q
|
|
117
|
+
if _flipped(new, base) and (flip_rate is None or flip_rate < 1 - th.audit_control_quantile):
|
|
118
|
+
verdict = "flipped"
|
|
119
|
+
elif rel < th.audit_robust:
|
|
120
|
+
verdict = "robust"
|
|
121
|
+
elif not beyond:
|
|
122
|
+
verdict = "nonspecific"
|
|
123
|
+
elif rel >= th.audit_sensitive:
|
|
124
|
+
verdict = "sensitive"
|
|
125
|
+
else:
|
|
126
|
+
verdict = "shifted"
|
|
127
|
+
rows.append(
|
|
128
|
+
AuditRow(
|
|
129
|
+
key=key,
|
|
130
|
+
ablation=name,
|
|
131
|
+
n_dropped=k,
|
|
132
|
+
baseline=base,
|
|
133
|
+
value=new,
|
|
134
|
+
rel_change=rel,
|
|
135
|
+
control_q=ctrl_q,
|
|
136
|
+
control_flip_rate=flip_rate,
|
|
137
|
+
verdict=verdict,
|
|
138
|
+
note=note,
|
|
139
|
+
)
|
|
140
|
+
)
|
|
141
|
+
return baseline, rows
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def load_finding(spec: str) -> tuple[str, FindingFn]:
|
|
145
|
+
"""Resolve ``path/to/file.py:func`` or ``package.module:func``."""
|
|
146
|
+
if ":" not in spec:
|
|
147
|
+
raise ValueError(f"finding must look like file.py:func or module:func, got {spec!r}")
|
|
148
|
+
target, func = spec.rsplit(":", 1)
|
|
149
|
+
if target.endswith(".py") or Path(target).exists():
|
|
150
|
+
path = Path(target).resolve()
|
|
151
|
+
if not path.exists():
|
|
152
|
+
raise FileNotFoundError(path)
|
|
153
|
+
mod_name = f"_ayar_finding_{path.stem}"
|
|
154
|
+
spec_obj = importlib.util.spec_from_file_location(mod_name, path)
|
|
155
|
+
if spec_obj is None or spec_obj.loader is None:
|
|
156
|
+
raise ImportError(f"cannot import {path}")
|
|
157
|
+
module = importlib.util.module_from_spec(spec_obj)
|
|
158
|
+
sys.modules[mod_name] = module
|
|
159
|
+
spec_obj.loader.exec_module(module)
|
|
160
|
+
else:
|
|
161
|
+
module = importlib.import_module(target)
|
|
162
|
+
fn = getattr(module, func, None)
|
|
163
|
+
if not callable(fn):
|
|
164
|
+
raise AttributeError(f"{target} has no callable {func!r}")
|
|
165
|
+
return f"{Path(target).name if target.endswith('.py') else target}:{func}", fn
|
ayar/bundle.py
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""The ayar bundle: the one input contract every detector reads.
|
|
2
|
+
|
|
3
|
+
A bundle is the output of *any* spike sorter reduced to what ayar needs:
|
|
4
|
+
|
|
5
|
+
* ``spike_times`` (seconds) and ``spike_clusters`` (unit id per spike): tier 0
|
|
6
|
+
* ``amplitudes`` and/or ``spike_depths`` / ``unit_depths``: tier 1
|
|
7
|
+
* ``events`` (trial times, optional condition labels): needed for confound checks
|
|
8
|
+
|
|
9
|
+
Bundles are saved as plain ``.npz`` files that numpy can read without ayar.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Iterable, Mapping
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
FORMAT_VERSION = 1
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Events:
|
|
24
|
+
"""Trial or event times, with an optional condition label per event."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, times: Iterable[float], labels: Iterable[Any] | None = None):
|
|
27
|
+
times = np.asarray(times, dtype=float).ravel()
|
|
28
|
+
if not np.all(np.isfinite(times)):
|
|
29
|
+
raise ValueError("event times must be finite")
|
|
30
|
+
order = np.argsort(times, kind="stable")
|
|
31
|
+
self.times = times[order]
|
|
32
|
+
if labels is None:
|
|
33
|
+
self.labels = None
|
|
34
|
+
else:
|
|
35
|
+
labels = np.asarray(labels).ravel()
|
|
36
|
+
if labels.shape[0] != times.shape[0]:
|
|
37
|
+
raise ValueError(
|
|
38
|
+
f"{labels.shape[0]} event labels for {times.shape[0]} event times"
|
|
39
|
+
)
|
|
40
|
+
self.labels = labels[order]
|
|
41
|
+
|
|
42
|
+
def __len__(self) -> int:
|
|
43
|
+
return int(self.times.shape[0])
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def label_values(self) -> np.ndarray:
|
|
47
|
+
if self.labels is None:
|
|
48
|
+
return np.array([])
|
|
49
|
+
return np.unique(self.labels)
|
|
50
|
+
|
|
51
|
+
def __repr__(self) -> str:
|
|
52
|
+
n_lab = len(self.label_values)
|
|
53
|
+
return f"Events(n={len(self)}, labels={n_lab if self.labels is not None else None})"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class Bundle:
|
|
57
|
+
"""Sorted spikes plus whatever extra evidence the sorter provides."""
|
|
58
|
+
|
|
59
|
+
def __init__(
|
|
60
|
+
self,
|
|
61
|
+
spike_times: Iterable[float],
|
|
62
|
+
spike_clusters: Iterable[int],
|
|
63
|
+
*,
|
|
64
|
+
duration: float | None = None,
|
|
65
|
+
amplitudes: Iterable[float] | None = None,
|
|
66
|
+
spike_depths: Iterable[float] | None = None,
|
|
67
|
+
unit_ids: Iterable[int] | None = None,
|
|
68
|
+
unit_depths: Iterable[float] | Mapping[int, float] | None = None,
|
|
69
|
+
events: Events | tuple | Mapping | None = None,
|
|
70
|
+
meta: Mapping[str, Any] | None = None,
|
|
71
|
+
):
|
|
72
|
+
st = np.asarray(spike_times, dtype=float).ravel()
|
|
73
|
+
sc = np.asarray(spike_clusters).ravel()
|
|
74
|
+
if st.shape != sc.shape:
|
|
75
|
+
raise ValueError(f"{st.size} spike times but {sc.size} spike clusters")
|
|
76
|
+
if not np.all(np.isfinite(st)):
|
|
77
|
+
raise ValueError("spike times must be finite")
|
|
78
|
+
if st.size and st.min() < 0:
|
|
79
|
+
raise ValueError("spike times must be >= 0 (seconds from session start)")
|
|
80
|
+
sc = sc.astype(np.int64)
|
|
81
|
+
|
|
82
|
+
if st.size > 1 and np.any(np.diff(st) < 0):
|
|
83
|
+
order = np.argsort(st, kind="stable")
|
|
84
|
+
else:
|
|
85
|
+
order = np.arange(st.size) # already sorted: skip the O(n log n) sort
|
|
86
|
+
self.spike_times = st[order]
|
|
87
|
+
self.spike_clusters = sc[order]
|
|
88
|
+
self.amplitudes = self._per_spike(amplitudes, order, "amplitudes")
|
|
89
|
+
self.spike_depths = self._per_spike(spike_depths, order, "spike_depths")
|
|
90
|
+
|
|
91
|
+
if unit_ids is None:
|
|
92
|
+
self.unit_ids = np.unique(self.spike_clusters)
|
|
93
|
+
else:
|
|
94
|
+
self.unit_ids = np.unique(np.asarray(list(unit_ids), dtype=np.int64))
|
|
95
|
+
missing = np.setdiff1d(np.unique(self.spike_clusters), self.unit_ids)
|
|
96
|
+
if missing.size:
|
|
97
|
+
raise ValueError(f"spikes belong to units not in unit_ids: {missing[:10].tolist()}")
|
|
98
|
+
|
|
99
|
+
self.unit_depths = self._unit_depths(unit_depths)
|
|
100
|
+
|
|
101
|
+
last = float(self.spike_times[-1]) if self.spike_times.size else 0.0
|
|
102
|
+
if duration is None:
|
|
103
|
+
duration = last
|
|
104
|
+
if duration < last:
|
|
105
|
+
raise ValueError(f"duration {duration} s is shorter than the last spike at {last} s")
|
|
106
|
+
self.duration = float(duration)
|
|
107
|
+
|
|
108
|
+
self.events = self._events(events)
|
|
109
|
+
self.meta: dict[str, Any] = dict(meta or {})
|
|
110
|
+
self._index: dict[int, np.ndarray] | None = None
|
|
111
|
+
|
|
112
|
+
# ------------------------------------------------------------------ build
|
|
113
|
+
def _per_spike(self, values, order, name):
|
|
114
|
+
if values is None:
|
|
115
|
+
return None
|
|
116
|
+
v = np.asarray(values, dtype=float).ravel()
|
|
117
|
+
if v.shape[0] != order.shape[0]:
|
|
118
|
+
raise ValueError(f"{name} has {v.shape[0]} values for {order.shape[0]} spikes")
|
|
119
|
+
return v[order]
|
|
120
|
+
|
|
121
|
+
def _unit_depths(self, unit_depths):
|
|
122
|
+
if unit_depths is None:
|
|
123
|
+
return None
|
|
124
|
+
if isinstance(unit_depths, Mapping):
|
|
125
|
+
d = np.array([float(unit_depths.get(int(u), np.nan)) for u in self.unit_ids])
|
|
126
|
+
else:
|
|
127
|
+
d = np.asarray(unit_depths, dtype=float).ravel()
|
|
128
|
+
if d.shape[0] != self.unit_ids.shape[0]:
|
|
129
|
+
raise ValueError(
|
|
130
|
+
f"unit_depths has {d.shape[0]} values for {self.unit_ids.shape[0]} units"
|
|
131
|
+
)
|
|
132
|
+
return d
|
|
133
|
+
|
|
134
|
+
@staticmethod
|
|
135
|
+
def _events(events):
|
|
136
|
+
if events is None or isinstance(events, Events):
|
|
137
|
+
return events
|
|
138
|
+
if isinstance(events, Mapping):
|
|
139
|
+
return Events(events["times"], events.get("labels"))
|
|
140
|
+
if isinstance(events, tuple):
|
|
141
|
+
return Events(*events)
|
|
142
|
+
return Events(events)
|
|
143
|
+
|
|
144
|
+
def _build_index(self) -> dict[int, np.ndarray]:
|
|
145
|
+
order = np.argsort(self.spike_clusters, kind="stable") # stable keeps time order
|
|
146
|
+
sorted_c = self.spike_clusters[order]
|
|
147
|
+
starts = np.searchsorted(sorted_c, self.unit_ids, side="left")
|
|
148
|
+
ends = np.searchsorted(sorted_c, self.unit_ids, side="right")
|
|
149
|
+
return {int(u): order[s:e] for u, s, e in zip(self.unit_ids, starts, ends)}
|
|
150
|
+
|
|
151
|
+
# ----------------------------------------------------------------- access
|
|
152
|
+
def spike_indices(self, unit: int) -> np.ndarray:
|
|
153
|
+
if self._index is None:
|
|
154
|
+
self._index = self._build_index()
|
|
155
|
+
try:
|
|
156
|
+
return self._index[int(unit)]
|
|
157
|
+
except KeyError:
|
|
158
|
+
raise KeyError(f"no unit {unit} in bundle") from None
|
|
159
|
+
|
|
160
|
+
def train(self, unit: int) -> np.ndarray:
|
|
161
|
+
"""Sorted spike times (s) of one unit."""
|
|
162
|
+
return self.spike_times[self.spike_indices(unit)]
|
|
163
|
+
|
|
164
|
+
def trains(self) -> dict[int, np.ndarray]:
|
|
165
|
+
return {int(u): self.train(u) for u in self.unit_ids}
|
|
166
|
+
|
|
167
|
+
def unit_amplitudes(self, unit: int) -> np.ndarray | None:
|
|
168
|
+
if self.amplitudes is None:
|
|
169
|
+
return None
|
|
170
|
+
return self.amplitudes[self.spike_indices(unit)]
|
|
171
|
+
|
|
172
|
+
def unit_spike_depths(self, unit: int) -> np.ndarray | None:
|
|
173
|
+
if self.spike_depths is None:
|
|
174
|
+
return None
|
|
175
|
+
return self.spike_depths[self.spike_indices(unit)]
|
|
176
|
+
|
|
177
|
+
def depth(self, unit: int) -> float | None:
|
|
178
|
+
"""Unit depth (µm) from ``unit_depths``, else the median spike depth."""
|
|
179
|
+
if self.unit_depths is not None:
|
|
180
|
+
i = int(np.searchsorted(self.unit_ids, int(unit)))
|
|
181
|
+
v = float(self.unit_depths[i])
|
|
182
|
+
if np.isfinite(v):
|
|
183
|
+
return v
|
|
184
|
+
d = self.unit_spike_depths(unit)
|
|
185
|
+
if d is not None and d.size:
|
|
186
|
+
return float(np.median(d))
|
|
187
|
+
return None
|
|
188
|
+
|
|
189
|
+
def rate(self, unit: int) -> float:
|
|
190
|
+
return len(self.spike_indices(unit)) / self.duration if self.duration > 0 else 0.0
|
|
191
|
+
|
|
192
|
+
@property
|
|
193
|
+
def n_units(self) -> int:
|
|
194
|
+
return int(self.unit_ids.shape[0])
|
|
195
|
+
|
|
196
|
+
@property
|
|
197
|
+
def n_spikes(self) -> int:
|
|
198
|
+
return int(self.spike_times.shape[0])
|
|
199
|
+
|
|
200
|
+
@property
|
|
201
|
+
def has_amplitudes(self) -> bool:
|
|
202
|
+
return self.amplitudes is not None
|
|
203
|
+
|
|
204
|
+
@property
|
|
205
|
+
def has_depths(self) -> bool:
|
|
206
|
+
return self.unit_depths is not None or self.spike_depths is not None
|
|
207
|
+
|
|
208
|
+
@property
|
|
209
|
+
def tier(self) -> int:
|
|
210
|
+
"""0: spike times only. 1: plus amplitudes or depths."""
|
|
211
|
+
return 1 if (self.has_amplitudes or self.has_depths) else 0
|
|
212
|
+
|
|
213
|
+
# -------------------------------------------------------------- transform
|
|
214
|
+
def without(self, units: Iterable[int]) -> "Bundle":
|
|
215
|
+
"""A new bundle with the given units removed (used by the finding audit)."""
|
|
216
|
+
drop = np.asarray(sorted({int(u) for u in units}), dtype=np.int64)
|
|
217
|
+
keep_unit = ~np.isin(self.unit_ids, drop)
|
|
218
|
+
if self._index is None:
|
|
219
|
+
self._index = self._build_index()
|
|
220
|
+
keep_spk = np.ones(self.n_spikes, dtype=bool)
|
|
221
|
+
for u in drop:
|
|
222
|
+
if int(u) in self._index:
|
|
223
|
+
keep_spk[self._index[int(u)]] = False
|
|
224
|
+
new_pos = np.cumsum(keep_spk) - 1
|
|
225
|
+
child_index = {
|
|
226
|
+
int(u): new_pos[self._index[int(u)]] for u in self.unit_ids[keep_unit]
|
|
227
|
+
}
|
|
228
|
+
# internal fast path: the parent is already validated and sorted
|
|
229
|
+
child = object.__new__(Bundle)
|
|
230
|
+
child.spike_times = self.spike_times[keep_spk]
|
|
231
|
+
child.spike_clusters = self.spike_clusters[keep_spk]
|
|
232
|
+
child.amplitudes = None if self.amplitudes is None else self.amplitudes[keep_spk]
|
|
233
|
+
child.spike_depths = None if self.spike_depths is None else self.spike_depths[keep_spk]
|
|
234
|
+
child.unit_ids = self.unit_ids[keep_unit]
|
|
235
|
+
child.unit_depths = None if self.unit_depths is None else self.unit_depths[keep_unit]
|
|
236
|
+
child.duration = self.duration
|
|
237
|
+
child.events = self.events
|
|
238
|
+
child.meta = self.meta
|
|
239
|
+
child._index = child_index
|
|
240
|
+
return child
|
|
241
|
+
|
|
242
|
+
def summary(self) -> dict[str, Any]:
|
|
243
|
+
ev = None
|
|
244
|
+
if self.events is not None:
|
|
245
|
+
ev = {
|
|
246
|
+
"n": len(self.events),
|
|
247
|
+
"labels": [str(v) for v in self.events.label_values] if self.events.labels is not None else None,
|
|
248
|
+
}
|
|
249
|
+
return {
|
|
250
|
+
"n_units": self.n_units,
|
|
251
|
+
"n_spikes": self.n_spikes,
|
|
252
|
+
"duration_s": round(self.duration, 3),
|
|
253
|
+
"tier": self.tier,
|
|
254
|
+
"amplitudes": self.has_amplitudes,
|
|
255
|
+
"depths": self.has_depths,
|
|
256
|
+
"events": ev,
|
|
257
|
+
"source": self.meta.get("source"),
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
def __repr__(self) -> str:
|
|
261
|
+
return (
|
|
262
|
+
f"Bundle(units={self.n_units}, spikes={self.n_spikes}, "
|
|
263
|
+
f"duration={self.duration:.1f}s, tier={self.tier})"
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
# --------------------------------------------------------------------- io
|
|
267
|
+
def save(self, path: str | Path) -> Path:
|
|
268
|
+
path = Path(path)
|
|
269
|
+
if path.suffix != ".npz":
|
|
270
|
+
path = path.with_suffix(".npz")
|
|
271
|
+
arrays: dict[str, np.ndarray] = {
|
|
272
|
+
"format_version": np.array(FORMAT_VERSION),
|
|
273
|
+
"spike_times": self.spike_times,
|
|
274
|
+
"spike_clusters": self.spike_clusters,
|
|
275
|
+
"unit_ids": self.unit_ids,
|
|
276
|
+
"duration": np.array(self.duration),
|
|
277
|
+
"meta_json": np.array(json.dumps(self.meta, default=_json_default)),
|
|
278
|
+
}
|
|
279
|
+
if self.amplitudes is not None:
|
|
280
|
+
arrays["amplitudes"] = self.amplitudes
|
|
281
|
+
if self.spike_depths is not None:
|
|
282
|
+
arrays["spike_depths"] = self.spike_depths
|
|
283
|
+
if self.unit_depths is not None:
|
|
284
|
+
arrays["unit_depths"] = self.unit_depths
|
|
285
|
+
if self.events is not None:
|
|
286
|
+
arrays["event_times"] = self.events.times
|
|
287
|
+
if self.events.labels is not None:
|
|
288
|
+
labels = self.events.labels
|
|
289
|
+
if labels.dtype == object:
|
|
290
|
+
labels = labels.astype(str)
|
|
291
|
+
arrays["event_labels"] = labels
|
|
292
|
+
np.savez_compressed(path, **arrays)
|
|
293
|
+
return path
|
|
294
|
+
|
|
295
|
+
@classmethod
|
|
296
|
+
def load(cls, path: str | Path) -> "Bundle":
|
|
297
|
+
with np.load(Path(path), allow_pickle=False) as z:
|
|
298
|
+
version = int(z["format_version"]) if "format_version" in z else 1
|
|
299
|
+
if version > FORMAT_VERSION:
|
|
300
|
+
raise ValueError(
|
|
301
|
+
f"bundle format {version} is newer than this ayar supports ({FORMAT_VERSION})"
|
|
302
|
+
)
|
|
303
|
+
get = lambda k: z[k] if k in z else None # noqa: E731
|
|
304
|
+
events = None
|
|
305
|
+
if "event_times" in z:
|
|
306
|
+
events = Events(z["event_times"], get("event_labels"))
|
|
307
|
+
meta = json.loads(str(z["meta_json"])) if "meta_json" in z else {}
|
|
308
|
+
return cls(
|
|
309
|
+
z["spike_times"],
|
|
310
|
+
z["spike_clusters"],
|
|
311
|
+
duration=float(z["duration"]) if "duration" in z else None,
|
|
312
|
+
amplitudes=get("amplitudes"),
|
|
313
|
+
spike_depths=get("spike_depths"),
|
|
314
|
+
unit_ids=get("unit_ids"),
|
|
315
|
+
unit_depths=get("unit_depths"),
|
|
316
|
+
events=events,
|
|
317
|
+
meta=meta,
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _json_default(o):
|
|
322
|
+
if isinstance(o, np.integer):
|
|
323
|
+
return int(o)
|
|
324
|
+
if isinstance(o, np.floating):
|
|
325
|
+
return float(o)
|
|
326
|
+
if isinstance(o, np.ndarray):
|
|
327
|
+
return o.tolist()
|
|
328
|
+
raise TypeError(f"not JSON serialisable: {type(o).__name__}")
|
ayar/cli.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""Command-line interface: ``ayar synth | scan | validate | info | doctor``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import platform
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _thresholds(args):
|
|
15
|
+
from .config import DEFAULTS
|
|
16
|
+
|
|
17
|
+
return DEFAULTS.override_strings(args.set or [])
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def cmd_synth(args) -> int:
|
|
21
|
+
from .synth import SynthConfig, make_session
|
|
22
|
+
|
|
23
|
+
cfg = SynthConfig(n_units=args.units, duration=args.duration, design=args.design)
|
|
24
|
+
b = make_session(args.seed, cfg)
|
|
25
|
+
path = b.save(args.out)
|
|
26
|
+
t = b.meta["truth"]
|
|
27
|
+
print(f"wrote {path}: {b.n_units} units, {b.n_spikes:,} spikes, {b.duration:g} s, design={args.design}")
|
|
28
|
+
print(f"planted: " + ", ".join(f"{x['code']} {x['units']}" for x in t))
|
|
29
|
+
return 0
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def cmd_scan(args) -> int:
|
|
33
|
+
from .audit import load_finding
|
|
34
|
+
from .io import load
|
|
35
|
+
from .render import format_text
|
|
36
|
+
from .scan import scan
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
th = _thresholds(args)
|
|
40
|
+
kw = {}
|
|
41
|
+
if args.sample_rate:
|
|
42
|
+
kw["sample_rate"] = args.sample_rate
|
|
43
|
+
bundle = load(args.bundle, **kw)
|
|
44
|
+
if args.events is not None:
|
|
45
|
+
from .bundle import Events
|
|
46
|
+
import numpy as np
|
|
47
|
+
|
|
48
|
+
with np.load(args.events, allow_pickle=False) as z:
|
|
49
|
+
bundle.events = Events(z["times"], z["labels"] if "labels" in z else None)
|
|
50
|
+
finding, name = None, None
|
|
51
|
+
if args.finding:
|
|
52
|
+
name, finding = load_finding(args.finding)
|
|
53
|
+
report = scan(bundle, finding, thresholds=th, detectors=args.only, finding_name=name)
|
|
54
|
+
except (FileNotFoundError, ValueError, KeyError, TypeError, ImportError, AttributeError) as e:
|
|
55
|
+
print(f"ayar: error: {e}", file=sys.stderr)
|
|
56
|
+
return 2
|
|
57
|
+
|
|
58
|
+
if args.json:
|
|
59
|
+
Path(args.json).write_text(report.to_json())
|
|
60
|
+
if args.format == "json":
|
|
61
|
+
print(report.to_json())
|
|
62
|
+
else:
|
|
63
|
+
print(format_text(report, verbose=args.verbose))
|
|
64
|
+
if args.json:
|
|
65
|
+
print(f"report written to {args.json}")
|
|
66
|
+
return report.exit_code(args.fail_on)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cmd_validate(args) -> int:
|
|
70
|
+
from .synth import SynthConfig
|
|
71
|
+
from .validate import validate
|
|
72
|
+
|
|
73
|
+
res = validate(range(args.seed, args.seed + args.sessions), SynthConfig(duration=args.duration),
|
|
74
|
+
_thresholds(args))
|
|
75
|
+
if args.format == "json":
|
|
76
|
+
print(json.dumps(res.to_dict(), indent=2))
|
|
77
|
+
else:
|
|
78
|
+
print(res.format_text())
|
|
79
|
+
ok = res.passed(args.min_sensitivity, args.min_precision)
|
|
80
|
+
if args.format != "json":
|
|
81
|
+
print(f"\n {'PASS' if ok else 'FAIL'} (min sensitivity {args.min_sensitivity}, "
|
|
82
|
+
f"min precision {args.min_precision}, no control alarms)")
|
|
83
|
+
return 0 if ok else 1
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def cmd_info(args) -> int:
|
|
87
|
+
from .io import load
|
|
88
|
+
|
|
89
|
+
b = load(args.bundle)
|
|
90
|
+
print(json.dumps(b.summary(), indent=2))
|
|
91
|
+
return 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def cmd_doctor(args) -> int:
|
|
95
|
+
import numpy
|
|
96
|
+
import scipy
|
|
97
|
+
|
|
98
|
+
print(f"ayar {__version__}")
|
|
99
|
+
print(f"python {platform.python_version()} ({sys.executable})")
|
|
100
|
+
print(f"numpy {numpy.__version__}, scipy {scipy.__version__}")
|
|
101
|
+
for mod in ("spikeinterface", "pynwb", "daftar"):
|
|
102
|
+
try:
|
|
103
|
+
m = __import__(mod)
|
|
104
|
+
print(f"{mod} {getattr(m, '__version__', '?')} (adapter planned)")
|
|
105
|
+
except ImportError:
|
|
106
|
+
print(f"{mod}: not installed (optional)")
|
|
107
|
+
from .examples import condition_contrast
|
|
108
|
+
from .scan import scan
|
|
109
|
+
from .synth import make_session
|
|
110
|
+
|
|
111
|
+
b = make_session(0)
|
|
112
|
+
r = scan(b, condition_contrast)
|
|
113
|
+
planted = {(t["code"], frozenset(t["units"])) for t in b.meta["truth"]}
|
|
114
|
+
found = {(f.code, frozenset(f.units)) for f in r.flags}
|
|
115
|
+
ok = planted <= found
|
|
116
|
+
print(f"self-check: {len(planted & found)}/{len(planted)} planted artifacts found "
|
|
117
|
+
f"in a 600 s synthetic session -> {'ok' if ok else 'PROBLEM'}")
|
|
118
|
+
return 0 if ok else 1
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
122
|
+
p = argparse.ArgumentParser(
|
|
123
|
+
prog="ayar",
|
|
124
|
+
description="Assay your spike-sorted finding before you publish.",
|
|
125
|
+
)
|
|
126
|
+
p.add_argument("--version", action="version", version=f"ayar {__version__}")
|
|
127
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
128
|
+
|
|
129
|
+
s = sub.add_parser("scan", help="flag sorting artifacts and audit a finding")
|
|
130
|
+
s.add_argument("bundle", help="a .npz bundle or a Phy/Kilosort output folder")
|
|
131
|
+
s.add_argument("--finding", help="your analysis: file.py:func or module:func")
|
|
132
|
+
s.add_argument("--events", help=".npz with 'times' (s) and optional 'labels' arrays")
|
|
133
|
+
s.add_argument("--sample-rate", type=float, help="for Phy folders without params.py")
|
|
134
|
+
s.add_argument("--only", nargs="+", metavar="CODE", help="run only these detectors")
|
|
135
|
+
s.add_argument("--set", action="append", metavar="NAME=VALUE", help="override a threshold")
|
|
136
|
+
s.add_argument("--format", choices=["text", "json"], default="text")
|
|
137
|
+
s.add_argument("--json", metavar="PATH", help="also write the JSON report here")
|
|
138
|
+
s.add_argument("--fail-on", choices=["error", "warning", "never"], default="error",
|
|
139
|
+
help="exit 1 when flags reach this severity (default: error)")
|
|
140
|
+
s.add_argument("-v", "--verbose", action="store_true", help="explain every flag")
|
|
141
|
+
s.set_defaults(fn=cmd_scan)
|
|
142
|
+
|
|
143
|
+
y = sub.add_parser("synth", help="write a synthetic session with planted artifacts")
|
|
144
|
+
y.add_argument("out", help="output .npz path")
|
|
145
|
+
y.add_argument("--seed", type=int, default=0)
|
|
146
|
+
y.add_argument("--units", type=int, default=40)
|
|
147
|
+
y.add_argument("--duration", type=float, default=600.0)
|
|
148
|
+
y.add_argument("--design", choices=["block", "interleaved"], default="block")
|
|
149
|
+
y.set_defaults(fn=cmd_synth)
|
|
150
|
+
|
|
151
|
+
v = sub.add_parser("validate", help="measure detector accuracy on synthetic sessions")
|
|
152
|
+
v.add_argument("--sessions", type=int, default=5)
|
|
153
|
+
v.add_argument("--seed", type=int, default=0)
|
|
154
|
+
v.add_argument("--duration", type=float, default=600.0)
|
|
155
|
+
v.add_argument("--min-sensitivity", type=float, default=0.8)
|
|
156
|
+
v.add_argument("--min-precision", type=float, default=0.8)
|
|
157
|
+
v.add_argument("--set", action="append", metavar="NAME=VALUE")
|
|
158
|
+
v.add_argument("--format", choices=["text", "json"], default="text")
|
|
159
|
+
v.set_defaults(fn=cmd_validate)
|
|
160
|
+
|
|
161
|
+
i = sub.add_parser("info", help="summarise a bundle")
|
|
162
|
+
i.add_argument("bundle")
|
|
163
|
+
i.set_defaults(fn=cmd_info)
|
|
164
|
+
|
|
165
|
+
d = sub.add_parser("doctor", help="check the installation")
|
|
166
|
+
d.set_defaults(fn=cmd_doctor)
|
|
167
|
+
return p
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def main(argv: list[str] | None = None) -> int:
|
|
171
|
+
args = build_parser().parse_args(argv)
|
|
172
|
+
return args.fn(args)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
if __name__ == "__main__":
|
|
176
|
+
sys.exit(main())
|