unitrefine 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. unitrefine/__init__.py +75 -0
  2. unitrefine/apply_models.py +111 -0
  3. unitrefine/cli.py +251 -0
  4. unitrefine/evaluation.py +457 -0
  5. unitrefine/figures.py +1458 -0
  6. unitrefine/import_labels.py +176 -0
  7. unitrefine/launch_sigui.py +138 -0
  8. unitrefine/launch_sigui_validate.py +243 -0
  9. unitrefine/launcher.py +239 -0
  10. unitrefine/main.py +2395 -0
  11. unitrefine/manage_models.py +193 -0
  12. unitrefine/metrics.py +146 -0
  13. unitrefine/model_selection.py +159 -0
  14. unitrefine/models_hub.py +118 -0
  15. unitrefine/phy_curation.py +306 -0
  16. unitrefine/phy_import.py +428 -0
  17. unitrefine/predictions.py +507 -0
  18. unitrefine/resources/curation_gui.jpg +0 -0
  19. unitrefine/resources/fonts/Inter-Bold.ttf +0 -0
  20. unitrefine/resources/fonts/Inter-LICENSE.txt +92 -0
  21. unitrefine/resources/fonts/Inter-Regular.ttf +0 -0
  22. unitrefine/resources/logo.png +0 -0
  23. unitrefine/resources/model_training_example.jpg +0 -0
  24. unitrefine/resources/predict-gui.JPG +0 -0
  25. unitrefine/resources/re-label.JPG +0 -0
  26. unitrefine/resources/si-gui.JPG +0 -0
  27. unitrefine/resources/train_and_load_gui.JPG +0 -0
  28. unitrefine/resources/unit_types_example.jpg +0 -0
  29. unitrefine/resources/unitrefine_gui.JPG +0 -0
  30. unitrefine/retraining_history.py +220 -0
  31. unitrefine/sigui_progress.py +157 -0
  32. unitrefine/theme.py +254 -0
  33. unitrefine/train.py +622 -0
  34. unitrefine/widgets.py +961 -0
  35. unitrefine-0.2.0.dist-info/METADATA +153 -0
  36. unitrefine-0.2.0.dist-info/RECORD +40 -0
  37. unitrefine-0.2.0.dist-info/WHEEL +5 -0
  38. unitrefine-0.2.0.dist-info/entry_points.txt +4 -0
  39. unitrefine-0.2.0.dist-info/licenses/LICENSE +21 -0
  40. unitrefine-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,176 @@
1
+ """
2
+ Import unit labels produced outside UnitRefine (Phy, SortingView, SpikeInterface curation
3
+ json, or a plain csv) and save them in the project, in the same format as labels made
4
+ with the SpikeInterface-GUI, so they can be used to train a model.
5
+
6
+ Supported files:
7
+ - Phy: cluster_group.tsv, cluster_info.tsv, cluster_KSLabel.tsv (columns `cluster_id` + `group`/`KSLabel`)
8
+ - SortingView curation json: {"labelsByUnit": {"<unit_id>": ["good"], ...}}
9
+ - SpikeInterface curation json: {"manual_labels": [...]}
10
+ - csv/tsv with columns `unit_id` and `quality`
11
+ """
12
+
13
+ import json
14
+ from pathlib import Path
15
+
16
+ import pandas as pd
17
+ from spikeinterface.core import load_sorting_analyzer
18
+
19
+ # Maps (lowercased) external label names to the labels used by UnitRefine.
20
+ LABEL_MAP = {
21
+ "good": "good",
22
+ "sua": "good",
23
+ "single": "good",
24
+ "mua": "MUA",
25
+ "multi": "MUA",
26
+ "multi-unit": "MUA",
27
+ "noise": "noise",
28
+ "artifact": "noise",
29
+ }
30
+
31
+
32
+ def read_label_file(label_file):
33
+ """Reads a label file and returns a DataFrame with columns `unit_id` and `raw_label`."""
34
+
35
+ label_file = Path(label_file)
36
+
37
+ if label_file.suffix.lower() == ".json":
38
+ with open(label_file) as f:
39
+ curation = json.load(f)
40
+
41
+ rows = []
42
+ if "labelsByUnit" in curation:
43
+ # SortingView
44
+ for unit_id, unit_labels in curation["labelsByUnit"].items():
45
+ rows.append({"unit_id": unit_id, "raw_label": _first_known_label(unit_labels)})
46
+ if len(curation.get("mergeGroups", [])) > 0:
47
+ print("Warning: merge groups in the SortingView curation are ignored, only labels are imported.")
48
+ elif "manual_labels" in curation:
49
+ # SpikeInterface curation format (v1 and v2)
50
+ for row in curation["manual_labels"]:
51
+ unit_labels = row.get("labels", row).get("quality", [])
52
+ rows.append({"unit_id": row["unit_id"], "raw_label": _first_known_label(unit_labels)})
53
+ else:
54
+ raise ValueError(f"{label_file} is not a recognised SortingView or SpikeInterface curation json.")
55
+
56
+ return pd.DataFrame(rows, columns=["unit_id", "raw_label"])
57
+
58
+ sep = "\t" if label_file.suffix.lower() == ".tsv" else ","
59
+ table = pd.read_csv(label_file, sep=sep)
60
+
61
+ unit_id_column = next((col for col in ["unit_id", "cluster_id", "id"] if col in table.columns), None)
62
+ # Phy stores manual labels in `group`; `KSLabel` holds Kilosort's automatic labels.
63
+ label_column = next((col for col in ["quality", "group", "KSLabel", "label"] if col in table.columns), None)
64
+
65
+ if unit_id_column is None or label_column is None:
66
+ raise ValueError(
67
+ f"Could not find unit id and label columns in {label_file}. "
68
+ f"Found columns: {list(table.columns)}"
69
+ )
70
+
71
+ print(f"Reading labels from column '{label_column}' and unit ids from column '{unit_id_column}'.")
72
+ return pd.DataFrame({"unit_id": table[unit_id_column].values, "raw_label": table[label_column].values})
73
+
74
+
75
+ def _first_known_label(unit_labels):
76
+ if isinstance(unit_labels, str):
77
+ unit_labels = [unit_labels]
78
+ for label in unit_labels:
79
+ if str(label).lower() in LABEL_MAP:
80
+ return label
81
+ return unit_labels[0] if len(unit_labels) > 0 else None
82
+
83
+
84
+ def compute_all_metrics(analyzer):
85
+ """Concatenates the quality and template metrics of an analyzer, indexed by unit id."""
86
+
87
+ for extension_name in ["quality_metrics", "template_metrics"]:
88
+ # load_extension raises if the extension was never computed
89
+ try:
90
+ analyzer.load_extension(extension_name)
91
+ except Exception:
92
+ pass
93
+
94
+ if analyzer.has_extension("template_metrics"):
95
+ tms = analyzer.get_extension("template_metrics").get_data()
96
+ else:
97
+ tms = pd.DataFrame()
98
+
99
+ if analyzer.has_extension("quality_metrics"):
100
+ qms = analyzer.get_extension("quality_metrics").get_data()
101
+ else:
102
+ qms = pd.DataFrame()
103
+
104
+ return pd.concat([qms, tms], axis=1)
105
+
106
+
107
+ def import_labels(label_file, analyzer_path, save_folder, existing_labels=None, file_prefix=""):
108
+ """
109
+ Matches the labels in `label_file` to the units of the analyzer at `analyzer_path` and saves
110
+ `labels.csv`, `labelled_metrics.csv` and `all_metrics.csv` in `save_folder`, their names starting
111
+ with `file_prefix` (used for new analyzers, whose files share one folder).
112
+
113
+ Imported labels replace existing labels of the same units; other existing labels are kept.
114
+ Returns the labels DataFrame (columns `quality`, `unit_id`).
115
+ """
116
+
117
+ save_folder = Path(save_folder)
118
+
119
+ raw_labels = read_label_file(label_file)
120
+
121
+ analyzer = load_sorting_analyzer(analyzer_path, load_extensions=False)
122
+ all_metrics = compute_all_metrics(analyzer)
123
+ if all_metrics.empty:
124
+ raise ValueError(
125
+ "The analyzer has no quality or template metrics. Please compute them before importing labels."
126
+ )
127
+
128
+ # unit ids in label files are often stored as int or str, independent of the analyzer
129
+ analyzer_unit_ids = {str(unit_id): unit_id for unit_id in analyzer.unit_ids}
130
+
131
+ unit_ids, qualities = [], []
132
+ missing_units, unknown_labels = [], set()
133
+ for unit_id, raw_label in zip(raw_labels["unit_id"], raw_labels["raw_label"]):
134
+ if pd.isna(raw_label):
135
+ continue
136
+ quality = LABEL_MAP.get(str(raw_label).strip().lower())
137
+ if quality is None:
138
+ unknown_labels.add(str(raw_label))
139
+ continue
140
+ analyzer_unit_id = analyzer_unit_ids.get(str(unit_id))
141
+ if analyzer_unit_id is None:
142
+ missing_units.append(unit_id)
143
+ continue
144
+ unit_ids.append(analyzer_unit_id)
145
+ qualities.append(quality)
146
+
147
+ if len(unknown_labels) > 0:
148
+ print(f"Skipped units with labels not used by UnitRefine: {sorted(unknown_labels)}")
149
+ if len(missing_units) > 0:
150
+ print(f"Skipped {len(missing_units)} units not found in the analyzer: {missing_units[:20]}")
151
+ if len(unit_ids) == 0:
152
+ raise ValueError("No labels could be matched to units in the analyzer.")
153
+
154
+ labels_df = pd.DataFrame({"quality": qualities, "unit_id": unit_ids})
155
+
156
+ if existing_labels is not None and len(existing_labels) > 0:
157
+ kept = existing_labels[["quality", "unit_id"]].copy()
158
+ # labels.csv is re-read with pandas' inferred dtype, so map back to the analyzer's unit ids
159
+ kept["unit_id"] = kept["unit_id"].astype(str).map(analyzer_unit_ids)
160
+ kept = kept.dropna(subset=["unit_id"])
161
+ kept = kept[~kept["unit_id"].isin(labels_df["unit_id"])]
162
+ labels_df = pd.concat([kept, labels_df])
163
+
164
+ labels_df = labels_df.sort_values("unit_id").reset_index(drop=True)
165
+ # keep labelled_metrics in the same order as labels.csv, which TrainWindow relies on
166
+ labelled_metrics = all_metrics.loc[labels_df["unit_id"].values]
167
+
168
+ save_folder.mkdir(exist_ok=True, parents=True)
169
+ all_metrics.to_csv(save_folder / f"{file_prefix}all_metrics.csv", index_label="unit_id")
170
+ labelled_metrics.to_csv(save_folder / f"{file_prefix}labelled_metrics.csv", index_label="unit_id")
171
+ labels_df.to_csv(save_folder / f"{file_prefix}labels.csv", index=False)
172
+
173
+ counts = labels_df["quality"].value_counts().to_dict()
174
+ print(f"Imported labels for {len(labels_df)}/{len(all_metrics)} units: {counts}")
175
+
176
+ return labels_df
@@ -0,0 +1,138 @@
1
+ import argparse
2
+ import functools
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ import pandas as pd
7
+ import spikeinterface.full as si
8
+ from PyQt5.QtGui import QCloseEvent
9
+ from PyQt5.QtWidgets import QWidget
10
+ from pyqtgraph import mkQApp
11
+ from PyQt5.QtCore import QTimer
12
+ from unitrefine import (
13
+ SIGUI_SHOWN_MARKER, check_sigui_version, report_sigui_step, save_labels_or_backup, unwritable_file_message,
14
+ )
15
+
16
+ check_sigui_version()
17
+ from spikeinterface_gui.backend_qt import QtMainWindow
18
+ from spikeinterface_gui.controller import Controller
19
+
20
+ import warnings
21
+ warnings.filterwarnings("ignore")
22
+
23
+
24
+ def my_custom_close_handler(event: QCloseEvent, window: QWidget, save_folder, analyzer):
25
+ """
26
+ This function will be called instead of the original closeEvent.
27
+ """
28
+
29
+ labelled_unit_ids = []
30
+ labels = []
31
+ analyzer_unit_ids = {str(unit_id): unit_id for unit_id in analyzer.unit_ids}
32
+
33
+ # the labels as they are now: spikeinterface-gui >= 0.13 works on its own copy of the curation data,
34
+ # so the dictionary it was given does not change
35
+ for row in window.controller.curation_data['manual_labels']:
36
+ quality = row.get('quality') if row.get('labels') is None else row['labels'].get('quality')
37
+ if quality is None or len(quality) == 0:
38
+ continue
39
+ labelled_unit_ids.append(analyzer_unit_ids[str(row['unit_id'])])
40
+ labels.append(quality[0])
41
+
42
+ if analyzer.has_extension('template_metrics'):
43
+ tms = analyzer.get_extension("template_metrics").get_data()
44
+ else:
45
+ tms = pd.DataFrame()
46
+
47
+ if analyzer.has_extension('quality_metrics'):
48
+ qms = analyzer.get_extension("quality_metrics").get_data()
49
+ else:
50
+ qms = pd.DataFrame()
51
+
52
+ all_metrics = pd.concat([qms, tms], axis=1)
53
+
54
+ labels_df = pd.DataFrame()
55
+ labels_df['quality'] = labels
56
+ labels_df['unit_id'] = labelled_unit_ids
57
+ labels_df = labels_df.sort_values('unit_id')
58
+
59
+ labelled_metrics = all_metrics[all_metrics.index.isin(labelled_unit_ids)]
60
+
61
+ # the labels first, so they are saved (or backed up) even if the metrics files can't be written
62
+ save_labels_or_backup(labels_df, save_folder / "labels.csv", index=False)
63
+ try:
64
+ all_metrics.to_csv(save_folder / "all_metrics.csv", index_label="unit_id")
65
+ labelled_metrics.to_csv(save_folder / "labelled_metrics.csv", index_label="unit_id")
66
+ except OSError as error:
67
+ print(unwritable_file_message(save_folder, error), flush=True)
68
+
69
+
70
+ argv = sys.argv[1:]
71
+
72
+ parser = argparse.ArgumentParser(description='spikeinterface-gui')
73
+ parser.add_argument('analyzer_folder', help='SortingAnalyzer folder path', default=None, nargs='?')
74
+ parser.add_argument('project_folder', help='Project folder path', default=None, nargs='?')
75
+ parser.add_argument('analyzer_index', help='Project folder path', default=None, nargs='?')
76
+
77
+ args = parser.parse_args(argv)
78
+
79
+
80
+ analyzer_folder = args.analyzer_folder
81
+
82
+ if '//' not in analyzer_folder:
83
+ analyzer_folder = Path(analyzer_folder)
84
+
85
+ project_folder = Path(args.project_folder)
86
+ analyzer_index = int(args.analyzer_index)
87
+
88
+ save_folder = project_folder / (f"analyzers/analyzer_{analyzer_index}")
89
+
90
+ if Path(save_folder / "labels.csv").is_file():
91
+ decisions = pd.read_csv(save_folder / "labels.csv")
92
+ else:
93
+ decisions = pd.DataFrame(columns=['unit_id', 'quality'])
94
+
95
+ report_sigui_step("loading")
96
+ analyzer = si.load_sorting_analyzer(str(analyzer_folder), load_extensions=False)
97
+
98
+ # the existing labels fill the quality column. Unit ids are matched as text: labels.csv is read back with
99
+ # pandas' inferred type (e.g. numbers), while an analyzer's unit ids can be text
100
+ existing_labels = dict(zip(decisions['unit_id'].astype(str), decisions['quality']))
101
+ manual_labels = []
102
+ for unit_id in analyzer.unit_ids:
103
+ quality = existing_labels.get(str(unit_id))
104
+ if isinstance(quality, str):
105
+ # curation format 2: the labels of a unit are under "labels"
106
+ manual_labels.append({"unit_id": unit_id, "labels": {"quality": [quality]}})
107
+
108
+ label_definitions = {
109
+ "quality": dict(name="quality", label_options=["good", "MUA", "noise"], exclusive=True),
110
+ }
111
+
112
+ curation_dict = dict(
113
+ format_version="2",
114
+ unit_ids=analyzer.unit_ids,
115
+ manual_labels=manual_labels,
116
+ label_definitions=label_definitions,
117
+ )
118
+
119
+ report_sigui_step("views")
120
+ controller = Controller(
121
+ analyzer, backend="qt", curation=True, curation_data=curation_dict, skip_extensions=['waveforms', 'principal_components', 'spike_locations', 'isi_histograms', 'template_similarity', 'amplitudescalings'],
122
+ )
123
+
124
+ layout_dict={'zone1': ['unitlist', 'mainsettings'], 'zone2': [], 'zone3': ['waveform'], 'zone4': ['correlogram'], 'zone5': ['spikeamplitude'], 'zone6': [], 'zone7': [], 'zone8': ['spikerate']}
125
+
126
+
127
+ app = mkQApp()
128
+ win = QtMainWindow(controller, layout_dict=layout_dict, user_settings=None)
129
+ win.closeEvent = functools.partial(my_custom_close_handler, window=win, save_folder=save_folder, analyzer=analyzer)
130
+
131
+ win.show()
132
+ # tell the UnitRefine window that the SpikeInterface-GUI is open, once the event loop has started
133
+ QTimer.singleShot(0, lambda: print(SIGUI_SHOWN_MARKER, flush=True))
134
+ app.exec()
135
+
136
+
137
+
138
+
@@ -0,0 +1,243 @@
1
+ import argparse
2
+ import functools
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ import pandas as pd
7
+ import spikeinterface.full as si
8
+ from PyQt5.QtGui import QCloseEvent
9
+ from PyQt5.QtWidgets import QWidget
10
+ from pyqtgraph import mkQApp
11
+ from PyQt5.QtCore import QTimer
12
+ from unitrefine import (
13
+ SIGUI_SHOWN_MARKER, check_sigui_version, report_sigui_step, save_labels_or_backup, unwritable_file_message,
14
+ )
15
+
16
+ check_sigui_version()
17
+ from unitrefine.import_labels import compute_all_metrics
18
+ from spikeinterface_gui.backend_qt import QtMainWindow
19
+ from spikeinterface_gui.controller import Controller
20
+ import numpy as np
21
+
22
+ from unitrefine.predictions import (
23
+ get_saved_or_new_predictions,
24
+ predictions_table,
25
+ predictions_file_name,
26
+ describe_prediction_error,
27
+ save_csv_atomic,
28
+ )
29
+
30
+ import warnings
31
+ warnings.filterwarnings("ignore")
32
+
33
+ # extensions the SpikeInterface-GUI does not need here, and which can be large
34
+ SKIPPED_EXTENSIONS = ['waveforms', 'principal_components', 'spike_locations', 'isi_histograms', 'template_similarity', 'amplitudescalings']
35
+
36
+
37
+ def get_quality_labels(curation_dict):
38
+ """The quality labels set in the SpikeInterface-GUI, as a Series indexed by unit id."""
39
+ quality_labels = {}
40
+ for row in curation_dict['manual_labels']:
41
+ if row.get('labels') is None:
42
+ quality = row.get('quality')
43
+ else:
44
+ quality = row['labels'].get('quality')
45
+ if quality is not None and len(quality) > 0:
46
+ quality_labels[row['unit_id']] = quality[0]
47
+ return pd.Series(quality_labels, dtype=object)
48
+
49
+
50
+ def my_custom_close_handler(event: QCloseEvent, window: QWidget, save_folder, file_prefix, analyzer, selection_key, shown_unit_ids):
51
+ """
52
+ This function will be called instead of the original closeEvent.
53
+ """
54
+
55
+ # the labels as they are now: spikeinterface-gui >= 0.13 works on its own copy of the curation data,
56
+ # so the dictionary it was given does not change
57
+ quality_labels = get_quality_labels(window.controller.curation_data)
58
+
59
+ # the quality column starts with the human labels the models were trained on; only labels which differ
60
+ # from them (or units which had none) are new, so only those are saved as relabelled units
61
+ new_labels = quality_labels[[original_quality.get(unit_id) != quality for unit_id, quality in quality_labels.items()]]
62
+
63
+ labels_df = pd.DataFrame({'quality': new_labels.values, 'unit_id': new_labels.index})
64
+
65
+ # keep the labels from earlier sessions for units which were not shown this time
66
+ if existing_relabels is not None:
67
+ shown = set(str(unit_id) for unit_id in shown_unit_ids)
68
+ kept = existing_relabels[~existing_relabels['unit_id'].astype(str).isin(shown)]
69
+ labels_df = pd.concat([kept[['quality', 'unit_id']], labels_df])
70
+
71
+ labels_df = labels_df.sort_values('unit_id', key=lambda unit_ids: unit_ids.astype(str))
72
+ save_labels_or_backup(labels_df, relabelled_units_path, write=lambda table, path: save_csv_atomic(table, path, index=False),
73
+ index=False)
74
+
75
+ # the metrics of all units, which Retrain uses; `analyzer` is the full analyzer, not only the units shown
76
+ all_metrics_path = save_folder / f"{file_prefix}all_metrics.csv"
77
+ if not all_metrics_path.is_file():
78
+ try:
79
+ compute_all_metrics(analyzer).to_csv(all_metrics_path, index_label="unit_id")
80
+ except OSError as error:
81
+ # Retrain makes this file again if it is missing
82
+ print(unwritable_file_message(all_metrics_path, error), flush=True)
83
+
84
+ argv = sys.argv[1:]
85
+
86
+ parser = argparse.ArgumentParser(description='spikeinterface-gui')
87
+ parser.add_argument('analyzer_folder', help='SortingAnalyzer folder path')
88
+ parser.add_argument('--save_folder', required=True, help='Folder for the predictions and relabelled units')
89
+ parser.add_argument('--file_prefix', default='', help='Prefix for the saved file names')
90
+ parser.add_argument('--mode', choices=['binary', 'cascade'], default='binary')
91
+ parser.add_argument('--model_folders', nargs='+', required=True, help='Binary: the models. Cascade: layer 1 then layer 2.')
92
+ parser.add_argument('--selection_key', required=True, help='Name used for the relabelled units file')
93
+ parser.add_argument('--human_labels', default=None, help='csv with the curated labels (unit_id, quality), which fill the quality column')
94
+ parser.add_argument('--relabel', action='store_true', help='Only show the units in --units_file')
95
+ parser.add_argument('--units_file', default=None, help='csv with the unit_id of the units to relabel')
96
+
97
+
98
+ args = parser.parse_args(argv)
99
+
100
+ save_folder = Path(args.save_folder)
101
+ file_prefix = args.file_prefix
102
+ analyzer_folder = args.analyzer_folder
103
+ mode = args.mode
104
+ model_folders = [Path(model_folder) for model_folder in args.model_folders]
105
+ selection_key = args.selection_key
106
+ relabel = args.relabel
107
+
108
+ if '//' not in analyzer_folder:
109
+ analyzer_folder = Path(analyzer_folder)
110
+
111
+ save_folder.mkdir(exist_ok=True, parents=True)
112
+
113
+ # extensions are loaded when they are needed, so large ones like waveforms or principal components are not read
114
+ report_sigui_step("loading")
115
+ sorting_analyzer = si.load_sorting_analyzer(analyzer_folder, load_extensions=False)
116
+ # kept for saving the metrics of all units, also when only some units are shown for relabelling
117
+ full_sorting_analyzer = sorting_analyzer
118
+ # unit ids read back from csv files get pandas' inferred dtype, so match them as strings
119
+ analyzer_unit_ids = {str(unit_id): unit_id for unit_id in sorting_analyzer.unit_ids}
120
+
121
+ if mode == "cascade":
122
+ print(f"\nPredictions of the cascade: {model_folders[0].name} (layer 1), then {model_folders[1].name} (layer 2) on the neural units.\n")
123
+ else:
124
+ print(f"\nPredictions made by: {[model_folder.name for model_folder in model_folders]}.\n")
125
+
126
+ report_sigui_step("predictions")
127
+ predictions_path = save_folder / predictions_file_name(file_prefix, selection_key)
128
+ try:
129
+ # reuse the predictions saved by an earlier Inspect, Relabel or Apply, when they are up to date
130
+ label_columns, confidence_columns, selection_confidence = get_saved_or_new_predictions(
131
+ sorting_analyzer, analyzer_folder, mode, model_folders, predictions_path
132
+ )
133
+ except Exception as e:
134
+ print(f"\nCould not compute predictions for {analyzer_folder}: {describe_prediction_error(e)}\n")
135
+ sys.exit(1)
136
+
137
+
138
+ # the human labels the models were trained on, for analyzers which were curated or imported. As in Curate,
139
+ # they fill the quality column, where the n, g and m keyboard shortcuts set labels
140
+ original_quality = {}
141
+ if args.human_labels is not None and Path(args.human_labels).is_file():
142
+ human_labels_df = pd.read_csv(args.human_labels)
143
+ original_quality = {
144
+ analyzer_unit_ids[str(unit_id)]: quality
145
+ for unit_id, quality in zip(human_labels_df['unit_id'], human_labels_df['quality'])
146
+ if str(unit_id) in analyzer_unit_ids
147
+ }
148
+
149
+ # save every model's predictions, named after the model they come from, with the human labels next to them
150
+ predictions_df = predictions_table(
151
+ sorting_analyzer.unit_ids, {"human_label": pd.Series(original_quality, dtype=object), **label_columns}, confidence_columns
152
+ )
153
+ save_csv_atomic(predictions_df, predictions_path)
154
+
155
+ # labels from earlier sessions with this model selection are shown again, and kept when saving
156
+ relabelled_units_path = save_folder / f"{file_prefix}relabelled_units_{selection_key}.csv"
157
+ existing_relabels = pd.read_csv(relabelled_units_path) if relabelled_units_path.is_file() else None
158
+ existing_quality = {}
159
+ if existing_relabels is not None:
160
+ for unit_id, quality in zip(existing_relabels['unit_id'], existing_relabels['quality']):
161
+ if str(unit_id) in analyzer_unit_ids:
162
+ existing_quality[analyzer_unit_ids[str(unit_id)]] = quality
163
+
164
+ unit_ids_to_show = list(sorting_analyzer.unit_ids)
165
+ if relabel:
166
+ units_to_relabel = pd.read_csv(args.units_file)['unit_id'].astype(str)
167
+ unit_ids_to_show = [analyzer_unit_ids[unit_id] for unit_id in units_to_relabel if unit_id in analyzer_unit_ids]
168
+ # the selected units are copied into a new analyzer in memory, which only has the extensions loaded now,
169
+ # so load the ones the SpikeInterface-GUI shows (not the large ones it skips)
170
+ for extension_name in sorting_analyzer.get_saved_extension_names():
171
+ if extension_name not in SKIPPED_EXTENSIONS:
172
+ sorting_analyzer.load_extension(extension_name)
173
+ sorting_analyzer = sorting_analyzer.select_units(unit_ids=unit_ids_to_show)
174
+
175
+ manual_labels = []
176
+ for unit_id in unit_ids_to_show:
177
+ unit_labels = {}
178
+ # labels from earlier sessions take priority over the original human labels. When relabelling, the
179
+ # original labels are not shown, so the uncertain units get a fresh look without being nudged
180
+ if unit_id in existing_quality:
181
+ unit_labels["quality"] = [existing_quality[unit_id]]
182
+ elif unit_id in original_quality and not relabel:
183
+ unit_labels["quality"] = [original_quality[unit_id]]
184
+ for column_name, labels in label_columns.items():
185
+ if unit_id in labels.index:
186
+ unit_labels[column_name] = [labels[unit_id]]
187
+ # curation format 2: the labels of a unit are under "labels". In this format the n/g/m keys can set the
188
+ # quality label of every unit, also of units shown with a model's label but no quality label yet
189
+ manual_labels.append({"unit_id": unit_id, "labels": unit_labels})
190
+
191
+ label_definitions = {
192
+ "quality": dict(name="quality", label_options=["good", "MUA", "noise"], exclusive=True),
193
+ }
194
+ for column_name, labels in label_columns.items():
195
+ label_definitions[column_name] = dict(name=column_name, label_options=sorted(labels.unique()), exclusive=True)
196
+
197
+ unit_positions = {unit_id: position for position, unit_id in enumerate(predictions_df.index)}
198
+ shown_positions = [unit_positions[unit_id] for unit_id in unit_ids_to_show]
199
+ extra_unit_properties = {column_name: np.asarray(confidences)[shown_positions] for column_name, confidences in confidence_columns.items()}
200
+
201
+ curation_dict = dict(
202
+ format_version="2",
203
+ unit_ids=sorting_analyzer.unit_ids,
204
+ manual_labels=manual_labels,
205
+ label_definitions=label_definitions,
206
+ )
207
+
208
+ report_sigui_step("views")
209
+ controller = Controller(
210
+ sorting_analyzer,
211
+ backend="qt",
212
+ curation=True,
213
+ curation_data=curation_dict,
214
+ extra_unit_properties=extra_unit_properties,
215
+ # the label columns are shown by the curation view, and the confidence columns are added from extra_unit_properties
216
+ displayed_unit_properties=['firing_rate', 'snr', 'x', 'y', 'rp_violations'],
217
+ skip_extensions=SKIPPED_EXTENSIONS,
218
+ )
219
+
220
+ layout_dict={'zone1': ['unitlist', 'mainsettings'], 'zone2': [], 'zone3': ['waveform'], 'zone4': ['correlogram'], 'zone5': ['spikeamplitude'], 'zone6': [], 'zone7': [], 'zone8': ['spikerate']}
221
+
222
+ if relabel:
223
+ print(f"Launching SpikeInterface-GUI to relabel the {len(unit_ids_to_show)} least confident units of analyzer at {analyzer_folder}")
224
+ else:
225
+ print(f"Launching SpikeInterface-GUI to inspect model predictions for analyzer at {analyzer_folder}")
226
+
227
+ print("Main UnitRefine window will remain inactive until you close the SpikeInterface-GUI window")
228
+ print("Label units as noise, good and MUA by pressing 'n', 'g' and 'm' on your keyboard.")
229
+ if relabel:
230
+ print("Your labels are shown in the 'quality' column, which starts empty (except units you relabelled before).")
231
+ else:
232
+ print("Your labels are shown in the 'quality' column, which starts with your existing human labels.")
233
+ print("To change a label, select the unit and press another key: the new label replaces the old one.")
234
+ print("Your labels are saved automatically when you close the SpikeInterface-GUI.")
235
+
236
+ app = mkQApp()
237
+ win = QtMainWindow(controller, layout_dict=layout_dict, user_settings=None)
238
+ win.closeEvent = functools.partial(my_custom_close_handler, window=win, save_folder=save_folder, file_prefix=file_prefix, analyzer=full_sorting_analyzer, selection_key=selection_key, shown_unit_ids=unit_ids_to_show)
239
+
240
+ win.show()
241
+ # tell the UnitRefine window that the SpikeInterface-GUI is open, once the event loop has started
242
+ QTimer.singleShot(0, lambda: print(SIGUI_SHOWN_MARKER, flush=True))
243
+ app.exec()