TwoPy 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- twopy/analysis/background_subtraction.py +120 -21
- twopy/analysis/dff.py +33 -0
- twopy/analysis/kernels/__init__.py +2 -2
- twopy/analysis/persistence.py +19 -4
- twopy/analysis/response_maps.py +57 -7
- twopy/analysis_cache.py +86 -23
- twopy/config.py +47 -8
- twopy/conversion/alignment_crop.py +18 -13
- twopy/conversion/api.py +21 -5
- twopy/conversion/hdf5_writing.py +27 -4
- twopy/conversion/orientation.py +80 -0
- twopy/conversion/types.py +1 -1
- twopy/converted.py +26 -3
- twopy/custom/__init__.py +8 -0
- twopy/custom/native_workflows/direction_selectivity.py +13 -12
- twopy/custom/native_workflows/response_kernels.py +2 -7
- twopy/custom/parameters.py +4 -3
- twopy/custom/provenance.py +14 -0
- twopy/custom/runner.py +244 -0
- twopy/custom/types.py +247 -8
- twopy/database/search.py +1 -1
- twopy/napari/controls.py +147 -26
- twopy/napari/custom_tab.py +168 -20
- twopy/napari/database_search.py +41 -1
- twopy/napari/display.py +10 -85
- twopy/napari/display_paths.py +26 -0
- twopy/napari/group_matching/__init__.py +29 -0
- twopy/napari/group_matching/cards.py +127 -0
- twopy/napari/group_matching/fov_assignment.py +679 -0
- twopy/napari/group_matching/fov_cards.py +143 -0
- twopy/napari/{group_matching_images.py → group_matching/images.py} +2 -2
- twopy/napari/group_matching/response_preview.py +334 -0
- twopy/napari/group_matching/responses.py +130 -0
- twopy/napari/{group_matching_roi.py → group_matching/roi_assignment.py} +590 -532
- twopy/napari/group_matching/roi_cards.py +371 -0
- twopy/napari/group_matching/style.py +269 -0
- twopy/napari/group_matching/tables.py +27 -0
- twopy/napari/group_matching/window.py +267 -0
- twopy/napari/live_analysis.py +75 -32
- twopy/napari/load_controller.py +400 -0
- twopy/napari/load_progress.py +168 -0
- twopy/napari/load_workflow.py +225 -83
- twopy/napari/loaded_recordings_csv.py +35 -4
- twopy/napari/loading.py +59 -3
- twopy/napari/plotting/dff_options.py +1 -0
- twopy/napari/plotting/docks/dynamic_options.py +60 -14
- twopy/napari/plotting/docks/options_panel.py +15 -16
- twopy/napari/plotting/docks/paths.py +5 -6
- twopy/napari/plotting/docks/plot_area.py +42 -0
- twopy/napari/plotting/docks/response_map_area.py +5 -8
- twopy/napari/plotting/docks/response_plot_widget.py +255 -101
- twopy/napari/plotting/docks/visibility_state.py +29 -2
- twopy/napari/plotting/export.py +11 -19
- twopy/napari/plotting/export_controls.py +5 -6
- twopy/napari/plotting/normalization_options.py +1 -0
- twopy/napari/plotting/options.py +1 -1
- twopy/napari/plotting/panels.py +90 -5
- twopy/napari/plotting/preview_strip.py +218 -0
- twopy/napari/plotting/processing_options.py +1 -0
- twopy/napari/plotting/widgets.py +403 -26
- twopy/napari/responses.py +58 -1
- twopy/napari/roi.py +91 -22
- twopy/napari/roi_signatures.py +20 -0
- twopy/napari/session.py +20 -3
- twopy/napari/sidebar.py +27 -14
- twopy/napari/state.py +113 -27
- twopy/napari/viewer.py +206 -67
- twopy/roi.py +26 -5
- twopy/spatial.py +3 -3
- twopy/spatial_orientation.py +78 -0
- {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/METADATA +8 -4
- {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/RECORD +75 -59
- twopy/napari/group_matching.py +0 -596
- {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/WHEEL +0 -0
- {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/entry_points.txt +0 -0
- {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/top_level.txt +0 -0
|
@@ -15,6 +15,7 @@ from typing import Literal
|
|
|
15
15
|
|
|
16
16
|
import numpy as np
|
|
17
17
|
import numpy.typing as npt
|
|
18
|
+
from scipy import sparse
|
|
18
19
|
|
|
19
20
|
from twopy.converted import RecordingData
|
|
20
21
|
from twopy.frame_ranges import normalize_frame_range
|
|
@@ -278,6 +279,7 @@ def _extract_mean_traces_from_indices(
|
|
|
278
279
|
rather than total recording length.
|
|
279
280
|
"""
|
|
280
281
|
traces = np.empty((frame_stop - frame_start, len(pixel_indices_by_trace)))
|
|
282
|
+
projection = _mean_projection(pixel_indices_by_trace)
|
|
281
283
|
_check_cancelled(check_cancelled)
|
|
282
284
|
for chunk_start, chunk_stop, frames in recording.movie.iter_frame_batches(
|
|
283
285
|
chunk_frames=chunk_frames,
|
|
@@ -289,10 +291,7 @@ def _extract_mean_traces_from_indices(
|
|
|
289
291
|
chunk_offset = chunk_start - frame_start
|
|
290
292
|
chunk_slice = slice(chunk_offset, chunk_offset + (chunk_stop - chunk_start))
|
|
291
293
|
chunk_flat = frames.reshape(frames.shape[0], -1)
|
|
292
|
-
|
|
293
|
-
traces[chunk_slice, trace_index] = chunk_flat[:, pixel_indices].mean(
|
|
294
|
-
axis=1,
|
|
295
|
-
)
|
|
294
|
+
traces[chunk_slice, :] = _project_trace_pixels(chunk_flat, projection)
|
|
296
295
|
_check_cancelled(check_cancelled)
|
|
297
296
|
|
|
298
297
|
return traces
|
|
@@ -344,6 +343,7 @@ def _extract_movie_global_percentile_corrected_traces(
|
|
|
344
343
|
)
|
|
345
344
|
frame_count = frame_stop - frame_start
|
|
346
345
|
roi_count = len(roi_pixel_indices)
|
|
346
|
+
projection = _mean_projection(roi_pixel_indices)
|
|
347
347
|
raw_values = np.empty((frame_count, roi_count), dtype=np.float64)
|
|
348
348
|
background_values = np.empty((frame_count, roi_count), dtype=np.float64)
|
|
349
349
|
corrected_values = np.empty((frame_count, roi_count), dtype=np.float64)
|
|
@@ -359,14 +359,19 @@ def _extract_movie_global_percentile_corrected_traces(
|
|
|
359
359
|
chunk_slice = slice(chunk_offset, chunk_offset + (chunk_stop - chunk_start))
|
|
360
360
|
chunk_flat = frames.reshape(frames.shape[0], -1)
|
|
361
361
|
background_trace = chunk_flat[:, background_indices].mean(axis=1)
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
362
|
+
roi_pixels = chunk_flat[:, projection.pixel_indices]
|
|
363
|
+
raw_values[chunk_slice, :] = _project_selected_pixels(roi_pixels, projection)
|
|
364
|
+
background_values[chunk_slice, :] = background_trace[:, np.newaxis]
|
|
365
|
+
# Clamp pixels before averaging so dim pixels cannot pull the
|
|
366
|
+
# corrected ROI trace below zero.
|
|
367
|
+
corrected_pixels = np.maximum(
|
|
368
|
+
roi_pixels - background_trace[:, np.newaxis],
|
|
369
|
+
0.0,
|
|
370
|
+
)
|
|
371
|
+
corrected_values[chunk_slice, :] = _project_selected_pixels(
|
|
372
|
+
corrected_pixels,
|
|
373
|
+
projection,
|
|
374
|
+
)
|
|
370
375
|
_check_cancelled(check_cancelled)
|
|
371
376
|
|
|
372
377
|
metadata: dict[str, BackgroundMetadataValue] = {
|
|
@@ -440,6 +445,7 @@ def _extract_movie_y_stripe_percentile_corrected_traces(
|
|
|
440
445
|
)
|
|
441
446
|
frame_count = frame_stop - frame_start
|
|
442
447
|
roi_count = len(roi_pixel_indices)
|
|
448
|
+
projection = _mean_projection(roi_pixel_indices)
|
|
443
449
|
raw_values = np.empty((frame_count, roi_count), dtype=np.float64)
|
|
444
450
|
background_values = np.empty((frame_count, roi_count), dtype=np.float64)
|
|
445
451
|
corrected_values = np.empty((frame_count, roi_count), dtype=np.float64)
|
|
@@ -460,15 +466,21 @@ def _extract_movie_y_stripe_percentile_corrected_traces(
|
|
|
460
466
|
)
|
|
461
467
|
chunk_flat = frames.reshape(frames.shape[0], -1)
|
|
462
468
|
background_flat = background_field.reshape(background_field.shape[0], -1)
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
469
|
+
roi_pixels = chunk_flat[:, projection.pixel_indices]
|
|
470
|
+
roi_background_pixels = background_flat[:, projection.pixel_indices]
|
|
471
|
+
raw_values[chunk_slice, :] = _project_selected_pixels(roi_pixels, projection)
|
|
472
|
+
background_values[chunk_slice, :] = _project_selected_pixels(
|
|
473
|
+
roi_background_pixels,
|
|
474
|
+
projection,
|
|
475
|
+
)
|
|
476
|
+
corrected_pixels = np.maximum(
|
|
477
|
+
roi_pixels - roi_background_pixels,
|
|
478
|
+
0.0,
|
|
479
|
+
)
|
|
480
|
+
corrected_values[chunk_slice, :] = _project_selected_pixels(
|
|
481
|
+
corrected_pixels,
|
|
482
|
+
projection,
|
|
483
|
+
)
|
|
472
484
|
_check_cancelled(check_cancelled)
|
|
473
485
|
|
|
474
486
|
metadata: dict[str, BackgroundMetadataValue] = {
|
|
@@ -593,6 +605,93 @@ def _roi_pixel_indices_from_masks(
|
|
|
593
605
|
return tuple(np.flatnonzero(mask).astype(np.int64) for mask in flat_masks)
|
|
594
606
|
|
|
595
607
|
|
|
608
|
+
@dataclass(frozen=True)
|
|
609
|
+
class _MeanProjection:
|
|
610
|
+
"""Sparse averaging matrix plus selected flattened pixel indices."""
|
|
611
|
+
|
|
612
|
+
pixel_indices: npt.NDArray[np.int64]
|
|
613
|
+
matrix: sparse.csc_matrix
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _mean_projection(
|
|
617
|
+
pixel_indices_by_trace: tuple[npt.NDArray[np.int64], ...],
|
|
618
|
+
) -> _MeanProjection:
|
|
619
|
+
"""Return selected pixels and a sparse pixel-to-trace averaging matrix.
|
|
620
|
+
|
|
621
|
+
Args:
|
|
622
|
+
pixel_indices_by_trace: One flattened pixel-index array per trace.
|
|
623
|
+
|
|
624
|
+
Returns:
|
|
625
|
+
Selected flattened pixel indices plus a sparse matrix shaped
|
|
626
|
+
``(selected_pixels, traces)`` whose columns average each trace.
|
|
627
|
+
|
|
628
|
+
Pixel indices are concatenated per trace instead of deduplicated. That
|
|
629
|
+
preserves the existing behavior for overlapping ROIs, where the same movie
|
|
630
|
+
pixel can contribute independently to more than one ROI mean.
|
|
631
|
+
"""
|
|
632
|
+
selected_pixel_indices: list[npt.NDArray[np.int64]] = []
|
|
633
|
+
rows: list[npt.NDArray[np.int64]] = []
|
|
634
|
+
columns: list[npt.NDArray[np.int64]] = []
|
|
635
|
+
values: list[npt.NDArray[np.float64]] = []
|
|
636
|
+
row_start = 0
|
|
637
|
+
for trace_index, pixel_indices in enumerate(pixel_indices_by_trace):
|
|
638
|
+
if pixel_indices.size == 0:
|
|
639
|
+
continue
|
|
640
|
+
selected_pixel_indices.append(pixel_indices)
|
|
641
|
+
rows.append(
|
|
642
|
+
np.arange(
|
|
643
|
+
row_start,
|
|
644
|
+
row_start + pixel_indices.size,
|
|
645
|
+
dtype=np.int64,
|
|
646
|
+
),
|
|
647
|
+
)
|
|
648
|
+
row_start += pixel_indices.size
|
|
649
|
+
columns.append(
|
|
650
|
+
np.full(pixel_indices.shape, trace_index, dtype=np.int64),
|
|
651
|
+
)
|
|
652
|
+
values.append(
|
|
653
|
+
np.full(
|
|
654
|
+
pixel_indices.shape,
|
|
655
|
+
1.0 / float(pixel_indices.size),
|
|
656
|
+
dtype=np.float64,
|
|
657
|
+
),
|
|
658
|
+
)
|
|
659
|
+
if len(selected_pixel_indices) == 0:
|
|
660
|
+
return _MeanProjection(
|
|
661
|
+
pixel_indices=np.array([], dtype=np.int64),
|
|
662
|
+
matrix=sparse.csc_matrix((0, len(pixel_indices_by_trace))),
|
|
663
|
+
)
|
|
664
|
+
return _MeanProjection(
|
|
665
|
+
pixel_indices=np.concatenate(selected_pixel_indices),
|
|
666
|
+
matrix=sparse.csc_matrix(
|
|
667
|
+
(
|
|
668
|
+
np.concatenate(values),
|
|
669
|
+
(np.concatenate(rows), np.concatenate(columns)),
|
|
670
|
+
),
|
|
671
|
+
shape=(row_start, len(pixel_indices_by_trace)),
|
|
672
|
+
),
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def _project_trace_pixels(
|
|
677
|
+
chunk_flat: npt.NDArray[np.float64],
|
|
678
|
+
projection: _MeanProjection,
|
|
679
|
+
) -> npt.NDArray[np.float64]:
|
|
680
|
+
"""Return trace means from a flattened movie chunk and projection."""
|
|
681
|
+
return _project_selected_pixels(chunk_flat[:, projection.pixel_indices], projection)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def _project_selected_pixels(
|
|
685
|
+
selected_pixels: npt.NDArray[np.float64],
|
|
686
|
+
projection: _MeanProjection,
|
|
687
|
+
) -> npt.NDArray[np.float64]:
|
|
688
|
+
"""Return trace means from selected pixel columns."""
|
|
689
|
+
return np.asarray(
|
|
690
|
+
selected_pixels @ projection.matrix,
|
|
691
|
+
dtype=np.float64,
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
|
|
596
695
|
def _check_cancelled(check_cancelled: Callable[[], None] | None) -> None:
|
|
597
696
|
"""Raise ``CancelledError`` when an interactive caller superseded this work."""
|
|
598
697
|
if check_cancelled is None:
|
twopy/analysis/dff.py
CHANGED
|
@@ -115,6 +115,10 @@ def compute_roi_delta_f_over_f(
|
|
|
115
115
|
data_rate_hz=data_rate_hz,
|
|
116
116
|
baseline_sample_seconds=baseline_sample_seconds,
|
|
117
117
|
)
|
|
118
|
+
_validate_baseline_samples_for_background_method(
|
|
119
|
+
traces=traces,
|
|
120
|
+
baseline_samples=baseline_samples,
|
|
121
|
+
)
|
|
118
122
|
tau = _fit_shared_tau(
|
|
119
123
|
frame_numbers=baseline_samples.frame_numbers,
|
|
120
124
|
mean_fluorescence=baseline_samples.values.mean(axis=1),
|
|
@@ -274,6 +278,35 @@ def _baseline_samples(
|
|
|
274
278
|
)
|
|
275
279
|
|
|
276
280
|
|
|
281
|
+
def _validate_baseline_samples_for_background_method(
|
|
282
|
+
*,
|
|
283
|
+
traces: BackgroundCorrectedRoiTraces,
|
|
284
|
+
baseline_samples: _BaselineSamples,
|
|
285
|
+
) -> None:
|
|
286
|
+
"""Reject background-corrected baselines that cannot support dF/F."""
|
|
287
|
+
if traces.method != "roi_y_stripe_percentile":
|
|
288
|
+
return
|
|
289
|
+
|
|
290
|
+
invalid = ~np.isfinite(baseline_samples.values) | (baseline_samples.values <= 0.0)
|
|
291
|
+
if not np.any(invalid):
|
|
292
|
+
return
|
|
293
|
+
|
|
294
|
+
roi_indices = np.flatnonzero(np.any(invalid, axis=0))
|
|
295
|
+
labels = ", ".join(traces.labels[index] for index in roi_indices[:5])
|
|
296
|
+
if roi_indices.size > 5:
|
|
297
|
+
labels = f"{labels}, ..."
|
|
298
|
+
minimum = float(np.nanmin(baseline_samples.values[:, roi_indices]))
|
|
299
|
+
msg = (
|
|
300
|
+
"ROI y-stripe P% background subtraction over-subtracted baseline "
|
|
301
|
+
f"fluorescence for {labels}; minimum corrected baseline sample is "
|
|
302
|
+
f"{minimum:.6g}. dF/F needs positive baseline fluorescence, so the "
|
|
303
|
+
"local y-stripe pixels are not a valid additive background for those "
|
|
304
|
+
"ROIs. Use shared y-stripe P%, global percentile, or revise the ROIs "
|
|
305
|
+
"to leave dim local background pixels."
|
|
306
|
+
)
|
|
307
|
+
raise ValueError(msg)
|
|
308
|
+
|
|
309
|
+
|
|
277
310
|
def _validate_baseline_local_range(
|
|
278
311
|
*,
|
|
279
312
|
window: FrameWindow,
|
|
@@ -80,8 +80,8 @@ class StimulusKernelOptions:
|
|
|
80
80
|
baseline_epoch_number: int = 1
|
|
81
81
|
discard_first_stimulus_epoch: bool = True
|
|
82
82
|
selected_epoch_numbers: tuple[int, ...] | None = None
|
|
83
|
-
num_stim_past: int =
|
|
84
|
-
num_stim_future: int =
|
|
83
|
+
num_stim_past: int = 100
|
|
84
|
+
num_stim_future: int = 10
|
|
85
85
|
method: KernelFitMethod = "ols"
|
|
86
86
|
hemisphere: Hemisphere | None = None
|
|
87
87
|
|
twopy/analysis/persistence.py
CHANGED
|
@@ -48,6 +48,10 @@ from twopy.hdf5_utils import (
|
|
|
48
48
|
write_string_dataset as _write_string_dataset,
|
|
49
49
|
)
|
|
50
50
|
from twopy.roi import RoiSet, TraceStatistic, make_roi_set
|
|
51
|
+
from twopy.spatial_orientation import (
|
|
52
|
+
require_twopy_spatial_orientation,
|
|
53
|
+
write_twopy_spatial_orientation,
|
|
54
|
+
)
|
|
51
55
|
from twopy.typing_guards import (
|
|
52
56
|
require_bool_array,
|
|
53
57
|
require_float64_array,
|
|
@@ -238,7 +242,9 @@ def load_analysis_outputs(path: Path) -> LoadedAnalysisOutputs:
|
|
|
238
242
|
return LoadedAnalysisOutputs(
|
|
239
243
|
path=input_path,
|
|
240
244
|
roi_set=(
|
|
241
|
-
_read_roi_set(h5_file["roi_set"])
|
|
245
|
+
_read_roi_set(h5_file["roi_set"], path=input_path)
|
|
246
|
+
if "roi_set" in h5_file
|
|
247
|
+
else None
|
|
242
248
|
),
|
|
243
249
|
traces=(
|
|
244
250
|
_read_background_traces(h5_file["traces"])
|
|
@@ -347,19 +353,28 @@ def _write_roi_set(group: h5py.Group, roi_set: RoiSet) -> None:
|
|
|
347
353
|
Returns:
|
|
348
354
|
None.
|
|
349
355
|
"""
|
|
350
|
-
group
|
|
356
|
+
write_twopy_spatial_orientation(group)
|
|
357
|
+
masks = group.create_dataset("masks", data=roi_set.masks, compression="gzip")
|
|
358
|
+
write_twopy_spatial_orientation(masks)
|
|
351
359
|
_write_string_dataset(group, "labels", roi_set.labels)
|
|
352
360
|
|
|
353
361
|
|
|
354
|
-
def _read_roi_set(group: h5py.Group) -> RoiSet:
|
|
362
|
+
def _read_roi_set(group: h5py.Group, *, path: Path) -> RoiSet:
|
|
355
363
|
"""Read a persisted ROI set.
|
|
356
364
|
|
|
357
365
|
Args:
|
|
358
366
|
group: HDF5 ``roi_set`` group.
|
|
367
|
+
path: Analysis output path used in validation errors.
|
|
359
368
|
|
|
360
369
|
Returns:
|
|
361
370
|
``RoiSet`` with validated masks and labels.
|
|
362
371
|
"""
|
|
372
|
+
require_twopy_spatial_orientation(group, path=path, label="roi_set")
|
|
373
|
+
require_twopy_spatial_orientation(
|
|
374
|
+
group["masks"],
|
|
375
|
+
path=path,
|
|
376
|
+
label="roi_set/masks",
|
|
377
|
+
)
|
|
363
378
|
return make_roi_set(
|
|
364
379
|
require_bool_array(group["masks"][()], name="roi_set/masks", ndim=3),
|
|
365
380
|
labels=_read_string_dataset(group, "labels"),
|
|
@@ -869,7 +884,7 @@ def _trial_response_row(
|
|
|
869
884
|
"stop_frame": trial.stop_frame,
|
|
870
885
|
"frame_count": trial.stop_frame - trial.start_frame,
|
|
871
886
|
}
|
|
872
|
-
row.update(
|
|
887
|
+
row.update(dict.fromkeys(time_columns, ""))
|
|
873
888
|
for time_seconds, value in zip(
|
|
874
889
|
trial.time_seconds,
|
|
875
890
|
trial.values[:, roi_index],
|
twopy/analysis/response_maps.py
CHANGED
|
@@ -43,6 +43,10 @@ from twopy.analysis.timing import resolve_recording_timing
|
|
|
43
43
|
from twopy.analysis.trials import EpochFrameWindow, is_baseline_epoch_name
|
|
44
44
|
from twopy.converted import RecordingData, recording_frame_rate_hz
|
|
45
45
|
from twopy.spatial import SpatialCrop
|
|
46
|
+
from twopy.spatial_orientation import (
|
|
47
|
+
require_twopy_spatial_orientation,
|
|
48
|
+
write_twopy_spatial_orientation,
|
|
49
|
+
)
|
|
46
50
|
|
|
47
51
|
__all__ = [
|
|
48
52
|
"EpochResponseMap",
|
|
@@ -247,29 +251,33 @@ def save_response_map_data(path: Path, map_data: ResponseMapData) -> None:
|
|
|
247
251
|
recording-level movie products. Values are stored as normalized signed maps
|
|
248
252
|
with the original dF/F scale in ``response_scale`` for audit.
|
|
249
253
|
"""
|
|
254
|
+
_require_response_map_shapes(map_data)
|
|
250
255
|
output_path = path.expanduser()
|
|
251
256
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
252
257
|
with h5py.File(output_path, "w") as h5_file:
|
|
253
258
|
h5_file.attrs["twopy_format"] = RESPONSE_MAP_FILE_FORMAT
|
|
254
259
|
h5_file.attrs["response_scale"] = map_data.response_scale
|
|
260
|
+
write_twopy_spatial_orientation(h5_file)
|
|
255
261
|
_write_response_map_options(h5_file.create_group("options"), map_data.options)
|
|
256
262
|
_write_spatial_crop(h5_file.create_group("spatial_crop"), map_data.spatial_crop)
|
|
257
|
-
h5_file.create_dataset(
|
|
263
|
+
mean_image = h5_file.create_dataset(
|
|
258
264
|
"mean_image",
|
|
259
265
|
data=map_data.mean_image,
|
|
260
266
|
compression="gzip",
|
|
261
267
|
)
|
|
268
|
+
write_twopy_spatial_orientation(mean_image)
|
|
262
269
|
epochs_group = h5_file.create_group("epochs")
|
|
263
270
|
for index, epoch in enumerate(map_data.epochs):
|
|
264
271
|
epoch_group = epochs_group.create_group(f"{index:04d}")
|
|
265
272
|
epoch_group.attrs["epoch_name"] = epoch.epoch_name
|
|
266
273
|
epoch_group.attrs["epoch_number"] = epoch.epoch_number
|
|
267
274
|
epoch_group.attrs["trial_count"] = epoch.trial_count
|
|
268
|
-
epoch_group.create_dataset(
|
|
275
|
+
response_values = epoch_group.create_dataset(
|
|
269
276
|
"response_values",
|
|
270
277
|
data=epoch.response_values,
|
|
271
278
|
compression="gzip",
|
|
272
279
|
)
|
|
280
|
+
write_twopy_spatial_orientation(response_values)
|
|
273
281
|
|
|
274
282
|
|
|
275
283
|
def load_response_map_data(path: Path) -> ResponseMapData:
|
|
@@ -287,21 +295,34 @@ def load_response_map_data(path: Path) -> ResponseMapData:
|
|
|
287
295
|
input_path = path.expanduser()
|
|
288
296
|
with h5py.File(input_path, "r") as h5_file:
|
|
289
297
|
_require_response_map_format(h5_file, input_path)
|
|
298
|
+
require_twopy_spatial_orientation(
|
|
299
|
+
h5_file,
|
|
300
|
+
path=input_path,
|
|
301
|
+
label="response heatmap file",
|
|
302
|
+
)
|
|
290
303
|
options = _read_response_map_options(h5_file["options"])
|
|
291
304
|
spatial_crop = _read_spatial_crop(h5_file["spatial_crop"])
|
|
292
|
-
|
|
305
|
+
mean_image_dataset = h5_file["mean_image"]
|
|
306
|
+
require_twopy_spatial_orientation(
|
|
307
|
+
mean_image_dataset,
|
|
308
|
+
path=input_path,
|
|
309
|
+
label="mean_image",
|
|
310
|
+
)
|
|
311
|
+
mean_image = np.asarray(mean_image_dataset, dtype=np.float64)
|
|
293
312
|
epochs_group = h5_file["epochs"]
|
|
294
313
|
epochs = tuple(
|
|
295
|
-
_read_epoch_response_map(epochs_group[name])
|
|
314
|
+
_read_epoch_response_map(epochs_group[name], path=input_path)
|
|
296
315
|
for name in sorted(epochs_group.keys())
|
|
297
316
|
)
|
|
298
|
-
|
|
317
|
+
map_data = ResponseMapData(
|
|
299
318
|
mean_image=mean_image,
|
|
300
319
|
epochs=epochs,
|
|
301
320
|
options=options,
|
|
302
321
|
spatial_crop=spatial_crop,
|
|
303
322
|
response_scale=float(h5_file.attrs["response_scale"]),
|
|
304
323
|
)
|
|
324
|
+
_require_response_map_shapes(map_data)
|
|
325
|
+
return map_data
|
|
305
326
|
|
|
306
327
|
|
|
307
328
|
def _write_response_map_options(
|
|
@@ -357,16 +378,45 @@ def _read_spatial_crop(group: h5py.Group) -> SpatialCrop:
|
|
|
357
378
|
)
|
|
358
379
|
|
|
359
380
|
|
|
360
|
-
def _read_epoch_response_map(group: h5py.Group) -> EpochResponseMap:
|
|
381
|
+
def _read_epoch_response_map(group: h5py.Group, *, path: Path) -> EpochResponseMap:
|
|
361
382
|
"""Read one persisted epoch response map."""
|
|
383
|
+
response_values = group["response_values"]
|
|
384
|
+
require_twopy_spatial_orientation(
|
|
385
|
+
response_values,
|
|
386
|
+
path=path,
|
|
387
|
+
label=f"{group.name}/response_values",
|
|
388
|
+
)
|
|
362
389
|
return EpochResponseMap(
|
|
363
390
|
epoch_name=_string_attr(group, "epoch_name"),
|
|
364
391
|
epoch_number=int(group.attrs["epoch_number"]),
|
|
365
|
-
response_values=np.asarray(
|
|
392
|
+
response_values=np.asarray(response_values, dtype=np.float64),
|
|
366
393
|
trial_count=int(group.attrs["trial_count"]),
|
|
367
394
|
)
|
|
368
395
|
|
|
369
396
|
|
|
397
|
+
def _require_response_map_shapes(map_data: ResponseMapData) -> None:
|
|
398
|
+
"""Require heatmap spatial arrays to match their recorded crop."""
|
|
399
|
+
if map_data.mean_image.ndim != 2:
|
|
400
|
+
msg = (
|
|
401
|
+
f"response heatmap mean_image must be 2-D; got {map_data.mean_image.shape}"
|
|
402
|
+
)
|
|
403
|
+
raise ValueError(msg)
|
|
404
|
+
if map_data.mean_image.shape != map_data.spatial_crop.shape:
|
|
405
|
+
msg = (
|
|
406
|
+
"response heatmap mean_image shape must match spatial_crop shape; "
|
|
407
|
+
f"got {map_data.mean_image.shape} and {map_data.spatial_crop.shape}"
|
|
408
|
+
)
|
|
409
|
+
raise ValueError(msg)
|
|
410
|
+
for epoch in map_data.epochs:
|
|
411
|
+
if epoch.response_values.shape != map_data.mean_image.shape:
|
|
412
|
+
msg = (
|
|
413
|
+
"response heatmap response_values shape must match mean_image "
|
|
414
|
+
f"shape; got {epoch.response_values.shape} for "
|
|
415
|
+
f"{epoch.epoch_name!r} and {map_data.mean_image.shape}"
|
|
416
|
+
)
|
|
417
|
+
raise ValueError(msg)
|
|
418
|
+
|
|
419
|
+
|
|
370
420
|
def _require_response_map_format(h5_file: h5py.File, path: Path) -> None:
|
|
371
421
|
"""Raise when an HDF5 file is not a twopy response heatmap file."""
|
|
372
422
|
actual = _string_attr(h5_file, "twopy_format")
|
twopy/analysis_cache.py
CHANGED
|
@@ -4,12 +4,13 @@ Inputs: configured recording roots, local cache folders, and saved analysis file
|
|
|
4
4
|
Outputs: cache refreshes and publish-sync plans for twopy-owned outputs.
|
|
5
5
|
|
|
6
6
|
The cache keeps interactive reads and writes on local storage. The publish sync
|
|
7
|
-
copies
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
7
|
+
copies twopy-owned converted files and user-visible analysis outputs back to the
|
|
8
|
+
configured output folder. Exported figure files are removed from the local cache
|
|
9
|
+
after they copy successfully because the publish destination is their durable
|
|
10
|
+
location.
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
|
+
import os
|
|
13
14
|
import shutil
|
|
14
15
|
import tempfile
|
|
15
16
|
from concurrent.futures import Executor, Future
|
|
@@ -24,7 +25,9 @@ from twopy.config import (
|
|
|
24
25
|
)
|
|
25
26
|
from twopy.converted import RecordingData
|
|
26
27
|
from twopy.filenames import (
|
|
28
|
+
ALIGNED_MOVIE_FILENAME,
|
|
27
29
|
ANALYSIS_OUTPUT_FILENAME,
|
|
30
|
+
RECORDING_DATA_FILENAME,
|
|
28
31
|
RESPONSE_HEATMAPS_FILENAME,
|
|
29
32
|
ROI_FILENAME,
|
|
30
33
|
)
|
|
@@ -35,11 +38,14 @@ __all__ = [
|
|
|
35
38
|
"build_analysis_sync_plan",
|
|
36
39
|
"copy_file_atomically",
|
|
37
40
|
"copy_analysis_sync_plan",
|
|
41
|
+
"copy_converted_files_to_publish",
|
|
38
42
|
"refresh_cached_analysis_outputs",
|
|
39
43
|
"start_analysis_sync",
|
|
40
44
|
]
|
|
41
45
|
|
|
42
46
|
_CACHE_REFRESH_FILENAMES = (
|
|
47
|
+
RECORDING_DATA_FILENAME,
|
|
48
|
+
ALIGNED_MOVIE_FILENAME,
|
|
43
49
|
ROI_FILENAME,
|
|
44
50
|
ANALYSIS_OUTPUT_FILENAME,
|
|
45
51
|
RESPONSE_HEATMAPS_FILENAME,
|
|
@@ -90,8 +96,8 @@ def refresh_cached_analysis_outputs(
|
|
|
90
96
|
Local paths refreshed from published outputs.
|
|
91
97
|
|
|
92
98
|
Missing config or disabled caching means there is no cache refresh to do.
|
|
93
|
-
|
|
94
|
-
|
|
99
|
+
HDF5 files needed by the GUI are pulled down; CSV and image exports are
|
|
100
|
+
regenerated from saved analysis when needed.
|
|
95
101
|
"""
|
|
96
102
|
try:
|
|
97
103
|
config = load_config(config_path)
|
|
@@ -136,21 +142,60 @@ def build_analysis_sync_plan(
|
|
|
136
142
|
config_path: YAML config file with cache and publish settings.
|
|
137
143
|
|
|
138
144
|
Returns:
|
|
139
|
-
A sync plan, or ``None`` when
|
|
140
|
-
|
|
145
|
+
A sync plan, or ``None`` when config is missing, the publish root is the
|
|
146
|
+
local root, or no syncable files were written.
|
|
141
147
|
"""
|
|
142
148
|
try:
|
|
143
149
|
config = load_config(config_path)
|
|
144
150
|
except FileNotFoundError:
|
|
145
151
|
return None
|
|
146
152
|
try:
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
153
|
+
plan = _build_sync_plan_from_config(
|
|
154
|
+
source_session_dir=recording.source_session_dir,
|
|
155
|
+
local_root=recording.path.expanduser().parent,
|
|
156
|
+
local_paths=(*_converted_recording_paths(recording), *local_paths),
|
|
150
157
|
config=config,
|
|
151
158
|
)
|
|
152
159
|
except ValueError:
|
|
153
160
|
return None
|
|
161
|
+
if len(plan.local_paths) == 0:
|
|
162
|
+
return None
|
|
163
|
+
return plan
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def copy_converted_files_to_publish(
|
|
167
|
+
*,
|
|
168
|
+
recording_data_path: Path,
|
|
169
|
+
movie_path: Path,
|
|
170
|
+
source_session_dir: Path,
|
|
171
|
+
config_path: Path = DEFAULT_CONFIG_PATH,
|
|
172
|
+
) -> AnalysisSyncResult | None:
|
|
173
|
+
"""Copy converted recording files to the configured publish folder.
|
|
174
|
+
|
|
175
|
+
Args:
|
|
176
|
+
recording_data_path: Local ``recording_data.h5`` file.
|
|
177
|
+
movie_path: Local ``aligned_movie.h5`` file.
|
|
178
|
+
source_session_dir: Source microscope recording folder that defines the
|
|
179
|
+
configured publish location.
|
|
180
|
+
config_path: YAML config file with cache and publish settings.
|
|
181
|
+
|
|
182
|
+
Returns:
|
|
183
|
+
Sync result, or ``None`` when config is missing, the publish root is the
|
|
184
|
+
local root, or publish routing does not apply.
|
|
185
|
+
"""
|
|
186
|
+
try:
|
|
187
|
+
config = load_config(config_path)
|
|
188
|
+
plan = _build_sync_plan_from_config(
|
|
189
|
+
source_session_dir=source_session_dir,
|
|
190
|
+
local_root=recording_data_path.expanduser().parent,
|
|
191
|
+
local_paths=(recording_data_path, movie_path),
|
|
192
|
+
config=config,
|
|
193
|
+
)
|
|
194
|
+
except (FileNotFoundError, ValueError):
|
|
195
|
+
return None
|
|
196
|
+
if len(plan.local_paths) == 0:
|
|
197
|
+
return None
|
|
198
|
+
return copy_analysis_sync_plan(plan)
|
|
154
199
|
|
|
155
200
|
|
|
156
201
|
def start_analysis_sync(
|
|
@@ -193,6 +238,8 @@ def copy_analysis_sync_plan(plan: AnalysisSyncPlan) -> AnalysisSyncResult:
|
|
|
193
238
|
publish_path = plan.publish_root / relative
|
|
194
239
|
if _same_path(local_path, publish_path):
|
|
195
240
|
continue
|
|
241
|
+
if _copy_is_current(local_path, publish_path):
|
|
242
|
+
continue
|
|
196
243
|
copy_file_atomically(local_path, publish_path)
|
|
197
244
|
copied.append(publish_path)
|
|
198
245
|
if _should_remove_after_sync(local_path):
|
|
@@ -205,20 +252,21 @@ def copy_analysis_sync_plan(plan: AnalysisSyncPlan) -> AnalysisSyncResult:
|
|
|
205
252
|
)
|
|
206
253
|
|
|
207
254
|
|
|
208
|
-
def
|
|
255
|
+
def _build_sync_plan_from_config(
|
|
209
256
|
*,
|
|
210
|
-
|
|
257
|
+
source_session_dir: Path,
|
|
258
|
+
local_root: Path,
|
|
211
259
|
local_paths: tuple[Path | None, ...],
|
|
212
260
|
config: TwopyConfig,
|
|
213
|
-
) -> AnalysisSyncPlan
|
|
261
|
+
) -> AnalysisSyncPlan:
|
|
214
262
|
"""Return a sync plan from an already-loaded config."""
|
|
215
|
-
|
|
216
|
-
return None
|
|
217
|
-
|
|
218
|
-
local_root = recording.path.expanduser().parent
|
|
219
|
-
publish_root = resolve_analysis_output_dir(config, recording.source_session_dir)
|
|
263
|
+
publish_root = resolve_analysis_output_dir(config, source_session_dir)
|
|
220
264
|
if _same_path(local_root, publish_root):
|
|
221
|
-
return
|
|
265
|
+
return AnalysisSyncPlan(
|
|
266
|
+
local_root=local_root,
|
|
267
|
+
publish_root=publish_root,
|
|
268
|
+
local_paths=(),
|
|
269
|
+
)
|
|
222
270
|
|
|
223
271
|
syncable: list[Path] = []
|
|
224
272
|
seen: set[Path] = set()
|
|
@@ -235,8 +283,6 @@ def _build_analysis_sync_plan_from_config(
|
|
|
235
283
|
seen.add(local_path)
|
|
236
284
|
syncable.append(local_path)
|
|
237
285
|
|
|
238
|
-
if len(syncable) == 0:
|
|
239
|
-
return None
|
|
240
286
|
return AnalysisSyncPlan(
|
|
241
287
|
local_root=local_root,
|
|
242
288
|
publish_root=publish_root,
|
|
@@ -266,7 +312,7 @@ def copy_file_atomically(source: Path, target: Path) -> None:
|
|
|
266
312
|
temp_path = Path(temp_name)
|
|
267
313
|
try:
|
|
268
314
|
with (
|
|
269
|
-
|
|
315
|
+
os.fdopen(temp_descriptor, "wb") as temp_file,
|
|
270
316
|
source.open(
|
|
271
317
|
"rb",
|
|
272
318
|
) as source_file,
|
|
@@ -286,6 +332,23 @@ def _same_path(left: Path, right: Path) -> bool:
|
|
|
286
332
|
)
|
|
287
333
|
|
|
288
334
|
|
|
335
|
+
def _copy_is_current(source: Path, target: Path) -> bool:
|
|
336
|
+
"""Return whether the published file already matches this source version."""
|
|
337
|
+
if not target.is_file():
|
|
338
|
+
return False
|
|
339
|
+
source_stat = source.stat()
|
|
340
|
+
target_stat = target.stat()
|
|
341
|
+
return (
|
|
342
|
+
target_stat.st_size == source_stat.st_size
|
|
343
|
+
and target_stat.st_mtime_ns >= source_stat.st_mtime_ns
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _converted_recording_paths(recording: RecordingData) -> tuple[Path, Path]:
|
|
348
|
+
"""Return the converted HDF5 files that travel with every publish sync."""
|
|
349
|
+
return (recording.path, recording.movie.path)
|
|
350
|
+
|
|
351
|
+
|
|
289
352
|
def _should_remove_after_sync(path: Path) -> bool:
|
|
290
353
|
"""Return whether a local cache file is disposable after publish sync."""
|
|
291
354
|
return path.suffix.lower() in _REMOVE_AFTER_SYNC_SUFFIXES
|