TwoPy 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. twopy/analysis/background_subtraction.py +120 -21
  2. twopy/analysis/dff.py +33 -0
  3. twopy/analysis/kernels/__init__.py +2 -2
  4. twopy/analysis/persistence.py +19 -4
  5. twopy/analysis/response_maps.py +57 -7
  6. twopy/analysis_cache.py +86 -23
  7. twopy/config.py +47 -8
  8. twopy/conversion/alignment_crop.py +18 -13
  9. twopy/conversion/api.py +21 -5
  10. twopy/conversion/hdf5_writing.py +27 -4
  11. twopy/conversion/orientation.py +80 -0
  12. twopy/conversion/types.py +1 -1
  13. twopy/converted.py +26 -3
  14. twopy/custom/__init__.py +8 -0
  15. twopy/custom/native_workflows/direction_selectivity.py +13 -12
  16. twopy/custom/native_workflows/response_kernels.py +2 -7
  17. twopy/custom/parameters.py +4 -3
  18. twopy/custom/provenance.py +14 -0
  19. twopy/custom/runner.py +244 -0
  20. twopy/custom/types.py +247 -8
  21. twopy/database/search.py +1 -1
  22. twopy/napari/controls.py +147 -26
  23. twopy/napari/custom_tab.py +168 -20
  24. twopy/napari/database_search.py +41 -1
  25. twopy/napari/display.py +10 -85
  26. twopy/napari/display_paths.py +26 -0
  27. twopy/napari/group_matching/__init__.py +29 -0
  28. twopy/napari/group_matching/cards.py +127 -0
  29. twopy/napari/group_matching/fov_assignment.py +679 -0
  30. twopy/napari/group_matching/fov_cards.py +143 -0
  31. twopy/napari/{group_matching_images.py → group_matching/images.py} +2 -2
  32. twopy/napari/group_matching/response_preview.py +334 -0
  33. twopy/napari/group_matching/responses.py +130 -0
  34. twopy/napari/{group_matching_roi.py → group_matching/roi_assignment.py} +590 -532
  35. twopy/napari/group_matching/roi_cards.py +371 -0
  36. twopy/napari/group_matching/style.py +269 -0
  37. twopy/napari/group_matching/tables.py +27 -0
  38. twopy/napari/group_matching/window.py +267 -0
  39. twopy/napari/live_analysis.py +75 -32
  40. twopy/napari/load_controller.py +400 -0
  41. twopy/napari/load_progress.py +168 -0
  42. twopy/napari/load_workflow.py +225 -83
  43. twopy/napari/loaded_recordings_csv.py +35 -4
  44. twopy/napari/loading.py +59 -3
  45. twopy/napari/plotting/dff_options.py +1 -0
  46. twopy/napari/plotting/docks/dynamic_options.py +60 -14
  47. twopy/napari/plotting/docks/options_panel.py +15 -16
  48. twopy/napari/plotting/docks/paths.py +5 -6
  49. twopy/napari/plotting/docks/plot_area.py +42 -0
  50. twopy/napari/plotting/docks/response_map_area.py +5 -8
  51. twopy/napari/plotting/docks/response_plot_widget.py +255 -101
  52. twopy/napari/plotting/docks/visibility_state.py +29 -2
  53. twopy/napari/plotting/export.py +11 -19
  54. twopy/napari/plotting/export_controls.py +5 -6
  55. twopy/napari/plotting/normalization_options.py +1 -0
  56. twopy/napari/plotting/options.py +1 -1
  57. twopy/napari/plotting/panels.py +90 -5
  58. twopy/napari/plotting/preview_strip.py +218 -0
  59. twopy/napari/plotting/processing_options.py +1 -0
  60. twopy/napari/plotting/widgets.py +403 -26
  61. twopy/napari/responses.py +58 -1
  62. twopy/napari/roi.py +91 -22
  63. twopy/napari/roi_signatures.py +20 -0
  64. twopy/napari/session.py +20 -3
  65. twopy/napari/sidebar.py +27 -14
  66. twopy/napari/state.py +113 -27
  67. twopy/napari/viewer.py +206 -67
  68. twopy/roi.py +26 -5
  69. twopy/spatial.py +3 -3
  70. twopy/spatial_orientation.py +78 -0
  71. {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/METADATA +8 -4
  72. {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/RECORD +75 -59
  73. twopy/napari/group_matching.py +0 -596
  74. {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/WHEEL +0 -0
  75. {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/entry_points.txt +0 -0
  76. {twopy-0.2.0.dist-info → twopy-0.3.0.dist-info}/top_level.txt +0 -0
@@ -15,6 +15,7 @@ from typing import Literal
15
15
 
16
16
  import numpy as np
17
17
  import numpy.typing as npt
18
+ from scipy import sparse
18
19
 
19
20
  from twopy.converted import RecordingData
20
21
  from twopy.frame_ranges import normalize_frame_range
@@ -278,6 +279,7 @@ def _extract_mean_traces_from_indices(
278
279
  rather than total recording length.
279
280
  """
280
281
  traces = np.empty((frame_stop - frame_start, len(pixel_indices_by_trace)))
282
+ projection = _mean_projection(pixel_indices_by_trace)
281
283
  _check_cancelled(check_cancelled)
282
284
  for chunk_start, chunk_stop, frames in recording.movie.iter_frame_batches(
283
285
  chunk_frames=chunk_frames,
@@ -289,10 +291,7 @@ def _extract_mean_traces_from_indices(
289
291
  chunk_offset = chunk_start - frame_start
290
292
  chunk_slice = slice(chunk_offset, chunk_offset + (chunk_stop - chunk_start))
291
293
  chunk_flat = frames.reshape(frames.shape[0], -1)
292
- for trace_index, pixel_indices in enumerate(pixel_indices_by_trace):
293
- traces[chunk_slice, trace_index] = chunk_flat[:, pixel_indices].mean(
294
- axis=1,
295
- )
294
+ traces[chunk_slice, :] = _project_trace_pixels(chunk_flat, projection)
296
295
  _check_cancelled(check_cancelled)
297
296
 
298
297
  return traces
@@ -344,6 +343,7 @@ def _extract_movie_global_percentile_corrected_traces(
344
343
  )
345
344
  frame_count = frame_stop - frame_start
346
345
  roi_count = len(roi_pixel_indices)
346
+ projection = _mean_projection(roi_pixel_indices)
347
347
  raw_values = np.empty((frame_count, roi_count), dtype=np.float64)
348
348
  background_values = np.empty((frame_count, roi_count), dtype=np.float64)
349
349
  corrected_values = np.empty((frame_count, roi_count), dtype=np.float64)
@@ -359,14 +359,19 @@ def _extract_movie_global_percentile_corrected_traces(
359
359
  chunk_slice = slice(chunk_offset, chunk_offset + (chunk_stop - chunk_start))
360
360
  chunk_flat = frames.reshape(frames.shape[0], -1)
361
361
  background_trace = chunk_flat[:, background_indices].mean(axis=1)
362
- for roi_index, pixel_indices in enumerate(roi_pixel_indices):
363
- roi_pixels = chunk_flat[:, pixel_indices]
364
- raw_values[chunk_slice, roi_index] = roi_pixels.mean(axis=1)
365
- background_values[chunk_slice, roi_index] = background_trace
366
- # Clamp pixels before averaging so dim pixels cannot pull the
367
- # corrected ROI trace below zero.
368
- corrected_pixels = np.maximum(roi_pixels - background_trace[:, None], 0.0)
369
- corrected_values[chunk_slice, roi_index] = corrected_pixels.mean(axis=1)
362
+ roi_pixels = chunk_flat[:, projection.pixel_indices]
363
+ raw_values[chunk_slice, :] = _project_selected_pixels(roi_pixels, projection)
364
+ background_values[chunk_slice, :] = background_trace[:, np.newaxis]
365
+ # Clamp pixels before averaging so dim pixels cannot pull the
366
+ # corrected ROI trace below zero.
367
+ corrected_pixels = np.maximum(
368
+ roi_pixels - background_trace[:, np.newaxis],
369
+ 0.0,
370
+ )
371
+ corrected_values[chunk_slice, :] = _project_selected_pixels(
372
+ corrected_pixels,
373
+ projection,
374
+ )
370
375
  _check_cancelled(check_cancelled)
371
376
 
372
377
  metadata: dict[str, BackgroundMetadataValue] = {
@@ -440,6 +445,7 @@ def _extract_movie_y_stripe_percentile_corrected_traces(
440
445
  )
441
446
  frame_count = frame_stop - frame_start
442
447
  roi_count = len(roi_pixel_indices)
448
+ projection = _mean_projection(roi_pixel_indices)
443
449
  raw_values = np.empty((frame_count, roi_count), dtype=np.float64)
444
450
  background_values = np.empty((frame_count, roi_count), dtype=np.float64)
445
451
  corrected_values = np.empty((frame_count, roi_count), dtype=np.float64)
@@ -460,15 +466,21 @@ def _extract_movie_y_stripe_percentile_corrected_traces(
460
466
  )
461
467
  chunk_flat = frames.reshape(frames.shape[0], -1)
462
468
  background_flat = background_field.reshape(background_field.shape[0], -1)
463
- for roi_index, pixel_indices in enumerate(roi_pixel_indices):
464
- roi_pixels = chunk_flat[:, pixel_indices]
465
- roi_background_pixels = background_flat[:, pixel_indices]
466
- raw_values[chunk_slice, roi_index] = roi_pixels.mean(axis=1)
467
- background_values[chunk_slice, roi_index] = roi_background_pixels.mean(
468
- axis=1,
469
- )
470
- corrected_pixels = np.maximum(roi_pixels - roi_background_pixels, 0.0)
471
- corrected_values[chunk_slice, roi_index] = corrected_pixels.mean(axis=1)
469
+ roi_pixels = chunk_flat[:, projection.pixel_indices]
470
+ roi_background_pixels = background_flat[:, projection.pixel_indices]
471
+ raw_values[chunk_slice, :] = _project_selected_pixels(roi_pixels, projection)
472
+ background_values[chunk_slice, :] = _project_selected_pixels(
473
+ roi_background_pixels,
474
+ projection,
475
+ )
476
+ corrected_pixels = np.maximum(
477
+ roi_pixels - roi_background_pixels,
478
+ 0.0,
479
+ )
480
+ corrected_values[chunk_slice, :] = _project_selected_pixels(
481
+ corrected_pixels,
482
+ projection,
483
+ )
472
484
  _check_cancelled(check_cancelled)
473
485
 
474
486
  metadata: dict[str, BackgroundMetadataValue] = {
@@ -593,6 +605,93 @@ def _roi_pixel_indices_from_masks(
593
605
  return tuple(np.flatnonzero(mask).astype(np.int64) for mask in flat_masks)
594
606
 
595
607
 
608
+ @dataclass(frozen=True)
609
+ class _MeanProjection:
610
+ """Sparse averaging matrix plus selected flattened pixel indices."""
611
+
612
+ pixel_indices: npt.NDArray[np.int64]
613
+ matrix: sparse.csc_matrix
614
+
615
+
616
+ def _mean_projection(
617
+ pixel_indices_by_trace: tuple[npt.NDArray[np.int64], ...],
618
+ ) -> _MeanProjection:
619
+ """Return selected pixels and a sparse pixel-to-trace averaging matrix.
620
+
621
+ Args:
622
+ pixel_indices_by_trace: One flattened pixel-index array per trace.
623
+
624
+ Returns:
625
+ Selected flattened pixel indices plus a sparse matrix shaped
626
+ ``(selected_pixels, traces)`` whose columns average each trace.
627
+
628
+ Pixel indices are concatenated per trace instead of deduplicated. That
629
+ preserves the existing behavior for overlapping ROIs, where the same movie
630
+ pixel can contribute independently to more than one ROI mean.
631
+ """
632
+ selected_pixel_indices: list[npt.NDArray[np.int64]] = []
633
+ rows: list[npt.NDArray[np.int64]] = []
634
+ columns: list[npt.NDArray[np.int64]] = []
635
+ values: list[npt.NDArray[np.float64]] = []
636
+ row_start = 0
637
+ for trace_index, pixel_indices in enumerate(pixel_indices_by_trace):
638
+ if pixel_indices.size == 0:
639
+ continue
640
+ selected_pixel_indices.append(pixel_indices)
641
+ rows.append(
642
+ np.arange(
643
+ row_start,
644
+ row_start + pixel_indices.size,
645
+ dtype=np.int64,
646
+ ),
647
+ )
648
+ row_start += pixel_indices.size
649
+ columns.append(
650
+ np.full(pixel_indices.shape, trace_index, dtype=np.int64),
651
+ )
652
+ values.append(
653
+ np.full(
654
+ pixel_indices.shape,
655
+ 1.0 / float(pixel_indices.size),
656
+ dtype=np.float64,
657
+ ),
658
+ )
659
+ if len(selected_pixel_indices) == 0:
660
+ return _MeanProjection(
661
+ pixel_indices=np.array([], dtype=np.int64),
662
+ matrix=sparse.csc_matrix((0, len(pixel_indices_by_trace))),
663
+ )
664
+ return _MeanProjection(
665
+ pixel_indices=np.concatenate(selected_pixel_indices),
666
+ matrix=sparse.csc_matrix(
667
+ (
668
+ np.concatenate(values),
669
+ (np.concatenate(rows), np.concatenate(columns)),
670
+ ),
671
+ shape=(row_start, len(pixel_indices_by_trace)),
672
+ ),
673
+ )
674
+
675
+
676
+ def _project_trace_pixels(
677
+ chunk_flat: npt.NDArray[np.float64],
678
+ projection: _MeanProjection,
679
+ ) -> npt.NDArray[np.float64]:
680
+ """Return trace means from a flattened movie chunk and projection."""
681
+ return _project_selected_pixels(chunk_flat[:, projection.pixel_indices], projection)
682
+
683
+
684
+ def _project_selected_pixels(
685
+ selected_pixels: npt.NDArray[np.float64],
686
+ projection: _MeanProjection,
687
+ ) -> npt.NDArray[np.float64]:
688
+ """Return trace means from selected pixel columns."""
689
+ return np.asarray(
690
+ selected_pixels @ projection.matrix,
691
+ dtype=np.float64,
692
+ )
693
+
694
+
596
695
  def _check_cancelled(check_cancelled: Callable[[], None] | None) -> None:
597
696
  """Raise ``CancelledError`` when an interactive caller superseded this work."""
598
697
  if check_cancelled is None:
twopy/analysis/dff.py CHANGED
@@ -115,6 +115,10 @@ def compute_roi_delta_f_over_f(
115
115
  data_rate_hz=data_rate_hz,
116
116
  baseline_sample_seconds=baseline_sample_seconds,
117
117
  )
118
+ _validate_baseline_samples_for_background_method(
119
+ traces=traces,
120
+ baseline_samples=baseline_samples,
121
+ )
118
122
  tau = _fit_shared_tau(
119
123
  frame_numbers=baseline_samples.frame_numbers,
120
124
  mean_fluorescence=baseline_samples.values.mean(axis=1),
@@ -274,6 +278,35 @@ def _baseline_samples(
274
278
  )
275
279
 
276
280
 
281
+ def _validate_baseline_samples_for_background_method(
282
+ *,
283
+ traces: BackgroundCorrectedRoiTraces,
284
+ baseline_samples: _BaselineSamples,
285
+ ) -> None:
286
+ """Reject background-corrected baselines that cannot support dF/F."""
287
+ if traces.method != "roi_y_stripe_percentile":
288
+ return
289
+
290
+ invalid = ~np.isfinite(baseline_samples.values) | (baseline_samples.values <= 0.0)
291
+ if not np.any(invalid):
292
+ return
293
+
294
+ roi_indices = np.flatnonzero(np.any(invalid, axis=0))
295
+ labels = ", ".join(traces.labels[index] for index in roi_indices[:5])
296
+ if roi_indices.size > 5:
297
+ labels = f"{labels}, ..."
298
+ minimum = float(np.nanmin(baseline_samples.values[:, roi_indices]))
299
+ msg = (
300
+ "ROI y-stripe P% background subtraction over-subtracted baseline "
301
+ f"fluorescence for {labels}; minimum corrected baseline sample is "
302
+ f"{minimum:.6g}. dF/F needs positive baseline fluorescence, so the "
303
+ "local y-stripe pixels are not a valid additive background for those "
304
+ "ROIs. Use shared y-stripe P%, global percentile, or revise the ROIs "
305
+ "to leave dim local background pixels."
306
+ )
307
+ raise ValueError(msg)
308
+
309
+
277
310
  def _validate_baseline_local_range(
278
311
  *,
279
312
  window: FrameWindow,
@@ -80,8 +80,8 @@ class StimulusKernelOptions:
80
80
  baseline_epoch_number: int = 1
81
81
  discard_first_stimulus_epoch: bool = True
82
82
  selected_epoch_numbers: tuple[int, ...] | None = None
83
- num_stim_past: int = 150
84
- num_stim_future: int = 25
83
+ num_stim_past: int = 100
84
+ num_stim_future: int = 10
85
85
  method: KernelFitMethod = "ols"
86
86
  hemisphere: Hemisphere | None = None
87
87
 
@@ -48,6 +48,10 @@ from twopy.hdf5_utils import (
48
48
  write_string_dataset as _write_string_dataset,
49
49
  )
50
50
  from twopy.roi import RoiSet, TraceStatistic, make_roi_set
51
+ from twopy.spatial_orientation import (
52
+ require_twopy_spatial_orientation,
53
+ write_twopy_spatial_orientation,
54
+ )
51
55
  from twopy.typing_guards import (
52
56
  require_bool_array,
53
57
  require_float64_array,
@@ -238,7 +242,9 @@ def load_analysis_outputs(path: Path) -> LoadedAnalysisOutputs:
238
242
  return LoadedAnalysisOutputs(
239
243
  path=input_path,
240
244
  roi_set=(
241
- _read_roi_set(h5_file["roi_set"]) if "roi_set" in h5_file else None
245
+ _read_roi_set(h5_file["roi_set"], path=input_path)
246
+ if "roi_set" in h5_file
247
+ else None
242
248
  ),
243
249
  traces=(
244
250
  _read_background_traces(h5_file["traces"])
@@ -347,19 +353,28 @@ def _write_roi_set(group: h5py.Group, roi_set: RoiSet) -> None:
347
353
  Returns:
348
354
  None.
349
355
  """
350
- group.create_dataset("masks", data=roi_set.masks, compression="gzip")
356
+ write_twopy_spatial_orientation(group)
357
+ masks = group.create_dataset("masks", data=roi_set.masks, compression="gzip")
358
+ write_twopy_spatial_orientation(masks)
351
359
  _write_string_dataset(group, "labels", roi_set.labels)
352
360
 
353
361
 
354
- def _read_roi_set(group: h5py.Group) -> RoiSet:
362
+ def _read_roi_set(group: h5py.Group, *, path: Path) -> RoiSet:
355
363
  """Read a persisted ROI set.
356
364
 
357
365
  Args:
358
366
  group: HDF5 ``roi_set`` group.
367
+ path: Analysis output path used in validation errors.
359
368
 
360
369
  Returns:
361
370
  ``RoiSet`` with validated masks and labels.
362
371
  """
372
+ require_twopy_spatial_orientation(group, path=path, label="roi_set")
373
+ require_twopy_spatial_orientation(
374
+ group["masks"],
375
+ path=path,
376
+ label="roi_set/masks",
377
+ )
363
378
  return make_roi_set(
364
379
  require_bool_array(group["masks"][()], name="roi_set/masks", ndim=3),
365
380
  labels=_read_string_dataset(group, "labels"),
@@ -869,7 +884,7 @@ def _trial_response_row(
869
884
  "stop_frame": trial.stop_frame,
870
885
  "frame_count": trial.stop_frame - trial.start_frame,
871
886
  }
872
- row.update({column: "" for column in time_columns})
887
+ row.update(dict.fromkeys(time_columns, ""))
873
888
  for time_seconds, value in zip(
874
889
  trial.time_seconds,
875
890
  trial.values[:, roi_index],
@@ -43,6 +43,10 @@ from twopy.analysis.timing import resolve_recording_timing
43
43
  from twopy.analysis.trials import EpochFrameWindow, is_baseline_epoch_name
44
44
  from twopy.converted import RecordingData, recording_frame_rate_hz
45
45
  from twopy.spatial import SpatialCrop
46
+ from twopy.spatial_orientation import (
47
+ require_twopy_spatial_orientation,
48
+ write_twopy_spatial_orientation,
49
+ )
46
50
 
47
51
  __all__ = [
48
52
  "EpochResponseMap",
@@ -247,29 +251,33 @@ def save_response_map_data(path: Path, map_data: ResponseMapData) -> None:
247
251
  recording-level movie products. Values are stored as normalized signed maps
248
252
  with the original dF/F scale in ``response_scale`` for audit.
249
253
  """
254
+ _require_response_map_shapes(map_data)
250
255
  output_path = path.expanduser()
251
256
  output_path.parent.mkdir(parents=True, exist_ok=True)
252
257
  with h5py.File(output_path, "w") as h5_file:
253
258
  h5_file.attrs["twopy_format"] = RESPONSE_MAP_FILE_FORMAT
254
259
  h5_file.attrs["response_scale"] = map_data.response_scale
260
+ write_twopy_spatial_orientation(h5_file)
255
261
  _write_response_map_options(h5_file.create_group("options"), map_data.options)
256
262
  _write_spatial_crop(h5_file.create_group("spatial_crop"), map_data.spatial_crop)
257
- h5_file.create_dataset(
263
+ mean_image = h5_file.create_dataset(
258
264
  "mean_image",
259
265
  data=map_data.mean_image,
260
266
  compression="gzip",
261
267
  )
268
+ write_twopy_spatial_orientation(mean_image)
262
269
  epochs_group = h5_file.create_group("epochs")
263
270
  for index, epoch in enumerate(map_data.epochs):
264
271
  epoch_group = epochs_group.create_group(f"{index:04d}")
265
272
  epoch_group.attrs["epoch_name"] = epoch.epoch_name
266
273
  epoch_group.attrs["epoch_number"] = epoch.epoch_number
267
274
  epoch_group.attrs["trial_count"] = epoch.trial_count
268
- epoch_group.create_dataset(
275
+ response_values = epoch_group.create_dataset(
269
276
  "response_values",
270
277
  data=epoch.response_values,
271
278
  compression="gzip",
272
279
  )
280
+ write_twopy_spatial_orientation(response_values)
273
281
 
274
282
 
275
283
  def load_response_map_data(path: Path) -> ResponseMapData:
@@ -287,21 +295,34 @@ def load_response_map_data(path: Path) -> ResponseMapData:
287
295
  input_path = path.expanduser()
288
296
  with h5py.File(input_path, "r") as h5_file:
289
297
  _require_response_map_format(h5_file, input_path)
298
+ require_twopy_spatial_orientation(
299
+ h5_file,
300
+ path=input_path,
301
+ label="response heatmap file",
302
+ )
290
303
  options = _read_response_map_options(h5_file["options"])
291
304
  spatial_crop = _read_spatial_crop(h5_file["spatial_crop"])
292
- mean_image = np.asarray(h5_file["mean_image"], dtype=np.float64)
305
+ mean_image_dataset = h5_file["mean_image"]
306
+ require_twopy_spatial_orientation(
307
+ mean_image_dataset,
308
+ path=input_path,
309
+ label="mean_image",
310
+ )
311
+ mean_image = np.asarray(mean_image_dataset, dtype=np.float64)
293
312
  epochs_group = h5_file["epochs"]
294
313
  epochs = tuple(
295
- _read_epoch_response_map(epochs_group[name])
314
+ _read_epoch_response_map(epochs_group[name], path=input_path)
296
315
  for name in sorted(epochs_group.keys())
297
316
  )
298
- return ResponseMapData(
317
+ map_data = ResponseMapData(
299
318
  mean_image=mean_image,
300
319
  epochs=epochs,
301
320
  options=options,
302
321
  spatial_crop=spatial_crop,
303
322
  response_scale=float(h5_file.attrs["response_scale"]),
304
323
  )
324
+ _require_response_map_shapes(map_data)
325
+ return map_data
305
326
 
306
327
 
307
328
  def _write_response_map_options(
@@ -357,16 +378,45 @@ def _read_spatial_crop(group: h5py.Group) -> SpatialCrop:
357
378
  )
358
379
 
359
380
 
360
- def _read_epoch_response_map(group: h5py.Group) -> EpochResponseMap:
381
+ def _read_epoch_response_map(group: h5py.Group, *, path: Path) -> EpochResponseMap:
361
382
  """Read one persisted epoch response map."""
383
+ response_values = group["response_values"]
384
+ require_twopy_spatial_orientation(
385
+ response_values,
386
+ path=path,
387
+ label=f"{group.name}/response_values",
388
+ )
362
389
  return EpochResponseMap(
363
390
  epoch_name=_string_attr(group, "epoch_name"),
364
391
  epoch_number=int(group.attrs["epoch_number"]),
365
- response_values=np.asarray(group["response_values"], dtype=np.float64),
392
+ response_values=np.asarray(response_values, dtype=np.float64),
366
393
  trial_count=int(group.attrs["trial_count"]),
367
394
  )
368
395
 
369
396
 
397
+ def _require_response_map_shapes(map_data: ResponseMapData) -> None:
398
+ """Require heatmap spatial arrays to match their recorded crop."""
399
+ if map_data.mean_image.ndim != 2:
400
+ msg = (
401
+ f"response heatmap mean_image must be 2-D; got {map_data.mean_image.shape}"
402
+ )
403
+ raise ValueError(msg)
404
+ if map_data.mean_image.shape != map_data.spatial_crop.shape:
405
+ msg = (
406
+ "response heatmap mean_image shape must match spatial_crop shape; "
407
+ f"got {map_data.mean_image.shape} and {map_data.spatial_crop.shape}"
408
+ )
409
+ raise ValueError(msg)
410
+ for epoch in map_data.epochs:
411
+ if epoch.response_values.shape != map_data.mean_image.shape:
412
+ msg = (
413
+ "response heatmap response_values shape must match mean_image "
414
+ f"shape; got {epoch.response_values.shape} for "
415
+ f"{epoch.epoch_name!r} and {map_data.mean_image.shape}"
416
+ )
417
+ raise ValueError(msg)
418
+
419
+
370
420
  def _require_response_map_format(h5_file: h5py.File, path: Path) -> None:
371
421
  """Raise when an HDF5 file is not a twopy response heatmap file."""
372
422
  actual = _string_attr(h5_file, "twopy_format")
twopy/analysis_cache.py CHANGED
@@ -4,12 +4,13 @@ Inputs: configured recording roots, local cache folders, and saved analysis file
4
4
  Outputs: cache refreshes and publish-sync plans for twopy-owned outputs.
5
5
 
6
6
  The cache keeps interactive reads and writes on local storage. The publish sync
7
- copies only user-visible analysis outputs back to the configured output folder;
8
- converted movies remain derived local working files by default. Exported figure
9
- files are removed from the local cache after they copy successfully because the
10
- publish destination is their durable location.
7
+ copies twopy-owned converted files and user-visible analysis outputs back to the
8
+ configured output folder. Exported figure files are removed from the local cache
9
+ after they copy successfully because the publish destination is their durable
10
+ location.
11
11
  """
12
12
 
13
+ import os
13
14
  import shutil
14
15
  import tempfile
15
16
  from concurrent.futures import Executor, Future
@@ -24,7 +25,9 @@ from twopy.config import (
24
25
  )
25
26
  from twopy.converted import RecordingData
26
27
  from twopy.filenames import (
28
+ ALIGNED_MOVIE_FILENAME,
27
29
  ANALYSIS_OUTPUT_FILENAME,
30
+ RECORDING_DATA_FILENAME,
28
31
  RESPONSE_HEATMAPS_FILENAME,
29
32
  ROI_FILENAME,
30
33
  )
@@ -35,11 +38,14 @@ __all__ = [
35
38
  "build_analysis_sync_plan",
36
39
  "copy_file_atomically",
37
40
  "copy_analysis_sync_plan",
41
+ "copy_converted_files_to_publish",
38
42
  "refresh_cached_analysis_outputs",
39
43
  "start_analysis_sync",
40
44
  ]
41
45
 
42
46
  _CACHE_REFRESH_FILENAMES = (
47
+ RECORDING_DATA_FILENAME,
48
+ ALIGNED_MOVIE_FILENAME,
43
49
  ROI_FILENAME,
44
50
  ANALYSIS_OUTPUT_FILENAME,
45
51
  RESPONSE_HEATMAPS_FILENAME,
@@ -90,8 +96,8 @@ def refresh_cached_analysis_outputs(
90
96
  Local paths refreshed from published outputs.
91
97
 
92
98
  Missing config or disabled caching means there is no cache refresh to do.
93
- Only the HDF5 files needed by the GUI reload path are pulled down; CSV and
94
- image exports are regenerated from saved analysis when needed.
99
+ HDF5 files needed by the GUI are pulled down; CSV and image exports are
100
+ regenerated from saved analysis when needed.
95
101
  """
96
102
  try:
97
103
  config = load_config(config_path)
@@ -136,21 +142,60 @@ def build_analysis_sync_plan(
136
142
  config_path: YAML config file with cache and publish settings.
137
143
 
138
144
  Returns:
139
- A sync plan, or ``None`` when caching is disabled, config is missing,
140
- the publish root is the local root, or no syncable files were written.
145
+ A sync plan, or ``None`` when config is missing, the publish root is the
146
+ local root, or no syncable files were written.
141
147
  """
142
148
  try:
143
149
  config = load_config(config_path)
144
150
  except FileNotFoundError:
145
151
  return None
146
152
  try:
147
- return _build_analysis_sync_plan_from_config(
148
- recording=recording,
149
- local_paths=local_paths,
153
+ plan = _build_sync_plan_from_config(
154
+ source_session_dir=recording.source_session_dir,
155
+ local_root=recording.path.expanduser().parent,
156
+ local_paths=(*_converted_recording_paths(recording), *local_paths),
150
157
  config=config,
151
158
  )
152
159
  except ValueError:
153
160
  return None
161
+ if len(plan.local_paths) == 0:
162
+ return None
163
+ return plan
164
+
165
+
166
+ def copy_converted_files_to_publish(
167
+ *,
168
+ recording_data_path: Path,
169
+ movie_path: Path,
170
+ source_session_dir: Path,
171
+ config_path: Path = DEFAULT_CONFIG_PATH,
172
+ ) -> AnalysisSyncResult | None:
173
+ """Copy converted recording files to the configured publish folder.
174
+
175
+ Args:
176
+ recording_data_path: Local ``recording_data.h5`` file.
177
+ movie_path: Local ``aligned_movie.h5`` file.
178
+ source_session_dir: Source microscope recording folder that defines the
179
+ configured publish location.
180
+ config_path: YAML config file with cache and publish settings.
181
+
182
+ Returns:
183
+ Sync result, or ``None`` when config is missing, the publish root is the
184
+ local root, or publish routing does not apply.
185
+ """
186
+ try:
187
+ config = load_config(config_path)
188
+ plan = _build_sync_plan_from_config(
189
+ source_session_dir=source_session_dir,
190
+ local_root=recording_data_path.expanduser().parent,
191
+ local_paths=(recording_data_path, movie_path),
192
+ config=config,
193
+ )
194
+ except (FileNotFoundError, ValueError):
195
+ return None
196
+ if len(plan.local_paths) == 0:
197
+ return None
198
+ return copy_analysis_sync_plan(plan)
154
199
 
155
200
 
156
201
  def start_analysis_sync(
@@ -193,6 +238,8 @@ def copy_analysis_sync_plan(plan: AnalysisSyncPlan) -> AnalysisSyncResult:
193
238
  publish_path = plan.publish_root / relative
194
239
  if _same_path(local_path, publish_path):
195
240
  continue
241
+ if _copy_is_current(local_path, publish_path):
242
+ continue
196
243
  copy_file_atomically(local_path, publish_path)
197
244
  copied.append(publish_path)
198
245
  if _should_remove_after_sync(local_path):
@@ -205,20 +252,21 @@ def copy_analysis_sync_plan(plan: AnalysisSyncPlan) -> AnalysisSyncResult:
205
252
  )
206
253
 
207
254
 
208
- def _build_analysis_sync_plan_from_config(
255
+ def _build_sync_plan_from_config(
209
256
  *,
210
- recording: RecordingData,
257
+ source_session_dir: Path,
258
+ local_root: Path,
211
259
  local_paths: tuple[Path | None, ...],
212
260
  config: TwopyConfig,
213
- ) -> AnalysisSyncPlan | None:
261
+ ) -> AnalysisSyncPlan:
214
262
  """Return a sync plan from an already-loaded config."""
215
- if not config.analysis_caching:
216
- return None
217
-
218
- local_root = recording.path.expanduser().parent
219
- publish_root = resolve_analysis_output_dir(config, recording.source_session_dir)
263
+ publish_root = resolve_analysis_output_dir(config, source_session_dir)
220
264
  if _same_path(local_root, publish_root):
221
- return None
265
+ return AnalysisSyncPlan(
266
+ local_root=local_root,
267
+ publish_root=publish_root,
268
+ local_paths=(),
269
+ )
222
270
 
223
271
  syncable: list[Path] = []
224
272
  seen: set[Path] = set()
@@ -235,8 +283,6 @@ def _build_analysis_sync_plan_from_config(
235
283
  seen.add(local_path)
236
284
  syncable.append(local_path)
237
285
 
238
- if len(syncable) == 0:
239
- return None
240
286
  return AnalysisSyncPlan(
241
287
  local_root=local_root,
242
288
  publish_root=publish_root,
@@ -266,7 +312,7 @@ def copy_file_atomically(source: Path, target: Path) -> None:
266
312
  temp_path = Path(temp_name)
267
313
  try:
268
314
  with (
269
- open(temp_descriptor, "wb") as temp_file,
315
+ os.fdopen(temp_descriptor, "wb") as temp_file,
270
316
  source.open(
271
317
  "rb",
272
318
  ) as source_file,
@@ -286,6 +332,23 @@ def _same_path(left: Path, right: Path) -> bool:
286
332
  )
287
333
 
288
334
 
335
+ def _copy_is_current(source: Path, target: Path) -> bool:
336
+ """Return whether the published file already matches this source version."""
337
+ if not target.is_file():
338
+ return False
339
+ source_stat = source.stat()
340
+ target_stat = target.stat()
341
+ return (
342
+ target_stat.st_size == source_stat.st_size
343
+ and target_stat.st_mtime_ns >= source_stat.st_mtime_ns
344
+ )
345
+
346
+
347
+ def _converted_recording_paths(recording: RecordingData) -> tuple[Path, Path]:
348
+ """Return the converted HDF5 files that travel with every publish sync."""
349
+ return (recording.path, recording.movie.path)
350
+
351
+
289
352
  def _should_remove_after_sync(path: Path) -> bool:
290
353
  """Return whether a local cache file is disposable after publish sync."""
291
354
  return path.suffix.lower() in _REMOVE_AFTER_SYNC_SUFFIXES