optimouse 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modules/__init__.py +21 -0
- modules/bruker_concat_tif.py +650 -0
- modules/compute_zcorr.py +2040 -0
- modules/extraction.py +559 -0
- modules/motion_correction.py +982 -0
- modules/utils/plot_zones.py +125 -0
- modules/utils/save_z_motion_args.py +93 -0
- optimouse-0.1.0.dist-info/METADATA +321 -0
- optimouse-0.1.0.dist-info/RECORD +24 -0
- optimouse-0.1.0.dist-info/WHEEL +5 -0
- optimouse-0.1.0.dist-info/licenses/LICENSE.md +21 -0
- optimouse-0.1.0.dist-info/top_level.txt +2 -0
- pipeline/__init__.py +0 -0
- pipeline/configs/config_C57_O1M2_10022023_run1run2.json +107 -0
- pipeline/configs/config_C57_O1M2_10022023_run1run2run3run4.json +114 -0
- pipeline/configs/config_template_cnmf.json +123 -0
- pipeline/configs/config_template_suite2p.json +102 -0
- pipeline/configs/config_test.json +114 -0
- pipeline/pipeline_cnmf.py +263 -0
- pipeline/pipeline_mcorr.py +299 -0
- pipeline/roi_zcorr.py +65 -0
- pipeline/utils/__init__.py +0 -0
- pipeline/utils/config_loader.py +92 -0
- pipeline/utils/pipeline_utils.py +537 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Utilities for loading pipeline configuration files with extras.
|
|
2
|
+
|
|
3
|
+
Supports:
|
|
4
|
+
- `_env` blocks for defining reusable path prefixes or variables.
|
|
5
|
+
- `${VAR}` style environment substitution in all string fields.
|
|
6
|
+
- `__include` directives to merge reusable JSON snippets.
|
|
7
|
+
- Metadata keys prefixed with `__` that should be ignored downstream.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Dict
|
|
17
|
+
|
|
18
|
+
_VAR_PATTERN = re.compile(r"\$\{([^}]+)\}")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _expand_string(value: str, env: Dict[str, str]) -> str:
|
|
22
|
+
"""Expand ${VAR} placeholders using the provided environment mapping."""
|
|
23
|
+
|
|
24
|
+
def _replace(match: re.Match[str]) -> str:
|
|
25
|
+
key = match.group(1)
|
|
26
|
+
if key in env:
|
|
27
|
+
return env[key]
|
|
28
|
+
return os.environ.get(key, match.group(0))
|
|
29
|
+
|
|
30
|
+
return _VAR_PATTERN.sub(_replace, value)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _process_node(node: Any, base_dir: Path, env: Dict[str, str]) -> Any:
|
|
34
|
+
if isinstance(node, dict):
|
|
35
|
+
return _process_dict(node, base_dir, env)
|
|
36
|
+
if isinstance(node, list):
|
|
37
|
+
return [_process_node(item, base_dir, env) for item in node]
|
|
38
|
+
if isinstance(node, str):
|
|
39
|
+
return _expand_string(node, env)
|
|
40
|
+
return node
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _process_dict(data: Dict[str, Any], base_dir: Path, env: Dict[str, str]) -> Dict[str, Any]:
|
|
44
|
+
# Allow local environment overrides
|
|
45
|
+
local_env = env
|
|
46
|
+
if "_env" in data:
|
|
47
|
+
env_block = data.get("_env") or {}
|
|
48
|
+
local_env = env.copy()
|
|
49
|
+
for key, raw_val in env_block.items():
|
|
50
|
+
if key.startswith("__"):
|
|
51
|
+
continue
|
|
52
|
+
if raw_val is None:
|
|
53
|
+
continue
|
|
54
|
+
value = _expand_string(str(raw_val), local_env)
|
|
55
|
+
local_env[key] = value
|
|
56
|
+
|
|
57
|
+
result: Dict[str, Any] = {}
|
|
58
|
+
|
|
59
|
+
# Handle include directives before processing the rest of the keys
|
|
60
|
+
include_target = data.get("__include")
|
|
61
|
+
if include_target is not None:
|
|
62
|
+
include_path = _expand_string(str(include_target), local_env)
|
|
63
|
+
include_file = (base_dir / include_path).resolve()
|
|
64
|
+
if not include_file.exists():
|
|
65
|
+
raise FileNotFoundError(f"Included config file not found: {include_file}")
|
|
66
|
+
with include_file.open("r") as include_fp:
|
|
67
|
+
include_raw = json.load(include_fp)
|
|
68
|
+
included = _process_node(include_raw, include_file.parent, local_env)
|
|
69
|
+
if not isinstance(included, dict):
|
|
70
|
+
raise ValueError("Included JSON must define an object at the top level")
|
|
71
|
+
result.update(included)
|
|
72
|
+
|
|
73
|
+
for key, value in data.items():
|
|
74
|
+
if key in {"_env", "__include"}:
|
|
75
|
+
continue
|
|
76
|
+
if key.startswith("__"):
|
|
77
|
+
# Metadata keys are ignored downstream
|
|
78
|
+
continue
|
|
79
|
+
result[key] = _process_node(value, base_dir, local_env)
|
|
80
|
+
|
|
81
|
+
return result
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def load_config(config_path: str | Path) -> Dict[str, Any]:
|
|
85
|
+
"""Load a configuration file resolving includes and environment variables."""
|
|
86
|
+
|
|
87
|
+
path = Path(config_path)
|
|
88
|
+
with path.open("r") as fp:
|
|
89
|
+
data = json.load(fp)
|
|
90
|
+
|
|
91
|
+
initial_env = dict(os.environ)
|
|
92
|
+
return _process_node(data, path.parent, initial_env)
|
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
"""
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os, sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from tifffile import TiffFile
|
|
12
|
+
|
|
13
|
+
# Add project root to path (standardized approach)
|
|
14
|
+
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
|
15
|
+
if str(PROJECT_ROOT) not in sys.path:
|
|
16
|
+
sys.path.insert(0, str(PROJECT_ROOT))
|
|
17
|
+
|
|
18
|
+
from caiman.mmapping import load_memmap
|
|
19
|
+
from caiman.paths import decode_mmap_filename_dict
|
|
20
|
+
import logging
|
|
21
|
+
import numpy as np
|
|
22
|
+
import time
|
|
23
|
+
import psutil
|
|
24
|
+
import contextlib
|
|
25
|
+
import mesmerize_core as mc
|
|
26
|
+
import gc
|
|
27
|
+
import shutil
|
|
28
|
+
from PIL import Image
|
|
29
|
+
|
|
30
|
+
def is_logger_configured():
|
|
31
|
+
return len(logging.root.handlers) > 0
|
|
32
|
+
|
|
33
|
+
def log_and_print(message, level='info'):
|
|
34
|
+
if is_logger_configured():
|
|
35
|
+
if level == 'info':
|
|
36
|
+
logging.info(message)
|
|
37
|
+
elif level == 'warning':
|
|
38
|
+
logging.warning(message)
|
|
39
|
+
elif level == 'error':
|
|
40
|
+
logging.error(message)
|
|
41
|
+
elif level == 'critical':
|
|
42
|
+
logging.critical(message)
|
|
43
|
+
print(message)
|
|
44
|
+
@contextlib.contextmanager
|
|
45
|
+
|
|
46
|
+
def memory_manager(stage="operation"):
|
|
47
|
+
process = psutil.Process()
|
|
48
|
+
|
|
49
|
+
def print_memory_usage(stage_message):
|
|
50
|
+
mem_info = process.memory_info()
|
|
51
|
+
num_threads = process.num_threads()
|
|
52
|
+
print()
|
|
53
|
+
print("=" * 40)
|
|
54
|
+
print(f"{stage_message} - Thread count: {num_threads}, Memory usage: {mem_info.rss / (1024 ** 2):.2f} MB")
|
|
55
|
+
print("=" * 40)
|
|
56
|
+
print()
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
print_memory_usage(f"Process and memory usage before {stage}")
|
|
60
|
+
yield
|
|
61
|
+
finally:
|
|
62
|
+
gc.collect()
|
|
63
|
+
|
|
64
|
+
# -----------------------------------------------------------------------------
|
|
65
|
+
# Adapter: expose a CaImAn memmap with a Suite2p BinaryFile-like API
|
|
66
|
+
# -----------------------------------------------------------------------------
|
|
67
|
+
class _FrameAccessor:
|
|
68
|
+
"""Internal helper to mimic Suite2p's BinaryFile .file slicing interface."""
|
|
69
|
+
def __init__(self, parent: 'CaimanMemmapBinary'):
|
|
70
|
+
self._parent = parent
|
|
71
|
+
def __getitem__(self, indices):
|
|
72
|
+
return self._parent._get_frames(indices)
|
|
73
|
+
|
|
74
|
+
class CaimanMemmapBinary:
|
|
75
|
+
"""
|
|
76
|
+
BinaryFile-like wrapper for a CaImAn memmap (.mmap or .npy) without loading it fully.
|
|
77
|
+
|
|
78
|
+
Provides minimal interface compatibility with Suite2p's BinaryFile:
|
|
79
|
+
Attributes: Ly, Lx, filename, dtype, file, n_frames, shape, size
|
|
80
|
+
Methods: __getitem__, data(), close(), sampled_mean() (approximate)
|
|
81
|
+
|
|
82
|
+
Notes
|
|
83
|
+
-----
|
|
84
|
+
CaImAn stores data as a 2D memmap (pixels, T) where pixels = Ly*Lx*(d3).
|
|
85
|
+
The filename encodes dimensions and order (C or F). We reshape lazily when
|
|
86
|
+
frames are requested. This avoids allocating (T, Ly, Lx) in RAM.
|
|
87
|
+
"""
|
|
88
|
+
def __init__(self, mmap_path: str | Path):
|
|
89
|
+
self.filename = str(mmap_path)
|
|
90
|
+
self._decoded = decode_mmap_filename_dict(self.filename)
|
|
91
|
+
self.Ly = int(self._decoded['d1'])
|
|
92
|
+
self.Lx = int(self._decoded['d2'])
|
|
93
|
+
self._d3 = int(self._decoded['d3'])
|
|
94
|
+
if self._d3 != 1:
|
|
95
|
+
raise NotImplementedError("Adapter currently supports d3 == 1 (2D data)")
|
|
96
|
+
self._T = int(self._decoded['T'])
|
|
97
|
+
self._order = self._decoded['order'] # 'F' or 'C'
|
|
98
|
+
self.dtype = np.float32 # CaImAn memmaps are float32 by convention
|
|
99
|
+
pixels = self.Ly * self.Lx * self._d3
|
|
100
|
+
# Open underlying memmap (pixels, T)
|
|
101
|
+
self._mmap = np.memmap(self.filename, mode='r', dtype=self.dtype, shape=(pixels, self._T), order=self._order)
|
|
102
|
+
# Provide .file accessor mimicking Suite2p BinaryFile.file slicing
|
|
103
|
+
self.file = _FrameAccessor(self)
|
|
104
|
+
self._closed = False
|
|
105
|
+
|
|
106
|
+
# --- properties mirroring Suite2p BinaryFile ---
|
|
107
|
+
@property
|
|
108
|
+
def n_frames(self) -> int:
|
|
109
|
+
return self._T
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def shape(self):
|
|
113
|
+
return (self._T, self.Ly, self.Lx)
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def size(self):
|
|
117
|
+
return self._mmap.size
|
|
118
|
+
|
|
119
|
+
def _get_frames(self, indices):
|
|
120
|
+
"""Return frames (n, Ly, Lx) for given indices (slice/int/array)."""
|
|
121
|
+
# Normalize indices to an array of frame indices
|
|
122
|
+
if isinstance(indices, slice):
|
|
123
|
+
frame_inds = np.arange(*indices.indices(self._T))
|
|
124
|
+
elif isinstance(indices, (list, tuple, np.ndarray)):
|
|
125
|
+
frame_inds = np.asarray(indices)
|
|
126
|
+
else: # single int
|
|
127
|
+
frame_inds = np.asarray([indices])
|
|
128
|
+
# Fetch columns corresponding to frames
|
|
129
|
+
block = self._mmap[:, frame_inds] # shape (pixels, n_frames)
|
|
130
|
+
nF = block.shape[1]
|
|
131
|
+
# Reshape (Ly, Lx, nF) with proper order then transpose to (nF, Ly, Lx)
|
|
132
|
+
frames = block.reshape(self.Ly, self.Lx, nF, order=self._order).transpose(2, 0, 1)
|
|
133
|
+
if frames.shape[0] == 1 and not isinstance(indices, slice) and not isinstance(indices, (list, tuple, np.ndarray)):
|
|
134
|
+
return frames[0]
|
|
135
|
+
return frames
|
|
136
|
+
|
|
137
|
+
def __getitem__(self, indices):
|
|
138
|
+
return self._get_frames(indices)
|
|
139
|
+
|
|
140
|
+
def data(self):
|
|
141
|
+
"""Return entire movie (T, Ly, Lx). Beware of memory usage."""
|
|
142
|
+
return self._get_frames(slice(0, self._T))
|
|
143
|
+
|
|
144
|
+
def sampled_mean(self, nsamps: int = 1000):
|
|
145
|
+
"""Approximate mean by sampling up to nsamps frames evenly spaced."""
|
|
146
|
+
nsamps = min(nsamps, self._T)
|
|
147
|
+
inds = np.linspace(0, self._T - 1, nsamps, dtype=int)
|
|
148
|
+
frames = self[inds].astype(np.float32)
|
|
149
|
+
return frames.mean(axis=0)
|
|
150
|
+
|
|
151
|
+
def close(self):
|
|
152
|
+
if not self._closed:
|
|
153
|
+
try:
|
|
154
|
+
self._mmap._mmap.close()
|
|
155
|
+
except Exception:
|
|
156
|
+
pass
|
|
157
|
+
self._closed = True
|
|
158
|
+
|
|
159
|
+
def __enter__(self):
|
|
160
|
+
return self
|
|
161
|
+
|
|
162
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
163
|
+
self.close()
|
|
164
|
+
|
|
165
|
+
def get_batch_ids(batch_path):
|
|
166
|
+
df = mc.load_batch(batch_path)
|
|
167
|
+
batch_ids = []
|
|
168
|
+
for _, row in df.iterrows():
|
|
169
|
+
batch_ids.append(row.uuid)
|
|
170
|
+
try:
|
|
171
|
+
df.caiman.remove_item(row.uuid)
|
|
172
|
+
except Exception:
|
|
173
|
+
pass
|
|
174
|
+
del df
|
|
175
|
+
return batch_ids
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def cleanup_files(batch_path, export_path, preserve_batch=False):
|
|
179
|
+
if not preserve_batch:
|
|
180
|
+
try:
|
|
181
|
+
batch_ids = get_batch_ids(batch_path)
|
|
182
|
+
except Exception:
|
|
183
|
+
log_and_print(f"Could not get batch ids from {batch_path}.")
|
|
184
|
+
return
|
|
185
|
+
for batch_id in batch_ids:
|
|
186
|
+
batch_runfile = Path(export_path) / f"{batch_id}.runfile"
|
|
187
|
+
if batch_runfile.exists():
|
|
188
|
+
batch_runfile.unlink()
|
|
189
|
+
batch_dir = Path(export_path) / batch_id
|
|
190
|
+
if batch_dir.exists():
|
|
191
|
+
for attempt in range(3):
|
|
192
|
+
time.sleep(1 + attempt)
|
|
193
|
+
try:
|
|
194
|
+
shutil.rmtree(batch_dir)
|
|
195
|
+
break
|
|
196
|
+
except Exception as e:
|
|
197
|
+
if attempt == 2:
|
|
198
|
+
log_and_print(f"Could not delete {batch_dir}: {e}")
|
|
199
|
+
for pickle_file in Path(export_path).glob("batch_*.pickle"):
|
|
200
|
+
pickle_file.unlink()
|
|
201
|
+
|
|
202
|
+
# Remove concatenated tiff if present
|
|
203
|
+
cat_tiff_path = Path(export_path) / 'cat_tiff_bt.tiff'
|
|
204
|
+
if cat_tiff_path.exists():
|
|
205
|
+
cat_tiff_path.unlink()
|
|
206
|
+
# NOTE: Do not remove the final motion-corrected movie here.
|
|
207
|
+
# Downstream steps (ops creation and extraction) may rely on
|
|
208
|
+
# `mcorr_movie.*` being present in the export directory.
|
|
209
|
+
# A separate cleanup step is run by the batch script during wrapping.
|
|
210
|
+
|
|
211
|
+
# Remove concatenation sidecar JSONs if present
|
|
212
|
+
cat_sidecars = [
|
|
213
|
+
Path(export_path) / 'cat_tiff_bt.tiff.json',
|
|
214
|
+
Path(export_path) / 'cat_tiff.h5.json',
|
|
215
|
+
]
|
|
216
|
+
bt_sidecar = cat_sidecars[0]
|
|
217
|
+
if bt_sidecar.exists():
|
|
218
|
+
try:
|
|
219
|
+
mmap_candidates = sorted(
|
|
220
|
+
Path(export_path).rglob("*.mmap"),
|
|
221
|
+
key=lambda p: p.stat().st_mtime,
|
|
222
|
+
reverse=True,
|
|
223
|
+
)
|
|
224
|
+
if mmap_candidates:
|
|
225
|
+
dest_sidecar = Path(export_path) / "movie_attributes.json"
|
|
226
|
+
shutil.copy2(bt_sidecar, dest_sidecar)
|
|
227
|
+
log_and_print(f"Copied sidecar JSON to {dest_sidecar}")
|
|
228
|
+
except Exception as e:
|
|
229
|
+
log_and_print(f"Could not copy sidecar JSON for memmap: {e}", level="warning")
|
|
230
|
+
for sc in cat_sidecars:
|
|
231
|
+
if sc.exists():
|
|
232
|
+
try:
|
|
233
|
+
sc.unlink()
|
|
234
|
+
except Exception:
|
|
235
|
+
pass
|
|
236
|
+
log_and_print("Batch files deleted.\n")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def find_latest_batch(export_path):
|
|
240
|
+
export_path = Path(export_path)
|
|
241
|
+
batch_files = sorted(export_path.glob("batch_*.pickle"), key=lambda p: p.stat().st_mtime, reverse=True)
|
|
242
|
+
if batch_files:
|
|
243
|
+
return batch_files[0]
|
|
244
|
+
|
|
245
|
+
runfiles = sorted(export_path.glob("*.runfile"), key=lambda p: p.stat().st_mtime, reverse=True)
|
|
246
|
+
for runfile in runfiles:
|
|
247
|
+
candidate = export_path / runfile.stem
|
|
248
|
+
if candidate.exists():
|
|
249
|
+
return candidate
|
|
250
|
+
candidate_pickle = export_path / f"{runfile.stem}.pickle"
|
|
251
|
+
if candidate_pickle.exists():
|
|
252
|
+
return candidate_pickle
|
|
253
|
+
|
|
254
|
+
if runfiles:
|
|
255
|
+
raise FileNotFoundError(f"No batch file found for runfile(s) in {export_path}")
|
|
256
|
+
raise FileNotFoundError(f"No batch file found in {export_path}")
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def get_default_parameters(proc_step):
|
|
260
|
+
"""
|
|
261
|
+
Get the default parameters for motion correction or CNMF.
|
|
262
|
+
"""
|
|
263
|
+
if proc_step == 'mcorr':
|
|
264
|
+
default_params = \
|
|
265
|
+
{
|
|
266
|
+
'main':
|
|
267
|
+
{
|
|
268
|
+
'strides': [36, 36],
|
|
269
|
+
'overlaps': [24, 24],
|
|
270
|
+
'max_shifts': [12, 12],
|
|
271
|
+
'max_deviation_rigid': 6,
|
|
272
|
+
'border_nan': 'copy',
|
|
273
|
+
'pw_rigid': True,
|
|
274
|
+
'gSig_filt': None
|
|
275
|
+
},
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
elif proc_step == 'cnmf':
|
|
279
|
+
default_params = \
|
|
280
|
+
{
|
|
281
|
+
'main':
|
|
282
|
+
{
|
|
283
|
+
'fr': 20, # framerate, very important!
|
|
284
|
+
'p': 1,
|
|
285
|
+
'nb': 2,
|
|
286
|
+
'merge_thr': 0.85,
|
|
287
|
+
'rf': 20,
|
|
288
|
+
'stride': 10, # "stride" for cnmf, "strides" for mcorr
|
|
289
|
+
'K': 10,
|
|
290
|
+
'gSig': [5, 5],
|
|
291
|
+
'ssub': 1,
|
|
292
|
+
'tsub': 1,
|
|
293
|
+
'method_init': 'greedy_roi',
|
|
294
|
+
'min_SNR': 3.0,
|
|
295
|
+
'SNR_lowest': 1.0,
|
|
296
|
+
'rval_thr': 0.8,
|
|
297
|
+
'rval_lowest': 0.2,
|
|
298
|
+
'use_cnn': True,
|
|
299
|
+
'min_cnn_thr': 0.9,
|
|
300
|
+
'cnn_lowest': 0.2,
|
|
301
|
+
'decay_time': 0.15,
|
|
302
|
+
},
|
|
303
|
+
'refit': True, # If `True`, run a second iteration of CNMF
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
return default_params
|
|
307
|
+
|
|
308
|
+
def clip_range(array, clip_range='uint16'):
|
|
309
|
+
"""
|
|
310
|
+
Clip the values of an array to a specific range.
|
|
311
|
+
"""
|
|
312
|
+
if clip_range == 'uint16':
|
|
313
|
+
return np.clip(array, 0, 2**16 - 1)
|
|
314
|
+
elif clip_range == 'uint8':
|
|
315
|
+
return np.clip(array, 0, 2**8 - 1)
|
|
316
|
+
elif clip_range == 'int16':
|
|
317
|
+
return np.clip(array, -2**15, 2**15 - 1)
|
|
318
|
+
else:
|
|
319
|
+
return array
|
|
320
|
+
|
|
321
|
+
def safe_rename(src, dst):
|
|
322
|
+
"""Attempt to rename the file with retries upon encountering an access error."""
|
|
323
|
+
max_attempts = 5
|
|
324
|
+
attempt = 0
|
|
325
|
+
rename_succes = False
|
|
326
|
+
print(f"Renaming file {src} to {dst}.")
|
|
327
|
+
while attempt < max_attempts:
|
|
328
|
+
try:
|
|
329
|
+
src.rename(dst)
|
|
330
|
+
rename_succes = True
|
|
331
|
+
break
|
|
332
|
+
except PermissionError:
|
|
333
|
+
print(f"Attempt {attempt+1} failed, retrying in 5 seconds...")
|
|
334
|
+
time.sleep(5)
|
|
335
|
+
attempt += 1
|
|
336
|
+
if attempt == max_attempts:
|
|
337
|
+
print(f"Failed to rename file {src} to {dst} after {max_attempts} attempts.")
|
|
338
|
+
|
|
339
|
+
return rename_succes
|
|
340
|
+
|
|
341
|
+
def overwrite_movie_memmap(movie, original_mmap_path, clip=True, movie_type='mcorr', save_original=False, remove_input=False):
|
|
342
|
+
"""
|
|
343
|
+
Overwrite the original memmap file with the new movie.
|
|
344
|
+
"""
|
|
345
|
+
# Check if movie is a Path object or a numpy array
|
|
346
|
+
if isinstance(movie, Path):
|
|
347
|
+
# Load the movie from the memmap file
|
|
348
|
+
movie_array = load_mmap_movie(movie)
|
|
349
|
+
else:
|
|
350
|
+
movie_array = movie
|
|
351
|
+
print(f"Movie type: {type(movie_array)}, with shape: {movie_array.shape}")
|
|
352
|
+
first_frame = movie_array[0]
|
|
353
|
+
print(f"First frame dtype: {first_frame.dtype}, shape: {first_frame.shape}, min_val: {first_frame.min()}, max_val: {first_frame.max()}")
|
|
354
|
+
print(f"Movie median: {np.median(movie_array)}")
|
|
355
|
+
|
|
356
|
+
# # Compare original movie median
|
|
357
|
+
# with TiffFile(movie.parent.parent / "cat_tiff_bt.tiff") as tif:
|
|
358
|
+
# og_tiff = tif.asarray()
|
|
359
|
+
# print(f"Z-stack shape: {og_tiff.shape}")
|
|
360
|
+
# print(f"Z-stack median: {np.median(og_tiff)}")
|
|
361
|
+
|
|
362
|
+
if clip:
|
|
363
|
+
print(f"Clipping movie array to uint16 range.")
|
|
364
|
+
# Clip, but don't convert to uint16 yet (caiman expects 32 bit float) image
|
|
365
|
+
movie_array = clip_range(movie_array, 'uint16')
|
|
366
|
+
|
|
367
|
+
# Original array with dimensions (T, y, x). Transpose the array to (y, x, T)
|
|
368
|
+
transposed_array = movie_array.transpose(1, 2, 0)
|
|
369
|
+
|
|
370
|
+
# Flatten the transposed array in 'F' order (to align with the loading code's expectations, based on caiman demo notebook)
|
|
371
|
+
flattened_array = transposed_array.flatten(order='F')
|
|
372
|
+
|
|
373
|
+
# Create a new memmap file with write access
|
|
374
|
+
log_and_print(f"{movie_type} movie path: {original_mmap_path}")
|
|
375
|
+
flattened_movie_path = original_mmap_path.parent / f"flattened_{original_mmap_path.name}"
|
|
376
|
+
|
|
377
|
+
# Save the flattened array as a memmap
|
|
378
|
+
flattened_movie = np.memmap(flattened_movie_path, dtype='float32', mode='w+', shape=flattened_array.shape)
|
|
379
|
+
np.copyto(flattened_movie, flattened_array) # equivalent to flattened_movie[:] = flattened_array[:]
|
|
380
|
+
|
|
381
|
+
# Flush changes to disk and close the memmap and the original memmap
|
|
382
|
+
flattened_movie.flush()
|
|
383
|
+
del flattened_movie, flattened_array, transposed_array # Delete variables explicitly
|
|
384
|
+
gc.collect() # Force garbage collection
|
|
385
|
+
|
|
386
|
+
# Recompute the projections
|
|
387
|
+
proj_paths = dict()
|
|
388
|
+
for proj_type in ["mean", "std", "max"]:
|
|
389
|
+
p_img = getattr(np, f"nan{proj_type}")(movie_array, axis=0)
|
|
390
|
+
proj_paths[proj_type] = original_mmap_path.parent.joinpath(
|
|
391
|
+
f"{str(original_mmap_path.parent.stem)}_{proj_type}_projection.npy"
|
|
392
|
+
)
|
|
393
|
+
np.save(str(proj_paths[proj_type]), p_img)
|
|
394
|
+
del p_img # Delete projection images after saving
|
|
395
|
+
|
|
396
|
+
log_and_print(f"Projections recomputed and saved to {original_mmap_path.parent}.")
|
|
397
|
+
|
|
398
|
+
del movie_array
|
|
399
|
+
|
|
400
|
+
# The code below does not work on Windows while memmap is open
|
|
401
|
+
time.sleep(5)
|
|
402
|
+
|
|
403
|
+
# Create a backup copy of the original memmap file then delete the original
|
|
404
|
+
rename_success = safe_rename(original_mmap_path, original_mmap_path.parent / f"original_{original_mmap_path.name}")
|
|
405
|
+
# original_mmap_path.rename(original_mmap_path.parent / f"original_{original_mmap_path.name}")
|
|
406
|
+
# print(f"Original motion corrected movie saved to {original_mmap_path.parent / f'original_{original_mmap_path.name}'}")
|
|
407
|
+
|
|
408
|
+
if rename_success:
|
|
409
|
+
# Finally, rename the clipped memmap file to the original name
|
|
410
|
+
flattened_movie_path.rename(original_mmap_path)
|
|
411
|
+
# Delete the backup copy
|
|
412
|
+
if not save_original:
|
|
413
|
+
(original_mmap_path.parent / f"original_{original_mmap_path.name}").unlink()
|
|
414
|
+
|
|
415
|
+
if remove_input and isinstance(movie, Path):
|
|
416
|
+
movie.unlink()
|
|
417
|
+
|
|
418
|
+
return original_mmap_path, flattened_movie_path
|
|
419
|
+
|
|
420
|
+
else:
|
|
421
|
+
return flattened_movie_path, None
|
|
422
|
+
|
|
423
|
+
def save_mmap_movie(movie, movie_path):
|
|
424
|
+
"""
|
|
425
|
+
Save an array as a memmaped numpy array.
|
|
426
|
+
Original array must have dimensions T, y, x
|
|
427
|
+
"""
|
|
428
|
+
# Save the movie as a memmaped numpy array
|
|
429
|
+
movie_path = Path(movie_path)
|
|
430
|
+
# Transpose the array to (y, x, T)
|
|
431
|
+
transposed_array = movie.transpose(1, 2, 0)
|
|
432
|
+
# Flatten the transposed array in 'F' order (to align with the loading code)
|
|
433
|
+
flattened_array = transposed_array.flatten(order='F')
|
|
434
|
+
# Create a new memmap file with write access
|
|
435
|
+
movie_ = np.memmap(movie_path, dtype='float32', mode='w+', shape=flattened_array.shape)
|
|
436
|
+
movie_[:] = flattened_array[:]
|
|
437
|
+
# Flush changes to disk and close the memmap
|
|
438
|
+
del movie_
|
|
439
|
+
del flattened_array
|
|
440
|
+
|
|
441
|
+
def load_mmap_movie(movie_path):
|
|
442
|
+
"""
|
|
443
|
+
Load a memmaped numpy array.
|
|
444
|
+
Returns a fully materialized 3D numpy array (T, Ly, Lx).
|
|
445
|
+
For large datasets prefer using `load_caiman_memmap` to avoid loading all frames.
|
|
446
|
+
"""
|
|
447
|
+
# Load the movie from the memmap file
|
|
448
|
+
movie , dims, T = load_memmap(movie_path)
|
|
449
|
+
# Reshape the array to the desired dimensions
|
|
450
|
+
movie = np.reshape(movie.T, [T] + list(dims), order='F')
|
|
451
|
+
return movie
|
|
452
|
+
|
|
453
|
+
def load_caiman_memmap(movie_path: str | Path) -> CaimanMemmapBinary:
|
|
454
|
+
"""Convenience loader returning a BinaryFile-like adapter for a CaImAn memmap."""
|
|
455
|
+
return CaimanMemmapBinary(movie_path)
|
|
456
|
+
|
|
457
|
+
def create_mp4_movie(input_movie, export_path, filename=None):
|
|
458
|
+
"""
|
|
459
|
+
Create a mp4 movie from a numpy array, using ffmpeg.
|
|
460
|
+
"""
|
|
461
|
+
# Create a directory for temporary frame storage
|
|
462
|
+
temp_frame_dir = export_path / 'temp_frames'
|
|
463
|
+
os.makedirs(temp_frame_dir, exist_ok=True)
|
|
464
|
+
|
|
465
|
+
# Save each frame as an image
|
|
466
|
+
for i, frame in enumerate(input_movie):
|
|
467
|
+
# If height not divisible by 2, add padding, to avoid [libx264 @ 0x563ae05f8b40] height not divisible by 2 error
|
|
468
|
+
if frame.shape[0] % 2 != 0:
|
|
469
|
+
# Add a row of black pixels at the bottom
|
|
470
|
+
frame = np.pad(frame, ((0, 1), (0, 0)), 'constant')
|
|
471
|
+
frame_image = Image.fromarray(frame)
|
|
472
|
+
frame_image.save(temp_frame_dir / f"frame_{i:04d}.png")
|
|
473
|
+
|
|
474
|
+
# Get the size of the frames post-padding
|
|
475
|
+
frame_size = frame_image.size
|
|
476
|
+
|
|
477
|
+
if filename is None:
|
|
478
|
+
filename = 'movie.mp4'
|
|
479
|
+
|
|
480
|
+
# Construct the FFmpeg command to make an MP4 movie with the adjusted size
|
|
481
|
+
ffmpeg_command = f"ffmpeg -y -r 30 -f image2 -s {frame_size[0]}x{frame_size[1]} -i {temp_frame_dir}/frame_%04d.png -vcodec libx264 -crf 25 -pix_fmt yuv420p {Path(export_path, filename)}"
|
|
482
|
+
os.system(ffmpeg_command)
|
|
483
|
+
|
|
484
|
+
# Delete the temporary frame images after creating the video
|
|
485
|
+
# for file in os.listdir(temp_frame_dir):
|
|
486
|
+
# os.remove(temp_frame_dir / file)
|
|
487
|
+
# os.rmdir(temp_frame_dir)
|
|
488
|
+
shutil.rmtree(temp_frame_dir)
|
|
489
|
+
|
|
490
|
+
def cat_movies_to_mp4(movie1, movie2, movie_path):
|
|
491
|
+
"""
|
|
492
|
+
Concatenate two movies horizontally and save as a mp4 movie.
|
|
493
|
+
"""
|
|
494
|
+
# For visualization purposes only:
|
|
495
|
+
# If ranges are significantly different, consider rescaling
|
|
496
|
+
|
|
497
|
+
# Store the original data type
|
|
498
|
+
movie1_dtype = movie1.dtype
|
|
499
|
+
movie2_dtype = movie2.dtype
|
|
500
|
+
|
|
501
|
+
movie1_min, movie1_max = np.min(movie1), np.max(movie1)
|
|
502
|
+
movie2_min, movie2_max = np.min(movie2), np.max(movie2)
|
|
503
|
+
|
|
504
|
+
if movie1_min != movie2_min or movie1_max != movie2_max:
|
|
505
|
+
# Check if the movies are either uint16 or uint8
|
|
506
|
+
if movie1.dtype in (np.uint16, np.uint8) or movie2.dtype in (np.uint16, np.uint8):
|
|
507
|
+
# convert to float32
|
|
508
|
+
movie1 = movie1.astype(np.float32)
|
|
509
|
+
movie2 = movie2.astype(np.float32)
|
|
510
|
+
normalized_movie1 = (movie1 - movie1_min) / (movie1_max - movie1_min) * 65535
|
|
511
|
+
normalized_movie2 = (movie2 - movie2_min) / (movie2_max - movie2_min) * 65535
|
|
512
|
+
else:
|
|
513
|
+
normalized_movie1 = movie1
|
|
514
|
+
normalized_movie2 = movie2
|
|
515
|
+
|
|
516
|
+
# Convert to uint16 (or uint8 if that was the original datatype), after normalization
|
|
517
|
+
if movie1_dtype not in (np.uint16, np.uint8):
|
|
518
|
+
normalized_movie1 = normalized_movie1.astype(np.uint16)
|
|
519
|
+
else:
|
|
520
|
+
normalized_movie1 = normalized_movie1.astype(movie1_dtype)
|
|
521
|
+
if movie2_dtype not in (np.uint16, np.uint8):
|
|
522
|
+
normalized_movie2 = normalized_movie2.astype(np.uint16)
|
|
523
|
+
else:
|
|
524
|
+
normalized_movie2 = normalized_movie2.astype(movie2_dtype)
|
|
525
|
+
|
|
526
|
+
# first_frame_1 = normalized_movie1[0,:,:]
|
|
527
|
+
# first_frame_2 = normalized_movie2[0,:,:]
|
|
528
|
+
|
|
529
|
+
# Concatenate the two movies horizontally
|
|
530
|
+
cat_movie = np.concatenate((normalized_movie1, normalized_movie2), axis=2)
|
|
531
|
+
|
|
532
|
+
# Split the movie path into path and filename
|
|
533
|
+
export_path = movie_path.parent
|
|
534
|
+
filename = movie_path.name
|
|
535
|
+
|
|
536
|
+
# Create the mp4 movie
|
|
537
|
+
create_mp4_movie(cat_movie, export_path, filename)
|