micro-reader 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- micro_reader/__init__.py +23 -0
- micro_reader/_axes.py +196 -0
- micro_reader/_base.py +268 -0
- micro_reader/_codecs.py +439 -0
- micro_reader/_errors.py +14 -0
- micro_reader/_index.py +53 -0
- micro_reader/_io.py +83 -0
- micro_reader/_jpeg_restarts.py +174 -0
- micro_reader/_memory.py +161 -0
- micro_reader/_metadata.py +163 -0
- micro_reader/_pool.py +85 -0
- micro_reader/_registry.py +104 -0
- micro_reader/_stream.py +317 -0
- micro_reader/backends/__init__.py +1 -0
- micro_reader/backends/_czi_metadata.py +188 -0
- micro_reader/backends/_lif_metadata.py +125 -0
- micro_reader/backends/_nd2_legacy.py +370 -0
- micro_reader/backends/_nd2_metadata.py +224 -0
- micro_reader/backends/_nd2_structure.py +280 -0
- micro_reader/backends/_tiff_metadata.py +287 -0
- micro_reader/backends/czi.py +948 -0
- micro_reader/backends/ims.py +409 -0
- micro_reader/backends/lif.py +713 -0
- micro_reader/backends/mrc.py +184 -0
- micro_reader/backends/nd2.py +264 -0
- micro_reader/backends/oir.py +439 -0
- micro_reader/backends/raster.py +568 -0
- micro_reader/backends/tiff.py +858 -0
- micro_reader-0.0.1.dist-info/METADATA +741 -0
- micro_reader-0.0.1.dist-info/RECORD +32 -0
- micro_reader-0.0.1.dist-info/WHEEL +4 -0
- micro_reader-0.0.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""Metadata of ND2 images (version 3; docs/METADATA.md), from the picture
|
|
2
|
+
metadata (``ImageMetadataSeqLV|0``) and the experiment loops
|
|
3
|
+
(``ImageMetadataLV``). Field meanings follow nd2 (Talley Lambert, BSD-3).
|
|
4
|
+
|
|
5
|
+
- Pixel sizes: ``dCalibration`` (µm per pixel, x and y) when
|
|
6
|
+
``bCalibrated``; z: the Z loop's ``dZStep`` (µm); t: the time loop's
|
|
7
|
+
``dPeriod`` (the interval asked for, ms), else its measured
|
|
8
|
+
``dAvgPeriodDiff``.
|
|
9
|
+
- Channels: ``sPicturePlanes/sPlaneNew/a<k>``: ``sDescription``, ``uiColor``
|
|
10
|
+
(ABGR), and for the wavelengths the fluorescent probe's spectrum, else
|
|
11
|
+
the first filter's (a line, or a band's middle: ``_wavelength_of``).
|
|
12
|
+
- Position (docs/METADATA.md: the first pixel along the image's axes): the
|
|
13
|
+
XY loop's point for the image's position (else ``dXPos`` / ``dYPos``),
|
|
14
|
+
which NIS records for the centre of the field of view in stage axes,
|
|
15
|
+
brought into the image's axes through the inverse of the camera-to-stage
|
|
16
|
+
matrix ``dStgLgCT``, then moved to the first pixel; unknown without the
|
|
17
|
+
matrix (old files: a multipoint file without one is turned by 180
|
|
18
|
+
degrees, measured on its overlapping fields).
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from .._metadata import Channel, Metadata, Quantity, color_hex, quantity
|
|
23
|
+
from ._nd2_structure import NE_TIME_LOOP, TIME_LOOP, XY_POS_LOOP, Z_STACK_LOOP
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _loops(raw: dict) -> list:
|
|
27
|
+
"""Every loop of the experiment tree: (eType, parameters), outermost first."""
|
|
28
|
+
out = []
|
|
29
|
+
|
|
30
|
+
def walk(node):
|
|
31
|
+
node = node.get("SLxExperiment", node)
|
|
32
|
+
pars = node.get("uLoopPars") or {}
|
|
33
|
+
if list(pars) == ["i0000000000"]:
|
|
34
|
+
pars = pars["i0000000000"]
|
|
35
|
+
out.append((node.get("eType", 0), pars))
|
|
36
|
+
for item in (node.get("ppNextLevelEx") or {}).values():
|
|
37
|
+
walk(item)
|
|
38
|
+
if raw:
|
|
39
|
+
walk(raw)
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _valid_points(node_pars: dict, valid) -> list:
|
|
44
|
+
points = node_pars.get("Points")
|
|
45
|
+
if isinstance(points, dict):
|
|
46
|
+
points = [points[k] for k in sorted(points)]
|
|
47
|
+
elif "dPosX" in node_pars:
|
|
48
|
+
xs, ys = node_pars["dPosX"], node_pars["dPosY"]
|
|
49
|
+
points = [{"dPosX": xs[k], "dPosY": ys[k]} for k in sorted(xs)]
|
|
50
|
+
else:
|
|
51
|
+
return []
|
|
52
|
+
if valid:
|
|
53
|
+
flags = [v for _, v in sorted(valid.items())] if isinstance(valid, dict) else list(valid)
|
|
54
|
+
points = [p for p, ok in zip(points, flags) if ok]
|
|
55
|
+
return points
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _xy_points(raw: dict) -> list:
|
|
59
|
+
node = (raw or {}).get("SLxExperiment", raw or {})
|
|
60
|
+
stack = [node]
|
|
61
|
+
while stack:
|
|
62
|
+
n = stack.pop(0)
|
|
63
|
+
n = n.get("SLxExperiment", n)
|
|
64
|
+
if n.get("eType") == XY_POS_LOOP:
|
|
65
|
+
pars = n.get("uLoopPars") or {}
|
|
66
|
+
if list(pars) == ["i0000000000"]:
|
|
67
|
+
pars = pars["i0000000000"]
|
|
68
|
+
return _valid_points(pars, n.get("pItemValid"))
|
|
69
|
+
stack.extend((n.get("ppNextLevelEx") or {}).values())
|
|
70
|
+
return []
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
#: spectrum point types: a band's cut-on / cut-off edge, a single line
|
|
74
|
+
BAND_START, BAND_END, LINE = 2, 3, 4
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _wavelength_of(points: list) -> float:
|
|
78
|
+
"""One wavelength for a spectrum given as (type, wavelength, transmission)
|
|
79
|
+
points: the line with the highest transmission; else a band's middle;
|
|
80
|
+
else its one edge (a long- / short-pass filter) -- as Bio-Formats
|
|
81
|
+
reports filters (500-550 nm -> 525)."""
|
|
82
|
+
points = [p for p in points if p[1] and p[1] > 0]
|
|
83
|
+
lines = [p for p in points if p[0] == LINE]
|
|
84
|
+
if lines:
|
|
85
|
+
return max(lines, key=lambda p: p[2])[1]
|
|
86
|
+
starts = [p[1] for p in points if p[0] == BAND_START]
|
|
87
|
+
ends = [p[1] for p in points if p[0] == BAND_END]
|
|
88
|
+
if starts and ends:
|
|
89
|
+
return (starts[0] + ends[0]) / 2
|
|
90
|
+
return points[0][1] if points else 0.0
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _spectrum_peak(spectrum) -> float:
|
|
94
|
+
points = (spectrum or {}).get("pPoint") or {}
|
|
95
|
+
return _wavelength_of([(p.get("eType", 0), p.get("dWavelength", 0.0), p.get("dTValue", 0.0))
|
|
96
|
+
for p in points.values() if isinstance(p, dict)])
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _wavelengths(plane: dict) -> tuple:
|
|
100
|
+
"""(excitation, emission) nm, 0 when unknown -- nd2's rules."""
|
|
101
|
+
probe = plane.get("pFluorescentProbe") or {}
|
|
102
|
+
filters = (plane.get("pFilterPath") or {}).get("m_pFilter") or {}
|
|
103
|
+
filt = next(iter(filters.values()), {}) if isinstance(filters, dict) else {}
|
|
104
|
+
ex = _spectrum_peak(probe.get("m_ExcitationSpectrum"))
|
|
105
|
+
if not ex and filt:
|
|
106
|
+
ex = _spectrum_peak(filt.get("m_ExcitationSpectrum"))
|
|
107
|
+
em = _spectrum_peak(probe.get("m_EmissionSpectrum"))
|
|
108
|
+
if not em and filt:
|
|
109
|
+
em = _spectrum_peak(filt.get("m_EmissionSpectrum"))
|
|
110
|
+
return ex, em
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _abgr(value):
|
|
114
|
+
if not isinstance(value, int):
|
|
115
|
+
return None
|
|
116
|
+
return color_hex((value & 255, value >> 8 & 255, value >> 16 & 255))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def nd2_metadata(picture: dict, experiment: dict, position: int, shape: dict) -> Metadata:
|
|
120
|
+
"""*picture*: SLxPictureMetadata; *experiment*: the raw experiment;
|
|
121
|
+
*position*: the image's XY position; *shape*: axis -> size."""
|
|
122
|
+
sizes = {}
|
|
123
|
+
if picture.get("bCalibrated"):
|
|
124
|
+
q = quantity(picture.get("dCalibration"), "micrometer")
|
|
125
|
+
if q:
|
|
126
|
+
sizes["x"] = sizes["y"] = q
|
|
127
|
+
for kind, pars in _loops(experiment):
|
|
128
|
+
if kind == Z_STACK_LOOP and "z" not in sizes:
|
|
129
|
+
q = quantity(abs(pars.get("dZStep", 0.0) or 0.0), "micrometer")
|
|
130
|
+
if q:
|
|
131
|
+
sizes["z"] = q
|
|
132
|
+
elif kind == TIME_LOOP and "t" not in sizes:
|
|
133
|
+
q = quantity(pars.get("dPeriod"), "millisecond") or \
|
|
134
|
+
quantity(pars.get("dAvgPeriodDiff"), "millisecond")
|
|
135
|
+
if q:
|
|
136
|
+
sizes["t"] = q
|
|
137
|
+
elif kind == NE_TIME_LOOP and "t" not in sizes:
|
|
138
|
+
periods = pars.get("pPeriod") or {}
|
|
139
|
+
first = periods[min(periods)] if periods else {}
|
|
140
|
+
q = quantity(first.get("dPeriod"), "millisecond") or \
|
|
141
|
+
quantity(first.get("dAvgPeriodDiff"), "millisecond")
|
|
142
|
+
if q:
|
|
143
|
+
sizes["t"] = q
|
|
144
|
+
|
|
145
|
+
planes = ((picture.get("sPicturePlanes") or {}).get("sPlaneNew") or {})
|
|
146
|
+
channels = []
|
|
147
|
+
for key in sorted(planes, key=lambda k: int(k[1:]) if k[1:].isdigit() else 0):
|
|
148
|
+
plane = planes[key]
|
|
149
|
+
ex, em = _wavelengths(plane)
|
|
150
|
+
channels.append(Channel(
|
|
151
|
+
name=plane.get("sDescription") or None,
|
|
152
|
+
color=_abgr(plane.get("uiColor")),
|
|
153
|
+
emission_wavelength=quantity(em, "nanometer"),
|
|
154
|
+
excitation_wavelength=quantity(ex, "nanometer")))
|
|
155
|
+
n_c = max(1, shape.get("c", 1))
|
|
156
|
+
channels = (channels + [Channel()] * n_c)[:n_c]
|
|
157
|
+
|
|
158
|
+
points = _xy_points(experiment)
|
|
159
|
+
point = points[position] if position < len(points) else \
|
|
160
|
+
{"dPosX": picture.get("dXPos"), "dPosY": picture.get("dYPos")}
|
|
161
|
+
position_out = {}
|
|
162
|
+
centre = (quantity(point.get("dPosX"), "micrometer", positive=False),
|
|
163
|
+
quantity(point.get("dPosY"), "micrometer", positive=False))
|
|
164
|
+
if all(centre) and "x" in sizes:
|
|
165
|
+
# the field's centre along the image's axes: the stage position
|
|
166
|
+
# through the inverse of the camera-to-stage matrix (NIS: about -1
|
|
167
|
+
# on the diagonal for a camera turned by 180 degrees)
|
|
168
|
+
# without the matrix the camera's orientation is unknown (an old
|
|
169
|
+
# multipoint file without one is turned by 180 degrees): no position
|
|
170
|
+
keys = ("dStgLgCT11", "dStgLgCT12", "dStgLgCT21", "dStgLgCT22")
|
|
171
|
+
m11, m12, m21, m22 = (picture.get(k, 0.0) for k in keys)
|
|
172
|
+
det = m11 * m22 - m12 * m21
|
|
173
|
+
if all(k in picture for k in keys) and abs(det) > 1e-6:
|
|
174
|
+
sx, sy = centre[0].value, centre[1].value
|
|
175
|
+
cx = (m22 * sx - m12 * sy) / det
|
|
176
|
+
cy = (-m21 * sx + m11 * sy) / det
|
|
177
|
+
step = sizes["x"].value
|
|
178
|
+
position_out = {
|
|
179
|
+
"x": Quantity(cx + (0.5 - shape.get("x", 1) / 2) * step, "micrometer"),
|
|
180
|
+
"y": Quantity(cy + (0.5 - shape.get("y", 1) / 2) * step, "micrometer")}
|
|
181
|
+
return Metadata(pixel_sizes=sizes, position=position_out, channels=tuple(channels))
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# -- legacy ND2 (versions 1 / 2): the per-frame ``VIMD`` XML ----------------------
|
|
185
|
+
|
|
186
|
+
def _legacy_peak(spectrum) -> float:
|
|
187
|
+
points = (spectrum or {}).get("Point") or {}
|
|
188
|
+
return _wavelength_of([(p.get("Type", 0), p.get("Wavelength", 0.0), p.get("TValue", 0.0))
|
|
189
|
+
for p in points.values() if isinstance(p, dict)])
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def legacy_metadata(frame: dict, shape: dict) -> Metadata:
|
|
193
|
+
"""*frame*: the ``VIMD`` of the image's first frame (as nd2 reads it:
|
|
194
|
+
``Calibration`` µm per pixel, plane ``OpticalConfigName`` and ``Color``
|
|
195
|
+
(ABGR); emission from the probe's spectrum, else the filter's, as
|
|
196
|
+
Bio-Formats has it). No position: see below."""
|
|
197
|
+
sizes = {}
|
|
198
|
+
q = quantity(frame.get("Calibration"), "micrometer")
|
|
199
|
+
if q:
|
|
200
|
+
sizes["x"] = sizes["y"] = q
|
|
201
|
+
planes = ((frame.get("PicturePlanes") or {}).get("Plane") or {})
|
|
202
|
+
channels = []
|
|
203
|
+
for key in sorted(planes, key=lambda k: int(k) if str(k).isdigit() else 0):
|
|
204
|
+
plane = planes[key]
|
|
205
|
+
filters = (plane.get("FilterPath") or {}).get("Filter") or {}
|
|
206
|
+
filt = filters if "Name" in filters else next(iter(filters.values()), {})
|
|
207
|
+
filt = filt if isinstance(filt, dict) else {}
|
|
208
|
+
em = _legacy_peak((plane.get("FluorescentProbe") or {}).get("EmissionSpectrum")) or \
|
|
209
|
+
_legacy_peak(filt.get("EmissionSpectrum"))
|
|
210
|
+
ex = _legacy_peak((plane.get("FluorescentProbe") or {}).get("ExcitationSpectrum")) or \
|
|
211
|
+
_legacy_peak(filt.get("ExcitationSpectrum"))
|
|
212
|
+
channels.append(Channel(
|
|
213
|
+
name=plane.get("OpticalConfigName") or None,
|
|
214
|
+
color=_abgr(plane.get("Color")),
|
|
215
|
+
emission_wavelength=quantity(em, "nanometer"),
|
|
216
|
+
excitation_wavelength=quantity(ex, "nanometer")))
|
|
217
|
+
n_c = max(1, shape.get("c", 1))
|
|
218
|
+
if len(channels) != n_c: # channels are components of one plane
|
|
219
|
+
channels = []
|
|
220
|
+
channels = (channels + [Channel()] * n_c)[:n_c]
|
|
221
|
+
# XPos / YPos are stage coordinates of the field's centre, but old files
|
|
222
|
+
# record no camera-to-stage matrix: their direction along the image's
|
|
223
|
+
# axes is unknown, so no position (docs/METADATA.md)
|
|
224
|
+
return Metadata(pixel_sizes=sizes, channels=tuple(channels))
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""ND2 (Nikon, version 3) structure: chunk map, CLX-lite metadata, loops.
|
|
2
|
+
|
|
3
|
+
Only what is needed to locate pixels: where each frame is, how a frame is
|
|
4
|
+
laid out, and which (time, position, Z) each frame number stands for.
|
|
5
|
+
|
|
6
|
+
Ported, reduced, from nd2 (https://github.com/tlambert03/nd2), modules
|
|
7
|
+
``_parse/_chunk_decode.py``, ``_parse/_clx_lite.py`` and ``_parse/_parse.py``:
|
|
8
|
+
|
|
9
|
+
Copyright (c) 2021 Talley Lambert
|
|
10
|
+
|
|
11
|
+
Redistribution and use in source and binary forms, with or without
|
|
12
|
+
modification, are permitted provided that the following conditions are met:
|
|
13
|
+
|
|
14
|
+
1. Redistributions of source code must retain the above copyright notice,
|
|
15
|
+
this list of conditions and the following disclaimer.
|
|
16
|
+
2. Redistributions in binary form must reproduce the above copyright
|
|
17
|
+
notice, this list of conditions and the following disclaimer in the
|
|
18
|
+
documentation and/or other materials provided with the distribution.
|
|
19
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
20
|
+
contributors may be used to endorse or promote products derived from
|
|
21
|
+
this software without specific prior written permission.
|
|
22
|
+
|
|
23
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
|
24
|
+
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
25
|
+
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
26
|
+
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
|
|
27
|
+
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
28
|
+
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
29
|
+
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
30
|
+
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
31
|
+
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
32
|
+
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
33
|
+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
34
|
+
"""
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import io
|
|
38
|
+
import struct
|
|
39
|
+
import zlib
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
|
|
42
|
+
from .._errors import CorruptFileError, UnsupportedFormatError
|
|
43
|
+
|
|
44
|
+
CHUNK_MAGIC = 0x0ABECEDA
|
|
45
|
+
JP2_MAGIC = 0x0C000000 # legacy (version 1) ND2: JPEG 2000 based
|
|
46
|
+
FILE_SIGNATURE = b"ND2 FILE SIGNATURE CHUNK NAME01!"
|
|
47
|
+
CHUNKMAP_SIGNATURE = b"ND2 CHUNK MAP SIGNATURE 0000001!"
|
|
48
|
+
FILEMAP_SIGNATURE = b"ND2 FILEMAP SIGNATURE NAME 0001!"
|
|
49
|
+
CHUNK_HEADER = struct.Struct("<IIQ") # magic, name length, data length
|
|
50
|
+
START_CHUNK = struct.Struct("<IIQ32s64s") # header, name, data ("Ver3.0")
|
|
51
|
+
CHUNKMAP_LOCATION = struct.Struct("<32sQ") # the last 40 bytes of the file
|
|
52
|
+
|
|
53
|
+
# experiment loop types (``eType``)
|
|
54
|
+
TIME_LOOP, XY_POS_LOOP, Z_STACK_LOOP, SPECT_LOOP, CUSTOM_LOOP, NE_TIME_LOOP = 1, 2, 4, 6, 7, 8
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# -- file level ----------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
def version(head: bytes) -> tuple[int, int]:
|
|
60
|
+
"""(major, minor) from the first chunk; raises for anything else."""
|
|
61
|
+
if len(head) < START_CHUNK.size:
|
|
62
|
+
raise CorruptFileError("too short for an ND2 file")
|
|
63
|
+
magic, name_len, data_len, name, data = START_CHUNK.unpack_from(head)
|
|
64
|
+
if magic == JP2_MAGIC:
|
|
65
|
+
return (1, 0)
|
|
66
|
+
if magic != CHUNK_MAGIC or name != FILE_SIGNATURE:
|
|
67
|
+
raise CorruptFileError("not an ND2 file")
|
|
68
|
+
return int(chr(data[3])), int(chr(data[5]))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def chunkmap(fh) -> dict[bytes, tuple[int, int]]:
|
|
72
|
+
"""Chunk name -> (offset of its header, data size), from the end of file."""
|
|
73
|
+
fh.seek(-CHUNKMAP_LOCATION.size, 2)
|
|
74
|
+
sig, location = CHUNKMAP_LOCATION.unpack(fh.read(CHUNKMAP_LOCATION.size))
|
|
75
|
+
if sig != CHUNKMAP_SIGNATURE:
|
|
76
|
+
raise CorruptFileError("ND2 chunk map signature missing (truncated file?)")
|
|
77
|
+
data = read_chunk(fh, location, expect=FILEMAP_SIGNATURE)
|
|
78
|
+
out, pos = {}, 0
|
|
79
|
+
while True:
|
|
80
|
+
end = data.index(b"!", pos) + 1
|
|
81
|
+
name = data[pos:end]
|
|
82
|
+
if name == CHUNKMAP_SIGNATURE:
|
|
83
|
+
return out
|
|
84
|
+
offset, size = struct.unpack_from("<QQ", data, end)
|
|
85
|
+
out[name] = (offset, size)
|
|
86
|
+
pos = end + 16
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def data_offset(fh, offset: int) -> tuple[int, int]:
|
|
90
|
+
"""(file offset of a chunk's data, its length) from the chunk header."""
|
|
91
|
+
fh.seek(offset)
|
|
92
|
+
magic, name_len, data_len = CHUNK_HEADER.unpack(fh.read(CHUNK_HEADER.size))
|
|
93
|
+
if magic != CHUNK_MAGIC:
|
|
94
|
+
raise CorruptFileError(f"no ND2 chunk at byte {offset}")
|
|
95
|
+
return offset + CHUNK_HEADER.size + name_len, data_len
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def read_chunk(fh, offset: int, expect: bytes | None = None) -> bytes:
|
|
99
|
+
start, length = data_offset(fh, offset)
|
|
100
|
+
if expect is not None:
|
|
101
|
+
fh.seek(offset + CHUNK_HEADER.size)
|
|
102
|
+
if not fh.read(len(expect)) == expect:
|
|
103
|
+
raise CorruptFileError(f"expected ND2 chunk {expect!r} at byte {offset}")
|
|
104
|
+
fh.seek(start)
|
|
105
|
+
return fh.read(length)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# -- CLX-lite: Nikon's binary key/value metadata ---------------------------------
|
|
109
|
+
|
|
110
|
+
_BOOL, _INT32, _UINT32, _INT64, _UINT64, _DOUBLE, _VOIDPTR, _STRING, _BYTES, _DEPRECATED, \
|
|
111
|
+
_LEVEL = range(1, 12)
|
|
112
|
+
_COMPRESS = 76
|
|
113
|
+
_SCALARS = {_BOOL: "B", _INT32: "i", _UINT32: "I", _INT64: "q", _UINT64: "Q",
|
|
114
|
+
_DOUBLE: "d", _VOIDPTR: "Q"}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _string(stream: io.BytesIO) -> str:
|
|
118
|
+
value = stream.read(2)
|
|
119
|
+
while not value.endswith(b"\x00\x00"):
|
|
120
|
+
more = stream.read(2)
|
|
121
|
+
if not more:
|
|
122
|
+
break
|
|
123
|
+
value += more
|
|
124
|
+
try:
|
|
125
|
+
return value.decode("utf16")[:-1]
|
|
126
|
+
except UnicodeDecodeError:
|
|
127
|
+
return value.decode("utf8", "replace")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def decode_clx_lite(data: bytes | io.BytesIO, count: int = 1) -> dict:
|
|
131
|
+
"""CLX-lite bytes -> nested dicts. Keys keep their type prefix
|
|
132
|
+
(``uiCount``); lists (items with empty names) become ``{"i0000000000":
|
|
133
|
+
...}`` dicts, as in Nikon's own JSON export and in nd2."""
|
|
134
|
+
stream = data if isinstance(data, io.BytesIO) else io.BytesIO(data)
|
|
135
|
+
out: dict = {}
|
|
136
|
+
for _ in range(count):
|
|
137
|
+
start = stream.tell()
|
|
138
|
+
header = stream.read(2)
|
|
139
|
+
if len(header) < 2:
|
|
140
|
+
break
|
|
141
|
+
kind, name_len = header
|
|
142
|
+
if kind in (0, _DEPRECATED):
|
|
143
|
+
raise CorruptFileError(f"unknown ND2 metadata type {kind}")
|
|
144
|
+
if kind == _COMPRESS:
|
|
145
|
+
stream.seek(10, 1)
|
|
146
|
+
return decode_clx_lite(zlib.decompress(stream.read()))
|
|
147
|
+
name = stream.read(name_len * 2).decode("utf16")[:-1]
|
|
148
|
+
if kind == _LEVEL:
|
|
149
|
+
items, length = struct.unpack("<IQ", stream.read(12))
|
|
150
|
+
inner = decode_clx_lite(stream.read(length - (stream.tell() - start)), items)
|
|
151
|
+
stream.seek(items * 8, 1)
|
|
152
|
+
if len(inner) == 1 and "" in inner:
|
|
153
|
+
values = inner[""] if isinstance(inner[""], list) else [inner[""]]
|
|
154
|
+
value = {f"i{n:010}": v for n, v in enumerate(values)}
|
|
155
|
+
else:
|
|
156
|
+
value = inner
|
|
157
|
+
elif kind == _BYTES:
|
|
158
|
+
size, = struct.unpack("<Q", stream.read(8))
|
|
159
|
+
value = stream.read(size) # opaque; not needed to locate pixels
|
|
160
|
+
elif kind == _STRING:
|
|
161
|
+
value = _string(stream)
|
|
162
|
+
elif kind in _SCALARS:
|
|
163
|
+
fmt = "<" + _SCALARS[kind]
|
|
164
|
+
value, = struct.unpack(fmt, stream.read(struct.calcsize(fmt)))
|
|
165
|
+
else:
|
|
166
|
+
value = None
|
|
167
|
+
if name == "" and name in out:
|
|
168
|
+
if not isinstance(out[name], list):
|
|
169
|
+
out[name] = [out[name]]
|
|
170
|
+
out[name].append(value)
|
|
171
|
+
else:
|
|
172
|
+
out[name] = value
|
|
173
|
+
return out
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
# -- what the pixels need ----------------------------------------------------------
|
|
177
|
+
|
|
178
|
+
@dataclass
|
|
179
|
+
class Attributes:
|
|
180
|
+
height: int
|
|
181
|
+
width: int
|
|
182
|
+
width_bytes: int
|
|
183
|
+
components: int # values per pixel: channels x RGB components
|
|
184
|
+
channels: int
|
|
185
|
+
bits_in_memory: int
|
|
186
|
+
compression: str | None # "lossless", "lossy", or None
|
|
187
|
+
sequence_count: int
|
|
188
|
+
|
|
189
|
+
@property
|
|
190
|
+
def dtype(self) -> str:
|
|
191
|
+
item = self.bits_in_memory // 8
|
|
192
|
+
return f"<f{item}" if self.bits_in_memory == 32 else f"<u{item}"
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def attributes(raw_attrs: dict, raw_meta: dict) -> Attributes:
|
|
196
|
+
attrs = raw_attrs.get("SLxImageAttributes", raw_attrs)
|
|
197
|
+
meta = raw_meta.get("SLxPictureMetadata", raw_meta)
|
|
198
|
+
comp = attrs.get("eCompression", 2)
|
|
199
|
+
tile_w, tile_h = attrs.get("uiTileWidth", 0), attrs.get("uiTileHeight", 0)
|
|
200
|
+
if (0 < tile_w != attrs["uiWidth"]) or (0 < tile_h != attrs["uiHeight"]):
|
|
201
|
+
raise UnsupportedFormatError(f"tiled ND2 frames ({tile_h} x {tile_w})")
|
|
202
|
+
return Attributes(
|
|
203
|
+
height=attrs["uiHeight"], width=attrs["uiWidth"],
|
|
204
|
+
width_bytes=attrs["uiWidthBytes"], components=attrs["uiComp"],
|
|
205
|
+
channels=meta.get("sPicturePlanes", {}).get("uiCount", 1) or 1,
|
|
206
|
+
bits_in_memory=attrs["uiBpcInMemory"],
|
|
207
|
+
compression=["lossless", "lossy", None][comp] if 0 <= comp < 2 else None,
|
|
208
|
+
sequence_count=attrs["uiSequenceCount"],
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@dataclass
|
|
213
|
+
class Loop:
|
|
214
|
+
kind: str # "T", "P" or "Z"
|
|
215
|
+
count: int
|
|
216
|
+
level: int = 0
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _single_loop(exp: dict) -> Loop | None:
|
|
220
|
+
kind = exp.get("eType", 0)
|
|
221
|
+
pars = exp.get("uLoopPars", {})
|
|
222
|
+
if not pars:
|
|
223
|
+
return None
|
|
224
|
+
if list(pars) == ["i0000000000"]:
|
|
225
|
+
pars = next(iter(pars.values()))
|
|
226
|
+
if kind == TIME_LOOP:
|
|
227
|
+
return Loop("T", pars.get("uiCount", 0)) if pars.get("uiCount", 0) else None
|
|
228
|
+
if kind == XY_POS_LOOP:
|
|
229
|
+
valid = exp.get("pItemValid", [])
|
|
230
|
+
points = pars.get("Points")
|
|
231
|
+
count = len(points) if isinstance(points, dict) else len(pars.get("dPosX", {}))
|
|
232
|
+
if valid:
|
|
233
|
+
flags = [v for _, v in sorted(valid.items())] if isinstance(valid, dict) else list(valid)
|
|
234
|
+
if isinstance(flags, (bytes, bytearray)):
|
|
235
|
+
flags = list(flags)
|
|
236
|
+
count = sum(1 for _, ok in zip(range(count), flags) if ok)
|
|
237
|
+
return Loop("P", count)
|
|
238
|
+
if kind == Z_STACK_LOOP:
|
|
239
|
+
return Loop("Z", pars.get("uiCount", 0))
|
|
240
|
+
if kind == NE_TIME_LOOP:
|
|
241
|
+
valid = pars.get("pPeriodValid", [])
|
|
242
|
+
if isinstance(valid, dict):
|
|
243
|
+
valid = [valid[k] for k in sorted(valid)]
|
|
244
|
+
elif isinstance(valid, (bytes, bytearray)):
|
|
245
|
+
valid = list(valid)
|
|
246
|
+
count = sum(p.get("uiCount", 0) for p, ok in zip(pars["pPeriod"].values(), valid) if ok)
|
|
247
|
+
return Loop("T", count)
|
|
248
|
+
if kind == SPECT_LOOP:
|
|
249
|
+
return Loop("S", 0) # marker only: spectral planes are channels
|
|
250
|
+
if kind == CUSTOM_LOOP:
|
|
251
|
+
raise UnsupportedFormatError("ND2 with a custom experiment loop")
|
|
252
|
+
raise UnsupportedFormatError(f"ND2 experiment loop type {kind}")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def experiment(raw: dict, level: int = 0, dest: list | None = None) -> list[Loop]:
|
|
256
|
+
"""The experiment loops, outermost first (frame numbers run over them
|
|
257
|
+
in this order)."""
|
|
258
|
+
raw = raw.get("SLxExperiment", raw) if level == 0 else raw
|
|
259
|
+
dest = dest if dest is not None else []
|
|
260
|
+
loop = _single_loop(raw)
|
|
261
|
+
if loop is None or (loop.count == 0 and loop.kind != "S"):
|
|
262
|
+
return dest
|
|
263
|
+
if loop.kind == "S":
|
|
264
|
+
level -= 1
|
|
265
|
+
elif not dest or dest[-1].level < level:
|
|
266
|
+
loop.level = level
|
|
267
|
+
dest.append(loop)
|
|
268
|
+
else:
|
|
269
|
+
prev = dest[-1]
|
|
270
|
+
if prev.level == level and prev.kind == loop.kind and prev.count < loop.count:
|
|
271
|
+
loop.level = level
|
|
272
|
+
dest[-1] = loop
|
|
273
|
+
pars = raw.get("uLoopPars", {})
|
|
274
|
+
if isinstance(pars, dict) and "pSubLoops" in pars: # rare nested NE-time layout (nd2 #190)
|
|
275
|
+
sub = pars["pSubLoops"].get("i0000000000", {}).get("ppNextLevelEx", {})
|
|
276
|
+
if "i0000000000" in sub:
|
|
277
|
+
dest.extend(experiment(sub["i0000000000"].get("SLxExperiment", {}), 0, []))
|
|
278
|
+
for item in (raw.get("ppNextLevelEx") or {}).values():
|
|
279
|
+
dest = experiment(item, level + 1, dest)
|
|
280
|
+
return dest
|