micro-reader 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {micro_reader-0.2.0 → micro_reader-0.2.1}/PKG-INFO +1 -1
- {micro_reader-0.2.0 → micro_reader-0.2.1}/pyproject.toml +1 -1
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_errors.py +3 -2
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_version.py +1 -1
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_lif_metadata.py +63 -12
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/companion.py +57 -1
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/cv7000.py +95 -32
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/czi.py +7 -2
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/harmony.py +34 -9
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/lif.py +55 -8
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/metamorph.py +19 -6
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/metaxpress.py +77 -39
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/micromanager.py +50 -13
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/oif.py +59 -15
- {micro_reader-0.2.0 → micro_reader-0.2.1}/.gitignore +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/LICENSE +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/README.md +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/__init__.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_axes.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_base.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_cfb.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_codecs.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_eubi_guard.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_fileset.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_groups.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_index.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_io.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_jpeg_restarts.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_memory.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_metadata.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_mosaic.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_outline.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_parts.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_plate.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_pool.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_registry.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_spatial.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_stack.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/_stream.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/__init__.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_czi_metadata.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_ets.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_hdf5.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_nd2_legacy.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_nd2_metadata.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_nd2_structure.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_tiff_metadata.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_tiff_variants.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/_vsi_tree.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/bdv.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/dv.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/hdf5.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/ics.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/ilastik.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/ims.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/incell.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/mrc.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/nd2.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/ndpis.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/oir.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/omp2info.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/raster.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/tiff.py +0 -0
- {micro_reader-0.2.0 → micro_reader-0.2.1}/src/micro_reader/backends/vsi.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: micro-reader
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Low-level, memory-bounded pixel reading for microscopy formats: read exactly the rectangle you ask for.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bugraoezdemir/micro-reader
|
|
6
6
|
Project-URL: Source, https://github.com/bugraoezdemir/micro-reader
|
|
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "micro-reader"
|
|
9
|
-
version = "0.2.
|
|
9
|
+
version = "0.2.1"
|
|
10
10
|
description = "Low-level, memory-bounded pixel reading for microscopy formats: read exactly the rectangle you ask for."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
requires-python = ">=3.11"
|
|
@@ -45,8 +45,9 @@ class CompositeWarning(UserWarning):
|
|
|
45
45
|
|
|
46
46
|
class CompanionWarning(UserWarning):
|
|
47
47
|
"""An OME companion set's metadata came from somewhere other than the file
|
|
48
|
-
opened,
|
|
49
|
-
|
|
48
|
+
opened, a member path had to be resolved differently than written, or the
|
|
49
|
+
companion's bytes had to be read in another encoding than its XML
|
|
50
|
+
declares (docs/COMPANION.md)."""
|
|
50
51
|
|
|
51
52
|
|
|
52
53
|
class SkewedGridWarning(UserWarning):
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""micro-reader's version, in one place (the package, the import guard)."""
|
|
2
|
-
__version__ = "0.2.
|
|
2
|
+
__version__ = "0.2.1"
|
|
@@ -8,11 +8,14 @@ element.
|
|
|
8
8
|
``NumberOfElements - 1`` in m) is LAS's index space -- FLIM "Phasor
|
|
9
9
|
Plot", "Phasor Mask" and "Pattern Matching Scatter Plot", the only such
|
|
10
10
|
images in every local Leica file -- and gives no pixel size.
|
|
11
|
-
- Channels: colour from ``ChannelDescription`` ``LUTName``; names
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
- Channels: colour from ``ChannelDescription`` ``LUTName``; names from the
|
|
12
|
+
image's hardware setting (``_channel_names``): confocal, the active
|
|
13
|
+
detectors' dyes (``DyeName``, else the ``MultiBand`` dye; LAS's ``Leica/``
|
|
14
|
+
prefix dropped), else the detector's name; widefield, the user's channel
|
|
15
|
+
name (``UserDefName``), else the filter cube, else the contrasting method,
|
|
16
|
+
with the emission wavelength -- only when the setting describes exactly
|
|
17
|
+
the image's channels (sequential scans spread channels over several
|
|
18
|
+
settings: names unknown). Excitation wavelengths: out of scope for now.
|
|
16
19
|
- Index values (D5): the image's wavelength along a lambda (``emission``,
|
|
17
20
|
DimID 5) or excitation (``excitation``, DimID 9) dimension, ``Origin`` +
|
|
18
21
|
k x ``Length`` / (N - 1), in nm.
|
|
@@ -73,6 +76,57 @@ def _setting(element):
|
|
|
73
76
|
return None
|
|
74
77
|
|
|
75
78
|
|
|
79
|
+
#: LAS X's dye catalogue prefixes a dye's name (``Leica/ALEXA 488``)
|
|
80
|
+
DYE_PREFIXES = ("Leica/",)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _dye(name):
|
|
84
|
+
"""A dye's name as the user knows it: the catalogue prefix dropped; None for none."""
|
|
85
|
+
name = (name or "").strip()
|
|
86
|
+
for prefix in DYE_PREFIXES:
|
|
87
|
+
if name.startswith(prefix):
|
|
88
|
+
name = name[len(prefix):]
|
|
89
|
+
return name or None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _channel_names(setting, n_c: int, luts: list) -> tuple:
|
|
93
|
+
"""``([name or None, ...], [emission or None, ...])`` for the image's *n_c*
|
|
94
|
+
channels from its hardware setting, or ``([], [])`` when the setting does not
|
|
95
|
+
describe exactly them (a sequential scan spreads channels over settings).
|
|
96
|
+
|
|
97
|
+
Confocal (``ATLConfocalSettingDefinition``): the active detectors, in order; a
|
|
98
|
+
channel is named after its dye (the detector's ``DyeName``, else the ``MultiBand``
|
|
99
|
+
dye of its channel; ``_dye``), else after the detector (``HyD S 2``). Widefield
|
|
100
|
+
(``ATLCameraSettingDefinition``): the active ``WideFieldChannelInfo`` entries; when
|
|
101
|
+
there are more than channels (an autofocus channel beside the image's), those whose
|
|
102
|
+
``LUT`` matches the channels' ``LUTName``, in order. A channel is named as the user
|
|
103
|
+
named it (``UserDefName``), else after its filter cube (``FluoCubeName``), else its
|
|
104
|
+
contrasting method (``FLUO``, ``BF``); its ``EmissionWavelength`` is in nm."""
|
|
105
|
+
if setting.tag == "ATLCameraSettingDefinition":
|
|
106
|
+
infos = [w for w in setting.iter("WideFieldChannelInfo") if w.get("Active") == "1"]
|
|
107
|
+
if len(infos) != n_c and len(luts) == n_c:
|
|
108
|
+
picked, left = [], list(infos)
|
|
109
|
+
for lut in luts:
|
|
110
|
+
hit = next((w for w in left if (w.get("LUT") or "").lower() == (lut or "").lower()), None)
|
|
111
|
+
if hit is None:
|
|
112
|
+
break
|
|
113
|
+
picked.append(hit)
|
|
114
|
+
left.remove(hit)
|
|
115
|
+
infos = picked
|
|
116
|
+
if len(infos) != n_c:
|
|
117
|
+
return [], []
|
|
118
|
+
names = [w.get("UserDefName") or w.get("FluoCubeName") or w.get("ContrastingMethodName") or None
|
|
119
|
+
for w in infos]
|
|
120
|
+
emissions = [quantity(w.get("EmissionWavelength"), "nanometer") for w in infos]
|
|
121
|
+
return names, emissions
|
|
122
|
+
bands = {b.get("Channel"): b.get("DyeName") or None for b in setting.iter("MultiBand")}
|
|
123
|
+
active = [d for d in setting.iter("Detector") if d.get("IsActive") == "1"]
|
|
124
|
+
if len(active) != n_c:
|
|
125
|
+
return [], []
|
|
126
|
+
return [_dye(d.get("DyeName") or bands.get(d.get("Channel"))) or d.get("Name") or None
|
|
127
|
+
for d in active], []
|
|
128
|
+
|
|
129
|
+
|
|
76
130
|
def _stage_position(element, setting, tile) -> dict:
|
|
77
131
|
"""The stage position recorded for the image (*tile*: None) or for its
|
|
78
132
|
tile (``TileScanInfo`` PosX / PosY): {axis: Quantity}. Both are along
|
|
@@ -229,16 +283,13 @@ def lif_metadata(element, shape: dict, index: dict, tiled: bool = False,
|
|
|
229
283
|
if shape.get("s", 1) > 1: # interleaved RGB: one channel, no LUT colour
|
|
230
284
|
colors = []
|
|
231
285
|
setting = _setting(element)
|
|
232
|
-
names = []
|
|
233
|
-
|
|
234
|
-
bands = {b.get("Channel"): b.get("DyeName") or None for b in setting.iter("MultiBand")}
|
|
235
|
-
active = [d.get("Channel") for d in setting.iter("Detector") if d.get("IsActive") == "1"]
|
|
236
|
-
if len(active) == n_c:
|
|
237
|
-
names = [bands.get(ch) for ch in active]
|
|
286
|
+
names, emissions = ([], []) if setting is None else _channel_names(
|
|
287
|
+
setting, n_c, [c.get("LUTName") for c in descriptions])
|
|
238
288
|
channels = []
|
|
239
289
|
for k in range(n_c):
|
|
240
290
|
channels.append(Channel(name=names[k] if k < len(names) else None,
|
|
241
|
-
color=colors[k] if k < len(colors) else None
|
|
291
|
+
color=colors[k] if k < len(colors) else None,
|
|
292
|
+
emission_wavelength=emissions[k] if k < len(emissions) else None))
|
|
242
293
|
position = {}
|
|
243
294
|
stage = _stage_position(element, setting, tile)
|
|
244
295
|
if stage:
|
|
@@ -24,6 +24,9 @@ OME-TIFFs: each ``TiffData`` says which file (``UUID FileName``), which pages
|
|
|
24
24
|
not exist falls back to the same name in the companion's folder, with a
|
|
25
25
|
``CompanionWarning``.
|
|
26
26
|
- Missing members: as for every file set (``_fileset.keep_fields``).
|
|
27
|
+
- A companion whose bytes are not in the encoding it declares (Prairie View:
|
|
28
|
+
Windows ``µ`` under UTF-8) is read as cp1252, else latin-1, with a
|
|
29
|
+
``CompanionWarning``; one that is no XML either way is a ``CorruptFileError``.
|
|
27
30
|
- The parsed companion is cached per process (``CORE_CACHE`` companions, by
|
|
28
31
|
path, modification time and size), apart from the pixel caches.
|
|
29
32
|
|
|
@@ -34,8 +37,10 @@ separate images (docs/COMPANION.md, "A").
|
|
|
34
37
|
from __future__ import annotations
|
|
35
38
|
|
|
36
39
|
import builtins
|
|
40
|
+
import codecs
|
|
37
41
|
import math
|
|
38
42
|
import os
|
|
43
|
+
import re
|
|
39
44
|
import threading
|
|
40
45
|
import warnings
|
|
41
46
|
import xml.etree.ElementTree as ET
|
|
@@ -289,6 +294,11 @@ class Core:
|
|
|
289
294
|
|
|
290
295
|
#: the first bytes of a TIFF / BigTIFF, either byte order
|
|
291
296
|
TIFF_MAGIC = (b"II*\x00", b"MM\x00*", b"II+\x00", b"MM\x00+")
|
|
297
|
+
#: encodings tried in turn when a companion's bytes are not in the encoding
|
|
298
|
+
#: its XML declaration says: cp1252 first (what Windows software writes, and
|
|
299
|
+
#: what tifffile falls back to for a TIFF description), latin-1 for the few
|
|
300
|
+
#: bytes cp1252 leaves undefined
|
|
301
|
+
FALLBACK_ENCODINGS = ("cp1252", "latin-1")
|
|
292
302
|
|
|
293
303
|
|
|
294
304
|
def _ome_root(path: str):
|
|
@@ -298,7 +308,7 @@ def _ome_root(path: str):
|
|
|
298
308
|
with builtins.open(path, "rb") as fh:
|
|
299
309
|
magic = fh.read(4)
|
|
300
310
|
if magic not in TIFF_MAGIC:
|
|
301
|
-
return
|
|
311
|
+
return _xml_file_root(path)
|
|
302
312
|
import tifffile
|
|
303
313
|
try:
|
|
304
314
|
with tifffile.TiffFile(path, **PLAIN_PAGES) as tif:
|
|
@@ -310,6 +320,52 @@ def _ome_root(path: str):
|
|
|
310
320
|
return ET.fromstring(text)
|
|
311
321
|
|
|
312
322
|
|
|
323
|
+
def _xml_file_root(path: str):
|
|
324
|
+
"""The root element of the XML file *path*.
|
|
325
|
+
|
|
326
|
+
A file whose bytes are not in the encoding its declaration says (Prairie
|
|
327
|
+
View writes ``µ`` as the one Windows byte 0xB5 into every unit attribute,
|
|
328
|
+
under ``encoding="UTF-8"``) is decoded as cp1252, else latin-1
|
|
329
|
+
(``FALLBACK_ENCODINGS``), with a ``CompanionWarning`` naming the file and
|
|
330
|
+
the encoding used. A file that is no XML in any of them raises the
|
|
331
|
+
``ET.ParseError`` of its declared encoding, as before."""
|
|
332
|
+
try:
|
|
333
|
+
return ET.parse(path).getroot()
|
|
334
|
+
except ET.ParseError as exc:
|
|
335
|
+
error = exc
|
|
336
|
+
with builtins.open(path, "rb") as fh:
|
|
337
|
+
raw = fh.read()
|
|
338
|
+
if raw.startswith(codecs.BOM_UTF8): # a BOM is no character of the text
|
|
339
|
+
raw = raw[len(codecs.BOM_UTF8):]
|
|
340
|
+
declared = _declared_encoding(raw)
|
|
341
|
+
try:
|
|
342
|
+
raw.decode(declared)
|
|
343
|
+
except (UnicodeDecodeError, LookupError) as bad:
|
|
344
|
+
where = f"byte {bad.start}" if isinstance(bad, UnicodeDecodeError) else "unknown"
|
|
345
|
+
else:
|
|
346
|
+
raise error # the bytes are fine: it is no XML
|
|
347
|
+
for encoding in FALLBACK_ENCODINGS:
|
|
348
|
+
try:
|
|
349
|
+
text = raw.decode(encoding)
|
|
350
|
+
except UnicodeDecodeError:
|
|
351
|
+
continue
|
|
352
|
+
# the declaration is dropped: it names the encoding the bytes are not in
|
|
353
|
+
text = re.sub(r"^\s*<\?xml[^>]*\?>", "", text)
|
|
354
|
+
root = ET.fromstring(text) # ParseError: no XML in this encoding either
|
|
355
|
+
warnings.warn(CompanionWarning(
|
|
356
|
+
f"{path!r}: its XML declares {declared!r} but its bytes are not ({where}); read "
|
|
357
|
+
f"as {encoding!r}"), stacklevel=5)
|
|
358
|
+
return root
|
|
359
|
+
raise error
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _declared_encoding(raw: bytes) -> str:
|
|
363
|
+
"""The encoding an XML declaration at the start of *raw* names, else
|
|
364
|
+
"utf-8" (the XML default)."""
|
|
365
|
+
found = re.match(rb'\s*<\?xml[^>]*?encoding\s*=\s*["\']([A-Za-z0-9._-]+)["\']', raw)
|
|
366
|
+
return found.group(1).decode("ascii").lower() if found else "utf-8"
|
|
367
|
+
|
|
368
|
+
|
|
313
369
|
def _page_count(path: str) -> int:
|
|
314
370
|
"""Pages of a member file (an OME TiffData without IFD or PlaneCount covers
|
|
315
371
|
them all); 1 when the file is missing (it reads as zeros anyway)."""
|
|
@@ -13,7 +13,13 @@ micro-reader reads the measurement in that folder.
|
|
|
13
13
|
- ``MeasurementData.mlf``: one ``MeasurementRecord`` per plane: ``Type``
|
|
14
14
|
``IMG`` with 1-based ``Row`` / ``Column`` / ``TimePoint`` / ``FieldIndex``
|
|
15
15
|
/ ``ZIndex``, ``ActionIndex``, ``Ch``, stage ``X`` / ``Y`` / ``Z`` (um) and
|
|
16
|
-
the file name; ``ERR`` records log instrument errors (one warning).
|
|
16
|
+
the file name; ``ERR`` records log instrument errors (one warning). A
|
|
17
|
+
plane listed twice is read from the first record whose file exists
|
|
18
|
+
(Bio-Formats' rule: a metadata-only record never shadows a real one).
|
|
19
|
+
- The files name their elements and attributes in the BTS namespace with
|
|
20
|
+
any prefix (``bts:``, ``record:``, a default namespace) or none at all;
|
|
21
|
+
they are matched by namespace URI or local name, never by prefix. Numbers
|
|
22
|
+
may use a decimal comma (``0,65``).
|
|
17
23
|
- One micro-reader image per field, axes ``t c z y x``. A channel is an
|
|
18
24
|
(action, camera channel) pair, as Bio-Formats counts them. Indices:
|
|
19
25
|
``plate`` 0, ``acquisition`` 0, ``well`` (row then column), ``field`` =
|
|
@@ -40,13 +46,23 @@ Harmony-format export (``Index.idx.xml``) and the settings files (``.wpp``,
|
|
|
40
46
|
``.mes``, ``.ppp``, ``.ppf``, ``.icr``, ``*_parameter.xml``) are its parts and
|
|
41
47
|
open it. The plane TIFFs are the index's members (they open alone).
|
|
42
48
|
|
|
43
|
-
Missing files: as for every file set (``_fileset.keep_fields``).
|
|
44
|
-
|
|
45
|
-
|
|
49
|
+
Missing files: as for every file set (``_fileset.keep_fields``). The
|
|
50
|
+
timelines of the settings file (``TimelineIndex``: each images its own wells,
|
|
51
|
+
fields and actions, its TimePoint numbers running on from the timeline
|
|
52
|
+
before) are merged, as Bio-Formats merges them (checked against its expected
|
|
53
|
+
results for the 14 synthetic packages of Zenodo 21334895): each timeline's
|
|
54
|
+
timepoints are counted from its own first one, a field holds every plane
|
|
55
|
+
recorded for it, a well imaged by two timelines holds the channels of both,
|
|
56
|
+
and the plate's channels are the (action, camera channel) pairs of all its
|
|
57
|
+
records; a plane a second timeline images with the channel and timepoint of
|
|
58
|
+
the first becomes a channel of its own, named after its timeline (Bio-Formats'
|
|
59
|
+
"timeline-qualified overflow"). Refused: channels imaged at different
|
|
60
|
+
timepoints, channels with different pixel sizes or image sizes.
|
|
46
61
|
"""
|
|
47
62
|
from __future__ import annotations
|
|
48
63
|
|
|
49
64
|
import os
|
|
65
|
+
import re
|
|
50
66
|
import warnings
|
|
51
67
|
import xml.etree.ElementTree as ET
|
|
52
68
|
from dataclasses import replace
|
|
@@ -65,8 +81,10 @@ NAME = "cv7000"
|
|
|
65
81
|
SET_INDEX = True
|
|
66
82
|
BTS = "{http://www.yokogawa.co.jp/BTS/BTSSchema/1.0}"
|
|
67
83
|
DETAIL, DATA = "MeasurementDetail.mrf", "MeasurementData.mlf"
|
|
68
|
-
#: the root
|
|
69
|
-
|
|
84
|
+
#: the root element of a file that opens a measurement (not .wpp / .mes),
|
|
85
|
+
#: with any namespace prefix or none (``<bts:WellPlate ``, ``<plate:WellPlate ``,
|
|
86
|
+
#: ``<WellPlate>``; not ``<bts:WellPlateProduct``)
|
|
87
|
+
ENTRY_ROOT = re.compile(rb"<(?:[A-Za-z_][\w.-]*:)?(?:WellPlate|MeasurementDetail|MeasurementData)[\s/>]")
|
|
70
88
|
|
|
71
89
|
|
|
72
90
|
#: the CellVoyager software's default channel ``Target``: it names no channel
|
|
@@ -131,11 +149,21 @@ def may_index(name: str) -> bool:
|
|
|
131
149
|
|
|
132
150
|
|
|
133
151
|
def sniff(head: bytes, path: str) -> bool:
|
|
134
|
-
return b"yokogawa.co.jp/BTS" in head and
|
|
152
|
+
return b"yokogawa.co.jp/BTS" in head and ENTRY_ROOT.search(head) is not None
|
|
135
153
|
|
|
136
154
|
|
|
137
155
|
def _a(e, name):
|
|
138
|
-
|
|
156
|
+
"""Attribute *name* of *e*: in the BTS namespace (any prefix), else
|
|
157
|
+
unprefixed (no namespace, also under a default namespace)."""
|
|
158
|
+
value = e.get(BTS + name)
|
|
159
|
+
return e.get(name) if value is None else value
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _number(value):
|
|
163
|
+
"""A number's text with a decimal comma (``0,65``) made a point."""
|
|
164
|
+
if isinstance(value, str) and value.count(",") == 1 and "." not in value:
|
|
165
|
+
return value.replace(",", ".")
|
|
166
|
+
return value
|
|
139
167
|
|
|
140
168
|
|
|
141
169
|
def _int(e, name):
|
|
@@ -147,7 +175,7 @@ def _int(e, name):
|
|
|
147
175
|
|
|
148
176
|
def _float(e, name):
|
|
149
177
|
try:
|
|
150
|
-
return float(_a(e, name))
|
|
178
|
+
return float(_number(_a(e, name)))
|
|
151
179
|
except (TypeError, ValueError):
|
|
152
180
|
return None
|
|
153
181
|
|
|
@@ -162,7 +190,7 @@ def _parse(path):
|
|
|
162
190
|
|
|
163
191
|
|
|
164
192
|
class _CVImage(PlaneImage):
|
|
165
|
-
"""A field (or a part of a ragged one); *channels*: its (action, Ch)
|
|
193
|
+
"""A field (or a part of a ragged one); *channels*: its (action, Ch, run)
|
|
166
194
|
keys, *records*: its planes' (t, key, z) -> (X, Y, Z, Time)."""
|
|
167
195
|
|
|
168
196
|
def __init__(self, *args, info: dict, channels: list, dz, records: dict, times: list,
|
|
@@ -204,8 +232,19 @@ class _CVImage(PlaneImage):
|
|
|
204
232
|
position = {a: q for a, q in position.items() if q is not None}
|
|
205
233
|
return Metadata(pixel_sizes={a: q for a, q in sizes.items() if q}, position=position,
|
|
206
234
|
coordinate_system=self._system if position else None,
|
|
207
|
-
channels=tuple(
|
|
208
|
-
for act, ch in self._channels))
|
|
235
|
+
channels=tuple(self._channel(names, act, ch, run)
|
|
236
|
+
for act, ch, run in self._channels))
|
|
237
|
+
|
|
238
|
+
@staticmethod
|
|
239
|
+
def _channel(names: dict, act: int, ch: int, run: int) -> Channel:
|
|
240
|
+
"""The channel's description: the settings file's for camera channel *ch*,
|
|
241
|
+
else a name from the action and channel; a channel a second timeline
|
|
242
|
+
added (*run*: its TimelineIndex, 0 for the first) says so."""
|
|
243
|
+
channel = names.get(ch, Channel(name=f"Action {act}, Ch {ch}"))
|
|
244
|
+
if run:
|
|
245
|
+
channel = replace(channel, name=f"{channel.name or f'Action {act}, Ch {ch}'} "
|
|
246
|
+
f"(timeline {run})")
|
|
247
|
+
return channel
|
|
209
248
|
|
|
210
249
|
|
|
211
250
|
class _CVFile(File):
|
|
@@ -219,35 +258,59 @@ class _CVFile(File):
|
|
|
219
258
|
detail = _parse(os.path.join(folder, DETAIL))
|
|
220
259
|
data = _parse(os.path.join(folder, DATA))
|
|
221
260
|
rows, columns = _int(detail, "RowCount"), _int(detail, "ColumnCount")
|
|
222
|
-
sample = detail.find(
|
|
261
|
+
sample = detail.find("{*}MeasurementSamplePlate")
|
|
223
262
|
plate_name = (_a(sample, "Name") if sample is not None else None) or os.path.basename(folder)
|
|
224
|
-
cameras = {_int(e, "Ch"): e for e in detail.findall(
|
|
263
|
+
cameras = {_int(e, "Ch"): e for e in detail.findall("{*}MeasurementChannel")}
|
|
225
264
|
settings = _a(detail, "MeasurementSettingFileName")
|
|
226
265
|
channel_info = self._channels(os.path.join(folder, settings)) if settings else {}
|
|
227
266
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
267
|
+
# every IMG record of the measurement; the timelines of the settings file
|
|
268
|
+
# (TimelineIndex) are merged as Bio-Formats merges them: each timeline's
|
|
269
|
+
# timepoints counted from its own first one (TimePoint numbers run on from
|
|
270
|
+
# one timeline to the next), then the planes of a field put together
|
|
271
|
+
# (module docstring)
|
|
272
|
+
found: list = [] # (timeline, (row, col, field), (TimePoint, (action, ch), z), path, rec)
|
|
273
|
+
errors = []
|
|
274
|
+
for e in data.findall("{*}MeasurementRecord"):
|
|
232
275
|
kind = _a(e, "Type")
|
|
233
276
|
if kind == "ERR":
|
|
234
277
|
errors.append((e.text or "").strip())
|
|
235
278
|
continue
|
|
236
279
|
if kind != "IMG":
|
|
237
280
|
continue
|
|
238
|
-
|
|
281
|
+
timeline = _int(e, "TimelineIndex") if _a(e, "TimelineIndex") is not None else 1
|
|
239
282
|
key = (_int(e, "Row") - 1, _int(e, "Column") - 1, _int(e, "FieldIndex"))
|
|
240
283
|
plane = (_int(e, "TimePoint"), (_int(e, "ActionIndex"), _int(e, "Ch")),
|
|
241
284
|
_int(e, "ZIndex"))
|
|
242
285
|
name = (e.text or "").strip()
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
286
|
+
found.append((timeline, key, plane, os.path.join(folder, name) if name else None,
|
|
287
|
+
(_float(e, "X"), _float(e, "Y"), _float(e, "Z"),
|
|
288
|
+
self._time(_a(e, "Time")))))
|
|
289
|
+
ranks = {} # timeline -> {TimePoint: its rank in the timeline}
|
|
290
|
+
for timeline, _key, (tp, _ch, _z), _p, _rec in found:
|
|
291
|
+
ranks.setdefault(timeline, set()).add(tp)
|
|
292
|
+
ranks = {line: {tp: k for k, tp in enumerate(sorted(tps))} for line, tps in ranks.items()}
|
|
293
|
+
listed: dict = {} # (row, col, field) -> {(t, (action, ch), z): path}
|
|
294
|
+
records: dict = {} # (row, col, field) -> {(t, (action, ch), z): (X, Y, Z, Time)}
|
|
295
|
+
owner: dict = {} # (row, col, field), plane -> the timeline it came from
|
|
296
|
+
for timeline, key, (tp, (action, ch), z), file, rec in found:
|
|
297
|
+
# a channel is (action, camera channel, 0); a plane another timeline
|
|
298
|
+
# imaged already with the same channel and timepoint becomes a channel
|
|
299
|
+
# of its own, qualified by its timeline (Bio-Formats' "overflow")
|
|
300
|
+
plane = (ranks[timeline][tp], (action, ch, 0), z)
|
|
301
|
+
field = listed.setdefault(key, {})
|
|
302
|
+
if plane in field and owner[(key, plane)] != timeline and field[plane] != file:
|
|
303
|
+
plane = (plane[0], (action, ch, timeline), z)
|
|
304
|
+
if plane in field:
|
|
305
|
+
# a plane listed again: the first record whose file exists is kept
|
|
306
|
+
# (a metadata-only record never shadows a real one, module docstring)
|
|
307
|
+
if field[plane] and os.path.isfile(field[plane]):
|
|
308
|
+
continue
|
|
309
|
+
field[plane] = file
|
|
310
|
+
owner[(key, plane)] = timeline
|
|
311
|
+
records.setdefault(key, {})[plane] = rec
|
|
246
312
|
if not listed:
|
|
247
313
|
raise CorruptFileError(f"{path!r}: the measurement lists no images")
|
|
248
|
-
if len(timelines) > 1:
|
|
249
|
-
raise UnsupportedFormatError(
|
|
250
|
-
f"{path!r}: {len(timelines)} timelines in one measurement are not read yet")
|
|
251
314
|
if errors:
|
|
252
315
|
warnings.warn(f"{path!r}: the instrument logged {len(errors)} error"
|
|
253
316
|
f"{'' if len(errors) == 1 else 's'} (e.g. {errors[0]!r})", stacklevel=3)
|
|
@@ -255,7 +318,7 @@ class _CVFile(File):
|
|
|
255
318
|
wanted = sorted({p for field in listed.values() for p in field})
|
|
256
319
|
times = sorted({t for t, _k, _z in wanted})
|
|
257
320
|
check_times(wanted, path)
|
|
258
|
-
used = sorted({ch for _t, (_a_, ch), _z in wanted})
|
|
321
|
+
used = sorted({ch for _t, (_a_, ch, _run), _z in wanted})
|
|
259
322
|
sizes = {(_float(cameras[ch], "HorizontalPixelDimension"),
|
|
260
323
|
_float(cameras[ch], "VerticalPixelDimension"),
|
|
261
324
|
_a(cameras[ch], "HorizontalPixels"), _a(cameras[ch], "VerticalPixels"))
|
|
@@ -274,8 +337,8 @@ class _CVFile(File):
|
|
|
274
337
|
shape_hint = (_int(camera, "VerticalPixels"), _int(camera, "HorizontalPixels"))
|
|
275
338
|
dtype, unit, shape_yx = self._members.plane_info(first, shape_hint)
|
|
276
339
|
info = {"pixel_sizes": {} if camera is None else {
|
|
277
|
-
"x": quantity(
|
|
278
|
-
"y": quantity(
|
|
340
|
+
"x": quantity(_float(camera, "HorizontalPixelDimension"), "micrometer"),
|
|
341
|
+
"y": quantity(_float(camera, "VerticalPixelDimension"), "micrometer")},
|
|
279
342
|
"channels": channel_info}
|
|
280
343
|
|
|
281
344
|
wells_rc = sorted({(r, c) for r, c, _f in kept})
|
|
@@ -332,13 +395,13 @@ class _CVFile(File):
|
|
|
332
395
|
if not os.path.isfile(mes):
|
|
333
396
|
return {}
|
|
334
397
|
root = _parse(mes)
|
|
335
|
-
lights = {e
|
|
336
|
-
for e in root.
|
|
398
|
+
lights = {_a(e, "Name"): _float(e, "WaveLength")
|
|
399
|
+
for e in root.iterfind(".//{*}LightSource")}
|
|
337
400
|
out, fallback = {}, {}
|
|
338
|
-
for e in root.
|
|
401
|
+
for e in root.iterfind(".//{*}Channel"):
|
|
339
402
|
if _a(e, "Ch") is None:
|
|
340
403
|
continue
|
|
341
|
-
light = e.find(
|
|
404
|
+
light = e.find("{*}LightSourceName")
|
|
342
405
|
light = light.text.strip() if light is not None and light.text else None
|
|
343
406
|
ex = lights.get(light) if light else None
|
|
344
407
|
em = None
|
|
@@ -315,6 +315,10 @@ class _PlaneTiles:
|
|
|
315
315
|
self.rects = [(s.y - oy, s.y - oy + s.height, s.x - ox, s.x - ox + s.width)
|
|
316
316
|
for s in subblocks]
|
|
317
317
|
self.cell = (max(s.height for s in subblocks), max(s.width for s in subblocks))
|
|
318
|
+
# a compressed tile is decoded whole: a band shorter than it would decode the
|
|
319
|
+
# same tile again for every band it crosses (the cache holds a few tiles)
|
|
320
|
+
self.decoded_rows = max((s.height for s in subblocks if s.compression != UNCOMPRESSED),
|
|
321
|
+
default=0)
|
|
318
322
|
self.grid: dict = {}
|
|
319
323
|
for i, (y0, y1, x0, x1) in enumerate(self.rects):
|
|
320
324
|
for cy in range(y0 // self.cell[0], (y1 - 1) // self.cell[0] + 1):
|
|
@@ -782,8 +786,9 @@ class _CziImage(Image):
|
|
|
782
786
|
if tiles is None:
|
|
783
787
|
continue # nothing acquired: background
|
|
784
788
|
dst = out[tuple(k - s.start for k, s in zip(key, plane_spans))]
|
|
785
|
-
|
|
786
|
-
|
|
789
|
+
rows_per_task = max(per_task, tiles.decoded_rows)
|
|
790
|
+
for b0 in range(ys.start, ys.stop, rows_per_task):
|
|
791
|
+
b1 = min(ys.stop, b0 + rows_per_task)
|
|
787
792
|
part = dst[b0 - ys.start:b1 - ys.start]
|
|
788
793
|
tasks.append(lambda a=(tiles, b0, b1, xs, part): self._draw_band(*a))
|
|
789
794
|
read = _pool.run(tasks)
|
|
@@ -13,6 +13,9 @@ the index file: micro-reader reads the plate it describes.
|
|
|
13
13
|
0 or 1 depending on the instrument).
|
|
14
14
|
- Indices: ``plate`` 0, ``acquisition`` 0 (one measurement per index file),
|
|
15
15
|
``well`` (the wells with images, row then column), ``field`` = FieldID - 1.
|
|
16
|
+
- The plate's name: the index's ``Name``, else its ``PlateID``, else the
|
|
17
|
+
measurement folder's name (the folder above ``Images``; an index whose
|
|
18
|
+
``Name`` and ``PlateID`` are empty is common), as outputs are named.
|
|
16
19
|
- Missing planes (empty ``URL`` or no file; vendors list planes they did not
|
|
17
20
|
write): a field missing some reads them as zeros, a field missing all is
|
|
18
21
|
left out, with one ``MissingImagesWarning`` (``_fileset.keep_fields``).
|
|
@@ -25,9 +28,12 @@ the index file: micro-reader reads the plate it describes.
|
|
|
25
28
|
the field's well.
|
|
26
29
|
|
|
27
30
|
The measurement folder is also a composite (docs/COMPOSITE.md): next to
|
|
28
|
-
``Images/`` it holds ``Assaylayout/`` (the plate map)
|
|
29
|
-
flat-field model)
|
|
30
|
-
|
|
31
|
+
``Images/`` it holds ``Assaylayout/`` (the plate map), ``FFC_Profile/`` (the
|
|
32
|
+
flat-field model) and, once Harmony's analysis has run, ``Evaluation1/``,
|
|
33
|
+
``Evaluation2/``, ... (its results: ``PlateResults.xml``, ``ObjectList_*.xml``,
|
|
34
|
+
the ``.aas`` analysis script); their files, those folders, the other files in
|
|
35
|
+
the measurement folder and the measurement folder itself are parts of
|
|
36
|
+
``Images/Index.idx.xml`` and open it.
|
|
31
37
|
"""
|
|
32
38
|
from __future__ import annotations
|
|
33
39
|
|
|
@@ -50,14 +56,22 @@ SET_INDEX = True
|
|
|
50
56
|
#: where a measurement folder keeps its index (letter case aside: the "hs"
|
|
51
57
|
#: archive of Harmony 5.2 / 6 writes ``images/index.xml``), and its side
|
|
52
58
|
#: folders (lower case): classic ``Assaylayout`` / ``FFC_Profile``, hs
|
|
53
|
-
#: ``flatfieldcorrection`` / ``index`` (measurement descriptions)
|
|
59
|
+
#: ``flatfieldcorrection`` / ``index`` (measurement descriptions), and the
|
|
60
|
+
#: analysis results ``Evaluation<n>`` (``SIDE_FOLDER_PATTERN``)
|
|
54
61
|
IMAGES_FOLDERS = ("Images", "images")
|
|
55
62
|
INDEX_FILES = ("Index.idx.xml", "Index.xml", "index.xml", "index.idx.xml")
|
|
56
63
|
SIDE_FOLDERS = ("assaylayout", "ffc_profile", "flatfieldcorrection", "index")
|
|
64
|
+
SIDE_FOLDER_PATTERN = re.compile(r"evaluation\d+")
|
|
57
65
|
#: plane files: members of the index, opened as themselves (never parts)
|
|
58
66
|
PLANE_SUFFIXES = (".tif", ".tiff")
|
|
59
67
|
|
|
60
68
|
|
|
69
|
+
def _side_folder(name: str) -> bool:
|
|
70
|
+
"""Is a folder called *name* (any letter case) a measurement's side folder?"""
|
|
71
|
+
name = name.lower()
|
|
72
|
+
return name in SIDE_FOLDERS or SIDE_FOLDER_PATTERN.fullmatch(name) is not None
|
|
73
|
+
|
|
74
|
+
|
|
61
75
|
def _measurement_index(measurement: str):
|
|
62
76
|
"""The index file of the measurement folder *measurement*, or None (the
|
|
63
77
|
spellings above only: ``owner`` runs on every open, so no folder listing)."""
|
|
@@ -71,6 +85,16 @@ def _measurement_index(measurement: str):
|
|
|
71
85
|
return None
|
|
72
86
|
|
|
73
87
|
|
|
88
|
+
def _measurement_name(images: str) -> str:
|
|
89
|
+
"""The name of the measurement folder of the index in folder *images*: the
|
|
90
|
+
folder above ``Images``, or *images* itself when the index sits among its
|
|
91
|
+
planes (zenodo-15149351); ``""`` at a drive root."""
|
|
92
|
+
folder = os.path.abspath(images)
|
|
93
|
+
if os.path.basename(folder).lower() == "images":
|
|
94
|
+
folder = os.path.dirname(folder)
|
|
95
|
+
return os.path.basename(folder)
|
|
96
|
+
|
|
97
|
+
|
|
74
98
|
def _spelled(path: str) -> str:
|
|
75
99
|
"""*path* with its folder's and file's names as stored (a case-insensitive
|
|
76
100
|
file system finds ``Index.xml`` for ``index.xml``)."""
|
|
@@ -91,7 +115,7 @@ def may_own(folder: str, name: str, folders) -> bool:
|
|
|
91
115
|
in a measurement folder (next to ``Images``) or, unless it is a plane,
|
|
92
116
|
in ``Images`` may be a part."""
|
|
93
117
|
base = folders.base(folder).lower()
|
|
94
|
-
return base
|
|
118
|
+
return _side_folder(base) or "images" in folders.names(folder) or \
|
|
95
119
|
(base == "images" and not name.lower().endswith(PLANE_SUFFIXES))
|
|
96
120
|
|
|
97
121
|
|
|
@@ -102,10 +126,10 @@ def owner(path: str, head):
|
|
|
102
126
|
path = os.path.abspath(path)
|
|
103
127
|
parent = os.path.dirname(path)
|
|
104
128
|
if os.path.isdir(path):
|
|
105
|
-
if os.path.basename(path)
|
|
129
|
+
if _side_folder(os.path.basename(path)):
|
|
106
130
|
path = os.path.dirname(path)
|
|
107
131
|
index = _measurement_index(path)
|
|
108
|
-
elif os.path.basename(parent)
|
|
132
|
+
elif _side_folder(os.path.basename(parent)):
|
|
109
133
|
index = _measurement_index(os.path.dirname(parent))
|
|
110
134
|
elif os.path.basename(parent).lower() == "images":
|
|
111
135
|
if path.lower().endswith(PLANE_SUFFIXES):
|
|
@@ -296,8 +320,9 @@ class _HarmonyFile(File):
|
|
|
296
320
|
shape = (len(tps), len(chs), len(pls)) + shape_yx
|
|
297
321
|
position = {(tp, ch, pl): (tps.index(tp), chs.index(ch), pls.index(pl))
|
|
298
322
|
for tp, ch, pl in wanted}
|
|
299
|
-
plate_name = _text(plate_e, "Name") or _text(plate_e, "PlateID")
|
|
300
|
-
|
|
323
|
+
plate_name = (_text(plate_e, "Name") or _text(plate_e, "PlateID")
|
|
324
|
+
or _measurement_name(folder) or "")
|
|
325
|
+
base = plate_name or os.path.basename(path)
|
|
301
326
|
row_names, column_names = layout(rows, columns, wells_rc, path)
|
|
302
327
|
plates = [Plate(name=plate_name, rows=row_names, columns=column_names,
|
|
303
328
|
wells=tuple(Well(r, c, f"{row_label(r)}{c + 1}") for r, c in wells_rc))]
|