micro-reader 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- micro_reader/__init__.py +23 -0
- micro_reader/_axes.py +196 -0
- micro_reader/_base.py +268 -0
- micro_reader/_codecs.py +439 -0
- micro_reader/_errors.py +14 -0
- micro_reader/_index.py +53 -0
- micro_reader/_io.py +83 -0
- micro_reader/_jpeg_restarts.py +174 -0
- micro_reader/_memory.py +161 -0
- micro_reader/_metadata.py +163 -0
- micro_reader/_pool.py +85 -0
- micro_reader/_registry.py +104 -0
- micro_reader/_stream.py +317 -0
- micro_reader/backends/__init__.py +1 -0
- micro_reader/backends/_czi_metadata.py +188 -0
- micro_reader/backends/_lif_metadata.py +125 -0
- micro_reader/backends/_nd2_legacy.py +370 -0
- micro_reader/backends/_nd2_metadata.py +224 -0
- micro_reader/backends/_nd2_structure.py +280 -0
- micro_reader/backends/_tiff_metadata.py +287 -0
- micro_reader/backends/czi.py +948 -0
- micro_reader/backends/ims.py +409 -0
- micro_reader/backends/lif.py +713 -0
- micro_reader/backends/mrc.py +184 -0
- micro_reader/backends/nd2.py +264 -0
- micro_reader/backends/oir.py +439 -0
- micro_reader/backends/raster.py +568 -0
- micro_reader/backends/tiff.py +858 -0
- micro_reader-0.0.1.dist-info/METADATA +741 -0
- micro_reader-0.0.1.dist-info/RECORD +32 -0
- micro_reader-0.0.1.dist-info/WHEEL +4 -0
- micro_reader-0.0.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,568 @@
|
|
|
1
|
+
"""Plain raster images: JPEG, PNG and BMP files, one image each.
|
|
2
|
+
|
|
3
|
+
Axes ``yx``, or ``yxs`` for colour (R, G, B (A)). None of these formats can
|
|
4
|
+
be read by rectangle, so the first read decodes the whole picture (the
|
|
5
|
+
read unit is the whole image) and keeps it while it fits the cache.
|
|
6
|
+
|
|
7
|
+
- **JPEG**: baseline, progressive and lossless (8 / 16 bit), grey or colour,
|
|
8
|
+
through ``_codecs``. CMYK / YCCK JPEGs are refused by name. EXIF
|
|
9
|
+
orientation is not applied (as Bio-Formats).
|
|
10
|
+
- **PNG**: every colour type and depth, interlaced or not. Grey depths
|
|
11
|
+
below 8 bits keep their stored values (0-15 for 4 bits, as TIFF packed
|
|
12
|
+
depths); 16-bit samples are native-endian.
|
|
13
|
+
- **BMP**: 1 / 2 / 4 / 8-bit palettes (uncompressed, RLE4, RLE8), 16 / 24 /
|
|
14
|
+
32-bit (uncompressed, bit fields, with alpha when a mask says so), JPEG or
|
|
15
|
+
PNG payloads, top-down or bottom-up, Windows and OS/2 headers. Colour
|
|
16
|
+
fields narrower than 8 bits (5-5-5, 5-6-5) are scaled to 8 bits.
|
|
17
|
+
|
|
18
|
+
Palettes, as Bio-Formats handles them: a palette holding only greys is
|
|
19
|
+
applied (the pixels are the greys); a colour palette is not -- the pixels
|
|
20
|
+
are the indices and ``image.palette`` holds the colours ((n, 3) R G B, or
|
|
21
|
+
(n, 4) with PNG transparency), so label images keep their labels.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import builtins
|
|
26
|
+
import os
|
|
27
|
+
import struct
|
|
28
|
+
import threading
|
|
29
|
+
import zlib
|
|
30
|
+
from dataclasses import dataclass, field
|
|
31
|
+
from typing import Optional
|
|
32
|
+
|
|
33
|
+
import numpy as np
|
|
34
|
+
|
|
35
|
+
from .. import _codecs, _jpeg_restarts, _memory, _stream
|
|
36
|
+
from .._base import File, Image
|
|
37
|
+
from .._errors import CorruptFileError, UnsupportedFormatError
|
|
38
|
+
|
|
39
|
+
NAME = "raster"
|
|
40
|
+
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
|
41
|
+
JPEG_MAGIC = b"\xff\xd8\xff"
|
|
42
|
+
#: BMP header sizes: OS/2 1.x (12) and 2.x (16, 64), Windows 3.x (40) to v5 (124)
|
|
43
|
+
BMP_HEADERS = (12, 16, 40, 52, 56, 64, 108, 124)
|
|
44
|
+
HEAD_BYTES = 1 << 20 # read at open for the header; the whole file if needed
|
|
45
|
+
STREAM_PIECE_BYTES = 8 << 20 # rows of a streamed PNG decoded at a time
|
|
46
|
+
CACHE_BYTES = 256 << 20 # decoded pictures up to this size are kept
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def kind_of(head: bytes) -> Optional[str]:
|
|
50
|
+
""""jpeg", "png", "bmp" by content, or None."""
|
|
51
|
+
if head[:3] == JPEG_MAGIC:
|
|
52
|
+
return "jpeg"
|
|
53
|
+
if head[:8] == PNG_MAGIC:
|
|
54
|
+
return "png"
|
|
55
|
+
if head[:2] == b"BM" and len(head) >= 18 and \
|
|
56
|
+
struct.unpack_from("<I", head, 14)[0] in BMP_HEADERS:
|
|
57
|
+
return "bmp"
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def sniff(head: bytes, path: str) -> bool:
|
|
62
|
+
return kind_of(head) is not None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class Info:
|
|
67
|
+
"""What a picture's header says: enough to know its shape and decode it."""
|
|
68
|
+
kind: str
|
|
69
|
+
height: int
|
|
70
|
+
width: int
|
|
71
|
+
samples: int
|
|
72
|
+
dtype: np.dtype
|
|
73
|
+
bits: int = 8
|
|
74
|
+
#: colour palette, (n, 3 or 4) uint8: the pixels are indices into it
|
|
75
|
+
palette: Optional[np.ndarray] = None
|
|
76
|
+
#: grey palette, applied on decoding: pixel = lut[index]
|
|
77
|
+
lut: Optional[np.ndarray] = None
|
|
78
|
+
extra: dict = field(default_factory=dict)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _set_palette(info: Info, rgb: np.ndarray, alpha: Optional[np.ndarray] = None) -> None:
|
|
82
|
+
"""Grey palettes are applied, colour palettes kept (module docstring)."""
|
|
83
|
+
rgb = np.ascontiguousarray(rgb, np.uint8)
|
|
84
|
+
opaque = alpha is None or bool((alpha == 255).all())
|
|
85
|
+
if opaque and (rgb[:, 0] == rgb[:, 1]).all() and (rgb[:, 1] == rgb[:, 2]).all():
|
|
86
|
+
lut = np.zeros(256, np.uint8) # indices beyond the palette: 0
|
|
87
|
+
lut[:min(256, len(rgb))] = rgb[:256, 0]
|
|
88
|
+
info.lut = lut
|
|
89
|
+
else:
|
|
90
|
+
info.palette = rgb if opaque else np.ascontiguousarray(
|
|
91
|
+
np.concatenate([rgb, alpha[:, None].astype(np.uint8)], axis=1))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def apply_palette(indices: np.ndarray, palette: np.ndarray) -> np.ndarray:
|
|
95
|
+
"""Indices -> their colours (indices beyond the palette: black)."""
|
|
96
|
+
full = np.zeros((256, palette.shape[1]), np.uint8)
|
|
97
|
+
full[:min(256, len(palette))] = palette[:256]
|
|
98
|
+
return full[indices]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def info_of(data: bytes) -> Info:
|
|
102
|
+
"""The header of a JPEG / PNG / BMP file (*data*: its start or all of it)."""
|
|
103
|
+
kind = kind_of(data)
|
|
104
|
+
if kind == "jpeg":
|
|
105
|
+
return _jpeg_info(data)
|
|
106
|
+
if kind == "png":
|
|
107
|
+
return _png_info(data)
|
|
108
|
+
if kind == "bmp":
|
|
109
|
+
return _bmp_info(data)
|
|
110
|
+
raise UnsupportedFormatError("not a JPEG, PNG or BMP file")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def decode(data: bytes, info: Info) -> np.ndarray:
|
|
114
|
+
"""The whole picture, shaped (height, width[, samples])."""
|
|
115
|
+
if "outer_offset" in info.extra: # a JPEG / PNG inside a BMP
|
|
116
|
+
data = data[info.extra["outer_offset"]:]
|
|
117
|
+
out = {"jpeg": _jpeg_decode, "png": _png_decode, "bmp": _bmp_decode}[info.kind](data, info)
|
|
118
|
+
if info.lut is not None:
|
|
119
|
+
out = info.lut[out]
|
|
120
|
+
want = (info.height, info.width) + ((info.samples,) if info.samples > 1 else ())
|
|
121
|
+
if out.shape != want:
|
|
122
|
+
raise CorruptFileError(f"{info.kind.upper()} decodes to {out.shape}, "
|
|
123
|
+
f"its header says {want}")
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# -- JPEG ---------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
def jpeg_header(data: bytes):
|
|
130
|
+
"""(height, width, components, precision) from the first SOF marker."""
|
|
131
|
+
pos = 2
|
|
132
|
+
while pos + 4 <= len(data):
|
|
133
|
+
if data[pos] != 0xFF:
|
|
134
|
+
pos += 1
|
|
135
|
+
continue
|
|
136
|
+
marker = data[pos + 1]
|
|
137
|
+
if marker in (0xD8, 0x01, 0xFF) or 0xD0 <= marker <= 0xD7:
|
|
138
|
+
pos += 1 if marker == 0xFF else 2
|
|
139
|
+
continue
|
|
140
|
+
length = struct.unpack(">H", data[pos + 2:pos + 4])[0]
|
|
141
|
+
if 0xC0 <= marker <= 0xCF and marker not in (0xC4, 0xC8, 0xCC):
|
|
142
|
+
if pos + 10 > len(data):
|
|
143
|
+
break
|
|
144
|
+
bits, height, width, comps = struct.unpack(">BHHB", data[pos + 4:pos + 10])
|
|
145
|
+
return height, width, comps, bits
|
|
146
|
+
pos += 2 + length
|
|
147
|
+
raise CorruptFileError("JPEG without a frame header")
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _jpeg_info(data: bytes) -> Info:
|
|
151
|
+
height, width, comps, bits = jpeg_header(data)
|
|
152
|
+
if comps == 4:
|
|
153
|
+
raise UnsupportedFormatError("JPEG with 4 components (CMYK / YCCK)")
|
|
154
|
+
if comps not in (1, 3):
|
|
155
|
+
raise UnsupportedFormatError(f"JPEG with {comps} components")
|
|
156
|
+
if height == 0 or width == 0:
|
|
157
|
+
raise UnsupportedFormatError("JPEG whose size is set later (DNL marker)")
|
|
158
|
+
return Info("jpeg", height, width, comps, np.dtype("u1") if bits <= 8 else np.dtype("<u2"),
|
|
159
|
+
bits=bits)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _jpeg_decode(data: bytes, info: Info) -> np.ndarray:
|
|
163
|
+
shape = (info.height, info.width) + ((info.samples,) if info.samples > 1 else ())
|
|
164
|
+
out = np.empty(shape, info.dtype)
|
|
165
|
+
_codecs.jpeg_decode_into(data, out)
|
|
166
|
+
return out
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# -- PNG ----------------------------------------------------------------------------
|
|
170
|
+
|
|
171
|
+
def _png_chunks(data: bytes):
|
|
172
|
+
"""(type, body) of each chunk; a chunk running past *data* is an error
|
|
173
|
+
(at open: read more of the file)."""
|
|
174
|
+
pos = 8
|
|
175
|
+
while pos + 8 <= len(data):
|
|
176
|
+
length, ctype = struct.unpack_from(">I4s", data, pos)
|
|
177
|
+
if pos + 12 + length > len(data):
|
|
178
|
+
raise CorruptFileError("PNG ends inside a chunk")
|
|
179
|
+
yield ctype, data[pos + 8:pos + 8 + length]
|
|
180
|
+
if ctype == b"IEND":
|
|
181
|
+
return
|
|
182
|
+
pos += 12 + length
|
|
183
|
+
raise CorruptFileError("PNG ends before its image data")
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _png_info(data: bytes) -> Info:
|
|
187
|
+
if len(data) < 29 or data[12:16] != b"IHDR":
|
|
188
|
+
raise CorruptFileError("PNG without a header chunk")
|
|
189
|
+
width, height, bits, colour, _, _, interlace = struct.unpack(">IIBBBBB", data[16:29])
|
|
190
|
+
samples = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}.get(colour)
|
|
191
|
+
if samples is None or bits not in (1, 2, 4, 8, 16):
|
|
192
|
+
raise CorruptFileError(f"PNG colour type {colour} at {bits} bits")
|
|
193
|
+
plte = trns = None
|
|
194
|
+
for ctype, body in _png_chunks(data):
|
|
195
|
+
if ctype == b"PLTE":
|
|
196
|
+
plte = body
|
|
197
|
+
elif ctype == b"tRNS":
|
|
198
|
+
trns = body
|
|
199
|
+
elif ctype == b"IDAT":
|
|
200
|
+
break
|
|
201
|
+
info = Info("png", height, width, samples,
|
|
202
|
+
np.dtype("<u2") if bits == 16 else np.dtype("u1"), bits=bits,
|
|
203
|
+
extra={"colour": colour, "interlace": interlace})
|
|
204
|
+
if colour == 3:
|
|
205
|
+
if plte is None:
|
|
206
|
+
raise CorruptFileError("palette PNG without a palette (PLTE)")
|
|
207
|
+
rgb = np.frombuffer(plte, np.uint8, len(plte) // 3 * 3).reshape(-1, 3)
|
|
208
|
+
alpha = None
|
|
209
|
+
if trns:
|
|
210
|
+
alpha = np.full(len(rgb), 255, np.uint8)
|
|
211
|
+
n = min(len(trns), len(rgb))
|
|
212
|
+
alpha[:n] = np.frombuffer(trns, np.uint8, n)
|
|
213
|
+
_set_palette(info, rgb, alpha)
|
|
214
|
+
return info
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _png_decode(data: bytes, info: Info) -> np.ndarray:
|
|
218
|
+
h, w, s, bits = info.height, info.width, info.samples, info.bits
|
|
219
|
+
shape = (h, w, s) if s > 1 else (h, w)
|
|
220
|
+
if bits >= 8 and info.extra["colour"] != 3:
|
|
221
|
+
out = np.empty(shape, info.dtype)
|
|
222
|
+
_codecs.png_decode_into(data, out)
|
|
223
|
+
return out
|
|
224
|
+
# palette indices and depths below 8 bits: packed rows, unpacked here
|
|
225
|
+
row = (w * s * bits + 7) // 8
|
|
226
|
+
if _codecs._use_native(None, "png_decode_into"):
|
|
227
|
+
packed = np.empty(h * row, np.uint8)
|
|
228
|
+
_codecs.png_decode_into(data, packed)
|
|
229
|
+
else:
|
|
230
|
+
packed = _png_rows(data, info, row)
|
|
231
|
+
out = np.empty((h, w * s), np.uint8)
|
|
232
|
+
if bits == 8:
|
|
233
|
+
out[...] = packed.reshape(h, row)
|
|
234
|
+
else:
|
|
235
|
+
_codecs.unpack_bits_into(packed, out, bits, w * s)
|
|
236
|
+
return out.reshape(shape)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _png_rows(data: bytes, info: Info, row: int) -> np.ndarray:
|
|
240
|
+
"""Reference for packed rows: inflate and unfilter in numpy (samples
|
|
241
|
+
below 8 bits or palette indices: one byte per filter unit)."""
|
|
242
|
+
if info.extra["interlace"]:
|
|
243
|
+
raise UnsupportedFormatError(
|
|
244
|
+
"interlaced palette / low-bit PNG needs the native decoders (micro-reader-codecs)")
|
|
245
|
+
idat = b"".join(body for ctype, body in _png_chunks(data) if ctype == b"IDAT")
|
|
246
|
+
try:
|
|
247
|
+
raw = np.frombuffer(zlib.decompress(idat), np.uint8)
|
|
248
|
+
except zlib.error as exc:
|
|
249
|
+
raise CorruptFileError(f"corrupt PNG data: {exc}") from exc
|
|
250
|
+
h = info.height
|
|
251
|
+
if raw.size < h * (row + 1):
|
|
252
|
+
raise CorruptFileError("PNG data ends early")
|
|
253
|
+
raw = raw[:h * (row + 1)].reshape(h, row + 1)
|
|
254
|
+
out = np.empty((h, row), np.uint8)
|
|
255
|
+
prev = np.zeros(row, np.uint8)
|
|
256
|
+
for y in range(h):
|
|
257
|
+
kind, line = raw[y, 0], raw[y, 1:]
|
|
258
|
+
if kind == 0:
|
|
259
|
+
cur = line.copy()
|
|
260
|
+
elif kind == 1: # sub
|
|
261
|
+
cur = np.cumsum(line, dtype=np.uint8)
|
|
262
|
+
elif kind == 2: # up
|
|
263
|
+
cur = line + prev
|
|
264
|
+
elif kind in (3, 4): # average, Paeth
|
|
265
|
+
cur = np.empty(row, np.uint8)
|
|
266
|
+
left = 0
|
|
267
|
+
for i in range(row):
|
|
268
|
+
up = int(prev[i])
|
|
269
|
+
if kind == 3:
|
|
270
|
+
pred = (left + up) // 2
|
|
271
|
+
else:
|
|
272
|
+
corner = int(prev[i - 1]) if i else 0
|
|
273
|
+
p = left + up - corner
|
|
274
|
+
pa, pb, pc = abs(p - left), abs(p - up), abs(p - corner)
|
|
275
|
+
pred = left if pa <= pb and pa <= pc else up if pb <= pc else corner
|
|
276
|
+
left = cur[i] = (int(line[i]) + pred) & 0xFF
|
|
277
|
+
else:
|
|
278
|
+
raise CorruptFileError(f"PNG filter type {kind}")
|
|
279
|
+
out[y] = prev = cur
|
|
280
|
+
return out.reshape(-1)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# -- BMP ----------------------------------------------------------------------------
|
|
284
|
+
|
|
285
|
+
def _bmp_info(data: bytes) -> Info:
|
|
286
|
+
offset, dib = struct.unpack_from("<II", data, 10)
|
|
287
|
+
if dib == 12: # OS/2 1.x
|
|
288
|
+
width, height, _, bpp = struct.unpack_from("<HHHH", data, 18)
|
|
289
|
+
compression, colours, entry = 0, 0, 3
|
|
290
|
+
else:
|
|
291
|
+
width, height, _, bpp = struct.unpack_from("<iiHH", data, 18)
|
|
292
|
+
compression = struct.unpack_from("<I", data, 30)[0] if dib >= 20 else 0
|
|
293
|
+
colours = struct.unpack_from("<I", data, 46)[0] if dib >= 36 else 0
|
|
294
|
+
entry = 4
|
|
295
|
+
if width <= 0 or height == 0:
|
|
296
|
+
raise CorruptFileError(f"BMP of {width} x {height} pixels")
|
|
297
|
+
info = Info("bmp", abs(height), width, 3, np.dtype("u1"), bits=bpp,
|
|
298
|
+
extra={"offset": offset, "compression": compression, "top_down": height < 0})
|
|
299
|
+
if compression in (4, 5): # a JPEG / PNG inside
|
|
300
|
+
inner = info_of(data[offset:])
|
|
301
|
+
inner.extra["outer_offset"] = offset
|
|
302
|
+
return inner
|
|
303
|
+
if bpp in (1, 2, 4, 8):
|
|
304
|
+
if not (compression == 0 or (compression, bpp) in ((1, 8), (2, 4))):
|
|
305
|
+
raise UnsupportedFormatError(
|
|
306
|
+
f"BMP with {bpp} bits per pixel, compression {compression}")
|
|
307
|
+
n = colours or (1 << bpp)
|
|
308
|
+
at = 14 + dib
|
|
309
|
+
if len(data) < at + n * entry:
|
|
310
|
+
raise CorruptFileError("BMP ends inside its palette")
|
|
311
|
+
table = np.frombuffer(data, np.uint8, n * entry, at).reshape(n, entry)
|
|
312
|
+
info.samples = 1
|
|
313
|
+
_set_palette(info, table[:, 2::-1]) # stored B, G, R (, 0)
|
|
314
|
+
return info
|
|
315
|
+
if bpp == 24 and compression == 0:
|
|
316
|
+
return info
|
|
317
|
+
if bpp in (16, 32) and compression in (0, 3, 6):
|
|
318
|
+
if compression == 0:
|
|
319
|
+
masks = [0x7C00, 0x3E0, 0x1F] if bpp == 16 else [0xFF0000, 0xFF00, 0xFF]
|
|
320
|
+
else:
|
|
321
|
+
masks = list(struct.unpack_from("<III", data, 54))
|
|
322
|
+
if (dib >= 56 or compression == 6) and len(data) >= 70:
|
|
323
|
+
alpha = struct.unpack_from("<I", data, 66)[0]
|
|
324
|
+
if alpha:
|
|
325
|
+
masks.append(alpha)
|
|
326
|
+
for m in masks:
|
|
327
|
+
if not m or bin(m).count("1") > 8:
|
|
328
|
+
raise UnsupportedFormatError(f"BMP bit field mask {m:#x}")
|
|
329
|
+
info.samples = len(masks)
|
|
330
|
+
info.extra["masks"] = masks
|
|
331
|
+
return info
|
|
332
|
+
raise UnsupportedFormatError(f"BMP with {bpp} bits per pixel, compression {compression}")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _bmp_decode(data: bytes, info: Info) -> np.ndarray:
|
|
336
|
+
e, h, w, bpp = info.extra, info.height, info.width, info.bits
|
|
337
|
+
off, compression = e["offset"], e["compression"]
|
|
338
|
+
if compression in (1, 2):
|
|
339
|
+
pixels = _bmp_rle(memoryview(data)[off:], w, h, bpp)
|
|
340
|
+
else:
|
|
341
|
+
stride = (w * bpp + 31) // 32 * 4
|
|
342
|
+
if len(data) < off + stride * h:
|
|
343
|
+
raise CorruptFileError("BMP pixel data ends early")
|
|
344
|
+
raw = np.frombuffer(data, np.uint8, stride * h, off).reshape(h, stride)
|
|
345
|
+
if bpp <= 8:
|
|
346
|
+
pixels = np.empty((h, w), np.uint8)
|
|
347
|
+
if bpp == 8:
|
|
348
|
+
pixels[...] = raw[:, :w]
|
|
349
|
+
else:
|
|
350
|
+
_codecs.unpack_bits_into(np.ascontiguousarray(raw[:, :(w * bpp + 7) // 8]),
|
|
351
|
+
pixels, bpp, w)
|
|
352
|
+
elif bpp == 24:
|
|
353
|
+
pixels = raw[:, :w * 3].reshape(h, w, 3)[..., ::-1]
|
|
354
|
+
else:
|
|
355
|
+
values = np.ascontiguousarray(raw[:, :w * bpp // 8]).view(
|
|
356
|
+
"<u2" if bpp == 16 else "<u4").astype(np.uint32)
|
|
357
|
+
chans = []
|
|
358
|
+
for m in e["masks"]:
|
|
359
|
+
shift = (m & -m).bit_length() - 1
|
|
360
|
+
top = (1 << bin(m).count("1")) - 1
|
|
361
|
+
c = (values >> shift) & top
|
|
362
|
+
if top < 255: # scale to 8 bits
|
|
363
|
+
c = (c * 255 + top // 2) // top
|
|
364
|
+
chans.append(c.astype(np.uint8))
|
|
365
|
+
pixels = np.stack(chans, axis=-1)
|
|
366
|
+
if not e["top_down"]: # stored bottom-up
|
|
367
|
+
pixels = pixels[::-1]
|
|
368
|
+
return np.ascontiguousarray(pixels)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _bmp_rle(data, w: int, h: int, bpp: int) -> np.ndarray:
|
|
372
|
+
"""RLE8 / RLE4 -> (rows, cols) indices in stored (bottom-up) row order;
|
|
373
|
+
pixels the stream skips stay 0."""
|
|
374
|
+
out = np.zeros((h, w), np.uint8)
|
|
375
|
+
x = y = i = 0
|
|
376
|
+
n = len(data)
|
|
377
|
+
|
|
378
|
+
def put(values):
|
|
379
|
+
nonlocal x
|
|
380
|
+
if y < h and x < w:
|
|
381
|
+
k = min(len(values), w - x)
|
|
382
|
+
out[y, x:x + k] = values[:k]
|
|
383
|
+
x += len(values)
|
|
384
|
+
|
|
385
|
+
def nibbles(byte_values, count):
|
|
386
|
+
b = np.frombuffer(bytes(byte_values), np.uint8)
|
|
387
|
+
return np.stack([b >> 4, b & 15], axis=1).reshape(-1)[:count]
|
|
388
|
+
|
|
389
|
+
while i + 1 < n:
|
|
390
|
+
count, value = data[i], data[i + 1]
|
|
391
|
+
i += 2
|
|
392
|
+
if count: # a run
|
|
393
|
+
if bpp == 8:
|
|
394
|
+
put(np.full(count, value, np.uint8))
|
|
395
|
+
else:
|
|
396
|
+
put(nibbles([value] * ((count + 1) // 2), count))
|
|
397
|
+
elif value == 0: # end of line
|
|
398
|
+
x, y = 0, y + 1
|
|
399
|
+
elif value == 1: # end of bitmap
|
|
400
|
+
break
|
|
401
|
+
elif value == 2: # delta
|
|
402
|
+
if i + 1 >= n:
|
|
403
|
+
break
|
|
404
|
+
x, y = x + data[i], y + data[i + 1]
|
|
405
|
+
i += 2
|
|
406
|
+
else: # absolute run
|
|
407
|
+
size = value if bpp == 8 else (value + 1) // 2
|
|
408
|
+
if i + size > n:
|
|
409
|
+
raise CorruptFileError("BMP RLE data ends early")
|
|
410
|
+
chunk = data[i:i + size]
|
|
411
|
+
put(np.frombuffer(bytes(chunk), np.uint8) if bpp == 8 else nibbles(chunk, value))
|
|
412
|
+
i += size + (size & 1) # runs end on a 16-bit word
|
|
413
|
+
return out
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
# -- the backend --------------------------------------------------------------------
|
|
417
|
+
|
|
418
|
+
class _RasterImage(Image):
|
|
419
|
+
"""The picture of a JPEG / PNG / BMP file."""
|
|
420
|
+
|
|
421
|
+
def __init__(self, path: str, info: Info):
|
|
422
|
+
shape = (info.height, info.width) + ((info.samples,) if info.samples > 1 else ())
|
|
423
|
+
self._path, self._info = path, info
|
|
424
|
+
#: the colours of a colour palette ((n, 3) R G B, or (n, 4) with
|
|
425
|
+
#: alpha): the pixels are indices into it. None otherwise.
|
|
426
|
+
self.palette = info.palette
|
|
427
|
+
self._lock = threading.Lock()
|
|
428
|
+
self._cached = None
|
|
429
|
+
self._restarts = False # the JPEG's RestartIndex once built (None: none)
|
|
430
|
+
self._closed = False
|
|
431
|
+
super().__init__(0, "yx" + ("s" if info.samples > 1 else ""), shape, info.dtype,
|
|
432
|
+
name=os.path.basename(path), read_unit=shape)
|
|
433
|
+
|
|
434
|
+
def _pixels(self):
|
|
435
|
+
"""The decoded picture, and the samples decoded to get it now."""
|
|
436
|
+
with self._lock:
|
|
437
|
+
if self._cached is not None:
|
|
438
|
+
return self._cached, 0
|
|
439
|
+
with builtins.open(self._path, "rb") as fh:
|
|
440
|
+
data = fh.read()
|
|
441
|
+
try:
|
|
442
|
+
info = self._info
|
|
443
|
+
nbytes = int(np.prod(self.shape)) * self.dtype.itemsize
|
|
444
|
+
with _memory.reserve(nbytes, f"{self._path!r}: a {info.kind.upper()} picture "
|
|
445
|
+
f"of {self.shape[1]} x {self.shape[0]} pixels"):
|
|
446
|
+
pixels = decode(data, info)
|
|
447
|
+
except (CorruptFileError, UnsupportedFormatError) as exc:
|
|
448
|
+
raise type(exc)(f"{self._path!r}: {exc}") from exc
|
|
449
|
+
if pixels.nbytes <= CACHE_BYTES:
|
|
450
|
+
self._cached = pixels
|
|
451
|
+
return pixels, pixels.size
|
|
452
|
+
|
|
453
|
+
def _read_block(self, block):
|
|
454
|
+
if self._closed:
|
|
455
|
+
raise ValueError("this image has been closed: its file was closed "
|
|
456
|
+
"(File.close() or the end of a with block)")
|
|
457
|
+
if self._cached is None and self._streams():
|
|
458
|
+
return self._streamed(block)
|
|
459
|
+
pixels, read = self._pixels()
|
|
460
|
+
return np.array(pixels[block], copy=True), read
|
|
461
|
+
|
|
462
|
+
def _streams(self) -> bool:
|
|
463
|
+
"""A picture too large to keep decoded (``CACHE_BYTES``), read by
|
|
464
|
+
rows: a PNG not interlaced (``_stream``; streamed rows cannot be
|
|
465
|
+
reused, and PNG has no checkpoints), or a JPEG with restart markers
|
|
466
|
+
(``_jpeg_restarts``)."""
|
|
467
|
+
info = self._info
|
|
468
|
+
nbytes = int(np.prod(self.shape)) * self.dtype.itemsize
|
|
469
|
+
if nbytes <= CACHE_BYTES:
|
|
470
|
+
return False
|
|
471
|
+
if info.kind == "jpeg":
|
|
472
|
+
row_bytes = info.width * info.samples * info.dtype.itemsize
|
|
473
|
+
return _jpeg_restarts.usable(self._restart_index(), row_bytes, _memory.fits)
|
|
474
|
+
return info.kind == "png" and not info.extra.get("interlace") and _stream.streamable("png")
|
|
475
|
+
|
|
476
|
+
def _restart_index(self):
|
|
477
|
+
"""The JPEG's ``RestartIndex`` (None: it cannot be cut), built once."""
|
|
478
|
+
with self._lock:
|
|
479
|
+
if self._restarts is False:
|
|
480
|
+
with builtins.open(self._path, "rb") as fh:
|
|
481
|
+
fh.seek(self._info.extra.get("outer_offset", 0))
|
|
482
|
+
self._restarts = _jpeg_restarts.RestartIndex.build(fh.read())
|
|
483
|
+
return self._restarts
|
|
484
|
+
|
|
485
|
+
def _jpeg_rows(self, block):
|
|
486
|
+
"""Rows ``block[0]`` of a JPEG cut at its restart markers, in bands."""
|
|
487
|
+
info, ys = self._info, block[0]
|
|
488
|
+
offset = info.extra.get("outer_offset", 0)
|
|
489
|
+
out = np.empty(tuple(b.stop - b.start for b in block), self.dtype)
|
|
490
|
+
|
|
491
|
+
def read(start, stop):
|
|
492
|
+
with builtins.open(self._path, "rb") as fh:
|
|
493
|
+
fh.seek(offset + start)
|
|
494
|
+
return fh.read(stop - start)
|
|
495
|
+
|
|
496
|
+
tail = (info.samples,) if info.samples > 1 else ()
|
|
497
|
+
for a, b, rows in _jpeg_restarts.read_rows(
|
|
498
|
+
self._restart_index(), read, ys.start, ys.stop, _codecs.jpeg_decode_into, tail,
|
|
499
|
+
info.dtype, min(STREAM_PIECE_BYTES, _memory.memory_limit() // 4),
|
|
500
|
+
lambda n: _memory.reserve(n, f"{self._path!r}: rows of a JPEG")):
|
|
501
|
+
out[a - ys.start:b - ys.start] = rows[(slice(None),) + tuple(block[1:])]
|
|
502
|
+
return out, (ys.stop - ys.start) * info.width * info.samples
|
|
503
|
+
|
|
504
|
+
def _streamed(self, block):
|
|
505
|
+
"""Rows ``block[0]`` of a PNG, decoded as a stream in pieces of rows,
|
|
506
|
+
unpacked like a whole decode (``_png_decode``, ``decode``)."""
|
|
507
|
+
info = self._info
|
|
508
|
+
if info.kind == "jpeg":
|
|
509
|
+
return self._jpeg_rows(block)
|
|
510
|
+
ys = block[0]
|
|
511
|
+
w, s, bits = info.width, info.samples, info.bits
|
|
512
|
+
packed = bits < 8 or info.extra["colour"] == 3
|
|
513
|
+
row = (w * s * bits + 7) // 8 if packed else w * s * info.dtype.itemsize
|
|
514
|
+
offset = info.extra.get("outer_offset", 0)
|
|
515
|
+
out = np.empty(tuple(b.stop - b.start for b in block), self.dtype)
|
|
516
|
+
per_piece = max(1, min(STREAM_PIECE_BYTES, _memory.memory_limit() // 4) // row)
|
|
517
|
+
for a in range(ys.start, ys.stop, per_piece):
|
|
518
|
+
b = min(ys.stop, a + per_piece)
|
|
519
|
+
with _memory.reserve((b - a) * row * 2, f"{self._path!r}: rows of a PNG"):
|
|
520
|
+
buf = np.empty((b - a) * row, np.uint8)
|
|
521
|
+
n = _stream.read(self._path, offset, 0, "png", a * row, buf,
|
|
522
|
+
size=info.height * row)
|
|
523
|
+
if n < buf.nbytes:
|
|
524
|
+
raise CorruptFileError(f"{self._path!r}: PNG data ends early")
|
|
525
|
+
if not packed:
|
|
526
|
+
rows = buf.view(info.dtype).reshape(b - a, w * s)
|
|
527
|
+
elif bits == 8:
|
|
528
|
+
rows = buf.reshape(b - a, w * s)
|
|
529
|
+
else:
|
|
530
|
+
rows = np.empty((b - a, w * s), np.uint8)
|
|
531
|
+
_codecs.unpack_bits_into(buf, rows, bits, w * s)
|
|
532
|
+
if info.lut is not None:
|
|
533
|
+
rows = info.lut[rows]
|
|
534
|
+
rows = rows.reshape((b - a, w, s) if s > 1 else (b - a, w))
|
|
535
|
+
out[a - ys.start:b - ys.start] = rows[(slice(None),) + tuple(block[1:])]
|
|
536
|
+
return out, (ys.stop - ys.start) * w * s
|
|
537
|
+
|
|
538
|
+
def close(self) -> None:
|
|
539
|
+
self._closed = True
|
|
540
|
+
self._cached = None
|
|
541
|
+
_stream.forget(self._path)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
class _RasterFile(File):
|
|
545
|
+
backend = NAME
|
|
546
|
+
|
|
547
|
+
def __init__(self, path: str, info: Info):
|
|
548
|
+
#: "jpeg", "png" or "bmp" (also for a BMP holding a JPEG / PNG)
|
|
549
|
+
self.format = "bmp" if "outer_offset" in info.extra else info.kind
|
|
550
|
+
super().__init__(path, [_RasterImage(path, info)])
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def open(path: str, tiles: str = "separate", resolutions: str = "base") -> _RasterFile: # noqa: A001 - backend protocol
|
|
554
|
+
with builtins.open(path, "rb") as fh:
|
|
555
|
+
head = fh.read(HEAD_BYTES)
|
|
556
|
+
try:
|
|
557
|
+
try:
|
|
558
|
+
info = info_of(head)
|
|
559
|
+
except (CorruptFileError, struct.error, IndexError, ValueError):
|
|
560
|
+
if len(head) < HEAD_BYTES: # that was the whole file
|
|
561
|
+
raise
|
|
562
|
+
fh.seek(0)
|
|
563
|
+
info = info_of(fh.read()) # a header beyond the first MB
|
|
564
|
+
except (CorruptFileError, UnsupportedFormatError) as exc:
|
|
565
|
+
raise type(exc)(f"{path!r}: {exc}") from exc
|
|
566
|
+
except (struct.error, IndexError, ValueError) as exc:
|
|
567
|
+
raise CorruptFileError(f"{path!r}: truncated or damaged header ({exc})") from exc
|
|
568
|
+
return _RasterFile(path, info)
|