fmg 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fmg/__init__.py +27 -0
- fmg/__version__.py +3 -0
- fmg/core.py +757 -0
- fmg-0.4.0.dist-info/METADATA +340 -0
- fmg-0.4.0.dist-info/RECORD +8 -0
- fmg-0.4.0.dist-info/WHEEL +5 -0
- fmg-0.4.0.dist-info/licenses/LICENSE +21 -0
- fmg-0.4.0.dist-info/top_level.txt +1 -0
fmg/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""fmg -- Fast Memory Graphics for big images, used like OpenSlide."""
|
|
3
|
+
from .__version__ import __version__
|
|
4
|
+
from .core import (
|
|
5
|
+
AbstractSlide,
|
|
6
|
+
CorruptTileError,
|
|
7
|
+
FastMemoryGraphics,
|
|
8
|
+
FmgError,
|
|
9
|
+
FORMAT_VERSION,
|
|
10
|
+
configure,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
'open_slide', 'create_slide', 'FastMemoryGraphics', 'AbstractSlide',
|
|
15
|
+
'FmgError', 'CorruptTileError', 'configure', 'FORMAT_VERSION',
|
|
16
|
+
'__version__',
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def open_slide(filename, **kwargs):
|
|
21
|
+
"""Open an existing .fmg slide for reading (lazy, fork-safe)."""
|
|
22
|
+
return FastMemoryGraphics(filename, mode='r', **kwargs)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def create_slide(filename, **kwargs):
|
|
26
|
+
"""Create a writer for a new .fmg slide."""
|
|
27
|
+
return FastMemoryGraphics(filename, mode='w', **kwargs)
|
fmg/__version__.py
ADDED
fmg/core.py
ADDED
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Fast Memory Graphics: a tiled, LMDB-backed whole-slide image.
|
|
3
|
+
|
|
4
|
+
Storage layout (unchanged, so existing .fmg files stay readable)
|
|
5
|
+
---------------------------------------------------------------
|
|
6
|
+
<basename>.fmg JSON metadata
|
|
7
|
+
<basename>/level0/ LMDB env, keys = str((x, y, x_max, y_max))
|
|
8
|
+
<basename>/level1/
|
|
9
|
+
...
|
|
10
|
+
|
|
11
|
+
A tile is encoded with ``cv2.imencode`` and decoded with ``cv2.imdecode``.
|
|
12
|
+
The two are inverse operations, so the array that went in comes back out
|
|
13
|
+
with the same channel order (the library treats it as RGB at the API edge).
|
|
14
|
+
|
|
15
|
+
Concurrency contract
|
|
16
|
+
--------------------
|
|
17
|
+
* An ``Environment`` and its ``Transaction`` objects are never created in
|
|
18
|
+
``__init__``; they are opened lazily, once per process.
|
|
19
|
+
* If the object is inherited by a forked child (e.g. a PyTorch DataLoader
|
|
20
|
+
worker), the inherited handles are dropped without being closed and a
|
|
21
|
+
fresh environment is opened in the child. Reusing a forked LMDB env is
|
|
22
|
+
undefined behaviour and is the usual cause of sporadic SIGBUS/SIGSEGV.
|
|
23
|
+
* Every ``read_region`` uses its own short-lived read transaction, so the
|
|
24
|
+
same slide object is safe to use from multiple threads of one process.
|
|
25
|
+
* Within one process, all slides pointing at the same directory share one
|
|
26
|
+
refcounted read-only Environment: py-lmdb refuses to open the same path
|
|
27
|
+
twice, and unpickling a slide in its parent process would otherwise fail.
|
|
28
|
+
"""
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import json
|
|
32
|
+
import math
|
|
33
|
+
import os
|
|
34
|
+
import shutil
|
|
35
|
+
import threading
|
|
36
|
+
import warnings
|
|
37
|
+
from collections import OrderedDict
|
|
38
|
+
|
|
39
|
+
import cv2
|
|
40
|
+
import lmdb
|
|
41
|
+
import numpy as np
|
|
42
|
+
from PIL import Image
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
'FmgError', 'CorruptTileError', 'AbstractSlide',
|
|
46
|
+
'FastMemoryGraphics', 'configure', 'FORMAT_VERSION',
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
FORMAT_VERSION = 2
|
|
50
|
+
DEFAULT_PATCH_SIZE = 256
|
|
51
|
+
# Readers pass map_size=0, so LMDB reads the real value from the file's meta
|
|
52
|
+
# page (works for files written by 0.3.0 with a 1 TiB map too). Writers get
|
|
53
|
+
# a size derived from the actual image instead of blindly mapping 1 TiB per
|
|
54
|
+
# level per process.
|
|
55
|
+
MIN_WRITE_MAP_SIZE = 1 << 28 # 256 MiB
|
|
56
|
+
MAX_WRITE_MAP_SIZE = 1 << 40 # 1 TiB, same as the original writer
|
|
57
|
+
DEFAULT_COMMIT_EVERY = 2048
|
|
58
|
+
SUPPORTED_EXT = {'.jpg', '.jpeg', '.png', '.webp', '.bmp', '.tiff', '.tif'}
|
|
59
|
+
|
|
60
|
+
_cv2_pid = None
|
|
61
|
+
_cv2_lock = threading.Lock()
|
|
62
|
+
|
|
63
|
+
# ---------------------------------------------------------------------------
|
|
64
|
+
# Process-local registry of read-only LMDB environments.
|
|
65
|
+
#
|
|
66
|
+
# py-lmdb refuses to open the same path twice in one process ("The environment
|
|
67
|
+
# ... is already open in this process"). That happens as soon as two slide
|
|
68
|
+
# objects point at the same .fmg: a Dataset plus a pickled/unpickled clone,
|
|
69
|
+
# two Dataset instances over one slide, or a validation loader alongside the
|
|
70
|
+
# training loader. Sharing ONE read-only Environment (with a fresh short
|
|
71
|
+
# transaction per read) is safe and avoids mapping the same file N times.
|
|
72
|
+
# ---------------------------------------------------------------------------
|
|
73
|
+
_env_registry = {} # (realpath, map_size) -> {'env': ..., 'refs': n}
|
|
74
|
+
_env_registry_pid = os.getpid()
|
|
75
|
+
_env_registry_lock = threading.RLock()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _after_fork_in_child():
|
|
79
|
+
"""Drop inherited handles in the child; never close them here.
|
|
80
|
+
|
|
81
|
+
``mdb_env_close()`` in a forked child may clear the parent's reader slot
|
|
82
|
+
/ shared lock file, so the inherited Environment objects are simply
|
|
83
|
+
forgotten. The child opens its own on first use. Rebuilding the locks
|
|
84
|
+
also avoids a deadlock when another thread held one at fork time.
|
|
85
|
+
"""
|
|
86
|
+
global _env_registry, _env_registry_pid, _env_registry_lock
|
|
87
|
+
global _cv2_pid, _cv2_lock
|
|
88
|
+
_env_registry = {}
|
|
89
|
+
_env_registry_pid = os.getpid()
|
|
90
|
+
_env_registry_lock = threading.RLock()
|
|
91
|
+
_cv2_pid = None
|
|
92
|
+
_cv2_lock = threading.Lock()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
if hasattr(os, 'register_at_fork'):
|
|
96
|
+
os.register_at_fork(after_in_child=_after_fork_in_child)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _acquire_read_env(path, map_size):
|
|
100
|
+
"""Return a process-local read-only Environment shared by all slides."""
|
|
101
|
+
global _env_registry_pid
|
|
102
|
+
key = (os.path.realpath(path), int(map_size or 0))
|
|
103
|
+
with _env_registry_lock:
|
|
104
|
+
if _env_registry_pid != os.getpid(): # platforms without register_at_fork
|
|
105
|
+
_env_registry.clear()
|
|
106
|
+
_env_registry_pid = os.getpid()
|
|
107
|
+
entry = _env_registry.get(key)
|
|
108
|
+
if entry is not None:
|
|
109
|
+
entry['refs'] += 1
|
|
110
|
+
return entry['env']
|
|
111
|
+
env = lmdb.open(
|
|
112
|
+
path,
|
|
113
|
+
subdir=True,
|
|
114
|
+
readonly=True,
|
|
115
|
+
create=False,
|
|
116
|
+
lock=False, # MDB_NOLOCK: no reader table, fork-safe
|
|
117
|
+
readahead=False, # WSI reads are random, prefetch hurts
|
|
118
|
+
meminit=False,
|
|
119
|
+
map_size=map_size or 0, # 0 = use the size stored in meta
|
|
120
|
+
max_spare_txns=8, # makes begin() almost free
|
|
121
|
+
)
|
|
122
|
+
_env_registry[key] = {'env': env, 'refs': 1}
|
|
123
|
+
return env
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _release_read_env(env):
|
|
127
|
+
"""Release one reference; close only when the last user goes away."""
|
|
128
|
+
if env is None:
|
|
129
|
+
return
|
|
130
|
+
with _env_registry_lock:
|
|
131
|
+
for key, entry in list(_env_registry.items()):
|
|
132
|
+
if entry['env'] is env:
|
|
133
|
+
entry['refs'] -= 1
|
|
134
|
+
if entry['refs'] <= 0:
|
|
135
|
+
del _env_registry[key]
|
|
136
|
+
try:
|
|
137
|
+
env.close()
|
|
138
|
+
except Exception:
|
|
139
|
+
pass
|
|
140
|
+
return
|
|
141
|
+
# Not registered (e.g. inherited across fork): leaving the reference
|
|
142
|
+
# alone is the safe choice, see _after_fork_in_child().
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _tune_cv2_for_worker():
|
|
146
|
+
"""Make OpenCV fork-safe, once per process.
|
|
147
|
+
|
|
148
|
+
OpenCV's internal thread pool / OpenCL context do not survive ``fork()``.
|
|
149
|
+
A child that inherits an initialised pool can deadlock or crash inside
|
|
150
|
+
``imdecode``. ``setNumThreads(0)`` disables the pool; it costs nothing
|
|
151
|
+
for single-image JPEG decoding.
|
|
152
|
+
"""
|
|
153
|
+
global _cv2_pid
|
|
154
|
+
pid = os.getpid()
|
|
155
|
+
with _cv2_lock:
|
|
156
|
+
if _cv2_pid == pid:
|
|
157
|
+
return
|
|
158
|
+
try:
|
|
159
|
+
cv2.setNumThreads(0)
|
|
160
|
+
except Exception: # pragma: no cover - exotic OpenCV builds
|
|
161
|
+
pass
|
|
162
|
+
try:
|
|
163
|
+
cv2.ocl.setUseOpenCL(False)
|
|
164
|
+
except Exception: # pragma: no cover
|
|
165
|
+
pass
|
|
166
|
+
_cv2_pid = pid
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def configure(num_threads=0, use_opencl=False):
|
|
170
|
+
"""Explicitly configure OpenCV for a DataLoader worker.
|
|
171
|
+
|
|
172
|
+
Normally the library calls this automatically when it detects that it is
|
|
173
|
+
running inside a forked child. Call it yourself in ``worker_init_fn``
|
|
174
|
+
when using the ``spawn`` start method, or when you want to control the
|
|
175
|
+
setting from outside.
|
|
176
|
+
"""
|
|
177
|
+
global _cv2_pid
|
|
178
|
+
with _cv2_lock:
|
|
179
|
+
try:
|
|
180
|
+
cv2.setNumThreads(int(num_threads))
|
|
181
|
+
except Exception:
|
|
182
|
+
pass
|
|
183
|
+
try:
|
|
184
|
+
cv2.ocl.setUseOpenCL(bool(use_opencl))
|
|
185
|
+
except Exception:
|
|
186
|
+
pass
|
|
187
|
+
_cv2_pid = os.getpid()
|
|
188
|
+
return os.getpid()
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class FmgError(Exception):
|
|
192
|
+
"""Base exception for this package."""
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class CorruptTileError(FmgError):
|
|
196
|
+
"""A tile is missing, empty or cannot be decoded."""
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _tile_key(box):
|
|
200
|
+
"""Key format is part of the on-disk format; keep tuple repr."""
|
|
201
|
+
return str(tuple(int(v) for v in box)).encode('ascii')
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _default_params(ext):
|
|
205
|
+
if ext in ('.png',):
|
|
206
|
+
return [int(cv2.IMWRITE_PNG_COMPRESSION), 3]
|
|
207
|
+
return [int(cv2.IMWRITE_JPEG_QUALITY), 90]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class AbstractSlide:
|
|
211
|
+
"""Minimal OpenSlide-like interface."""
|
|
212
|
+
|
|
213
|
+
def __enter__(self):
|
|
214
|
+
return self
|
|
215
|
+
|
|
216
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
217
|
+
self.close()
|
|
218
|
+
return False
|
|
219
|
+
|
|
220
|
+
def close(self):
|
|
221
|
+
raise NotImplementedError
|
|
222
|
+
|
|
223
|
+
@property
|
|
224
|
+
def level_count(self):
|
|
225
|
+
raise NotImplementedError
|
|
226
|
+
|
|
227
|
+
@property
|
|
228
|
+
def level_dimensions(self):
|
|
229
|
+
"""[(width, height), ...] in level coordinates."""
|
|
230
|
+
raise NotImplementedError
|
|
231
|
+
|
|
232
|
+
@property
|
|
233
|
+
def dimensions(self):
|
|
234
|
+
"""(width, height) of level 0."""
|
|
235
|
+
return self.level_dimensions[0]
|
|
236
|
+
|
|
237
|
+
@property
|
|
238
|
+
def level_downsamples(self):
|
|
239
|
+
"""[float, ...], one downsample factor per level."""
|
|
240
|
+
raise NotImplementedError
|
|
241
|
+
|
|
242
|
+
@property
|
|
243
|
+
def properties(self):
|
|
244
|
+
raise NotImplementedError
|
|
245
|
+
|
|
246
|
+
def read_region(self, location, level, size):
|
|
247
|
+
"""Return a PIL.Image.
|
|
248
|
+
|
|
249
|
+
location: (x, y) in the **level 0** reference frame (OpenSlide
|
|
250
|
+
semantics).
|
|
251
|
+
level: pyramid level.
|
|
252
|
+
size: (width, height) in **level** coordinates.
|
|
253
|
+
"""
|
|
254
|
+
raise NotImplementedError
|
|
255
|
+
|
|
256
|
+
def save_from_numpy(self, data, ext='.jpg', params=None, pyramid=False,
|
|
257
|
+
mpp=None, patch_size=DEFAULT_PATCH_SIZE,
|
|
258
|
+
commit_every=DEFAULT_COMMIT_EVERY, map_size=None):
|
|
259
|
+
raise NotImplementedError
|
|
260
|
+
|
|
261
|
+
def save_from_tiles(self, patches, locations, level, ext, params, txn=None):
|
|
262
|
+
raise NotImplementedError
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
class FastMemoryGraphics(AbstractSlide):
|
|
266
|
+
"""A tiled whole-slide image stored in LMDB.
|
|
267
|
+
|
|
268
|
+
Parameters
|
|
269
|
+
----------
|
|
270
|
+
filename : str
|
|
271
|
+
Path to the ``.fmg`` JSON file.
|
|
272
|
+
mode : {'r', 'w'}
|
|
273
|
+
patch_cache : int
|
|
274
|
+
Per-process LRU cache size for decoded tiles (0 disables it). Very
|
|
275
|
+
effective for multi-epoch training, where every worker reads the
|
|
276
|
+
same tiles again and again.
|
|
277
|
+
map_size : int or None
|
|
278
|
+
Explicit LMDB map size. ``None`` (default) means: use the size
|
|
279
|
+
stored in the file when reading, and derive a size from the image
|
|
280
|
+
when writing. No more blind 1 TiB mapping per level per process.
|
|
281
|
+
strict : bool
|
|
282
|
+
Raise ``CorruptTileError`` on a missing/broken tile instead of
|
|
283
|
+
painting that tile black and warning once.
|
|
284
|
+
"""
|
|
285
|
+
|
|
286
|
+
def __init__(self, filename, mode='r', *, patch_cache=128, map_size=None,
|
|
287
|
+
strict=False, auto_worker_tuning=True):
|
|
288
|
+
if mode not in ('r', 'w'):
|
|
289
|
+
raise ValueError("mode must be 'r' or 'w'")
|
|
290
|
+
self._filename = os.fspath(filename)
|
|
291
|
+
self._mode = mode
|
|
292
|
+
self._data_dir = os.path.splitext(self._filename)[0] # same as old [:-4] for '.fmg'
|
|
293
|
+
self._patch_cache_size = max(0, int(patch_cache))
|
|
294
|
+
self._map_size = int(map_size) if map_size else None
|
|
295
|
+
self._strict = bool(strict)
|
|
296
|
+
self._auto_worker_tuning = bool(auto_worker_tuning)
|
|
297
|
+
|
|
298
|
+
self._lock = threading.RLock()
|
|
299
|
+
self._cache = OrderedDict()
|
|
300
|
+
self._envs = None # {level: lmdb.Environment} owned by THIS process
|
|
301
|
+
self._pid = None # pid that owns self._envs
|
|
302
|
+
self._creator_pid = os.getpid() # survives fork(), not pickling
|
|
303
|
+
self._closed = False
|
|
304
|
+
self._missing_warned = False
|
|
305
|
+
|
|
306
|
+
self._osr = {}
|
|
307
|
+
self._stored_dims = [] # (grid_x, grid_y) as written on disk
|
|
308
|
+
self._public_dims = [] # (width, height) exposed to callers
|
|
309
|
+
self._downsamples = []
|
|
310
|
+
self._legacy = False
|
|
311
|
+
|
|
312
|
+
if mode == 'r':
|
|
313
|
+
self._load_metadata()
|
|
314
|
+
|
|
315
|
+
# ------------------------------------------------------------------ #
|
|
316
|
+
# metadata
|
|
317
|
+
# ------------------------------------------------------------------ #
|
|
318
|
+
def _load_metadata(self):
|
|
319
|
+
# 'with' fixes the leaked file descriptor of the original code.
|
|
320
|
+
with open(self._filename, 'r', encoding='utf-8') as fh:
|
|
321
|
+
meta = json.load(fh)
|
|
322
|
+
if not isinstance(meta, dict) or 'level_dimensions' not in meta:
|
|
323
|
+
raise FmgError('not a valid .fmg metadata file: %s' % self._filename)
|
|
324
|
+
self._osr = meta
|
|
325
|
+
self._legacy = int(meta.get('format_version', 1)) < FORMAT_VERSION
|
|
326
|
+
self._stored_dims = [tuple(int(v) for v in d)
|
|
327
|
+
for d in meta['level_dimensions']]
|
|
328
|
+
if any(len(d) != 2 or d[0] <= 0 or d[1] <= 0 for d in self._stored_dims):
|
|
329
|
+
raise FmgError('invalid level_dimensions in %s' % self._filename)
|
|
330
|
+
# Legacy files stored [height, width] AND used that order as the tile
|
|
331
|
+
# grid, so the grid itself is fine; only the public (w, h) view is
|
|
332
|
+
# reversed. New files store (width, height) == grid (x, y).
|
|
333
|
+
if self._legacy:
|
|
334
|
+
self._public_dims = [(d[1], d[0]) for d in self._stored_dims]
|
|
335
|
+
warnings.warn(
|
|
336
|
+
'%s uses the legacy format (level_dimensions stored as '
|
|
337
|
+
'[height, width] and non-square images may miss tiles). '
|
|
338
|
+
'Re-write the slide with save_from_numpy() to upgrade.'
|
|
339
|
+
% self._filename, RuntimeWarning, stacklevel=2)
|
|
340
|
+
else:
|
|
341
|
+
self._public_dims = list(self._stored_dims)
|
|
342
|
+
|
|
343
|
+
raw_ds = meta.get('level_downsamples') or [1.0] * len(self._stored_dims)
|
|
344
|
+
self._downsamples = []
|
|
345
|
+
for v in raw_ds:
|
|
346
|
+
# legacy: [[dx, dy], ...]; new: [ds, ...]
|
|
347
|
+
self._downsamples.append(float(v[0] if isinstance(v, (list, tuple)) else v))
|
|
348
|
+
self._downsamples = (self._downsamples + [1.0] * len(self._stored_dims))[:len(self._stored_dims)]
|
|
349
|
+
for i, d in enumerate(self._downsamples):
|
|
350
|
+
if d <= 0 or not math.isfinite(d):
|
|
351
|
+
self._downsamples[i] = 1.0
|
|
352
|
+
|
|
353
|
+
self._osr.setdefault('patch_size', DEFAULT_PATCH_SIZE)
|
|
354
|
+
self._osr.setdefault('ext', '.jpg')
|
|
355
|
+
self._osr.setdefault('params', _default_params(self._osr['ext']))
|
|
356
|
+
self._osr.setdefault('mpp', None)
|
|
357
|
+
self._osr.setdefault('level_count', len(self._stored_dims))
|
|
358
|
+
|
|
359
|
+
def __repr__(self):
|
|
360
|
+
return f'{self.__class__.__name__}({self._filename!r}, mode={self._mode!r})'
|
|
361
|
+
|
|
362
|
+
@property
|
|
363
|
+
def level_count(self):
|
|
364
|
+
return len(self._stored_dims)
|
|
365
|
+
|
|
366
|
+
@property
|
|
367
|
+
def level_dimensions(self):
|
|
368
|
+
return [list(d) for d in self._public_dims]
|
|
369
|
+
|
|
370
|
+
@property
|
|
371
|
+
def dimensions(self):
|
|
372
|
+
return self._public_dims[0]
|
|
373
|
+
|
|
374
|
+
@property
|
|
375
|
+
def level_downsamples(self):
|
|
376
|
+
return list(self._downsamples)
|
|
377
|
+
|
|
378
|
+
@property
|
|
379
|
+
def properties(self):
|
|
380
|
+
out = dict(self._osr)
|
|
381
|
+
out['level_dimensions'] = self.level_dimensions
|
|
382
|
+
out['dimension'] = list(self.dimensions)
|
|
383
|
+
out['level_downsamples'] = self.level_downsamples
|
|
384
|
+
return out
|
|
385
|
+
|
|
386
|
+
@property
|
|
387
|
+
def patch_size(self):
|
|
388
|
+
return int(self._osr.get('patch_size', DEFAULT_PATCH_SIZE))
|
|
389
|
+
|
|
390
|
+
# ------------------------------------------------------------------ #
|
|
391
|
+
# process / thread safe environment management
|
|
392
|
+
# ------------------------------------------------------------------ #
|
|
393
|
+
def _ensure_open(self, level):
|
|
394
|
+
"""Return an LMDB Environment owned by the *current* process.
|
|
395
|
+
|
|
396
|
+
Only the requested level is opened, on first use. A forked child
|
|
397
|
+
discards inherited handles and opens its own; this is the single most
|
|
398
|
+
important fix for DataLoader crashes.
|
|
399
|
+
"""
|
|
400
|
+
if self._mode != 'r':
|
|
401
|
+
raise FmgError('slide is open for writing, not reading')
|
|
402
|
+
if self._closed:
|
|
403
|
+
raise FmgError('slide is closed')
|
|
404
|
+
pid = os.getpid()
|
|
405
|
+
is_forked_child = (self._creator_pid is not None
|
|
406
|
+
and self._creator_pid != pid)
|
|
407
|
+
with self._lock:
|
|
408
|
+
if is_forked_child and self._auto_worker_tuning:
|
|
409
|
+
# Detected even when the parent never opened an env (lazy
|
|
410
|
+
# case): OpenCV's inherited thread pool is still unsafe.
|
|
411
|
+
_tune_cv2_for_worker()
|
|
412
|
+
if self._envs is None or self._pid != pid:
|
|
413
|
+
if self._envs is not None:
|
|
414
|
+
# NEVER close an inherited env: mdb_env_close() in the
|
|
415
|
+
# child may clear the parent's reader slot / shared lock
|
|
416
|
+
# file. Dropping the reference alone is safe because we
|
|
417
|
+
# open with lock=False (no reader table is used at all).
|
|
418
|
+
self._cache.clear()
|
|
419
|
+
self._envs = {}
|
|
420
|
+
self._pid = pid
|
|
421
|
+
env = self._envs.get(level)
|
|
422
|
+
if env is not None:
|
|
423
|
+
return env
|
|
424
|
+
path = os.path.join(self._data_dir, 'level' + str(level))
|
|
425
|
+
env = _acquire_read_env(path, self._map_size)
|
|
426
|
+
self._envs[level] = env
|
|
427
|
+
return env
|
|
428
|
+
|
|
429
|
+
def close(self):
|
|
430
|
+
"""Idempotent. Only closes handles owned by the current process."""
|
|
431
|
+
with self._lock:
|
|
432
|
+
if self._closed:
|
|
433
|
+
return
|
|
434
|
+
self._closed = True
|
|
435
|
+
envs, owner = self._envs, self._pid
|
|
436
|
+
self._envs = None
|
|
437
|
+
self._cache.clear()
|
|
438
|
+
if not envs or owner != os.getpid():
|
|
439
|
+
return
|
|
440
|
+
for env in envs.values():
|
|
441
|
+
_release_read_env(env)
|
|
442
|
+
|
|
443
|
+
def __del__(self):
|
|
444
|
+
try:
|
|
445
|
+
self.close()
|
|
446
|
+
except Exception:
|
|
447
|
+
pass
|
|
448
|
+
|
|
449
|
+
# ------------------------------------------------------------------ #
|
|
450
|
+
# pickling: a forked/pickled slide must not carry an LMDB Environment.
|
|
451
|
+
# The clone reopens on first read; in the SAME process that read hits the
|
|
452
|
+
# shared registry, in a new process (spawn) it opens its own env.
|
|
453
|
+
# ------------------------------------------------------------------ #
|
|
454
|
+
def __getstate__(self):
|
|
455
|
+
state = self.__dict__.copy()
|
|
456
|
+
state['_envs'] = None
|
|
457
|
+
state['_pid'] = None
|
|
458
|
+
state['_creator_pid'] = None # unpickled object is native to its process
|
|
459
|
+
state['_lock'] = None
|
|
460
|
+
state['_cache'] = OrderedDict()
|
|
461
|
+
state['_closed'] = False
|
|
462
|
+
state['_missing_warned'] = False
|
|
463
|
+
return state
|
|
464
|
+
|
|
465
|
+
def __setstate__(self, state):
|
|
466
|
+
self.__dict__.update(state)
|
|
467
|
+
self._lock = threading.RLock()
|
|
468
|
+
self._cache = OrderedDict()
|
|
469
|
+
self._envs = None
|
|
470
|
+
self._pid = None
|
|
471
|
+
self._creator_pid = os.getpid()
|
|
472
|
+
self._closed = False
|
|
473
|
+
self._missing_warned = False
|
|
474
|
+
|
|
475
|
+
# ------------------------------------------------------------------ #
|
|
476
|
+
# tile cache
|
|
477
|
+
# ------------------------------------------------------------------ #
|
|
478
|
+
def _cache_get(self, key):
|
|
479
|
+
if not self._patch_cache_size:
|
|
480
|
+
return None
|
|
481
|
+
with self._lock:
|
|
482
|
+
tile = self._cache.get(key)
|
|
483
|
+
if tile is not None:
|
|
484
|
+
self._cache.move_to_end(key)
|
|
485
|
+
return tile
|
|
486
|
+
|
|
487
|
+
def _cache_put(self, key, tile):
|
|
488
|
+
if not self._patch_cache_size:
|
|
489
|
+
return
|
|
490
|
+
with self._lock:
|
|
491
|
+
self._cache[key] = tile
|
|
492
|
+
self._cache.move_to_end(key)
|
|
493
|
+
while len(self._cache) > self._patch_cache_size:
|
|
494
|
+
self._cache.popitem(last=False)
|
|
495
|
+
|
|
496
|
+
def _broken(self, level, box, reason):
|
|
497
|
+
msg = 'level %d tile %s: %s' % (level, tuple(box), reason)
|
|
498
|
+
if self._strict:
|
|
499
|
+
raise CorruptTileError(msg)
|
|
500
|
+
if not self._missing_warned:
|
|
501
|
+
warnings.warn(msg + ' (painted black)', RuntimeWarning, stacklevel=3)
|
|
502
|
+
self._missing_warned = True
|
|
503
|
+
|
|
504
|
+
# ------------------------------------------------------------------ #
|
|
505
|
+
# reading
|
|
506
|
+
# ------------------------------------------------------------------ #
|
|
507
|
+
def _read_tile(self, level, box, txn):
|
|
508
|
+
cache_key = b'%d/' % level + _tile_key(box)
|
|
509
|
+
tile = self._cache_get(cache_key)
|
|
510
|
+
if tile is not None:
|
|
511
|
+
return tile
|
|
512
|
+
|
|
513
|
+
raw = txn.get(key=_tile_key(box))
|
|
514
|
+
if raw is None or len(raw) == 0:
|
|
515
|
+
self._broken(level, box, 'missing or empty tile')
|
|
516
|
+
return None
|
|
517
|
+
try:
|
|
518
|
+
arr = np.frombuffer(raw, dtype=np.uint8)
|
|
519
|
+
tile = cv2.imdecode(arr, cv2.IMREAD_COLOR)
|
|
520
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
521
|
+
self._broken(level, box, 'decode failed: %r' % (exc,))
|
|
522
|
+
return None
|
|
523
|
+
if tile is None or tile.ndim != 3 or tile.shape[2] != 3:
|
|
524
|
+
self._broken(level, box, 'invalid decoded tile')
|
|
525
|
+
return None
|
|
526
|
+
self._cache_put(cache_key, tile)
|
|
527
|
+
return tile
|
|
528
|
+
|
|
529
|
+
def read_region(self, location, level, size):
|
|
530
|
+
"""Return a PIL.Image of ``size`` (width, height) in level coordinates.
|
|
531
|
+
|
|
532
|
+
``location`` is given in the level-0 frame, as in OpenSlide. Regions
|
|
533
|
+
outside the slide are padded with black instead of raising.
|
|
534
|
+
"""
|
|
535
|
+
if self._mode != 'r':
|
|
536
|
+
raise FmgError('slide is open for writing, not reading')
|
|
537
|
+
if not 0 <= level < self.level_count:
|
|
538
|
+
raise ValueError('level %d out of range [0, %d)' % (level, self.level_count))
|
|
539
|
+
width, height = int(size[0]), int(size[1])
|
|
540
|
+
if width <= 0 or height <= 0:
|
|
541
|
+
raise ValueError('size must be positive, got %r' % (size,))
|
|
542
|
+
x0, y0 = int(location[0]), int(location[1])
|
|
543
|
+
|
|
544
|
+
env = self._ensure_open(level)
|
|
545
|
+
|
|
546
|
+
ds = self._downsamples[level]
|
|
547
|
+
lx0 = int(round(x0 / ds))
|
|
548
|
+
ly0 = int(round(y0 / ds))
|
|
549
|
+
lx1, ly1 = lx0 + width, ly0 + height
|
|
550
|
+
|
|
551
|
+
step = self.patch_size
|
|
552
|
+
grid_x, grid_y = self._stored_dims[level]
|
|
553
|
+
out = np.zeros((height, width, 3), dtype=np.uint8)
|
|
554
|
+
|
|
555
|
+
x_first = math.floor(lx0 / step) * step
|
|
556
|
+
y_first = math.floor(ly0 / step) * step
|
|
557
|
+
x_last = min(grid_x, math.ceil(lx1 / step) * step)
|
|
558
|
+
y_last = min(grid_y, math.ceil(ly1 / step) * step)
|
|
559
|
+
|
|
560
|
+
with env.begin(buffers=True) as txn:
|
|
561
|
+
for ty in range(y_first, y_last, step):
|
|
562
|
+
if ty < 0:
|
|
563
|
+
continue
|
|
564
|
+
ty_max = min(ty + step, grid_y)
|
|
565
|
+
iy0, iy1 = max(ty, ly0), min(ty_max, ly1)
|
|
566
|
+
if iy0 >= iy1:
|
|
567
|
+
continue
|
|
568
|
+
for tx in range(x_first, x_last, step):
|
|
569
|
+
if tx < 0:
|
|
570
|
+
continue
|
|
571
|
+
tx_max = min(tx + step, grid_x)
|
|
572
|
+
ix0, ix1 = max(tx, lx0), min(tx_max, lx1)
|
|
573
|
+
if ix0 >= ix1:
|
|
574
|
+
continue
|
|
575
|
+
box = (tx, ty, tx_max, ty_max)
|
|
576
|
+
tile = self._read_tile(level, box, txn)
|
|
577
|
+
if tile is None:
|
|
578
|
+
continue
|
|
579
|
+
src = tile[iy0 - ty:iy1 - ty, ix0 - tx:ix1 - tx]
|
|
580
|
+
dst = out[iy0 - ly0:iy1 - ly0, ix0 - lx0:ix1 - lx0]
|
|
581
|
+
if src.shape != dst.shape:
|
|
582
|
+
# Damaged / mismatched legacy tile; do not crash a
|
|
583
|
+
# whole training job over one bad patch.
|
|
584
|
+
self._broken(level, box, 'tile shape %s != %s'
|
|
585
|
+
% (src.shape, dst.shape))
|
|
586
|
+
continue
|
|
587
|
+
dst[:] = src
|
|
588
|
+
return Image.fromarray(out)
|
|
589
|
+
|
|
590
|
+
# ------------------------------------------------------------------ #
|
|
591
|
+
# writing
|
|
592
|
+
# ------------------------------------------------------------------ #
|
|
593
|
+
@staticmethod
|
|
594
|
+
def _as_uint8_hwc(data):
|
|
595
|
+
arr = np.asarray(data)
|
|
596
|
+
if arr.ndim == 2:
|
|
597
|
+
arr = arr[:, :, None]
|
|
598
|
+
if arr.ndim != 3:
|
|
599
|
+
raise ValueError('data must be HWC or HW, got shape %s' % (arr.shape,))
|
|
600
|
+
if arr.shape[2] == 1:
|
|
601
|
+
arr = np.repeat(arr, 3, axis=2)
|
|
602
|
+
elif arr.shape[2] == 4:
|
|
603
|
+
arr = arr[:, :, :3]
|
|
604
|
+
elif arr.shape[2] != 3:
|
|
605
|
+
raise ValueError('data must have 1, 3 or 4 channels, got %d' % arr.shape[2])
|
|
606
|
+
if arr.dtype != np.uint8:
|
|
607
|
+
arr = np.clip(arr, 0, 255).astype(np.uint8)
|
|
608
|
+
return arr
|
|
609
|
+
|
|
610
|
+
@staticmethod
|
|
611
|
+
def _put_tiles(txn, patches, locations, ext, params):
|
|
612
|
+
for patch, box in zip(patches, locations):
|
|
613
|
+
arr = np.ascontiguousarray(FastMemoryGraphics._as_uint8_hwc(patch))
|
|
614
|
+
ok, buff = cv2.imencode(ext, arr, params)
|
|
615
|
+
if not ok or buff is None or buff.size == 0:
|
|
616
|
+
raise FmgError('cv2.imencode failed for tile %s' % (tuple(box),))
|
|
617
|
+
# memoryview avoids the extra bytes() copy of the original code.
|
|
618
|
+
txn.put(key=_tile_key(box), value=memoryview(buff))
|
|
619
|
+
|
|
620
|
+
@staticmethod
|
|
621
|
+
def _estimate_map_size(dims):
|
|
622
|
+
raw = sum(w * h for w, h in dims) * 3 # 3 channels, uint8
|
|
623
|
+
return max(MIN_WRITE_MAP_SIZE,
|
|
624
|
+
min(MAX_WRITE_MAP_SIZE, int(raw * 3 + (1 << 20) * len(dims))))
|
|
625
|
+
|
|
626
|
+
def _write_env(self, level, dims, map_size):
|
|
627
|
+
path = os.path.join(self._data_dir, 'level' + str(level))
|
|
628
|
+
env = lmdb.open(path, subdir=True, create=True,
|
|
629
|
+
map_size=map_size, max_spare_txns=1)
|
|
630
|
+
txn = env.begin(write=True)
|
|
631
|
+
txn.put(b'patch_size', str(self._osr['patch_size']).encode())
|
|
632
|
+
txn.put(b'mpp', str(self._osr['mpp']).encode())
|
|
633
|
+
txn.put(b'dimension', str(list(dims)).encode())
|
|
634
|
+
txn.commit()
|
|
635
|
+
return env
|
|
636
|
+
|
|
637
|
+
def save_from_numpy(self, data, ext='.jpg', params=None, pyramid=False,
|
|
638
|
+
mpp=None, patch_size=DEFAULT_PATCH_SIZE,
|
|
639
|
+
commit_every=DEFAULT_COMMIT_EVERY, map_size=None):
|
|
640
|
+
"""Write a whole slide.
|
|
641
|
+
|
|
642
|
+
``level_dimensions`` is now stored as (width, height) and the tile
|
|
643
|
+
grid iterates x over width and y over height -- the original code
|
|
644
|
+
mixed the two up, which silently corrupted every non-square image.
|
|
645
|
+
"""
|
|
646
|
+
if self._mode != 'w':
|
|
647
|
+
raise FmgError('slide is open for reading, not writing')
|
|
648
|
+
ext = ext.lower()
|
|
649
|
+
if ext not in SUPPORTED_EXT:
|
|
650
|
+
raise ValueError('unsupported ext %r, choose from %s' % (ext, sorted(SUPPORTED_EXT)))
|
|
651
|
+
params = [int(p) for p in params] if params is not None else _default_params(ext)
|
|
652
|
+
patch_size = int(patch_size)
|
|
653
|
+
if patch_size <= 0:
|
|
654
|
+
raise ValueError('patch_size must be positive')
|
|
655
|
+
|
|
656
|
+
level0 = self._as_uint8_hwc(data)
|
|
657
|
+
h, w, _ = level0.shape
|
|
658
|
+
|
|
659
|
+
dims = [(w, h)]
|
|
660
|
+
downsamples = [1.0]
|
|
661
|
+
nw, nh = w, h
|
|
662
|
+
# Only the *sizes* are planned here; each level image is generated
|
|
663
|
+
# once, directly from level 0, inside the write loop below. The old
|
|
664
|
+
# code kept a list of full copies of every level in RAM.
|
|
665
|
+
while pyramid and max(nw, nh) > 512 and len(dims) < 12:
|
|
666
|
+
nw, nh = max(1, nw // 2), max(1, nh // 2)
|
|
667
|
+
dims.append((nw, nh))
|
|
668
|
+
downsamples.append(downsamples[-1] * 2.0)
|
|
669
|
+
|
|
670
|
+
map_size = int(map_size) if map_size else (
|
|
671
|
+
self._map_size or self._estimate_map_size(dims))
|
|
672
|
+
map_size = max(MIN_WRITE_MAP_SIZE, map_size)
|
|
673
|
+
|
|
674
|
+
properties = {
|
|
675
|
+
'format_version': FORMAT_VERSION,
|
|
676
|
+
'map_size': map_size,
|
|
677
|
+
'grid_axes': 'xy', # level_dimensions == tile grid (x, y)
|
|
678
|
+
'level_count': len(dims),
|
|
679
|
+
'level_dimensions': [list(d) for d in dims],
|
|
680
|
+
'dimension': list(dims[0]),
|
|
681
|
+
'level_downsamples': downsamples,
|
|
682
|
+
'mpp': float(mpp) if mpp is not None else None,
|
|
683
|
+
'ext': ext,
|
|
684
|
+
'params': params,
|
|
685
|
+
'patch_size': patch_size,
|
|
686
|
+
'channels': 3,
|
|
687
|
+
'tile_codec': 'opencv',
|
|
688
|
+
}
|
|
689
|
+
self._osr = properties
|
|
690
|
+
self._stored_dims = [tuple(d) for d in dims]
|
|
691
|
+
self._public_dims = [tuple(d) for d in dims]
|
|
692
|
+
self._downsamples = list(downsamples)
|
|
693
|
+
self._legacy = False
|
|
694
|
+
|
|
695
|
+
if os.path.exists(self._data_dir):
|
|
696
|
+
shutil.rmtree(self._data_dir)
|
|
697
|
+
os.makedirs(self._data_dir)
|
|
698
|
+
|
|
699
|
+
for level, (gw, gh) in enumerate(dims):
|
|
700
|
+
layer = (level0 if level == 0 else
|
|
701
|
+
cv2.resize(level0, (gw, gh), interpolation=cv2.INTER_AREA))
|
|
702
|
+
env = self._write_env(level, (gw, gh), map_size)
|
|
703
|
+
try:
|
|
704
|
+
txn = env.begin(write=True)
|
|
705
|
+
written = 0
|
|
706
|
+
for x in range(0, gw, patch_size):
|
|
707
|
+
for y in range(0, gh, patch_size):
|
|
708
|
+
x_max, y_max = min(x + patch_size, gw), min(y + patch_size, gh)
|
|
709
|
+
arr = np.ascontiguousarray(layer[y:y_max, x:x_max, :])
|
|
710
|
+
ok, buff = cv2.imencode(ext, arr, params)
|
|
711
|
+
if not ok or buff is None or buff.size == 0:
|
|
712
|
+
raise FmgError('cv2.imencode failed for tile (%d,%d,%d,%d)'
|
|
713
|
+
% (x, y, x_max, y_max))
|
|
714
|
+
txn.put(key=_tile_key((x, y, x_max, y_max)), value=memoryview(buff))
|
|
715
|
+
written += 1
|
|
716
|
+
if written % commit_every == 0:
|
|
717
|
+
txn.commit()
|
|
718
|
+
txn = env.begin(write=True)
|
|
719
|
+
txn.commit()
|
|
720
|
+
env.sync()
|
|
721
|
+
finally:
|
|
722
|
+
env.close()
|
|
723
|
+
|
|
724
|
+
# Atomic metadata write: never leave a .fmg pointing at half-written data.
|
|
725
|
+
tmp = self._filename + '.tmp'
|
|
726
|
+
with open(tmp, 'w', encoding='utf-8') as fh:
|
|
727
|
+
json.dump(properties, fh, separators=(',', ':'))
|
|
728
|
+
os.replace(tmp, self._filename)
|
|
729
|
+
return properties
|
|
730
|
+
|
|
731
|
+
def save_from_tiles(self, patches, locations, level, ext, params=None, txn=None):
|
|
732
|
+
"""Append tiles to ``level``.
|
|
733
|
+
|
|
734
|
+
Existing callers work unchanged. A write transaction may be supplied
|
|
735
|
+
(``txn=``) so a caller can batch thousands of tiles into one commit.
|
|
736
|
+
"""
|
|
737
|
+
if self._mode != 'w':
|
|
738
|
+
raise FmgError('slide is open for reading, not writing')
|
|
739
|
+
if not 0 <= level < self.level_count:
|
|
740
|
+
raise ValueError('level %d out of range' % level)
|
|
741
|
+
ext = ext.lower()
|
|
742
|
+
if ext not in SUPPORTED_EXT:
|
|
743
|
+
raise ValueError('unsupported ext %r' % ext)
|
|
744
|
+
params = [int(p) for p in params] if params is not None else _default_params(ext)
|
|
745
|
+
if txn is not None:
|
|
746
|
+
self._put_tiles(txn, patches, locations, ext, params)
|
|
747
|
+
return
|
|
748
|
+
path = os.path.join(self._data_dir, 'level' + str(level))
|
|
749
|
+
map_size = self._map_size or self._estimate_map_size(self._stored_dims)
|
|
750
|
+
env = lmdb.open(path, subdir=True, create=True,
|
|
751
|
+
map_size=map_size, max_spare_txns=1)
|
|
752
|
+
try:
|
|
753
|
+
with env.begin(write=True) as t:
|
|
754
|
+
self._put_tiles(t, patches, locations, ext, params)
|
|
755
|
+
env.sync()
|
|
756
|
+
finally:
|
|
757
|
+
env.close()
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fmg
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Fast Memory Graphics for big image, use like OpenSlide
|
|
5
|
+
Home-page: https://github.com/khtao/fmg
|
|
6
|
+
Author: khtao
|
|
7
|
+
Author-email: khtao@live.cn
|
|
8
|
+
Maintainer: khtao
|
|
9
|
+
Maintainer-email: khtao@live.cn
|
|
10
|
+
License: MIT
|
|
11
|
+
Project-URL: Source, https://github.com/khtao/fmg
|
|
12
|
+
Project-URL: Issues, https://github.com/khtao/fmg/issues
|
|
13
|
+
Project-URL: Optimization notes, https://github.com/khtao/fmg/blob/main/OPTIMIZATION.md
|
|
14
|
+
Project-URL: Release guide, https://github.com/khtao/fmg/blob/main/RELEASE.md
|
|
15
|
+
Keywords: whole slide image,WSI,digital pathology,LMDB,OpenSlide,tiled image,microscopy,remote sensing
|
|
16
|
+
Classifier: Development Status :: 4 - Beta
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Intended Audience :: Developers
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
20
|
+
Classifier: Operating System :: OS Independent
|
|
21
|
+
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
27
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
28
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
29
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
30
|
+
Requires-Python: >=3.7
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
License-File: LICENSE
|
|
33
|
+
Requires-Dist: lmdb>=1.0
|
|
34
|
+
Requires-Dist: opencv-python>=4.0
|
|
35
|
+
Requires-Dist: numpy>=1.17
|
|
36
|
+
Requires-Dist: pillow>=7.0
|
|
37
|
+
Dynamic: author
|
|
38
|
+
Dynamic: author-email
|
|
39
|
+
Dynamic: classifier
|
|
40
|
+
Dynamic: description
|
|
41
|
+
Dynamic: description-content-type
|
|
42
|
+
Dynamic: home-page
|
|
43
|
+
Dynamic: keywords
|
|
44
|
+
Dynamic: license
|
|
45
|
+
Dynamic: license-file
|
|
46
|
+
Dynamic: maintainer
|
|
47
|
+
Dynamic: maintainer-email
|
|
48
|
+
Dynamic: project-url
|
|
49
|
+
Dynamic: requires-dist
|
|
50
|
+
Dynamic: requires-python
|
|
51
|
+
Dynamic: summary
|
|
52
|
+
|
|
53
|
+
# fmg
|
|
54
|
+
|
|
55
|
+
[](https://pypi.org/project/fmg/)
|
|
56
|
+
[](https://pypi.org/project/fmg/)
|
|
57
|
+
[](https://opensource.org/licenses/MIT)
|
|
58
|
+
|
|
59
|
+
> 把超大图像切成 tile 存进 LMDB,用 OpenSlide 的方式读,然后——在 `num_workers=8` 的 DataLoader 里,忘掉它的存在。
|
|
60
|
+
|
|
61
|
+
**fmg 0.4.0** · Python ≥ 3.7 · MIT · LMDB / OpenCV / NumPy / Pillow
|
|
62
|
+
|
|
63
|
+
面向数字病理全切片(WSI)、遥感大图、高分辨率扫描件这类「一张图几十 GB」的场景。它不把整图塞进内存,而是按 tile 随机读取,并提供一个你和 OpenSlide 一样熟悉的接口。
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## 你有没有被这几个报错折磨过
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
DataLoader worker (pid 12345) exited unexpectedly
|
|
71
|
+
lmdb.BadRslotError: MDB_BAD_RSLOT
|
|
72
|
+
lmdb.Error: The environment '.../level0' is already open in this process
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
然后是更隐蔽的那种:**不报错,但非正方形图像写出来就是错的**——宽方向多出空 patch、高方向少了 tile,训练安静地喂进一堆黑块。
|
|
76
|
+
|
|
77
|
+
这些不是你的代码写错了。0.3.0 在父进程就打开 LMDB 并长期持有一个事务,`fork` 出来的 worker 共享同一个 env、同一段 mmap 和同一个 reader slot;再加一个 `[h, w]` 当成 `[w, h]` 的笔误,问题就齐了。
|
|
78
|
+
|
|
79
|
+
**0.4.0 把这一整类问题从库里拿掉了。** 你要做的只是把 `open_slide()` 放回 `Dataset.__init__`。
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## 为什么值得换过来
|
|
84
|
+
|
|
85
|
+
### 🛡 多 worker 是安全的,不再靠运气
|
|
86
|
+
|
|
87
|
+
| 做法 | 效果 |
|
|
88
|
+
|---|---|
|
|
89
|
+
| 懒打开 | `open_slide()` 只读 JSON 元数据,不碰 LMDB;第一次 `read_region()` 才建环境 |
|
|
90
|
+
| 进程感知重开 | 检测到 pid 变化就丢弃继承来的句柄(**不调用 `close()`**,以免清掉父进程的 reader slot),在子进程重新 `lmdb.open()` |
|
|
91
|
+
| 短事务 | 每次读取一个独立只读事务,`max_spare_txns=8` 让创建近乎零成本,天然线程安全 |
|
|
92
|
+
| `lock=False` | MDB_NOLOCK 完全绕开 reader lock table,多进程只读没有 slot 竞争 |
|
|
93
|
+
| OpenCV fork 处理 | 子进程自动 `setNumThreads(0)` + 关闭 OpenCL,线程池不再在 `imdecode` 里挂住 |
|
|
94
|
+
| 可 pickle | 剔除 env 与锁,`fork` / `spawn` 两种启动方式都能用 |
|
|
95
|
+
|
|
96
|
+
### 🤝 同一张片子开几次,只占一份映射
|
|
97
|
+
|
|
98
|
+
进程内按 `(路径, map_size)` 共享**一个**只读 Environment,引用计数管理生命周期:`close()` 只减引用,最后一个使用者离开才真正关闭。于是这些场景全部不再冲突——
|
|
99
|
+
|
|
100
|
+
- `pickle` 往返产生的克隆对象
|
|
101
|
+
- train / val 两个 Dataset 指向同一张 slide
|
|
102
|
+
- 多个 Dataset 实例、多个 loader
|
|
103
|
+
- `spawn` 反序列化出的 slide
|
|
104
|
+
|
|
105
|
+
`os.register_at_fork` 会在子进程清空注册表并重建锁,避免 fork 时恰好有人持锁导致死锁。
|
|
106
|
+
|
|
107
|
+
### ⚡ 读得更快,而且省内存
|
|
108
|
+
|
|
109
|
+
零拷贝(`buffers=True` + `memoryview`)、直接分配目标画布 + 只拷贝相交 tile 的源/目标交集(不再先铺整个包围盒)、按 level 懒开环境、`map_size=0` 从文件 meta 页自动读真实尺寸、`readahead=False` 避免随机读时的无用预读。
|
|
110
|
+
|
|
111
|
+
写侧则是**流式金字塔**:每层由 level-0 一次 `cv2.INTER_AREA` 降采样得到、用完即弃,不再同时持有所有层的完整 numpy 副本(老写法峰值约 1.33× 原图,WSI 上就是几十 GB);配合每 2048 个 tile 批量提交、`memoryview` 代替 `tobytes()`。
|
|
112
|
+
|
|
113
|
+
### ✅ 正确性与健壮性
|
|
114
|
+
|
|
115
|
+
- **宽高顺序彻底修正**:新格式统一 `(width, height)`,网格 x 遍历宽、y 遍历高;`h, w, _ = data.shape`;`level_downsamples` 存标量。
|
|
116
|
+
- **OpenSlide 语义**:`location` 是 level-0 坐标,`size` 是 level 坐标尺寸,内部按 downsample 缩放;越界区域补黑而不是切片错乱。
|
|
117
|
+
- **坏 tile 不拖垮训练**:缺 tile / 解码失败 / 形状不符默认涂黑并只警告一次;`strict=True` 改为抛 `CorruptTileError`。
|
|
118
|
+
- **不会留下半个文件**:元数据先写 `.tmp` 再 `os.replace()` 原子替换;`imencode` / `imdecode` 返回值全部检查。
|
|
119
|
+
- **向后兼容**:0.3.0 写出的旧 `.fmg` 直接可读,自动把对外视图翻回 `(w, h)`、把 downsample 取成标量并给一次警告;新版写入 `format_version=2`。
|
|
120
|
+
- `close()` 幂等、`__del__` 补齐、`with slide:` 正常可用;`mpp=None` 不再 `TypeError`;`os.path.splitext()` 代替硬切 `[:-4]`。
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## 性能
|
|
125
|
+
|
|
126
|
+
1024×1024 图像、`patch_size=256`,随机读 200 个 512×512 区域:
|
|
127
|
+
|
|
128
|
+
| 项目 | 0.3.0 | 0.4.0 |
|
|
129
|
+
|---|---|---|
|
|
130
|
+
| 写入一张图 | 2.11 s | 1.94 s |
|
|
131
|
+
| 随机读 200 个区域 | 1.54 s | **0.48 s**(约 3.2×,关缓存) |
|
|
132
|
+
| 同上,`patch_cache=256` | — | 0.04 s |
|
|
133
|
+
|
|
134
|
+
复测(py-lmdb 3.0.0)读侧为 2.10 s → 0.64 s / 0.06 s,约 3.3×,与上表一致。
|
|
135
|
+
|
|
136
|
+
> **口径提醒**:带缓存那一行是「同一批 tile 被反复读」的理想场景,多 epoch 训练接近这个收益;单 epoch 顺序遍历全图时命中率会低很多,别按这个数字估吞吐。写侧瓶颈在 `cv2.imencode`,小图提升有限,大图上批量提交与流式金字塔的收益才明显。数字受硬件与文件系统影响,请以自己数据上的 `selftest.py` / `_bench.py` 为准。
|
|
137
|
+
|
|
138
|
+
另外,用原版跑 fork 对照时**有时 4 个 worker 会全部正常退出**——这恰恰说明它是低概率崩溃,取决于时序。别因为「我这儿没崩」就认为旧写法安全。
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## 快速开始
|
|
143
|
+
|
|
144
|
+
### 安装
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
pip install fmg==0.4.0
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
从源码安装(或想跑自测 / 基准):
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
git clone https://github.com/khtao/fmg.git && cd fmg
|
|
154
|
+
pip install .
|
|
155
|
+
python selftest.py
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
依赖会自动装上:`lmdb>=1.0`、`opencv-python>=4.0`、`numpy>=1.17`、`pillow>=7.0`。无 GUI 的服务器若装不上 `opencv-python`(缺 `libGL`),可在自己的环境里改用 `opencv-python-headless`。
|
|
159
|
+
|
|
160
|
+
想给项目做一次发布?作者操作手册见 [`RELEASE.md`](https://github.com/khtao/fmg/blob/main/RELEASE.md)。
|
|
161
|
+
|
|
162
|
+
### 写入一张图
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
import numpy as np
|
|
166
|
+
import fmg
|
|
167
|
+
|
|
168
|
+
img = np.imread('huge.png') # HWC,uint8 / 其它 dtype 均可
|
|
169
|
+
|
|
170
|
+
meta = fmg.create_slide('slide.fmg').save_from_numpy(
|
|
171
|
+
img,
|
|
172
|
+
ext='.png', # jpg / jpeg / png / webp / bmp / tiff / tif
|
|
173
|
+
pyramid=True, # 用 INTER_AREA 生成金字塔,不再是隔点采样的锯齿
|
|
174
|
+
mpp=0.25, # 可省略(None 也合法)
|
|
175
|
+
patch_size=256,
|
|
176
|
+
)
|
|
177
|
+
print(meta['level_dimensions']) # [[w, h], [w/2, h/2], ...]
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
也可以只补写若干 tile:`save_from_tiles(patches, locations, level, ext, params=None, txn=None)`,外部传入 `txn=` 就能把成千上万个 tile 并进一次提交。
|
|
181
|
+
|
|
182
|
+
### 读取
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
import fmg
|
|
186
|
+
|
|
187
|
+
slide = fmg.open_slide('slide.fmg', patch_cache=256)
|
|
188
|
+
w, h = slide.dimensions # (宽, 高)
|
|
189
|
+
print(slide.level_count, slide.level_dimensions, slide.level_downsamples)
|
|
190
|
+
|
|
191
|
+
patch = slide.read_region((1000, 2000), 0, (512, 512)) # -> PIL.Image
|
|
192
|
+
# location 是 level-0 坐标;size 是 level 坐标尺寸
|
|
193
|
+
low = slide.read_region((1000, 2000), 1, (256, 256)) # 自动按 downsample 缩放
|
|
194
|
+
|
|
195
|
+
with slide: # 或者 slide.close()
|
|
196
|
+
pass
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### 接进 PyTorch DataLoader
|
|
200
|
+
|
|
201
|
+
```python
|
|
202
|
+
import numpy as np
|
|
203
|
+
import torch
|
|
204
|
+
from torch.utils.data import Dataset, DataLoader, get_worker_info
|
|
205
|
+
import fmg
|
|
206
|
+
|
|
207
|
+
class SlideDataset(Dataset):
|
|
208
|
+
def __init__(self, path, size=512):
|
|
209
|
+
# 放心放在 __init__:这里只读元数据,不打开 LMDB
|
|
210
|
+
self.slide = fmg.open_slide(path, patch_cache=256)
|
|
211
|
+
self.w, self.h = self.slide.dimensions
|
|
212
|
+
self.size = size
|
|
213
|
+
|
|
214
|
+
def __len__(self):
|
|
215
|
+
return (self.w // self.size) * (self.h // self.size)
|
|
216
|
+
|
|
217
|
+
def __getitem__(self, i):
|
|
218
|
+
x = (i % (self.w // self.size)) * self.size
|
|
219
|
+
y = (i // (self.w // self.size)) * self.size
|
|
220
|
+
arr = np.asarray(self.slide.read_region((x, y), 0, (self.size, self.size)))
|
|
221
|
+
return torch.from_numpy(np.ascontiguousarray(arr.transpose(2, 0, 1)))
|
|
222
|
+
|
|
223
|
+
def worker_init_fn(worker_id):
|
|
224
|
+
# fork 下库会自动处理;spawn 下必须显式调用这一句
|
|
225
|
+
fmg.configure(num_threads=0)
|
|
226
|
+
info = get_worker_info()
|
|
227
|
+
if info is not None:
|
|
228
|
+
seed = info.seed if info.seed is not None else torch.initial_seed()
|
|
229
|
+
np.random.seed(seed % (2 ** 32))
|
|
230
|
+
info.dataset.slide.read_region((0, 0), 0, (1, 1)) # 预热,错开首次开 env
|
|
231
|
+
|
|
232
|
+
loader = DataLoader(
|
|
233
|
+
SlideDataset('slide.fmg'),
|
|
234
|
+
batch_size=128, shuffle=True,
|
|
235
|
+
num_workers=4,
|
|
236
|
+
persistent_workers=True, # 每 epoch 不重开 env / 不清缓存
|
|
237
|
+
pin_memory=True,
|
|
238
|
+
prefetch_factor=2,
|
|
239
|
+
worker_init_fn=worker_init_fn,
|
|
240
|
+
multiprocessing_context='fork', # 换成 'spawn' 也行,slide 可 pickle
|
|
241
|
+
)
|
|
242
|
+
for batch in loader:
|
|
243
|
+
...
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
完整可运行版本见 [`test.py`](https://github.com/khtao/fmg/blob/main/test.py)。
|
|
247
|
+
|
|
248
|
+
---
|
|
249
|
+
|
|
250
|
+
## API 速查
|
|
251
|
+
|
|
252
|
+
| 入口 | 说明 |
|
|
253
|
+
|---|---|
|
|
254
|
+
| `fmg.open_slide(path, **kw)` | 打开已有 `.fmg`(只读,懒打开) |
|
|
255
|
+
| `fmg.create_slide(path, **kw)` | 创建写对象 |
|
|
256
|
+
| `fmg.configure(num_threads=0, use_opencl=False)` | 在 `worker_init_fn` 里显式配置 OpenCV |
|
|
257
|
+
| `slide.read_region(location, level, size)` | 返回 `PIL.Image`;越界补黑 |
|
|
258
|
+
| `slide.dimensions` / `level_dimensions` | `(w, h)` / `[[w, h], ...]` |
|
|
259
|
+
| `slide.level_count` / `level_downsamples` / `properties` / `patch_size` | 元数据 |
|
|
260
|
+
| `save_from_numpy(...)` / `save_from_tiles(...)` | 整图写入 / 追加 tile |
|
|
261
|
+
| `slide.close()` / `with slide:` | 幂等关闭,只释放本进程持有的引用 |
|
|
262
|
+
| `fmg.FmgError` / `fmg.CorruptTileError` | 异常类型 |
|
|
263
|
+
| `fmg.FORMAT_VERSION` | 当前为 `2` |
|
|
264
|
+
|
|
265
|
+
**`open_slide` 参数**
|
|
266
|
+
|
|
267
|
+
| 参数 | 默认 | 含义 |
|
|
268
|
+
|---|---|---|
|
|
269
|
+
| `patch_cache` | `128` | 进程内解码 tile 的 LRU 缓存大小,`0` 关闭 |
|
|
270
|
+
| `map_size` | `None` | 显式 LMDB map size;默认从文件 meta 页读真实值(含旧版 1 TiB 文件) |
|
|
271
|
+
| `strict` | `False` | 坏 tile 改为抛 `CorruptTileError` 而不是涂黑 |
|
|
272
|
+
| `auto_worker_tuning` | `True` | fork 子进程自动关闭 OpenCV 线程池 / OpenCL |
|
|
273
|
+
|
|
274
|
+
---
|
|
275
|
+
|
|
276
|
+
## 磁盘格式
|
|
277
|
+
|
|
278
|
+
```
|
|
279
|
+
slide.fmg JSON 元数据(格式版本、各层尺寸、downsample、mpp、编码参数…)
|
|
280
|
+
slide/
|
|
281
|
+
├── level0/ LMDB,key = str((x, y, x_max, y_max))
|
|
282
|
+
│ ├── data.mdb
|
|
283
|
+
│ └── lock.mdb
|
|
284
|
+
├── level1/
|
|
285
|
+
└── ...
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
每个 tile 由 `cv2.imencode` 编码、读时 `cv2.imdecode` 还原,二者互逆,通道顺序保持一致。格式版本 2 与 1 的目录布局相同,因此**旧文件可以直接读**。
|
|
289
|
+
|
|
290
|
+
---
|
|
291
|
+
|
|
292
|
+
## 从 0.3.0 迁移
|
|
293
|
+
|
|
294
|
+
1. 旧 `.fmg` 可以直接读,无需转换。但旧文件的 `level_dimensions` 是 `[h, w]`,且非正方形图像的 tile 本来就不完整——**建议用新版 `save_from_numpy` 重灌一次**以获得正确格式。
|
|
295
|
+
2. `level_dimensions` / `dimensions` 现在严格返回 `(width, height)`;`level_downsamples` 返回标量列表。
|
|
296
|
+
3. `level>0` 时 `location` 按 OpenSlide 语义解释为 level-0 坐标。若你的代码依赖旧行为(把 location 当 level 坐标),乘上 `level_downsamples[level]` 即可等价。
|
|
297
|
+
4. `read_region` 的签名是 `read_region(location, level, size)`,不是五个位置参数。
|
|
298
|
+
5. `map_size` 一般不用再指定:读侧自适应、写侧按数据量估算。
|
|
299
|
+
6. `spawn` 启动时记得在 `worker_init_fn` 里调 `fmg.configure(num_threads=0)`;`fork` 下自动处理。
|
|
300
|
+
7. 覆盖写一张正在被读的 `.fmg` 时,先 `close()` 所有读句柄——`save_from_numpy` 会删除并重建底层目录。
|
|
301
|
+
|
|
302
|
+
完整的根因分析、修复清单与迁移细节见 [`OPTIMIZATION.md`](https://github.com/khtao/fmg/blob/main/OPTIMIZATION.md)。
|
|
303
|
+
|
|
304
|
+
---
|
|
305
|
+
|
|
306
|
+
## 常见问题
|
|
307
|
+
|
|
308
|
+
**`patch_cache` 开多大?** 多 epoch 训练很值,单 epoch 顺序遍历全图命中率低。它是进程内的,不跨进程共享内存;显存紧张时给 `0` 也不会错。
|
|
309
|
+
|
|
310
|
+
**为什么 `spawn` 还要手动 `configure`?** 新进程里库检测不到「pid 变了」这件事(对象是被反序列化出来的,不是继承来的),所以 OpenCV 线程池需要你显式关掉。`fork` 路径下这一步是自动的。
|
|
311
|
+
|
|
312
|
+
**一个进程里开两个 slide 会冲突吗?** 不会,共享同一个 env。但别在还有读句柄时覆盖写同一个文件。
|
|
313
|
+
|
|
314
|
+
**`strict` 什么时候用?** 数据入库校验阶段打开它,坏 tile 立刻暴露;训练阶段保持默认 `False`,一个坏 patch 不该中断整轮训练。
|
|
315
|
+
|
|
316
|
+
**坏 tile 会静默吗?** 不会,第一次会发 `RuntimeWarning` 并打印 level 与 tile 坐标,之后只涂黑不刷屏。
|
|
317
|
+
|
|
318
|
+
---
|
|
319
|
+
|
|
320
|
+
## 验证
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
python selftest.py # 35 项检查:非方形图一致性、越界补黑、level>0 语义、
|
|
324
|
+
# pickle / spawn / fork 4 worker 并发、共享 env、
|
|
325
|
+
# JPEG 路径、0.3.0 旧文件兼容
|
|
326
|
+
python _bench.py # 0.3.0 / 0.4.0 读写对照基准
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
`selftest.py` 里有一组专门复现原崩溃场景的用例:**父进程已打开 LMDB,fork 出 4 个 worker 各读 250 次,父进程同时读 200 次**,每个 worker 都在自己的 pid 下重新持有 Environment。
|
|
330
|
+
|
|
331
|
+
---
|
|
332
|
+
|
|
333
|
+
## 参考
|
|
334
|
+
|
|
335
|
+
- LMDB Reader Lock Table / MDB_NOLOCK:<http://www.lmdb.tech/doc/group__readers.html>
|
|
336
|
+
- py-lmdb 并发与线程模型:<https://deepwiki.com/jnwatson/py-lmdb/4.3-concurrency-and-threading>
|
|
337
|
+
- `MDB_BAD_RSLOT` 多进程案例:<https://stackoverflow.com/questions/56905502/>
|
|
338
|
+
- PyTorch DataLoader + LMDB 懒打开:<https://blog.csdn.net/evilfromshadow/article/details/119537968>
|
|
339
|
+
|
|
340
|
+
原项目:<https://github.com/khtao/fmg> · 作者 khtao · MIT
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
fmg/__init__.py,sha256=AHEY-02IM3EQ_bcI_ueBTtpKITz2bl44gntfpkXISuQ,757
|
|
2
|
+
fmg/__version__.py,sha256=Hz9hqlnehve1l3B0uhkko4gq3haSC1MwFHMRBO6BJhE,63
|
|
3
|
+
fmg/core.py,sha256=0CVB3rgu4Ok1CSdSlkalkNdzCK4bFlMNut93D7QF2xU,30100
|
|
4
|
+
fmg-0.4.0.dist-info/licenses/LICENSE,sha256=jsr7I2LK-hm_tCDQCfjIsDol6l00fzz5yZaidMXCs0c,1062
|
|
5
|
+
fmg-0.4.0.dist-info/METADATA,sha256=-AsxtQpQAFLSDvxXHKXm49dVmU9lMyeRw2-lTMSBi30,15643
|
|
6
|
+
fmg-0.4.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
7
|
+
fmg-0.4.0.dist-info/top_level.txt,sha256=XM6f5rNKsWRNU5f-fIRMEGbCrEPSMbXm9VjpuJ8F-yA,4
|
|
8
|
+
fmg-0.4.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 khtao
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fmg
|