tmapslide 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tmapslide-0.2.1 → tmapslide-0.2.3}/PKG-INFO +37 -20
- {tmapslide-0.2.1 → tmapslide-0.2.3}/README.md +36 -19
- {tmapslide-0.2.1 → tmapslide-0.2.3}/pyproject.toml +1 -1
- {tmapslide-0.2.1 → tmapslide-0.2.3}/src/tmapslide/__init__.py +1 -1
- {tmapslide-0.2.1 → tmapslide-0.2.3}/src/tmapslide/_slide.py +68 -19
- {tmapslide-0.2.1 → tmapslide-0.2.3}/src/tmapslide/_tmapformat.py +174 -76
- {tmapslide-0.2.1 → tmapslide-0.2.3}/tests/test_synthetic.py +80 -0
- {tmapslide-0.2.1 → tmapslide-0.2.3}/tests/tmap_fixtures.py +26 -9
- {tmapslide-0.2.1 → tmapslide-0.2.3}/.gitignore +0 -0
- {tmapslide-0.2.1 → tmapslide-0.2.3}/LICENSE +0 -0
- {tmapslide-0.2.1 → tmapslide-0.2.3}/examples/read_region.py +0 -0
- {tmapslide-0.2.1 → tmapslide-0.2.3}/src/tmapslide/_cache.py +0 -0
- {tmapslide-0.2.1 → tmapslide-0.2.3}/src/tmapslide/_exceptions.py +0 -0
- {tmapslide-0.2.1 → tmapslide-0.2.3}/tests/test_basic.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tmapslide
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Pure Python UNIC TMAP whole-slide image reader with OpenSlide-compatible API
|
|
5
5
|
Project-URL: Homepage, https://github.com/yifanfeng97/tmapslide
|
|
6
6
|
Project-URL: Documentation, https://github.com/yifanfeng97/tmapslide#readme
|
|
@@ -31,11 +31,7 @@ Description-Content-Type: text/markdown
|
|
|
31
31
|
|
|
32
32
|
<div align="center">
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
<br/>
|
|
37
|
-
|
|
38
|
-
**TmapSlide**
|
|
34
|
+
# TmapSlide
|
|
39
35
|
|
|
40
36
|
**Pure Python reader for UNIC TMAP whole-slide images — no SDK, no native deps.**
|
|
41
37
|
|
|
@@ -54,9 +50,6 @@ Description-Content-Type: text/markdown
|
|
|
54
50
|
<a href="LICENSE">
|
|
55
51
|
<img src="https://img.shields.io/badge/license-MIT-06b6d4?style=for-the-badge&logo=openaccess&logoColor=white&labelColor=1a1a2e" alt="License">
|
|
56
52
|
</a>
|
|
57
|
-
<a href="https://github.com/yifanfeng97/tmapslide/actions/workflows/test.yml">
|
|
58
|
-
<img src="https://img.shields.io/github/actions/yifanfeng97/tmapslide/test.yml?branch=main&style=for-the-badge&logo=githubactions&logoColor=white&labelColor=1a1a2e&label=tests" alt="Tests">
|
|
59
|
-
</a>
|
|
60
53
|
<a href="https://github.com/yifanfeng97/tmapslide/stargazers">
|
|
61
54
|
<img src="https://img.shields.io/github/stars/yifanfeng97/tmapslide?style=for-the-badge&logo=github&labelColor=1a1a2e&color=facc15" alt="GitHub Stars">
|
|
62
55
|
</a>
|
|
@@ -64,6 +57,10 @@ Description-Content-Type: text/markdown
|
|
|
64
57
|
|
|
65
58
|
[📖 English](#-quick-start) · [中文说明](#-中文说明)
|
|
66
59
|
|
|
60
|
+
<br/>
|
|
61
|
+
|
|
62
|
+
<img src="docs/hero.jpg" alt="TmapSlide — UNIC TMAP whole-slide images in pure Python" width="800" style="max-width: 100%;">
|
|
63
|
+
|
|
67
64
|
</div>
|
|
68
65
|
|
|
69
66
|
## ⚡ Quick Start
|
|
@@ -99,6 +96,14 @@ macro = slide.associated_images["macro"]
|
|
|
99
96
|
`associated_images`
|
|
100
97
|
- **Both known TMAP variants** — `TMAP06` (3-level pyramid) and
|
|
101
98
|
`TMAP07` (up to 10 levels)
|
|
99
|
+
- **Multi-file slides** — TMAP06 slides that spill tiles into `.DT1`
|
|
100
|
+
sidecar files are read transparently
|
|
101
|
+
- **Trusted pixel size** — several TMAP06 *and* TMAP07 headers carry an
|
|
102
|
+
impossible `pixel_size` (e.g. 6.88e-05 mm at 40x, implying ~2 µm
|
|
103
|
+
nuclei); tmapslide cross-checks it against the objective power and
|
|
104
|
+
falls back to 10 µm / magnification, exposing the result via
|
|
105
|
+
`openslide.mpp-x` / `openslide.mpp-y` (the raw header value stays in
|
|
106
|
+
`tmap.pixel_size_mm`, the decision in `tmap.mpp_source`)
|
|
102
107
|
- **Fork-safe file handles** — safe with PyTorch `DataLoader` workers
|
|
103
108
|
- **LRU decoded-tile cache** — fast repeated reads
|
|
104
109
|
- **Thread-safe reads** — concurrent `read_region` from worker threads
|
|
@@ -110,14 +115,17 @@ TMAP backend (median of 5 runs, same files, same machine):
|
|
|
110
115
|
|
|
111
116
|
| Scenario | TMAP07 | TMAP06 |
|
|
112
117
|
|---|---|---|
|
|
113
|
-
| Open slide |
|
|
114
|
-
| Cold 1024² region @L0 | **
|
|
115
|
-
| Random 512² region @L0 | **1.
|
|
116
|
-
| Warm 512² region ×50 | **
|
|
117
|
-
|
|
118
|
-
>
|
|
119
|
-
>
|
|
120
|
-
>
|
|
118
|
+
| Open slide | 68 ms vs 46 ms | **80 ms vs 220 ms (2.8×)** |
|
|
119
|
+
| Cold 1024² region @L0 | **3.6 ms vs 15.7 ms (4.4×)** | **2.8 ms vs 12.3 ms (4.4×)** |
|
|
120
|
+
| Random 512² region @L0 | **1.0 ms vs 5.2 ms (5.0×)** | **1.0 ms vs 5.6 ms (5.6×)** |
|
|
121
|
+
| Warm 512² region ×50 | **45 ms vs 242 ms (5.4×)** | **55 ms vs 373 ms (6.8×)** |
|
|
122
|
+
|
|
123
|
+
> The two libraries pay for indexing in opposite places. tmapslide builds a
|
|
124
|
+
> validated tile index (grid + lazy tile tables) once at open; ASlide defers
|
|
125
|
+
> that work into every read, re-decoding tiles per call. That is why TMAP07
|
|
126
|
+
> open runs ~20 ms slower while every read is 4–7× faster — for any workload
|
|
127
|
+
> that reads more than a handful of regions, tmapslide comes out far ahead.
|
|
128
|
+
> Slide open is a once-per-process cost.
|
|
121
129
|
|
|
122
130
|
## 📖 API
|
|
123
131
|
|
|
@@ -129,11 +137,12 @@ TMAP backend (median of 5 runs, same files, same machine):
|
|
|
129
137
|
| `level_count` | number of pyramid levels |
|
|
130
138
|
| `level_dimensions` | `(w, h)` per level |
|
|
131
139
|
| `level_downsamples` | downsample factor per level |
|
|
132
|
-
| `properties` | read-only metadata mapping (`openslide.vendor=unic`, `tmap.*`) |
|
|
140
|
+
| `properties` | read-only metadata mapping (`openslide.vendor=unic`, `openslide.mpp-x/y`, `tmap.*`) |
|
|
133
141
|
| `associated_images` | lazy mapping, typically `macro` / `label` / `thumbnail` |
|
|
134
142
|
| `read_region(loc, level, size)` | `PIL.Image` (RGBA) of the region |
|
|
135
|
-
| `get_thumbnail(size)` | thumbnail
|
|
143
|
+
| `get_thumbnail(size)` | stored thumbnail when available, else lowest level |
|
|
136
144
|
| `get_best_level_for_downsample(ds)` | best level for a downsample factor |
|
|
145
|
+
| `iter_tiles(level=0)` | yields `(x, y, load)` per stored tile |
|
|
137
146
|
| `close()` / context manager | release resources |
|
|
138
147
|
|
|
139
148
|
### `tmapslide.open_slide(filename)`
|
|
@@ -144,7 +153,7 @@ Alias of `OpenSlide(filename)`.
|
|
|
144
153
|
|
|
145
154
|
| Format | Extension | Vendor | Backend |
|
|
146
155
|
| --- | --- | --- | --- |
|
|
147
|
-
| TMAP 06 | `.TMAP` | UNIC (United Imaging) | Pure Python |
|
|
156
|
+
| TMAP 06 | `.TMAP` (+ optional `.DT1` sidecars) | UNIC (United Imaging) | Pure Python |
|
|
148
157
|
| TMAP 07 | `.TMAP` | UNIC (United Imaging) | Pure Python |
|
|
149
158
|
|
|
150
159
|
## 🧪 Testing
|
|
@@ -172,6 +181,14 @@ encryption.
|
|
|
172
181
|
are JPEG-compressed at that scale.
|
|
173
182
|
- `iter_tiles()` exposes the stored tile grid directly — useful for
|
|
174
183
|
tile-based ML pipelines.
|
|
184
|
+
- **Metadata caveat**: the header `pixel_size` field cannot be trusted.
|
|
185
|
+
Measured on real slides (cell-nucleus diameters, canvas physical size,
|
|
186
|
+
cross-checked with ASlide), both the Henan TMAP06 batch (6.88e-05 mm)
|
|
187
|
+
and the Shanxi TMAP07 batch (1.01e-04 mm) carry corrupt values at 40x;
|
|
188
|
+
the true resolution is 0.25 µm/px. tmapslide keeps the raw value in
|
|
189
|
+
`tmap.pixel_size_mm`, exposes the corrected one via
|
|
190
|
+
`openslide.mpp-x/y`, and records the decision in `tmap.mpp_source`
|
|
191
|
+
(`header` / `derived`).
|
|
175
192
|
|
|
176
193
|
## 📄 License
|
|
177
194
|
|
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
<br/>
|
|
6
|
-
|
|
7
|
-
**TmapSlide**
|
|
3
|
+
# TmapSlide
|
|
8
4
|
|
|
9
5
|
**Pure Python reader for UNIC TMAP whole-slide images — no SDK, no native deps.**
|
|
10
6
|
|
|
@@ -23,9 +19,6 @@
|
|
|
23
19
|
<a href="LICENSE">
|
|
24
20
|
<img src="https://img.shields.io/badge/license-MIT-06b6d4?style=for-the-badge&logo=openaccess&logoColor=white&labelColor=1a1a2e" alt="License">
|
|
25
21
|
</a>
|
|
26
|
-
<a href="https://github.com/yifanfeng97/tmapslide/actions/workflows/test.yml">
|
|
27
|
-
<img src="https://img.shields.io/github/actions/yifanfeng97/tmapslide/test.yml?branch=main&style=for-the-badge&logo=githubactions&logoColor=white&labelColor=1a1a2e&label=tests" alt="Tests">
|
|
28
|
-
</a>
|
|
29
22
|
<a href="https://github.com/yifanfeng97/tmapslide/stargazers">
|
|
30
23
|
<img src="https://img.shields.io/github/stars/yifanfeng97/tmapslide?style=for-the-badge&logo=github&labelColor=1a1a2e&color=facc15" alt="GitHub Stars">
|
|
31
24
|
</a>
|
|
@@ -33,6 +26,10 @@
|
|
|
33
26
|
|
|
34
27
|
[📖 English](#-quick-start) · [中文说明](#-中文说明)
|
|
35
28
|
|
|
29
|
+
<br/>
|
|
30
|
+
|
|
31
|
+
<img src="docs/hero.jpg" alt="TmapSlide — UNIC TMAP whole-slide images in pure Python" width="800" style="max-width: 100%;">
|
|
32
|
+
|
|
36
33
|
</div>
|
|
37
34
|
|
|
38
35
|
## ⚡ Quick Start
|
|
@@ -68,6 +65,14 @@ macro = slide.associated_images["macro"]
|
|
|
68
65
|
`associated_images`
|
|
69
66
|
- **Both known TMAP variants** — `TMAP06` (3-level pyramid) and
|
|
70
67
|
`TMAP07` (up to 10 levels)
|
|
68
|
+
- **Multi-file slides** — TMAP06 slides that spill tiles into `.DT1`
|
|
69
|
+
sidecar files are read transparently
|
|
70
|
+
- **Trusted pixel size** — several TMAP06 *and* TMAP07 headers carry an
|
|
71
|
+
impossible `pixel_size` (e.g. 6.88e-05 mm at 40x, implying ~2 µm
|
|
72
|
+
nuclei); tmapslide cross-checks it against the objective power and
|
|
73
|
+
falls back to 10 µm / magnification, exposing the result via
|
|
74
|
+
`openslide.mpp-x` / `openslide.mpp-y` (the raw header value stays in
|
|
75
|
+
`tmap.pixel_size_mm`, the decision in `tmap.mpp_source`)
|
|
71
76
|
- **Fork-safe file handles** — safe with PyTorch `DataLoader` workers
|
|
72
77
|
- **LRU decoded-tile cache** — fast repeated reads
|
|
73
78
|
- **Thread-safe reads** — concurrent `read_region` from worker threads
|
|
@@ -79,14 +84,17 @@ TMAP backend (median of 5 runs, same files, same machine):
|
|
|
79
84
|
|
|
80
85
|
| Scenario | TMAP07 | TMAP06 |
|
|
81
86
|
|---|---|---|
|
|
82
|
-
| Open slide |
|
|
83
|
-
| Cold 1024² region @L0 | **
|
|
84
|
-
| Random 512² region @L0 | **1.
|
|
85
|
-
| Warm 512² region ×50 | **
|
|
86
|
-
|
|
87
|
-
>
|
|
88
|
-
>
|
|
89
|
-
>
|
|
87
|
+
| Open slide | 68 ms vs 46 ms | **80 ms vs 220 ms (2.8×)** |
|
|
88
|
+
| Cold 1024² region @L0 | **3.6 ms vs 15.7 ms (4.4×)** | **2.8 ms vs 12.3 ms (4.4×)** |
|
|
89
|
+
| Random 512² region @L0 | **1.0 ms vs 5.2 ms (5.0×)** | **1.0 ms vs 5.6 ms (5.6×)** |
|
|
90
|
+
| Warm 512² region ×50 | **45 ms vs 242 ms (5.4×)** | **55 ms vs 373 ms (6.8×)** |
|
|
91
|
+
|
|
92
|
+
> The two libraries pay for indexing in opposite places. tmapslide builds a
|
|
93
|
+
> validated tile index (grid + lazy tile tables) once at open; ASlide defers
|
|
94
|
+
> that work into every read, re-decoding tiles per call. That is why TMAP07
|
|
95
|
+
> open runs ~20 ms slower while every read is 4–7× faster — for any workload
|
|
96
|
+
> that reads more than a handful of regions, tmapslide comes out far ahead.
|
|
97
|
+
> Slide open is a once-per-process cost.
|
|
90
98
|
|
|
91
99
|
## 📖 API
|
|
92
100
|
|
|
@@ -98,11 +106,12 @@ TMAP backend (median of 5 runs, same files, same machine):
|
|
|
98
106
|
| `level_count` | number of pyramid levels |
|
|
99
107
|
| `level_dimensions` | `(w, h)` per level |
|
|
100
108
|
| `level_downsamples` | downsample factor per level |
|
|
101
|
-
| `properties` | read-only metadata mapping (`openslide.vendor=unic`, `tmap.*`) |
|
|
109
|
+
| `properties` | read-only metadata mapping (`openslide.vendor=unic`, `openslide.mpp-x/y`, `tmap.*`) |
|
|
102
110
|
| `associated_images` | lazy mapping, typically `macro` / `label` / `thumbnail` |
|
|
103
111
|
| `read_region(loc, level, size)` | `PIL.Image` (RGBA) of the region |
|
|
104
|
-
| `get_thumbnail(size)` | thumbnail
|
|
112
|
+
| `get_thumbnail(size)` | stored thumbnail when available, else lowest level |
|
|
105
113
|
| `get_best_level_for_downsample(ds)` | best level for a downsample factor |
|
|
114
|
+
| `iter_tiles(level=0)` | yields `(x, y, load)` per stored tile |
|
|
106
115
|
| `close()` / context manager | release resources |
|
|
107
116
|
|
|
108
117
|
### `tmapslide.open_slide(filename)`
|
|
@@ -113,7 +122,7 @@ Alias of `OpenSlide(filename)`.
|
|
|
113
122
|
|
|
114
123
|
| Format | Extension | Vendor | Backend |
|
|
115
124
|
| --- | --- | --- | --- |
|
|
116
|
-
| TMAP 06 | `.TMAP` | UNIC (United Imaging) | Pure Python |
|
|
125
|
+
| TMAP 06 | `.TMAP` (+ optional `.DT1` sidecars) | UNIC (United Imaging) | Pure Python |
|
|
117
126
|
| TMAP 07 | `.TMAP` | UNIC (United Imaging) | Pure Python |
|
|
118
127
|
|
|
119
128
|
## 🧪 Testing
|
|
@@ -141,6 +150,14 @@ encryption.
|
|
|
141
150
|
are JPEG-compressed at that scale.
|
|
142
151
|
- `iter_tiles()` exposes the stored tile grid directly — useful for
|
|
143
152
|
tile-based ML pipelines.
|
|
153
|
+
- **Metadata caveat**: the header `pixel_size` field cannot be trusted.
|
|
154
|
+
Measured on real slides (cell-nucleus diameters, canvas physical size,
|
|
155
|
+
cross-checked with ASlide), both the Henan TMAP06 batch (6.88e-05 mm)
|
|
156
|
+
and the Shanxi TMAP07 batch (1.01e-04 mm) carry corrupt values at 40x;
|
|
157
|
+
the true resolution is 0.25 µm/px. tmapslide keeps the raw value in
|
|
158
|
+
`tmap.pixel_size_mm`, exposes the corrected one via
|
|
159
|
+
`openslide.mpp-x/y`, and records the decision in `tmap.mpp_source`
|
|
160
|
+
(`header` / `derived`).
|
|
144
161
|
|
|
145
162
|
## 📄 License
|
|
146
163
|
|
|
@@ -114,9 +114,10 @@ class OpenSlide:
|
|
|
114
114
|
"_info",
|
|
115
115
|
"_properties",
|
|
116
116
|
"_associated_images",
|
|
117
|
-
"_tile_lookup",
|
|
118
117
|
"_background_rgba",
|
|
119
118
|
"_io_lock",
|
|
119
|
+
"_data_paths",
|
|
120
|
+
"_data_handles",
|
|
120
121
|
)
|
|
121
122
|
|
|
122
123
|
def __init__(self, filename: str):
|
|
@@ -127,18 +128,15 @@ class OpenSlide:
|
|
|
127
128
|
self._pid = os.getpid()
|
|
128
129
|
# Serialises seek+read on the shared file handle across threads.
|
|
129
130
|
self._io_lock = threading.Lock()
|
|
131
|
+
# Auxiliary .DT1/.DT2 data files (TMAP06 multi-file slides).
|
|
132
|
+
self._data_paths: Dict[int, str] = {}
|
|
133
|
+
self._data_handles: Dict[int, io.BufferedReader] = {}
|
|
130
134
|
|
|
131
135
|
try:
|
|
132
136
|
self._info: TmapFileInfo = parse_tmap_file(filename)
|
|
133
137
|
except Exception as e:
|
|
134
138
|
raise OpenSlideUnsupportedFormatError(f"Cannot parse TMAP file: {e}")
|
|
135
139
|
|
|
136
|
-
# (level, x, y) -> tile index, for grid lookup in read_region.
|
|
137
|
-
self._tile_lookup: Dict[Tuple[int, int, int], int] = {}
|
|
138
|
-
for level in self._info.levels:
|
|
139
|
-
for idx, t in enumerate(level.tiles):
|
|
140
|
-
self._tile_lookup[(level.index, t.x, t.y)] = idx
|
|
141
|
-
|
|
142
140
|
try:
|
|
143
141
|
self._file_handle = open(filename, "rb")
|
|
144
142
|
except Exception as e:
|
|
@@ -151,6 +149,7 @@ class OpenSlide:
|
|
|
151
149
|
}
|
|
152
150
|
props.update(self._info.properties)
|
|
153
151
|
self._properties = _PropertyMap(props)
|
|
152
|
+
self._data_paths = dict(self._info.data_files)
|
|
154
153
|
if "tmap.objective_power" in self._info.properties:
|
|
155
154
|
self._properties._data["openslide.objective-power"] = self._info.properties[
|
|
156
155
|
"tmap.objective_power"
|
|
@@ -217,8 +216,12 @@ class OpenSlide:
|
|
|
217
216
|
tile = self._tile_cache.get(key)
|
|
218
217
|
if tile is not None:
|
|
219
218
|
return tile
|
|
220
|
-
fh = self._ensure_open_handle()
|
|
221
219
|
t = level.tiles[idx]
|
|
220
|
+
fh = self._get_file_handle(t.file_id)
|
|
221
|
+
if fh is None:
|
|
222
|
+
raise OpenSlideError(
|
|
223
|
+
f"Data file for file_id {t.file_id} is missing"
|
|
224
|
+
)
|
|
222
225
|
with self._io_lock:
|
|
223
226
|
fh.seek(t.offset)
|
|
224
227
|
jpeg = fh.read(t.size)
|
|
@@ -226,13 +229,42 @@ class OpenSlide:
|
|
|
226
229
|
self._tile_cache.put(key, tile)
|
|
227
230
|
return tile
|
|
228
231
|
|
|
229
|
-
def
|
|
232
|
+
def _get_file_handle(self, file_id: int):
|
|
233
|
+
"""Return the handle for a data file: 0 = main TMAP, 1+ = .DT sidecars.
|
|
234
|
+
|
|
235
|
+
Sidecar handles open lazily and reopen when the PID changes (same
|
|
236
|
+
fork-safety rule as the main handle). Returns None if the sidecar
|
|
237
|
+
file is missing."""
|
|
238
|
+
self._check_open()
|
|
239
|
+
if file_id == 0:
|
|
240
|
+
return self._ensure_open_handle()
|
|
241
|
+
path = self._data_paths.get(file_id)
|
|
242
|
+
if path is None:
|
|
243
|
+
return None
|
|
244
|
+
handle = self._data_handles.get(file_id)
|
|
245
|
+
if handle is None or os.getpid() != getattr(handle, "_tmap_pid", -1):
|
|
246
|
+
try:
|
|
247
|
+
handle.close()
|
|
248
|
+
except Exception:
|
|
249
|
+
pass
|
|
250
|
+
try:
|
|
251
|
+
handle = open(path, "rb")
|
|
252
|
+
except OSError:
|
|
253
|
+
self._data_paths.pop(file_id, None)
|
|
254
|
+
return None
|
|
255
|
+
handle._tmap_pid = os.getpid()
|
|
256
|
+
self._data_handles[file_id] = handle
|
|
257
|
+
return handle
|
|
258
|
+
|
|
259
|
+
def _get_cached_tile(self, offset: int, length: int, file_id: int = 0) -> Image.Image:
|
|
230
260
|
"""Read + decode a raw tile by file range, with caching."""
|
|
231
|
-
key = ("raw", offset, length)
|
|
261
|
+
key = ("raw", file_id, offset, length)
|
|
232
262
|
tile = self._tile_cache.get(key)
|
|
233
263
|
if tile is not None:
|
|
234
264
|
return tile
|
|
235
|
-
fh = self.
|
|
265
|
+
fh = self._get_file_handle(file_id)
|
|
266
|
+
if fh is None:
|
|
267
|
+
raise OpenSlideError(f"Data file for file_id {file_id} is missing")
|
|
236
268
|
with self._io_lock:
|
|
237
269
|
fh.seek(offset)
|
|
238
270
|
data = fh.read(length)
|
|
@@ -260,6 +292,12 @@ class OpenSlide:
|
|
|
260
292
|
except Exception:
|
|
261
293
|
pass
|
|
262
294
|
self._file_handle = None
|
|
295
|
+
for handle in self._data_handles.values():
|
|
296
|
+
try:
|
|
297
|
+
handle.close()
|
|
298
|
+
except Exception:
|
|
299
|
+
pass
|
|
300
|
+
self._data_handles.clear()
|
|
263
301
|
|
|
264
302
|
# ------------------------------------------------------------------
|
|
265
303
|
# Properties
|
|
@@ -371,13 +409,22 @@ class OpenSlide:
|
|
|
371
409
|
|
|
372
410
|
self._ensure_open_handle()
|
|
373
411
|
|
|
412
|
+
grid = lvl.grid
|
|
413
|
+
gcols = lvl.grid_cols
|
|
414
|
+
glen = len(grid) if grid is not None else 0
|
|
415
|
+
|
|
374
416
|
for ty in range(ty_start, ty_end):
|
|
417
|
+
tile_y = ty * th
|
|
375
418
|
for tx in range(tx_start, tx_end):
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
if
|
|
419
|
+
if grid is None or tx < 0:
|
|
420
|
+
continue
|
|
421
|
+
cell = ty * gcols + tx
|
|
422
|
+
if cell >= glen:
|
|
380
423
|
continue
|
|
424
|
+
idx = grid[cell]
|
|
425
|
+
if idx < 0:
|
|
426
|
+
continue
|
|
427
|
+
tile_x = tx * tw
|
|
381
428
|
tile = self._read_decoded_tile(lvl, idx)
|
|
382
429
|
crop_x0 = max(0, x0 - tile_x)
|
|
383
430
|
crop_y0 = max(0, y0 - tile_y)
|
|
@@ -409,7 +456,6 @@ class OpenSlide:
|
|
|
409
456
|
img_w = int(props.get("tmap.img_width", 2448))
|
|
410
457
|
img_h = int(props.get("tmap.img_height", 2048))
|
|
411
458
|
tile_img_w, tile_img_h = info.__dict__.get("_tile_jpeg_size", (612, 512))
|
|
412
|
-
fh = self._ensure_open_handle()
|
|
413
459
|
|
|
414
460
|
# clamp level to what we can render
|
|
415
461
|
if shrinks:
|
|
@@ -463,11 +509,10 @@ class OpenSlide:
|
|
|
463
509
|
|
|
464
510
|
if level >= 2 and shrinks:
|
|
465
511
|
# ShrinkTiles: pre-rendered tiles for deep zoom-out levels.
|
|
466
|
-
for layer_no, n_x, n_y, off, length in shrinks:
|
|
512
|
+
for layer_no, n_x, n_y, off, length, fid in shrinks:
|
|
467
513
|
if layer_no != level:
|
|
468
514
|
continue
|
|
469
515
|
scale = ratio_step**layer_no
|
|
470
|
-
tile_img = self._get_cached_tile(off, length)
|
|
471
516
|
cover_w = info.__dict__["_tile_jpeg_size"][0] * scale
|
|
472
517
|
cover_h = info.__dict__["_tile_jpeg_size"][1] * scale
|
|
473
518
|
if (
|
|
@@ -477,6 +522,7 @@ class OpenSlide:
|
|
|
477
522
|
or n_y > y0_l0 + h0
|
|
478
523
|
):
|
|
479
524
|
continue
|
|
525
|
+
tile_img = self._get_cached_tile(off, length, fid)
|
|
480
526
|
paste_scaled(tile_img, n_x, n_y, cover_w, cover_h)
|
|
481
527
|
return out
|
|
482
528
|
|
|
@@ -489,6 +535,9 @@ class OpenSlide:
|
|
|
489
535
|
if n_x + img_w < x0_l0 or n_x > x0_l0 + w0 or n_y + img_h < y0_l0 or n_y > y0_l0 + h0:
|
|
490
536
|
continue
|
|
491
537
|
jw, jh = info.__dict__.get("_tile_jpeg_size", (tile_img_w, tile_img_h))
|
|
538
|
+
fid = blk.get("file_id", 0)
|
|
539
|
+
if self._get_file_handle(fid) is None:
|
|
540
|
+
continue # sidecar file missing; degrade to background
|
|
492
541
|
for layer_no, col, row, off, length in blk["tiles"]:
|
|
493
542
|
if layer_no != level:
|
|
494
543
|
continue
|
|
@@ -508,7 +557,7 @@ class OpenSlide:
|
|
|
508
557
|
or tile_y0 >= y0_l0 + h0
|
|
509
558
|
):
|
|
510
559
|
continue
|
|
511
|
-
tile_img = self._get_cached_tile(off, length)
|
|
560
|
+
tile_img = self._get_cached_tile(off, length, fid)
|
|
512
561
|
paste_scaled(tile_img, tile_x0, tile_y0, cover_w, cover_h)
|
|
513
562
|
return out
|
|
514
563
|
|
|
@@ -39,7 +39,9 @@ TMAP06 layout
|
|
|
39
39
|
"""
|
|
40
40
|
|
|
41
41
|
import io
|
|
42
|
+
import os
|
|
42
43
|
import struct
|
|
44
|
+
from collections.abc import Sequence
|
|
43
45
|
from dataclasses import dataclass, field
|
|
44
46
|
from typing import Dict, List, Optional, Tuple
|
|
45
47
|
|
|
@@ -59,9 +61,14 @@ REC07_SIZE = 40
|
|
|
59
61
|
REC07_HEADER_SKIP = 24
|
|
60
62
|
|
|
61
63
|
|
|
62
|
-
@dataclass
|
|
64
|
+
@dataclass(slots=True)
|
|
63
65
|
class TmapTile:
|
|
64
|
-
"""One pyramid tile pointing at a JPEG blob in
|
|
66
|
+
"""One pyramid tile pointing at a JPEG blob in a data file.
|
|
67
|
+
|
|
68
|
+
``file_id`` selects the file: 0 is the main TMAP, 1+ are .DT sidecar
|
|
69
|
+
files (TMAP06 multi-file slides). Slotted for fast bulk construction:
|
|
70
|
+
TMAP07 slides materialize 100k+ tiles at open time.
|
|
71
|
+
"""
|
|
65
72
|
|
|
66
73
|
level: int
|
|
67
74
|
offset: int
|
|
@@ -70,6 +77,47 @@ class TmapTile:
|
|
|
70
77
|
y: int
|
|
71
78
|
width: int
|
|
72
79
|
height: int
|
|
80
|
+
file_id: int = 0
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class _TileList(Sequence):
|
|
84
|
+
"""Lazily materializes TmapTile objects from a level's raw record table.
|
|
85
|
+
|
|
86
|
+
A TMAP07 level-0 table holds 100k+ records; constructing every tile
|
|
87
|
+
up front dominated slide-open time. The record table is kept as raw
|
|
88
|
+
bytes and a tile is unpacked on first access. Public behavior is
|
|
89
|
+
identical to a list of TmapTile.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
__slots__ = ("_level", "_raw", "_rec_idx", "_tw", "_th")
|
|
93
|
+
|
|
94
|
+
def __init__(self, level: int, raw: bytes, rec_idx: List[int], tw: int, th: int):
|
|
95
|
+
self._level = level
|
|
96
|
+
self._raw = raw
|
|
97
|
+
self._rec_idx = rec_idx
|
|
98
|
+
self._tw = tw
|
|
99
|
+
self._th = th
|
|
100
|
+
|
|
101
|
+
def __len__(self) -> int:
|
|
102
|
+
return len(self._rec_idx)
|
|
103
|
+
|
|
104
|
+
def __getitem__(self, i):
|
|
105
|
+
if isinstance(i, slice):
|
|
106
|
+
return [self[j] for j in range(*i.indices(len(self)))]
|
|
107
|
+
base = self._rec_idx[i] * TMAP07_REC_SIZE
|
|
108
|
+
_layer, _focus, x, y, _w, _h, off, size = _REC07.unpack_from(self._raw, base)
|
|
109
|
+
return TmapTile(self._level, off, size, x, y, self._tw, self._th)
|
|
110
|
+
|
|
111
|
+
def __iter__(self):
|
|
112
|
+
unpack = _REC07.unpack_from
|
|
113
|
+
raw = self._raw
|
|
114
|
+
level, tw, th = self._level, self._tw, self._th
|
|
115
|
+
for i in self._rec_idx:
|
|
116
|
+
_layer, _focus, x, y, _w, _h, off, size = unpack(raw, i * TMAP07_REC_SIZE)
|
|
117
|
+
yield TmapTile(level, off, size, x, y, tw, th)
|
|
118
|
+
|
|
119
|
+
def __repr__(self) -> str:
|
|
120
|
+
return f"_TileList(len={len(self._rec_idx)})"
|
|
73
121
|
|
|
74
122
|
|
|
75
123
|
@dataclass
|
|
@@ -82,7 +130,12 @@ class TmapLevel:
|
|
|
82
130
|
height: int
|
|
83
131
|
tile_width: int
|
|
84
132
|
tile_height: int
|
|
85
|
-
tiles:
|
|
133
|
+
tiles: Sequence[TmapTile] = field(default_factory=list)
|
|
134
|
+
# TMAP07: map from grid cell (row * grid_cols + col) to the index of
|
|
135
|
+
# that tile in `tiles`, or -1 for blank cells. Replaces a hash lookup
|
|
136
|
+
# at read time; None for TMAP06 levels (block-based geometry).
|
|
137
|
+
grid: Optional[List[int]] = None
|
|
138
|
+
grid_cols: int = 0
|
|
86
139
|
|
|
87
140
|
|
|
88
141
|
@dataclass
|
|
@@ -104,6 +157,44 @@ class TmapFileInfo:
|
|
|
104
157
|
assoc_images: List[TmapAssocImage]
|
|
105
158
|
tile_count: int
|
|
106
159
|
properties: Dict[str, str] = field(default_factory=dict)
|
|
160
|
+
# file_id -> path of auxiliary data files (0 is the main TMAP itself).
|
|
161
|
+
data_files: Dict[int, str] = field(default_factory=dict)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# Header pixel_size fields are unreliable in the wild (TMAP06 slides ship
|
|
165
|
+
# with values like 6.88e-05 mm at 40x, implying biologically impossible
|
|
166
|
+
# ~2 µm nuclei), so derived values are used whenever the header value is
|
|
167
|
+
# outside the plausible mpp range for whole-slide imaging. 0.1 µm/px is
|
|
168
|
+
# the practical floor (100x); no light-microscopy WSI scanner goes finer.
|
|
169
|
+
MPP_MIN, MPP_MAX = 0.1, 5.0
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _mpp_properties(pixel_size_mm: float, scan_scale: float) -> Dict[str, str]:
|
|
173
|
+
"""openslide.mpp-x/y with plausibility checks and a documented fallback.
|
|
174
|
+
|
|
175
|
+
Two known-corrupt cases in real files: (1) TMAP06 headers carrying
|
|
176
|
+
6.88e-05 mm at 40x (biologically impossible ~2 µm nuclei), and
|
|
177
|
+
(2) TMAP07 headers carrying 1.01e-04 mm at 40x (implies a ~99x
|
|
178
|
+
objective that no WSI scanner has, and canvas physical sizes of 6-9 mm
|
|
179
|
+
instead of the standard 21-24 mm). A header value is trusted only if it
|
|
180
|
+
is inside the plausible mpp range AND within 25% of the 10 µm /
|
|
181
|
+
objective-power physics; otherwise 10 µm / objective is used. The raw
|
|
182
|
+
header value is always preserved in ``tmap.pixel_size_mm`` and
|
|
183
|
+
``tmap.mpp_source`` records which value won.
|
|
184
|
+
"""
|
|
185
|
+
mpp = pixel_size_mm * 1000.0
|
|
186
|
+
source = "header"
|
|
187
|
+
expected = 10.0 / scan_scale if scan_scale >= 4 else 0.0
|
|
188
|
+
plausible = MPP_MIN <= mpp <= MPP_MAX
|
|
189
|
+
consistent = expected <= 0 or abs(mpp - expected) / expected <= 0.25
|
|
190
|
+
if not (plausible and consistent):
|
|
191
|
+
mpp = expected if MPP_MIN <= expected <= MPP_MAX else 0.25
|
|
192
|
+
source = "derived"
|
|
193
|
+
return {
|
|
194
|
+
"openslide.mpp-x": f"{mpp:g}",
|
|
195
|
+
"openslide.mpp-y": f"{mpp:g}",
|
|
196
|
+
"tmap.mpp_source": source,
|
|
197
|
+
}
|
|
107
198
|
|
|
108
199
|
|
|
109
200
|
def _read_at(f, offset: int, size: int) -> bytes:
|
|
@@ -145,19 +236,6 @@ _ASSOC07_NAMES = {
|
|
|
145
236
|
}
|
|
146
237
|
|
|
147
238
|
|
|
148
|
-
def _find_tile_data_start07(f, file_size: int) -> int:
|
|
149
|
-
"""First JPEG SOI at/after the record region -> start of tile data."""
|
|
150
|
-
pos = 0
|
|
151
|
-
step = 1024 * 1024
|
|
152
|
-
while pos < file_size:
|
|
153
|
-
chunk = _read_at(f, pos, min(step, file_size - pos))
|
|
154
|
-
p = chunk.find(b"\xff\xd8\xff")
|
|
155
|
-
if p != -1:
|
|
156
|
-
return pos + p
|
|
157
|
-
pos += len(chunk)
|
|
158
|
-
raise ValueError("TMAP07: no tile data found")
|
|
159
|
-
|
|
160
|
-
|
|
161
239
|
def _parse_assoc07(f, file_size: int, tile_data_end: int) -> List[TmapAssocImage]:
|
|
162
240
|
"""Collect TMAP07 associated images.
|
|
163
241
|
|
|
@@ -240,32 +318,7 @@ def _parse_layer_table07(f, base: int = TMAP07_LAYERINFO_BASE) -> List[dict]:
|
|
|
240
318
|
return entries
|
|
241
319
|
|
|
242
320
|
|
|
243
|
-
|
|
244
|
-
"""Read one level's tile table.
|
|
245
|
-
|
|
246
|
-
Record (40 B), indexed by (row * tile_col + col):
|
|
247
|
-
layer_no u32, focus_no u32, x u32, y u32, w u32, h u32,
|
|
248
|
-
data_offset u64, data_length u32, pad u32
|
|
249
|
-
Blank cells store offset == 0.
|
|
250
|
-
"""
|
|
251
|
-
cols, rows = entry["tile_col"], entry["tile_row"]
|
|
252
|
-
total = cols * rows
|
|
253
|
-
raw = _read_at(f, entry["index_offset"], total * TMAP07_REC_SIZE + 64)
|
|
254
|
-
recs: List[Optional[tuple]] = []
|
|
255
|
-
for i in range(total):
|
|
256
|
-
base = i * TMAP07_REC_SIZE
|
|
257
|
-
if base + TMAP07_REC_SIZE > len(raw):
|
|
258
|
-
break
|
|
259
|
-
layer_no, focus_no, x, y, w, h, off, size = struct.unpack_from(
|
|
260
|
-
"<IIIIIIQI", raw, base
|
|
261
|
-
)
|
|
262
|
-
if off == 0 and size == 0:
|
|
263
|
-
recs.append(None)
|
|
264
|
-
continue
|
|
265
|
-
if size == 0 or off + size > file_size:
|
|
266
|
-
break
|
|
267
|
-
recs.append((off, size, layer_no, focus_no, x, y, w, h))
|
|
268
|
-
return recs
|
|
321
|
+
_REC07 = struct.Struct("<IIIIIIQI") # layer_no, focus_no, x, y, w, h, off u64, size
|
|
269
322
|
|
|
270
323
|
|
|
271
324
|
def _parse_tmap07(path: str) -> TmapFileInfo:
|
|
@@ -283,8 +336,6 @@ def _parse_tmap07(path: str) -> TmapFileInfo:
|
|
|
283
336
|
declared_tiles = struct.unpack_from("<I", head, 0x1C)[0]
|
|
284
337
|
magic = head[:6].decode("ascii", errors="replace")
|
|
285
338
|
|
|
286
|
-
data_start = _find_tile_data_start07(f, file_size)
|
|
287
|
-
|
|
288
339
|
# TMAP07 LayerInfo table at 560.
|
|
289
340
|
entries = _parse_layer_table07(f)
|
|
290
341
|
if not entries:
|
|
@@ -292,32 +343,55 @@ def _parse_tmap07(path: str) -> TmapFileInfo:
|
|
|
292
343
|
|
|
293
344
|
levels: List[TmapLevel] = []
|
|
294
345
|
prop_layer_info = []
|
|
346
|
+
tile_data_end = 0
|
|
295
347
|
for order, e in enumerate(entries):
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
348
|
+
# Single pass over the record table: validate records, fix the
|
|
349
|
+
# level tile size from the first stored record, build the
|
|
350
|
+
# grid-position index, and keep the table as raw bytes — tile
|
|
351
|
+
# objects are materialized lazily by _TileList (100k+ tiles on
|
|
352
|
+
# level 0 would otherwise dominate slide-open time).
|
|
353
|
+
cols = e["tile_col"]
|
|
354
|
+
total = cols * e["tile_row"]
|
|
355
|
+
raw = _read_at(f, e["index_offset"], total * TMAP07_REC_SIZE + 64)
|
|
356
|
+
n = min(total, len(raw) // TMAP07_REC_SIZE)
|
|
357
|
+
unpack = _REC07.unpack_from
|
|
358
|
+
grid = [-1] * total
|
|
359
|
+
rec_idx: List[int] = []
|
|
360
|
+
ri_append = rec_idx.append
|
|
361
|
+
tw = th = None
|
|
362
|
+
level_end = 0
|
|
363
|
+
pos = 0
|
|
364
|
+
for i in range(n):
|
|
365
|
+
_layer, _focus, x, y, w, h, off, size = unpack(raw, i * TMAP07_REC_SIZE)
|
|
366
|
+
if off == 0 and size == 0:
|
|
367
|
+
continue
|
|
368
|
+
if size == 0 or off + size > file_size:
|
|
369
|
+
break
|
|
370
|
+
if tw is None:
|
|
371
|
+
tw = w if 16 <= w <= 4096 else 256
|
|
372
|
+
th = h if 16 <= h <= 4096 else 256
|
|
373
|
+
grid[i] = pos
|
|
374
|
+
pos += 1
|
|
375
|
+
ri_append(i)
|
|
376
|
+
if off + size > level_end:
|
|
377
|
+
level_end = off + size
|
|
378
|
+
if not rec_idx:
|
|
299
379
|
continue
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
x, y = r[4], r[5]
|
|
314
|
-
lvl.tiles.append(
|
|
315
|
-
TmapTile(
|
|
316
|
-
level=order, offset=off, size=size, x=x, y=y,
|
|
317
|
-
width=tw, height=th,
|
|
318
|
-
)
|
|
380
|
+
if level_end > tile_data_end:
|
|
381
|
+
tile_data_end = level_end
|
|
382
|
+
levels.append(
|
|
383
|
+
TmapLevel(
|
|
384
|
+
index=order,
|
|
385
|
+
downsample=float(2**order),
|
|
386
|
+
width=e["width"],
|
|
387
|
+
height=e["height"],
|
|
388
|
+
tile_width=tw,
|
|
389
|
+
tile_height=th,
|
|
390
|
+
tiles=_TileList(order, raw, rec_idx, tw, th),
|
|
391
|
+
grid=grid,
|
|
392
|
+
grid_cols=cols,
|
|
319
393
|
)
|
|
320
|
-
|
|
394
|
+
)
|
|
321
395
|
prop_layer_info.append(
|
|
322
396
|
f"{e['width']}x{e['height']}:{e['tile_col']}x{e['tile_row']}"
|
|
323
397
|
)
|
|
@@ -325,7 +399,6 @@ def _parse_tmap07(path: str) -> TmapFileInfo:
|
|
|
325
399
|
if not levels:
|
|
326
400
|
raise ValueError("TMAP07: no tile records parsed")
|
|
327
401
|
|
|
328
|
-
tile_data_end = max(t.offset + t.size for l in levels for t in l.tiles)
|
|
329
402
|
assoc = _parse_assoc07(f, file_size, tile_data_end)
|
|
330
403
|
|
|
331
404
|
return TmapFileInfo(
|
|
@@ -348,6 +421,7 @@ def _parse_tmap07(path: str) -> TmapFileInfo:
|
|
|
348
421
|
"tmap.declared_tiles": str(declared_tiles),
|
|
349
422
|
"tmap.layer_infos": ",".join(prop_layer_info),
|
|
350
423
|
"tmap.objective_power": str(scan_scale),
|
|
424
|
+
**_mpp_properties(pixel_size, scan_scale),
|
|
351
425
|
},
|
|
352
426
|
)
|
|
353
427
|
|
|
@@ -409,6 +483,18 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
409
483
|
if width <= 0 or height <= 0:
|
|
410
484
|
raise ValueError("TMAP06: invalid canvas dimensions")
|
|
411
485
|
|
|
486
|
+
# Multi-file slides: file_num counts the main TMAP plus .DT
|
|
487
|
+
# sidecars; tiles with file_id k live in <stem>.DTk.
|
|
488
|
+
data_files: Dict[int, str] = {0: path}
|
|
489
|
+
data_sizes: Dict[int, int] = {0: file_size}
|
|
490
|
+
stem = os.path.splitext(path)[0]
|
|
491
|
+
for fid in range(1, max(1, file_num)):
|
|
492
|
+
for cand in (f"{stem}.DT{fid}", f"{stem}.dt{fid}"):
|
|
493
|
+
if os.path.exists(cand):
|
|
494
|
+
data_files[fid] = cand
|
|
495
|
+
data_sizes[fid] = os.path.getsize(cand)
|
|
496
|
+
break
|
|
497
|
+
|
|
412
498
|
# ExtInfo at 0x3C: assoc images as (type, offset, length) triplets.
|
|
413
499
|
# ext type 1 = combined macro+label image, 2 = thumbnail.
|
|
414
500
|
assoc: List[TmapAssocImage] = []
|
|
@@ -514,18 +600,20 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
514
600
|
blocks = {} # (n_img_col, n_img_row) -> layer info dict
|
|
515
601
|
for k in range(min(image_num, len(li_raw) // 308)):
|
|
516
602
|
base = k * 308
|
|
603
|
+
fid = li_raw[base]
|
|
517
604
|
n_col, n_row = struct.unpack_from("<HH", li_raw, base + 8)
|
|
518
605
|
if n_col == 0xFFFF and n_row == 0xFFFF:
|
|
519
606
|
continue
|
|
520
607
|
n_x, n_y = struct.unpack_from("<ii", li_raw, base + 12)
|
|
608
|
+
limit = data_sizes.get(fid, 0)
|
|
521
609
|
tiles = []
|
|
522
610
|
for t in range(24):
|
|
523
611
|
tb = base + 20 + t * 12
|
|
524
612
|
layer_no, col, row = struct.unpack_from("<BBB", li_raw, tb)
|
|
525
613
|
off, length = struct.unpack_from("<II", li_raw, tb + 4)
|
|
526
|
-
if off > 0 and length > 0 and off + length <=
|
|
614
|
+
if off > 0 and length > 0 and off + length <= limit:
|
|
527
615
|
tiles.append((layer_no, col, row, off, length))
|
|
528
|
-
blocks[(n_col, n_row)] = dict(n_x=n_x, n_y=n_y, tiles=tiles, file_id=
|
|
616
|
+
blocks[(n_col, n_row)] = dict(n_x=n_x, n_y=n_y, tiles=tiles, file_id=fid)
|
|
529
617
|
|
|
530
618
|
# ShrinkTileInfo right after the LayerInfo table.
|
|
531
619
|
st_base = li_base + image_num * 308
|
|
@@ -533,10 +621,10 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
533
621
|
st_raw = _read_at(f, st_base, min(shrink_tile_num, 100000) * 20)
|
|
534
622
|
for k in range(min(shrink_tile_num, len(st_raw) // 20)):
|
|
535
623
|
base = k * 20
|
|
536
|
-
|
|
624
|
+
fid, layer_no = struct.unpack_from("<BB", st_raw, base)
|
|
537
625
|
n_x, n_y, off, length = struct.unpack_from("<iiII", st_raw, base + 4)
|
|
538
|
-
if off > 0 and length > 0 and off + length <=
|
|
539
|
-
shrinks.append((layer_no, n_x, n_y, off, length))
|
|
626
|
+
if off > 0 and length > 0 and off + length <= data_sizes.get(fid, 0):
|
|
627
|
+
shrinks.append((layer_no, n_x, n_y, off, length, fid))
|
|
540
628
|
|
|
541
629
|
# Pyramid levels: ratio_step division from the level-0 canvas.
|
|
542
630
|
levels_meta = []
|
|
@@ -548,9 +636,14 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
548
636
|
h = (h + ratio_step - 1) // ratio_step
|
|
549
637
|
scale = scale // ratio_step
|
|
550
638
|
|
|
551
|
-
def _tile_img(off: int, length: int) -> Optional[Image.Image]:
|
|
552
|
-
|
|
639
|
+
def _tile_img(fid: int, off: int, length: int) -> Optional[Image.Image]:
|
|
640
|
+
p = data_files.get(fid)
|
|
641
|
+
if p is None:
|
|
642
|
+
return None
|
|
553
643
|
try:
|
|
644
|
+
with open(p, "rb") as df:
|
|
645
|
+
df.seek(off)
|
|
646
|
+
data = df.read(length)
|
|
554
647
|
return Image.open(io.BytesIO(data))
|
|
555
648
|
except Exception:
|
|
556
649
|
return None
|
|
@@ -560,7 +653,7 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
560
653
|
for info in blocks.values():
|
|
561
654
|
for layer_no, col, row, off, length in info["tiles"]:
|
|
562
655
|
if layer_no == 0:
|
|
563
|
-
im = _tile_img(off, length)
|
|
656
|
+
im = _tile_img(info["file_id"], off, length)
|
|
564
657
|
if im is not None:
|
|
565
658
|
tw, th = im.size
|
|
566
659
|
break
|
|
@@ -585,6 +678,7 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
585
678
|
|
|
586
679
|
for (n_col, n_row), blk in blocks.items():
|
|
587
680
|
n_x, n_y = blk["n_x"], blk["n_y"]
|
|
681
|
+
fid = blk["file_id"]
|
|
588
682
|
for layer_no, col, row, off, length in blk["tiles"]:
|
|
589
683
|
if layer_no == 0:
|
|
590
684
|
levels[0].tiles.append(
|
|
@@ -596,6 +690,7 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
596
690
|
y=n_y + row * th,
|
|
597
691
|
width=tw,
|
|
598
692
|
height=th,
|
|
693
|
+
file_id=fid,
|
|
599
694
|
)
|
|
600
695
|
)
|
|
601
696
|
elif layer_no == 1 and len(levels) > 1:
|
|
@@ -608,6 +703,7 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
608
703
|
y=n_y // ratio_step,
|
|
609
704
|
width=tw,
|
|
610
705
|
height=th,
|
|
706
|
+
file_id=fid,
|
|
611
707
|
)
|
|
612
708
|
)
|
|
613
709
|
|
|
@@ -631,7 +727,9 @@ def _parse_tmap06(path: str) -> TmapFileInfo:
|
|
|
631
727
|
"tmap.blocks": str(len(blocks)),
|
|
632
728
|
"tmap.shrink_tiles": str(len(shrinks)),
|
|
633
729
|
"tmap.bkg_color": str(bkg_color),
|
|
730
|
+
**_mpp_properties(pixel_size, scan_scale),
|
|
634
731
|
},
|
|
732
|
+
data_files=data_files,
|
|
635
733
|
# extra fields for read_region
|
|
636
734
|
)
|
|
637
735
|
info.__dict__["_blocks"] = blocks
|
|
@@ -122,3 +122,83 @@ def test_synthetic_matches_reference_reading(tmp_path):
|
|
|
122
122
|
s = OpenSlide(str(p))
|
|
123
123
|
assert s.level_count == 2
|
|
124
124
|
s.close()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_synthetic_tmap06_multifile(tmp_path):
|
|
128
|
+
"""TMAP06 file_num=2: block 1's tiles live in a .DT1 sidecar file and
|
|
129
|
+
must be routed to it for every read path (region, iter_tiles)."""
|
|
130
|
+
main = tmp_path / "s.TMAP"
|
|
131
|
+
side = tmp_path / "s.DT1"
|
|
132
|
+
meta = make_tmap06(str(main), sidecar_path=str(side))
|
|
133
|
+
assert side.exists()
|
|
134
|
+
assert sum(1 for t in meta["tiles"] if t["file_id"] == 1) == 9
|
|
135
|
+
s = OpenSlide(str(main))
|
|
136
|
+
|
|
137
|
+
# layer-0 sidecar tile colour through read_region
|
|
138
|
+
want = (
|
|
139
|
+
(120 + 1 * 20 + 2 * 25) % 256,
|
|
140
|
+
(60 + 1 * 30 + 0 * 45) % 256,
|
|
141
|
+
(150 + 2 * 8) % 256,
|
|
142
|
+
)
|
|
143
|
+
assert close(center(s.read_region((2 * 612 + 10, 1024 + 10), 0, (300, 200))), want)
|
|
144
|
+
|
|
145
|
+
# layer-1 whole-block sidecar tile renders at 1/4 scale
|
|
146
|
+
assert close(center(s.read_region((624, 1036), 1, (300, 250))), (37, 200, 90))
|
|
147
|
+
|
|
148
|
+
# iter_tiles load() must decode all 16 layer-0 tiles (8 of them sidecar)
|
|
149
|
+
loads = [load for _, _, load in s.iter_tiles()]
|
|
150
|
+
assert all(load().size == (612, 512) for load in loads)
|
|
151
|
+
|
|
152
|
+
# level-1 iter_tiles too (one whole-block tile per block)
|
|
153
|
+
for _, _, load in s.iter_tiles(level=1):
|
|
154
|
+
assert load().size == (612, 512)
|
|
155
|
+
|
|
156
|
+
# missing sidecar file degrades to background instead of crashing
|
|
157
|
+
s.close()
|
|
158
|
+
side.unlink()
|
|
159
|
+
s2 = OpenSlide(str(main))
|
|
160
|
+
edge = s2.read_region((2 * 612 + 10, 1024 + 10), 0, (8, 8)).getpixel((4, 4))
|
|
161
|
+
assert edge[3] == 255 # opaque background colour, not an exception
|
|
162
|
+
s2.close()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_tmap06_mpp_fallback(tmp_path):
|
|
166
|
+
"""Corrupt vendor pixel_size (real TMAP06 bug: 6.88e-05 mm at 40x, which
|
|
167
|
+
implies impossible 2 µm nuclei) must not leak into openslide.mpp-x/y —
|
|
168
|
+
fall back to 10 µm / objective power; the raw value stays in
|
|
169
|
+
tmap.pixel_size_mm."""
|
|
170
|
+
make_tmap06(str(tmp_path / "s.TMAP"), pixel_size_mm=6.88e-05)
|
|
171
|
+
s = OpenSlide(str(tmp_path / "s.TMAP"))
|
|
172
|
+
assert float(s.properties["tmap.pixel_size_mm"]) == pytest.approx(6.88e-05)
|
|
173
|
+
assert float(s.properties["openslide.mpp-x"]) == pytest.approx(0.25)
|
|
174
|
+
assert float(s.properties["openslide.mpp-y"]) == pytest.approx(0.25)
|
|
175
|
+
assert s.properties["tmap.mpp_source"] == "derived"
|
|
176
|
+
s.close()
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def test_tmap07_mpp_inconsistent_header(tmp_path):
|
|
180
|
+
"""Real Shanxi TMAP07 headers carry 1.01e-04 mm at 40x — biologically
|
|
181
|
+
impossible (~99x objective, 6-9 mm canvas). The header value must lose
|
|
182
|
+
to the 10 µm / 40x = 0.25 physics even though it sits inside the
|
|
183
|
+
absolute plausibility range."""
|
|
184
|
+
make_tmap07(str(tmp_path / "a.TMAP"), pixel_size_mm=1.0117647e-4)
|
|
185
|
+
s = OpenSlide(str(tmp_path / "a.TMAP"))
|
|
186
|
+
assert float(s.properties["tmap.pixel_size_mm"]) == pytest.approx(1.0117647e-4)
|
|
187
|
+
assert float(s.properties["openslide.mpp-x"]) == pytest.approx(0.25)
|
|
188
|
+
assert s.properties["tmap.mpp_source"] == "derived"
|
|
189
|
+
s.close()
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def test_mpp_from_header(tmp_path):
|
|
193
|
+
"""A plausible header pixel size is used as-is."""
|
|
194
|
+
make_tmap07(str(tmp_path / "a.TMAP"))
|
|
195
|
+
s = OpenSlide(str(tmp_path / "a.TMAP"))
|
|
196
|
+
assert float(s.properties["openslide.mpp-x"]) == pytest.approx(0.25)
|
|
197
|
+
assert s.properties["tmap.mpp_source"] == "header"
|
|
198
|
+
s.close()
|
|
199
|
+
|
|
200
|
+
make_tmap06(str(tmp_path / "b.TMAP"), pixel_size_mm=2.5e-4)
|
|
201
|
+
s = OpenSlide(str(tmp_path / "b.TMAP"))
|
|
202
|
+
assert float(s.properties["openslide.mpp-x"]) == pytest.approx(0.25)
|
|
203
|
+
assert s.properties["tmap.mpp_source"] == "header"
|
|
204
|
+
s.close()
|
|
@@ -19,7 +19,7 @@ def _jpeg(color, size):
|
|
|
19
19
|
return buf.getvalue()
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
def make_tmap07(path, background=250):
|
|
22
|
+
def make_tmap07(path, background=250, pixel_size_mm=2.5e-4):
|
|
23
23
|
"""Tiny TMAP07: level 0 = 2x2 grid of 256 px tiles, level 1 = one tile."""
|
|
24
24
|
W = H = 512
|
|
25
25
|
cols = rows = 2
|
|
@@ -53,7 +53,7 @@ def make_tmap07(path, background=250):
|
|
|
53
53
|
struct.pack_into("<BBB", head, 8, 0, 80, 1) # fmt, quality, focus
|
|
54
54
|
head[11] = 40 # scan scale
|
|
55
55
|
head[12] = background
|
|
56
|
-
struct.pack_into("<f", head, 0x10,
|
|
56
|
+
struct.pack_into("<f", head, 0x10, pixel_size_mm) # pixel size (mm)
|
|
57
57
|
struct.pack_into("<I", head, 0x14, 2) # image num
|
|
58
58
|
struct.pack_into("<I", head, 0x18, 2) # layer num
|
|
59
59
|
struct.pack_into("<I", head, 0x1C, 5) # tile num
|
|
@@ -83,14 +83,20 @@ def make_tmap07(path, background=250):
|
|
|
83
83
|
return dict(dims=(W, H), background=background)
|
|
84
84
|
|
|
85
85
|
|
|
86
|
-
def make_tmap06(path, background=250):
|
|
86
|
+
def make_tmap06(path, background=250, pixel_size_mm=2.5e-4, sidecar_path=None):
|
|
87
87
|
"""Tiny TMAP06: a 1x2 grid of blocks (each 2448x1024 = 4x2 tiles of
|
|
88
88
|
612x512), one layer-1 whole-block tile per block, 3 pyramid levels
|
|
89
|
-
(ratio step 4).
|
|
89
|
+
(ratio step 4).
|
|
90
|
+
|
|
91
|
+
With ``sidecar_path`` set, the header declares file_num=2 and block 1's
|
|
92
|
+
JPEG blobs are written to a separate .DT1-style file (512-byte header
|
|
93
|
+
then packed tiles); its LayerInfo offsets address that sidecar.
|
|
94
|
+
"""
|
|
90
95
|
TW, TH = 612, 512
|
|
91
96
|
IMG_W, IMG_H = 4 * TW, 2 * TH # 2448 x 1024 per block
|
|
92
97
|
N_BLOCKS = 2
|
|
93
98
|
CANVAS_W, CANVAS_H = IMG_W, N_BLOCKS * IMG_H # 2448 x 2048
|
|
99
|
+
use_sidecar = sidecar_path is not None
|
|
94
100
|
|
|
95
101
|
li_base = 192
|
|
96
102
|
li_size = 308
|
|
@@ -98,8 +104,8 @@ def make_tmap06(path, background=250):
|
|
|
98
104
|
|
|
99
105
|
out = bytearray(60)
|
|
100
106
|
out[0:6] = b"TMAP06"
|
|
101
|
-
out[6:16] = bytes([1, 0, 0, 3, 24, 0, 4, 10, 0, background])
|
|
102
|
-
struct.pack_into("<f", out, 0x10,
|
|
107
|
+
out[6:16] = bytes([1, 0, 2 if use_sidecar else 0, 3, 24, 0, 4, 10, 0, background])
|
|
108
|
+
struct.pack_into("<f", out, 0x10, pixel_size_mm) # pixel size (mm)
|
|
103
109
|
struct.pack_into("<I", out, 0x14, N_BLOCKS) # image num (LayerInfo count)
|
|
104
110
|
struct.pack_into(
|
|
105
111
|
"<HHHHHHHH", out, 0x18,
|
|
@@ -116,10 +122,14 @@ def make_tmap06(path, background=250):
|
|
|
116
122
|
|
|
117
123
|
layerinfos = bytearray()
|
|
118
124
|
tiles_blob = bytearray()
|
|
125
|
+
side_blob = bytearray()
|
|
119
126
|
tile_colors = []
|
|
120
127
|
for b in range(N_BLOCKS):
|
|
128
|
+
fid = 1 if (use_sidecar and b == 1) else 0
|
|
129
|
+
target = side_blob if fid else tiles_blob
|
|
130
|
+
base_off = 512 if fid else data_off # sidecar tiles follow a 512 B header
|
|
121
131
|
li = bytearray(li_size)
|
|
122
|
-
li[0] =
|
|
132
|
+
li[0] = fid # file id
|
|
123
133
|
li[1] = 0 # layer
|
|
124
134
|
struct.pack_into("<HH", li, 8, 0, b) # n_img_col, n_img_row
|
|
125
135
|
n_x, n_y = 0, b * IMG_H
|
|
@@ -136,11 +146,11 @@ def make_tmap06(path, background=250):
|
|
|
136
146
|
l1_color = (b * 37 % 256, 200, 90)
|
|
137
147
|
entries.append((1, 0, 0, _jpeg(l1_color, (TW, TH)), l1_color))
|
|
138
148
|
for t, (layer, col, row, j, color) in enumerate(entries[:24]):
|
|
139
|
-
off =
|
|
149
|
+
off = base_off + len(target)
|
|
140
150
|
li[20 + t * 12 : 20 + t * 12 + 12] = struct.pack(
|
|
141
151
|
"<BBBB II", layer, col, row, 0, off, len(j)
|
|
142
152
|
)
|
|
143
|
-
|
|
153
|
+
target += j
|
|
144
154
|
tile_colors.append(
|
|
145
155
|
dict(
|
|
146
156
|
layer=layer,
|
|
@@ -149,6 +159,7 @@ def make_tmap06(path, background=250):
|
|
|
149
159
|
w=TW,
|
|
150
160
|
h=TH,
|
|
151
161
|
color=color,
|
|
162
|
+
file_id=fid,
|
|
152
163
|
)
|
|
153
164
|
)
|
|
154
165
|
layerinfos += li
|
|
@@ -159,6 +170,12 @@ def make_tmap06(path, background=250):
|
|
|
159
170
|
|
|
160
171
|
with open(path, "wb") as f:
|
|
161
172
|
f.write(bytes(out))
|
|
173
|
+
if use_sidecar:
|
|
174
|
+
side = bytearray(512)
|
|
175
|
+
side[0:4] = b"DT01"
|
|
176
|
+
side += side_blob
|
|
177
|
+
with open(sidecar_path, "wb") as f:
|
|
178
|
+
f.write(bytes(side))
|
|
162
179
|
return dict(
|
|
163
180
|
dims=(CANVAS_W, CANVAS_H),
|
|
164
181
|
background=background,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|