scrollscout 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrollscout/__init__.py +81 -0
- scrollscout/catalog.py +103 -0
- scrollscout/cli.py +95 -0
- scrollscout/reader.py +112 -0
- scrollscout/render.py +42 -0
- scrollscout-0.1.0.dist-info/METADATA +125 -0
- scrollscout-0.1.0.dist-info/RECORD +11 -0
- scrollscout-0.1.0.dist-info/WHEEL +5 -0
- scrollscout-0.1.0.dist-info/entry_points.txt +2 -0
- scrollscout-0.1.0.dist-info/licenses/LICENSE +21 -0
- scrollscout-0.1.0.dist-info/top_level.txt +1 -0
scrollscout/__init__.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""scrollscout -- scout Vesuvius Challenge scrolls before you commit disk.
|
|
2
|
+
|
|
3
|
+
Stream and preview Herculaneum scroll CT volumes from a laptop, without
|
|
4
|
+
downloading terabytes and without tripping the server's rate limiter.
|
|
5
|
+
|
|
6
|
+
Quick start:
|
|
7
|
+
|
|
8
|
+
import scrollscout as ss
|
|
9
|
+
vol = ss.open_scroll(1) # streams a few KB of metadata
|
|
10
|
+
img = ss.thumbnail(1, z_frac=0.5, width=1000, out="scroll1.png")
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
from .catalog import Level, ScrollVolume, open_scroll, volume_url
|
|
17
|
+
from .reader import RateLimited, read_slice, read_window
|
|
18
|
+
from .render import autocontrast, to_png
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"open_scroll",
|
|
24
|
+
"volume_url",
|
|
25
|
+
"ScrollVolume",
|
|
26
|
+
"Level",
|
|
27
|
+
"read_slice",
|
|
28
|
+
"read_window",
|
|
29
|
+
"RateLimited",
|
|
30
|
+
"autocontrast",
|
|
31
|
+
"to_png",
|
|
32
|
+
"thumbnail",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def thumbnail(
|
|
37
|
+
scroll_id: int,
|
|
38
|
+
*,
|
|
39
|
+
z_frac: float = 0.5,
|
|
40
|
+
z: Optional[int] = None,
|
|
41
|
+
width: int = 1000,
|
|
42
|
+
out: Optional[str] = None,
|
|
43
|
+
tile: int = 512,
|
|
44
|
+
inter_tile_pause: float = 0.15,
|
|
45
|
+
verbose: bool = False,
|
|
46
|
+
):
|
|
47
|
+
"""Stream one cross-section of a scroll and return it (optionally save a PNG).
|
|
48
|
+
|
|
49
|
+
Parameters
|
|
50
|
+
----------
|
|
51
|
+
scroll_id : which scroll (1-5).
|
|
52
|
+
z_frac : depth as a fraction 0..1 of the volume height (ignored if ``z`` given).
|
|
53
|
+
z : explicit slice index at the *chosen pyramid level*.
|
|
54
|
+
width : desired output width in pixels; picks the coarsest pyramid level
|
|
55
|
+
that still meets it, keeping the read small and rate-limit-safe.
|
|
56
|
+
out : if given, save a contrast-stretched PNG here.
|
|
57
|
+
"""
|
|
58
|
+
vol = open_scroll(scroll_id)
|
|
59
|
+
level = vol.level_for_width(width)
|
|
60
|
+
zi = z if z is not None else int(round(z_frac * (level.shape[0] - 1)))
|
|
61
|
+
|
|
62
|
+
def _prog(done, total):
|
|
63
|
+
if verbose:
|
|
64
|
+
print(f" tile {done}/{total}", end="\r", flush=True)
|
|
65
|
+
|
|
66
|
+
def _back(delay, attempt):
|
|
67
|
+
if verbose:
|
|
68
|
+
print(f"\n 429 backoff {delay:.1f}s (attempt {attempt})", flush=True)
|
|
69
|
+
|
|
70
|
+
arr = read_slice(
|
|
71
|
+
vol, level, zi,
|
|
72
|
+
tile=tile, inter_tile_pause=inter_tile_pause,
|
|
73
|
+
on_progress=_prog, on_backoff=_back,
|
|
74
|
+
)
|
|
75
|
+
if verbose:
|
|
76
|
+
print()
|
|
77
|
+
if out:
|
|
78
|
+
w, h = to_png(arr, out)
|
|
79
|
+
if verbose:
|
|
80
|
+
print(f"saved {out} ({w}x{h}) from scroll {scroll_id} level {level.path} z={zi}")
|
|
81
|
+
return arr
|
scrollscout/catalog.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Scroll catalog: map a scroll id to its multiscale OME-Zarr URL and pyramid levels.
|
|
2
|
+
|
|
3
|
+
We stream everything. Nothing here downloads a full volume; opening a zarr array
|
|
4
|
+
only reads small metadata (`.zarray` / `.zattrs`), a few KB.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from functools import lru_cache
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
import fsspec
|
|
14
|
+
import zarr
|
|
15
|
+
|
|
16
|
+
# Canonical scroll -> volume zarr URL. Sourced from the official `vesuvius`
|
|
17
|
+
# catalogue (`vesuvius.list_files()`); hard-coded as a fallback so scrollscout
|
|
18
|
+
# works even if that optional dependency / network catalogue is unavailable.
|
|
19
|
+
_FALLBACK_VOLUMES = {
|
|
20
|
+
1: "https://dl.ash2txt.org/full-scrolls/Scroll1/PHercParis4.volpkg/volumes_zarr_standardized/54keV_7.91um_Scroll1A.zarr/",
|
|
21
|
+
2: "https://dl.ash2txt.org/full-scrolls/Scroll2/PHercParis3.volpkg/volumes_zarr_standardized/54keV_7.91um_Scroll2A.zarr/",
|
|
22
|
+
3: "https://dl.ash2txt.org/full-scrolls/Scroll3/PHerc332.volpkg/volumes_zarr_standardized/53keV_7.91um_Scroll3.zarr/",
|
|
23
|
+
4: "https://dl.ash2txt.org/full-scrolls/Scroll4/PHerc1667.volpkg/volumes_zarr/20231117161658.zarr/",
|
|
24
|
+
5: "https://dl.ash2txt.org/full-scrolls/Scroll5/PHerc172.volpkg/volumes_zarr/20241024131838.zarr/",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Level:
|
|
30
|
+
"""One resolution level of a multiscale pyramid."""
|
|
31
|
+
|
|
32
|
+
path: str # dataset key inside the zarr group, e.g. "0", "3"
|
|
33
|
+
shape: tuple # (z, y, x)
|
|
34
|
+
chunks: tuple
|
|
35
|
+
dtype: str
|
|
36
|
+
scale: tuple # downsample factor vs level 0, e.g. (8, 8, 8)
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def voxels(self) -> int:
|
|
40
|
+
z, y, x = self.shape
|
|
41
|
+
return z * y * x
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class ScrollVolume:
|
|
46
|
+
scroll_id: int
|
|
47
|
+
url: str
|
|
48
|
+
levels: tuple # tuple[Level], finest first
|
|
49
|
+
|
|
50
|
+
def level_for_width(self, target_px: int) -> Level:
|
|
51
|
+
"""Coarsest level whose in-plane width is still >= target_px.
|
|
52
|
+
|
|
53
|
+
Picking the smallest sufficient level is what keeps us off the
|
|
54
|
+
rate limiter: a 1000px view from level 3 is ~64 chunks, not ~4000.
|
|
55
|
+
"""
|
|
56
|
+
candidates = [lv for lv in self.levels if lv.shape[2] >= target_px]
|
|
57
|
+
if not candidates:
|
|
58
|
+
return self.levels[0] # target bigger than full res; use finest
|
|
59
|
+
return min(candidates, key=lambda lv: lv.voxels)
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def finest(self) -> Level:
|
|
63
|
+
return self.levels[0]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def volume_url(scroll_id: int) -> str:
|
|
67
|
+
"""Resolve a scroll's zarr URL, preferring the live `vesuvius` catalogue."""
|
|
68
|
+
try:
|
|
69
|
+
from vesuvius import list_files # optional dependency
|
|
70
|
+
|
|
71
|
+
files = list_files()
|
|
72
|
+
entry = files.get(scroll_id) or files.get(str(scroll_id))
|
|
73
|
+
if entry:
|
|
74
|
+
# entry: {energy: {resolution: {"volume": url, ...}}}
|
|
75
|
+
for energy in entry.values():
|
|
76
|
+
for res in energy.values():
|
|
77
|
+
if isinstance(res, dict) and res.get("volume"):
|
|
78
|
+
return res["volume"]
|
|
79
|
+
except Exception:
|
|
80
|
+
pass
|
|
81
|
+
if scroll_id not in _FALLBACK_VOLUMES:
|
|
82
|
+
raise KeyError(f"Unknown scroll id {scroll_id!r}; known: {sorted(_FALLBACK_VOLUMES)}")
|
|
83
|
+
return _FALLBACK_VOLUMES[scroll_id]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@lru_cache(maxsize=16)
|
|
87
|
+
def open_scroll(scroll_id: int, url: Optional[str] = None) -> ScrollVolume:
|
|
88
|
+
"""Open a scroll's multiscale metadata (streams only a few KB)."""
|
|
89
|
+
url = url or volume_url(scroll_id)
|
|
90
|
+
fs = fsspec.filesystem("https")
|
|
91
|
+
attrs = json.loads(fs.open(url.rstrip("/") + "/.zattrs").read())
|
|
92
|
+
datasets = attrs["multiscales"][0]["datasets"]
|
|
93
|
+
levels = []
|
|
94
|
+
for d in datasets:
|
|
95
|
+
p = d["path"]
|
|
96
|
+
scale = tuple(d["coordinateTransformations"][0]["scale"])
|
|
97
|
+
arr = zarr.open(url.rstrip("/") + "/" + p, mode="r")
|
|
98
|
+
levels.append(
|
|
99
|
+
Level(path=p, shape=tuple(arr.shape), chunks=tuple(arr.chunks),
|
|
100
|
+
dtype=str(arr.dtype), scale=scale)
|
|
101
|
+
)
|
|
102
|
+
levels.sort(key=lambda lv: lv.voxels, reverse=True) # finest (biggest) first
|
|
103
|
+
return ScrollVolume(scroll_id=scroll_id, url=url.rstrip("/") + "/", levels=tuple(levels))
|
scrollscout/cli.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Command-line interface for scrollscout."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from . import __version__
|
|
8
|
+
from .catalog import open_scroll
|
|
9
|
+
from .reader import read_slice
|
|
10
|
+
from .render import to_png
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _cmd_info(args) -> int:
|
|
14
|
+
vol = open_scroll(args.scroll)
|
|
15
|
+
print(f"Scroll {vol.scroll_id}")
|
|
16
|
+
print(f" url: {vol.url}")
|
|
17
|
+
print(f" pyramid levels ({len(vol.levels)}):")
|
|
18
|
+
print(f" {'level':>5} {'shape (z,y,x)':>24} {'chunks':>14} {'scale':>12} approx full size")
|
|
19
|
+
for lv in vol.levels:
|
|
20
|
+
gb = lv.voxels / 1e9 # uint8 => 1 byte/voxel
|
|
21
|
+
print(f" {lv.path:>5} {str(lv.shape):>24} {str(lv.chunks):>14} "
|
|
22
|
+
f"{str(lv.scale):>12} {gb:8.2f} GB")
|
|
23
|
+
return 0
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _cmd_thumb(args) -> int:
|
|
27
|
+
vol = open_scroll(args.scroll)
|
|
28
|
+
level = vol.level_for_width(args.width)
|
|
29
|
+
zi = args.z if args.z is not None else int(round(args.z_frac * (level.shape[0] - 1)))
|
|
30
|
+
print(f"scroll {vol.scroll_id}: level {level.path} {level.shape}, slice z={zi}, "
|
|
31
|
+
f"target width {args.width}px")
|
|
32
|
+
|
|
33
|
+
def prog(done, total):
|
|
34
|
+
print(f" streaming tile {done}/{total}", end="\r", flush=True)
|
|
35
|
+
|
|
36
|
+
def back(delay, attempt):
|
|
37
|
+
print(f"\n rate-limited; backing off {delay:.1f}s (attempt {attempt})", flush=True)
|
|
38
|
+
|
|
39
|
+
arr = read_slice(vol, level, zi, tile=args.tile,
|
|
40
|
+
inter_tile_pause=args.pause, on_progress=prog, on_backoff=back)
|
|
41
|
+
print()
|
|
42
|
+
w, h = to_png(arr, args.out, max_px=args.max_px)
|
|
43
|
+
print(f"saved {args.out} ({w}x{h})")
|
|
44
|
+
return 0
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _cmd_scrolls(_args) -> int:
|
|
48
|
+
from .catalog import _FALLBACK_VOLUMES
|
|
49
|
+
print("Known scrolls:", ", ".join(str(k) for k in sorted(_FALLBACK_VOLUMES)))
|
|
50
|
+
return 0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
54
|
+
p = argparse.ArgumentParser(
|
|
55
|
+
prog="scrollscout",
|
|
56
|
+
description="Scout Vesuvius Challenge scrolls -- stream and preview CT "
|
|
57
|
+
"volumes without downloading terabytes.",
|
|
58
|
+
)
|
|
59
|
+
p.add_argument("--version", action="version", version=f"scrollscout {__version__}")
|
|
60
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
61
|
+
|
|
62
|
+
ps = sub.add_parser("scrolls", help="list known scroll ids")
|
|
63
|
+
ps.set_defaults(func=_cmd_scrolls)
|
|
64
|
+
|
|
65
|
+
pi = sub.add_parser("info", help="show a scroll's multiscale pyramid")
|
|
66
|
+
pi.add_argument("scroll", type=int)
|
|
67
|
+
pi.set_defaults(func=_cmd_info)
|
|
68
|
+
|
|
69
|
+
pt = sub.add_parser("thumb", help="stream a cross-section to a PNG")
|
|
70
|
+
pt.add_argument("scroll", type=int)
|
|
71
|
+
pt.add_argument("-o", "--out", default="scroll.png", help="output PNG path")
|
|
72
|
+
pt.add_argument("-w", "--width", type=int, default=1000,
|
|
73
|
+
help="target width px (selects pyramid level)")
|
|
74
|
+
pt.add_argument("--z", type=int, default=None, help="explicit slice index at chosen level")
|
|
75
|
+
pt.add_argument("--z-frac", type=float, default=0.5, dest="z_frac",
|
|
76
|
+
help="depth as fraction 0..1 (default middle)")
|
|
77
|
+
pt.add_argument("--tile", type=int, default=512, help="tile size in px")
|
|
78
|
+
pt.add_argument("--pause", type=float, default=0.15, help="seconds between tiles")
|
|
79
|
+
pt.add_argument("--max-px", type=int, default=1600, dest="max_px",
|
|
80
|
+
help="cap output dimension")
|
|
81
|
+
pt.set_defaults(func=_cmd_thumb)
|
|
82
|
+
return p
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def main(argv=None) -> int:
|
|
86
|
+
args = build_parser().parse_args(argv)
|
|
87
|
+
try:
|
|
88
|
+
return args.func(args)
|
|
89
|
+
except KeyboardInterrupt:
|
|
90
|
+
print("\ninterrupted", file=sys.stderr)
|
|
91
|
+
return 130
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
if __name__ == "__main__":
|
|
95
|
+
raise SystemExit(main())
|
scrollscout/reader.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Throttle-safe streaming reads from the Vesuvius data server.
|
|
2
|
+
|
|
3
|
+
The scroll volumes live on dl.ash2txt.org as chunked OME-Zarr. A naive
|
|
4
|
+
`array[z, y0:y1, x0:x1]` over a large region makes zarr fire one HTTP request
|
|
5
|
+
per chunk, all at once -- which trips the server's HTTP 429 rate limiter and
|
|
6
|
+
kills the read. Every laptop contributor hits this.
|
|
7
|
+
|
|
8
|
+
This module reads a 2D slice in small, chunk-aligned tiles with a bounded
|
|
9
|
+
number of tiles in flight and exponential backoff on 429, so a read *slows
|
|
10
|
+
down* under pressure instead of failing.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import time
|
|
15
|
+
from typing import Callable, Optional
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import zarr
|
|
19
|
+
|
|
20
|
+
from .catalog import Level, ScrollVolume
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class RateLimited(RuntimeError):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _is_429(exc: Exception) -> bool:
|
|
28
|
+
s = str(exc)
|
|
29
|
+
return "429" in s or "Too Many Requests" in s
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _fetch_tile(arr, z: int, y0: int, y1: int, x0: int, x1: int,
|
|
33
|
+
tries: int, base_delay: float,
|
|
34
|
+
on_backoff: Optional[Callable[[float, int], None]]) -> np.ndarray:
|
|
35
|
+
"""Fetch one 2D tile with retry+backoff on rate limiting."""
|
|
36
|
+
for attempt in range(tries):
|
|
37
|
+
try:
|
|
38
|
+
return np.asarray(arr[z, y0:y1, x0:x1])
|
|
39
|
+
except Exception as exc: # noqa: BLE001 - zarr wraps the HTTP error deeply
|
|
40
|
+
if _is_429(exc) and attempt < tries - 1:
|
|
41
|
+
delay = base_delay * (2 ** attempt)
|
|
42
|
+
if on_backoff:
|
|
43
|
+
on_backoff(delay, attempt + 1)
|
|
44
|
+
time.sleep(delay)
|
|
45
|
+
continue
|
|
46
|
+
if _is_429(exc):
|
|
47
|
+
raise RateLimited(
|
|
48
|
+
f"Rate limited after {tries} attempts on tile "
|
|
49
|
+
f"z={z} y[{y0}:{y1}] x[{x0}:{x1}]"
|
|
50
|
+
) from exc
|
|
51
|
+
raise
|
|
52
|
+
raise RateLimited("unreachable")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def read_slice(
|
|
56
|
+
scroll: ScrollVolume,
|
|
57
|
+
level: Level,
|
|
58
|
+
z: int,
|
|
59
|
+
*,
|
|
60
|
+
tile: int = 512,
|
|
61
|
+
inter_tile_pause: float = 0.15,
|
|
62
|
+
tries: int = 6,
|
|
63
|
+
base_delay: float = 1.0,
|
|
64
|
+
on_progress: Optional[Callable[[int, int], None]] = None,
|
|
65
|
+
on_backoff: Optional[Callable[[float, int], None]] = None,
|
|
66
|
+
) -> np.ndarray:
|
|
67
|
+
"""Read a full 2D (y, x) cross-section at index ``z`` of ``level``.
|
|
68
|
+
|
|
69
|
+
Reads tile-by-tile with a short pause between tiles so we stay polite to
|
|
70
|
+
the server. Returns a 2D numpy array of the level's native dtype.
|
|
71
|
+
"""
|
|
72
|
+
arr = zarr.open(scroll.url + level.path, mode="r")
|
|
73
|
+
_, ny, nx = level.shape
|
|
74
|
+
if not (0 <= z < level.shape[0]):
|
|
75
|
+
raise IndexError(f"z={z} out of range 0..{level.shape[0]-1} for level {level.path}")
|
|
76
|
+
|
|
77
|
+
out = np.empty((ny, nx), dtype=level.dtype)
|
|
78
|
+
y_tiles = list(range(0, ny, tile))
|
|
79
|
+
x_tiles = list(range(0, nx, tile))
|
|
80
|
+
total = len(y_tiles) * len(x_tiles)
|
|
81
|
+
done = 0
|
|
82
|
+
for y0 in y_tiles:
|
|
83
|
+
y1 = min(y0 + tile, ny)
|
|
84
|
+
for x0 in x_tiles:
|
|
85
|
+
x1 = min(x0 + tile, nx)
|
|
86
|
+
out[y0:y1, x0:x1] = _fetch_tile(
|
|
87
|
+
arr, z, y0, y1, x0, x1, tries, base_delay, on_backoff
|
|
88
|
+
)
|
|
89
|
+
done += 1
|
|
90
|
+
if on_progress:
|
|
91
|
+
on_progress(done, total)
|
|
92
|
+
if inter_tile_pause and done < total:
|
|
93
|
+
time.sleep(inter_tile_pause)
|
|
94
|
+
return out
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def read_window(
|
|
98
|
+
scroll: ScrollVolume,
|
|
99
|
+
level: Level,
|
|
100
|
+
z: int,
|
|
101
|
+
y0: int,
|
|
102
|
+
y1: int,
|
|
103
|
+
x0: int,
|
|
104
|
+
x1: int,
|
|
105
|
+
*,
|
|
106
|
+
tries: int = 6,
|
|
107
|
+
base_delay: float = 1.0,
|
|
108
|
+
on_backoff: Optional[Callable[[float, int], None]] = None,
|
|
109
|
+
) -> np.ndarray:
|
|
110
|
+
"""Read a single small (y, x) window -- one throttle-safe request."""
|
|
111
|
+
arr = zarr.open(scroll.url + level.path, mode="r")
|
|
112
|
+
return _fetch_tile(arr, z, y0, y1, x0, x1, tries, base_delay, on_backoff)
|
scrollscout/render.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Turn a streamed cross-section into a viewable, contrast-stretched image."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Optional, Tuple
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def autocontrast(a: np.ndarray, low_pct: float = 1.0, high_pct: float = 99.5) -> np.ndarray:
|
|
10
|
+
"""Percentile contrast stretch to uint8.
|
|
11
|
+
|
|
12
|
+
Scroll CT data is low-contrast (papyrus vs air vs resin sit close together);
|
|
13
|
+
a percentile stretch makes the wound sheets legible without clipping detail.
|
|
14
|
+
"""
|
|
15
|
+
a = a.astype(np.float32)
|
|
16
|
+
lo, hi = np.percentile(a, [low_pct, high_pct])
|
|
17
|
+
if hi <= lo:
|
|
18
|
+
hi = lo + 1.0
|
|
19
|
+
out = np.clip((a - lo) / (hi - lo) * 255.0, 0, 255)
|
|
20
|
+
return out.astype(np.uint8)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def to_png(
|
|
24
|
+
a: np.ndarray,
|
|
25
|
+
path: str,
|
|
26
|
+
*,
|
|
27
|
+
max_px: Optional[int] = 1600,
|
|
28
|
+
stretch: bool = True,
|
|
29
|
+
) -> Tuple[int, int]:
|
|
30
|
+
"""Save a 2D array as a PNG, optionally auto-contrasted and downscaled.
|
|
31
|
+
|
|
32
|
+
Returns the saved (width, height).
|
|
33
|
+
"""
|
|
34
|
+
from PIL import Image
|
|
35
|
+
|
|
36
|
+
img8 = autocontrast(a) if stretch else a.astype(np.uint8)
|
|
37
|
+
im = Image.fromarray(img8)
|
|
38
|
+
if max_px and max(im.size) > max_px:
|
|
39
|
+
r = max_px / max(im.size)
|
|
40
|
+
im = im.resize((max(1, int(im.size[0] * r)), max(1, int(im.size[1] * r))), Image.LANCZOS)
|
|
41
|
+
im.save(path)
|
|
42
|
+
return im.size
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scrollscout
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scout Vesuvius Challenge scrolls: stream and preview Herculaneum CT volumes from a laptop without downloading terabytes or tripping rate limits.
|
|
5
|
+
Author: Alexandre Dufour-Richard
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/vonduffen/scrollscout
|
|
8
|
+
Project-URL: Repository, https://github.com/vonduffen/scrollscout
|
|
9
|
+
Project-URL: Issues, https://github.com/vonduffen/scrollscout/issues
|
|
10
|
+
Project-URL: Vesuvius Challenge, https://scrollprize.org
|
|
11
|
+
Keywords: vesuvius-challenge,herculaneum,ome-zarr,ct,papyrology
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Visualization
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: zarr<3,>=2.14
|
|
19
|
+
Requires-Dist: numpy>=1.23
|
|
20
|
+
Requires-Dist: fsspec>=2023.1
|
|
21
|
+
Requires-Dist: aiohttp>=3.8
|
|
22
|
+
Requires-Dist: requests>=2.28
|
|
23
|
+
Requires-Dist: Pillow>=9.0
|
|
24
|
+
Provides-Extra: catalog
|
|
25
|
+
Requires-Dist: vesuvius>=0.1.9; extra == "catalog"
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# scrollscout
|
|
31
|
+
|
|
32
|
+
**Scout [Vesuvius Challenge](https://scrollprize.org) scrolls from a laptop — stream and preview the Herculaneum CT volumes without downloading terabytes or tripping the data server's rate limiter.**
|
|
33
|
+
|
|
34
|
+
The scroll volumes are enormous. Scroll 1 alone is **14376 × 7888 × 8096 voxels ≈ 918 GB** at full resolution. If you're on a laptop with limited disk, you can't (and shouldn't) download them to take a look. The data *is* streamable as multiscale OME-Zarr — but two things bite newcomers immediately:
|
|
35
|
+
|
|
36
|
+
1. **You don't know which resolution level to read.** Ask for a full-res cross-section and you'll try to pull hundreds of GB.
|
|
37
|
+
2. **Naive reads get rate-limited.** A large `array[z, y0:y1, x0:x1]` makes zarr fire one HTTP request per chunk, all at once. The server replies `429 Too Many Requests` and your read dies:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
aiohttp.client_exceptions.ClientResponseError: 429, message='Too Many Requests',
|
|
41
|
+
url='https://dl.ash2txt.org/.../54keV_7.91um_Scroll1A.zarr/0/56/19/26'
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`scrollscout` handles both for you: it **auto-selects the coarsest pyramid level** that satisfies your requested pixel width, and reads it in **small, chunk-aligned tiles with exponential backoff on 429**, so a read *slows down* under pressure instead of failing.
|
|
45
|
+
|
|
46
|
+

|
|
47
|
+
|
|
48
|
+
*Scroll 1 (PHerc. Paris 4), middle cross-section — streamed with `scrollscout thumb 1 -w 1000`. Total transfer: a few MB, not 918 GB.*
|
|
49
|
+
|
|
50
|
+
## Install
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install scrollscout
|
|
54
|
+
# optional: use the live scroll catalogue from the official `vesuvius` package
|
|
55
|
+
pip install "scrollscout[catalog]"
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
You must accept the Vesuvius Challenge [data license](https://scrollprize.org/data) before accessing scroll data. scrollscout does not redistribute any scroll data; it streams the public server on your behalf.
|
|
59
|
+
|
|
60
|
+
## Command line
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# list a scroll's multiscale pyramid — see the sizes before you read anything
|
|
64
|
+
scrollscout info 1
|
|
65
|
+
|
|
66
|
+
# stream a middle cross-section to a PNG (auto-picks the pyramid level)
|
|
67
|
+
scrollscout thumb 1 --width 1000 -o scroll1.png
|
|
68
|
+
|
|
69
|
+
# a shallower slice of scroll 2, larger
|
|
70
|
+
scrollscout thumb 2 --z-frac 0.25 --width 1200 -o scroll2_quarter.png
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`scrollscout info 1` prints:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
Scroll 1
|
|
77
|
+
pyramid levels (6):
|
|
78
|
+
level shape (z,y,x) chunks scale approx full size
|
|
79
|
+
0 (14376, 7888, 8096) (128, 128, 128) (1.0, 1.0, 1.0) 918.07 GB
|
|
80
|
+
1 (7188, 3944, 4048) (128, 128, 128) (2.0, 2.0, 2.0) 114.76 GB
|
|
81
|
+
2 (3594, 1972, 2024) (128, 128, 128) (4.0, 4.0, 4.0) 14.34 GB
|
|
82
|
+
3 (1797, 986, 1012) (128, 128, 128) (8.0, 8.0, 8.0) 1.79 GB
|
|
83
|
+
4 (899, 493, 506) (128, 128, 128) (16.0,16.0,16.0) 0.22 GB
|
|
84
|
+
5 (450, 247, 253) (128, 128, 128) (32.0,32.0,32.0) 0.03 GB
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Python API
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
import scrollscout as ss
|
|
91
|
+
|
|
92
|
+
vol = ss.open_scroll(1) # streams only ~KB of metadata
|
|
93
|
+
print(vol.level_for_width(1000)) # -> Level(path='3', ...)
|
|
94
|
+
|
|
95
|
+
# get the cross-section as a numpy array (throttle-safe under the hood)
|
|
96
|
+
arr = ss.thumbnail(1, z_frac=0.5, width=1000, out="scroll1.png", verbose=True)
|
|
97
|
+
print(arr.shape, arr.dtype) # (986, 1012) uint8
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## How the throttle-safe read works
|
|
101
|
+
|
|
102
|
+
- `level_for_width(target_px)` picks the **coarsest** pyramid level whose in-plane
|
|
103
|
+
width still meets your target. A 1000 px view comes from level 3 (~64 chunks),
|
|
104
|
+
never level 0 (~4000 chunks). Under-fetching is the first line of defense.
|
|
105
|
+
- `read_slice()` walks the chosen slice in `tile`-sized blocks with a short pause
|
|
106
|
+
between tiles, and retries each tile with exponential backoff (`base_delay * 2**n`)
|
|
107
|
+
when the server returns 429. Non-rate-limit errors propagate immediately.
|
|
108
|
+
|
|
109
|
+
## Scope & limitations
|
|
110
|
+
|
|
111
|
+
- Reads scroll **volumes** (the `volumes_zarr*` OME-Zarr). Segment/surface
|
|
112
|
+
meshes and ink predictions are out of scope for v0.1.
|
|
113
|
+
- Preview/QA tool, not an analysis pipeline — it gets you *looking* at the data
|
|
114
|
+
fast so you can decide where to work.
|
|
115
|
+
- Cross-sections are on the `z` axis (the scan slice axis) for now.
|
|
116
|
+
|
|
117
|
+
## License
|
|
118
|
+
|
|
119
|
+
MIT (see `LICENSE`). Scroll data itself is governed by the Vesuvius Challenge /
|
|
120
|
+
EduceLab-Scrolls terms and is **not** included or redistributed here.
|
|
121
|
+
|
|
122
|
+
If you use scroll data in published work, cite EduceLab-Scrolls: Parsons, S.,
|
|
123
|
+
Parker, C. S., Chapman, C., Hayashida, M., & Seales, W. B. (2023).
|
|
124
|
+
*EduceLab-Scrolls: Verifiable Recovery of Text from Herculaneum Papyri using
|
|
125
|
+
X-ray CT.* arXiv:2304.02084.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
scrollscout/__init__.py,sha256=AO2QOyBYfw-jLtxuIoNd7f-U4RzazzuwNJ_qQAHC8vQ,2385
|
|
2
|
+
scrollscout/catalog.py,sha256=dGVwSirAVrtrZedp0g8fnhtmCJS1gPO-SKDJZ6hx-vI,4055
|
|
3
|
+
scrollscout/cli.py,sha256=FR_d4NFJ08T2ug3Uf8-VzWAUb3ROExB305jexwL9IRg,3541
|
|
4
|
+
scrollscout/reader.py,sha256=Q0VGgTZ9SpVkHnOgtxvK-8DRhR60WYvVXy37Sh1L3D8,3676
|
|
5
|
+
scrollscout/render.py,sha256=7z9GoQelnYoIS-pwqC3OCwRDDZPWdF7m_7ZAyTWIBBM,1279
|
|
6
|
+
scrollscout-0.1.0.dist-info/licenses/LICENSE,sha256=fqoCMbltjt0-rd4v3jQbTjAL8tT7_DaApF3-mT_yKmM,1081
|
|
7
|
+
scrollscout-0.1.0.dist-info/METADATA,sha256=JPgDV3rZBvTZuAsUXo_G_GXzyOscwv__L4QQv2eL2vk,5747
|
|
8
|
+
scrollscout-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
9
|
+
scrollscout-0.1.0.dist-info/entry_points.txt,sha256=_HPS8ClBc0s4rgwgg_0lFKXTfX4OwYPDU-AOEEQY5rg,53
|
|
10
|
+
scrollscout-0.1.0.dist-info/top_level.txt,sha256=H_5S-xPMRf7KenFZ780pEhRfdEP2FGbfYl76603rwm4,12
|
|
11
|
+
scrollscout-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Alexandre Dufour-Richard
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
scrollscout
|