spatialdata-io 0.5.1__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.github/workflows/prepare_test_data.yaml +10 -1
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.gitignore +3 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.pre-commit-config.yaml +3 -3
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/PKG-INFO +3 -1
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/_version.py +2 -2
- spatialdata_io-0.6.0/asv.conf.json +23 -0
- spatialdata_io-0.6.0/benchmarks/__init__.py +1 -0
- spatialdata_io-0.6.0/benchmarks/benchmark_image.py +180 -0
- spatialdata_io-0.6.0/benchmarks/benchmark_xenium.py +108 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/pyproject.toml +3 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/_constants/_constants.py +1 -3
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/converters/legacy_anndata.py +68 -55
- spatialdata_io-0.6.0/src/spatialdata_io/readers/_utils/_image.py +202 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/_utils/_utils.py +1 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/cosmx.py +1 -1
- spatialdata_io-0.6.0/src/spatialdata_io/readers/generic.py +240 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/macsima.py +372 -43
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/seqfish.py +110 -49
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/steinbock.py +2 -2
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/visium_hd.py +37 -16
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/xenium.py +112 -89
- spatialdata_io-0.6.0/tests/readers/test_utils_image.py +103 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/test_generic.py +69 -0
- spatialdata_io-0.6.0/tests/test_macsima.py +640 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/test_seqfish.py +3 -2
- spatialdata_io-0.5.1/src/spatialdata_io/readers/generic.py +0 -83
- spatialdata_io-0.5.1/tests/test_macsima.py +0 -231
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.bumpversion.cfg +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.codecov.yaml +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.editorconfig +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.github/codecov.yml +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.github/release.yml +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.github/workflows/release.yaml +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.github/workflows/test.yaml +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.mypy.ini +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/.readthedocs.yaml +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/LICENSE +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/README.md +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/Makefile +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/_static/.gitkeep +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/_templates/.gitkeep +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/_templates/autosummary/class.rst +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/api.md +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/changelog.md +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/cli.md +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/conf.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/contributing.md +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/extensions/.gitkeep +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/extensions/typed_returns.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/index.md +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/make.bat +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/references.bib +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/docs/references.md +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/__main__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/_constants/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/_constants/_enum.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/_docs.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/_utils.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/converters/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/converters/generic_to_zarr.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/experimental/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/_utils/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/_utils/_read_10x_h5.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/codex.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/curio.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/dbit.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/iss.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/mcmicro.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/merscope.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/stereoseq.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/readers/visium.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/_utils.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/conftest.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/converters/__init__.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/converters/test_legacy_anndata.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/test_basic.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/test_visium_hd.py +0 -0
- {spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/tests/test_xenium.py +0 -0
|
@@ -4,7 +4,8 @@ on:
|
|
|
4
4
|
schedule:
|
|
5
5
|
- cron: "0 0 1 * *" # run once a month to prevent artifact expiration
|
|
6
6
|
workflow_dispatch:
|
|
7
|
-
#
|
|
7
|
+
# Uncomment and adjust the branch name if you need to add new datasets to the artifact.
|
|
8
|
+
# It needs to be a branch in the spatialdata-io origin repository, not from a fork.
|
|
8
9
|
# push:
|
|
9
10
|
# branches:
|
|
10
11
|
# - main
|
|
@@ -57,6 +58,14 @@ jobs:
|
|
|
57
58
|
# Spatial Genomics seqFISH v2
|
|
58
59
|
curl -O https://s3.embl.de/spatialdata/raw_data/seqfish-2-test-dataset.zip
|
|
59
60
|
|
|
61
|
+
# -------
|
|
62
|
+
# MACSima OMAP datasets are licensed as CC BY 4.0
|
|
63
|
+
# OMAP23 for format v1.x.x
|
|
64
|
+
curl -o OMAP23_small.zip "https://zenodo.org/api/records/18196452/files-archive"
|
|
65
|
+
|
|
66
|
+
# OMAP10 for format v0.x.x
|
|
67
|
+
curl -o OMAP10_small.zip "https://zenodo.org/api/records/18196366/files-archive"
|
|
68
|
+
|
|
60
69
|
- name: Unzip files
|
|
61
70
|
run: |
|
|
62
71
|
cd ./data
|
|
@@ -7,17 +7,17 @@ default_stages:
|
|
|
7
7
|
minimum_pre_commit_version: 2.16.0
|
|
8
8
|
repos:
|
|
9
9
|
- repo: https://github.com/rbubley/mirrors-prettier
|
|
10
|
-
rev: v3.
|
|
10
|
+
rev: v3.7.4
|
|
11
11
|
hooks:
|
|
12
12
|
- id: prettier
|
|
13
13
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
14
|
-
rev: v0.14.
|
|
14
|
+
rev: v0.14.11
|
|
15
15
|
hooks:
|
|
16
16
|
- id: ruff
|
|
17
17
|
args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
|
|
18
18
|
- id: ruff-format
|
|
19
19
|
- repo: https://github.com/pre-commit/mirrors-mypy
|
|
20
|
-
rev: v1.
|
|
20
|
+
rev: v1.19.1
|
|
21
21
|
hooks:
|
|
22
22
|
- id: mypy
|
|
23
23
|
additional_dependencies: [numpy, types-PyYAML]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: spatialdata-io
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: SpatialData IO for common techs
|
|
5
5
|
Project-URL: Documentation, https://spatialdata-io.readthedocs.io/
|
|
6
6
|
Project-URL: Source, https://github.com/scverse/spatialdata-io
|
|
@@ -52,6 +52,7 @@ Requires-Dist: scanpy
|
|
|
52
52
|
Requires-Dist: scikit-image
|
|
53
53
|
Requires-Dist: spatialdata>=0.2.6
|
|
54
54
|
Requires-Dist: tifffile>=2023.8.12
|
|
55
|
+
Requires-Dist: xmltodict
|
|
55
56
|
Provides-Extra: dev
|
|
56
57
|
Requires-Dist: bump2version; extra == 'dev'
|
|
57
58
|
Requires-Dist: pre-commit; extra == 'dev'
|
|
@@ -68,6 +69,7 @@ Requires-Dist: sphinxcontrib-bibtex>=1.0.0; extra == 'doc'
|
|
|
68
69
|
Provides-Extra: pre
|
|
69
70
|
Requires-Dist: spatialdata>=0.4.0rc0; extra == 'pre'
|
|
70
71
|
Provides-Extra: test
|
|
72
|
+
Requires-Dist: pyarrow!=22; extra == 'test'
|
|
71
73
|
Requires-Dist: pytest; extra == 'test'
|
|
72
74
|
Requires-Dist: pytest-cov; extra == 'test'
|
|
73
75
|
Description-Content-Type: text/markdown
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.6.0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 6, 0)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"project": "spatialdata-io",
|
|
4
|
+
"project_url": "https://github.com/scverse/spatialdata-io",
|
|
5
|
+
"repo": ".",
|
|
6
|
+
"branches": ["image-reader-chunkwise"],
|
|
7
|
+
"dvcs": "git",
|
|
8
|
+
"environment_type": "virtualenv",
|
|
9
|
+
"pythons": ["3.12"],
|
|
10
|
+
"build_command": [],
|
|
11
|
+
"install_command": ["python -m pip install {build_dir}[test]"],
|
|
12
|
+
"uninstall_command": ["python -m pip uninstall -y {project}"],
|
|
13
|
+
"env_dir": ".asv/env",
|
|
14
|
+
"results_dir": ".asv/results",
|
|
15
|
+
"html_dir": ".asv/html",
|
|
16
|
+
"benchmark_dir": "benchmarks",
|
|
17
|
+
"hash_length": 8,
|
|
18
|
+
"build_cache_size": 2,
|
|
19
|
+
"install_timeout": 600,
|
|
20
|
+
"repeat": 3,
|
|
21
|
+
"processes": 1,
|
|
22
|
+
"attribute_selection": ["time_*", "peakmem_*"]
|
|
23
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# ASV benchmarks for spatialdata-io
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Benchmarks for SpatialData IO operations for large images.
|
|
2
|
+
|
|
3
|
+
Instructions:
|
|
4
|
+
See benchmark_xenium.py for instructions.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
import logging.handlers
|
|
9
|
+
import tempfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import tifffile
|
|
15
|
+
from spatialdata import SpatialData
|
|
16
|
+
from spatialdata._logging import logger
|
|
17
|
+
from xarray import DataArray
|
|
18
|
+
|
|
19
|
+
from spatialdata_io import image # type: ignore[attr-defined]
|
|
20
|
+
|
|
21
|
+
# =============================================================================
|
|
22
|
+
# CONFIGURATION - Edit these values to match your setup
|
|
23
|
+
# =============================================================================
|
|
24
|
+
# Image dimensions: (channels, height, width)
|
|
25
|
+
IMAGE_SHAPE = (3, 30000, 30000)
|
|
26
|
+
# =============================================================================
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class IOBenchmarkImage:
|
|
30
|
+
"""Benchmark IO read operations with different parameter combinations."""
|
|
31
|
+
|
|
32
|
+
timeout = 3600
|
|
33
|
+
repeat = 3
|
|
34
|
+
number = 1
|
|
35
|
+
warmup_time = 0
|
|
36
|
+
processes = 1
|
|
37
|
+
|
|
38
|
+
# Parameter combinations: scale_factors, (use_tiff_memmap, compressed), chunks
|
|
39
|
+
# Combinations: (memmap=False, compressed=True), (memmap=False, compressed=False), (memmap=True, compressed=False)
|
|
40
|
+
params = [
|
|
41
|
+
[None, [2, 2]], # scale_factors
|
|
42
|
+
[(False, True), (False, False), (True, False)], # (use_tiff_memmap, compressed)
|
|
43
|
+
[(1, 250, 250), (3, 250, 250)], # chunks
|
|
44
|
+
]
|
|
45
|
+
param_names = ["scale_factors", "memmap_compressed", "chunks"]
|
|
46
|
+
|
|
47
|
+
# Class-level temp directory for image files (persists across all benchmarks)
|
|
48
|
+
_images_temp_dir: tempfile.TemporaryDirectory[str] | None = None
|
|
49
|
+
_path_read_uncompressed: Path | None = None
|
|
50
|
+
_path_read_compressed: Path | None = None
|
|
51
|
+
|
|
52
|
+
@classmethod
|
|
53
|
+
def _setup_images(cls) -> None:
|
|
54
|
+
"""Create fake image data once for all benchmarks."""
|
|
55
|
+
if cls._images_temp_dir is not None:
|
|
56
|
+
return
|
|
57
|
+
|
|
58
|
+
cls._images_temp_dir = tempfile.TemporaryDirectory()
|
|
59
|
+
images_dir = Path(cls._images_temp_dir.name)
|
|
60
|
+
cls._path_read_uncompressed = images_dir / "image_uncompressed.tif"
|
|
61
|
+
cls._path_read_compressed = images_dir / "image_compressed.tif"
|
|
62
|
+
|
|
63
|
+
# Generate fake image data
|
|
64
|
+
rng = np.random.default_rng(42)
|
|
65
|
+
data = rng.integers(0, 255, size=IMAGE_SHAPE, dtype=np.uint8)
|
|
66
|
+
|
|
67
|
+
# Write uncompressed TIFF (memmappable)
|
|
68
|
+
tifffile.imwrite(cls._path_read_uncompressed, data, compression=None)
|
|
69
|
+
# Write compressed TIFF (not memmappable)
|
|
70
|
+
tifffile.imwrite(cls._path_read_compressed, data, compression="zlib")
|
|
71
|
+
|
|
72
|
+
def setup(self, *_: Any) -> None:
|
|
73
|
+
"""Set up paths for benchmarking."""
|
|
74
|
+
# Create images once (shared across all benchmark runs)
|
|
75
|
+
self._setup_images()
|
|
76
|
+
self.path_read_uncompressed = self._path_read_uncompressed
|
|
77
|
+
self.path_read_compressed = self._path_read_compressed
|
|
78
|
+
|
|
79
|
+
# Create a separate temp directory for output (cleaned up after each run)
|
|
80
|
+
self._output_temp_dir = tempfile.TemporaryDirectory()
|
|
81
|
+
self.path_write = Path(self._output_temp_dir.name) / "data_benchmark.zarr"
|
|
82
|
+
|
|
83
|
+
def teardown(self, *_: Any) -> None:
|
|
84
|
+
"""Clean up output directory after each benchmark run."""
|
|
85
|
+
if hasattr(self, "_output_temp_dir"):
|
|
86
|
+
self._output_temp_dir.cleanup()
|
|
87
|
+
|
|
88
|
+
def _convert_image(
|
|
89
|
+
self, scale_factors: list[int] | None, memmap_compressed: tuple[bool, bool], chunks: tuple[int, ...]
|
|
90
|
+
) -> SpatialData:
|
|
91
|
+
"""Read image data with specified parameters."""
|
|
92
|
+
use_tiff_memmap, compressed = memmap_compressed
|
|
93
|
+
# Select file based on compression setting
|
|
94
|
+
path_read = self.path_read_compressed if compressed else self.path_read_uncompressed
|
|
95
|
+
assert path_read is not None
|
|
96
|
+
|
|
97
|
+
# Capture log messages to verify memmappable warning behavior
|
|
98
|
+
log_capture = logging.handlers.MemoryHandler(capacity=100)
|
|
99
|
+
log_capture.setLevel(logging.WARNING)
|
|
100
|
+
logger.addHandler(log_capture)
|
|
101
|
+
original_propagate = logger.propagate
|
|
102
|
+
logger.propagate = True
|
|
103
|
+
|
|
104
|
+
try:
|
|
105
|
+
im = image(
|
|
106
|
+
input=path_read,
|
|
107
|
+
data_axes=("c", "y", "x"),
|
|
108
|
+
coordinate_system="global",
|
|
109
|
+
use_tiff_memmap=use_tiff_memmap,
|
|
110
|
+
chunks=chunks,
|
|
111
|
+
scale_factors=scale_factors,
|
|
112
|
+
)
|
|
113
|
+
finally:
|
|
114
|
+
logger.removeHandler(log_capture)
|
|
115
|
+
logger.propagate = original_propagate
|
|
116
|
+
|
|
117
|
+
# Check warning behavior: when use_tiff_memmap=True with uncompressed file, no warning should be raised
|
|
118
|
+
log_messages = [record.getMessage() for record in log_capture.buffer]
|
|
119
|
+
has_memmap_warning = any("image data is not memory-mappable" in msg for msg in log_messages)
|
|
120
|
+
if use_tiff_memmap and not compressed:
|
|
121
|
+
assert not has_memmap_warning, (
|
|
122
|
+
"Uncompressed TIFF with memmap=True should not trigger memory-mappable warning"
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
sdata = SpatialData.init_from_elements({"image": im})
|
|
126
|
+
# sanity check: chunks is (c, y, x)
|
|
127
|
+
if scale_factors is None:
|
|
128
|
+
assert isinstance(sdata["image"], DataArray)
|
|
129
|
+
if chunks is not None:
|
|
130
|
+
assert (
|
|
131
|
+
sdata["image"].chunksizes["x"][0] == chunks[2]
|
|
132
|
+
or sdata["image"].chunksizes["x"][0] == sdata["image"].shape[2]
|
|
133
|
+
)
|
|
134
|
+
assert (
|
|
135
|
+
sdata["image"].chunksizes["y"][0] == chunks[1]
|
|
136
|
+
or sdata["image"].chunksizes["y"][0] == sdata["image"].shape[1]
|
|
137
|
+
)
|
|
138
|
+
else:
|
|
139
|
+
assert len(sdata["image"].keys()) == len(scale_factors) + 1
|
|
140
|
+
if chunks is not None:
|
|
141
|
+
assert (
|
|
142
|
+
sdata["image"]["scale0"]["image"].chunksizes["x"][0] == chunks[2]
|
|
143
|
+
or sdata["image"]["scale0"]["image"].chunksizes["x"][0]
|
|
144
|
+
== sdata["image"]["scale0"]["image"].shape[2]
|
|
145
|
+
)
|
|
146
|
+
assert (
|
|
147
|
+
sdata["image"]["scale0"]["image"].chunksizes["y"][0] == chunks[1]
|
|
148
|
+
or sdata["image"]["scale0"]["image"].chunksizes["y"][0]
|
|
149
|
+
== sdata["image"]["scale0"]["image"].shape[1]
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
return sdata
|
|
153
|
+
|
|
154
|
+
def time_io(
|
|
155
|
+
self, scale_factors: list[int] | None, memmap_compressed: tuple[bool, bool], chunks: tuple[int, ...]
|
|
156
|
+
) -> None:
|
|
157
|
+
"""Walltime for data parsing."""
|
|
158
|
+
sdata = self._convert_image(scale_factors, memmap_compressed, chunks)
|
|
159
|
+
sdata.write(self.path_write)
|
|
160
|
+
|
|
161
|
+
def peakmem_io(
|
|
162
|
+
self, scale_factors: list[int] | None, memmap_compressed: tuple[bool, bool], chunks: tuple[int, ...]
|
|
163
|
+
) -> None:
|
|
164
|
+
"""Peak memory for data parsing."""
|
|
165
|
+
sdata = self._convert_image(scale_factors, memmap_compressed, chunks)
|
|
166
|
+
sdata.write(self.path_write)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
# if __name__ == "__main__":
|
|
170
|
+
# # Run a single test case for quick verification
|
|
171
|
+
# bench = IOBenchmarkImage()
|
|
172
|
+
#
|
|
173
|
+
# bench.setup()
|
|
174
|
+
# bench.time_io(None, (True, False), (1, 5000, 5000))
|
|
175
|
+
# bench.teardown()
|
|
176
|
+
#
|
|
177
|
+
# # Clean up the shared images temp directory at the end
|
|
178
|
+
# if IOBenchmarkImage._images_temp_dir is not None:
|
|
179
|
+
# IOBenchmarkImage._images_temp_dir.cleanup()
|
|
180
|
+
# IOBenchmarkImage._images_temp_dir = None
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Benchmarks for SpatialData IO operations.
|
|
2
|
+
|
|
3
|
+
Configuration:
|
|
4
|
+
Edit SANDBOX_DIR and DATASET below to point to your data.
|
|
5
|
+
|
|
6
|
+
Setup:
|
|
7
|
+
cd <SANDBOX_DIR>/<DATASET>
|
|
8
|
+
python download.py # use the same env where spatialdata is installed
|
|
9
|
+
|
|
10
|
+
Running:
|
|
11
|
+
cd /path/to/spatialdata-io
|
|
12
|
+
|
|
13
|
+
# Quick benchmark (single run, for testing):
|
|
14
|
+
asv run --python=same -b IOBenchmarkXenium --quick --show-stderr -v
|
|
15
|
+
|
|
16
|
+
# Full benchmark (multiple runs, for accurate results):
|
|
17
|
+
asv run --python=same -b IOBenchmarkXenium --show-stderr -v
|
|
18
|
+
|
|
19
|
+
Comparing branches:
|
|
20
|
+
# Run on specific commits:
|
|
21
|
+
asv run main^! -b IOBenchmarkXenium --show-stderr -v
|
|
22
|
+
asv run xenium-labels-dask^! -b IOBenchmarkXenium --show-stderr -v
|
|
23
|
+
|
|
24
|
+
# Or compare two branches directly:
|
|
25
|
+
asv continuous main xenium-labels-dask -b IOBenchmarkXenium --show-stderr -v
|
|
26
|
+
|
|
27
|
+
# View comparison:
|
|
28
|
+
asv compare main xenium-labels-dask
|
|
29
|
+
|
|
30
|
+
Results:
|
|
31
|
+
- Console output shows timing and memory after each run
|
|
32
|
+
- JSON results saved to: .asv/results/
|
|
33
|
+
- Generate HTML report: asv publish && asv preview
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
import inspect
|
|
37
|
+
import shutil
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
|
|
40
|
+
from spatialdata import SpatialData
|
|
41
|
+
|
|
42
|
+
from spatialdata_io import xenium # type: ignore[attr-defined]
|
|
43
|
+
|
|
44
|
+
# =============================================================================
|
|
45
|
+
# CONFIGURATION - Edit these paths to match your setup
|
|
46
|
+
# =============================================================================
|
|
47
|
+
SANDBOX_DIR = Path(__file__).parent.parent.parent / "spatialdata-sandbox"
|
|
48
|
+
DATASET = "xenium_2.0.0_io"
|
|
49
|
+
# =============================================================================
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def get_paths() -> tuple[Path, Path]:
|
|
53
|
+
"""Get paths for benchmark data."""
|
|
54
|
+
path = SANDBOX_DIR / DATASET
|
|
55
|
+
path_read = path / "data"
|
|
56
|
+
path_write = path / "data_benchmark.zarr"
|
|
57
|
+
|
|
58
|
+
if not path_read.exists():
|
|
59
|
+
raise ValueError(f"Data directory not found: {path_read}")
|
|
60
|
+
|
|
61
|
+
return path_read, path_write
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class IOBenchmarkXenium:
|
|
65
|
+
timeout = 3600
|
|
66
|
+
repeat = 3
|
|
67
|
+
number = 1
|
|
68
|
+
warmup_time = 0
|
|
69
|
+
processes = 1
|
|
70
|
+
|
|
71
|
+
def setup(self) -> None:
|
|
72
|
+
"""Set up paths for benchmarking."""
|
|
73
|
+
self.path_read, self.path_write = get_paths()
|
|
74
|
+
if self.path_write.exists():
|
|
75
|
+
shutil.rmtree(self.path_write)
|
|
76
|
+
|
|
77
|
+
def _read_xenium(self) -> SpatialData:
|
|
78
|
+
"""Read xenium data with version-compatible kwargs."""
|
|
79
|
+
signature = inspect.signature(xenium)
|
|
80
|
+
kwargs = {}
|
|
81
|
+
if "cleanup_labels_zarr_tmpdir" in signature.parameters:
|
|
82
|
+
kwargs["cleanup_labels_zarr_tmpdir"] = False
|
|
83
|
+
|
|
84
|
+
return xenium(
|
|
85
|
+
path=str(self.path_read),
|
|
86
|
+
n_jobs=8,
|
|
87
|
+
cell_boundaries=True,
|
|
88
|
+
nucleus_boundaries=True,
|
|
89
|
+
morphology_focus=True,
|
|
90
|
+
cells_as_circles=True,
|
|
91
|
+
**kwargs,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
def time_io(self) -> None:
|
|
95
|
+
"""Walltime for data parsing."""
|
|
96
|
+
sdata = self._read_xenium()
|
|
97
|
+
sdata.write(self.path_write)
|
|
98
|
+
|
|
99
|
+
def peakmem_io(self) -> None:
|
|
100
|
+
"""Peak memory for data parsing."""
|
|
101
|
+
sdata = self._read_xenium()
|
|
102
|
+
sdata.write(self.path_write)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
if __name__ == "__main__":
|
|
106
|
+
benchmark = IOBenchmarkXenium()
|
|
107
|
+
benchmark.setup()
|
|
108
|
+
benchmark.time_io()
|
|
@@ -36,6 +36,7 @@ dependencies = [
|
|
|
36
36
|
"readfcs",
|
|
37
37
|
"tifffile>=2023.8.12",
|
|
38
38
|
"ome-types",
|
|
39
|
+
"xmltodict",
|
|
39
40
|
]
|
|
40
41
|
|
|
41
42
|
[project.optional-dependencies]
|
|
@@ -59,6 +60,8 @@ doc = [
|
|
|
59
60
|
test = [
|
|
60
61
|
"pytest",
|
|
61
62
|
"pytest-cov",
|
|
63
|
+
# https://github.com/scverse/spatialdata-io/issues/334
|
|
64
|
+
"pyarrow!=22"
|
|
62
65
|
]
|
|
63
66
|
# this will be used by readthedocs and will make pip also look for pre-releases, generally installing the latest available version
|
|
64
67
|
# update: readthedocs doens't seem to try to install pre-releases even if when trying to install the pre optional-dependency. For
|
|
@@ -68,7 +68,6 @@ class SeqfishKeys(ModeEnum):
|
|
|
68
68
|
TIFF_FILE = ".tiff"
|
|
69
69
|
GEOJSON_FILE = ".geojson"
|
|
70
70
|
# file identifiers
|
|
71
|
-
ROI = "Roi"
|
|
72
71
|
TRANSCRIPT_COORDINATES = "TranscriptList"
|
|
73
72
|
DAPI = "DAPI"
|
|
74
73
|
COUNTS_FILE = "CellxGene"
|
|
@@ -78,6 +77,7 @@ class SeqfishKeys(ModeEnum):
|
|
|
78
77
|
# transcripts
|
|
79
78
|
TRANSCRIPTS_X = "x"
|
|
80
79
|
TRANSCRIPTS_Y = "y"
|
|
80
|
+
TRANSCRIPTS_Z = "z"
|
|
81
81
|
FEATURE_KEY = "name"
|
|
82
82
|
INSTANCE_KEY_POINTS = "cell"
|
|
83
83
|
# cells
|
|
@@ -88,8 +88,6 @@ class SeqfishKeys(ModeEnum):
|
|
|
88
88
|
SPATIAL_KEY = "spatial"
|
|
89
89
|
REGION_KEY = "region"
|
|
90
90
|
INSTANCE_KEY_TABLE = "instance_id"
|
|
91
|
-
SCALEFEFACTOR_X = "PhysicalSizeX"
|
|
92
|
-
SCALEFEFACTOR_Y = "PhysicalSizeY"
|
|
93
91
|
|
|
94
92
|
|
|
95
93
|
@unique
|
{spatialdata_io-0.5.1 → spatialdata_io-0.6.0}/src/spatialdata_io/converters/legacy_anndata.py
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
from collections.abc import Mapping
|
|
4
4
|
from typing import TYPE_CHECKING
|
|
5
5
|
|
|
6
6
|
import numpy as np
|
|
@@ -12,6 +12,7 @@ from spatialdata import (
|
|
|
12
12
|
to_circles,
|
|
13
13
|
)
|
|
14
14
|
from spatialdata._core.operations._utils import transform_to_data_extent
|
|
15
|
+
from spatialdata._logging import logger
|
|
15
16
|
from spatialdata.models import Image2DModel, ShapesModel, TableModel, get_table_keys
|
|
16
17
|
from spatialdata.transformations import Identity, Scale
|
|
17
18
|
|
|
@@ -220,7 +221,7 @@ def to_legacy_anndata(
|
|
|
220
221
|
return adata
|
|
221
222
|
|
|
222
223
|
|
|
223
|
-
def from_legacy_anndata(adata: AnnData) -> SpatialData:
|
|
224
|
+
def from_legacy_anndata(adata: AnnData, rgb: bool | None = None) -> SpatialData:
|
|
224
225
|
"""Convert (legacy) spatial AnnData object to SpatialData object.
|
|
225
226
|
|
|
226
227
|
This is useful for parsing a (legacy) spatial AnnData object, for example the ones produced by Scanpy and older
|
|
@@ -230,6 +231,9 @@ def from_legacy_anndata(adata: AnnData) -> SpatialData:
|
|
|
230
231
|
----------
|
|
231
232
|
adata
|
|
232
233
|
(legacy) spatial AnnData object
|
|
234
|
+
rgb
|
|
235
|
+
Argument passed to `spatialdata.models.Image2DModel.parse()`. If `None`,
|
|
236
|
+
3-(4) channels images will be interpreted as RGB(A).
|
|
233
237
|
|
|
234
238
|
Returns
|
|
235
239
|
-------
|
|
@@ -250,6 +254,7 @@ def from_legacy_anndata(adata: AnnData) -> SpatialData:
|
|
|
250
254
|
IMAGES = "images"
|
|
251
255
|
HIRES = "hires"
|
|
252
256
|
LOWRES = "lowres"
|
|
257
|
+
DEFAULT_SCALE_FACTOR = 1.0
|
|
253
258
|
|
|
254
259
|
# SpatialData keys
|
|
255
260
|
REGION = "locations"
|
|
@@ -265,72 +270,80 @@ def from_legacy_anndata(adata: AnnData) -> SpatialData:
|
|
|
265
270
|
if SPATIAL in adata.uns:
|
|
266
271
|
dataset_ids = list(adata.uns[SPATIAL].keys())
|
|
267
272
|
for dataset_id in dataset_ids:
|
|
268
|
-
|
|
269
|
-
keys = set(
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
if
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
273
|
+
dataset_data = adata.uns[SPATIAL][dataset_id]
|
|
274
|
+
keys = set(dataset_data.keys())
|
|
275
|
+
scalefactors_raw = dataset_data.get(SCALEFACTORS, {}) if isinstance(dataset_data, Mapping) else {}
|
|
276
|
+
scalefactors = scalefactors_raw if isinstance(scalefactors_raw, Mapping) else {}
|
|
277
|
+
|
|
278
|
+
if SPOT_DIAMETER_FULLRES in scalefactors:
|
|
279
|
+
spot_diameter_fullres_list.append(scalefactors[SPOT_DIAMETER_FULLRES])
|
|
280
|
+
|
|
281
|
+
image_data = dataset_data.get(IMAGES, {}) if IMAGES in keys and isinstance(dataset_data, Mapping) else {}
|
|
282
|
+
if not isinstance(image_data, Mapping):
|
|
283
|
+
image_data = {}
|
|
284
|
+
for image_key, image_value in image_data.items():
|
|
285
|
+
if image_key not in (HIRES, LOWRES):
|
|
286
|
+
logger.warning(
|
|
287
|
+
"Found non-standard image key '%s' in dataset '%s' - attempting to parse.",
|
|
288
|
+
image_key,
|
|
289
|
+
dataset_id,
|
|
290
|
+
)
|
|
291
|
+
# prefer scalefactors keyed by the image name, fall back to tissue_<image_key>_scalef
|
|
292
|
+
# then to legacy hires/lowres names, finally default to 1.0.
|
|
293
|
+
scalefactor = None
|
|
294
|
+
scalefactor_source = None
|
|
295
|
+
if image_key in scalefactors:
|
|
296
|
+
scalefactor = scalefactors[image_key]
|
|
297
|
+
scalefactor_source = image_key
|
|
298
|
+
elif f"tissue_{image_key}_scalef" in scalefactors:
|
|
299
|
+
scalefactor = scalefactors[f"tissue_{image_key}_scalef"]
|
|
300
|
+
scalefactor_source = f"tissue_{image_key}_scalef"
|
|
301
|
+
elif image_key == HIRES and TISSUE_HIRES_SCALEF in scalefactors:
|
|
302
|
+
scalefactor = scalefactors[TISSUE_HIRES_SCALEF]
|
|
303
|
+
scalefactor_source = TISSUE_HIRES_SCALEF
|
|
304
|
+
elif image_key == LOWRES and TISSUE_LOWRES_SCALEF in scalefactors:
|
|
305
|
+
scalefactor = scalefactors[TISSUE_LOWRES_SCALEF]
|
|
306
|
+
scalefactor_source = TISSUE_LOWRES_SCALEF
|
|
307
|
+
|
|
308
|
+
if scalefactor is None:
|
|
309
|
+
scalefactor = DEFAULT_SCALE_FACTOR
|
|
310
|
+
logger.warning(
|
|
311
|
+
"Scale factor missing for image '%s' in dataset '%s'; defaulting to %s",
|
|
312
|
+
image_key,
|
|
313
|
+
dataset_id,
|
|
314
|
+
DEFAULT_SCALE_FACTOR,
|
|
315
|
+
)
|
|
316
|
+
else:
|
|
317
|
+
logger.debug(
|
|
318
|
+
"Using scalefactor '%s' for image '%s' in dataset '%s'",
|
|
319
|
+
scalefactor_source,
|
|
320
|
+
image_key,
|
|
321
|
+
dataset_id,
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
transform_name = f"{dataset_id}_{image_key}"
|
|
325
|
+
image_name = f"{dataset_id}_{image_key}"
|
|
326
|
+
images[image_name] = Image2DModel.parse(
|
|
327
|
+
image_value,
|
|
328
|
+
dims=("y", "x", "c"),
|
|
329
|
+
transformations={transform_name: Identity()},
|
|
330
|
+
rgb=rgb,
|
|
294
331
|
)
|
|
295
|
-
|
|
296
|
-
hires, dims=("y", "x", "c"), transformations={f"{dataset_id}_downscaled_hires": Identity()}
|
|
297
|
-
)
|
|
298
|
-
images[f"{dataset_id}_hires_image"] = hires_image
|
|
299
|
-
|
|
300
|
-
# prepare the transformation to the hires image for the shapes
|
|
301
|
-
scale_hires = Scale([tissue_hires_scalef, tissue_hires_scalef], axes=("x", "y"))
|
|
302
|
-
shapes_transformations[f"{dataset_id}_downscaled_hires"] = scale_hires
|
|
303
|
-
if lowres is not None:
|
|
304
|
-
# prepare the lowres image
|
|
305
|
-
assert tissue_lowres_scalef is not None, (
|
|
306
|
-
"tissue_lowres_scalef is required when an the lowres image is present"
|
|
307
|
-
)
|
|
308
|
-
lowres_image = Image2DModel.parse(
|
|
309
|
-
lowres, dims=("y", "x", "c"), transformations={f"{dataset_id}_downscaled_lowres": Identity()}
|
|
310
|
-
)
|
|
311
|
-
images[f"{dataset_id}_lowres_image"] = lowres_image
|
|
312
|
-
|
|
313
|
-
# prepare the transformation to the lowres image for the shapes
|
|
314
|
-
scale_lowres = Scale([tissue_lowres_scalef, tissue_lowres_scalef], axes=("x", "y"))
|
|
315
|
-
shapes_transformations[f"{dataset_id}_downscaled_lowres"] = scale_lowres
|
|
332
|
+
shapes_transformations[transform_name] = Scale([scalefactor, scalefactor], axes=("x", "y"))
|
|
316
333
|
|
|
317
334
|
# validate the spot_diameter_fullres value
|
|
318
335
|
if len(spot_diameter_fullres_list) > 0:
|
|
319
336
|
d = np.array(spot_diameter_fullres_list)
|
|
320
337
|
if not np.allclose(d, d[0]):
|
|
321
|
-
|
|
338
|
+
logger.warning(
|
|
322
339
|
"spot_diameter_fullres is not constant across datasets. Using the average value.",
|
|
323
|
-
UserWarning,
|
|
324
|
-
stacklevel=2,
|
|
325
340
|
)
|
|
326
341
|
spot_diameter_fullres = d.mean()
|
|
327
342
|
else:
|
|
328
343
|
spot_diameter_fullres = d[0]
|
|
329
344
|
else:
|
|
330
|
-
|
|
345
|
+
logger.warning(
|
|
331
346
|
f"spot_diameter_fullres is not present. Using {SPOT_DIAMETER_FULLRES_DEFAULT} as default value.",
|
|
332
|
-
UserWarning,
|
|
333
|
-
stacklevel=2,
|
|
334
347
|
)
|
|
335
348
|
spot_diameter_fullres = SPOT_DIAMETER_FULLRES_DEFAULT
|
|
336
349
|
|