shortcog 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shortcog-0.5.0/.gitignore +69 -0
- shortcog-0.5.0/PKG-INFO +31 -0
- shortcog-0.5.0/README.md +3 -0
- shortcog-0.5.0/hatch_build.py +20 -0
- shortcog-0.5.0/pyproject.toml +89 -0
- shortcog-0.5.0/shortcog/__init__.py +9 -0
- shortcog-0.5.0/shortcog/_ffi.py +141 -0
- shortcog-0.5.0/shortcog/_read.py +161 -0
- shortcog-0.5.0/shortcog/_repr.py +114 -0
- shortcog-0.5.0/shortcog/_spec.py +85 -0
- shortcog-0.5.0/tests/.gitkeep +0 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.egg-info/
|
|
6
|
+
.pytest_cache/
|
|
7
|
+
.mypy_cache/
|
|
8
|
+
.ruff_cache/
|
|
9
|
+
.tox/
|
|
10
|
+
.coverage
|
|
11
|
+
coverage.xml
|
|
12
|
+
htmlcov/
|
|
13
|
+
.hypothesis/
|
|
14
|
+
|
|
15
|
+
# c++ / cmake
|
|
16
|
+
build/
|
|
17
|
+
build-*/
|
|
18
|
+
cmake-build-*/
|
|
19
|
+
*.o
|
|
20
|
+
*.obj
|
|
21
|
+
*.so
|
|
22
|
+
*.so.*
|
|
23
|
+
*.dylib
|
|
24
|
+
*.dll
|
|
25
|
+
*.a
|
|
26
|
+
*.lib
|
|
27
|
+
*.pdb
|
|
28
|
+
*.exp
|
|
29
|
+
CMakeCache.txt
|
|
30
|
+
CMakeFiles/
|
|
31
|
+
CMakeUserPresets.json
|
|
32
|
+
cmake_install.cmake
|
|
33
|
+
*.ninja
|
|
34
|
+
.ninja_deps
|
|
35
|
+
.ninja_log
|
|
36
|
+
compile_commands.json
|
|
37
|
+
.cache/
|
|
38
|
+
|
|
39
|
+
# virtualenvs
|
|
40
|
+
.venv/
|
|
41
|
+
venv/
|
|
42
|
+
env/
|
|
43
|
+
|
|
44
|
+
# distribution
|
|
45
|
+
dist/
|
|
46
|
+
wheelhouse/
|
|
47
|
+
*.whl
|
|
48
|
+
|
|
49
|
+
# editors
|
|
50
|
+
.vscode/
|
|
51
|
+
.idea/
|
|
52
|
+
*.swp
|
|
53
|
+
*.swo
|
|
54
|
+
*~
|
|
55
|
+
|
|
56
|
+
# os
|
|
57
|
+
.DS_Store
|
|
58
|
+
Thumbs.db
|
|
59
|
+
desktop.ini
|
|
60
|
+
|
|
61
|
+
# misc
|
|
62
|
+
*.log
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# test data
|
|
66
|
+
core/tests/fixtures/data/
|
|
67
|
+
demo.py
|
|
68
|
+
|
|
69
|
+
T30TYK_B04_shortcog.tif
|
shortcog-0.5.0/PKG-INFO
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: shortcog
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: A COG profile for AI training data.
|
|
5
|
+
Project-URL: Homepage, https://github.com/asterisk-labs/shortcog
|
|
6
|
+
Project-URL: Repository, https://github.com/asterisk-labs/shortcog
|
|
7
|
+
Project-URL: Issues, https://github.com/asterisk-labs/shortcog/issues
|
|
8
|
+
Author-email: Cesar Aybar <cesar@asterisk.coop>
|
|
9
|
+
License: MIT
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: C++
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.12
|
|
23
|
+
Requires-Dist: cffi>=1.17
|
|
24
|
+
Requires-Dist: numpy>=1.24
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# shortcog (python)
|
|
30
|
+
|
|
31
|
+
Python bindings for the shortcog reader. See the top level `README.md` for the public API and usage examples.
|
shortcog-0.5.0/README.md
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from hatchling.builders.hooks.plugin.interface import BuildHookInterface
|
|
4
|
+
|
|
5
|
+
_LIB_GLOBS = ("*.so", "*.so.*", "*.dylib", "*.dll")
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CustomBuildHook(BuildHookInterface):
|
|
9
|
+
def initialize(self, version, build_data):
|
|
10
|
+
lib_dir = Path(self.root) / "shortcog" / "_lib"
|
|
11
|
+
found = [p for g in _LIB_GLOBS for p in lib_dir.glob(g)]
|
|
12
|
+
if not found:
|
|
13
|
+
raise RuntimeError(
|
|
14
|
+
f"no native library in {lib_dir}. Build it first: "
|
|
15
|
+
"`make lib` from the repo root."
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
# Binary inside, so tag the wheel for this platform, not py3-none-any.
|
|
19
|
+
build_data["pure_python"] = False
|
|
20
|
+
build_data["infer_tag"] = True
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
[project]
|
|
7
|
+
name = "shortcog"
|
|
8
|
+
dynamic = ["version"]
|
|
9
|
+
description = "A COG profile for AI training data."
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
requires-python = ">=3.12"
|
|
13
|
+
|
|
14
|
+
authors = [
|
|
15
|
+
{ name = "Cesar Aybar", email = "cesar@asterisk.coop" },
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
dependencies = [
|
|
19
|
+
"numpy>=1.24",
|
|
20
|
+
"cffi>=1.17",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Development Status :: 3 - Alpha",
|
|
25
|
+
"Intended Audience :: Developers",
|
|
26
|
+
"Intended Audience :: Science/Research",
|
|
27
|
+
"License :: OSI Approved :: MIT License",
|
|
28
|
+
"Operating System :: MacOS",
|
|
29
|
+
"Operating System :: POSIX :: Linux",
|
|
30
|
+
"Programming Language :: C++",
|
|
31
|
+
"Programming Language :: Python :: 3",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Topic :: Scientific/Engineering :: GIS",
|
|
35
|
+
"Typing :: Typed",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
test = [
|
|
40
|
+
"pytest>=7",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/asterisk-labs/shortcog"
|
|
45
|
+
Repository = "https://github.com/asterisk-labs/shortcog"
|
|
46
|
+
Issues = "https://github.com/asterisk-labs/shortcog/issues"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
[tool.hatch.version]
|
|
50
|
+
path = "../../VERSION"
|
|
51
|
+
pattern = "(?P<version>\\S+)"
|
|
52
|
+
|
|
53
|
+
[tool.hatch.build.targets.wheel]
|
|
54
|
+
packages = ["shortcog"]
|
|
55
|
+
artifacts = [
|
|
56
|
+
"shortcog/_lib/*.so",
|
|
57
|
+
"shortcog/_lib/*.so.*",
|
|
58
|
+
"shortcog/_lib/*.dylib",
|
|
59
|
+
"shortcog/_lib/*.dll",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
[tool.hatch.build.targets.wheel.hooks.custom]
|
|
63
|
+
path = "hatch_build.py"
|
|
64
|
+
|
|
65
|
+
[tool.hatch.build.targets.sdist]
|
|
66
|
+
include = [
|
|
67
|
+
"shortcog/**",
|
|
68
|
+
"tests/**",
|
|
69
|
+
"hatch_build.py",
|
|
70
|
+
"README.md",
|
|
71
|
+
]
|
|
72
|
+
exclude = [
|
|
73
|
+
"shortcog/_lib/*.so",
|
|
74
|
+
"shortcog/_lib/*.so.*",
|
|
75
|
+
"shortcog/_lib/*.dylib",
|
|
76
|
+
"shortcog/_lib/*.dll",
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
[tool.pytest.ini_options]
|
|
81
|
+
testpaths = ["tests"]
|
|
82
|
+
addopts = "-ra --strict-markers"
|
|
83
|
+
|
|
84
|
+
[tool.ruff]
|
|
85
|
+
line-length = 100
|
|
86
|
+
target-version = "py312"
|
|
87
|
+
|
|
88
|
+
[tool.ruff.lint]
|
|
89
|
+
select = ["E", "F", "I", "B", "UP", "NPY"]
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""shortcog, a COG profile for AI training data."""
|
|
2
|
+
|
|
3
|
+
from ._ffi import ffi, lib
|
|
4
|
+
from ._read import index_file, parse, read
|
|
5
|
+
from ._spec import Spec
|
|
6
|
+
|
|
7
|
+
__all__ = ["Spec", "__version__", "index_file", "parse", "read"]
|
|
8
|
+
|
|
9
|
+
__version__ = ffi.string(lib.shortcog_version_string()).decode("ascii")
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import ctypes.util
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from cffi import FFI
|
|
6
|
+
|
|
7
|
+
# The C definitions for the API.
|
|
8
|
+
_CDEF = """
|
|
9
|
+
typedef enum {
|
|
10
|
+
SHORTCOG_OK = 0,
|
|
11
|
+
SHORTCOG_ERR_INVALID = 1,
|
|
12
|
+
SHORTCOG_ERR_IO = 2,
|
|
13
|
+
SHORTCOG_ERR_PARSE = 3,
|
|
14
|
+
SHORTCOG_ERR_FORMAT = 4,
|
|
15
|
+
SHORTCOG_ERR_DECODE = 5,
|
|
16
|
+
SHORTCOG_ERR_OOM = 6,
|
|
17
|
+
SHORTCOG_ERR_UNSUPPORTED = 7,
|
|
18
|
+
SHORTCOG_ERR_INTERNAL = 99
|
|
19
|
+
} shortcog_status;
|
|
20
|
+
|
|
21
|
+
typedef struct {
|
|
22
|
+
uint32_t image_width;
|
|
23
|
+
uint32_t image_length;
|
|
24
|
+
uint16_t tile_width;
|
|
25
|
+
uint16_t tile_length;
|
|
26
|
+
uint16_t samples_per_pixel;
|
|
27
|
+
uint8_t bits_per_sample;
|
|
28
|
+
uint8_t sample_format;
|
|
29
|
+
uint8_t predictor;
|
|
30
|
+
uint8_t _reserved;
|
|
31
|
+
uint32_t tiles_across;
|
|
32
|
+
uint32_t tiles_down;
|
|
33
|
+
uint64_t base_tiles_offset;
|
|
34
|
+
} shortcog_header;
|
|
35
|
+
|
|
36
|
+
typedef struct {
|
|
37
|
+
int64_t shape[4];
|
|
38
|
+
int ndim;
|
|
39
|
+
int64_t sn;
|
|
40
|
+
int64_t sb;
|
|
41
|
+
int64_t sy;
|
|
42
|
+
int64_t sx;
|
|
43
|
+
int native;
|
|
44
|
+
} shortcog_layout;
|
|
45
|
+
|
|
46
|
+
typedef struct shortcog_spec shortcog_spec;
|
|
47
|
+
|
|
48
|
+
int shortcog_api_version(void);
|
|
49
|
+
const char* shortcog_version_string(void);
|
|
50
|
+
const char* shortcog_last_error(void);
|
|
51
|
+
void shortcog_clear_error(void);
|
|
52
|
+
void shortcog_free(void* ptr);
|
|
53
|
+
|
|
54
|
+
shortcog_status
|
|
55
|
+
shortcog_index_file(const char* path, unsigned char** out_blob, size_t* out_size);
|
|
56
|
+
|
|
57
|
+
shortcog_status
|
|
58
|
+
shortcog_compile_layout(const char* pattern,
|
|
59
|
+
int64_t n, int64_t b, int64_t y, int64_t x,
|
|
60
|
+
shortcog_layout* out);
|
|
61
|
+
|
|
62
|
+
shortcog_status
|
|
63
|
+
shortcog_spec_parse(const unsigned char* blob, size_t blob_size,
|
|
64
|
+
shortcog_spec** out);
|
|
65
|
+
|
|
66
|
+
void shortcog_spec_destroy(shortcog_spec* spec);
|
|
67
|
+
|
|
68
|
+
shortcog_status
|
|
69
|
+
shortcog_spec_header(const shortcog_spec* spec, shortcog_header* out);
|
|
70
|
+
|
|
71
|
+
shortcog_status
|
|
72
|
+
shortcog_read(const char* path, const shortcog_spec* spec,
|
|
73
|
+
const int* bands, size_t n_bands,
|
|
74
|
+
int y_off, int y_size, int x_off, int x_size,
|
|
75
|
+
const char* pattern, int num_threads,
|
|
76
|
+
void* dst, size_t dst_size);
|
|
77
|
+
|
|
78
|
+
shortcog_status
|
|
79
|
+
shortcog_read_stack(const char* const* paths,
|
|
80
|
+
const shortcog_spec* const* specs, size_t n_images,
|
|
81
|
+
const int* n_index, size_t n_n,
|
|
82
|
+
const int* bands, size_t n_bands,
|
|
83
|
+
int y_off, int y_size, int x_off, int x_size,
|
|
84
|
+
const char* pattern, int num_threads,
|
|
85
|
+
void* dst, size_t dst_size);
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
ffi = FFI()
|
|
90
|
+
ffi.cdef(_CDEF)
|
|
91
|
+
|
|
92
|
+
_LIB_GLOBS = ("*.so", "*.so.*", "*.dylib", "*.dll")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _bundled_lib():
|
|
96
|
+
lib_dir = Path(__file__).parent / "_lib"
|
|
97
|
+
for pattern in _LIB_GLOBS:
|
|
98
|
+
for path in sorted(lib_dir.glob(pattern)):
|
|
99
|
+
return str(path)
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _load_lib():
|
|
104
|
+
# SHORTCOG_LIB beats everything, then the bundled wheel copy, then the OS path.
|
|
105
|
+
env_path = os.environ.get("SHORTCOG_LIB")
|
|
106
|
+
candidate = env_path or _bundled_lib() or ctypes.util.find_library("shortcog")
|
|
107
|
+
if candidate is None:
|
|
108
|
+
raise OSError(
|
|
109
|
+
"libshortcog not found. Install it or set SHORTCOG_LIB to its path."
|
|
110
|
+
)
|
|
111
|
+
try:
|
|
112
|
+
return ffi.dlopen(candidate)
|
|
113
|
+
except OSError as exc:
|
|
114
|
+
raise OSError(
|
|
115
|
+
f"failed to load libshortcog from {candidate!r}: {exc}. "
|
|
116
|
+
"It links GDAL; put a compatible libgdal on the loader path."
|
|
117
|
+
) from exc
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
lib = _load_lib()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
_STATUS_TO_EXC = {
|
|
124
|
+
lib.SHORTCOG_ERR_INVALID: ValueError,
|
|
125
|
+
lib.SHORTCOG_ERR_IO: IOError,
|
|
126
|
+
lib.SHORTCOG_ERR_PARSE: ValueError,
|
|
127
|
+
lib.SHORTCOG_ERR_FORMAT: ValueError,
|
|
128
|
+
lib.SHORTCOG_ERR_DECODE: IOError,
|
|
129
|
+
lib.SHORTCOG_ERR_OOM: MemoryError,
|
|
130
|
+
lib.SHORTCOG_ERR_UNSUPPORTED: NotImplementedError,
|
|
131
|
+
lib.SHORTCOG_ERR_INTERNAL: RuntimeError,
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _check(rc):
|
|
136
|
+
if rc == lib.SHORTCOG_OK:
|
|
137
|
+
return
|
|
138
|
+
err = lib.shortcog_last_error()
|
|
139
|
+
msg = (ffi.string(err).decode("utf-8", errors="replace")
|
|
140
|
+
if err != ffi.NULL else "(no error message)")
|
|
141
|
+
raise _STATUS_TO_EXC.get(rc, RuntimeError)(msg)
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from collections.abc import Sequence
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from ._ffi import _check, ffi, lib
|
|
7
|
+
from ._spec import Spec
|
|
8
|
+
|
|
9
|
+
# (start, stop) slice, explicit 0-based indices, or None for all.
|
|
10
|
+
Axis = tuple[int, int] | list[int] | None
|
|
11
|
+
PathLike = str | bytes | os.PathLike
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _enc(path: PathLike) -> bytes:
|
|
15
|
+
return path.encode("utf-8") if isinstance(path, str) else os.fsencode(path)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# Tuple selectors are Python slices (start, stop), 0-based. Lists are 0-based
|
|
19
|
+
# explicit indices in the order requested. Both convert to 1-based for the C
|
|
20
|
+
# API. None means all and gets passed through as NULL/0.
|
|
21
|
+
def _resolve_axis(sel: Axis, name: str, total: int) -> list[int] | None:
|
|
22
|
+
if sel is None:
|
|
23
|
+
return None
|
|
24
|
+
if isinstance(sel, tuple):
|
|
25
|
+
if len(sel) != 2:
|
|
26
|
+
raise ValueError(f"{name}: tuple must be (start, stop)")
|
|
27
|
+
start, stop = sel
|
|
28
|
+
if not (0 <= start < stop <= total):
|
|
29
|
+
raise ValueError(f"{name}: slice ({start}, {stop}) out of [0, {total}]")
|
|
30
|
+
return list(range(start + 1, stop + 1))
|
|
31
|
+
if isinstance(sel, list):
|
|
32
|
+
out = [int(i) + 1 for i in sel]
|
|
33
|
+
for x in out:
|
|
34
|
+
if not (1 <= x <= total):
|
|
35
|
+
raise ValueError(f"{name}: index {x - 1} out of [0, {total})")
|
|
36
|
+
return out
|
|
37
|
+
raise TypeError(f"{name}: expected tuple or list, got {type(sel).__name__}")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _resolve_window(sel: tuple[int, int] | None, name: str,
|
|
41
|
+
total: int) -> tuple[int, int]:
|
|
42
|
+
if sel is None:
|
|
43
|
+
return 0, total
|
|
44
|
+
if isinstance(sel, tuple) and len(sel) == 2:
|
|
45
|
+
start, stop = sel
|
|
46
|
+
if not (0 <= start < stop <= total):
|
|
47
|
+
raise ValueError(f"{name}: window ({start}, {stop}) out of [0, {total}]")
|
|
48
|
+
return start, stop - start
|
|
49
|
+
raise TypeError(f"{name}: expected (start, stop) tuple")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _to_c(lst: list[int] | None):
|
|
53
|
+
if lst is None:
|
|
54
|
+
return ffi.NULL, 0
|
|
55
|
+
return ffi.new("int[]", lst), len(lst)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def index_file(path: PathLike) -> bytes:
|
|
59
|
+
blob_out = ffi.new("unsigned char**")
|
|
60
|
+
size_out = ffi.new("size_t*")
|
|
61
|
+
_check(lib.shortcog_index_file(_enc(path), blob_out, size_out))
|
|
62
|
+
try:
|
|
63
|
+
return bytes(ffi.buffer(blob_out[0], size_out[0]))
|
|
64
|
+
finally:
|
|
65
|
+
lib.shortcog_free(blob_out[0])
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def parse(blob: bytes | bytearray | memoryview) -> Spec:
|
|
69
|
+
return Spec(blob)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _read_one(path: PathLike, spec: Spec, pattern: str | None,
|
|
73
|
+
b: Axis, y: tuple[int, int] | None, x: tuple[int, int] | None,
|
|
74
|
+
num_threads: int) -> np.ndarray:
|
|
75
|
+
h = spec._header
|
|
76
|
+
bands = _resolve_axis(b, "b", h.samples_per_pixel)
|
|
77
|
+
y_off, y_size = _resolve_window(y, "y", h.image_length)
|
|
78
|
+
x_off, x_size = _resolve_window(x, "x", h.image_width)
|
|
79
|
+
|
|
80
|
+
n_bands = len(bands) if bands is not None else h.samples_per_pixel
|
|
81
|
+
if pattern is None:
|
|
82
|
+
pattern = "b y x"
|
|
83
|
+
|
|
84
|
+
layout = ffi.new("shortcog_layout*")
|
|
85
|
+
_check(lib.shortcog_compile_layout(
|
|
86
|
+
pattern.encode("ascii"), 1, n_bands, y_size, x_size, layout
|
|
87
|
+
))
|
|
88
|
+
shape = tuple(layout.shape[i] for i in range(layout.ndim))
|
|
89
|
+
arr = np.empty(shape, dtype=spec._dtype)
|
|
90
|
+
|
|
91
|
+
bands_c, n_bands_c = _to_c(bands)
|
|
92
|
+
_check(lib.shortcog_read(
|
|
93
|
+
_enc(path), spec._handle, bands_c, n_bands_c,
|
|
94
|
+
y_off, y_size, x_off, x_size,
|
|
95
|
+
pattern.encode("ascii"), num_threads,
|
|
96
|
+
ffi.cast("void*", arr.ctypes.data), arr.nbytes,
|
|
97
|
+
))
|
|
98
|
+
return arr
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _read_stack(paths: Sequence[PathLike], specs: Sequence[Spec],
|
|
102
|
+
pattern: str | None, n: Axis, b: Axis,
|
|
103
|
+
y: tuple[int, int] | None, x: tuple[int, int] | None,
|
|
104
|
+
num_threads: int) -> np.ndarray:
|
|
105
|
+
paths = list(paths)
|
|
106
|
+
specs = list(specs)
|
|
107
|
+
if len(paths) != len(specs):
|
|
108
|
+
raise ValueError(
|
|
109
|
+
f"paths and specs length mismatch: {len(paths)} vs {len(specs)}"
|
|
110
|
+
)
|
|
111
|
+
if not paths:
|
|
112
|
+
raise ValueError("read requires at least one image")
|
|
113
|
+
|
|
114
|
+
h = specs[0]._header
|
|
115
|
+
n_sel = _resolve_axis(n, "n", len(specs))
|
|
116
|
+
bands = _resolve_axis(b, "b", h.samples_per_pixel)
|
|
117
|
+
y_off, y_size = _resolve_window(y, "y", h.image_length)
|
|
118
|
+
x_off, x_size = _resolve_window(x, "x", h.image_width)
|
|
119
|
+
|
|
120
|
+
n_count = len(n_sel) if n_sel is not None else len(specs)
|
|
121
|
+
n_bands = len(bands) if bands is not None else h.samples_per_pixel
|
|
122
|
+
if pattern is None:
|
|
123
|
+
pattern = "n b y x" if n_count > 1 else "b y x"
|
|
124
|
+
|
|
125
|
+
layout = ffi.new("shortcog_layout*")
|
|
126
|
+
_check(lib.shortcog_compile_layout(
|
|
127
|
+
pattern.encode("ascii"), n_count, n_bands, y_size, x_size, layout
|
|
128
|
+
))
|
|
129
|
+
shape = tuple(layout.shape[i] for i in range(layout.ndim))
|
|
130
|
+
arr = np.empty(shape, dtype=specs[0]._dtype)
|
|
131
|
+
|
|
132
|
+
paths_c = [ffi.new("char[]", _enc(p)) for p in paths]
|
|
133
|
+
paths_arr = ffi.new("char*[]", paths_c)
|
|
134
|
+
specs_arr = ffi.new("shortcog_spec*[]", [s._handle for s in specs])
|
|
135
|
+
|
|
136
|
+
n_c, n_count_c = _to_c(n_sel)
|
|
137
|
+
bands_c, n_bands_c = _to_c(bands)
|
|
138
|
+
_check(lib.shortcog_read_stack(
|
|
139
|
+
paths_arr, specs_arr, len(specs),
|
|
140
|
+
n_c, n_count_c, bands_c, n_bands_c,
|
|
141
|
+
y_off, y_size, x_off, x_size,
|
|
142
|
+
pattern.encode("ascii"), num_threads,
|
|
143
|
+
ffi.cast("void*", arr.ctypes.data), arr.nbytes,
|
|
144
|
+
))
|
|
145
|
+
return arr
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# Polymorphic and stateless. A single path/spec reads one image, lists read a
|
|
149
|
+
# stack. Opens, reads, closes, with nothing kept alive between calls.
|
|
150
|
+
def read(paths: PathLike | Sequence[PathLike],
|
|
151
|
+
specs: Spec | Sequence[Spec],
|
|
152
|
+
pattern: str | None = None, *,
|
|
153
|
+
n: Axis = None, b: Axis = None,
|
|
154
|
+
y: tuple[int, int] | None = None,
|
|
155
|
+
x: tuple[int, int] | None = None,
|
|
156
|
+
num_threads: int = 1) -> np.ndarray:
|
|
157
|
+
if isinstance(paths, (str, bytes, os.PathLike)):
|
|
158
|
+
if n is not None:
|
|
159
|
+
raise ValueError("n applies to a stack; pass lists of paths/specs")
|
|
160
|
+
return _read_one(paths, specs, pattern, b, y, x, num_threads)
|
|
161
|
+
return _read_stack(paths, specs, pattern, n, b, y, x, num_threads)
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import html
|
|
2
|
+
import itertools
|
|
3
|
+
|
|
4
|
+
_FACE = "#FAEEDA"
|
|
5
|
+
_TOP = "#FAC775"
|
|
6
|
+
_SIDE = "#EF9F27"
|
|
7
|
+
_LINE = "#854F0B"
|
|
8
|
+
_EDGE = "#633806"
|
|
9
|
+
|
|
10
|
+
_counter = itertools.count()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# At depth t the band slice is front-top-left -> front-top-right ->
|
|
14
|
+
# front-bottom-right, each shifted by t*(33, -33).
|
|
15
|
+
def _sheet(t):
|
|
16
|
+
dx, dy = 33 * t, -33 * t
|
|
17
|
+
return (f'<polyline points="{55+dx},{72+dy} {145+dx},{72+dy} {145+dx},{185+dy}" '
|
|
18
|
+
f'fill="none" stroke="{_EDGE}" stroke-width="1" opacity=".78"/>')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _bands(b):
|
|
22
|
+
if b <= 1:
|
|
23
|
+
return ""
|
|
24
|
+
if b < 10:
|
|
25
|
+
return "".join(_sheet(k / b) for k in range(1, b))
|
|
26
|
+
fronts = [0.06, 0.12, 0.18, 0.24] if b <= 50 else \
|
|
27
|
+
[0.04, 0.07, 0.10, 0.13, 0.16, 0.19]
|
|
28
|
+
sheets = "".join(_sheet(t) for t in fronts) + _sheet(0.88)
|
|
29
|
+
dots = "".join(f'<circle cx="{105+i*8}" cy="50" r="2.1" fill="{_EDGE}"/>'
|
|
30
|
+
for i in range(3))
|
|
31
|
+
return sheets + dots
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _cube(b, x, y):
|
|
35
|
+
return (
|
|
36
|
+
'<svg width="100%" viewBox="0 0 215 205" '
|
|
37
|
+
f'role="img" aria-label="shortcog image cube, {b} bands">'
|
|
38
|
+
f'<polygon points="55,72 88,39 178,39 145,72" fill="{_TOP}" '
|
|
39
|
+
f'stroke="{_LINE}" stroke-width="1.3"/>'
|
|
40
|
+
f'<polygon points="145,72 178,39 178,152 145,185" fill="{_SIDE}" '
|
|
41
|
+
f'stroke="{_LINE}" stroke-width="1.3"/>'
|
|
42
|
+
f'{_bands(b)}'
|
|
43
|
+
f'<rect x="55" y="72" width="90" height="113" fill="{_FACE}" '
|
|
44
|
+
f'stroke="{_EDGE}" stroke-width="1.5"/>'
|
|
45
|
+
f'<g stroke="{_LINE}" stroke-width="0.5" opacity=".35">'
|
|
46
|
+
'<line x1="85" y1="72" x2="85" y2="185"/>'
|
|
47
|
+
'<line x1="115" y1="72" x2="115" y2="185"/>'
|
|
48
|
+
'<line x1="55" y1="110" x2="145" y2="110"/>'
|
|
49
|
+
'<line x1="55" y1="147" x2="145" y2="147"/></g>'
|
|
50
|
+
f'<text x="100" y="199" text-anchor="middle" font-size="10.5" '
|
|
51
|
+
f'font-family="monospace" fill="currentColor" opacity=".7">X: {x}</text>'
|
|
52
|
+
f'<text x="44" y="128" text-anchor="middle" font-size="10.5" '
|
|
53
|
+
f'font-family="monospace" fill="currentColor" opacity=".7" '
|
|
54
|
+
f'transform="rotate(-90,44,128)">Y: {y}</text>'
|
|
55
|
+
f'<text x="182" y="37" font-size="10.5" font-family="monospace" '
|
|
56
|
+
f'fill="currentColor" opacity=".7">B: {b}</text>'
|
|
57
|
+
'</svg>'
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
_CSS = """
|
|
62
|
+
#ID{font-family:ui-monospace,Menlo,monospace;font-size:13px;color:inherit;
|
|
63
|
+
display:inline-block;line-height:1.5}
|
|
64
|
+
#ID .box{display:flex;gap:6px;align-items:center;
|
|
65
|
+
background:rgba(128,128,128,.06);border:1px solid rgba(128,128,128,.25);
|
|
66
|
+
border-radius:8px;padding:12px 16px}
|
|
67
|
+
#ID .hdr{margin-bottom:9px}
|
|
68
|
+
#ID .cls{opacity:.6}
|
|
69
|
+
#ID .dim{font-weight:600}
|
|
70
|
+
#ID table{border-collapse:collapse;font-size:12.5px}
|
|
71
|
+
#ID td.k{opacity:.6;padding:2px 16px 2px 0}
|
|
72
|
+
#ID td.sub{opacity:.45;padding-left:10px}
|
|
73
|
+
#ID .g{flex:0 0 auto;width:170px}
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _wrap(inner):
|
|
78
|
+
uid = f"scog{next(_counter)}"
|
|
79
|
+
return (f'<div class="shortcog-repr" id="{uid}"><style>'
|
|
80
|
+
f'{_CSS.replace("#ID", f"#{uid}")}</style>{inner}</div>')
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def text(f):
|
|
84
|
+
if not f["ok"]:
|
|
85
|
+
return "<shortcog.Spec (unreadable)>"
|
|
86
|
+
tw, tl = f["tile"]
|
|
87
|
+
return "\n".join([
|
|
88
|
+
f"<shortcog.Spec ({f['b']}, {f['y']}, {f['x']})>",
|
|
89
|
+
f" dtype : {f['dtype']}",
|
|
90
|
+
f" tile : {tw} x {tl}",
|
|
91
|
+
f" tiles : {f['tiles']}",
|
|
92
|
+
f" tiles/band : {f['across'] * f['down']}",
|
|
93
|
+
f" codec : {f['codec']}",
|
|
94
|
+
])
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def html_(f):
|
|
98
|
+
if not f["ok"]:
|
|
99
|
+
return _wrap('<div class="hdr"><span class="cls">shortcog.Spec</span> '
|
|
100
|
+
'<span class="dim">(unreadable)</span></div>')
|
|
101
|
+
tw, tl = f["tile"]
|
|
102
|
+
e = html.escape
|
|
103
|
+
rows = (
|
|
104
|
+
f'<tr><td class="k">dtype</td><td>{e(f["dtype"])}</td></tr>'
|
|
105
|
+
f'<tr><td class="k">tile</td><td>{tw} \u00d7 {tl}</td></tr>'
|
|
106
|
+
f'<tr><td class="k">tiles</td><td><b>{f["tiles"]:,}</b></td></tr>'
|
|
107
|
+
f'<tr><td class="k">tiles/band</td><td>{f["across"] * f["down"]:,}</td></tr>'
|
|
108
|
+
f'<tr><td class="k">codec</td><td>{e(f["codec"])}</td></tr>'
|
|
109
|
+
)
|
|
110
|
+
meta = (f'<div><div class="hdr"><span class="cls">shortcog.Spec</span> '
|
|
111
|
+
f'<span class="dim">({f["b"]}, {f["y"]}, {f["x"]})</span></div>'
|
|
112
|
+
f'<table>{rows}</table></div>')
|
|
113
|
+
return _wrap(f'<div class="box">{meta}'
|
|
114
|
+
f'<div class="g">{_cube(f["b"], f["x"], f["y"])}</div></div>')
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from . import _repr
|
|
4
|
+
from ._ffi import _check, ffi, lib
|
|
5
|
+
|
|
6
|
+
# Spec dtype encoding (sample_format, bits_per_sample) -> numpy dtype.
|
|
7
|
+
# Complex int and 16-bit complex float have no numpy equivalent and raise.
|
|
8
|
+
_DTYPE: dict[tuple[int, int], type[np.generic]] = {
|
|
9
|
+
(1, 8): np.uint8,
|
|
10
|
+
(1, 16): np.uint16,
|
|
11
|
+
(1, 32): np.uint32,
|
|
12
|
+
(1, 64): np.uint64,
|
|
13
|
+
(2, 8): np.int8,
|
|
14
|
+
(2, 16): np.int16,
|
|
15
|
+
(2, 32): np.int32,
|
|
16
|
+
(2, 64): np.int64,
|
|
17
|
+
(3, 16): np.float16,
|
|
18
|
+
(3, 32): np.float32,
|
|
19
|
+
(3, 64): np.float64,
|
|
20
|
+
(6, 64): np.complex64,
|
|
21
|
+
(6, 128): np.complex128,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# predictor doubles as codec selector. 1/2 are ZSTD (TIFF predictor none/
|
|
26
|
+
# horizontal); reserved high values pick a codec that needs no predictor.
|
|
27
|
+
_CODEC = {1: "ZSTD", 2: "ZSTD + horizontal", 42: "OpenZL"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _np_dtype(sf: int, bps: int) -> type[np.generic]:
|
|
31
|
+
try:
|
|
32
|
+
return _DTYPE[(sf, bps)]
|
|
33
|
+
except KeyError:
|
|
34
|
+
raise NotImplementedError(
|
|
35
|
+
f"no numpy dtype for (sample_format={sf}, bits_per_sample={bps})"
|
|
36
|
+
) from None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Spec:
|
|
40
|
+
"""Parsed header blob. Pure memory, no file handle, reusable across reads."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, blob: bytes | bytearray | memoryview) -> None:
|
|
43
|
+
if not isinstance(blob, (bytes, bytearray, memoryview)):
|
|
44
|
+
raise TypeError(f"blob must be bytes-like, got {type(blob).__name__}")
|
|
45
|
+
|
|
46
|
+
blob_buf = ffi.from_buffer("unsigned char[]", blob)
|
|
47
|
+
handle_out = ffi.new("shortcog_spec**")
|
|
48
|
+
_check(lib.shortcog_spec_parse(blob_buf, len(blob), handle_out))
|
|
49
|
+
# ffi.gc frees the handle whenever it goes away, even mid-__init__.
|
|
50
|
+
self._handle = ffi.gc(handle_out[0], lib.shortcog_spec_destroy)
|
|
51
|
+
|
|
52
|
+
header = ffi.new("shortcog_header*")
|
|
53
|
+
_check(lib.shortcog_spec_header(self._handle, header))
|
|
54
|
+
self._header = header
|
|
55
|
+
self._dtype = _np_dtype(header.sample_format, header.bits_per_sample)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def shape(self) -> tuple[int, int, int]:
|
|
59
|
+
h = self._header
|
|
60
|
+
return (h.samples_per_pixel, h.image_length, h.image_width)
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def dtype(self) -> type[np.generic]:
|
|
64
|
+
return self._dtype
|
|
65
|
+
|
|
66
|
+
def _facts(self) -> dict:
|
|
67
|
+
try:
|
|
68
|
+
h = self._header
|
|
69
|
+
return {
|
|
70
|
+
"ok": True,
|
|
71
|
+
"b": h.samples_per_pixel, "y": h.image_length, "x": h.image_width,
|
|
72
|
+
"dtype": self._dtype.__name__,
|
|
73
|
+
"tile": (h.tile_width, h.tile_length),
|
|
74
|
+
"across": h.tiles_across, "down": h.tiles_down,
|
|
75
|
+
"tiles": h.tiles_across * h.tiles_down * h.samples_per_pixel,
|
|
76
|
+
"codec": _CODEC.get(h.predictor, "ZSTD"),
|
|
77
|
+
}
|
|
78
|
+
except Exception:
|
|
79
|
+
return {"ok": False}
|
|
80
|
+
|
|
81
|
+
def __repr__(self) -> str:
|
|
82
|
+
return _repr.text(self._facts())
|
|
83
|
+
|
|
84
|
+
def _repr_html_(self) -> str:
|
|
85
|
+
return _repr.html_(self._facts())
|
|
File without changes
|