xvrt 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xvrt/__init__.py +16 -0
- xvrt/_coords.py +90 -0
- xvrt/_dtype.py +63 -0
- xvrt/_errors.py +11 -0
- xvrt/_sources.py +200 -0
- xvrt/_stac.py +301 -0
- xvrt/_writer.py +1231 -0
- xvrt/_xml.py +160 -0
- xvrt/accessor.py +65 -0
- xvrt-0.1.0.dist-info/METADATA +170 -0
- xvrt-0.1.0.dist-info/RECORD +13 -0
- xvrt-0.1.0.dist-info/WHEEL +4 -0
- xvrt-0.1.0.dist-info/licenses/LICENSE +21 -0
xvrt/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""xvrt: write GDAL multidimensional VRT from xarray Datasets and STAC catalogues.
|
|
2
|
+
|
|
3
|
+
A pure-Python writer. ``write_mdim_vrt`` takes an ``xarray.Dataset`` plus the
|
|
4
|
+
source files and emits a GDAL multidim VRT; ``from_stac`` builds one from
|
|
5
|
+
STAC catalogue fields alone. Output validates against GDAL's ``gdalvrt.xsd``.
|
|
6
|
+
|
|
7
|
+
Documentation: https://hypertidy.github.io/xvrt/
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from ._writer import write_mdim_vrt, encode_datetime_coord
|
|
12
|
+
from ._errors import VrtWriterError
|
|
13
|
+
from ._stac import from_stac
|
|
14
|
+
|
|
15
|
+
__all__ = ["write_mdim_vrt", "from_stac", "encode_datetime_coord", "VrtWriterError"]
|
|
16
|
+
__version__ = "0.1.0"
|
xvrt/_coords.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Coord value carrier dispatch.
|
|
2
|
+
|
|
3
|
+
Three carriers, per the plan:
|
|
4
|
+
|
|
5
|
+
- ``<RegularlySpacedValues start= increment=/>`` when uniform spacing is
|
|
6
|
+
detected within a tolerance
|
|
7
|
+
- ``<InlineValuesWithValueElement><Value>...</Value>...</InlineValuesWithValueElement>``
|
|
8
|
+
for small non-regular coords
|
|
9
|
+
- ``<Source>...</Source>`` pointing at a coord variable in one of the source
|
|
10
|
+
files, for large coords (typical 2D lon/lat grids)
|
|
11
|
+
|
|
12
|
+
The ``coord_mode`` argument on :func:`xvrt.write_mdim_vrt` selects policy:
|
|
13
|
+
|
|
14
|
+
- ``"auto"``: regular-spacing detection, else inline below threshold, else
|
|
15
|
+
source-ref
|
|
16
|
+
- ``"inline"``: always inline (errors if coord exceeds XML practicality)
|
|
17
|
+
- ``"source"``: always source-ref (caller promises the coord is in the source)
|
|
18
|
+
- ``dict``: per-coord override, e.g. ``{"time": "inline", "lat": "source"}``
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from typing import Literal, Mapping
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
from ._errors import VrtWriterError
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
CoordMode = Literal["auto", "inline", "source"]
|
|
30
|
+
|
|
31
|
+
_DEFAULT_INLINE_THRESHOLD = 10_000
|
|
32
|
+
_REGULAR_RTOL = 1e-9
|
|
33
|
+
_REGULAR_ATOL = 1e-15
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def detect_regular(values: np.ndarray) -> tuple[float, float] | None:
|
|
37
|
+
"""Return (start, increment) if the values are uniformly spaced, else None.
|
|
38
|
+
|
|
39
|
+
Uses :func:`numpy.isclose` with tight tolerances — suitable for real-world
|
|
40
|
+
coord axes (daily time stamps, regular lat/lon grids) but strict enough
|
|
41
|
+
that float drift in concatenated chunks won't be misidentified.
|
|
42
|
+
|
|
43
|
+
Empty or single-element arrays are not regular.
|
|
44
|
+
"""
|
|
45
|
+
if values.ndim != 1 or values.size < 2:
|
|
46
|
+
return None
|
|
47
|
+
# Float-safe: require all diffs equal to within tolerance.
|
|
48
|
+
diffs = np.diff(values.astype(np.float64))
|
|
49
|
+
if diffs.size == 0:
|
|
50
|
+
return None
|
|
51
|
+
first = diffs[0]
|
|
52
|
+
if not np.all(np.isclose(diffs, first, rtol=_REGULAR_RTOL, atol=_REGULAR_ATOL)):
|
|
53
|
+
return None
|
|
54
|
+
# Reject zero increment — degenerate.
|
|
55
|
+
if first == 0:
|
|
56
|
+
return None
|
|
57
|
+
# Increment from the end points, not the first difference: the first
|
|
58
|
+
# difference of decoded floats carries rounding noise (0.01 comes back as
|
|
59
|
+
# 0.010000000000005116), and the end-point estimate averages it away.
|
|
60
|
+
v = values.astype(np.float64)
|
|
61
|
+
return float(v[0]), float((v[-1] - v[0]) / (v.size - 1))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def resolve_mode(
|
|
65
|
+
coord_name: str,
|
|
66
|
+
values: np.ndarray,
|
|
67
|
+
mode: CoordMode | Mapping[str, CoordMode],
|
|
68
|
+
threshold: int = _DEFAULT_INLINE_THRESHOLD,
|
|
69
|
+
) -> CoordMode:
|
|
70
|
+
"""Resolve the coord-mode for a specific coord.
|
|
71
|
+
|
|
72
|
+
``mode`` may be a scalar policy or a per-coord dict. ``"auto"`` chooses
|
|
73
|
+
between inline and source based on ``values.size`` and ``threshold``;
|
|
74
|
+
regular-spacing detection is applied *before* this dispatch in
|
|
75
|
+
:func:`xvrt._writer`.
|
|
76
|
+
"""
|
|
77
|
+
if isinstance(mode, Mapping):
|
|
78
|
+
chosen = mode.get(coord_name, "auto")
|
|
79
|
+
else:
|
|
80
|
+
chosen = mode
|
|
81
|
+
|
|
82
|
+
if chosen not in ("auto", "inline", "source"):
|
|
83
|
+
raise VrtWriterError(
|
|
84
|
+
f"coord_mode for {coord_name!r}: expected 'auto'|'inline'|'source', "
|
|
85
|
+
f"got {chosen!r}"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
if chosen == "auto":
|
|
89
|
+
chosen = "inline" if values.size <= threshold else "source"
|
|
90
|
+
return chosen
|
xvrt/_dtype.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""numpy dtype → GDAL VRT DataType string.
|
|
2
|
+
|
|
3
|
+
In the multidim XSD, ``<DataType>`` is ``xs:string`` — not constrained to the
|
|
4
|
+
classic ``DataTypeType`` enumeration — so technically we could emit anything.
|
|
5
|
+
In practice we stick to canonical GDAL names so the output round-trips through
|
|
6
|
+
``osgeo.gdal``'s mdim API without surprises.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from ._errors import VrtWriterError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# Mirrors ``_gdal_to_vrt_dtype`` in vrtstack/R. Kept aligned deliberately —
|
|
16
|
+
# the two writers should produce the same <DataType> string for the same
|
|
17
|
+
# logical dtype.
|
|
18
|
+
_NUMPY_TO_VRT = {
|
|
19
|
+
"bool": "Byte", # GDAL has no bool; promote.
|
|
20
|
+
"int8": "Int8",
|
|
21
|
+
"uint8": "Byte",
|
|
22
|
+
"int16": "Int16",
|
|
23
|
+
"uint16": "UInt16",
|
|
24
|
+
"int32": "Int32",
|
|
25
|
+
"uint32": "UInt32",
|
|
26
|
+
"int64": "Int64",
|
|
27
|
+
"uint64": "UInt64",
|
|
28
|
+
"float16": "Float16",
|
|
29
|
+
"float32": "Float32",
|
|
30
|
+
"float64": "Float64",
|
|
31
|
+
"complex64": "CFloat32",
|
|
32
|
+
"complex128": "CFloat64",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def numpy_to_vrt_dtype(dtype: np.dtype | str) -> str:
|
|
37
|
+
"""Map a numpy dtype to a canonical GDAL VRT DataType name.
|
|
38
|
+
|
|
39
|
+
Raises :class:`VrtWriterError` for datetime / timedelta / string / object
|
|
40
|
+
dtypes. See :func:`xvrt.encode_datetime_coord` for the datetime escape
|
|
41
|
+
hatch.
|
|
42
|
+
"""
|
|
43
|
+
dt = np.dtype(dtype)
|
|
44
|
+
name = dt.name
|
|
45
|
+
|
|
46
|
+
if name in _NUMPY_TO_VRT:
|
|
47
|
+
return _NUMPY_TO_VRT[name]
|
|
48
|
+
|
|
49
|
+
kind = dt.kind
|
|
50
|
+
if kind == "M":
|
|
51
|
+
raise VrtWriterError(
|
|
52
|
+
f"datetime64 dtype ({dt}) not supported in v0. Convert the coord "
|
|
53
|
+
"to numeric first, e.g. via xvrt.encode_datetime_coord(da, "
|
|
54
|
+
"unit='days', since='1970-01-01'), and attach CF units in attrs."
|
|
55
|
+
)
|
|
56
|
+
if kind == "m":
|
|
57
|
+
raise VrtWriterError(f"timedelta64 dtype ({dt}) not supported in v0.")
|
|
58
|
+
if kind in ("U", "S", "O"):
|
|
59
|
+
raise VrtWriterError(
|
|
60
|
+
f"string/object dtype ({dt}) not supported by the VRT mdim model "
|
|
61
|
+
"used here. Convert categorical strings to integer codes first."
|
|
62
|
+
)
|
|
63
|
+
raise VrtWriterError(f"unsupported dtype: {dt}")
|
xvrt/_errors.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Single exception hierarchy so the rules R1..R7 raise something catchable."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class VrtWriterError(ValueError):
|
|
6
|
+
"""Raised when the writer's rules are violated.
|
|
7
|
+
|
|
8
|
+
Subclasses ``ValueError`` so that naive except-clauses still catch it;
|
|
9
|
+
specific enough that callers can ``except VrtWriterError`` to distinguish
|
|
10
|
+
rule violations from generic bad input.
|
|
11
|
+
"""
|
xvrt/_sources.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""Source specification — normalise user input.
|
|
2
|
+
|
|
3
|
+
Accepts:
|
|
4
|
+
- ``list[str | Path]`` — each source is a path/URL, writer infers the source
|
|
5
|
+
array name per composition mode
|
|
6
|
+
- ``list[dict]`` — each dict carries ``{"path": ..., "array": ..., ...}``
|
|
7
|
+
|
|
8
|
+
Produces a uniform list of :class:`Source` records for the emitter.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import Any, Mapping, Sequence
|
|
15
|
+
|
|
16
|
+
from ._errors import VrtWriterError
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class Source:
|
|
21
|
+
"""One participating source.
|
|
22
|
+
|
|
23
|
+
Attributes
|
|
24
|
+
----------
|
|
25
|
+
path
|
|
26
|
+
The string that goes into ``<SourceFilename>``. May be a local path,
|
|
27
|
+
a URL with ``/vsicurl/`` prefix applied, or a plain ``/vsi*`` VSI path.
|
|
28
|
+
array
|
|
29
|
+
The ``<SourceArray>`` name — the full-name of the array inside the
|
|
30
|
+
source (e.g. ``"/temp"``, or ``"temp"`` for root-group arrays).
|
|
31
|
+
relative_to_vrt
|
|
32
|
+
Emitted as ``relativeToVRT="1"`` when True.
|
|
33
|
+
size
|
|
34
|
+
For ``composition="concat"``: number of elements this source
|
|
35
|
+
contributes along the concat dim. ``None`` for stack (always 1).
|
|
36
|
+
original
|
|
37
|
+
The source string as the user originally passed it, before VSI or
|
|
38
|
+
relative-to-VRT rewriting. Used by size auto-discovery so the
|
|
39
|
+
fallback opener sees a resolvable path.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
path: str
|
|
43
|
+
array: str
|
|
44
|
+
relative_to_vrt: bool = False
|
|
45
|
+
size: int | None = None
|
|
46
|
+
extra: Mapping[str, Any] = field(default_factory=dict)
|
|
47
|
+
original: str = ""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
_VSI_PREFIXES = ("/vsi",)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _is_url(s: str) -> bool:
|
|
54
|
+
return s.startswith("http://") or s.startswith("https://")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _to_vsi(s: str) -> str:
|
|
58
|
+
"""Prefix http(s) URLs with /vsicurl/; pass /vsi* through unchanged."""
|
|
59
|
+
if s.startswith(_VSI_PREFIXES):
|
|
60
|
+
return s
|
|
61
|
+
if _is_url(s):
|
|
62
|
+
return f"/vsicurl/{s}"
|
|
63
|
+
if s.startswith("s3://"):
|
|
64
|
+
return f"/vsis3/{s[5:]}"
|
|
65
|
+
return s
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def normalise_sources(
|
|
69
|
+
sources: Sequence[Any],
|
|
70
|
+
*,
|
|
71
|
+
default_array: str,
|
|
72
|
+
vrt_path: Path | None = None,
|
|
73
|
+
composition: str,
|
|
74
|
+
n_vars: int = 1,
|
|
75
|
+
) -> list[Source]:
|
|
76
|
+
"""Resolve user-supplied sources into :class:`Source` records.
|
|
77
|
+
|
|
78
|
+
Parameters
|
|
79
|
+
----------
|
|
80
|
+
sources
|
|
81
|
+
``list[str | Path | dict]``.
|
|
82
|
+
default_array
|
|
83
|
+
Array name to use when a dict-form source omits ``"array"``. Typically
|
|
84
|
+
the data-variable name from ``ds``.
|
|
85
|
+
vrt_path
|
|
86
|
+
Path the VRT will be written to. Used to decide whether to mark
|
|
87
|
+
``relativeToVRT="1"``. If ``None``, always emit absolute paths.
|
|
88
|
+
composition
|
|
89
|
+
``"stack"`` | ``"concat"`` | ``"mosaic"``. Stack sources must not
|
|
90
|
+
carry a per-source ``size``.
|
|
91
|
+
n_vars
|
|
92
|
+
Number of data variables being written. With more than one, a
|
|
93
|
+
dict-form ``"array"`` is ambiguous and refused; use ``"arrays"``
|
|
94
|
+
(a ``{var: source_array_name}`` mapping) instead.
|
|
95
|
+
"""
|
|
96
|
+
if not sources:
|
|
97
|
+
raise VrtWriterError("at least one source is required")
|
|
98
|
+
|
|
99
|
+
out: list[Source] = []
|
|
100
|
+
for i, raw in enumerate(sources):
|
|
101
|
+
out.append(_coerce_one(raw, i, default_array, vrt_path, composition, n_vars))
|
|
102
|
+
return out
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _coerce_one(
|
|
106
|
+
raw: Any,
|
|
107
|
+
index: int,
|
|
108
|
+
default_array: str,
|
|
109
|
+
vrt_path: Path | None,
|
|
110
|
+
composition: str,
|
|
111
|
+
n_vars: int = 1,
|
|
112
|
+
) -> Source:
|
|
113
|
+
if isinstance(raw, Source):
|
|
114
|
+
return raw
|
|
115
|
+
|
|
116
|
+
# R6: no open handles. Check before type dispatch — a Dataset is not a
|
|
117
|
+
# str/Path/dict and would otherwise hit the generic "expected ..." error.
|
|
118
|
+
if hasattr(raw, "variables") and hasattr(raw, "dims"):
|
|
119
|
+
raise VrtWriterError(
|
|
120
|
+
f"source #{index}: open xarray/netCDF4 handles are not accepted. "
|
|
121
|
+
"Pass the path string instead (R6)."
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
if isinstance(raw, (str, Path, PurePosixPath)):
|
|
125
|
+
path_str = str(raw)
|
|
126
|
+
arr = default_array
|
|
127
|
+
size = None
|
|
128
|
+
rel = False
|
|
129
|
+
extra: Mapping[str, Any] = {}
|
|
130
|
+
elif isinstance(raw, Mapping):
|
|
131
|
+
if "path" not in raw:
|
|
132
|
+
raise VrtWriterError(
|
|
133
|
+
f"source #{index}: dict form requires a 'path' key"
|
|
134
|
+
)
|
|
135
|
+
path_str = str(raw["path"])
|
|
136
|
+
if n_vars > 1 and "array" in raw:
|
|
137
|
+
raise VrtWriterError(
|
|
138
|
+
f"source #{index}: 'array' names one source array but "
|
|
139
|
+
f"{n_vars} variables are being written. Use "
|
|
140
|
+
"'arrays': {var: '/name', ...} for renames."
|
|
141
|
+
)
|
|
142
|
+
arrays = raw.get("arrays")
|
|
143
|
+
if arrays is not None and not isinstance(arrays, Mapping):
|
|
144
|
+
raise VrtWriterError(
|
|
145
|
+
f"source #{index}: 'arrays' must be a dict {{var: name}}"
|
|
146
|
+
)
|
|
147
|
+
arr = raw.get("array", default_array)
|
|
148
|
+
size = raw.get("size", None)
|
|
149
|
+
rel = bool(raw.get("relative_to_vrt", False))
|
|
150
|
+
extra = {k: v for k, v in raw.items()
|
|
151
|
+
if k not in {"path", "array", "size", "relative_to_vrt"}}
|
|
152
|
+
else:
|
|
153
|
+
raise VrtWriterError(
|
|
154
|
+
f"source #{index}: expected str, Path, or dict — got {type(raw).__name__}"
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
# Classic raster sources (GeoTIFF/COG): <SourceBand> instead of
|
|
158
|
+
# <SourceArray>. Explicit with "band": N in dict form; implied band 1 for
|
|
159
|
+
# .tif/.tiff paths that name no array.
|
|
160
|
+
band = extra.get("band") if isinstance(extra, Mapping) else None
|
|
161
|
+
if band is None and not (isinstance(raw, Mapping) and
|
|
162
|
+
("array" in raw or "arrays" in raw)):
|
|
163
|
+
if path_str.lower().split("?")[0].endswith((".tif", ".tiff")):
|
|
164
|
+
band = 1
|
|
165
|
+
if band is not None:
|
|
166
|
+
extra = {**extra, "band": int(band)}
|
|
167
|
+
if composition == "concat":
|
|
168
|
+
raise VrtWriterError(
|
|
169
|
+
f"source #{index}: a raster band source is 2D and has no "
|
|
170
|
+
"concat dim to contribute; use composition='stack'."
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
if composition == "stack" and size is not None and size != 1:
|
|
174
|
+
raise VrtWriterError(
|
|
175
|
+
f"source #{index}: stack composition contributes one element per "
|
|
176
|
+
"source; do not pass size=. Use concat if each source has >1 "
|
|
177
|
+
"element along the dim."
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
# Apply /vsicurl/ prefixing and relativity.
|
|
181
|
+
path_emitted = _to_vsi(path_str)
|
|
182
|
+
if (vrt_path is not None and not _is_url(path_str)
|
|
183
|
+
and not path_str.startswith(("/vsi", "s3://"))):
|
|
184
|
+
try:
|
|
185
|
+
rel_path = Path(path_str).resolve().relative_to(vrt_path.parent.resolve())
|
|
186
|
+
# Only auto-relativise if the caller didn't explicitly set it.
|
|
187
|
+
if rel is False:
|
|
188
|
+
path_emitted = str(rel_path)
|
|
189
|
+
rel = True
|
|
190
|
+
except (ValueError, OSError):
|
|
191
|
+
pass
|
|
192
|
+
|
|
193
|
+
return Source(
|
|
194
|
+
path=path_emitted,
|
|
195
|
+
array=str(arr),
|
|
196
|
+
relative_to_vrt=rel,
|
|
197
|
+
size=size,
|
|
198
|
+
extra=extra,
|
|
199
|
+
original=path_str,
|
|
200
|
+
)
|
xvrt/_stac.py
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Build a multidim VRT straight from STAC items: no file is opened.
|
|
2
|
+
|
|
3
|
+
The catalogue already says everything the VRT needs: the grid
|
|
4
|
+
(``proj:shape``, ``proj:transform``), the CRS (``proj:epsg`` / ``proj:code``
|
|
5
|
+
/ ``proj:wkt2``), each item's datetime, and per asset the band's storage
|
|
6
|
+
(``raster:bands``: ``data_type``, ``nodata``, ``scale``, ``offset``,
|
|
7
|
+
``unit``). From that we build a template :class:`xarray.Dataset` whose data
|
|
8
|
+
variables are zero-memory broadcast placeholders of the full shape, plus one
|
|
9
|
+
source list per asset, and hand both to :func:`xvrt.write_mdim_vrt`.
|
|
10
|
+
|
|
11
|
+
This is the catalogue-driven counterpart of the R package vrtstack: "the grid
|
|
12
|
+
comes from the catalogue, identical grid or refuse", one array per asset, and
|
|
13
|
+
items that lack an asset leave nodata at their position.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import datetime as _dt
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Iterable, Mapping, Sequence
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
import xarray as xr
|
|
23
|
+
|
|
24
|
+
from ._errors import VrtWriterError
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def from_stac(
|
|
28
|
+
items: str | Path | Iterable[Mapping[str, Any]],
|
|
29
|
+
path: str | Path,
|
|
30
|
+
*,
|
|
31
|
+
assets: Sequence[str] | None = None,
|
|
32
|
+
dim: str = "time",
|
|
33
|
+
crs: str | None = None,
|
|
34
|
+
block_size: Sequence[int] | Mapping[str, Sequence[int]] | None = None,
|
|
35
|
+
) -> Path:
|
|
36
|
+
"""Write a multidim VRT stacking STAC items along ``dim``.
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
items
|
|
41
|
+
A stac-geoparquet file (path or URL; needs ``pyarrow``), or an
|
|
42
|
+
iterable of STAC item dicts (``pystac.Item.to_dict()`` works). Both
|
|
43
|
+
the flat geoparquet layout and the nested ``"properties"`` layout are
|
|
44
|
+
read.
|
|
45
|
+
path
|
|
46
|
+
Output ``.vrt`` path.
|
|
47
|
+
assets
|
|
48
|
+
Asset keys to write, one array each, in this order. Default: every
|
|
49
|
+
asset with ``raster:bands``, in first-seen order.
|
|
50
|
+
dim
|
|
51
|
+
Name of the stack dimension (one position per distinct datetime).
|
|
52
|
+
crs
|
|
53
|
+
WKT override. Default: from the items' projection fields, turned into
|
|
54
|
+
WKT with ``pyproj`` (or GDAL's ``osr``) when only an EPSG code is
|
|
55
|
+
given, so the axis mapping can be computed.
|
|
56
|
+
block_size
|
|
57
|
+
Optional ``<BlockSize>``; the catalogue does not record it.
|
|
58
|
+
|
|
59
|
+
Raises
|
|
60
|
+
------
|
|
61
|
+
VrtWriterError
|
|
62
|
+
Items on different grids, duplicate datetimes, a band description
|
|
63
|
+
that changes between items, or no usable CRS.
|
|
64
|
+
"""
|
|
65
|
+
from ._writer import write_mdim_vrt
|
|
66
|
+
|
|
67
|
+
rows = [_flatten(it) for it in _read_items(items)]
|
|
68
|
+
if not rows:
|
|
69
|
+
raise VrtWriterError("no STAC items")
|
|
70
|
+
|
|
71
|
+
times = [_item_time(r) for r in rows]
|
|
72
|
+
order = np.argsort(np.array(times, dtype="datetime64[ns]"), kind="stable")
|
|
73
|
+
rows = [rows[i] for i in order]
|
|
74
|
+
times = np.array([times[i] for i in order], dtype="datetime64[ns]")
|
|
75
|
+
if len(np.unique(times)) != len(times):
|
|
76
|
+
dup = times[1:][times[1:] == times[:-1]]
|
|
77
|
+
raise VrtWriterError(
|
|
78
|
+
f"duplicate datetimes in items (first: {dup[0]}); one item per "
|
|
79
|
+
f"position along {dim!r} is required."
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
keys = list(assets) if assets is not None else _band_assets(rows)
|
|
83
|
+
if not keys:
|
|
84
|
+
raise VrtWriterError("no assets with raster:bands found in the items")
|
|
85
|
+
|
|
86
|
+
shape, transform = _common_grid(rows, keys)
|
|
87
|
+
ny, nx = shape
|
|
88
|
+
a, b, c, d, e, f = transform[:6]
|
|
89
|
+
if b != 0 or d != 0:
|
|
90
|
+
raise VrtWriterError(
|
|
91
|
+
f"rotated/sheared transform {transform[:6]} is not supported"
|
|
92
|
+
)
|
|
93
|
+
x = c + a * (np.arange(nx) + 0.5)
|
|
94
|
+
y = f + e * (np.arange(ny) + 0.5)
|
|
95
|
+
|
|
96
|
+
wkt = crs or _common_crs_wkt(rows, keys)
|
|
97
|
+
|
|
98
|
+
data_vars = {}
|
|
99
|
+
sources: dict[str, list[dict[str, Any]]] = {}
|
|
100
|
+
for k in keys:
|
|
101
|
+
band = _common_band(rows, k)
|
|
102
|
+
dtype = np.dtype(band.get("data_type") or "float32")
|
|
103
|
+
placeholder = np.broadcast_to(np.zeros((), dtype=dtype), (len(rows), ny, nx))
|
|
104
|
+
da = xr.DataArray(placeholder, dims=(dim, "y", "x"), name=k)
|
|
105
|
+
enc: dict[str, Any] = {"dtype": dtype}
|
|
106
|
+
if band.get("nodata") is not None:
|
|
107
|
+
enc["_FillValue"] = band["nodata"]
|
|
108
|
+
if band.get("scale") is not None:
|
|
109
|
+
enc["scale_factor"] = band["scale"]
|
|
110
|
+
if band.get("offset") is not None:
|
|
111
|
+
enc["add_offset"] = band["offset"]
|
|
112
|
+
da.encoding.update(enc)
|
|
113
|
+
if band.get("unit"):
|
|
114
|
+
da.attrs["units"] = band["unit"]
|
|
115
|
+
data_vars[k] = da
|
|
116
|
+
sources[k] = [
|
|
117
|
+
{"path": r["assets"][k]["href"], "band": 1, "index": i}
|
|
118
|
+
for i, r in enumerate(rows)
|
|
119
|
+
if (r.get("assets") or {}).get(k)
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
ds = xr.Dataset(
|
|
123
|
+
data_vars,
|
|
124
|
+
coords={dim: (dim, times), "y": ("y", y), "x": ("x", x)},
|
|
125
|
+
)
|
|
126
|
+
return write_mdim_vrt(
|
|
127
|
+
ds, sources, path,
|
|
128
|
+
composition="stack", concat_dim=dim, crs=wkt, block_size=block_size,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# -- reading -------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
def _read_items(items) -> list[Mapping[str, Any]]:
|
|
135
|
+
if isinstance(items, (str, Path)):
|
|
136
|
+
try:
|
|
137
|
+
import pyarrow.parquet as pq
|
|
138
|
+
except ImportError as exc: # pragma: no cover
|
|
139
|
+
raise VrtWriterError(
|
|
140
|
+
"reading stac-geoparquet needs pyarrow (pip install pyarrow)"
|
|
141
|
+
) from exc
|
|
142
|
+
src = str(items)
|
|
143
|
+
if src.startswith(("http://", "https://")):
|
|
144
|
+
import io
|
|
145
|
+
import urllib.request
|
|
146
|
+
|
|
147
|
+
from . import __version__
|
|
148
|
+
|
|
149
|
+
# Some hosts refuse urllib's default User-Agent (HTTP 403).
|
|
150
|
+
req = urllib.request.Request(src, headers={"User-Agent": f"xvrt/{__version__}"})
|
|
151
|
+
with urllib.request.urlopen(req) as resp: # noqa: S310
|
|
152
|
+
return pq.read_table(io.BytesIO(resp.read())).to_pylist()
|
|
153
|
+
return pq.read_table(src).to_pylist()
|
|
154
|
+
return [it.to_dict() if hasattr(it, "to_dict") else it for it in items]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _flatten(item: Mapping[str, Any]) -> dict[str, Any]:
|
|
158
|
+
"""Geoparquet rows are flat; STAC JSON nests fields under properties."""
|
|
159
|
+
out = dict(item.get("properties") or {})
|
|
160
|
+
out.update({k: v for k, v in item.items() if k != "properties"})
|
|
161
|
+
return out
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _item_time(r: Mapping[str, Any]) -> np.datetime64:
|
|
165
|
+
t = r.get("datetime") or r.get("start_datetime")
|
|
166
|
+
if t is None:
|
|
167
|
+
raise VrtWriterError(f"item {r.get('id')!r} has no datetime")
|
|
168
|
+
if isinstance(t, _dt.datetime):
|
|
169
|
+
if t.tzinfo is not None:
|
|
170
|
+
t = t.astimezone(_dt.timezone.utc).replace(tzinfo=None)
|
|
171
|
+
return np.datetime64(t, "ns")
|
|
172
|
+
s = str(t).replace("Z", "")
|
|
173
|
+
if s.endswith("+00:00"):
|
|
174
|
+
s = s[:-6]
|
|
175
|
+
return np.datetime64(s, "ns")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _band_assets(rows) -> list[str]:
|
|
179
|
+
seen: dict[str, None] = {}
|
|
180
|
+
for r in rows:
|
|
181
|
+
for k, a in (r.get("assets") or {}).items():
|
|
182
|
+
if a and a.get("raster:bands"):
|
|
183
|
+
seen.setdefault(k, None)
|
|
184
|
+
return list(seen)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _field(r, asset, name):
|
|
188
|
+
"""A projection field from the asset, falling back to the item."""
|
|
189
|
+
a = (r.get("assets") or {}).get(asset) or {}
|
|
190
|
+
v = a.get(name)
|
|
191
|
+
return v if v is not None else r.get(name)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _common_grid(rows, keys):
|
|
195
|
+
grids = set()
|
|
196
|
+
for r in rows:
|
|
197
|
+
for k in keys:
|
|
198
|
+
if not (r.get("assets") or {}).get(k):
|
|
199
|
+
continue
|
|
200
|
+
shape = _field(r, k, "proj:shape")
|
|
201
|
+
tr = _field(r, k, "proj:transform")
|
|
202
|
+
if shape is None or tr is None:
|
|
203
|
+
raise VrtWriterError(
|
|
204
|
+
f"item {r.get('id')!r} asset {k!r} lacks proj:shape / "
|
|
205
|
+
"proj:transform; the VRT cannot be built from the "
|
|
206
|
+
"catalogue alone."
|
|
207
|
+
)
|
|
208
|
+
grids.add((tuple(int(v) for v in shape),
|
|
209
|
+
tuple(round(float(v), 12) for v in tr[:6])))
|
|
210
|
+
if len(grids) != 1:
|
|
211
|
+
raise VrtWriterError(
|
|
212
|
+
f"items are on {len(grids)} different grids (identical grid or "
|
|
213
|
+
f"refuse): {sorted(grids)[:3]}"
|
|
214
|
+
)
|
|
215
|
+
shape, tr = grids.pop()
|
|
216
|
+
return shape, tr
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
_STORAGE = ("data_type", "nodata", "scale", "offset")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _common_band(rows, k) -> dict[str, Any]:
|
|
223
|
+
"""One storage description per asset.
|
|
224
|
+
|
|
225
|
+
Storage fields (dtype, nodata, scale, offset) must agree across items:
|
|
226
|
+
they decide how bytes decode. The unit is advisory: items that omit it
|
|
227
|
+
take the unit other items give, and conflicting non-empty units warn and
|
|
228
|
+
use the most common.
|
|
229
|
+
"""
|
|
230
|
+
storage = set()
|
|
231
|
+
units: dict[str, int] = {}
|
|
232
|
+
first = None
|
|
233
|
+
for r in rows:
|
|
234
|
+
a = (r.get("assets") or {}).get(k)
|
|
235
|
+
if not a:
|
|
236
|
+
continue
|
|
237
|
+
b = (a.get("raster:bands") or [{}])[0]
|
|
238
|
+
st = tuple(b.get(f) for f in _STORAGE)
|
|
239
|
+
storage.add(st)
|
|
240
|
+
if b.get("unit"):
|
|
241
|
+
units[b["unit"]] = units.get(b["unit"], 0) + 1
|
|
242
|
+
first = first or dict(zip(_STORAGE, st))
|
|
243
|
+
if len(storage) > 1:
|
|
244
|
+
raise VrtWriterError(
|
|
245
|
+
f"asset {k!r}: raster:bands storage ({', '.join(_STORAGE)}) differs "
|
|
246
|
+
f"between items: {sorted(storage, key=str)[:3]}; one array needs one "
|
|
247
|
+
"storage description."
|
|
248
|
+
)
|
|
249
|
+
out = dict(first or {})
|
|
250
|
+
if units:
|
|
251
|
+
unit = max(units, key=units.get)
|
|
252
|
+
if len(units) > 1:
|
|
253
|
+
import warnings
|
|
254
|
+
warnings.warn(
|
|
255
|
+
f"asset {k!r}: items give different units {sorted(units)}; "
|
|
256
|
+
f"using {unit!r}",
|
|
257
|
+
UserWarning, stacklevel=3,
|
|
258
|
+
)
|
|
259
|
+
out["unit"] = unit
|
|
260
|
+
return out
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _common_crs_wkt(rows, keys) -> str:
|
|
264
|
+
found = set()
|
|
265
|
+
for r in rows[:: max(1, len(rows) // 50)] + rows[-1:]:
|
|
266
|
+
for k in keys:
|
|
267
|
+
if not (r.get("assets") or {}).get(k):
|
|
268
|
+
continue
|
|
269
|
+
for name in ("proj:wkt2", "proj:code", "proj:epsg"):
|
|
270
|
+
v = _field(r, k, name)
|
|
271
|
+
if v is not None:
|
|
272
|
+
found.add((name, str(v)))
|
|
273
|
+
break
|
|
274
|
+
if not found:
|
|
275
|
+
raise VrtWriterError("items carry no proj:epsg / proj:code / proj:wkt2; pass crs=")
|
|
276
|
+
if len({v for _, v in found}) > 1:
|
|
277
|
+
raise VrtWriterError(f"items declare different CRSs: {sorted(found)}")
|
|
278
|
+
name, v = found.pop()
|
|
279
|
+
if name == "proj:wkt2":
|
|
280
|
+
return v
|
|
281
|
+
code = v if ":" in v else f"EPSG:{v}"
|
|
282
|
+
return _code_to_wkt(code)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _code_to_wkt(code: str) -> str:
|
|
286
|
+
try:
|
|
287
|
+
import pyproj
|
|
288
|
+
return pyproj.CRS.from_user_input(code).to_wkt()
|
|
289
|
+
except ImportError:
|
|
290
|
+
pass
|
|
291
|
+
try:
|
|
292
|
+
from osgeo import osr
|
|
293
|
+
srs = osr.SpatialReference()
|
|
294
|
+
srs.SetFromUserInput(code)
|
|
295
|
+
return srs.ExportToWkt(["FORMAT=WKT2_2019"])
|
|
296
|
+
except ImportError:
|
|
297
|
+
pass
|
|
298
|
+
raise VrtWriterError(
|
|
299
|
+
f"{code} needs pyproj (or GDAL) to become WKT, which the axis mapping "
|
|
300
|
+
"depends on; install pyproj or pass crs=<WKT>."
|
|
301
|
+
)
|