pointclouds 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pointclouds/__init__.py +20 -0
- pointclouds/classify/__init__.py +1 -0
- pointclouds/classify/ground.py +1 -0
- pointclouds/classify/trees.py +1 -0
- pointclouds/convert.py +35 -0
- pointclouds/io/__init__.py +1 -0
- pointclouds/io/_records.py +196 -0
- pointclouds/io/copc.py +570 -0
- pointclouds/io/ept.py +724 -0
- pointclouds/io/las.py +119 -0
- pointclouds/ops/__init__.py +1 -0
- pointclouds/ops/clip.py +99 -0
- pointclouds/ops/decimate.py +107 -0
- pointclouds/ops/filter.py +109 -0
- pointclouds/ops/grid.py +241 -0
- pointclouds/ops/merge.py +107 -0
- pointclouds/points.py +801 -0
- pointclouds-0.0.0.dist-info/METADATA +95 -0
- pointclouds-0.0.0.dist-info/RECORD +21 -0
- pointclouds-0.0.0.dist-info/WHEEL +4 -0
- pointclouds-0.0.0.dist-info/licenses/LICENSE.md +675 -0
pointclouds/io/ept.py
ADDED
|
@@ -0,0 +1,724 @@
|
|
|
1
|
+
"""EPT (Entwine Point Tile) spatially-indexed windowed reads: a bounding box, or a coarser level of detail.
|
|
2
|
+
|
|
3
|
+
Where COPC packs an octree into a single LAZ file (:mod:`pointclouds.io.copc`), EPT spreads the same idea across
|
|
4
|
+
a **directory tree**: an ``ept.json`` manifest, a ``ept-hierarchy/`` of octree index pages, and a ``ept-data/``
|
|
5
|
+
of one file per node. That shape is what a windowed read exploits - given a bounding box, only the nodes whose
|
|
6
|
+
octree cube overlaps it are fetched; given a target resolution, only the shallow octree depths are visited. The
|
|
7
|
+
alternative, reading every node and masking, defeats the point of the tiling.
|
|
8
|
+
|
|
9
|
+
``laspy`` has no EPT reader, so this module walks the tree itself:
|
|
10
|
+
|
|
11
|
+
- :func:`read_header` reads ``ept.json`` alone - ``count`` / ``bounds`` / ``crs`` / ``span`` - touching no node
|
|
12
|
+
data, the same header-first contract the LAS and COPC engines keep.
|
|
13
|
+
- :func:`read_window` selects nodes from the hierarchy, fetches and decodes each, and returns record arrays
|
|
14
|
+
under the contract in :mod:`pointclouds.io._records`, so an EPT window maps onto
|
|
15
|
+
:class:`~pointclouds.points.PointCloud` exactly as every other read does.
|
|
16
|
+
|
|
17
|
+
**Data types.** EPT stores each node as ``laszip`` (a standalone LAZ, read with ``laspy``), ``binary`` (the
|
|
18
|
+
schema's dimensions packed little-endian, read with ``numpy``), or ``zstandard`` (binary under zstd framing).
|
|
19
|
+
The first two are supported; ``zstandard`` raises a clear error naming what is missing, rather than returning
|
|
20
|
+
wrong data.
|
|
21
|
+
|
|
22
|
+
**Local and remote.** A source may be a path to an ``ept.json`` or to the directory holding it, or an
|
|
23
|
+
``http(s)://`` / ``s3://`` URL to either. Remote access goes through ``fsspec`` behind the ``[cloud]`` extra;
|
|
24
|
+
the core imports with ``numpy`` / ``laspy`` / ``pyproj`` alone and never touches GDAL. A node's data is fetched
|
|
25
|
+
through one indirection (:class:`_Store`), so the local and remote paths differ only in how bytes are read.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import io
|
|
31
|
+
import json
|
|
32
|
+
import math
|
|
33
|
+
import os
|
|
34
|
+
from dataclasses import dataclass
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
import laspy
|
|
39
|
+
import numpy as np
|
|
40
|
+
from pyproj import CRS
|
|
41
|
+
|
|
42
|
+
from pointclouds.io import _records
|
|
43
|
+
|
|
44
|
+
__all__ = ["EptInfo", "is_ept", "read_header", "read_window"]
|
|
45
|
+
|
|
46
|
+
#: EPT schema dimension names (case-insensitive) mapped onto this package's canonical lower-case names. A name
|
|
47
|
+
#: EPT carries but this package does not map (``Red``, ``GpsTime``, ...) is simply not read.
|
|
48
|
+
_SCHEMA_ALIASES: dict[str, str] = {
|
|
49
|
+
"x": "x",
|
|
50
|
+
"y": "y",
|
|
51
|
+
"z": "z",
|
|
52
|
+
"intensity": "intensity",
|
|
53
|
+
"classification": "classification",
|
|
54
|
+
"returnnumber": "return_number",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
#: NumPy dtype for each EPT schema ``{type, size}`` pair. EPT types are ``signed`` / ``unsigned`` / ``float``.
|
|
58
|
+
_NUMPY_TYPES: dict[tuple[str, int], type] = {
|
|
59
|
+
("signed", 1): np.int8,
|
|
60
|
+
("signed", 2): np.int16,
|
|
61
|
+
("signed", 4): np.int32,
|
|
62
|
+
("signed", 8): np.int64,
|
|
63
|
+
("unsigned", 1): np.uint8,
|
|
64
|
+
("unsigned", 2): np.uint16,
|
|
65
|
+
("unsigned", 4): np.uint32,
|
|
66
|
+
("unsigned", 8): np.uint64,
|
|
67
|
+
("float", 4): np.float32,
|
|
68
|
+
("float", 8): np.float64,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
#: The manifest filename at the root of every EPT dataset.
|
|
72
|
+
_MANIFEST = "ept.json"
|
|
73
|
+
|
|
74
|
+
#: A source may be a path or a URL to the manifest or its directory.
|
|
75
|
+
Source = str | os.PathLike[str]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class _Store:
|
|
79
|
+
"""Read-only access to an EPT dataset's files, local or remote, by path relative to the dataset root.
|
|
80
|
+
|
|
81
|
+
The one indirection the local and remote paths share: :func:`read_header` and :func:`read_window` ask for
|
|
82
|
+
``ept.json``, ``ept-hierarchy/<key>.json`` and ``ept-data/<key>.<ext>`` by relative path, and this resolves
|
|
83
|
+
each against the root - a local directory, or a URL through ``fsspec``.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def __init__(self, root: str, fs: Any | None) -> None:
|
|
87
|
+
"""Build a store.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
root: the dataset root - a local directory path, or a URL (with a trailing-slash-insensitive join).
|
|
91
|
+
fs: an ``fsspec`` filesystem for a remote root, or ``None`` for the local filesystem.
|
|
92
|
+
"""
|
|
93
|
+
self._root = root.rstrip("/")
|
|
94
|
+
self._fs = fs
|
|
95
|
+
|
|
96
|
+
def read_bytes(self, relative: str) -> bytes:
|
|
97
|
+
"""Return the bytes of the file at ``relative`` under the root.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
relative: a path relative to the dataset root, e.g. ``"ept-data/0-0-0-0.laz"``.
|
|
101
|
+
|
|
102
|
+
Returns:
|
|
103
|
+
The file's raw bytes.
|
|
104
|
+
"""
|
|
105
|
+
if self._fs is None:
|
|
106
|
+
return Path(self._root, relative).read_bytes()
|
|
107
|
+
with self._fs.open(f"{self._root}/{relative}", "rb") as handle:
|
|
108
|
+
return handle.read()
|
|
109
|
+
|
|
110
|
+
def exists(self, relative: str) -> bool:
|
|
111
|
+
"""Whether the file at ``relative`` exists.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
relative: a path relative to the dataset root.
|
|
115
|
+
|
|
116
|
+
Returns:
|
|
117
|
+
``True`` if it exists.
|
|
118
|
+
"""
|
|
119
|
+
if self._fs is None:
|
|
120
|
+
return Path(self._root, relative).is_file()
|
|
121
|
+
return bool(self._fs.exists(f"{self._root}/{relative}"))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _store(source: Source) -> _Store:
|
|
125
|
+
"""Resolve a source to a :class:`_Store` rooted at the dataset directory.
|
|
126
|
+
|
|
127
|
+
Args:
|
|
128
|
+
source: a path or URL to an ``ept.json`` or to the directory containing it.
|
|
129
|
+
|
|
130
|
+
Returns:
|
|
131
|
+
A store whose root is the dataset directory.
|
|
132
|
+
|
|
133
|
+
Raises:
|
|
134
|
+
FileNotFoundError: when a local path has no ``ept.json``.
|
|
135
|
+
ModuleNotFoundError: when a remote URL is given but the ``[cloud]`` extra is not installed.
|
|
136
|
+
"""
|
|
137
|
+
text = str(source)
|
|
138
|
+
if "://" in text:
|
|
139
|
+
try:
|
|
140
|
+
import fsspec
|
|
141
|
+
except ModuleNotFoundError as exc: # pragma: no cover - environment-specific
|
|
142
|
+
raise ModuleNotFoundError(
|
|
143
|
+
"reading a remote EPT needs the optional cloud stack (fsspec). Install it with: "
|
|
144
|
+
"pip install pointclouds[cloud]"
|
|
145
|
+
) from exc
|
|
146
|
+
root = text[: -len(_MANIFEST)] if text.endswith(_MANIFEST) else text
|
|
147
|
+
fs, _, (path,) = fsspec.get_fs_token_paths(root.rstrip("/"))
|
|
148
|
+
return _Store(path, fs)
|
|
149
|
+
|
|
150
|
+
path = Path(source)
|
|
151
|
+
base = path.parent if path.name == _MANIFEST else path
|
|
152
|
+
if not (base / _MANIFEST).is_file():
|
|
153
|
+
raise FileNotFoundError(f"no EPT manifest at {base / _MANIFEST}")
|
|
154
|
+
return _Store(str(base), None)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@dataclass(frozen=True)
|
|
158
|
+
class EptInfo:
|
|
159
|
+
"""Everything an EPT ``ept.json`` answers without reading a single node.
|
|
160
|
+
|
|
161
|
+
Attributes:
|
|
162
|
+
count: the dataset's total point count, from the manifest.
|
|
163
|
+
bounds: ``(xmin, ymin, xmax, ymax)`` of the *conforming* (tight) bounds, in the dataset's CRS.
|
|
164
|
+
crs: the dataset's CRS as a :class:`pyproj.CRS`, or ``None`` when the manifest carries no SRS.
|
|
165
|
+
dimensions: every dimension name the schema declares, in schema order.
|
|
166
|
+
version: the EPT version string, e.g. ``"1.0.0"``.
|
|
167
|
+
span: the octree's points-per-edge resolution (e.g. ``128``); the number ``resolution=`` is measured
|
|
168
|
+
against. A node at depth ``d`` has point spacing roughly ``cube_side / (span * 2**d)``.
|
|
169
|
+
data_type: the node encoding - ``"laszip"``, ``"binary"`` or ``"zstandard"``.
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
count: int
|
|
173
|
+
bounds: tuple[float, float, float, float]
|
|
174
|
+
crs: CRS | None
|
|
175
|
+
dimensions: tuple[str, ...]
|
|
176
|
+
version: str
|
|
177
|
+
span: int
|
|
178
|
+
data_type: str
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _parse_crs(manifest: dict[str, Any]) -> CRS | None:
|
|
182
|
+
"""Build a :class:`pyproj.CRS` from an EPT ``srs`` block, or ``None`` when absent.
|
|
183
|
+
|
|
184
|
+
Args:
|
|
185
|
+
manifest: the parsed ``ept.json``.
|
|
186
|
+
|
|
187
|
+
Returns:
|
|
188
|
+
The CRS, or ``None``. A WKT string is preferred; else an ``authority``/``horizontal`` code pair.
|
|
189
|
+
"""
|
|
190
|
+
srs = manifest.get("srs") or {}
|
|
191
|
+
if srs.get("wkt"):
|
|
192
|
+
return CRS.from_wkt(srs["wkt"])
|
|
193
|
+
if srs.get("authority") and srs.get("horizontal"):
|
|
194
|
+
return CRS.from_authority(srs["authority"], srs["horizontal"])
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _read_manifest(store: _Store) -> dict[str, Any]:
|
|
199
|
+
"""Read and parse ``ept.json``.
|
|
200
|
+
|
|
201
|
+
Args:
|
|
202
|
+
store: the dataset store.
|
|
203
|
+
|
|
204
|
+
Returns:
|
|
205
|
+
The parsed manifest.
|
|
206
|
+
"""
|
|
207
|
+
return json.loads(store.read_bytes(_MANIFEST).decode("utf-8"))
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def is_ept(source: Source) -> bool:
|
|
211
|
+
"""Report whether ``source`` is an EPT dataset.
|
|
212
|
+
|
|
213
|
+
Cheap: it checks for an ``ept.json``, reading no node data.
|
|
214
|
+
|
|
215
|
+
Args:
|
|
216
|
+
source: a path or URL to inspect.
|
|
217
|
+
|
|
218
|
+
Returns:
|
|
219
|
+
``True`` when an ``ept.json`` is present at or under ``source``, ``False`` otherwise.
|
|
220
|
+
|
|
221
|
+
Examples:
|
|
222
|
+
>>> is_ept("examples/data/sample-ept") # doctest: +SKIP
|
|
223
|
+
True
|
|
224
|
+
"""
|
|
225
|
+
text = str(source)
|
|
226
|
+
if "://" in text:
|
|
227
|
+
try:
|
|
228
|
+
return _store(source).exists(_MANIFEST)
|
|
229
|
+
except (ModuleNotFoundError, FileNotFoundError, ValueError):
|
|
230
|
+
return False
|
|
231
|
+
path = Path(source)
|
|
232
|
+
if path.name == _MANIFEST:
|
|
233
|
+
return path.is_file()
|
|
234
|
+
return (path / _MANIFEST).is_file()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def read_header(source: Source) -> EptInfo:
|
|
238
|
+
"""Read an EPT dataset's manifest, touching no node data.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
source: a path or URL to an ``ept.json`` or its directory.
|
|
242
|
+
|
|
243
|
+
Returns:
|
|
244
|
+
An :class:`EptInfo` with count, bounds, CRS, dimensions, version, span and data type.
|
|
245
|
+
|
|
246
|
+
Raises:
|
|
247
|
+
FileNotFoundError: when there is no ``ept.json`` at the source.
|
|
248
|
+
ModuleNotFoundError: for a remote URL without the ``[cloud]`` extra.
|
|
249
|
+
|
|
250
|
+
Examples:
|
|
251
|
+
>>> info = read_header("examples/data/sample-ept") # doctest: +SKIP
|
|
252
|
+
>>> info.count, info.span # doctest: +SKIP
|
|
253
|
+
(1600, 128)
|
|
254
|
+
"""
|
|
255
|
+
manifest = _read_manifest(_store(source))
|
|
256
|
+
conforming = manifest.get("boundsConforming") or manifest["bounds"]
|
|
257
|
+
schema = manifest.get("schema", [])
|
|
258
|
+
return EptInfo(
|
|
259
|
+
count=int(manifest.get("points", 0)),
|
|
260
|
+
bounds=(float(conforming[0]), float(conforming[1]), float(conforming[3]), float(conforming[4])),
|
|
261
|
+
crs=_parse_crs(manifest),
|
|
262
|
+
dimensions=tuple(str(entry["name"]) for entry in schema),
|
|
263
|
+
version=str(manifest.get("version", "")),
|
|
264
|
+
span=int(manifest.get("span", 0)),
|
|
265
|
+
data_type=str(manifest["dataType"]),
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
@dataclass(frozen=True)
|
|
270
|
+
class _Cube:
|
|
271
|
+
"""An octree node's cubic bounds, carried as min/max corners in the dataset's CRS."""
|
|
272
|
+
|
|
273
|
+
mins: np.ndarray
|
|
274
|
+
maxs: np.ndarray
|
|
275
|
+
|
|
276
|
+
def child(self, dx: int, dy: int, dz: int) -> _Cube:
|
|
277
|
+
"""Return the child sub-cube in octant ``(dx, dy, dz)`` (each 0 or 1).
|
|
278
|
+
|
|
279
|
+
Args:
|
|
280
|
+
dx: x octant, 0 (low) or 1 (high).
|
|
281
|
+
dy: y octant.
|
|
282
|
+
dz: z octant.
|
|
283
|
+
|
|
284
|
+
Returns:
|
|
285
|
+
The child cube, half the side in every axis.
|
|
286
|
+
"""
|
|
287
|
+
mid = (self.mins + self.maxs) / 2.0
|
|
288
|
+
lo = np.where([dx, dy, dz], mid, self.mins)
|
|
289
|
+
hi = np.where([dx, dy, dz], self.maxs, mid)
|
|
290
|
+
return _Cube(lo, hi)
|
|
291
|
+
|
|
292
|
+
def overlaps_xy(self, bbox: tuple[float, float, float, float]) -> bool:
|
|
293
|
+
"""Whether this cube's x/y footprint overlaps a 2-D bounding box.
|
|
294
|
+
|
|
295
|
+
Args:
|
|
296
|
+
bbox: ``(xmin, ymin, xmax, ymax)``.
|
|
297
|
+
|
|
298
|
+
Returns:
|
|
299
|
+
``True`` if the footprints intersect (touching edges count).
|
|
300
|
+
"""
|
|
301
|
+
xmin, ymin, xmax, ymax = bbox
|
|
302
|
+
return not (self.maxs[0] < xmin or self.mins[0] > xmax or self.maxs[1] < ymin or self.mins[1] > ymax)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _root_cube(manifest: dict[str, Any]) -> _Cube:
|
|
306
|
+
"""The octree root cube, from the manifest's cubic ``bounds``.
|
|
307
|
+
|
|
308
|
+
Args:
|
|
309
|
+
manifest: the parsed ``ept.json``.
|
|
310
|
+
|
|
311
|
+
Returns:
|
|
312
|
+
The root :class:`_Cube`.
|
|
313
|
+
"""
|
|
314
|
+
b = manifest["bounds"]
|
|
315
|
+
return _Cube(np.array([b[0], b[1], b[2]], dtype=float), np.array([b[3], b[4], b[5]], dtype=float))
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _max_depth(manifest: dict[str, Any], resolution: float | None, depth: int | None) -> int | None:
|
|
319
|
+
"""Translate a ``resolution`` or an explicit ``depth`` into the deepest octree level to visit.
|
|
320
|
+
|
|
321
|
+
Args:
|
|
322
|
+
manifest: the parsed ``ept.json`` (for ``span`` and the cube side).
|
|
323
|
+
resolution: target point spacing in CRS units, or ``None``.
|
|
324
|
+
depth: an explicit maximum depth, or ``None``.
|
|
325
|
+
|
|
326
|
+
Returns:
|
|
327
|
+
The maximum depth (inclusive) to descend to, or ``None`` for "no limit - the full dataset".
|
|
328
|
+
"""
|
|
329
|
+
if depth is not None:
|
|
330
|
+
return depth
|
|
331
|
+
if resolution is None:
|
|
332
|
+
return None
|
|
333
|
+
b = manifest["bounds"]
|
|
334
|
+
side = float(b[3]) - float(b[0])
|
|
335
|
+
span = int(manifest.get("span", 0)) or 1
|
|
336
|
+
# Spacing at depth d is side / (span * 2**d); descend until it is <= resolution.
|
|
337
|
+
ratio = side / (span * resolution)
|
|
338
|
+
return max(0, math.ceil(math.log2(ratio))) if ratio > 1 else 0
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class _Hierarchy:
|
|
342
|
+
"""The EPT octree index, loaded page by page on demand.
|
|
343
|
+
|
|
344
|
+
The root page (``ept-hierarchy/0-0-0-0.json``) maps ``"d-x-y-z"`` keys to point counts; a count of ``-1``
|
|
345
|
+
means that node's subtree lives in its own page, fetched lazily the first time the walk reaches it.
|
|
346
|
+
"""
|
|
347
|
+
|
|
348
|
+
def __init__(self, store: _Store) -> None:
|
|
349
|
+
"""Build a hierarchy over ``store``, loading only the root page.
|
|
350
|
+
|
|
351
|
+
Args:
|
|
352
|
+
store: the dataset store.
|
|
353
|
+
"""
|
|
354
|
+
self._store = store
|
|
355
|
+
self._counts: dict[str, int] = {}
|
|
356
|
+
self._load_page("0-0-0-0")
|
|
357
|
+
|
|
358
|
+
def _load_page(self, key: str) -> None:
|
|
359
|
+
"""Merge the hierarchy page named ``key`` into the known counts.
|
|
360
|
+
|
|
361
|
+
Args:
|
|
362
|
+
key: the ``"d-x-y-z"`` key whose page to load.
|
|
363
|
+
"""
|
|
364
|
+
page = json.loads(self._store.read_bytes(f"ept-hierarchy/{key}.json").decode("utf-8"))
|
|
365
|
+
self._counts.update({str(k): int(v) for k, v in page.items()})
|
|
366
|
+
|
|
367
|
+
def count(self, key: str) -> int:
|
|
368
|
+
"""Return the point count for ``key``, loading a nested page if the index points to one.
|
|
369
|
+
|
|
370
|
+
Args:
|
|
371
|
+
key: the ``"d-x-y-z"`` node key.
|
|
372
|
+
|
|
373
|
+
Returns:
|
|
374
|
+
The node's point count, ``0`` when the node is absent. A ``-1`` placeholder is resolved by loading
|
|
375
|
+
the node's own page and re-reading the count.
|
|
376
|
+
"""
|
|
377
|
+
value = self._counts.get(key, 0)
|
|
378
|
+
if value == -1:
|
|
379
|
+
self._load_page(key)
|
|
380
|
+
value = self._counts.get(key, 0)
|
|
381
|
+
return value
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _select_nodes(
|
|
385
|
+
hierarchy: _Hierarchy,
|
|
386
|
+
root: _Cube,
|
|
387
|
+
bbox: tuple[float, float, float, float] | None,
|
|
388
|
+
max_depth: int | None,
|
|
389
|
+
) -> list[str]:
|
|
390
|
+
"""Walk the octree and return the keys of every node to read.
|
|
391
|
+
|
|
392
|
+
Args:
|
|
393
|
+
hierarchy: the loaded hierarchy index.
|
|
394
|
+
root: the root cube.
|
|
395
|
+
bbox: a 2-D bounding box to prune against, or ``None`` for the whole extent.
|
|
396
|
+
max_depth: the deepest level to descend to, or ``None`` for no limit.
|
|
397
|
+
|
|
398
|
+
Returns:
|
|
399
|
+
The ``"d-x-y-z"`` keys of the non-empty nodes whose cube overlaps ``bbox`` and whose depth is within
|
|
400
|
+
``max_depth``.
|
|
401
|
+
"""
|
|
402
|
+
selected: list[str] = []
|
|
403
|
+
stack: list[tuple[int, int, int, int, _Cube]] = [(0, 0, 0, 0, root)]
|
|
404
|
+
while stack:
|
|
405
|
+
d, x, y, z, cube = stack.pop()
|
|
406
|
+
if bbox is not None and not cube.overlaps_xy(bbox):
|
|
407
|
+
continue
|
|
408
|
+
if hierarchy.count(f"{d}-{x}-{y}-{z}") <= 0:
|
|
409
|
+
continue
|
|
410
|
+
selected.append(f"{d}-{x}-{y}-{z}")
|
|
411
|
+
if max_depth is not None and d >= max_depth:
|
|
412
|
+
continue
|
|
413
|
+
for dx in (0, 1):
|
|
414
|
+
for dy in (0, 1):
|
|
415
|
+
for dz in (0, 1):
|
|
416
|
+
stack.append((d + 1, 2 * x + dx, 2 * y + dy, 2 * z + dz, cube.child(dx, dy, dz)))
|
|
417
|
+
return selected
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _decode_laszip(data: bytes, requested: tuple[str, ...]) -> dict[str, np.ndarray]:
|
|
421
|
+
"""Decode a ``laszip`` node (a standalone LAZ) into the requested record arrays.
|
|
422
|
+
|
|
423
|
+
Args:
|
|
424
|
+
data: the node file's bytes.
|
|
425
|
+
requested: the resolved canonical dimension names.
|
|
426
|
+
|
|
427
|
+
Returns:
|
|
428
|
+
A record mapping, via the shared :func:`pointclouds.io._records.extract`.
|
|
429
|
+
"""
|
|
430
|
+
return _records.extract(laspy.read(io.BytesIO(data)), requested)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _binary_dtype(schema: list[dict[str, Any]]) -> np.dtype:
|
|
434
|
+
"""Build the structured dtype a ``binary`` node is packed as, from the EPT schema.
|
|
435
|
+
|
|
436
|
+
Args:
|
|
437
|
+
schema: the manifest's ``schema`` list of ``{name, type, size}`` entries.
|
|
438
|
+
|
|
439
|
+
Returns:
|
|
440
|
+
A little-endian structured :class:`numpy.dtype`, one field per schema entry in order.
|
|
441
|
+
|
|
442
|
+
Raises:
|
|
443
|
+
ValueError: when an entry's ``{type, size}`` is not an EPT numeric type.
|
|
444
|
+
"""
|
|
445
|
+
fields = []
|
|
446
|
+
for entry in schema:
|
|
447
|
+
key = (str(entry["type"]), int(entry["size"]))
|
|
448
|
+
if key not in _NUMPY_TYPES:
|
|
449
|
+
raise ValueError(f"unsupported EPT schema type {key} for dimension {entry.get('name')!r}")
|
|
450
|
+
fields.append((str(entry["name"]), np.dtype(_NUMPY_TYPES[key]).newbyteorder("<")))
|
|
451
|
+
return np.dtype(fields)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _decode_binary(data: bytes, schema: list[dict[str, Any]], requested: tuple[str, ...]) -> dict[str, np.ndarray]:
|
|
455
|
+
"""Decode a ``binary`` node into the requested record arrays, applying each dimension's scale/offset.
|
|
456
|
+
|
|
457
|
+
Args:
|
|
458
|
+
data: the node file's bytes - the schema's dimensions packed little-endian, one record per point.
|
|
459
|
+
schema: the manifest's ``schema`` list, which defines the packing and any per-dimension scale/offset.
|
|
460
|
+
requested: the resolved canonical dimension names.
|
|
461
|
+
|
|
462
|
+
Returns:
|
|
463
|
+
A record mapping under the shared contract: every requested name present, a dimension the schema does
|
|
464
|
+
not carry coming back as a zero-length array of the canonical dtype.
|
|
465
|
+
"""
|
|
466
|
+
records = np.frombuffer(data, dtype=_binary_dtype(schema))
|
|
467
|
+
# canonical name -> the schema entry that feeds it (first match wins).
|
|
468
|
+
by_canonical: dict[str, dict[str, Any]] = {}
|
|
469
|
+
for entry in schema:
|
|
470
|
+
canonical = _SCHEMA_ALIASES.get(str(entry["name"]).lower())
|
|
471
|
+
if canonical is not None:
|
|
472
|
+
by_canonical.setdefault(canonical, entry)
|
|
473
|
+
|
|
474
|
+
arrays: dict[str, np.ndarray] = {}
|
|
475
|
+
for name in requested:
|
|
476
|
+
dtype = _records.CANONICAL[name]
|
|
477
|
+
feed = by_canonical.get(name)
|
|
478
|
+
if feed is None:
|
|
479
|
+
arrays[name] = np.empty(0, dtype=dtype)
|
|
480
|
+
continue
|
|
481
|
+
column = records[str(feed["name"])]
|
|
482
|
+
scale, offset = feed.get("scale"), feed.get("offset")
|
|
483
|
+
if scale is not None or offset is not None:
|
|
484
|
+
factor = float(scale) if scale is not None else 1.0
|
|
485
|
+
shift = float(offset) if offset is not None else 0.0
|
|
486
|
+
column = column * factor + shift
|
|
487
|
+
arrays[name] = np.array(column, dtype=dtype, copy=True)
|
|
488
|
+
return arrays
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _decode(
|
|
492
|
+
data: bytes, data_type: str, schema: list[dict[str, Any]], requested: tuple[str, ...]
|
|
493
|
+
) -> dict[str, np.ndarray]:
|
|
494
|
+
"""Decode one node's bytes according to the dataset's ``dataType``.
|
|
495
|
+
|
|
496
|
+
Args:
|
|
497
|
+
data: the node file's bytes.
|
|
498
|
+
data_type: the manifest's ``dataType`` - ``"laszip"`` or ``"binary"``.
|
|
499
|
+
schema: the manifest's ``schema`` (used by the binary path).
|
|
500
|
+
requested: the resolved canonical dimension names.
|
|
501
|
+
|
|
502
|
+
Returns:
|
|
503
|
+
A record mapping for that node.
|
|
504
|
+
|
|
505
|
+
Raises:
|
|
506
|
+
NotImplementedError: for ``"zstandard"`` - recognised but not decoded here.
|
|
507
|
+
ValueError: for an unknown data type.
|
|
508
|
+
"""
|
|
509
|
+
if data_type == "laszip":
|
|
510
|
+
return _decode_laszip(data, requested)
|
|
511
|
+
if data_type == "binary":
|
|
512
|
+
return _decode_binary(data, schema, requested)
|
|
513
|
+
if data_type == "zstandard":
|
|
514
|
+
raise NotImplementedError(
|
|
515
|
+
"EPT 'zstandard' node encoding is not supported yet; re-export the dataset as 'laszip' or 'binary'"
|
|
516
|
+
)
|
|
517
|
+
raise ValueError(f"unknown EPT dataType {data_type!r}; expected 'laszip', 'binary' or 'zstandard'")
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _node_extension(data_type: str) -> str:
|
|
521
|
+
"""The data-file extension for a given ``dataType``.
|
|
522
|
+
|
|
523
|
+
Args:
|
|
524
|
+
data_type: the manifest's ``dataType``.
|
|
525
|
+
|
|
526
|
+
Returns:
|
|
527
|
+
``"laz"`` for laszip, else ``"bin"``.
|
|
528
|
+
"""
|
|
529
|
+
return "laz" if data_type == "laszip" else "bin"
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _concatenate(parts: list[dict[str, np.ndarray]], requested: tuple[str, ...]) -> dict[str, np.ndarray]:
|
|
533
|
+
"""Join per-node record mappings into one, preserving the no-missing-key contract.
|
|
534
|
+
|
|
535
|
+
Args:
|
|
536
|
+
parts: one record mapping per node read.
|
|
537
|
+
requested: the resolved canonical dimension names.
|
|
538
|
+
|
|
539
|
+
Returns:
|
|
540
|
+
A single mapping, each dimension the concatenation of that dimension across nodes. An empty selection
|
|
541
|
+
gives zero-length arrays throughout.
|
|
542
|
+
"""
|
|
543
|
+
if not parts:
|
|
544
|
+
return {name: np.empty(0, dtype=_records.CANONICAL[name]) for name in requested}
|
|
545
|
+
return {name: np.concatenate([part[name] for part in parts]) for name in requested}
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _check_bbox(bbox: tuple[float, float, float, float]) -> None:
|
|
549
|
+
"""Validate a bounding box's shape, finiteness and orientation. Needs no dataset, so it runs before any read.
|
|
550
|
+
|
|
551
|
+
Args:
|
|
552
|
+
bbox: ``(xmin, ymin, xmax, ymax)`` in the dataset's CRS.
|
|
553
|
+
|
|
554
|
+
Raises:
|
|
555
|
+
ValueError: when ``bbox`` is not four numbers, holds a non-finite value, or either axis is inverted.
|
|
556
|
+
"""
|
|
557
|
+
if len(bbox) != 4:
|
|
558
|
+
raise ValueError(f"bbox must be (xmin, ymin, xmax, ymax); got {len(bbox)} value(s): {bbox!r}")
|
|
559
|
+
xmin, ymin, xmax, ymax = (float(value) for value in bbox)
|
|
560
|
+
if not all(math.isfinite(value) for value in (xmin, ymin, xmax, ymax)):
|
|
561
|
+
raise ValueError(f"bbox corners must be finite; got (xmin, ymin, xmax, ymax) = {(xmin, ymin, xmax, ymax)!r}")
|
|
562
|
+
if xmin > xmax or ymin > ymax:
|
|
563
|
+
raise ValueError(f"bbox is inverted: (xmin, ymin, xmax, ymax) = {(xmin, ymin, xmax, ymax)!r}")
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def read_window(
|
|
567
|
+
source: Source,
|
|
568
|
+
*,
|
|
569
|
+
bbox: tuple[float, float, float, float] | None = None,
|
|
570
|
+
resolution: float | None = None,
|
|
571
|
+
depth: int | None = None,
|
|
572
|
+
dimensions: tuple[str, ...] | None = None,
|
|
573
|
+
) -> dict[str, np.ndarray]:
|
|
574
|
+
"""Read only the points a window selects from an EPT dataset, using its octree rather than every node.
|
|
575
|
+
|
|
576
|
+
With no argument this reads the whole dataset - the honest default, and the expensive one; the point is to
|
|
577
|
+
pass a window.
|
|
578
|
+
|
|
579
|
+
Args:
|
|
580
|
+
source: a path or URL to an ``ept.json`` or its directory.
|
|
581
|
+
bbox: ``(xmin, ymin, xmax, ymax)`` in the dataset's CRS. Only nodes whose octree cube overlaps the box
|
|
582
|
+
are fetched, and the points are then clipped exactly to the box, to the float edge. 2-D: the full z
|
|
583
|
+
range is kept.
|
|
584
|
+
resolution: target point spacing in CRS units, measured against :attr:`EptInfo.span`. The octree is
|
|
585
|
+
visited only to the depth that satisfies it, so a wide view stays cheap. Mutually exclusive with
|
|
586
|
+
``depth``.
|
|
587
|
+
depth: an explicit maximum octree depth (``0`` is the root), inclusive, when a caller would rather name
|
|
588
|
+
a depth than a spacing. Mutually exclusive with ``resolution``.
|
|
589
|
+
dimensions: dimension names to return, case-insensitive. ``None`` returns every dimension the package
|
|
590
|
+
maps onto a :class:`~pointclouds.points.PointCloud` field. Selects what comes back, not what is
|
|
591
|
+
decoded.
|
|
592
|
+
|
|
593
|
+
Returns:
|
|
594
|
+
A mapping of lower-case dimension name to array, under the same contract as every other reader: a
|
|
595
|
+
dimension the schema does not carry comes back as a zero-length array, never as a missing key. A window
|
|
596
|
+
that selects nothing gives zero-length arrays throughout.
|
|
597
|
+
|
|
598
|
+
Raises:
|
|
599
|
+
FileNotFoundError: when there is no ``ept.json`` at the source.
|
|
600
|
+
ModuleNotFoundError: for a remote URL without the ``[cloud]`` extra.
|
|
601
|
+
ValueError: when ``bbox`` is malformed, non-finite or inverted; when ``resolution`` is not positive and
|
|
602
|
+
finite; when ``depth`` is negative; or when ``resolution`` and ``depth`` are both given.
|
|
603
|
+
KeyError: when a requested dimension is not one this package reads.
|
|
604
|
+
NotImplementedError: when the dataset's ``dataType`` is ``"zstandard"``.
|
|
605
|
+
|
|
606
|
+
Examples:
|
|
607
|
+
>>> arrays = read_window( # doctest: +SKIP
|
|
608
|
+
... "examples/data/sample-ept", bbox=(155000, 463000, 155020, 463020)
|
|
609
|
+
... )
|
|
610
|
+
>>> arrays["x"].size # doctest: +SKIP
|
|
611
|
+
121
|
|
612
|
+
"""
|
|
613
|
+
requested = _validate_window(bbox, resolution, depth, dimensions) # before any read: no I/O on bad input
|
|
614
|
+
store = _store(source)
|
|
615
|
+
manifest = _read_manifest(store)
|
|
616
|
+
return _window_from(store, manifest, bbox=bbox, resolution=resolution, depth=depth, requested=requested)
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _validate_window(
|
|
620
|
+
bbox: tuple[float, float, float, float] | None,
|
|
621
|
+
resolution: float | None,
|
|
622
|
+
depth: int | None,
|
|
623
|
+
dimensions: tuple[str, ...] | None,
|
|
624
|
+
) -> tuple[str, ...]:
|
|
625
|
+
"""Run every source-independent window check and resolve the dimension list, before any file is read.
|
|
626
|
+
|
|
627
|
+
None of these needs the dataset, so running them first lets a malformed window be rejected before paying
|
|
628
|
+
the manifest, hierarchy and node fetches - which matters on a remote source.
|
|
629
|
+
|
|
630
|
+
Args:
|
|
631
|
+
bbox: see :func:`read_window`.
|
|
632
|
+
resolution: see :func:`read_window`.
|
|
633
|
+
depth: see :func:`read_window`.
|
|
634
|
+
dimensions: see :func:`read_window`.
|
|
635
|
+
|
|
636
|
+
Returns:
|
|
637
|
+
The resolved, lower-cased, de-duplicated dimension names.
|
|
638
|
+
|
|
639
|
+
Raises:
|
|
640
|
+
ValueError: on a bad bbox, resolution, depth, or both resolution and depth.
|
|
641
|
+
KeyError: when a requested dimension is not one this package reads.
|
|
642
|
+
"""
|
|
643
|
+
if resolution is not None and depth is not None:
|
|
644
|
+
raise ValueError("pass either resolution= or depth=, not both; they are two ways to say the same thing")
|
|
645
|
+
if resolution is not None and not (math.isfinite(resolution) and resolution > 0):
|
|
646
|
+
raise ValueError(f"resolution must be a positive, finite number; got {resolution!r}")
|
|
647
|
+
if depth is not None and (isinstance(depth, bool) or not isinstance(depth, int) or depth < 0):
|
|
648
|
+
raise ValueError(f"depth must be a non-negative octree depth; got {depth!r}")
|
|
649
|
+
if bbox is not None:
|
|
650
|
+
_check_bbox(bbox)
|
|
651
|
+
return _records.resolve(dimensions)
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def _window_from(
|
|
655
|
+
store: _Store,
|
|
656
|
+
manifest: dict[str, Any],
|
|
657
|
+
*,
|
|
658
|
+
bbox: tuple[float, float, float, float] | None,
|
|
659
|
+
resolution: float | None,
|
|
660
|
+
depth: int | None,
|
|
661
|
+
requested: tuple[str, ...],
|
|
662
|
+
) -> dict[str, np.ndarray]:
|
|
663
|
+
"""Run a window against an already-loaded store and manifest, with pre-validated arguments.
|
|
664
|
+
|
|
665
|
+
Shared by :func:`read_window` and :func:`_read_window_with_crs`; both call :func:`_validate_window` first,
|
|
666
|
+
so this does no validation.
|
|
667
|
+
|
|
668
|
+
Args:
|
|
669
|
+
store: the dataset store.
|
|
670
|
+
manifest: the parsed ``ept.json``.
|
|
671
|
+
bbox: see :func:`read_window`.
|
|
672
|
+
resolution: see :func:`read_window`.
|
|
673
|
+
depth: see :func:`read_window`.
|
|
674
|
+
requested: the resolved dimension names from :func:`_validate_window`.
|
|
675
|
+
|
|
676
|
+
Returns:
|
|
677
|
+
The record-array mapping, as :func:`read_window` documents.
|
|
678
|
+
"""
|
|
679
|
+
# An exact bbox clip needs x and y even when the caller did not ask for them; read them, then drop them.
|
|
680
|
+
needed = requested if bbox is None else tuple(dict.fromkeys(("x", "y", *requested)))
|
|
681
|
+
|
|
682
|
+
data_type = str(manifest["dataType"])
|
|
683
|
+
schema = manifest.get("schema", [])
|
|
684
|
+
extension = _node_extension(data_type)
|
|
685
|
+
hierarchy = _Hierarchy(store)
|
|
686
|
+
keys = _select_nodes(hierarchy, _root_cube(manifest), bbox, _max_depth(manifest, resolution, depth))
|
|
687
|
+
|
|
688
|
+
parts = [_decode(store.read_bytes(f"ept-data/{key}.{extension}"), data_type, schema, needed) for key in keys]
|
|
689
|
+
arrays = _concatenate(parts, needed)
|
|
690
|
+
if bbox is not None:
|
|
691
|
+
arrays = _records.clip_to_box(arrays, bbox)
|
|
692
|
+
arrays = {name: arrays[name] for name in requested}
|
|
693
|
+
return arrays
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def _read_window_with_crs(
|
|
697
|
+
source: Source,
|
|
698
|
+
*,
|
|
699
|
+
bbox: tuple[float, float, float, float] | None = None,
|
|
700
|
+
resolution: float | None = None,
|
|
701
|
+
depth: int | None = None,
|
|
702
|
+
dimensions: tuple[str, ...] | None = None,
|
|
703
|
+
) -> tuple[CRS | None, dict[str, np.ndarray]]:
|
|
704
|
+
"""Read a window and the dataset's CRS from a single manifest read.
|
|
705
|
+
|
|
706
|
+
:class:`~pointclouds.points.PointCloud` needs both to build a windowed cloud; reading the manifest once
|
|
707
|
+
avoids a second fetch of ``ept.json`` on a remote source.
|
|
708
|
+
|
|
709
|
+
Args:
|
|
710
|
+
source: a path or URL to an ``ept.json`` or its directory.
|
|
711
|
+
bbox: see :func:`read_window`.
|
|
712
|
+
resolution: see :func:`read_window`.
|
|
713
|
+
depth: see :func:`read_window`.
|
|
714
|
+
dimensions: see :func:`read_window`.
|
|
715
|
+
|
|
716
|
+
Returns:
|
|
717
|
+
``(crs, arrays)`` - the dataset's CRS (or ``None``) and the record-array mapping.
|
|
718
|
+
"""
|
|
719
|
+
requested = _validate_window(bbox, resolution, depth, dimensions) # before any read: no I/O on bad input
|
|
720
|
+
store = _store(source)
|
|
721
|
+
manifest = _read_manifest(store)
|
|
722
|
+
crs = _parse_crs(manifest)
|
|
723
|
+
arrays = _window_from(store, manifest, bbox=bbox, resolution=resolution, depth=depth, requested=requested)
|
|
724
|
+
return crs, arrays
|