lunar-toolkit 0.1.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lunar_toolkit/__init__.py +6 -0
- lunar_toolkit/asp.py +432 -0
- lunar_toolkit/cli/__init__.py +88 -0
- lunar_toolkit/cli/_jobs.py +437 -0
- lunar_toolkit/cli/_status.py +368 -0
- lunar_toolkit/cli/_utils.py +128 -0
- lunar_toolkit/cli/app.py +205 -0
- lunar_toolkit/cli/build_db.py +130 -0
- lunar_toolkit/cli/download.py +111 -0
- lunar_toolkit/cli/kaguya.py +253 -0
- lunar_toolkit/cli/ode_search.py +144 -0
- lunar_toolkit/cli/pairs.py +236 -0
- lunar_toolkit/cli/process.py +138 -0
- lunar_toolkit/cli/query.py +70 -0
- lunar_toolkit/cli/reference.py +233 -0
- lunar_toolkit/cli/search.py +90 -0
- lunar_toolkit/cli/serve.py +706 -0
- lunar_toolkit/cli/sfs.py +344 -0
- lunar_toolkit/cli/sfs_report.py +148 -0
- lunar_toolkit/cli/shadowcam.py +246 -0
- lunar_toolkit/cli/stereo.py +85 -0
- lunar_toolkit/cli/workspace.py +84 -0
- lunar_toolkit/config.py +345 -0
- lunar_toolkit/constants.py +217 -0
- lunar_toolkit/csm.py +340 -0
- lunar_toolkit/downloader.py +447 -0
- lunar_toolkit/geometry.py +556 -0
- lunar_toolkit/isis.py +222 -0
- lunar_toolkit/kaguya/__init__.py +45 -0
- lunar_toolkit/kaguya/mi.py +274 -0
- lunar_toolkit/kaguya/tc.py +370 -0
- lunar_toolkit/lroc/__init__.py +6 -0
- lunar_toolkit/lroc/cumindex.py +1197 -0
- lunar_toolkit/lroc/nac.py +702 -0
- lunar_toolkit/observation.py +127 -0
- lunar_toolkit/ode.py +485 -0
- lunar_toolkit/provenance.py +312 -0
- lunar_toolkit/reference.py +1111 -0
- lunar_toolkit/selection.py +714 -0
- lunar_toolkit/sfs.py +1534 -0
- lunar_toolkit/sfs_products.py +676 -0
- lunar_toolkit/sfs_report.py +1328 -0
- lunar_toolkit/shadowcam/__init__.py +29 -0
- lunar_toolkit/shadowcam/sc.py +392 -0
- lunar_toolkit/static/index.html +3259 -0
- lunar_toolkit/stereo.py +519 -0
- lunar_toolkit/stereo_dem.py +145 -0
- lunar_toolkit/tools.py +264 -0
- lunar_toolkit/workspace.py +1173 -0
- lunar_toolkit-0.1.0.dev2.dist-info/METADATA +177 -0
- lunar_toolkit-0.1.0.dev2.dist-info/RECORD +55 -0
- lunar_toolkit-0.1.0.dev2.dist-info/WHEEL +5 -0
- lunar_toolkit-0.1.0.dev2.dist-info/entry_points.txt +2 -0
- lunar_toolkit-0.1.0.dev2.dist-info/licenses/LICENSE +28 -0
- lunar_toolkit-0.1.0.dev2.dist-info/top_level.txt +1 -0
lunar_toolkit/config.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""Flat YAML run configuration for ``lunar sfs``: load, validate, merge, resolve regions and dump."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import dataclasses
|
|
6
|
+
import difflib
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import types
|
|
12
|
+
import typing
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Union
|
|
17
|
+
|
|
18
|
+
import yaml
|
|
19
|
+
|
|
20
|
+
from .sfs import SfsConfig
|
|
21
|
+
|
|
22
|
+
log = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
PGDA = "pgda"
|
|
25
|
+
RUNS_DIR_ENV = "LUNAR_RUNS_DIR"
|
|
26
|
+
"""``dem: pgda`` (or no ``dem`` with a ``region``): the region's PGDA LOLA 5 m ``surf`` DEM."""
|
|
27
|
+
|
|
28
|
+
OPTION_KEYS = ("bundle_adjust_options", "refine_options", "sfs_options")
|
|
29
|
+
FLOW_KEYS = ("projwin", "stereo_convergence", "crop_center")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ConfigError(ValueError):
|
|
33
|
+
"""An invalid configuration: unknown key, wrong type or inconsistent settings."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class RunConfig:
|
|
38
|
+
"""Everything a ``lunar sfs`` run needs: inputs, region and the :class:`SfsConfig`.
|
|
39
|
+
|
|
40
|
+
In YAML the run keys below and the :class:`SfsConfig` fields share one flat
|
|
41
|
+
mapping. Relative paths are relative to the current directory.
|
|
42
|
+
|
|
43
|
+
Parameters
|
|
44
|
+
----------
|
|
45
|
+
work_dir : Path
|
|
46
|
+
Run directory; a relative path is taken under *runs_dir*.
|
|
47
|
+
runs_dir : Path, optional
|
|
48
|
+
Where relative ``work_dir`` paths live (default: ``$LUNAR_RUNS_DIR``, else the
|
|
49
|
+
current directory).
|
|
50
|
+
Working / output directory.
|
|
51
|
+
dem : str, optional
|
|
52
|
+
Prior DEM path, or ``"pgda"`` for the region's PGDA LOLA 5 m DEM (the
|
|
53
|
+
default with a *region*).
|
|
54
|
+
region : str or int, optional
|
|
55
|
+
Artemis III region (:func:`lunar_toolkit.reference.get_region`): sets
|
|
56
|
+
the ``projwin`` (its ROI, or a *crop_km* window), ``region_id`` and the
|
|
57
|
+
image search AOI.
|
|
58
|
+
crop_km : float, optional
|
|
59
|
+
Side of a square window at the region centre (:func:`~lunar_toolkit.reference.crop_window`).
|
|
60
|
+
crop_center : (lat, lon), optional
|
|
61
|
+
Centre of that window (default: the ROI centre).
|
|
62
|
+
aoi : str, optional
|
|
63
|
+
AOI (GeoJSON / WKT file or WKT) for the image search and, without
|
|
64
|
+
``projwin``, the crop.
|
|
65
|
+
sources : list of str
|
|
66
|
+
NAC product IDs or ``.IMG`` / ``.cub`` paths (default: search the AOI).
|
|
67
|
+
from_file : Path, optional
|
|
68
|
+
Text file of more sources, one per line.
|
|
69
|
+
max_slew, max_pixel_scale : float, optional
|
|
70
|
+
Image search cuts (deg, m/px). No slew cut by default: slewed images give the
|
|
71
|
+
stereo convergence for registration, and ``max_sfs_slew`` keeps them out of SfS.
|
|
72
|
+
max_images : int, optional
|
|
73
|
+
Cap on the searched images before download (stereo pairs + illumination coverage;
|
|
74
|
+
:func:`lunar_toolkit.sfs.cap_candidates`).
|
|
75
|
+
pgda_dir : Path, optional
|
|
76
|
+
Where PGDA files are downloaded (default: the reference cache).
|
|
77
|
+
data_dir : Path, optional
|
|
78
|
+
Parquet index directory.
|
|
79
|
+
checksums : bool
|
|
80
|
+
MD5 the input files (under 2 GB) in ``provenance.json``.
|
|
81
|
+
sfs : SfsConfig
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
work_dir: Path = Path("sfs_work")
|
|
85
|
+
runs_dir: Path | None = None
|
|
86
|
+
dem: str | None = None
|
|
87
|
+
region: str | int | None = None
|
|
88
|
+
crop_km: float | None = None
|
|
89
|
+
crop_center: tuple[float, float] | None = None
|
|
90
|
+
aoi: str | None = None
|
|
91
|
+
sources: list[str] = field(default_factory=list)
|
|
92
|
+
from_file: Path | None = None
|
|
93
|
+
max_slew: float | None = None
|
|
94
|
+
max_pixel_scale: float | None = 1.5
|
|
95
|
+
max_images: int | None = None
|
|
96
|
+
pgda_dir: Path | None = None
|
|
97
|
+
data_dir: Path | None = None
|
|
98
|
+
checksums: bool = True
|
|
99
|
+
sfs: SfsConfig = field(default_factory=SfsConfig)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _hints(cls: type) -> dict[str, Any]:
|
|
103
|
+
names = [f.name for f in dataclasses.fields(cls)]
|
|
104
|
+
hints = typing.get_type_hints(cls)
|
|
105
|
+
return {n: hints[n] for n in names if n != "sfs"}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
RUN_KEYS: dict[str, Any] = _hints(RunConfig)
|
|
109
|
+
SFS_KEYS: dict[str, Any] = _hints(SfsConfig)
|
|
110
|
+
KEYS: dict[str, Any] = {**RUN_KEYS, **SFS_KEYS}
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _describe(hint: Any) -> str:
|
|
114
|
+
if typing.get_origin(hint) in (Union, types.UnionType):
|
|
115
|
+
return " or ".join(_describe(a) for a in typing.get_args(hint))
|
|
116
|
+
origin, args = typing.get_origin(hint), typing.get_args(hint)
|
|
117
|
+
if origin is tuple:
|
|
118
|
+
return f"list of {len(args)} {_describe(args[0])}s"
|
|
119
|
+
if origin is list:
|
|
120
|
+
return f"list of {_describe(args[0])}s"
|
|
121
|
+
if origin is dict:
|
|
122
|
+
return "mapping"
|
|
123
|
+
names = {bool: "boolean", int: "integer", float: "number", str: "string", Path: "path"}
|
|
124
|
+
return "null" if hint is type(None) else names.get(hint, str(hint))
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _coerce(key: str, value: Any, hint: Any) -> Any:
|
|
128
|
+
origin, args = typing.get_origin(hint), typing.get_args(hint)
|
|
129
|
+
if hint is Any:
|
|
130
|
+
return value
|
|
131
|
+
if origin in (Union, types.UnionType):
|
|
132
|
+
if value is None and type(None) in args:
|
|
133
|
+
return None
|
|
134
|
+
for arm in args:
|
|
135
|
+
if arm is not type(None):
|
|
136
|
+
try:
|
|
137
|
+
return _coerce(key, value, arm)
|
|
138
|
+
except ConfigError:
|
|
139
|
+
pass
|
|
140
|
+
elif hint is bool and isinstance(value, bool):
|
|
141
|
+
return value
|
|
142
|
+
elif hint is int and isinstance(value, int) and not isinstance(value, bool):
|
|
143
|
+
return value
|
|
144
|
+
elif hint is float and isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
145
|
+
return float(value)
|
|
146
|
+
elif hint is str and isinstance(value, str):
|
|
147
|
+
return value
|
|
148
|
+
elif hint is Path and isinstance(value, (str, Path)):
|
|
149
|
+
return Path(value)
|
|
150
|
+
elif origin is tuple and isinstance(value, (list, tuple)) and len(value) == len(args):
|
|
151
|
+
return tuple(_coerce(key, v, a) for v, a in zip(value, args))
|
|
152
|
+
elif origin is list and isinstance(value, (list, tuple)):
|
|
153
|
+
return [_coerce(key, v, args[0]) for v in value]
|
|
154
|
+
elif origin is dict and isinstance(value, Mapping) and all(isinstance(k, str) for k in value):
|
|
155
|
+
return {k.replace("-", "_"): v for k, v in value.items()}
|
|
156
|
+
raise ConfigError(f"{key}: expected {_describe(hint)}, got {value!r}")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def from_dict(data: Mapping[str, Any], source: str = "config") -> RunConfig:
|
|
160
|
+
"""A validated :class:`RunConfig` from a flat mapping (keys as in :data:`KEYS`).
|
|
161
|
+
|
|
162
|
+
Raises
|
|
163
|
+
------
|
|
164
|
+
ConfigError
|
|
165
|
+
For unknown keys (with suggestions), wrong types and invalid choices.
|
|
166
|
+
"""
|
|
167
|
+
if not isinstance(data, Mapping):
|
|
168
|
+
raise ConfigError(f"{source}: expected a mapping of settings, got {type(data).__name__}")
|
|
169
|
+
norm = {str(k).replace("-", "_"): v for k, v in data.items()}
|
|
170
|
+
for key in norm:
|
|
171
|
+
if key not in KEYS:
|
|
172
|
+
close = difflib.get_close_matches(key, list(KEYS), n=1)
|
|
173
|
+
hint = f" (did you mean {close[0]!r}?)" if close else ""
|
|
174
|
+
raise ConfigError(f"{source}: unknown key {key!r}{hint}")
|
|
175
|
+
try:
|
|
176
|
+
values = {k: _coerce(k, v, KEYS[k]) for k, v in norm.items()}
|
|
177
|
+
except ConfigError as exc:
|
|
178
|
+
raise ConfigError(f"{source}: {exc}") from None
|
|
179
|
+
try:
|
|
180
|
+
sfs = SfsConfig(**{k: v for k, v in values.items() if k in SFS_KEYS})
|
|
181
|
+
except ValueError as exc:
|
|
182
|
+
raise ConfigError(f"{source}: {exc}") from None
|
|
183
|
+
return RunConfig(**{k: v for k, v in values.items() if k in RUN_KEYS}, sfs=sfs)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _plain(value: Any) -> Any:
|
|
187
|
+
if isinstance(value, Path):
|
|
188
|
+
return str(value)
|
|
189
|
+
if isinstance(value, (list, tuple)):
|
|
190
|
+
return [_plain(v) for v in value]
|
|
191
|
+
if isinstance(value, dict):
|
|
192
|
+
return {k: _plain(v) for k, v in value.items()}
|
|
193
|
+
return value
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def to_dict(cfg: RunConfig) -> dict[str, Any]:
|
|
197
|
+
"""Flat, YAML / JSON-ready mapping of every setting (run keys first)."""
|
|
198
|
+
out = {k: _plain(getattr(cfg, k)) for k in RUN_KEYS}
|
|
199
|
+
out.update({k: _plain(getattr(cfg.sfs, k)) for k in SFS_KEYS})
|
|
200
|
+
return out
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def merge(cfg: RunConfig, overrides: Mapping[str, Any], source: str = "overrides") -> RunConfig:
|
|
204
|
+
"""*cfg* with *overrides* applied; option dicts are merged key by key, other values replaced."""
|
|
205
|
+
data = to_dict(cfg)
|
|
206
|
+
for key, value in overrides.items():
|
|
207
|
+
key = key.replace("-", "_")
|
|
208
|
+
if key in OPTION_KEYS and isinstance(value, Mapping) and isinstance(data.get(key), dict):
|
|
209
|
+
data[key] = {**data[key], **value}
|
|
210
|
+
else:
|
|
211
|
+
data[key] = value
|
|
212
|
+
return from_dict(data, source)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def load_config(path: Path) -> RunConfig:
|
|
216
|
+
"""Read and validate a YAML config (an empty file gives the defaults)."""
|
|
217
|
+
path = Path(path)
|
|
218
|
+
try:
|
|
219
|
+
data = yaml.safe_load(path.read_text())
|
|
220
|
+
except OSError as exc:
|
|
221
|
+
raise ConfigError(f"Cannot read config {path}: {exc.strerror or exc}") from None
|
|
222
|
+
except yaml.YAMLError as exc:
|
|
223
|
+
raise ConfigError(f"{path}: invalid YAML: {exc}") from None
|
|
224
|
+
return from_dict(data if data is not None else {}, str(path))
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def dump_config(cfg: RunConfig, path: Path | None = None) -> str:
|
|
228
|
+
"""YAML text of every setting (written to *path* when given)."""
|
|
229
|
+
from .constants import APP_VERSION
|
|
230
|
+
|
|
231
|
+
lines = [f"# lunar sfs configuration (lunar-toolkit {APP_VERSION})\n"]
|
|
232
|
+
for key, value in to_dict(cfg).items():
|
|
233
|
+
if key == next(iter(SFS_KEYS)):
|
|
234
|
+
lines.append("\n# SfS pipeline settings (lunar_toolkit.sfs.SfsConfig)\n")
|
|
235
|
+
if key in FLOW_KEYS or (isinstance(value, (list, dict)) and not value):
|
|
236
|
+
flow = yaml.safe_dump(value, default_flow_style=True, width=10**6).strip()
|
|
237
|
+
lines.append(f"{key}: {flow.removesuffix('...').strip()}\n")
|
|
238
|
+
else:
|
|
239
|
+
lines.append(yaml.safe_dump({key: value}, default_flow_style=False, sort_keys=False, width=110))
|
|
240
|
+
text = "".join(lines)
|
|
241
|
+
if path is not None:
|
|
242
|
+
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
243
|
+
Path(path).write_text(text)
|
|
244
|
+
return text
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def config_hash(cfg: RunConfig) -> str:
|
|
248
|
+
"""SHA-256 of the settings (canonical JSON of :func:`to_dict`)."""
|
|
249
|
+
return hashlib.sha256(json.dumps(to_dict(cfg), sort_keys=True).encode()).hexdigest()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def pgda_file(region: Any, kind: str, out_dir: Path | None = None, download: bool = True) -> Path:
|
|
253
|
+
"""Local path of a region's PGDA LOLA 5 m raster (``surf``, ``ldec``, ``toterr``, ...).
|
|
254
|
+
|
|
255
|
+
Downloaded into ``out_dir/<site>/`` when missing and *download*; otherwise the
|
|
256
|
+
expected path (``<site>_final_adj_5mpp_<kind>.tif``, ``<site>_final_adj.xyzi``)
|
|
257
|
+
is returned without touching the network or the disk.
|
|
258
|
+
"""
|
|
259
|
+
from . import reference as ref
|
|
260
|
+
|
|
261
|
+
site = ref._site_name(region)
|
|
262
|
+
out_dir = Path(out_dir) if out_dir else ref.REFERENCE_CACHE_DIR / "pgda"
|
|
263
|
+
name = f"{site}_final_adj.xyzi" if kind == "xyzi" else f"{site}_final_adj_5mpp_{kind}.tif"
|
|
264
|
+
expected = out_dir / site / name
|
|
265
|
+
if expected.exists() or not download:
|
|
266
|
+
return expected
|
|
267
|
+
files = ref.download_pgda(site, [kind], out_dir)[kind]
|
|
268
|
+
if not files:
|
|
269
|
+
raise ConfigError(f"PGDA site {site} has no {kind} file")
|
|
270
|
+
return files[0]
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def region_id(region: Any) -> str:
|
|
274
|
+
"""Bertone et al. product prefix of a region, e.g. ``A311`` for Malapert Massif."""
|
|
275
|
+
from . import reference as ref
|
|
276
|
+
|
|
277
|
+
return f"A3{ref.get_region(region).id:02d}"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def resolve(cfg: RunConfig, download: bool = True) -> RunConfig:
|
|
281
|
+
"""Fill what *region* implies: ``projwin``, ``region_id`` and the PGDA prior DEM.
|
|
282
|
+
|
|
283
|
+
An explicit ``projwin`` wins, then ``aoi`` (the crop is computed from it at run
|
|
284
|
+
time), then a ``crop_km`` window, then the region ROI. ``dem: pgda`` (or none)
|
|
285
|
+
is the region's PGDA ``surf`` DEM, downloaded when missing and *download*.
|
|
286
|
+
|
|
287
|
+
Raises
|
|
288
|
+
------
|
|
289
|
+
ConfigError
|
|
290
|
+
For a ``crop_km`` / ``dem: pgda`` without a region, or an unknown region.
|
|
291
|
+
"""
|
|
292
|
+
from . import reference as ref
|
|
293
|
+
|
|
294
|
+
runs_dir = cfg.runs_dir or (Path(os.environ[RUNS_DIR_ENV]) if os.environ.get(RUNS_DIR_ENV) else None)
|
|
295
|
+
if runs_dir is not None and not Path(cfg.work_dir).is_absolute():
|
|
296
|
+
cfg = merge(cfg, {"work_dir": str(Path(runs_dir).expanduser() / cfg.work_dir), "runs_dir": None})
|
|
297
|
+
if cfg.region is None:
|
|
298
|
+
if cfg.crop_km is not None or cfg.crop_center is not None:
|
|
299
|
+
raise ConfigError("crop_km / crop_center need a region")
|
|
300
|
+
if cfg.dem == PGDA:
|
|
301
|
+
raise ConfigError("dem: pgda needs a region")
|
|
302
|
+
return cfg
|
|
303
|
+
try:
|
|
304
|
+
reg = ref.get_region(cfg.region)
|
|
305
|
+
except KeyError as exc:
|
|
306
|
+
raise ConfigError(f"region: {exc.args[0]}") from None
|
|
307
|
+
changes: dict[str, Any] = {}
|
|
308
|
+
if cfg.sfs.projwin is None and cfg.aoi is None:
|
|
309
|
+
if cfg.crop_km is not None:
|
|
310
|
+
box = ref.crop_window(reg, cfg.crop_km, cfg.crop_center)
|
|
311
|
+
else:
|
|
312
|
+
box = reg.roi
|
|
313
|
+
changes["projwin"] = box.projwin
|
|
314
|
+
if cfg.sfs.region_id is None:
|
|
315
|
+
changes["region_id"] = region_id(reg)
|
|
316
|
+
if cfg.dem in (None, PGDA):
|
|
317
|
+
try:
|
|
318
|
+
changes["dem"] = str(pgda_file(reg, "surf", cfg.pgda_dir, download=download))
|
|
319
|
+
except KeyError as exc:
|
|
320
|
+
raise ConfigError(f"dem: {exc.args[0]}") from None
|
|
321
|
+
return merge(cfg, changes, "region") if changes else cfg
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def search_aoi(cfg: RunConfig) -> str | None:
|
|
325
|
+
"""Image-search AOI: ``aoi``, else the region crop (``projwin`` as lon/lat WKT), else None."""
|
|
326
|
+
from . import reference as ref
|
|
327
|
+
|
|
328
|
+
if cfg.aoi is not None:
|
|
329
|
+
return cfg.aoi
|
|
330
|
+
if cfg.region is None or cfg.sfs.projwin is None:
|
|
331
|
+
return None
|
|
332
|
+
ulx, uly, lrx, lry = cfg.sfs.projwin
|
|
333
|
+
return ref.PsBox(ulx, lry, lrx, uly).wkt()
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def load_sources(cfg: RunConfig) -> list[str]:
|
|
337
|
+
"""``sources`` plus the non-comment lines of ``from_file``."""
|
|
338
|
+
sources = list(cfg.sources)
|
|
339
|
+
if cfg.from_file is not None:
|
|
340
|
+
try:
|
|
341
|
+
text = Path(cfg.from_file).read_text()
|
|
342
|
+
except OSError as exc:
|
|
343
|
+
raise ConfigError(f"from_file: cannot read {cfg.from_file}: {exc.strerror or exc}") from None
|
|
344
|
+
sources += [s.strip() for s in text.splitlines() if s.strip() and not s.strip().startswith("#")]
|
|
345
|
+
return sources
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""App name, data directories, PDS LROC archive URLs, datasets and shared defaults."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
# Application name used for the user data / cache directories and the HTTP
|
|
9
|
+
# User-Agent. The only place the package name is spelled out at runtime.
|
|
10
|
+
APP_NAME = "lunar_toolkit"
|
|
11
|
+
try:
|
|
12
|
+
APP_VERSION = version("lunar-toolkit")
|
|
13
|
+
except PackageNotFoundError: # running from a source tree that was never installed
|
|
14
|
+
APP_VERSION = "0.0.0"
|
|
15
|
+
|
|
16
|
+
# PDS archive roots
|
|
17
|
+
# The LROC SOC host (https://pds.lroc.im-ldi.com/data/<DATA_SET_ID>/...) now
|
|
18
|
+
# 302-redirects to the PDS Imaging Node S3 store below. Dataset directories
|
|
19
|
+
# there are lowercase and drop the "-V1.0" suffix, e.g.
|
|
20
|
+
# https://pds.lroc.im-ldi.com/data/LRO-L-LROC-2-EDR-V1.0/LROLRC_0066C/INDEX/
|
|
21
|
+
# → https://pds.mcp.nasa.gov/data/store/img/lunar_reconnaissance_orbiter/
|
|
22
|
+
# pds4/lroc/lro-l-lroc-2-edr/LROLRC_0066C/INDEX/
|
|
23
|
+
|
|
24
|
+
LEGACY_PDS_BASE_URL = "https://pds.lroc.im-ldi.com/data/"
|
|
25
|
+
|
|
26
|
+
PDS_STORE_URL = "https://pds.mcp.nasa.gov/data/store/img/"
|
|
27
|
+
PDS_LROC_PREFIX = "lunar_reconnaissance_orbiter/pds4/lroc/"
|
|
28
|
+
PDS_BASE_URL = PDS_STORE_URL + PDS_LROC_PREFIX
|
|
29
|
+
|
|
30
|
+
EDR_DATASET = "lro-l-lroc-2-edr/"
|
|
31
|
+
CDR_DATASET = "lro-l-lroc-3-cdr/"
|
|
32
|
+
RDR_DATASET = "lro-l-lroc-5-rdr/"
|
|
33
|
+
|
|
34
|
+
EDR_BASE_URL = PDS_BASE_URL + EDR_DATASET
|
|
35
|
+
CDR_BASE_URL = PDS_BASE_URL + CDR_DATASET
|
|
36
|
+
RDR_BASE_URL = PDS_BASE_URL + RDR_DATASET
|
|
37
|
+
|
|
38
|
+
# Fallback releases — the newest volume is auto-discovered from the S3 listing
|
|
39
|
+
# (see cumindex.discover_volumes); these are only used when that fails.
|
|
40
|
+
EDR_LATEST_RELEASE = "LROLRC_0068A/"
|
|
41
|
+
CDR_LATEST_RELEASE = "LROLRC_1068A/"
|
|
42
|
+
RDR_LATEST_RELEASE = "LROLRC_2001/"
|
|
43
|
+
|
|
44
|
+
INDEX_DIR = "INDEX/"
|
|
45
|
+
|
|
46
|
+
CUMINDEX_LBL_FILENAME = "CUMINDEX.LBL"
|
|
47
|
+
CUMINDEX_TAB_FILENAME = "CUMINDEX.TAB"
|
|
48
|
+
VOLINDEX_LBL_FILENAME = "INDEX.LBL"
|
|
49
|
+
VOLINDEX_TAB_FILENAME = "INDEX.TAB"
|
|
50
|
+
|
|
51
|
+
# Dataset registry
|
|
52
|
+
# Each entry drives CumIndex URL construction and the name of the parquet file
|
|
53
|
+
# built by `lunar build-db <key>`.
|
|
54
|
+
|
|
55
|
+
DATASETS: dict[str, dict] = {
|
|
56
|
+
"edr": {
|
|
57
|
+
"label": "EDR — Experiment Data Record (raw NAC/WAC images)",
|
|
58
|
+
"data_set_id": "LRO-L-LROC-2-EDR-V1.0",
|
|
59
|
+
"base_url": EDR_BASE_URL,
|
|
60
|
+
"release": EDR_LATEST_RELEASE,
|
|
61
|
+
"volume_re": r"LROLRC_0\d{3}[A-Z]?",
|
|
62
|
+
"index_dir": INDEX_DIR,
|
|
63
|
+
"lbl_filename": CUMINDEX_LBL_FILENAME,
|
|
64
|
+
"tab_filename": CUMINDEX_TAB_FILENAME,
|
|
65
|
+
"parquet": "cumindex_edr.parquet",
|
|
66
|
+
},
|
|
67
|
+
"cdr": {
|
|
68
|
+
"label": "CDR — Calibrated Data Record",
|
|
69
|
+
"data_set_id": "LRO-L-LROC-3-CDR-V1.0",
|
|
70
|
+
"base_url": CDR_BASE_URL,
|
|
71
|
+
"release": CDR_LATEST_RELEASE,
|
|
72
|
+
"volume_re": r"LROLRC_1\d{3}[A-Z]?",
|
|
73
|
+
"index_dir": INDEX_DIR,
|
|
74
|
+
"lbl_filename": CUMINDEX_LBL_FILENAME,
|
|
75
|
+
"tab_filename": CUMINDEX_TAB_FILENAME,
|
|
76
|
+
"parquet": "cumindex_cdr.parquet",
|
|
77
|
+
},
|
|
78
|
+
"rdr": {
|
|
79
|
+
"label": "RDR — Reduced Data Record (WAC mosaics, DEMs, etc.)",
|
|
80
|
+
"data_set_id": "LRO-L-LROC-5-RDR-V1.0",
|
|
81
|
+
"base_url": RDR_BASE_URL,
|
|
82
|
+
"release": RDR_LATEST_RELEASE,
|
|
83
|
+
"volume_re": r"LROLRC_2\d{3}[A-Z]?",
|
|
84
|
+
"index_dir": INDEX_DIR,
|
|
85
|
+
"lbl_filename": CUMINDEX_LBL_FILENAME,
|
|
86
|
+
"tab_filename": CUMINDEX_TAB_FILENAME,
|
|
87
|
+
"parquet": "cumindex_rdr.parquet",
|
|
88
|
+
},
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
# PDS URL helpers
|
|
92
|
+
_LEGACY_DS_RE = re.compile(r"^(LRO-L-LROC-\d-[A-Z]+)-V\d+\.\d+$", re.IGNORECASE)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def pds_url_from_file_spec(file_spec: str) -> str:
|
|
96
|
+
"""Map a CUMINDEX ``FILE_SPECIFICATION_NAME`` to a download URL.
|
|
97
|
+
|
|
98
|
+
``LRO-L-LROC-2-EDR-V1.0/LROLRC_0001/DATA/.../M101013931LE.IMG`` →
|
|
99
|
+
``<PDS_BASE_URL>lro-l-lroc-2-edr/LROLRC_0001/DATA/.../M101013931LE.IMG``.
|
|
100
|
+
Paths without a dataset prefix are joined to ``PDS_BASE_URL`` unchanged.
|
|
101
|
+
"""
|
|
102
|
+
spec = str(file_spec).strip().lstrip("/")
|
|
103
|
+
head, _, rest = spec.partition("/")
|
|
104
|
+
m = _LEGACY_DS_RE.match(head)
|
|
105
|
+
if m and rest:
|
|
106
|
+
return PDS_BASE_URL + m.group(1).lower() + "/" + rest
|
|
107
|
+
return PDS_BASE_URL + spec
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def modernize_pds_url(url: str) -> str:
|
|
111
|
+
"""Rewrite a legacy ``pds.lroc.im-ldi.com/data/`` URL to the PDS store host.
|
|
112
|
+
|
|
113
|
+
Other URLs are returned unchanged. (The legacy host still redirects, but
|
|
114
|
+
resolving up front saves a round-trip and keeps Range requests simple.)
|
|
115
|
+
"""
|
|
116
|
+
if url.startswith(LEGACY_PDS_BASE_URL):
|
|
117
|
+
return pds_url_from_file_spec(url[len(LEGACY_PDS_BASE_URL) :])
|
|
118
|
+
return url
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# Data directory resolution
|
|
122
|
+
# Environment variable overriding the data directory.
|
|
123
|
+
DATA_DIR_ENV_VAR = "LUNAR_DATA_DIR"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _user_data_dir(app_name: str = APP_NAME) -> Path:
|
|
127
|
+
try:
|
|
128
|
+
from platformdirs import user_data_dir
|
|
129
|
+
|
|
130
|
+
return Path(user_data_dir(app_name, appauthor=False))
|
|
131
|
+
except ImportError: # pragma: no cover - platformdirs is a declared dependency
|
|
132
|
+
base = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share")
|
|
133
|
+
return Path(base) / app_name
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def get_data_dir() -> Path:
|
|
137
|
+
"""Directory where Parquet databases are stored.
|
|
138
|
+
|
|
139
|
+
``LUNAR_DATA_DIR`` if set, else the per-user data directory
|
|
140
|
+
(``platformdirs.user_data_dir``), e.g. ``~/.local/share/lunar_toolkit``
|
|
141
|
+
on Linux (honours ``XDG_DATA_HOME``).
|
|
142
|
+
"""
|
|
143
|
+
env = os.environ.get(DATA_DIR_ENV_VAR)
|
|
144
|
+
return Path(env) if env else _user_data_dir()
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def find_parquet(name: str, data_dir: Path | None = None) -> Path:
|
|
148
|
+
"""Path of Parquet database *name* in *data_dir* (default: :func:`get_data_dir`)."""
|
|
149
|
+
return Path(data_dir or get_data_dir()) / name
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# User cache (raw TAB / LBL downloads)
|
|
153
|
+
DEFAULT_CACHE_DIR = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / APP_NAME
|
|
154
|
+
|
|
155
|
+
# Misc
|
|
156
|
+
DEFAULT_PROJECTION = "equirectangular"
|
|
157
|
+
|
|
158
|
+
# gdal_translate flags for map-projected GeoTIFFs: native values, lossless and tiled
|
|
159
|
+
GEOTIFF_OPTIONS = ["-of", "GTiff", "-co", "COMPRESS=DEFLATE", "-co", "TILED=YES", "-co", "BIGTIFF=IF_SAFER"]
|
|
160
|
+
|
|
161
|
+
# Mean lunar radius (km) used for polar stereographic footprints and searches
|
|
162
|
+
MOON_RADIUS_KM = 1737.4
|
|
163
|
+
|
|
164
|
+
# |latitude| above which footprints also get polar stereographic coordinates
|
|
165
|
+
POLAR_LAT_THRESHOLD = 60.0
|
|
166
|
+
|
|
167
|
+
# Index columns <corner>_LATITUDE / <corner>_LONGITUDE of a footprint, in ring order
|
|
168
|
+
FOOTPRINT_CORNERS = ("UPPER_LEFT", "UPPER_RIGHT", "LOWER_RIGHT", "LOWER_LEFT")
|
|
169
|
+
|
|
170
|
+
# NAC product IDs: <observation><side L/R><type E = EDR, C = CDR>; the short
|
|
171
|
+
# base form (M1096757719L) omits the type
|
|
172
|
+
NAC_ID_RE = re.compile(r"^([A-Z]\d{9,12})([LR])([EC]?)$", re.IGNORECASE)
|
|
173
|
+
WAC_ID_RE = re.compile(r"^[A-Z]\d{9,12}[CMUV]E$", re.IGNORECASE)
|
|
174
|
+
NAC_PRODUCT_TYPES = {"E": "edr", "C": "cdr"}
|
|
175
|
+
NAC_SUFFIXES = ("LE", "RE", "LC", "RC")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def parse_nac_id(product_id: str) -> tuple[str, str, str] | None:
|
|
179
|
+
"""Split a NAC product ID into ``(observation, side, type)``.
|
|
180
|
+
|
|
181
|
+
``"M1096757719LE"`` → ``("M1096757719", "L", "edr")``; a short base ID
|
|
182
|
+
(``"M1096757719L"``) gets type ``"edr"``. None for anything else (WAC,
|
|
183
|
+
ShadowCam, …).
|
|
184
|
+
"""
|
|
185
|
+
m = NAC_ID_RE.match(str(product_id).strip())
|
|
186
|
+
if m is None:
|
|
187
|
+
return None
|
|
188
|
+
obs, side, kind = m.groups()
|
|
189
|
+
return obs.upper(), side.upper(), NAC_PRODUCT_TYPES[(kind or "E").upper()]
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def nac_id(product_id: str, side: str | None = None, product_type: str | None = None) -> str | None:
|
|
193
|
+
"""Full NAC product ID of *product_id*, optionally for another side / type.
|
|
194
|
+
|
|
195
|
+
``nac_id("M1096757719L")`` → ``"M1096757719LE"``;
|
|
196
|
+
``nac_id("M1096757719LE", side="R", product_type="cdr")`` → ``"M1096757719RC"``.
|
|
197
|
+
None when *product_id* is not a NAC ID.
|
|
198
|
+
"""
|
|
199
|
+
parts = parse_nac_id(product_id)
|
|
200
|
+
if parts is None:
|
|
201
|
+
return None
|
|
202
|
+
obs, own_side, own_type = parts
|
|
203
|
+
letter = {v: k for k, v in NAC_PRODUCT_TYPES.items()}[(product_type or own_type).lower()]
|
|
204
|
+
return f"{obs}{(side or own_side).upper()}{letter}"
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def is_nac_id(product_id: str) -> bool:
|
|
208
|
+
"""True for a full NAC product ID (``LE`` / ``RE`` / ``LC`` / ``RC``)."""
|
|
209
|
+
m = NAC_ID_RE.match(str(product_id).strip())
|
|
210
|
+
return bool(m and m.group(3))
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
# SfS pipeline option values (lunar_toolkit.sfs.SfsConfig), here so the CLI stays light
|
|
214
|
+
REGISTRATION_MODES = ("stereo", "residuals", "none")
|
|
215
|
+
SELECTION_METHODS = ("coverage", "kmedoids", "image_subset", "all")
|
|
216
|
+
CAMERA_SOURCES = ("ba", "published")
|
|
217
|
+
SFS_CAMERA_MODELS = ("csm", "isis")
|